diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000000..e50cf0411f3 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,100 @@ +# Use `allow` to specify which dependencies to maintain + +version: 2 +updates: + - package-ecosystem: "npm" #npm + directory: "/" + schedule: + interval: "daily" + allow: + # Allow direct and indirect updates for all packages + - dependency-type: "all" + + - package-ecosystem: "composer" #composer + directory: "/" + schedule: + interval: "daily" + allow: + # Allow direct and indirect updates for all packages + - dependency-type: "all" + + - package-ecosystem: "pip" #python + directory: "/" + schedule: + interval: "daily" + allow: + # Allow direct and indirect updates for all packages + - dependency-type: "all" + + # Maintain dependencies for GitHub Actions + - package-ecosystem: "github-actions" #github-actions + directory: "/" + schedule: + interval: "daily" + allow: + # Allow direct and indirect updates for all packages + - dependency-type: "all" + + - package-ecosystem: "bundler" + directory: "/" + schedule: + interval: "daily" + allow: + # Allow direct and indirect updates for all packages + - dependency-type: "all" + + - package-ecosystem: "cargo" + directory: "/" + schedule: + interval: "daily" + allow: + # Allow direct and indirect updates for all packages + - dependency-type: "all" + + - package-ecosystem: "docker" + directory: "/" + schedule: + interval: "daily" + allow: + # Allow direct updates for all packages + - dependency-type: "all" + + - package-ecosystem: "gomod" + directory: "/" + schedule: + interval: "daily" + allow: + # Allow direct and indirect updates for all packages + - dependency-type: "all" + + - package-ecosystem: "gradle" + directory: "/" + schedule: + interval: "daily" + allow: + # Allow direct and indirect updates for all packages + - dependency-type: "all" + + - package-ecosystem: "maven" + directory: "/" + schedule: + interval: "daily" + allow: + # Allow direct and indirect updates for all packages + - dependency-type: "all" + + - package-ecosystem: "pub" + directory: "/" + schedule: + interval: "daily" + allow: + # Allow direct and indirect updates for all packages + - dependency-type: "all" + + - package-ecosystem: "terraform" + directory: "/" + schedule: + interval: "daily" + allow: + # Allow direct and indirect updates for all packages + - dependency-type: "all" diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 00000000000..6f60690904a --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,75 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL" + +on: + pull_request: + schedule: + - cron: '0 1 * * *' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: ['java', 'python'] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] + # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + - name: Use Java version + uses: actions/setup-java@v3 + with: + distribution: 'adopt' # See 'Supported distributions' for available options + java-version: '8' + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + + # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + # queries: security-extended,security-and-quality + + + # Autobuild attempts to build any compiled languages (C/C++, C#, Go, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + #- name: Autobuild + # uses: github/codeql-action/autobuild@v2 + + # ℹī¸ Command-line programs to run using the OS shell. + # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun + + # If the Autobuild fails above, remove it and uncomment the following three lines. + # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. + - name: Autobuild + run: | + echo "Run, Build Application using script" + chmod +x ./build.sh && ./build.sh + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 + with: + category: "/language:${{matrix.language}}" diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml new file mode 100644 index 00000000000..6c162f4a4dc --- /dev/null +++ b/.github/workflows/dependency-review.yml @@ -0,0 +1,13 @@ +name: 'Dependency Review' +on: [pull_request] +permissions: + contents: read +jobs: + dependency-review: + runs-on: ubuntu-latest + steps: + - name: 'Checkout Repository' + uses: actions/checkout@v3 + - name: Dependency Review + uses: actions/dependency-review-action@v2 + diff --git a/.github/workflows/main-ecr.yml b/.github/workflows/main-ecr.yml new file mode 100644 index 00000000000..acb8883f8d8 --- /dev/null +++ b/.github/workflows/main-ecr.yml @@ -0,0 +1,199 @@ +name: Build and Push Atlas Metastore docker image to ECR(deprecated as of Jan 11, 2023) + +on: + push: + branches: + - xoxo/deprecating-ecr/xoxo + +permissions: + id-token: write + contents: write + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - name: Checkout Repo + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Get branch name + run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})" + id: get_branch + + - name: Get repository name + run: echo "REPOSITORY_NAME=`echo "$GITHUB_REPOSITORY" | awk -F / '{print $2}' | sed -e "s/:refs//"`" >> $GITHUB_ENV + shell: bash + + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + with: + java-version: 1.8 + + - name: Cache Maven packages + uses: actions/cache@v2 + with: + path: ~/.m2 + key: ${{ runner.os }}-m2-${{ hashFiles('**/build.sh') }} + restore-keys: ${{ runner.os }}-m2 + + - name: Build with Maven + run: | + branch_name=${{ steps.get_branch.outputs.branch }} + if [[ $branch_name == 'main' || $branch_name == 'master' || $branch_name == 'lineageondemand' ]] + then + echo "build without dashboard" + chmod +x ./build.sh && ./build.sh build_without_dashboard + else + echo "build with dashboard" + chmod +x ./build.sh && ./build.sh + fi + + - name: Get IAM Role and Region to login into ECR + id: get_ecr-details + run: | + branch_name=${{ steps.get_branch.outputs.branch }} + if [[ $branch_name == 'main' || $branch_name == 'staging' || $branch_name == 'master' ]] + then + iamRole=${{ secrets.AWS_PROD_PLATFORM_ECR_ACCESS }} + region='us-east-1' + else + iamRole=${{ secrets.AWS_DEV_PLATFORM_ECR_ACCESS }} + region='ap-south-1' + fi + echo "##[set-output name=iamRole;]$(echo ${iamRole})" + echo "##[set-output name=region;]$(echo ${region})" + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ steps.get_ecr-details.outputs.iamRole }} + aws-region: ${{ steps.get_ecr-details.outputs.region }} + + - name: Check and create ECR Repo if it does not exist + uses: nick-fields/retry@v2 + id: check_and_create_repo + with: + timeout_minutes: 10 + max_attempts: 5 + command: | + aws ecr describe-repositories --repository-names atlanhq/${{ github.event.repository.name }} || aws ecr create-repository --repository-name atlanhq/${{ github.event.repository.name }} --image-tag-mutability IMMUTABLE + while ! $(aws ecr get-repository-policy --repository-name atlanhq/${{ github.event.repository.name }}) + do + aws ecr set-repository-policy --repository-name atlanhq/${{ github.event.repository.name }} --policy-text file://ecrorgcrossaccountpolicy.json + registryIdfrompolicy=$(aws ecr get-repository-policy --repository-name atlanhq/${{ github.event.repository.name }} | jq -r '.registryId') + registryIdfromfromecrrepo=$(aws ecr describe-repositories --repository-names atlanhq/${{ github.event.repository.name }} | jq -r '.repositories[0].registryId') + echo "registryIdfrompolicy : $registryIdfrompolicy and registryIdfromfromecrrepo : $registryIdfromfromecrrepo" + if [ $registryIdfrompolicy == $registryIdfromfromecrrepo ] + then + break + fi + done + + - name: Get Semantic Version tag + id: semver_tag + run: | + branch_name=${{ steps.get_branch.outputs.branch }} + tag=$(git tag --sort=v:refname | grep -E "^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)-$branch_name$" | tail -1) + if [[ -z $tag ]] + then + echo "Creating a new tag series" + updated_full_tag="1.0.0-$branch_name" + new_tag=1.0.0 + else + # echo "Current tag is $tag" + patch_version=$(echo $tag | cut -d '.' -f 3 | cut -d '-' -f 1) + minor_version=$(echo $tag | cut -d '.' -f 2) + major_version=$(echo $tag | cut -d '.' -f 1) + if [[ $patch_version == '999' && $minor_version == '999' ]] + then + bump_type='M' + elif [[ $patch_version == '999' && $minor_version != '999' ]] + then + bump_type='m' + else + bump_type='p' + fi + plain_tag_value=$major_version.$minor_version.$patch_version + chmod +x ./sem-ver-bump.sh + new_tag=$(./sem-ver-bump.sh -$bump_type $plain_tag_value) + updated_full_tag=$new_tag-$branch_name + # echo "Last tag is $tag, Bumping up to $updated_full_tag" + fi + + commit_tag=$(git describe --tags --exact-match $GITHUB_SHA 2>/dev/null || echo "no_tag_found") + if [[ $commit_tag == "no_tag_found" ]] + then + echo "No associated tag found, tagging $GITHUB_SHA with $updated_full_tag" + git tag $updated_full_tag $GITHUB_SHA + git push --tags + echo "##[set-output name=build;]$(echo true)" + echo "##[set-output name=new_tag;]$(echo $new_tag)" + echo "Image will be tagged with $new_tag" + else + echo "$GITHUB_SHA is already tagged with $commit_tag" + IMAGE_META=$(aws ecr describe-images --repository-name=atlanhq/${{ github.event.repository.name }} --image-ids=imageTag=$commit_tag 2>/dev/null || echo "no_tag_found") + if [[ $IMAGE_META == "no_tag_found" ]]; then + echo "Image with specified tag does not exist, proceeding to build and push" + existing_tag=$(echo $commit_tag | cut -d '-' -f 1) + echo "##[set-output name=new_tag;]$(echo $existing_tag)" + echo "##[set-output name=build;]$(echo true)" + echo "Image will be tagged with $existing_tag" + else + IMAGE_TAGS="$( echo ${IMAGE_META} | jq '.imageDetails[0].imageTags[0]' -r )" + echo "Image with the specified tag exists, Skipping build" + existing_tag=$(echo $commit_tag | cut -d '-' -f 1) + echo "##[set-output name=new_tag;]$(echo $existing_tag)" + echo "##[set-output name=build;]$(echo false)" + fi + fi + + - name: Keep last 10 tags and delete rest of the old tags + id: cleanup-tags + run: | + branch_name=${{ steps.get_branch.outputs.branch }} + git tag --sort=v:refname | grep -E "^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)-$branch_name" > tagsfile + cat tagsfile + + vars=($(awk -F= '{print $1}' tagsfile)) + + Npars=${#vars[@]} + echo "Npars : $Npars" + numberOfTagsToKeep=10 + if [ $Npars -gt $numberOfTagsToKeep ] + + then + for ((i=0; i<$Npars - $numberOfTagsToKeep; i++)); do + echo "var $i : ${vars[$i]}" + git push --delete origin ${vars[$i]} + done + fi + + - name: Login to Amazon ECR + id: login-ecr + if: steps.semver_tag.outputs.build == 'true' + uses: aws-actions/amazon-ecr-login@v1 + + - name: Set up Buildx + id: buildx + if: steps.semver_tag.outputs.build == 'true' + uses: docker/setup-buildx-action@v2 + with: + driver-opts: image=moby/buildkit:master + + - name: Build and push docker image + id: docker_build + if: steps.semver_tag.outputs.build == 'true' + uses: docker/build-push-action@v3 + with: + context: . + file: ./Dockerfile + push: true + tags: | + ${{ steps.login-ecr.outputs.registry }}/atlanhq/${{ github.event.repository.name }}:${{ steps.semver_tag.outputs.new_tag }}-${{ steps.get_branch.outputs.branch }} + ${{ steps.login-ecr.outputs.registry }}/atlanhq/${{ github.event.repository.name }}:${{ steps.get_branch.outputs.branch }}-${{ steps.semver_tag.outputs.new_tag }} + build-args: | + ACCESS_TOKEN_USR=$GITHUB_ACTOR + ACCESS_TOKEN_PWD=${{ secrets.my_pat }} \ No newline at end of file diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 83cdca3cea4..46e381c0355 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -25,6 +25,8 @@ on: - beta - development - master + - staging + - lineageondemand jobs: build: @@ -46,18 +48,27 @@ jobs: key: ${{ runner.os }}-m2-${{ hashFiles('**/build.sh') }} restore-keys: ${{ runner.os }}-m2 - - name: Build with Maven - run: chmod +x ./build.sh && ./build.sh - - name: Get branch name run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})" id: get_branch + - name: Build with Maven + run: | + branch_name=${{ steps.get_branch.outputs.branch }} + if [[ $branch_name == 'main' || $branch_name == 'master' || $branch_name == 'lineageondemand' ]] + then + echo "build without dashboard" + chmod +x ./build.sh && ./build.sh build_without_dashboard + else + echo "build with dashboard" + chmod +x ./build.sh && ./build.sh + fi + - run: echo "REPOSITORY_NAME=`echo "$GITHUB_REPOSITORY" | awk -F / '{print $2}' | sed -e "s/:refs//"`" >> $GITHUB_ENV shell: bash - name: Get version tag - run: echo "##[set-output name=version;]$(echo `git ls-remote https://${{ secrets.my_pat }}@github.com/atlanhq/${REPOSITORY_NAME}.git ${{ steps.get_branch.outputs.branch }} | awk '{ print $1}' | cut -c1-7`)abcd" + run: echo "##[set-output name=version;]$(echo `git ls-remote https://${{ secrets.my_pat }}@github.com/atlanhq/${REPOSITORY_NAME}.git refs/heads/${{ steps.get_branch.outputs.branch }} | awk '{ print $1}' | cut -c1-7`)abcd" id: get_version - name: Set up Buildx @@ -77,11 +88,29 @@ jobs: with: context: . file: ./Dockerfile + no-cache: true + sbom: true + provenance: true push: true tags: | ghcr.io/atlanhq/${{ github.event.repository.name }}-${{ steps.get_branch.outputs.branch }}:latest ghcr.io/atlanhq/${{ github.event.repository.name }}-${{ steps.get_branch.outputs.branch }}:${{ steps.get_version.outputs.version }} +<<<<<<< HEAD + +======= + - name: Scan Image + uses: aquasecurity/trivy-action@master + with: + image-ref: 'ubuntu:18.04' + vuln-type: 'os,library' + format: 'sarif' + output: 'trivy-image-results.sarif' + - name: Upload Trivy scan results to GitHub Security tab + uses: github/codeql-action/upload-sarif@v2.1.33 + with: + sarif_file: 'trivy-image-results.sarif' +>>>>>>> 7f3c811fb15361ba82bfb4edd7a8393798863ff0 - name: Check out into atlan repo uses: actions/checkout@v2 @@ -96,7 +125,10 @@ jobs: echo "- ${{ github.event.head_commit.message }}">>gitlog/${{ github.event.repository.name }}.txt chmod +x ./scripts/create_changelog.sh ./scripts/create_changelog.sh +<<<<<<< HEAD +======= +>>>>>>> 7f3c811fb15361ba82bfb4edd7a8393798863ff0 - name: Commit changes uses: EndBug/add-and-commit@v7 with: @@ -106,4 +138,7 @@ jobs: message: '${{ github.event.repository.name }}' default_author: user_info push: origin ${{ steps.get_branch.outputs.branch }} +<<<<<<< HEAD +======= +>>>>>>> 7f3c811fb15361ba82bfb4edd7a8393798863ff0 diff --git a/.github/workflows/trivy-docker-scan.yml b/.github/workflows/trivy-docker-scan.yml new file mode 100644 index 00000000000..0cf526a38c4 --- /dev/null +++ b/.github/workflows/trivy-docker-scan.yml @@ -0,0 +1,30 @@ +name: build +on: + push: + branches: + - alpha + - beta + - development + - master + - lineageondemand + - staging + +jobs: + build: + name: Build + runs-on: ubuntu-20.04 + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Run Trivy vulnerability scanner in IaC mode + uses: aquasecurity/trivy-action@master + with: + scan-type: 'config' + scan-ref: './Dockerfile' + hide-progress: false + template: '@/contrib/gitlab.tpl' + format: 'table' + exit-code: '1' + severity: 'CRITICAL,HIGH,MEDIUM' + diff --git a/Dockerfile b/Dockerfile index ebc2166c7f6..cf2b8f382f7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,7 +17,7 @@ # FROM scratch -FROM ubuntu:18.04 +FROM ubuntu:22.04 LABEL maintainer="engineering@atlan.com" ARG VERSION=3.0.0-SNAPSHOT @@ -28,7 +28,7 @@ RUN apt-get update \ && apt-get -y install apt-utils \ && apt-get -y install \ wget \ - python \ + python2 \ openjdk-8-jdk-headless \ patch \ netcat \ @@ -50,6 +50,8 @@ RUN cd / \ && mv /atlas-index-repair-tool-${VERSION}.jar /opt/apache-atlas/libext/ \ && rm -rf /atlas-index-repair-tool-${VERSION}.tar.gz +RUN ln -s /usr/bin/python2 /usr/bin/python + COPY atlas-hub/repair_index.py /opt/apache-atlas/bin/ RUN chmod +x /opt/apache-atlas/bin/repair_index.py diff --git a/README.txt b/README.txt index 2a8e5b55dab..337051b304f 100755 --- a/README.txt +++ b/README.txt @@ -73,5 +73,4 @@ Build Process distro/target/apache-atlas--storm-hook.tar.gz distro/target/apache-atlas--falcon-hook.tar.gz -4. For more details on installing and running Apache Atlas, please refer to https://atlas.apache.org/#/Installation - +4. For more details on installing and running Apache Atlas, please refer to https://atlas.apache.org/#/Installation. diff --git a/addons/elasticsearch/atlas-auth-es-schema.json b/addons/elasticsearch/atlas-auth-es-schema.json new file mode 100644 index 00000000000..58e7a9c0f64 --- /dev/null +++ b/addons/elasticsearch/atlas-auth-es-schema.json @@ -0,0 +1,128 @@ +{ + "mappings": { + "properties": { + "_expire_at_": { + "type": "date", + "store": true, + "doc_values": true + }, + "_ttl_": { + "type": "text", + "store": true + }, + "_version_": { + "type": "long", + "store": true, + "index": false + }, + "access": { + "type": "keyword" + }, + "action": { + "type": "keyword" + }, + "agent": { + "type": "keyword" + }, + "agentHost": { + "type": "keyword" + }, + "cliIP": { + "type": "keyword" + }, + "cliType": { + "type": "keyword" + }, + "cluster": { + "type": "keyword" + }, + "reqContext": { + "type": "keyword" + }, + "enforcer": { + "type": "keyword" + }, + "event_count": { + "type": "long", + "doc_values": true + }, + "event_dur_ms": { + "type": "long", + "doc_values": true + }, + "evtTime": { + "type": "date", + "doc_values": true + }, + "id": { + "type": "keyword", + "store": true + }, + "logType": { + "type": "keyword" + }, + "policy": { + "type": "keyword" + }, + "proxyUsers": { + "type": "keyword" + }, + "reason": { + "type": "text" + }, + "repo": { + "type": "keyword" + }, + "repoType": { + "type": "integer", + "doc_values": true + }, + "req_caller_id": { + "type": "keyword" + }, + "req_self_id": { + "type": "keyword" + }, + "reqData": { + "type": "text" + }, + "reqUser": { + "type": "keyword" + }, + "reqEntityGuid": { + "type": "keyword" + }, + "resType": { + "type": "keyword" + }, + "resource": { + "type": "keyword" + }, + "result": { + "type": "integer" + }, + "seq_num": { + "type": "long", + "doc_values": true + }, + "sess": { + "type": "keyword" + }, + "tags": { + "type": "keyword" + }, + "tags_str": { + "type": "text" + }, + "text": { + "type": "text" + }, + "zoneName": { + "type": "keyword" + }, + "policyVersion": { + "type": "long" + } + } + } +} \ No newline at end of file diff --git a/addons/elasticsearch/es-audit-mappings.json b/addons/elasticsearch/es-audit-mappings.json index 2f52c3b962e..350a2b44552 100644 --- a/addons/elasticsearch/es-audit-mappings.json +++ b/addons/elasticsearch/es-audit-mappings.json @@ -1,7 +1,17 @@ { "mappings": { "properties": { +<<<<<<< HEAD "entityId": { +======= + "entityQualifiedName": { + "type": "keyword" + }, + "entityId": { + "type": "keyword" + }, + "typeName": { +>>>>>>> 7f3c811fb15361ba82bfb4edd7a8393798863ff0 "type": "keyword" }, "created": { @@ -22,6 +32,18 @@ "eventKey": { "type": "keyword" } - } + }, + "dynamic_templates": [ + { + "atlan_headers_as_keyword": { + "path_match": "headers.x-atlan-*", + "mapping": { + "type": "keyword" + } + } + } + ], + "date_detection": false, + "numeric_detection": false } -} \ No newline at end of file +} diff --git a/addons/elasticsearch/es-mappings.json b/addons/elasticsearch/es-mappings.json new file mode 100644 index 00000000000..56d12b2c0d4 --- /dev/null +++ b/addons/elasticsearch/es-mappings.json @@ -0,0 +1,54 @@ +{ + "properties": { + "relationshipList": { + "type": "nested", + "properties": { + "typeName": { + "type": "keyword" + }, + "guid": { + "type": "keyword" + }, + "provenanceType": { + "type": "integer" + }, + "endName": { + "type": "keyword" + }, + "endGuid": { + "type": "keyword" + }, + "endTypeName": { + "type": "keyword" + }, + "endQualifiedName": { + "type": "keyword" + }, + "label": { + "type": "keyword" + }, + "propagateTags": { + "type": "keyword" + }, + "status": { + "type": "keyword" + }, + "createdBy": { + "type": "keyword" + }, + "updatedBy": { + "type": "keyword" + }, + "createTime": { + "type": "long" + }, + "updateTime": { + "type": "long" + }, + "version": { + "type": "long" + } + } + } + } +} \ No newline at end of file diff --git a/addons/elasticsearch/es-search-logs-mappings.json b/addons/elasticsearch/es-search-logs-mappings.json new file mode 100644 index 00000000000..43b960eb5b0 --- /dev/null +++ b/addons/elasticsearch/es-search-logs-mappings.json @@ -0,0 +1,185 @@ +{ + "mappings": { + "properties": { + "request.attributes": { + "type": "keyword" + }, + "request.relationAttributes": { + "type": "keyword" + }, + "request.dsl": { + "type": "flattened" + }, + "request.dslText": { + "type": "text", + "analyzer": "atlan_json_analyzer" + }, + + + "persona": { + "type": "keyword", + "fields": { + "text": { + "type": "text", + "analyzer": "atlan_text_analyzer" + } + } + }, + "purpose": { + "type": "keyword", + "fields": { + "text": { + "type": "text", + "analyzer": "atlan_text_analyzer" + } + } + }, + + "searchInput": { + "type": "text", + "analyzer": "atlan_text_analyzer", + "fields": { + "keyword": { + "type": "keyword" + }, + "keyword_lowercase": { + "type": "keyword", + "normalizer": "atlan_normalizer" + } + } + }, + + "userAgent": { + "type": "keyword" + }, + "host": { + "type": "keyword" + }, + "ipAddress": { + "type": "keyword" + }, + "userName": { + "type": "keyword" + }, + "entityGuidsAll": { + "type": "keyword" + }, + "entityQFNamesAll": { + "type": "keyword" + }, + "entityGuidsAllowed": { + "type": "keyword" + }, + "entityQFNamesAllowed": { + "type": "keyword" + }, + "entityGuidsDenied": { + "type": "keyword" + }, + "entityQFNamesDenied": { + "type": "keyword" + }, + + "entityTypeNamesAll": { + "type": "keyword" + }, + "entityTypeNamesAllowed": { + "type": "keyword" + }, + "entityTypeNamesDenied": { + "type": "keyword" + }, + + "utmTags": { + "type": "keyword" + }, + "hasResult": { + "type": "boolean" + }, + "isFailed": { + "type": "boolean" + }, + "errorDetails": { + "type": "text" + }, + "errorCode": { + "type": "keyword" + }, + "resultsCount": { + "type": "long" + }, + "timestamp": { + "type": "long" + }, + "createdAt": { + "type": "long" + }, + "responseTime": { + "type": "long" + } + } + }, + "settings": { + "analysis": { + "analyzer": { + "atlan_text_analyzer": { + "type": "custom", + "tokenizer": "atlan_tokenizer", + "filter": [ + "apostrophe", + "lowercase" + ], + "char_filter": [ + "number_filter" + ] + }, + "atlan_json_analyzer": { + "type": "custom", + "tokenizer": "atlan_json_tokenizer" + }, + "atlan_text_stemmer": { + "type": "custom", + "tokenizer": "atlan_tokenizer", + "filter": [ + "snowball_english", + "lowercase" + ], + "char_filter": [ + "number_filter" + ] + } + }, + "normalizer": { + "atlan_normalizer": { + "type": "custom", + "filter": [ + "lowercase" + ] + } + }, + "filter": { + "snowball_english": { + "type": "snowball", + "language": "English" + } + }, + "tokenizer": { + "atlan_tokenizer": { + "type": "pattern", + "pattern": "( |_|-|'|/|@)" + }, + "atlan_json_tokenizer": { + "type": "pattern", + "pattern": "(\"|\\{|\\}|:|\\[|\\]|\\,)" + } + }, + "char_filter": { + "number_filter": { + "type": "pattern_replace", + "pattern": "\\d+", + "replacement": " $0" + } + } + } + } +} \ No newline at end of file diff --git a/addons/elasticsearch/es-settings.json b/addons/elasticsearch/es-settings.json index 673039bb438..359482a09d4 100644 --- a/addons/elasticsearch/es-settings.json +++ b/addons/elasticsearch/es-settings.json @@ -1,6 +1,29 @@ { + "mapping" : { + "total_fields" : { + "limit" : "2000" + }, + "nested_objects" : { + "limit" : "100000" + } + }, + "similarity": { + "default": { + "type": "boolean" + } + }, "analysis": { "analyzer": { + "atlan_glossary_analyzer": { + "type": "custom", + "tokenizer": "atlan_glossary_tokenizer", + "filter": [ + "lowercase" + ], + "char_filter": [ + "letter_number_filter" + ] + }, "atlan_text_analyzer": { "type": "custom", "tokenizer": "atlan_tokenizer", @@ -34,6 +57,12 @@ "graph_synonyms", "lowercase" ], "tokenizer": "atlan_tokenizer" + }, + "truncate_analyzer": { + "char_filter": [ + "truncate_filter" + ], + "tokenizer": "standard" } }, "normalizer": { @@ -53,6 +82,14 @@ } }, "tokenizer": { + "atlan_glossary_tokenizer": { + "type": "char_group", + "tokenize_on_chars": [ + "whitespace", + "punctuation", + "symbol" + ] + }, "atlan_tokenizer": { "type": "pattern", "pattern": "( |_|-|'|/|@)" @@ -63,10 +100,21 @@ } }, "char_filter":{ + "letter_number_filter": { + "type": "pattern_replace", + "pattern": "\\d+", + "replacement": " $0 " + }, "number_filter":{ "type":"pattern_replace", "pattern":"\\d+", "replacement":" $0" + }, + "truncate_filter": { + "pattern": "(.{0,100000}).*", + "type": "pattern_replace", + "replacement": "$1", + "flags": "DOTALL" } } } diff --git a/addons/models/0000-Area0/0010-base_model.json b/addons/models/0000-Area0/0010-base_model.json index 885d35c9f36..fc8b63917b7 100644 --- a/addons/models/0000-Area0/0010-base_model.json +++ b/addons/models/0000-Area0/0010-base_model.json @@ -47,9 +47,160 @@ "value": "SERVER_STATE_ACTIVE" } ] + }, + { + "category": "ENUM", + "name": "AuthPolicyType", + "description": "Policy Item type", + "serviceType": "atlas_core", + "typeVersion": "1.1", + "elementDefs": + [ + { + "value": "allow", + "ordinal": 0 + }, + { + "value": "deny", + "ordinal": 1 + }, + { + "value": "allowExceptions", + "ordinal": 2 + }, + { + "value": "denyExceptions", + "ordinal": 3 + }, + { + "value": "dataMask", + "ordinal": 4 + }, + { + "value": "rowFilter", + "ordinal": 5 + } + ] + }, + { + "category": "ENUM", + "name": "AuthPolicyCategory", + "description": "Policy category", + "serviceType": "atlas_core", + "typeVersion": "1.1", + "elementDefs": [ + { + "value": "bootstrap", + "ordinal": 0 + }, + { + "value": "persona", + "ordinal": 1 + }, + { + "value": "purpose", + "ordinal": 2 + } + ] + }, + { + "category": "ENUM", + "name": "AuthPolicyResourceCategory", + "description": "Policy resource category", + "serviceType": "atlas_core", + "typeVersion": "1.1", + "elementDefs": + [ + { + "value": "ENTITY", + "ordinal": 0 + }, + { + "value": "RELATIONSHIP", + "ordinal": 1 + }, + { + "value": "TAG", + "ordinal": 2 + }, + { + "value": "CUSTOM", + "ordinal": 3 + }, + { + "value": "TYPEDEFS", + "ordinal": 4 + }, + { + "value": "ADMIN", + "ordinal": 5 + } + ] + } + ], + "structDefs": [ + { + "name": "AuthPolicyValiditySchedule", + "description": "Validity schedule struct for policy", + "serviceType": "atlan", + "typeVersion": "1.1", + "attributeDefs": + [ + { + "name": "policyValidityScheduleStartTime", + "typeName": "string", + "indexType": "STRING", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": false, + "isUnique": false + }, + { + "name": "policyValidityScheduleEndTime", + "typeName": "string", + "indexType": "STRING", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": false, + "isUnique": false + }, + { + "name": "policyValidityScheduleTimezone", + "typeName": "string", + "indexType": "STRING", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": false, + "isUnique": false + } + ] + }, + { + "name": "AuthPolicyCondition", + "description": "Policy condition schedule struct", + "serviceType": "atlan", + "typeVersion": "1.1", + "attributeDefs": + [ + { + "name": "policyConditionType", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": false, + "isUnique": false + }, + { + "name": "policyConditionValues", + "typeName": "array", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": false, + "isUnique": false + } + ] } ], - "structDefs": [], "classificationDefs": [], "entityDefs": [ { @@ -352,6 +503,533 @@ "isUnique": false } ] + }, + { + "name": "Referenceable", + "superTypes": [], + "serviceType": "atlas_core", + "typeVersion": "1.01", + "attributeDefs": [ + { + "name": "qualifiedName", + "typeName": "string", + "indexType": "STRING", + "cardinality": "SINGLE", + "isIndexable": true, + "isOptional": false, + "isUnique": true, + "skipScrubbing": true, + "includeInNotification": true, + "indexTypeESFields": { + "text": { + "type": "text", + "analyzer": "atlan_text_analyzer" + } + } + } + ] + }, + { + "name": "Asset", + "superTypes": [ + "Referenceable" + ], + "serviceType": "atlas_core", + "typeVersion": "1.01", + "attributeDefs": [ + { + "name": "name", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": true, + "isOptional": false, + "isUnique": false, + "searchWeight": 10, + "skipScrubbing": true, + "indexTypeESConfig": { + "analyzer": "atlan_text_analyzer" + }, + "indexTypeESFields": { + "keyword": { + "type": "keyword", + "normalizer": "atlan_normalizer" + }, + "stemmed": { + "type": "text", + "analyzer": "atlan_text_stemmer" + } + } + }, + { + "name": "displayName", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": true, + "isOptional": true, + "isUnique": false, + "searchWeight": 10, + "skipScrubbing": true, + "indexTypeESConfig": { + "analyzer": "atlan_text_analyzer" + }, + "indexTypeESFields": { + "keyword": { + "type": "keyword" + } + } + }, + { + "name": "description", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false, + "searchWeight": 9, + "skipScrubbing": true, + "includeInNotification": true, + "indexTypeESConfig": { + "analyzer": "atlan_text_analyzer" + }, + "indexTypeESFields": { + "keyword": { + "type": "keyword", + "normalizer": "atlan_normalizer" + } + } + }, + { + "name": "tenantId", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": true, + "isOptional": true, + "isUnique": false, + "skipScrubbing": true + } + ] + }, + { + "name": "AuthService", + "superTypes": [ + "Asset" + ], + "description": "Model to store auth service in Atlas", + "serviceType": "atlan", + "typeVersion": "1.1", + "attributeDefs": [ + { + "name": "authServiceType", + "typeName": "string", + "indexType": "STRING", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false, + "skipScrubbing": true, + "includeInNotification": true + }, + { + "name": "tagService", + "typeName": "string", + "indexType": "STRING", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false, + "skipScrubbing": true, + "includeInNotification": true + }, + { + "name": "authServiceIsEnabled", + "typeName": "boolean", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false, + "defaultValue": true, + "skipScrubbing": true, + "includeInNotification": true + }, + { + "name": "authServiceConfig", + "typeName": "map", + "isOptional": true, + "cardinality": "SINGLE", + "isUnique": false, + "skipScrubbing": true, + "isIndexable": true, + "includeInNotification": true + }, + { + "name": "authServicePolicyLastSync", + "typeName": "long", + "isOptional": true, + "defaultValue": 0, + "cardinality": "SINGLE", + "isUnique": false, + "skipScrubbing": true, + "isIndexable": true, + "includeInNotification": true + } + ] + }, + { + "name": "AuthPolicy", + "description": "Model to store an accesscontrol policy in Atlas", + "superTypes": [ + "Asset" + ], + "serviceType": "atlan", + "typeVersion": "1.1", + "attributeDefs": + [ + { + "name": "policyType", + "typeName": "AuthPolicyType", + "indexType": "STRING", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false, + "skipScrubbing": true, + "includeInNotification": true + }, + { + "name": "policyServiceName", + "typeName": "string", + "indexType": "STRING", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false, + "skipScrubbing": true, + "includeInNotification": true + }, + { + "name": "policyCategory", + "typeName": "string", + "indexType": "STRING", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false, + "skipScrubbing": true, + "includeInNotification": true + }, + { + "name": "policySubCategory", + "typeName": "string", + "indexType": "STRING", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false, + "skipScrubbing": true, + "includeInNotification": true + }, + { + "name": "policyUsers", + "typeName": "array", + "indexType": "STRING", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false, + "skipScrubbing": true, + "includeInNotification": true + }, + { + "name": "policyGroups", + "typeName": "array", + "indexType": "STRING", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false, + "skipScrubbing": true, + "includeInNotification": true + }, + { + "name": "policyRoles", + "typeName": "array", + "indexType": "STRING", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false, + "skipScrubbing": true, + "includeInNotification": true + }, + { + "name": "policyActions", + "typeName": "array", + "indexType": "STRING", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false, + "skipScrubbing": true, + "includeInNotification": true + }, + { + "name": "policyResources", + "typeName": "array", + "indexType": "STRING", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false, + "skipScrubbing": true, + "includeInNotification": true + }, + { + "name": "policyResourceCategory", + "typeName": "string", + "indexType": "STRING", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false, + "skipScrubbing": true, + "includeInNotification": true + }, + { + "name": "policyPriority", + "typeName": "int", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false, + "defaultValue": 0, + "skipScrubbing": true, + "includeInNotification": true + }, + { + "name": "isPolicyEnabled", + "typeName": "boolean", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false, + "defaultValue": true, + "skipScrubbing": true, + "includeInNotification": true + }, + { + "name": "policyMaskType", + "typeName": "string", + "indexType": "STRING", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false, + "skipScrubbing": true, + "includeInNotification": true + }, + { + "name": "policyValiditySchedule", + "typeName": "array", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false, + "defaultValue": true, + "skipScrubbing": true, + "includeInNotification": false + }, + { + "name": "policyResourceSignature", + "typeName": "string", + "indexType": "STRING", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false, + "skipScrubbing": true, + "includeInNotification": false + }, + { + "name": "policyDelegateAdmin", + "typeName": "boolean", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false, + "defaultValue": false, + "skipScrubbing": true, + "includeInNotification": true + }, + { + "name": "policyConditions", + "typeName": "array", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false, + "skipScrubbing": true, + "includeInNotification": false + } + ] + }, + { + "name": "AccessControl", + "superTypes": [ + "Asset" + ], + "description": "Atlan Type representing parent model for Persona, Purpose", + "serviceType": "atlan", + "typeVersion": "1.3", + "attributeDefs": [ + { + "name": "isAccessControlEnabled", + "typeName": "boolean", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "defaultValue": true, + "isUnique": false, + "skipScrubbing": true, + "includeInNotification": false + }, + { + "name": "denyCustomMetadataGuids", + "typeName": "array", + "indexType": "STRING", + "cardinality": "SET", + "isIndexable": false, + "isOptional": true, + "isUnique": false, + "skipScrubbing": true, + "includeInNotification": false + }, + { + "name": "denyAssetTabs", + "typeName": "array", + "indexType": "STRING", + "cardinality": "SET", + "isIndexable": false, + "isOptional": true, + "isUnique": false, + "skipScrubbing": true, + "includeInNotification": false + }, + { + "name": "denyAssetFilters", + "typeName": "array", + "cardinality": "SET", + "isIndexable": false, + "isOptional": true, + "isUnique": false, + "skipScrubbing": true, + "includeInNotification": false + }, + { + "name": "channelLink", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false, + "skipScrubbing": true, + "includeInNotification": false + }, + { + "name": "denyAssetTypes", + "typeName": "array", + "cardinality": "SET", + "isIndexable": false, + "isOptional": true, + "isUnique": false, + "skipScrubbing": true, + "includeInNotification": false + }, + { + "name": "denyNavigationPages", + "typeName": "array", + "cardinality": "SET", + "isIndexable": false, + "isOptional": true, + "isUnique": false, + "skipScrubbing": true, + "includeInNotification": false + }, + { + "name": "defaultNavigation", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false, + "skipScrubbing": true, + "includeInNotification": false + } + ] + }, + { + "name": "Persona", + "superTypes": [ + "AccessControl" + ], + "description": "Atlan Type representing a Persona model", + "serviceType": "atlan", + "typeVersion": "1.1", + "attributeDefs": [ + { + "name": "personaGroups", + "typeName": "array", + "indexType": "STRING", + "cardinality": "SET", + "isIndexable": false, + "isOptional": true, + "isUnique": false, + "skipScrubbing": true, + "includeInNotification": false + }, + { + "name": "personaUsers", + "typeName": "array", + "indexType": "STRING", + "cardinality": "SET", + "isIndexable": false, + "isOptional": true, + "isUnique": false, + "skipScrubbing": true, + "includeInNotification": false + }, + { + "name": "roleId", + "typeName": "string", + "indexType": "STRING", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false, + "includeInNotification": false + } + ] + }, + { + "name": "Purpose", + "superTypes": [ + "AccessControl" + ], + "description": "Atlan Type representing a Purpose model", + "serviceType": "atlan", + "typeVersion": "1.2", + "attributeDefs": [ + { + "name": "purposeClassifications", + "typeName": "array", + "indexType": "STRING", + "cardinality": "SET", + "isIndexable": false, + "isOptional": true, + "isUnique": false, + "skipScrubbing": true, + "includeInNotification": false + } + ] } ], "relationshipDefs": [ @@ -374,6 +1052,28 @@ "cardinality": "SINGLE" }, "propagateTags": "NONE" + }, + { + "name": "access_control_policies", + "typeVersion": "1.1", + "relationshipCategory": "AGGREGATION", + "relationshipLabel": "__AccessControl.policies", + "serviceType": "atlan", + "endDef1": + { + "type": "AccessControl", + "name": "policies", + "isContainer": true, + "cardinality": "SET" + }, + "endDef2": + { + "type": "AuthPolicy", + "name": "accessControl", + "isContainer": false, + "cardinality": "SINGLE" + }, + "propagateTags": "ONE_TO_TWO" } ] } diff --git a/addons/policies/atlas_service.json b/addons/policies/atlas_service.json new file mode 100644 index 00000000000..49473d26eff --- /dev/null +++ b/addons/policies/atlas_service.json @@ -0,0 +1,41 @@ +{ + "entities": [ + { + "typeName": "AuthService", + "attributes": + { + "qualifiedName": "auth_service_atlas_tag", + "name": "atlas_tag", + "authServiceType": "tag", + "authServiceConfig": { + "ranger.plugin.audit.filters": "[{'accessResult':'DENIED','isAudited':true}]" + } + } + }, + { + "typeName": "AuthService", + "attributes": + { + "qualifiedName": "auth_service_atlas", + "name": "atlas", + "authServiceType": "atlas", + "tagService": "atlas_tag", + "authServiceConfig": { + "ranger.plugin.audit.filters": "[ {'accessResult': 'DENIED', 'isAudited': true}, {'users':['atlas'] ,'isAudited':false} ]" + } + } + }, + { + "typeName": "AuthService", + "attributes": + { + "qualifiedName": "auth_service_heka", + "name": "heka", + "authServiceType": "heka", + "tagService": "atlas_tag", + "authServiceConfig": {} + } + } + ] +} + diff --git a/addons/policies/bootstrap_admin_policies.json b/addons/policies/bootstrap_admin_policies.json new file mode 100644 index 00000000000..3cd9d7a62fa --- /dev/null +++ b/addons/policies/bootstrap_admin_policies.json @@ -0,0 +1,54 @@ +{ + "entities": [ + { + "typeName": "AuthPolicy", + "attributes": { + "name": "RUN_REPAIR_INDEX", + "qualifiedName": "RUN_REPAIR_INDEX", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": [ + "service-account-atlan-argo", + "service-account-atlan-backend" + ], + "policyGroups": [], + "policyRoles": [], + "policyResourceCategory": "ADMIN", + "policyResources": [ + "atlas-service:*" + ], + "policyActions": [ + "admin-repair-index" + ] + } + }, + { + "typeName": "AuthPolicy", + "attributes": { + "name": "ADMIN_ALLOW_TASK_CUD", + "qualifiedName": "ADMIN_ALLOW_TASK_CUD", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": [ + "service-account-atlan-argo", + "service-account-atlan-backend" + ], + "policyGroups": [], + "policyRoles": [], + "policyResourceCategory": "ADMIN", + "policyResources": [ + "atlas-service:*" + ], + "policyActions": [ + "admin-task-cud" + ] + } + } + ] +} \ No newline at end of file diff --git a/addons/policies/bootstrap_entity_policies.json b/addons/policies/bootstrap_entity_policies.json new file mode 100644 index 00000000000..54b385e59dd --- /dev/null +++ b/addons/policies/bootstrap_entity_policies.json @@ -0,0 +1,2939 @@ +{ + "entities": + [ + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 27 + }, + "attributes": + { + "name": "CREATE_SAVEDFILTER", + "qualifiedName": "CREATE_SAVEDFILTER", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$guest", + "$member", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:__AtlasUserSavedSearch", + "entity-classification:*", + "entity:*" + ], + "policyActions": + [ + "entity-create" + ] + } + }, + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 27 + }, + "attributes": + { + "name": "READ_SAVEDFILTER", + "qualifiedName": "READ_SAVEDFILTER", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:__AtlasUserSavedSearch", + "entity-classification:*", + "entity:*" + ], + "policyActions": + [ + "entity-read" + ] + } + }, + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 27 + }, + "attributes": + { + "name": "UPDATE_SAVEDFILTER", + "qualifiedName": "UPDATE_SAVEDFILTER", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:__AtlasUserSavedSearch", + "entity-classification:*", + "entity:*" + ], + "policyActions": + [ + "entity-update" + ] + } + }, + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 27 + }, + "attributes": + { + "name": "DELETE_SAVEDFILTER", + "qualifiedName": "DELETE_SAVEDFILTER", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:__AtlasUserSavedSearch", + "entity-classification:*", + "entity:*" + ], + "policyActions": + [ + "entity-delete" + ] + } + }, + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 26 + }, + "attributes": + { + "name": "LIST_SAVEDFILTER_SELF", + "qualifiedName": "LIST_SAVEDFILTER_SELF", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$guest", + "$member" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:__AtlasUserSavedSearch", + "entity-classification:*", + "entity:{USER}", + "entity:{USER}:*" + ], + "policyActions": + [ + "entity-read" + ] + } + }, + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 26 + }, + "attributes": + { + "name": "UPDATE_SAVEDFILTER_SELF", + "qualifiedName": "UPDATE_SAVEDFILTER_SELF", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$guest", + "$member", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:__AtlasUserSavedSearch", + "entity-classification:*", + "entity:{USER}", + "entity:{USER}:*" + ], + "policyActions": + [ + "entity-update" + ] + } + }, + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 26 + }, + "attributes": + { + "name": "DELETE_SAVEDFILTER_SELF", + "qualifiedName": "DELETE_SAVEDFILTER_SELF", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$guest", + "$member", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:__AtlasUserSavedSearch", + "entity-classification:*", + "entity:{USER}", + "entity:{USER}:*" + ], + "policyActions": + [ + "entity-delete" + ] + } + }, + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 28 + }, + "attributes": + { + "name": "CREATE_GLOSSARY", + "qualifiedName": "CREATE_GLOSSARY", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:AtlasGlossary", + "entity-classification:*", + "entity:*" + ], + "policyActions": + [ + "entity-create" + ] + } + }, + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 28 + }, + "attributes": + { + "name": "READ_GLOSSARY", + "qualifiedName": "READ_GLOSSARY", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$guest", + "$member", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:AtlasGlossary", + "entity-classification:*", + "entity:*" + ], + "policyActions": + [ + "entity-read" + ] + } + }, + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 28 + }, + "attributes": + { + "name": "UPDATE_GLOSSARY", + "qualifiedName": "UPDATE_GLOSSARY", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:AtlasGlossary", + "entity-classification:*", + "entity:*" + ], + "policyActions": + [ + "entity-update" + ] + } + }, + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 28 + }, + "attributes": + { + "name": "DELETE_GLOSSARY", + "qualifiedName": "DELETE_GLOSSARY", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:AtlasGlossary", + "entity-classification:*", + "entity:*" + ], + "policyActions": + [ + "entity-delete" + ] + } + }, + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 29 + }, + "attributes": + { + "name": "CREATE_TERM", + "qualifiedName": "CREATE_TERM", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:AtlasGlossaryTerm", + "entity-classification:*", + "entity:*" + ], + "policyActions": + [ + "entity-create" + ] + } + }, + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 29 + }, + "attributes": + { + "name": "READ_TERM", + "qualifiedName": "READ_TERM", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$guest", + "$member", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:AtlasGlossaryTerm", + "entity-classification:*", + "entity:*" + ], + "policyActions": + [ + "entity-read" + ] + } + }, + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 29 + }, + "attributes": + { + "name": "UPDATE_TERM", + "qualifiedName": "UPDATE_TERM", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:AtlasGlossaryTerm", + "entity-classification:*", + "entity:*" + ], + "policyActions": + [ + "entity-update" + ] + } + }, + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 29 + }, + "attributes": + { + "name": "DELETE_TERM", + "qualifiedName": "DELETE_TERM", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:AtlasGlossaryTerm", + "entity-classification:*", + "entity:*" + ], + "policyActions": + [ + "entity-delete" + ] + } + }, + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 30 + }, + "attributes": + { + "name": "CREATE_CATEGORY", + "qualifiedName": "CREATE_CATEGORY", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:AtlasGlossaryCategory", + "entity-classification:*", + "entity:*" + ], + "policyActions": + [ + "entity-create" + ] + } + }, + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 30 + }, + "attributes": + { + "name": "READ_CATEGORY", + "qualifiedName": "READ_CATEGORY", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$guest", + "$member", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:AtlasGlossaryCategory", + "entity-classification:*", + "entity:*" + ], + "policyActions": + [ + "entity-read" + ] + } + }, + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 30 + }, + "attributes": + { + "name": "UPDATE_CATEGORY", + "qualifiedName": "UPDATE_CATEGORY", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:AtlasGlossaryCategory", + "entity-classification:*", + "entity:*" + ], + "policyActions": + [ + "entity-update" + ] + } + }, + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 30 + }, + "attributes": + { + "name": "DELETE_CATEGORY", + "qualifiedName": "DELETE_CATEGORY", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:AtlasGlossaryCategory", + "entity-classification:*", + "entity:*" + ], + "policyActions": + [ + "entity-delete" + ] + } + }, + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 34 + }, + "attributes": + { + "name": "CREATE_README", + "qualifiedName": "CREATE_README", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$member", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:Readme", + "entity-classification:*", + "entity:*" + ], + "policyActions": + [ + "entity-create" + ] + } + }, + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 34 + }, + "attributes": + { + "name": "READ_README", + "qualifiedName": "READ_README", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$member", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:Readme", + "entity-classification:*", + "entity:*" + ], + "policyActions": + [ + "entity-read" + ] + } + }, + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 34 + }, + "attributes": + { + "name": "UPDATE_README", + "qualifiedName": "UPDATE_README", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$member", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:Readme", + "entity-classification:*", + "entity:*" + ], + "policyActions": + [ + "entity-update" + ] + } + }, + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 34 + }, + "attributes": + { + "name": "DELETE_README", + "qualifiedName": "DELETE_README", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$member", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:Readme", + "entity-classification:*", + "entity:*" + ], + "policyActions": + [ + "entity-delete" + ] + } + }, + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 123 + }, + "attributes": + { + "name": "CREATE_BADGE", + "qualifiedName": "CREATE_BADGE", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:Badge", + "entity-classification:*", + "entity:*" + ], + "policyActions": + [ + "entity-create" + ] + } + }, + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 123 + }, + "attributes": + { + "name": "READ_BADGE", + "qualifiedName": "READ_BADGE", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$guest", + "$member", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:Badge", + "entity-classification:*", + "entity:*" + ], + "policyActions": + [ + "entity-read" + ] + } + }, + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 123 + }, + "attributes": + { + "name": "UPDATE_BADGE", + "qualifiedName": "UPDATE_BADGE", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:Badge", + "entity-classification:*", + "entity:*" + ], + "policyActions": + [ + "entity-update" + ] + } + }, + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 123 + }, + "attributes": + { + "name": "DELETE_BADGE", + "qualifiedName": "DELETE_BADGE", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:Badge", + "entity-classification:*", + "entity:*" + ], + "policyActions": + [ + "entity-delete" + ] + } + }, + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 35 + }, + "attributes": + { + "name": "CREATE_BOOKMARK", + "qualifiedName": "CREATE_BOOKMARK", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$guest", + "$member", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:AtlanBookmark", + "entity-classification:*", + "entity:{USER}", + "entity:{USER}:*" + ], + "policyActions": + [ + "entity-create" + ] + } + }, + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 35 + }, + "attributes": + { + "name": "READ_BOOKMARK_SELF", + "qualifiedName": "READ_BOOKMARK_SELF", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$guest", + "$member", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:AtlanBookmark", + "entity-classification:*", + "entity:{USER}", + "entity:{USER}:*" + ], + "policyActions": + [ + "entity-read" + ] + } + }, + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 35 + }, + "attributes": + { + "name": "UPDATE_BOOKMARK_SELF", + "qualifiedName": "UPDATE_BOOKMARK_SELF", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$guest", + "$member", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:AtlanBookmark", + "entity-classification:*", + "entity:{USER}", + "entity:{USER}:*" + ], + "policyActions": + [ + "entity-update" + ] + } + }, + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 35 + }, + "attributes": + { + "name": "DELETE_BOOKMARK_SELF", + "qualifiedName": "DELETE_BOOKMARK_SELF", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$guest", + "$member", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:AtlanBookmark", + "entity-classification:*", + "entity:{USER}", + "entity:{USER}:*" + ], + "policyActions": + [ + "entity-delete" + ] + } + }, + { + "typeName": "AuthPolicy", + "attributes": + { + "name": "UPDATE_QUERY_CLASSIFICATION", + "qualifiedName": "UPDATE_QUERY_CLASSIFICATION", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyUsers": + [ + "{USER}" + ], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$member", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:Query", + "entity-type:QueryFolder", + "entity-classification:*", + "entity:{USER}", + "entity:*/{USER}/*" + ], + "policyActions": + [ + "entity-add-classification", + "entity-update-classification", + "entity-remove-classification" + ] + } + }, + { + "typeName": "AuthPolicy", + "attributes": + { + "name": "ADD_TERM_CLASSIFICATION", + "qualifiedName": "ADD_TERM_CLASSIFICATION", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:AtlasGlossaryTerm", + "entity-classification:*", + "classification:*", + "entity:*" + ], + "policyActions": + [ + "entity-add-classification", + "entity-update-classification", + "entity-remove-classification" + ] + } + }, + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 103 + }, + "attributes": + { + "name": "CREATE_LINK", + "qualifiedName": "CREATE_LINK", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$member", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:Link", + "entity-classification:*", + "entity:*" + ], + "policyActions": + [ + "entity-create" + ] + } + }, + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 103 + }, + "attributes": + { + "name": "READ_LINK", + "qualifiedName": "READ_LINK", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$member", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:Link", + "entity-classification:*", + "entity:*" + ], + "policyActions": + [ + "entity-read" + ] + } + }, + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 103 + }, + "attributes": + { + "name": "UPDATE_LINK", + "qualifiedName": "UPDATE_LINK", + "policySubCategory": "default", + "policyCategory": "bootstrap", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$member", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:Link", + "entity-classification:*", + "entity:*" + ], + "policyActions": + [ + "entity-update" + ] + } + }, + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 103 + }, + "attributes": + { + "name": "DELETE_LINK", + "qualifiedName": "DELETE_LINK", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$member", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:Link", + "entity-classification:*", + "entity:*" + ], + "policyActions": + [ + "entity-delete" + ] + } + }, + { + "typeName": "AuthPolicy", + "attributes": + { + "name": "CREATE_QUERIES", + "qualifiedName": "CREATE_QUERIES", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:QueryFolderNamespace", + "entity-type:Query", + "entity-type:QueryFolder", + "entity-type:QueryFolderParent", + "entity-type:QueryFolderChild", + "entity-classification:*", + "entity:*" + ], + "policyActions": + [ + "entity-create" + ] + } + }, + { + "typeName": "AuthPolicy", + "attributes": + { + "name": "READ_QUERIES", + "qualifiedName": "READ_QUERIES", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:QueryFolderNamespace", + "entity-type:Query", + "entity-type:QueryFolder", + "entity-type:QueryFolderParent", + "entity-type:QueryFolderChild", + "entity-classification:*", + "entity:*" + ], + "policyActions": + [ + "entity-read" + ] + } + }, + { + "typeName": "AuthPolicy", + "attributes": + { + "name": "UPDATE_QUERIES", + "qualifiedName": "UPDATE_QUERIES", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:QueryFolderNamespace", + "entity-type:Query", + "entity-type:QueryFolder", + "entity-type:QueryFolderParent", + "entity-type:QueryFolderChild", + "entity-classification:*", + "entity:*" + ], + "policyActions": + [ + "entity-update" + ] + } + }, + { + "typeName": "AuthPolicy", + "attributes": + { + "name": "DELETE_QUERIES", + "qualifiedName": "DELETE_QUERIES", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:QueryFolderNamespace", + "entity-type:Query", + "entity-type:QueryFolder", + "entity-type:QueryFolderParent", + "entity-type:QueryFolderChild", + "entity-classification:*", + "entity:*" + ], + "policyActions": + [ + "entity-delete" + ] + } + }, + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 107 + }, + "attributes": + { + "name": "CREATE_CONNECTION", + "qualifiedName": "CREATE_CONNECTION", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:Connection", + "entity-classification:*", + "entity:*" + ], + "policyActions": + [ + "entity-create" + ] + } + }, + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 108 + }, + "attributes": + { + "name": "CREATE_COLLECTION", + "qualifiedName": "CREATE_COLLECTION", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$member", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:Collection", + "entity-classification:*", + "entity:{USER}", + "entity:*/{USER}/*" + ], + "policyActions": + [ + "entity-create" + ] + } + }, + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 109 + }, + "attributes": + { + "name": "BM_UPDATE_GLOSSARY", + "qualifiedName": "BM_UPDATE_GLOSSARY", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:AtlasGlossary", + "entity-classification:*", + "entity:*", + "entity-business-metadata:*" + ], + "policyActions": + [ + "entity-update-business-metadata" + ] + } + }, + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 110 + }, + "attributes": + { + "name": "BM_UPDATE_TERM", + "qualifiedName": "BM_UPDATE_TERM", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:AtlasGlossaryTerm", + "entity-classification:*", + "entity:*", + "entity-business-metadata:*" + ], + "policyActions": + [ + "entity-update-business-metadata" + ] + } + }, + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 114 + }, + "attributes": + { + "name": "BM_UPDATE_CATEGORY", + "qualifiedName": "BM_UPDATE_CATEGORY", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:AtlasGlossaryCategory", + "entity-classification:*", + "entity:*", + "entity-business-metadata:*" + ], + "policyActions": + [ + "entity-update-business-metadata" + ] + } + }, + { + "typeName": "AuthPolicy", + "attributes": + { + "name": "CREATE_PROCESS_ASSET", + "qualifiedName": "CREATE_PROCESS_ASSET", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:Process", + "entity-classification:*", + "entity:*" + ], + "policyActions": + [ + "entity-create" + ] + } + }, + { + "typeName": "AuthPolicy", + "attributes": + { + "name": "READ_PROCESS_ASSET", + "qualifiedName": "READ_PROCESS_ASSET", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:Process", + "entity-classification:*", + "entity:*" + ], + "policyActions": + [ + "entity-read" + ] + } + }, + { + "typeName": "AuthPolicy", + "attributes": + { + "name": "UPDATE_PROCESS_ASSET", + "qualifiedName": "UPDATE_PROCESS_ASSET", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:Process", + "entity-classification:*", + "entity:*" + ], + "policyActions": + [ + "entity-update" + ] + } + }, + { + "typeName": "AuthPolicy", + "attributes": + { + "name": "DELETE_PROCESS_ASSET", + "qualifiedName": "DELETE_PROCESS_ASSET", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:Process", + "entity-classification:*", + "entity:*" + ], + "policyActions": + [ + "entity-delete" + ] + } + }, + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 117 + }, + "attributes": + { + "name": "CREATE_README_TEMPLATE", + "qualifiedName": "CREATE_README_TEMPLATE", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:ReadmeTemplate", + "entity-classification:*", + "entity:*" + ], + "policyActions": + [ + "entity-create" + ] + } + }, + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 117 + }, + "attributes": + { + "name": "READ_README_TEMPLATE", + "qualifiedName": "READ_README_TEMPLATE", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$member", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:ReadmeTemplate", + "entity-classification:*", + "entity:*" + ], + "policyActions": + [ + "entity-read" + ] + } + }, + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 117 + }, + "attributes": + { + "name": "UPDATE_README_TEMPLATE", + "qualifiedName": "UPDATE_README_TEMPLATE", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:ReadmeTemplate", + "entity-classification:*", + "entity:*" + ], + "policyActions": + [ + "entity-update" + ] + } + }, + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 117 + }, + "attributes": + { + "name": "DELETE_README_TEMPLATE", + "qualifiedName": "DELETE_README_TEMPLATE", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:ReadmeTemplate", + "entity-classification:*", + "entity:*" + ], + "policyActions": + [ + "entity-delete" + ] + } + }, + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 119 + }, + "attributes": + { + "name": "CREATE_AUDITENTRY", + "qualifiedName": "CREATE_AUDITENTRY", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:__AtlasAuditEntry", + "entity-classification:*", + "entity:*" + ], + "policyActions": + [ + "entity-create" + ] + } + }, + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 119 + }, + "attributes": + { + "name": "READ_AUDITENTRY", + "qualifiedName": "READ_AUDITENTRY", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:__AtlasAuditEntry", + "entity-classification:*", + "entity:*" + ], + "policyActions": + [ + "entity-read" + ] + } + }, + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 119 + }, + "attributes": + { + "name": "UPDATE_AUDITENTRY", + "qualifiedName": "UPDATE_AUDITENTRY", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:__AtlasAuditEntry", + "entity-classification:*", + "entity:*" + ], + "policyActions": + [ + "entity-update" + ] + } + }, + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 119 + }, + "attributes": + { + "name": "DELETE_AUDITENTRY", + "qualifiedName": "DELETE_AUDITENTRY", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:__AtlasAuditEntry", + "entity-classification:*", + "entity:*" + ], + "policyActions": + [ + "entity-delete" + ] + } + }, + { + "typeName": "AuthPolicy", + "attributes": + { + "name": "UPDATE_ENTITY_BUSINESS_METADATA", + "qualifiedName": "UPDATE_ENTITY_BUSINESS_METADATA", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyUsers": + [ + "admin", + "service-account-atlan-argo", + "service-account-atlan-backend" + ], + "policyGroups": + [], + "policyRoles": + [], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:*", + "entity-classification:*", + "entity:*", + "entity-business-metadata:*" + ], + "policyActions": + [ + "entity-update-business-metadata" + ] + } + }, + { + "typeName": "AuthPolicy", + "attributes": + { + "name": "READ_ENTITY", + "qualifiedName": "READ_ENTITY", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyUsers": + [ + "rangertagsync" + ], + "policyGroups": + [], + "policyRoles": + [], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:*", + "entity-classification:*", + "entity:*" + ], + "policyActions": + [ + "entity-read" + ] + } + }, + { + "typeName": "AuthPolicy", + "attributes": + { + "name": "CRUD_ENTITY", + "qualifiedName": "CRUD_ENTITY", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyUsers": + [ + "admin", + "service-account-atlan-argo", + "service-account-atlan-backend" + ], + "policyGroups": + [], + "policyRoles": + [], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:*", + "entity-classification:*", + "entity:*" + ], + "policyActions": + [ + "entity-read", + "entity-create", + "entity-update", + "entity-delete" + ] + } + }, + { + "typeName": "AuthPolicy", + "attributes": + { + "name": "AUR_CLASSIFICATION", + "qualifiedName": "AUR_CLASSIFICATION", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyUsers": + [ + "admin", + "service-account-atlan-argo", + "service-account-atlan-backend" + ], + "policyGroups": + [], + "policyRoles": + [], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:*", + "entity-classification:*", + "entity:*", + "classification:*" + ], + "policyActions": + [ + "entity-add-classification", + "entity-update-classification", + "entity-remove-classification" + ] + } + }, + { + "typeName": "AuthPolicy", + "attributes": + { + "name": "AR_ENITY_LABEL", + "qualifiedName": "AR_ENITY_LABEL", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyUsers": + [ + "admin", + "service-account-atlan-argo", + "service-account-atlan-backend" + ], + "policyGroups": + [], + "policyRoles": + [], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:*", + "entity-classification:*", + "entity:*", + "entity-label:*" + ], + "policyActions": + [ + "entity-add-label", + "entity-remove-label" + ] + } + }, + { + "typeName": "AuthPolicy", + "attributes": + { + "name": "CRUD_ENTITY_SAVED_SEARCH", + "qualifiedName": "CRUD_ENTITY_SAVED_SEARCH", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyUsers": + [ + "{USER}" + ], + "policyGroups": + [], + "policyRoles": + [], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:__AtlasUserProfile", + "entity-type:__AtlasUserSavedSearch", + "entity-classification:*", + "entity:{USER}", + "entity:{USER}:*" + ], + "policyActions": + [ + "entity-read", + "entity-create", + "entity-update", + "entity-delete" + ] + } + }, + { + "typeName": "AuthPolicy", + "attributes": + { + "name": "CRUD_COLLECTION_FOLDER_QUERY", + "qualifiedName": "CRUD_COLLECTION_FOLDER_QUERY", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyUsers": + [ + "{USER}" + ], + "policyGroups": + [], + "policyRoles": + [ + "$member", + "$admin" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:Collection", + "entity-type:Query", + "entity-type:Folder", + "entity-classification:*", + "entity:*/{USER}/*" + ], + "policyActions": + [ + "entity-read", + "entity-create", + "entity-update", + "entity-delete" + ] + } + }, + { + "typeName": "AuthPolicy", + "attributes": + { + "name": "EIPA_ATLAS_SERVICE", + "qualifiedName": "EIPA_ATLAS_SERVICE", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyUsers": + [ + "admin", + "service-account-atlan-argo", + "service-account-atlan-backend" + ], + "policyGroups": + [], + "policyRoles": + [], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "atlas-service:*" + ], + "policyActions": + [ + "admin-export", + "admin-import", + "admin-purge", + "admin-audits" + ] + } + }, + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 120 + }, + "attributes": + { + "name": "CREATE_ACCESS_CONTROL_ENTITIES", + "qualifiedName": "CREATE_ACCESS_CONTROL_ENTITIES", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:Persona", + "entity-type:Purpose", + "entity-classification:*", + "entity:*" + ], + "policyActions": + [ + "entity-create" + ] + } + }, + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 120 + }, + "attributes": + { + "name": "READ_ACCESS_CONTROL_ENTITIES", + "qualifiedName": "READ_ACCESS_CONTROL_ENTITIES", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [ + "service-account-atlan-argo", + "service-account-atlan-backend" + ], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:Persona", + "entity-type:Purpose", + "entity-classification:*", + "entity:*" + ], + "policyActions": + [ + "entity-read" + ] + } + }, + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 120 + }, + "attributes": + { + "name": "UPDATE_ACCESS_CONTROL_ENTITIES", + "qualifiedName": "UPDATE_ACCESS_CONTROL_ENTITIES", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:Persona", + "entity-type:Purpose", + "entity-classification:*", + "entity:*" + ], + "policyActions": + [ + "entity-update" + ] + } + }, + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 120 + }, + "attributes": + { + "name": "DELETE_ACCESS_CONTROL_ENTITIES", + "qualifiedName": "DELETE_ACCESS_CONTROL_ENTITIES", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:Persona", + "entity-type:Purpose", + "entity-classification:*", + "entity:*" + ], + "policyActions": + [ + "entity-delete" + ] + } + }, + { + "typeName": "AuthPolicy", + "attributes": + { + "name": "CREATE_AUTH_POLICIES", + "qualifiedName": "CREATE_AUTH_POLICIES", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyUsers": + [ + "service-account-atlan-argo", + "service-account-atlan-backend" + ], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:AuthPolicy", + "entity-classification:*", + "entity:*" + ], + "policyActions": + [ + "entity-create" + ] + } + }, + { + "typeName": "AuthPolicy", + "attributes": + { + "name": "READ_AUTH_POLICIES", + "qualifiedName": "READ_AUTH_POLICIES", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [ + "service-account-atlan-argo", + "service-account-atlan-backend" + ], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:AuthPolicy", + "entity-classification:*", + "entity:*" + ], + "policyActions": + [ + "entity-read" + ] + } + }, + { + "typeName": "AuthPolicy", + "attributes": + { + "name": "UPDATE_AUTH_POLICIES", + "qualifiedName": "UPDATE_AUTH_POLICIES", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyUsers": + [ + "service-account-atlan-argo", + "service-account-atlan-backend" + ], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:AuthPolicy", + "entity-classification:*", + "entity:*" + ], + "policyActions": + [ + "entity-update" + ] + } + }, + { + "typeName": "AuthPolicy", + "attributes": + { + "name": "DELETE_AUTH_POLICIES", + "qualifiedName": "DELETE_AUTH_POLICIES", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyUsers": + [ + "service-account-atlan-argo", + "service-account-atlan-backend" + ], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:AuthPolicy", + "entity-classification:*", + "entity:*" + ], + "policyActions": + [ + "entity-delete" + ] + } + }, + { + "typeName": "AuthPolicy", + "attributes": + { + "name": "CUD_ENTITY_DENY_GUEST", + "qualifiedName": "CUD_ENTITY_DENY_GUEST", + "description": "cud deny for guest users", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "deny", + "policyPriority": 0, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$guest" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:AtlasGlossaryTerm", + "entity-type:AtlasGlossaryCategory", + "entity-type:AtlasGlossary", + "entity-type:Catalog", + "entity-type:Process", + "entity-type:Connection", + "entity-type:ProcessExecution", + "entity-type:File", + "entity-classification:*", + "entity:*" + ], + "policyActions": + [ + "entity-create", + "entity-update", + "entity-delete" + ] + } + }, + { + "typeName": "AuthPolicy", + "attributes": + { + "name": "BM_DENY_GUEST", + "qualifiedName": "BM_DENY_GUEST", + "description": "deny guest update entity-business-metadata", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "deny", + "policyPriority": 0, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$guest" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:*", + "entity-classification:*", + "entity:*", + "entity-business-metadata:*" + ], + "policyActions": + [ + "entity-update-business-metadata" + ] + } + }, + { + "typeName": "AuthPolicy", + "attributes": + { + "name": "CUD_CLASSIFICATION_DENY_GUEST", + "qualifiedName": "CUD_CLASSIFICATION_DENY_GUEST", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "deny", + "policyPriority": 0, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$guest" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:*", + "entity-classification:*", + "entity:*", + "classification:*" + ], + "policyActions": + [ + "entity-add-classification", + "entity-update-classification", + "entity-remove-classification" + ] + } + }, + { + "typeName": "AuthPolicy", + "attributes": + { + "name": "FILE_CRUD_ALLOW_USERS", + "qualifiedName": "FILE_CRUD_ALLOW_USERS", + "description": "Allows user to perform crud operation on file asset", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:File", + "entity-classification:*", + "entity:*" + ], + "policyActions": + [ + "entity-create", + "entity-read", + "entity-update", + "entity-delete" + ] + } + }, + { + "typeName": "AuthPolicy", + "attributes": + { + "name": "CONNECTION_UPDATE_DENY_MEMBER", + "qualifiedName": "CONNECTION_UPDATE_DENY_MEMBER", + "description": "Policy for connection update deny for members", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "deny", + "policyPriority": 0, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$member" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:Connection", + "entity-classification:*", + "entity:*" + ], + "policyActions": + [ + "entity-update" + ] + } + }, + { + "typeName": "AuthPolicy", + "attributes": + { + "name": "CRUD_DATA_MESH_ENTITIES", + "qualifiedName": "CRUD_DATA_MESH_ENTITIES", + "description": "Allows user to perform crud operation on data mesh assets", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:DataContract", + "entity-type:DataDomain", + "entity-type:DataProduct", + "entity-classification:*", + "entity:*" + ], + "policyActions": + [ + "entity-create", + "entity-read", + "entity-update", + "entity-delete" + ] + } + }, + { + "typeName": "AuthPolicy", + "attributes": + { + "name": "READ_DATA_MESH_ENTITIES", + "qualifiedName": "READ_DATA_MESH_ENTITIES", + "description": "Allows user to perform to do read operation on data mesh assets", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$member", + "$guest" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:DataContract", + "entity-type:DataDomain", + "entity-type:DataProduct", + "entity-classification:*", + "entity:*" + ], + "policyActions": + [ + "entity-read" + ] + } + } + ] +} \ No newline at end of file diff --git a/addons/policies/bootstrap_heka_policies.json b/addons/policies/bootstrap_heka_policies.json new file mode 100644 index 00000000000..d092186d789 --- /dev/null +++ b/addons/policies/bootstrap_heka_policies.json @@ -0,0 +1,35 @@ +{ + "entities": + [ + { + "typeName": "AuthPolicy", + "attributes": + { + "name": "DENY_DATA_ACCESS_GUEST", + "qualifiedName": "DENY_DATA_ACCESS_GUEST", + "description": "deny data access for guest users", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "heka", + "policyType": "deny", + "policyPriority": 1, + "policyUsers": [], + "policyGroups": [], + "policyRoles": + [ + "$guest" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity:*", + "entity-type:*" + ], + "policyActions": + [ + "select" + ] + } + } + ] +} \ No newline at end of file diff --git a/addons/policies/bootstrap_relationship_policies.json b/addons/policies/bootstrap_relationship_policies.json new file mode 100644 index 00000000000..f6845ff74a8 --- /dev/null +++ b/addons/policies/bootstrap_relationship_policies.json @@ -0,0 +1,763 @@ +{ + "entities": + [ + { + "typeName": "AuthPolicy", + "attributes": + { + "name": "LINK_GLOSSARY_TERM", + "qualifiedName": "LINK_GLOSSARY_TERM", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "RELATIONSHIP", + "policyResources": + [ + "end-one-entity-classification:*", + "end-two-entity-classification:*", + "end-one-entity:*", + "end-two-entity:*", + "end-one-entity-type:AtlasGlossary", + "end-two-entity-type:AtlasGlossaryTerm", + "relationship-type:*" + ], + "policyActions": + [ + "add-relationship", + "update-relationship", + "remove-relationship" + ] + } + }, + { + "typeName": "AuthPolicy", + "attributes": + { + "name": "LINK_GLOSSARY_TERM_CATEGORY", + "qualifiedName": "LINK_GLOSSARY_TERM_CATEGORY", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "RELATIONSHIP", + "policyResources": + [ + "end-one-entity-classification:*", + "end-two-entity-classification:*", + "end-one-entity:*", + "end-two-entity:*", + "end-one-entity-type:AtlasGlossaryCategory", + "end-two-entity-type:AtlasGlossaryTerm", + "relationship-type:*" + ], + "policyActions": + [ + "add-relationship", + "update-relationship", + "remove-relationship" + ] + } + }, + { + "typeName": "AuthPolicy", + "attributes": + { + "name": "LINK_GLOSSARY_CATEGORY", + "qualifiedName": "LINK_GLOSSARY_CATEGORY", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "RELATIONSHIP", + "policyResources": + [ + "end-one-entity-classification:*", + "end-two-entity-classification:*", + "end-one-entity:*", + "end-two-entity:*", + "end-one-entity-type:AtlasGlossary", + "end-two-entity-type:AtlasGlossaryCategory", + "relationship-type:*" + ], + "policyActions": + [ + "add-relationship", + "update-relationship", + "remove-relationship" + ] + } + }, + { + "typeName": "AuthPolicy", + "attributes": + { + "name": "LINK_FOLDERS_USER", + "qualifiedName": "LINK_FOLDERS_USER", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyUsers": + [ + "{USER}" + ], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$member" + ], + "policyResourceCategory": "RELATIONSHIP", + "policyResources": + [ + "end-one-entity-classification:*", + "end-two-entity-classification:*", + "end-one-entity:{USER}", + "end-one-entity:*/{USER}/*", + "end-two-entity:{USER}", + "end-two-entity:*/{USER}/*", + "end-one-entity-type:QueryFolder", + "end-one-entity-type:QueryFolderParent", + "end-two-entity-type:QueryFolder", + "end-two-entity-type:QueryFolderChild", + "relationship-type:*" + ], + "policyActions": + [ + "add-relationship", + "update-relationship", + "remove-relationship" + ] + } + }, + { + "typeName": "AuthPolicy", + "attributes": + { + "name": "LINK_CATEGORY_GLOSSARY", + "qualifiedName": "LINK_CATEGORY_GLOSSARY", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "RELATIONSHIP", + "policyResources": + [ + "end-one-entity-classification:*", + "end-two-entity-classification:*", + "end-one-entity:*", + "end-two-entity:*", + "end-one-entity-type:AtlasGlossaryCategory", + "end-two-entity-type:AtlasGlossaryCategory", + "relationship-type:*" + ], + "policyActions": + [ + "add-relationship", + "update-relationship", + "remove-relationship" + ] + } + }, + { + "typeName": "AuthPolicy", + "attributes": + { + "name": "LINK_FOLDERS", + "qualifiedName": "LINK_FOLDERS", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyUsers": + [ + "{USER}" + ], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$member" + ], + "policyResourceCategory": "RELATIONSHIP", + "policyResources": + [ + "end-one-entity-classification:*", + "end-two-entity-classification:*", + "end-one-entity:*", + "end-two-entity:*", + "end-one-entity-type:QueryFolder", + "end-one-entity-type:QueryFolderParent", + "end-one-entity-type:QueryFolderChild", + "end-one-entity-type:QueryFolderNamespace", + "end-two-entity-type:QueryFolder", + "end-two-entity-type:QueryFolderParent", + "end-two-entity-type:QueryFolderChild", + "end-two-entity-type:Query", + "relationship-type:*" + ], + "policyActions": + [ + "add-relationship", + "update-relationship", + "remove-relationship" + ] + } + }, + { + "typeName": "AuthPolicy", + "attributes": + { + "name": "LINK_GLOSSARY_ASSET", + "qualifiedName": "LINK_GLOSSARY_ASSET", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "RELATIONSHIP", + "policyResources": + [ + "end-one-entity-classification:*", + "end-two-entity-classification:*", + "end-one-entity:*", + "end-two-entity:*", + "end-one-entity-type:AtlasGlossary", + "end-two-entity-type:*", + "relationship-type:*" + ], + "policyActions": + [ + "add-relationship", + "update-relationship", + "remove-relationship" + ] + } + }, + { + "typeName": "AuthPolicy", + "attributes": + { + "name": "LINK_TERM_ASSET", + "qualifiedName": "LINK_TERM_ASSET", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "RELATIONSHIP", + "policyResources": + [ + "end-one-entity-classification:*", + "end-two-entity-classification:*", + "end-one-entity:*", + "end-two-entity:*", + "end-one-entity-type:AtlasGlossaryTerm", + "end-two-entity-type:*", + "relationship-type:*" + ], + "policyActions": + [ + "add-relationship", + "update-relationship", + "remove-relationship" + ] + } + }, + { + "typeName": "AuthPolicy", + "attributes": + { + "name": "LINK_CATEGORY_ASSET", + "qualifiedName": "LINK_CATEGORY_ASSET", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "RELATIONSHIP", + "policyResources": + [ + "end-one-entity-classification:*", + "end-two-entity-classification:*", + "end-one-entity:*", + "end-two-entity:*", + "end-one-entity-type:AtlasGlossaryCategory", + "end-two-entity-type:*", + "relationship-type:*" + ], + "policyActions": + [ + "add-relationship", + "update-relationship", + "remove-relationship" + ] + } + }, + { + "typeName": "AuthPolicy", + "attributes": + { + "name": "LINK_PROCESS_ASSET", + "qualifiedName": "LINK_PROCESS_ASSET", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "RELATIONSHIP", + "policyResources": + [ + "end-one-entity-classification:*", + "end-two-entity-classification:*", + "end-one-entity:*", + "end-two-entity:*", + "end-one-entity-type:Process", + "end-two-entity-type:Catalog", + "relationship-type:*" + ], + "policyActions": + [ + "add-relationship", + "update-relationship", + "remove-relationship" + ] + } + }, + { + "typeName": "AuthPolicy", + "attributes": + { + "name": "LINK_PROCESSES", + "qualifiedName": "LINK_PROCESSES", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "RELATIONSHIP", + "policyResources": + [ + "end-one-entity-classification:*", + "end-two-entity-classification:*", + "end-one-entity:*", + "end-two-entity:*", + "end-one-entity-type:Process", + "end-two-entity-type:Process", + "relationship-type:*" + ], + "policyActions": + [ + "add-relationship", + "update-relationship", + "remove-relationship" + ] + } + }, + { + "typeName": "AuthPolicy", + "attributes": + { + "name": "RELATIONSHIP_CRUD", + "qualifiedName": "RELATIONSHIP_CRUD", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyUsers": + [ + "admin", + "service-account-atlan-argo", + "service-account-atlan-backend" + ], + "policyGroups": + [], + "policyRoles": + [], + "policyResourceCategory": "RELATIONSHIP", + "policyResources": + [ + "end-one-entity-classification:*", + "end-two-entity-classification:*", + "end-one-entity:*", + "end-two-entity:*", + "end-one-entity-type:*", + "end-two-entity-type:*", + "relationship-type:*" + ], + "policyActions": + [ + "add-relationship", + "update-relationship", + "remove-relationship" + ] + } + }, + { + "typeName": "AuthPolicy", + "attributes": + { + "name": "LINK_COLLECTION_ASSET", + "qualifiedName": "LINK_COLLECTION_ASSET", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "description": "Policy for link assets to collection", + "policyUsers": + [ + "{USER}", + "admin", + "service-account-atlan-argo", + "service-account-atlan-backend" + ], + "policyGroups": + [], + "policyRoles": + [], + "policyResourceCategory": "RELATIONSHIP", + "policyResources": + [ + "end-one-entity-classification:*", + "end-two-entity-classification:*", + "end-one-entity:*/{USER}/*", + "end-two-entity:*", + "end-one-entity-type:*", + "end-two-entity-type:Catalog", + "end-two-entity-type:Connection", + "end-two-entity-type:Dataset", + "end-two-entity-type:Infrastructure", + "end-two-entity-type:Process", + "end-two-entity-type:ProcessExecution", + "end-two-entity-type:Namespace", + "relationship-type:*" + ], + "policyActions": + [ + "add-relationship", + "update-relationship", + "remove-relationship" + ] + } + }, + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 121 + }, + "attributes": + { + "name": "LINK_ACCESS_CONTROL_POLICY", + "qualifiedName": "LINK_ACCESS_CONTROL_POLICY", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "RELATIONSHIP", + "policyResources": + [ + "end-one-entity-classification:*", + "end-two-entity-classification:*", + "end-one-entity:*", + "end-two-entity:*", + "end-one-entity-type:AccessControl", + "end-two-entity-type:AuthPolicy", + "relationship-type:*" + ], + "policyActions": + [ + "add-relationship", + "update-relationship", + "remove-relationship" + ] + } + }, + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 122 + }, + "attributes": + { + "name": "LINK_ACCESS_CONTROL_RESOURCE", + "qualifiedName": "LINK_ACCESS_CONTROL_RESOURCE", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "RELATIONSHIP", + "policyResources": + [ + "end-one-entity-classification:*", + "end-two-entity-classification:*", + "end-one-entity:*", + "end-two-entity:*", + "end-one-entity-type:AccessControl", + "end-two-entity-type:Readme", + "end-two-entity-type:Link", + "relationship-type:*" + ], + "policyActions": + [ + "add-relationship", + "update-relationship", + "remove-relationship" + ] + } + }, + { + "typeName": "AuthPolicy", + "attributes": + { + "name": "CUD_RELATIONSHIP_DENY_GUEST", + "qualifiedName": "CUD_RELATIONSHIP_DENY_GUEST", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "deny", + "policyPriority": 0, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$guest" + ], + "policyResourceCategory": "RELATIONSHIP", + "policyResources": + [ + "end-one-entity-classification:*", + "end-two-entity-classification:*", + "end-one-entity:*", + "end-two-entity:*", + "end-one-entity-type:*", + "end-two-entity-type:*", + "relationship-type:*" + ], + "policyActions": + [ + "add-relationship", + "update-relationship", + "remove-relationship" + ] + } + }, + { + "typeName": "AuthPolicy", + "attributes": + { + "name": "LINK_MESH_DATA_PRODUCT_TO_DATA_CONTRACT", + "qualifiedName": "LINK_MESH_DATA_PRODUCT_TO_DATA_CONTRACT", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "RELATIONSHIP", + "policyResources": + [ + "end-one-entity-classification:*", + "end-two-entity-classification:*", + "end-one-entity:*", + "end-two-entity:*", + "end-one-entity-type:DataProduct", + "end-two-entity-type:DataContract", + "relationship-type:*" + ], + "policyActions": + [ + "add-relationship", + "update-relationship", + "remove-relationship" + ] + } + }, + { + "typeName": "AuthPolicy", + "attributes": + { + "name": "LINK_MESH_DATA_DOMAIN_TO_DATA_PRODUCT", + "qualifiedName": "LINK_MESH_DATA_PRODUCT_DATA_CONTRACT", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "RELATIONSHIP", + "policyResources": + [ + "end-one-entity-classification:*", + "end-two-entity-classification:*", + "end-one-entity:*", + "end-two-entity:*", + "end-one-entity-type:DataDomain", + "end-two-entity-type:DataProduct", + "relationship-type:*" + ], + "policyActions": + [ + "add-relationship", + "update-relationship", + "remove-relationship" + ] + } + }, + { + "typeName": "AuthPolicy", + "attributes": + { + "name": "LINK_MESH_DATA_DOMAIN_TO_DATA_DOMAIN", + "qualifiedName": "LINK_MESH_DATA_DOMAIN_TO_DATA_DOMAIN", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "RELATIONSHIP", + "policyResources": + [ + "end-one-entity-classification:*", + "end-two-entity-classification:*", + "end-one-entity:*", + "end-two-entity:*", + "end-one-entity-type:DataDomain", + "end-two-entity-type:DataDomain", + "relationship-type:*" + ], + "policyActions": + [ + "add-relationship", + "update-relationship", + "remove-relationship" + ] + } + } + ] +} \ No newline at end of file diff --git a/addons/policies/bootstrap_typedef_policies.json b/addons/policies/bootstrap_typedef_policies.json new file mode 100644 index 00000000000..092c91a4de5 --- /dev/null +++ b/addons/policies/bootstrap_typedef_policies.json @@ -0,0 +1,632 @@ +{ + "entities": + [ + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 1 + }, + "attributes": + { + "name": "CREATE_CLASSIFICATION", + "qualifiedName": "CREATE_CLASSIFICATION", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "TYPEDEFS", + "policyResources": + [ + "type-category:classification", + "type:*" + ], + "policyActions": + [ + "type-create" + ] + } + }, + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 1 + }, + "attributes": + { + "name": "READ_CLASSIFICATION", + "qualifiedName": "READ_CLASSIFICATION", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$member", + "$api-token-default-access" + ], + "policyResourceCategory": "TYPEDEFS", + "policyResources": + [ + "type-category:classification", + "type:*" + ], + "policyActions": + [ + "type-read" + ] + } + }, + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 1 + }, + "attributes": + { + "name": "UPDATE_CLASSIFICATION", + "qualifiedName": "UPDATE_CLASSIFICATION", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "TYPEDEFS", + "policyResources": + [ + "type-category:classification", + "type:*" + ], + "policyActions": + [ + "type-update" + ] + } + }, + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 1 + }, + "attributes": + { + "name": "DELETE_CLASSIFICATION", + "qualifiedName": "DELETE_CLASSIFICATION", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "TYPEDEFS", + "policyResources": + [ + "type-category:classification", + "type:*" + ], + "policyActions": + [ + "type-delete" + ] + } + }, + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 2 + }, + "attributes": + { + "name": "CREATE_BUSINESS_METADATA", + "qualifiedName": "CREATE_BUSINESS_METADATA", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "TYPEDEFS", + "policyResources": + [ + "type-category:business_metadata", + "type:*" + ], + "policyActions": + [ + "type-create" + ] + } + }, + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 2 + }, + "attributes": + { + "name": "READ_BUSINESS_METADATA", + "qualifiedName": "READ_BUSINESS_METADATA", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$member", + "$api-token-default-access" + ], + "policyResourceCategory": "TYPEDEFS", + "policyResources": + [ + "type-category:business_metadata", + "type:*" + ], + "policyActions": + [ + "type-read" + ] + } + }, + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 2 + }, + "attributes": + { + "name": "UPDATE_BUSINESS_METADATA", + "qualifiedName": "UPDATE_BUSINESS_METADATA", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "TYPEDEFS", + "policyResources": + [ + "type-category:business_metadata", + "type:*" + ], + "policyActions": + [ + "type-update" + ] + } + }, + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 2 + }, + "attributes": + { + "name": "DELETE_BUSINESS_METADATA", + "qualifiedName": "DELETE_BUSINESS_METADATA", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "TYPEDEFS", + "policyResources": + [ + "type-category:business_metadata", + "type:*" + ], + "policyActions": + [ + "type-delete" + ] + } + }, + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 3 + }, + "attributes": + { + "name": "CREATE_ENUM", + "qualifiedName": "CREATE_ENUM", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "TYPEDEFS", + "policyResources": + [ + "type-category:enum", + "type:*" + ], + "policyActions": + [ + "type-create" + ] + } + }, + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 3 + }, + "attributes": + { + "name": "READ_ENUM", + "qualifiedName": "READ_ENUM", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$member", + "$api-token-default-access" + ], + "policyResourceCategory": "TYPEDEFS", + "policyResources": + [ + "type-category:enum", + "type:*" + ], + "policyActions": + [ + "type-read" + ] + } + }, + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 3 + }, + "attributes": + { + "name": "UPDATE_ENUM", + "qualifiedName": "UPDATE_ENUM", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "TYPEDEFS", + "policyResources": + [ + "type-category:enum", + "type:*" + ], + "policyActions": + [ + "type-update" + ] + } + }, + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 3 + }, + "attributes": + { + "name": "DELETE_ENUM", + "qualifiedName": "DELETE_ENUM", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "TYPEDEFS", + "policyResources": + [ + "type-category:enum", + "type:*" + ], + "policyActions": + [ + "type-delete" + ] + } + }, + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 4 + }, + "attributes": + { + "name": "CREATE_TYPEDEF", + "qualifiedName": "CREATE_TYPEDEF", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [], + "policyResourceCategory": "TYPEDEFS", + "policyResources": + [ + "type-category:*", + "type:*" + ], + "policyActions": + [ + "type-create" + ] + } + }, + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 4 + }, + "attributes": + { + "name": "READ_TYPEDEF", + "qualifiedName": "READ_TYPEDEF", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [ + "admin", + "public" + ], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$guest", + "$member", + "$api-token-default-access" + ], + "policyResourceCategory": "TYPEDEFS", + "policyResources": + [ + "type-category:*", + "type:*" + ], + "policyActions": + [ + "type-read" + ] + } + }, + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 4 + }, + "attributes": + { + "name": "UPDATE_TYPEDEF", + "qualifiedName": "UPDATE_TYPEDEF", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [], + "policyResourceCategory": "TYPEDEFS", + "policyResources": + [ + "type-category:*", + "type:*" + ], + "policyActions": + [ + "type-update" + ] + } + }, + { + "typeName": "AuthPolicy", + "customAttributes": { + "internalId": 4 + }, + "attributes": + { + "name": "DELETE_TYPEDEF", + "qualifiedName": "DELETE_TYPEDEF", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [], + "policyResourceCategory": "TYPEDEFS", + "policyResources": + [ + "type-category:*", + "type:*" + ], + "policyActions": + [ + "type-delete" + ] + } + }, + { + "typeName": "AuthPolicy", + "attributes": + { + "name": "CRUD_TYPEDEF", + "qualifiedName": "CRUD_TYPEDEF", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyUsers": + [ + "admin", + "service-account-atlan-argo", + "service-account-atlan-backend" + ], + "policyGroups": + [], + "policyRoles": + [], + "policyResourceCategory": "TYPEDEFS", + "policyResources": + [ + "type-category:*", + "type:*" + ], + "policyActions": + [ + "type-create", + "type-read", + "type-update", + "type-delete" + ] + } + }, + { + "typeName": "AuthPolicy", + "attributes": + { + "name": "CUD_TYPEDEF_DENY_GUEST", + "qualifiedName": "CUD_TYPEDEF_DENY_GUEST", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "deny", + "policyPriority": 0, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$guest" + ], + "policyResourceCategory": "TYPEDEFS", + "policyResources": + [ + "type-category:*", + "type:*" + ], + "policyActions": + [ + "type-create", + "type-update", + "type-delete" + ] + } + } + ] +} \ No newline at end of file diff --git a/addons/static/templates/collection_bootstrap_policies.json b/addons/static/templates/collection_bootstrap_policies.json new file mode 100644 index 00000000000..d2e868a7c8a --- /dev/null +++ b/addons/static/templates/collection_bootstrap_policies.json @@ -0,0 +1,264 @@ +[ + { + "typeName": "AuthPolicy", + "guid": -1, + "attributes": { + "qualifiedName": "{guid}/collection-link-assets", + "name": "{name}-collection-link-assets", + "policyCategory": "bootstrap", + "policySubCategory": "collection", + "policyType": "allow", + "policyServiceName": "atlas", + "policyRoles": [ + "collection_admins_{guid}" + ], + "policyActions": [ + "add-relationship", + "remove-relationship" + ], + "policyResourceCategory": "RELATIONSHIP", + "policyResources": [ + "end-one-entity-classification:*", + "end-one-entity:{entity}", + "end-one-entity:{entity}/*", + "end-one-entity-type:*", + + "end-two-entity-classification:*", + "end-two-entity:*", + "end-two-entity-type:Catalog", + "end-two-entity-type:collection", + "end-two-entity-type:Process", + "end-two-entity-type:ProcessExecution", + "end-two-entity-type:Namespace", + + "relationship-type:*" + ] + + } + }, + { + "typeName": "AuthPolicy", + "guid": -2, + "attributes": { + "qualifiedName": "{guid}/collection-add-terms", + "name": "{name}-collection-add-terms", + "policyCategory": "bootstrap", + "policySubCategory": "collection", + "policyType": "allow", + "policyServiceName": "atlas", + "policyRoles": [ + "collection_admins_{guid}" + ], + "policyActions": [ + "add-relationship", + "remove-relationship" + ], + "policyResourceCategory": "RELATIONSHIP", + "policyResources": [ + "end-one-entity-classification:*", + "end-one-entity:*", + "end-one-entity-type:AtlasGlossaryTerm", + + "end-two-entity-classification:*", + "end-two-entity:{entity}", + "end-two-entity:{entity}/*", + "end-two-entity-type:*", + + "relationship-type:*" + ] + } + }, + { + "typeName": "AuthPolicy", + "guid": -3, + "attributes": { + "qualifiedName": "{guid}/collection-CRUD-admin", + "name": "{name}-collection-CRUD-admin", + "policyCategory": "bootstrap", + "policySubCategory": "collection", + "policyType": "allow", + "policyServiceName": "atlas", + "policyRoles": [ + "collection_admins_{guid}" + ], + "policyActions": [ + "entity-read", + "entity-create", + "entity-update", + "entity-delete" + ], + "policyResourceCategory": "ENTITY", + "policyResources": [ + "entity:{entity}", + "entity:{entity}/*", + "entity-type:*", + "entity-classification:*" + ] + } + }, + { + "typeName": "AuthPolicy", + "guid": -4, + "attributes": { + "qualifiedName": "{guid}/collection-CRUD-viewer", + "name": "{name}-collection-CRUD-viewer", + "policyCategory": "bootstrap", + "policySubCategory": "collection", + "policyType": "allow", + "policyServiceName": "atlas", + "policyRoles": [ + "collection_viewer_{guid}" + ], + "policyActions": [ + "entity-read" + ], + "policyResourceCategory": "ENTITY", + "policyResources": [ + "entity:{entity}", + "entity:{entity}/*", + "entity-type:*", + "entity-classification:*" + ] + } + }, + + { + "typeName": "AuthPolicy", + "guid": -5, + "attributes": { + "qualifiedName": "{guid}/collection-entity-business-metadata", + "name": "{name}-collection-entity-business-metadata", + "policyCategory": "bootstrap", + "policySubCategory": "collection", + "policyType": "allow", + "policyServiceName": "atlas", + "policyRoles": [ + "collection_admins_{guid}" + ], + "policyActions": [ + "entity-update-business-metadata" + ], + "policyResourceCategory": "RELATIONSHIP", + "policyResources": [ + "entity:{entity}", + "entity:{entity}/*", + "entity-type:*", + "entity-classification:*", + "entity-business-metadata:*" + ] + } + }, + { + "typeName": "AuthPolicy", + "guid": -6, + "attributes": { + "qualifiedName": "{guid}/collection-classification", + "name": "{name}-collection-classification", + "policyCategory": "bootstrap", + "policySubCategory": "collection", + "policyType": "allow", + "policyServiceName": "atlas", + "policyRoles": [ + "collection_admins_{guid}" + ], + "policyActions": [ + "entity-add-classification", + "entity-remove-classification", + "entity-update-classification" + ], + "policyResourceCategory": "ENTITY", + "policyResources": [ + "entity:{entity}", + "entity:{entity}/*", + "entity-type:*", + "entity-classification:*", + "classification:*" + ] + } + }, + { + "typeName": "AuthPolicy", + "guid": -7, + "attributes": { + "qualifiedName": "{guid}/collection-entity-label", + "name": "{name}-collection-entity-label", + "policyCategory": "bootstrap", + "policySubCategory": "collection", + "policyType": "allow", + "policyServiceName": "atlas", + "policyRoles": [ + "collection_admins_{guid}" + ], + "policyActions": [ + "entity-add-label", + "entity-remove-label" + ], + "policyResourceCategory": "ENTITY", + "policyResources": [ + "entity:{entity}", + "entity:{entity}/*", + "entity-type:*", + "entity-classification:*", + "entity-label:*" + ] + } + }, + { + "typeName": "AuthPolicy", + "guid": -8, + "attributes": { + "qualifiedName": "{guid}/collection-unlink-queries", + "name": "{name}-collection-unlink-queries", + "policyCategory": "bootstrap", + "policySubCategory": "collection", + "policyType": "allow", + "policyServiceName": "atlas", + "policyRoles": [ + "collection_admins_{guid}" + ], + "policyActions": [ + "remove-relationship" + ], + "policyResourceCategory": "RELATIONSHIP", + "policyResources": [ + "end-one-entity-classification:*", + "end-one-entity:*", + "end-one-entity-type:*", + + "end-two-entity-classification:*", + "end-two-entity:{entity}", + "end-two-entity:{entity}/*", + "end-two-entity-type:*", + + "relationship-type:*" + ] + + } + }, + { + "typeName": "AuthPolicy", + "guid": -9, + "attributes": { + "qualifiedName": "{guid}/dataPolicy-collection", + "name": "{name}-dataPolicy-collection", + "policyCategory": "bootstrap", + "policySubCategory": "collection", + "policyType": "allow", + "policyServiceName": "heka", + "policyRoles": [ + "collection_admins_{guid}" + ], + "policyActions": [ + "select" + ], + "policyResourceCategory": "ENTITY", + "policyResources": [ + "entity:{entity}", + "entity:{entity}/*", + "entity-type:*" + ] + } + } +] + + diff --git a/addons/static/templates/connection_bootstrap_policies.json b/addons/static/templates/connection_bootstrap_policies.json new file mode 100644 index 00000000000..4f0b8a448ce --- /dev/null +++ b/addons/static/templates/connection_bootstrap_policies.json @@ -0,0 +1,241 @@ +[ + { + "typeName": "AuthPolicy", + "guid": -1, + "attributes": { + "qualifiedName": "{guid}/connection-link-assets", + "name": "{name}-connection-link-assets", + "policyCategory": "bootstrap", + "policySubCategory": "connection", + "policyType": "allow", + "policyServiceName": "atlas", + "policyRoles": [ + "connection_admins_{guid}" + ], + "policyActions": [ + "add-relationship", + "remove-relationship" + ], + "policyResourceCategory": "RELATIONSHIP", + "policyResources": [ + "end-one-entity-classification:*", + "end-one-entity:{entity}", + "end-one-entity:{entity}/*", + "end-one-entity-type:*", + + "end-two-entity-classification:*", + "end-two-entity:*", + "end-two-entity-type:Catalog", + "end-two-entity-type:Connection", + "end-two-entity-type:Process", + "end-two-entity-type:ProcessExecution", + "end-two-entity-type:Namespace", + + "relationship-type:*" + ] + + } + }, + { + "typeName": "AuthPolicy", + "guid": -8, + "attributes": { + "qualifiedName": "{guid}/connection-link-assets-inverse", + "name": "{name}-connection-link-assets-inverse", + "policyCategory": "bootstrap", + "policySubCategory": "connection", + "policyType": "allow", + "policyServiceName": "atlas", + "policyRoles": [ + "connection_admins_{guid}" + ], + "policyActions": [ + "add-relationship", + "remove-relationship" + ], + "policyResourceCategory": "RELATIONSHIP", + "policyResources": [ + "end-one-entity-classification:*", + "end-one-entity:*", + "end-one-entity-type:Catalog", + "end-one-entity-type:Connection", + "end-one-entity-type:Process", + "end-one-entity-type:ProcessExecution", + "end-one-entity-type:Namespace", + + "end-two-entity-classification:*", + "end-two-entity:{entity}", + "end-two-entity:{entity}/*", + "end-two-entity-type:*", + + "relationship-type:*" + ] + } + }, + { + "typeName": "AuthPolicy", + "guid": -2, + "attributes": { + "qualifiedName": "{guid}/connection-add-terms", + "name": "{name}-connection-add-terms", + "policyCategory": "bootstrap", + "policySubCategory": "connection", + "policyType": "allow", + "policyServiceName": "atlas", + "policyRoles": [ + "connection_admins_{guid}" + ], + "policyActions": [ + "add-relationship", + "remove-relationship" + ], + "policyResourceCategory": "RELATIONSHIP", + "policyResources": [ + "end-one-entity-classification:*", + "end-one-entity:*", + "end-one-entity-type:AtlasGlossaryTerm", + + "end-two-entity-classification:*", + "end-two-entity:{entity}", + "end-two-entity:{entity}/*", + "end-two-entity-type:*", + + "relationship-type:*" + ] + } + }, + { + "typeName": "AuthPolicy", + "guid": -3, + "attributes": { + "qualifiedName": "{guid}/connection-CRUD", + "name": "{name}-connection-CRUD", + "policyCategory": "bootstrap", + "policySubCategory": "connection", + "policyType": "allow", + "policyServiceName": "atlas", + "policyRoles": [ + "connection_admins_{guid}" + ], + "policyActions": [ + "entity-read", + "entity-create", + "entity-update", + "entity-delete" + ], + "policyResourceCategory": "ENTITY", + "policyResources": [ + "entity:{entity}", + "entity:{entity}/*", + "entity-type:*", + "entity-classification:*" + ] + } + }, + + { + "typeName": "AuthPolicy", + "guid": -4, + "attributes": { + "qualifiedName": "{guid}/connection-entity-business-metadata", + "name": "{name}-connection-entity-business-metadata", + "policyCategory": "bootstrap", + "policySubCategory": "connection", + "policyType": "allow", + "policyServiceName": "atlas", + "policyRoles": [ + "connection_admins_{guid}" + ], + "policyActions": [ + "entity-update-business-metadata" + ], + "policyResourceCategory": "RELATIONSHIP", + "policyResources": [ + "entity:{entity}", + "entity:{entity}/*", + "entity-type:*", + "entity-classification:*", + "entity-business-metadata:*" + ] + } + }, + { + "typeName": "AuthPolicy", + "guid": -5, + "attributes": { + "qualifiedName": "{guid}/connection-classification", + "name": "{name}-connection-classification", + "policyCategory": "bootstrap", + "policySubCategory": "connection", + "policyType": "allow", + "policyServiceName": "atlas", + "policyRoles": [ + "connection_admins_{guid}" + ], + "policyActions": [ + "entity-add-classification", + "entity-remove-classification", + "entity-update-classification" + ], + "policyResourceCategory": "ENTITY", + "policyResources": [ + "entity:{entity}", + "entity:{entity}/*", + "entity-type:*", + "entity-classification:*", + "classification:*" + ] + } + }, + { + "typeName": "AuthPolicy", + "guid": -6, + "attributes": { + "qualifiedName": "{guid}/connection-entity-label", + "name": "{name}-connection-entity-label", + "policyCategory": "bootstrap", + "policySubCategory": "connection", + "policyType": "allow", + "policyServiceName": "atlas", + "policyRoles": [ + "connection_admins_{guid}" + ], + "policyActions": [ + "entity-add-label", + "entity-remove-label" + ], + "policyResourceCategory": "ENTITY", + "policyResources": [ + "entity:{entity}", + "entity:{entity}/*", + "entity-type:*", + "entity-classification:*", + "entity-label:*" + ] + } + }, + { + "typeName": "AuthPolicy", + "guid": -7, + "attributes": { + "qualifiedName": "{guid}/dataPolicy-connection", + "name": "{name}-dataPolicy-connection", + "policyCategory": "bootstrap", + "policySubCategory": "connection", + "policyType": "allow", + "policyServiceName": "heka", + "policyRoles": [ + "connection_admins_{guid}" + ], + "policyActions": [ + "select" + ], + "policyResourceCategory": "ENTITY", + "policyResources": [ + "entity:{entity}", + "entity:{entity}/*", + "entity-type:*" + ] + } + } +] \ No newline at end of file diff --git a/addons/static/templates/policy_cache_transformer_persona.json b/addons/static/templates/policy_cache_transformer_persona.json new file mode 100644 index 00000000000..e48eba784ac --- /dev/null +++ b/addons/static/templates/policy_cache_transformer_persona.json @@ -0,0 +1,387 @@ +{ + "persona-asset-read": [ + { + "policyType": "ACCESS", + "policyResourceCategory": "ENTITY", + "resources": [ + "entity:{entity}", + "entity:{entity}/*", + "entity-type:{entity-type}", + "entity-classification:*" + ], + "actions": ["entity-read"] + } + ], + "persona-asset-update": [ + { + "policyType": "ACCESS", + "policyResourceCategory": "ENTITY", + "resources": [ + "entity:{entity}", + "entity:{entity}/*", + "entity-type:{entity-type}", + "entity-classification:*" + ], + "actions": ["entity-update"] + }, + { + "policyType": "ACCESS", + "policyResourceCategory": "RELATIONSHIP", + "resources": [ + "relationship-type:*", + + "end-one-entity-type:{entity-type}", + "end-one-entity-classification:*", + "end-one-entity:{entity}", + "end-one-entity:{entity}/*", + + "end-two-entity-type:Catalog", + "end-two-entity-type:Connection", + "end-two-entity-type:Process", + "end-two-entity-type:Namespace", + "end-two-entity-type:ProcessExecution", + "end-two-entity-classification:*", + "end-two-entity:*" + ], + "actions": ["add-relationship", "remove-relationship"] + }, + { + "policyType": "ACCESS", + "policyResourceCategory": "RELATIONSHIP", + "resources": [ + "relationship-type:*", + + "end-one-entity-type:Catalog", + "end-one-entity-type:Connection", + "end-one-entity-type:Process", + "end-one-entity-type:Namespace", + "end-one-entity-type:ProcessExecution", + "end-one-entity-classification:*", + "end-one-entity:*", + + "end-two-entity-type:{entity-type}", + "end-two-entity-classification:*", + "end-two-entity:{entity}", + "end-two-entity:{entity}/*" + ], + "actions": ["add-relationship", "remove-relationship"] + } + ], + "persona-api-create": [ + { + "policyType": "ACCESS", + "policyResourceCategory": "ENTITY", + "resources": [ + "entity:{entity}", + "entity:{entity}/*", + "entity-type:{entity-type}", + "entity-classification:*" + ], + "actions": ["entity-create"] + } + ], + "persona-api-delete": [ + { + "policyType": "ACCESS", + "policyResourceCategory": "ENTITY", + "resources": [ + "entity:{entity}", + "entity:{entity}/*", + "entity-type:{entity-type}", + "entity-classification:*" + ], + "actions": ["entity-delete"] + } + ], + "persona-business-update-metadata": [ + { + "policyType": "ACCESS", + "policyResourceCategory": "ENTITY", + "resources": [ + "entity:{entity}", + "entity:{entity}/*", + "entity-type:{entity-type}", + "entity-classification:*", + "entity-business-metadata:*" + ], + "actions": ["entity-update-business-metadata"] + } + ], + "persona-entity-add-classification": [ + { + "policyType": "ACCESS", + "policyResourceCategory": "ENTITY", + "resources": [ + "entity:{entity}", + "entity:{entity}/*", + "entity-type:{entity-type}", + "entity-classification:*", + "classification:*" + ], + "actions": [ + "entity-add-classification", + "entity-update-classification" + ] + } + ], + "persona-entity-update-classification": [ + { + "policyType": "ACCESS", + "policyResourceCategory": "ENTITY", + "resources": [ + "entity:{entity}", + "entity:{entity}/*", + "entity-type:{entity-type}", + "entity-classification:*", + "classification:*" + ], + "actions": ["entity-update-classification"] + } + ], + "persona-entity-remove-classification": [ + { + "policyType": "ACCESS", + "policyResourceCategory": "ENTITY", + "resources": [ + "entity:{entity}", + "entity:{entity}/*", + "entity-type:{entity-type}", + "entity-classification:*", + "classification:*" + ], + "actions": [ + "entity-remove-classification", + "entity-update-classification" + ] + } + ], + "persona-add-terms": [ + { + "policyType": "ACCESS", + "policyResourceCategory": "RELATIONSHIP", + "resources": [ + "relationship-type:*", + + "end-one-entity-type:AtlasGlossaryTerm", + "end-one-entity-classification:*", + "end-one-entity:*", + + "end-two-entity-type:{entity-type}", + "end-two-entity-classification:*", + "end-two-entity:{entity}", + "end-two-entity:{entity}/*" + ], + "actions": ["add-relationship"] + } + ], + "persona-remove-terms": [ + { + "policyType": "ACCESS", + "policyResourceCategory": "RELATIONSHIP", + "resources": [ + "relationship-type:*", + + "end-one-entity-type:AtlasGlossaryTerm", + "end-one-entity-classification:*", + "end-one-entity:*", + + "end-two-entity-type:{entity-type}", + "end-two-entity-classification:*", + "end-two-entity:{entity}", + "end-two-entity:{entity}/*" + ], + "actions": ["remove-relationship"] + } + ], + + + + "persona-glossary-read": [ + { + "policyType": "ACCESS", + "policyResourceCategory": "ENTITY", + "resources": [ + "entity:*{entity}", + "entity-type:AtlasGlossary", + "entity-type:AtlasGlossaryTerm", + "entity-type:AtlasGlossaryCategory", + "entity-classification:*" + ], + "actions": ["entity-read"] + }, + { + "policyType": "ACCESS", + "policyResourceCategory": "RELATIONSHIP", + "resources": [ + "relationship-type:*", + + "end-one-entity-type:AtlasGlossary", + "end-one-entity-type:AtlasGlossaryTerm", + "end-one-entity-type:AtlasGlossaryCategory", + "end-one-entity-classification:*", + "end-one-entity:*{entity}", + + "end-two-entity-type:AtlasGlossary", + "end-two-entity-type:AtlasGlossaryTerm", + "end-two-entity-type:AtlasGlossaryCategory", + "end-two-entity-classification:*", + "end-two-entity:*{entity}" + ], + "actions": ["add-relationship", "update-relationship", "remove-relationship"] + } + ], + "persona-glossary-update": [ + { + "policyType": "ACCESS", + "policyResourceCategory": "ENTITY", + "resources": [ + "entity:*{entity}", + "entity-type:AtlasGlossary", + "entity-type:AtlasGlossaryTerm", + "entity-type:AtlasGlossaryCategory", + "entity-classification:*" + ], + "actions": ["entity-update"] + }, + { + "policyType": "ACCESS", + "policyResourceCategory": "RELATIONSHIP", + "resources": [ + "relationship-type:*", + + "end-one-entity-type:AtlasGlossary", + "end-one-entity-type:AtlasGlossaryTerm", + "end-one-entity-type:AtlasGlossaryCategory", + "end-one-entity-classification:*", + "end-one-entity:*{entity}", + + "end-two-entity-type:*", + "end-two-entity-classification:*", + "end-two-entity:*" + ], + "actions": ["add-relationship", "update-relationship", "remove-relationship"] + }, + { + "policyType": "ACCESS", + "policyResourceCategory": "RELATIONSHIP", + "resources": [ + "relationship-type:*", + + "end-one-entity-type:*", + "end-one-entity-classification:*", + "end-one-entity:*", + + "end-two-entity-type:*", + "end-two-entity-classification:*", + "end-two-entity:*{entity}" + ], + "actions": ["add-relationship", "update-relationship", "remove-relationship"] + } + ], + "persona-glossary-create": [ + { + "policyType": "ACCESS", + "policyResourceCategory": "ENTITY", + "resources": [ + "entity:*{entity}", + "entity-type:AtlasGlossary", + "entity-type:AtlasGlossaryTerm", + "entity-type:AtlasGlossaryCategory", + "entity-classification:*" + ], + "actions": ["entity-create"] + } + ], + "persona-glossary-delete": [ + { + "policyType": "ACCESS", + "policyResourceCategory": "ENTITY", + "resources": [ + "entity:*{entity}", + "entity-type:AtlasGlossary", + "entity-type:AtlasGlossaryTerm", + "entity-type:AtlasGlossaryCategory", + "entity-classification:*" + ], + "actions": ["entity-delete"] + } + ], + "persona-glossary-update-custom-metadata": [ + { + "policyType": "ACCESS", + "policyResourceCategory": "ENTITY", + "resources": [ + "entity:*{entity}", + "entity-type:AtlasGlossary", + "entity-type:AtlasGlossaryTerm", + "entity-type:AtlasGlossaryCategory", + "entity-classification:*", + "entity-business-metadata:*" + ], + "actions": ["entity-update-business-metadata"] + } + ], + "persona-glossary-add-classifications": [ + { + "policyType": "ACCESS", + "policyResourceCategory": "ENTITY", + "resources": [ + "entity:*{entity}", + "entity-type:AtlasGlossary", + "entity-type:AtlasGlossaryTerm", + "entity-type:AtlasGlossaryCategory", + "entity-classification:*", + "classification:*" + ], + "actions": [ + "entity-add-classification", + "entity-update-classification" + ] + } + ], + "persona-glossary-update-classifications": [ + { + "policyType": "ACCESS", + "policyResourceCategory": "ENTITY", + "resources": [ + "entity:*{entity}", + "entity-type:AtlasGlossary", + "entity-type:AtlasGlossaryTerm", + "entity-type:AtlasGlossaryCategory", + "entity-classification:*", + "classification:*" + ], + "actions": ["entity-update-classification"] + } + ], + "persona-glossary-delete-classifications": [ + { + "policyType": "ACCESS", + "policyResourceCategory": "ENTITY", + "resources": [ + "entity:*{entity}", + "entity-type:AtlasGlossary", + "entity-type:AtlasGlossaryTerm", + "entity-type:AtlasGlossaryCategory", + "entity-classification:*", + "classification:*" + ], + "actions": [ + "entity-remove-classification", + "entity-update-classification" + ] + } + ], + "select": [ + { + "policyType": "ACCESS", + "policyResourceCategory": "ENTITY", + "resources": [ + "entity:{entity}", + "entity:{entity}/*", + "entity-type:{entity-type}" + ], + "actions": ["select"] + } + ] +} \ No newline at end of file diff --git a/addons/static/templates/policy_cache_transformer_purpose.json b/addons/static/templates/policy_cache_transformer_purpose.json new file mode 100644 index 00000000000..79e6aa25dc7 --- /dev/null +++ b/addons/static/templates/policy_cache_transformer_purpose.json @@ -0,0 +1,68 @@ +{ + "entity-update": [ + { + "policyType": "ACCESS", + "policyResourceCategory": "TAG", + "resources": [ + "tag:{tag}" + ], + "actions": ["entity-update"] + }, + { + "policyServiceName": "atlas", + "policyType": "ACCESS", + "policyResourceCategory": "RELATIONSHIP", + "resources": [ + "end-one-entity-classification:{tag}", + "end-one-entity:*", + "end-one-entity-type:*", + + "end-two-entity:*", + "end-two-entity-type:Readme", + "end-two-entity-type:Link", + "end-two-entity-classification:*", + + "relationship-type:*" + ], + "actions": ["add-relationship", "remove-relationship"] + } + ], + "purpose-add-terms": [ + { + "policyServiceName": "atlas", + "policyType": "ACCESS", + "policyResourceCategory": "RELATIONSHIP", + "resources": [ + "relationship-type:*", + + "end-one-entity-classification:*", + "end-one-entity:*", + "end-one-entity-type:AtlasGlossaryTerm", + + "end-two-entity:*", + "end-two-entity-type:*", + "end-two-entity-classification:{tag}" + ], + "actions": ["add-relationship"] + } + ], + "purpose-remove-terms": [ + { + "policyServiceName": "atlas", + "policyType": "ACCESS", + "policyResourceCategory": "RELATIONSHIP", + "resources": [ + "relationship-type:*", + + "end-one-entity-classification:*", + "end-one-entity:*", + "end-one-entity-type:AtlasGlossaryTerm", + + "end-two-entity:*", + "end-two-entity-type:*", + "end-two-entity-classification:{tag}" + ], + "actions": ["remove-relationship"] + } + ] +} \ No newline at end of file diff --git a/addons/storm-bridge/pom.xml b/addons/storm-bridge/pom.xml index e8106afce0e..77dce71536b 100644 --- a/addons/storm-bridge/pom.xml +++ b/addons/storm-bridge/pom.xml @@ -346,9 +346,9 @@ - log4j - log4j - ${log4j.version} + ch.qos.reload4j + reload4j + ${reload4j.version} com.sun.jersey diff --git a/atlas-hub/pre-conf/ranger/disable-atlas-plugin.sh b/atlas-hub/pre-conf/ranger/disable-atlas-plugin.sh index badea189860..a15728f6950 100755 --- a/atlas-hub/pre-conf/ranger/disable-atlas-plugin.sh +++ b/atlas-hub/pre-conf/ranger/disable-atlas-plugin.sh @@ -712,29 +712,6 @@ addOrUpdatePropertyToFile(){ fi } -if [ "${HCOMPONENT_NAME}" = "atlas" ] -then - if [ "${action}" = "enable" ] - then - authName="org.apache.ranger.authorization.atlas.authorizer.RangerAtlasAuthorizer" - else - authName="org.apache.atlas.authorize.SimpleAtlasAuthorizer" - fi - - dt=`date '+%Y%m%d%H%M%S'` - fn=`ls ${HCOMPONENT_CONF_DIR}/atlas-application.properties 2> /dev/null` - if [ -f "${fn}" ] - then - dn=`dirname ${fn}` - bn=`basename ${fn}` - bf=${dn}/.${bn}.${dt} - echo "backup of ${fn} to ${bf} ..." - cp ${fn} ${bf} - echo "Updating properties file: [${fn}] ... " - updatePropertyToFile atlas.authorizer.impl $authName ${fn} - fi -fi - if [ "${HCOMPONENT_NAME}" = "sqoop" ] then if [ "${action}" = "enable" ] diff --git a/atlas-hub/pre-conf/ranger/enable-atlas-plugin.sh b/atlas-hub/pre-conf/ranger/enable-atlas-plugin.sh index 6af472ed2a9..10c7fcfb0ac 100755 --- a/atlas-hub/pre-conf/ranger/enable-atlas-plugin.sh +++ b/atlas-hub/pre-conf/ranger/enable-atlas-plugin.sh @@ -715,29 +715,6 @@ addOrUpdatePropertyToFile(){ fi } -if [ "${HCOMPONENT_NAME}" = "atlas" ] -then - if [ "${action}" = "enable" ] - then - authName="org.apache.ranger.authorization.atlas.authorizer.RangerAtlasAuthorizer" - else - authName="org.apache.atlas.authorize.SimpleAtlasAuthorizer" - fi - - dt=`date '+%Y%m%d%H%M%S'` - fn=`ls ${HCOMPONENT_CONF_DIR}/atlas-application.properties 2> /dev/null` - if [ -f "${fn}" ] - then - dn=`dirname ${fn}` - bn=`basename ${fn}` - bf=${dn}/.${bn}.${dt} - echo "backup of ${fn} to ${bf} ..." - cp ${fn} ${bf} - echo "Updating properties file: [${fn}] ... " - updatePropertyToFile atlas.authorizer.impl $authName ${fn} - fi -fi - if [ "${HCOMPONENT_NAME}" = "sqoop" ] then if [ "${action}" = "enable" ] diff --git a/atlas-hub/pre-conf/ranger/install/conf.templates/enable/atlas-atlas-audit.xml b/atlas-hub/pre-conf/ranger/install/conf.templates/enable/atlas-atlas-audit.xml new file mode 100755 index 00000000000..39dbcdc1c3d --- /dev/null +++ b/atlas-hub/pre-conf/ranger/install/conf.templates/enable/atlas-atlas-audit.xml @@ -0,0 +1,139 @@ + + + + + + xasecure.audit.is.enabled + true + + + + + + xasecure.audit.destination.solr + false + + + + xasecure.audit.destination.solr.urls + NONE + + + + xasecure.audit.destination.solr.zookeepers + + + + + xasecure.audit.destination.solr.collection + NONE + + + + + + xasecure.audit.destination.hdfs + false + + + + xasecure.audit.destination.hdfs.dir + hdfs://__REPLACE__NAME_NODE_HOST:8020/ranger/audit + + + + xasecure.audit.destination.hdfs.subdir + %app-type%/%time:yyyyMMdd% + + + + xasecure.audit.destination.hdfs.filename.format + %app-type%_ranger_audit_%hostname%.log + + + + xasecure.audit.destination.hdfs.file.rollover.sec + 86400 + + + + + + + xasecure.audit.destination.log4j + false + + + + xasecure.audit.destination.log4j.logger + AUTH_AUDIT + + + + + xasecure.audit.destination.elasticsearch + true + + + xasecure.audit.elasticsearch.is.enabled + true/ + + + xasecure.audit.destination.elasticsearch.urls + logging-master.logging.svc.cluster.local + + + xasecure.audit.destination.elasticsearch.index + ranger-audit + + diff --git a/atlas-hub/pre-conf/ranger/install/conf.templates/enable/atlas-atlas-security-changes.cfg b/atlas-hub/pre-conf/ranger/install/conf.templates/enable/atlas-atlas-security-changes.cfg new file mode 100755 index 00000000000..8fd6e092740 --- /dev/null +++ b/atlas-hub/pre-conf/ranger/install/conf.templates/enable/atlas-atlas-security-changes.cfg @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Change the original policy parameter to work with policy manager based. +# +# +atlas.plugin.atlas.service.name %REPOSITORY_NAME% mod create-if-not-exists + diff --git a/atlas-hub/pre-conf/ranger/install/conf.templates/enable/atlas-atlas-security.xml b/atlas-hub/pre-conf/ranger/install/conf.templates/enable/atlas-atlas-security.xml new file mode 100755 index 00000000000..6b16b6ee1ee --- /dev/null +++ b/atlas-hub/pre-conf/ranger/install/conf.templates/enable/atlas-atlas-security.xml @@ -0,0 +1,88 @@ + + + + + + atlas.plugin.atlas.service.name + atlas + + Name of the Ranger service containing policies for this YARN instance + + + + + atlas.plugin.atlas.policy.source.impl + org.apache.atlas.authz.admin.client.AtlasAuthRESTClient + + Class to retrieve policies from the source + + + + + atlas.plugin.atlas.authz.rest.url + localhost:21000/api/atlas/v2/auth + + URL to Ranger Admin + + + + + atlas.plugin.atlas.policy.rest.ssl.config.file + /etc/atlas/conf/ranger-policymgr-ssl.xml + + Path to the file containing SSL details to contact Ranger Admin + + + + + atlas.plugin.atlas.policy.pollIntervalMs + 30000 + + How often to poll for changes in policies? + + + + + atlas.plugin.atlas.policy.cache.dir + /etc/atlas/atlasdev/policycache + + Directory where Ranger policies are cached after successful retrieval from the source + + + + + atlas.plugin.atlas.policy.rest.client.connection.timeoutMs + 120000 + + RangerRestClient Connection Timeout in Milli Seconds + + + + + atlas.plugin.atlas.policy.rest.client.read.timeoutMs + 120000 + + AtlasAuth read Timeout in Milli Seconds + + + + + atlas.plugin.atlas.policyengine.option.disable.tag.retriever + true + + diff --git a/atlas-hub/pre-conf/ranger/install/lib/slf4j-api-1.7.25.jar b/atlas-hub/pre-conf/ranger/install/lib/slf4j-api-1.7.25.jar deleted file mode 100644 index 0143c099699..00000000000 Binary files a/atlas-hub/pre-conf/ranger/install/lib/slf4j-api-1.7.25.jar and /dev/null differ diff --git a/atlas-hub/pre-conf/ranger/install/lib/slf4j-api-1.7.30.jar b/atlas-hub/pre-conf/ranger/install/lib/slf4j-api-1.7.30.jar new file mode 100644 index 00000000000..29ac26fb8ca Binary files /dev/null and b/atlas-hub/pre-conf/ranger/install/lib/slf4j-api-1.7.30.jar differ diff --git a/atlas-hub/pre-conf/ranger/lib/ranger-atlas-plugin-impl/ranger-atlas-plugin-2.2.0.jar b/atlas-hub/pre-conf/ranger/lib/ranger-atlas-plugin-impl/ranger-atlas-plugin-2.2.0.jar new file mode 100644 index 00000000000..178b6191eb0 Binary files /dev/null and b/atlas-hub/pre-conf/ranger/lib/ranger-atlas-plugin-impl/ranger-atlas-plugin-2.2.0.jar differ diff --git a/atlas-hub/pre-conf/ranger/lib/ranger-atlas-plugin-impl/ranger-atlas-plugin-3.0.0-SNAPSHOT.jar b/atlas-hub/pre-conf/ranger/lib/ranger-atlas-plugin-impl/ranger-atlas-plugin-3.0.0-SNAPSHOT.jar deleted file mode 100644 index cea0a71da09..00000000000 Binary files a/atlas-hub/pre-conf/ranger/lib/ranger-atlas-plugin-impl/ranger-atlas-plugin-3.0.0-SNAPSHOT.jar and /dev/null differ diff --git a/atlas-hub/pre-conf/ranger/lib/ranger-atlas-plugin-impl/ranger-plugins-audit-2.2.0-SNAPSHOT.jar b/atlas-hub/pre-conf/ranger/lib/ranger-atlas-plugin-impl/ranger-plugins-audit-2.2.0-SNAPSHOT.jar deleted file mode 100644 index d9a0d222ab2..00000000000 Binary files a/atlas-hub/pre-conf/ranger/lib/ranger-atlas-plugin-impl/ranger-plugins-audit-2.2.0-SNAPSHOT.jar and /dev/null differ diff --git a/atlas-hub/pre-conf/ranger/lib/ranger-atlas-plugin-impl/ranger-plugins-audit-2.2.0.jar b/atlas-hub/pre-conf/ranger/lib/ranger-atlas-plugin-impl/ranger-plugins-audit-2.2.0.jar new file mode 100644 index 00000000000..e108d19e1d7 Binary files /dev/null and b/atlas-hub/pre-conf/ranger/lib/ranger-atlas-plugin-impl/ranger-plugins-audit-2.2.0.jar differ diff --git a/atlas-hub/pre-conf/ranger/lib/ranger-atlas-plugin-impl/ranger-plugins-common-3.0.0-SNAPSHOT.jar b/atlas-hub/pre-conf/ranger/lib/ranger-atlas-plugin-impl/ranger-plugins-common-2.2.0.jar similarity index 73% rename from atlas-hub/pre-conf/ranger/lib/ranger-atlas-plugin-impl/ranger-plugins-common-3.0.0-SNAPSHOT.jar rename to atlas-hub/pre-conf/ranger/lib/ranger-atlas-plugin-impl/ranger-plugins-common-2.2.0.jar index ae271152779..cc23b8a0b26 100644 Binary files a/atlas-hub/pre-conf/ranger/lib/ranger-atlas-plugin-impl/ranger-plugins-common-3.0.0-SNAPSHOT.jar and b/atlas-hub/pre-conf/ranger/lib/ranger-atlas-plugin-impl/ranger-plugins-common-2.2.0.jar differ diff --git a/atlas-hub/pre-conf/ranger/lib/ranger-atlas-plugin-shim-3.0.0-SNAPSHOT.jar b/atlas-hub/pre-conf/ranger/lib/ranger-atlas-plugin-shim-3.0.0-SNAPSHOT.jar index 6db07efc41b..89893ad6ce8 100644 Binary files a/atlas-hub/pre-conf/ranger/lib/ranger-atlas-plugin-shim-3.0.0-SNAPSHOT.jar and b/atlas-hub/pre-conf/ranger/lib/ranger-atlas-plugin-shim-3.0.0-SNAPSHOT.jar differ diff --git a/auth-agents-common/pom.xml b/auth-agents-common/pom.xml new file mode 100644 index 00000000000..e2885cebf3f --- /dev/null +++ b/auth-agents-common/pom.xml @@ -0,0 +1,143 @@ + + + + + apache-atlas + org.apache.atlas + 3.0.0-SNAPSHOT + + 4.0.0 + + auth-agents-common + + + 8 + 8 + + 1.9.13 + 31.1-jre + + + + + org.apache.atlas + atlas-intg + ${project.version} + + + + org.apache.atlas + atlas-repository + ${project.version} + + + + org.apache.atlas + client-keycloak + ${project.version} + + + + org.apache.atlas + auth-audits + ${project.version} + + + + javax.servlet + javax.servlet-api + + + + com.sun.jersey + jersey-client + + + + + com.fasterxml.jackson.jaxrs + jackson-jaxrs-json-provider + 2.15.0 + + + + + com.google.guava + guava + ${guava.version} + + + + com.squareup.okio + okio + 2.8.0 + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.2.4 + + + package + + shade + + + false + true + + + *:* + + **/*.md + **/*.txt + **/*.yml + **/*.properties + + + + + + org.slf4j:* + com.google.api.grpc:* + org.janusgraph:* + com.google.protobuf:* + com.squareup.okhttp:* + org.apache.lucene:* + org.apache.hbase:* + com.datastax.oss:* + org.apache.tinkerpop + org.keycloak:* + + + + + + + + + + \ No newline at end of file diff --git a/auth-agents-common/src/main/java/org/apache/atlas/admin/client/AbstractRangerAdminClient.java b/auth-agents-common/src/main/java/org/apache/atlas/admin/client/AbstractRangerAdminClient.java new file mode 100644 index 00000000000..0a3d6c63262 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/admin/client/AbstractRangerAdminClient.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.admin.client; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import org.apache.hadoop.conf.Configuration; +import org.apache.atlas.plugin.model.RangerRole; +import org.apache.atlas.plugin.util.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; + +public abstract class AbstractRangerAdminClient implements RangerAdminClient { + private static final Logger LOG = LoggerFactory.getLogger(AbstractRangerAdminClient.class); + + protected Gson gson; + + @Override + public void init(String serviceName, String appId, String configPropertyPrefix, Configuration config) { + Gson gson = null; + + try { + gson = new GsonBuilder().setDateFormat("yyyyMMdd-HH:mm:ss.SSS-Z").setPrettyPrinting().create(); + } catch(Throwable excp) { + LOG.error("AbstractRangerAdminClient: failed to create GsonBuilder object", excp); + } + + this.gson = gson; + } + + @Override + public ServicePolicies getServicePoliciesIfUpdated(long lastKnownVersion, long lastActivationTimeInMillis) throws Exception { + return null; + } + + @Override + public RangerRoles getRolesIfUpdated(long lastKnownRoleVersion, long lastActivationTimeInMillis) throws Exception { + return null; + } + + @Override + public RangerRole createRole(RangerRole request) throws Exception { + return null; + } + + @Override + public void dropRole(String execUser, String roleName) throws Exception { + + } + + @Override + public List getAllRoles(String execUser) throws Exception { + return null; + } + + @Override + public List getUserRoles(String execUser) throws Exception { + return null; + } + + @Override + public RangerRole getRole(String execUser, String roleName) throws Exception { + return null; + } + + @Override + public void grantRole(GrantRevokeRoleRequest request) throws Exception { + + } + + @Override + public void revokeRole(GrantRevokeRoleRequest request) throws Exception { + + } + + @Override + public void grantAccess(GrantRevokeRequest request) throws Exception { + + } + + @Override + public void revokeAccess(GrantRevokeRequest request) throws Exception { + + } + + @Override + public ServiceTags getServiceTagsIfUpdated(long lastKnownVersion, long lastActivationTimeInMillis) throws Exception { + return null; + } + + @Override + public List getTagTypes(String tagTypePattern) throws Exception { + return null; + } + + @Override + public RangerUserStore getUserStoreIfUpdated(long lastKnownUserStoreVersion, long lastActivationTimeInMillis) throws Exception { + return null; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/admin/client/RangerAdminClient.java b/auth-agents-common/src/main/java/org/apache/atlas/admin/client/RangerAdminClient.java new file mode 100644 index 00000000000..795ffad0211 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/admin/client/RangerAdminClient.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + package org.apache.atlas.admin.client; + + +import org.apache.hadoop.conf.Configuration; +import org.apache.atlas.plugin.model.RangerRole; +import org.apache.atlas.plugin.util.GrantRevokeRequest; +import org.apache.atlas.plugin.util.GrantRevokeRoleRequest; +import org.apache.atlas.plugin.util.RangerRoles; +import org.apache.atlas.plugin.util.RangerUserStore; +import org.apache.atlas.plugin.util.ServicePolicies; +import org.apache.atlas.plugin.util.ServiceTags; + +import java.util.List; + + +public interface RangerAdminClient { + + void init(String serviceName, String appId, String configPropertyPrefix, Configuration config); + + ServicePolicies getServicePoliciesIfUpdated(long lastKnownVersion, long lastActivationTimeInMillis) throws Exception; + + RangerRoles getRolesIfUpdated(long lastKnownRoleVersion, long lastActivationTimeInMills) throws Exception; + + RangerRole createRole(RangerRole request) throws Exception; + + void dropRole(String execUser, String roleName) throws Exception; + + List getAllRoles(String execUser) throws Exception; + + List getUserRoles(String execUser) throws Exception; + + RangerRole getRole(String execUser, String roleName) throws Exception; + + void grantRole(GrantRevokeRoleRequest request) throws Exception; + + void revokeRole(GrantRevokeRoleRequest request) throws Exception; + + void grantAccess(GrantRevokeRequest request) throws Exception; + + void revokeAccess(GrantRevokeRequest request) throws Exception; + + ServiceTags getServiceTagsIfUpdated(long lastKnownVersion, long lastActivationTimeInMillis) throws Exception; + + List getTagTypes(String tagTypePattern) throws Exception; + + RangerUserStore getUserStoreIfUpdated(long lastKnownUserStoreVersion, long lastActivationTimeInMillis) throws Exception; + +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/admin/client/RangerAdminRESTClient.java b/auth-agents-common/src/main/java/org/apache/atlas/admin/client/RangerAdminRESTClient.java new file mode 100644 index 00000000000..49522fd602b --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/admin/client/RangerAdminRESTClient.java @@ -0,0 +1,1500 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.admin.client; + + +import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.GenericType; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.AccessControlException; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.atlas.admin.client.datatype.RESTResponse; +import org.apache.atlas.audit.provider.MiscUtil; +import org.apache.atlas.authorization.hadoop.config.RangerPluginConfig; +import org.apache.atlas.authorization.utils.StringUtil; +import org.apache.atlas.plugin.model.RangerRole; +import org.apache.atlas.plugin.util.*; + +import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.core.Cookie; +import javax.ws.rs.core.NewCookie; +import java.io.UnsupportedEncodingException; +import java.lang.reflect.ParameterizedType; +import java.lang.reflect.Type; +import java.security.PrivilegedAction; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class RangerAdminRESTClient extends AbstractRangerAdminClient { + private static final Log LOG = LogFactory.getLog(RangerAdminRESTClient.class); + + private String serviceName; + private String serviceNameUrlParam; + private String pluginId; + private String clusterName; + private RangerRESTClient restClient; + private RangerRESTUtils restUtils = new RangerRESTUtils(); + private boolean supportsPolicyDeltas; + private boolean supportsTagDeltas; + private boolean isRangerCookieEnabled; + private String rangerAdminCookieName; + private Cookie policyDownloadSessionId = null; + private boolean isValidPolicyDownloadSessionCookie = false; + private Cookie tagDownloadSessionId = null; + private boolean isValidTagDownloadSessionCookie = false; + private Cookie roleDownloadSessionId = null; + private Cookie userStoreDownloadSessionId = null; + private boolean isValidRoleDownloadSessionCookie = false; + private boolean isValidUserStoreDownloadSessionCookie = false; + private final String pluginCapabilities = Long.toHexString(new RangerPluginCapability().getPluginCapabilities()); + + public static GenericType> getGenericType(final T clazz) { + + ParameterizedType parameterizedGenericType = new ParameterizedType() { + public Type[] getActualTypeArguments() { + return new Type[] { clazz.getClass() }; + } + + public Type getRawType() { + return List.class; + } + + public Type getOwnerType() { + return List.class; + } + }; + + return new GenericType>(parameterizedGenericType) {}; + } + + @Override + public void init(String serviceName, String appId, String propertyPrefix, Configuration config) { + super.init(serviceName, appId, propertyPrefix, config); + + this.serviceName = serviceName; + this.pluginId = restUtils.getPluginId(serviceName, appId); + + String url = ""; + String tmpUrl = config.get(propertyPrefix + ".policy.rest.url"); + String sslConfigFileName = config.get(propertyPrefix + ".policy.rest.ssl.config.file"); + clusterName = config.get(propertyPrefix + ".access.cluster.name", ""); + if(StringUtil.isEmpty(clusterName)){ + clusterName =config.get(propertyPrefix + ".ambari.cluster.name", ""); + if (StringUtil.isEmpty(clusterName)) { + if (config instanceof RangerPluginConfig) { + clusterName = ((RangerPluginConfig)config).getClusterName(); + } + } + } + int restClientConnTimeOutMs = config.getInt(propertyPrefix + ".policy.rest.client.connection.timeoutMs", 120 * 1000); + int restClientReadTimeOutMs = config.getInt(propertyPrefix + ".policy.rest.client.read.timeoutMs", 30 * 1000); + supportsPolicyDeltas = config.getBoolean(propertyPrefix + RangerCommonConstants.PLUGIN_CONFIG_SUFFIX_POLICY_DELTA, RangerCommonConstants.PLUGIN_CONFIG_SUFFIX_POLICY_DELTA_DEFAULT); + supportsTagDeltas = config.getBoolean(propertyPrefix + RangerCommonConstants.PLUGIN_CONFIG_SUFFIX_TAG_DELTA, RangerCommonConstants.PLUGIN_CONFIG_SUFFIX_TAG_DELTA_DEFAULT); + isRangerCookieEnabled = config.getBoolean(propertyPrefix + ".policy.rest.client.cookie.enabled", RangerCommonConstants.POLICY_REST_CLIENT_SESSION_COOKIE_ENABLED); + rangerAdminCookieName = config.get(propertyPrefix + ".policy.rest.client.session.cookie.name", RangerCommonConstants.DEFAULT_COOKIE_NAME); + + if (!StringUtil.isEmpty(tmpUrl)) { + url = tmpUrl.trim(); + } + if (url.endsWith("/")) { + url = url.substring(0, url.length() - 1); + } + + init(url, sslConfigFileName, restClientConnTimeOutMs , restClientReadTimeOutMs, config); + + try { + this.serviceNameUrlParam = URLEncoderUtil.encodeURIParam(serviceName); + } catch (UnsupportedEncodingException e) { + LOG.warn("Unsupported encoding, serviceName=" + serviceName); + this.serviceNameUrlParam = serviceName; + } + } + + @Override + public ServicePolicies getServicePoliciesIfUpdated(final long lastKnownVersion, final long lastActivationTimeInMillis) throws Exception { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerAdminRESTClient.getServicePoliciesIfUpdated(" + lastKnownVersion + ", " + lastActivationTimeInMillis + ")"); + } + + final ServicePolicies ret; + + if (isRangerCookieEnabled && policyDownloadSessionId != null && isValidPolicyDownloadSessionCookie) { + ret = getServicePoliciesIfUpdatedWithCookie(lastKnownVersion, lastActivationTimeInMillis); + } else { + ret = getServicePoliciesIfUpdatedWithCred(lastKnownVersion, lastActivationTimeInMillis); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerAdminRESTClient.getServicePoliciesIfUpdated(" + lastKnownVersion + ", " + lastActivationTimeInMillis + "): " + ret); + } + + return ret; + } + + @Override + public RangerRoles getRolesIfUpdated(final long lastKnownRoleVersion, final long lastActivationTimeInMillis) throws Exception { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerAdminRESTClient.getRolesIfUpdated(" + lastKnownRoleVersion + ", " + lastActivationTimeInMillis + ")"); + } + + final RangerRoles ret; + + if (isRangerCookieEnabled && roleDownloadSessionId != null && isValidRoleDownloadSessionCookie) { + ret = getRolesIfUpdatedWithCookie(lastKnownRoleVersion, lastActivationTimeInMillis); + } else { + ret = getRolesIfUpdatedWithCred(lastKnownRoleVersion, lastActivationTimeInMillis); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerAdminRESTClient.getRolesIfUpdated(" + lastKnownRoleVersion + ", " + lastActivationTimeInMillis + "): "); + } + + return ret; + } + + @Override + public RangerRole createRole(final RangerRole request) throws Exception { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerAdminRESTClient.createRole(" + request + ")"); + } + + RangerRole ret = null; + + ClientResponse response = null; + UserGroupInformation user = MiscUtil.getUGILoginUser(); + boolean isSecureMode = user != null && UserGroupInformation.isSecurityEnabled(); + String relativeURL = RangerRESTUtils.REST_URL_SERVICE_CREATE_ROLE; + + Map queryParams = new HashMap (); + queryParams.put(RangerRESTUtils.SERVICE_NAME_PARAM, serviceNameUrlParam); + + if (isSecureMode) { + PrivilegedAction action = new PrivilegedAction() { + public ClientResponse run() { + ClientResponse clientRes = null; + try { + clientRes = restClient.post(relativeURL, queryParams, request); + } catch (Exception e) { + LOG.error("Failed to get response, Error is : "+e.getMessage()); + } + return clientRes; + } + }; + if (LOG.isDebugEnabled()) { + LOG.debug("create role as user " + user); + } + response = user.doAs(action); + } else { + response = restClient.post(relativeURL, queryParams, request); + } + + if(response != null && response.getStatus() != HttpServletResponse.SC_OK) { + RESTResponse resp = RESTResponse.fromClientResponse(response); + LOG.error("createRole() failed: HTTP status=" + response.getStatus() + ", message=" + resp.getMessage() + ", isSecure=" + isSecureMode + (isSecureMode ? (", user=" + user) : "")); + + if(response.getStatus()==HttpServletResponse.SC_UNAUTHORIZED) { + throw new AccessControlException(); + } + + throw new Exception("HTTP " + response.getStatus() + " Error: " + resp.getMessage()); + } else if(response == null) { + throw new Exception("unknown error during createRole. roleName=" + request.getName()); + } else { + ret = response.getEntity(RangerRole.class); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerAdminRESTClient.createRole(" + request + ")"); + } + return ret; + } + + @Override + public void dropRole(final String execUser, final String roleName) throws Exception { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerAdminRESTClient.dropRole(" + roleName + ")"); + } + + ClientResponse response = null; + UserGroupInformation user = MiscUtil.getUGILoginUser(); + boolean isSecureMode = user != null && UserGroupInformation.isSecurityEnabled(); + + Map queryParams = new HashMap(); + queryParams.put(RangerRESTUtils.SERVICE_NAME_PARAM, serviceNameUrlParam); + queryParams.put(RangerRESTUtils.REST_PARAM_EXEC_USER, execUser); + + String relativeURL = RangerRESTUtils.REST_URL_SERVICE_DROP_ROLE + roleName; + + if (isSecureMode) { + PrivilegedAction action = new PrivilegedAction() { + public ClientResponse run() { + ClientResponse clientRes = null; + try { + clientRes = restClient.delete(relativeURL, queryParams); + } catch (Exception e) { + LOG.error("Failed to get response, Error is : "+e.getMessage()); + } + return clientRes; + } + }; + if (LOG.isDebugEnabled()) { + LOG.debug("drop role as user " + user); + } + response = user.doAs(action); + } else { + response = restClient.delete(relativeURL, queryParams); + } + if(response == null) { + throw new Exception("unknown error during deleteRole. roleName=" + roleName); + } else if(response.getStatus() != HttpServletResponse.SC_OK && response.getStatus() != HttpServletResponse.SC_NO_CONTENT) { + RESTResponse resp = RESTResponse.fromClientResponse(response); + LOG.error("createRole() failed: HTTP status=" + response.getStatus() + ", message=" + resp.getMessage() + ", isSecure=" + isSecureMode + (isSecureMode ? (", user=" + user) : "")); + + if(response.getStatus()==HttpServletResponse.SC_UNAUTHORIZED) { + throw new AccessControlException(); + } + + throw new Exception("HTTP " + response.getStatus() + " Error: " + resp.getMessage()); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerAdminRESTClient.deleteRole(" + roleName + ")"); + } + } + + @Override + public List getUserRoles(final String execUser) throws Exception { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerAdminRESTClient.getUserRoles(" + execUser + ")"); + } + + List ret = null; + String emptyString = ""; + ClientResponse response = null; + UserGroupInformation user = MiscUtil.getUGILoginUser(); + boolean isSecureMode = user != null && UserGroupInformation.isSecurityEnabled(); + String relativeURL = RangerRESTUtils.REST_URL_SERVICE_GET_USER_ROLES + execUser; + + if (isSecureMode) { + PrivilegedAction action = new PrivilegedAction() { + public ClientResponse run() { + ClientResponse clientRes = null; + try { + clientRes = restClient.get(relativeURL, null); + } catch (Exception e) { + LOG.error("Failed to get response, Error is : "+e.getMessage()); + } + return clientRes; + } + }; + if (LOG.isDebugEnabled()) { + LOG.debug("get roles as user " + user); + } + response = user.doAs(action); + } else { + response = restClient.get(relativeURL, null); + } + if(response != null) { + if (response.getStatus() != HttpServletResponse.SC_OK) { + RESTResponse resp = RESTResponse.fromClientResponse(response); + LOG.error("getUserRoles() failed: HTTP status=" + response.getStatus() + ", message=" + resp.getMessage() + ", isSecure=" + isSecureMode + (isSecureMode ? (", user=" + user) : "")); + + if (response.getStatus() == HttpServletResponse.SC_UNAUTHORIZED) { + throw new AccessControlException(); + } + + throw new Exception("HTTP " + response.getStatus() + " Error: " + resp.getMessage()); + } else { + ret = response.getEntity(getGenericType(emptyString)); + } + } else { + throw new Exception("unknown error during getUserRoles. execUser=" + execUser); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerAdminRESTClient.getUserRoles(" + execUser + ")"); + } + return ret; + } + + @Override + public List getAllRoles(final String execUser) throws Exception { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerAdminRESTClient.getAllRoles()"); + } + + List ret = null; + String emptyString = ""; + ClientResponse response = null; + UserGroupInformation user = MiscUtil.getUGILoginUser(); + boolean isSecureMode = user != null && UserGroupInformation.isSecurityEnabled(); + String relativeURL = RangerRESTUtils.REST_URL_SERVICE_GET_ALL_ROLES; + + Map queryParams = new HashMap(); + queryParams.put(RangerRESTUtils.SERVICE_NAME_PARAM, serviceNameUrlParam); + queryParams.put(RangerRESTUtils.REST_PARAM_EXEC_USER, execUser); + + if (isSecureMode) { + PrivilegedAction action = new PrivilegedAction() { + public ClientResponse run() { + ClientResponse clientRes = null; + try { + clientRes = restClient.get(relativeURL, queryParams); + } catch (Exception e) { + LOG.error("Failed to get response, Error is : "+e.getMessage()); + } + return clientRes; + } + }; + if (LOG.isDebugEnabled()) { + LOG.debug("get roles as user " + user); + } + response = user.doAs(action); + } else { + response = restClient.get(relativeURL, queryParams); + } + if(response != null) { + if (response.getStatus() != HttpServletResponse.SC_OK) { + RESTResponse resp = RESTResponse.fromClientResponse(response); + LOG.error("getAllRoles() failed: HTTP status=" + response.getStatus() + ", message=" + resp.getMessage() + ", isSecure=" + isSecureMode + (isSecureMode ? (", user=" + user) : "")); + + if (response.getStatus() == HttpServletResponse.SC_UNAUTHORIZED) { + throw new AccessControlException(); + } + + throw new Exception("HTTP " + response.getStatus() + " Error: " + resp.getMessage()); + } else { + ret = response.getEntity(getGenericType(emptyString)); + } + } else { + throw new Exception("unknown error during getAllRoles."); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerAdminRESTClient.getAllRoles()"); + } + return ret; + } + + @Override + public RangerRole getRole(final String execUser, final String roleName) throws Exception { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerAdminRESTClient.getPrincipalsForRole(" + roleName + ")"); + } + + RangerRole ret = null; + ClientResponse response = null; + UserGroupInformation user = MiscUtil.getUGILoginUser(); + boolean isSecureMode = user != null && UserGroupInformation.isSecurityEnabled(); + String relativeURL = RangerRESTUtils.REST_URL_SERVICE_GET_ROLE_INFO + roleName; + + Map queryParams = new HashMap(); + queryParams.put(RangerRESTUtils.SERVICE_NAME_PARAM, serviceNameUrlParam); + queryParams.put(RangerRESTUtils.REST_PARAM_EXEC_USER, execUser); + + if (isSecureMode) { + PrivilegedAction action = new PrivilegedAction() { + public ClientResponse run() { + ClientResponse clientResp = null; + try { + clientResp = restClient.get(relativeURL, queryParams); + } catch (Exception e) { + LOG.error("Failed to get response, Error is : "+e.getMessage()); + } + return clientResp; + } + }; + if (LOG.isDebugEnabled()) { + LOG.debug("get role info as user " + user); + } + response = user.doAs(action); + } else { + response = restClient.get(relativeURL, queryParams); + } + if(response != null) { + if (response.getStatus() != HttpServletResponse.SC_OK) { + RESTResponse resp = RESTResponse.fromClientResponse(response); + LOG.error("getPrincipalsForRole() failed: HTTP status=" + response.getStatus() + ", message=" + resp.getMessage() + ", isSecure=" + isSecureMode + (isSecureMode ? (", user=" + user) : "")); + + if (response.getStatus() == HttpServletResponse.SC_UNAUTHORIZED) { + throw new AccessControlException(); + } + + throw new Exception("HTTP " + response.getStatus() + " Error: " + resp.getMessage()); + } else { + ret = response.getEntity(RangerRole.class); + } + } else { + throw new Exception("unknown error during getPrincipalsForRole. roleName=" + roleName); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerAdminRESTClient.getPrincipalsForRole(" + roleName + ")"); + } + return ret; + } + + + @Override + public void grantRole(final GrantRevokeRoleRequest request) throws Exception { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerAdminRESTClient.grantRole(" + request + ")"); + } + + ClientResponse response = null; + UserGroupInformation user = MiscUtil.getUGILoginUser(); + boolean isSecureMode = user != null && UserGroupInformation.isSecurityEnabled(); + String relativeURL = RangerRESTUtils.REST_URL_SERVICE_GRANT_ROLE + serviceNameUrlParam; + + if (isSecureMode) { + PrivilegedAction action = new PrivilegedAction() { + public ClientResponse run() { + ClientResponse clientResp = null; + try { + clientResp = restClient.put(relativeURL, null, request); + } catch (Exception e) { + LOG.error("Failed to get response, Error is : "+e.getMessage()); + } + return clientResp; + } + }; + if (LOG.isDebugEnabled()) { + LOG.debug("grant role as user " + user); + } + response = user.doAs(action); + } else { + response = restClient.put(relativeURL, null, request); + } + if(response != null && response.getStatus() != HttpServletResponse.SC_OK) { + RESTResponse resp = RESTResponse.fromClientResponse(response); + LOG.error("grantRole() failed: HTTP status=" + response.getStatus() + ", message=" + resp.getMessage() + ", isSecure=" + isSecureMode + (isSecureMode ? (", user=" + user) : "")); + + if(response.getStatus()==HttpServletResponse.SC_UNAUTHORIZED) { + throw new AccessControlException(); + } + + throw new Exception("HTTP " + response.getStatus() + " Error: " + resp.getMessage()); + } else if(response == null) { + throw new Exception("unknown error during grantRole. serviceName=" + serviceName); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerAdminRESTClient.grantRole(" + request + ")"); + } + } + + @Override + public void revokeRole(final GrantRevokeRoleRequest request) throws Exception { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerAdminRESTClient.revokeRole(" + request + ")"); + } + + ClientResponse response = null; + UserGroupInformation user = MiscUtil.getUGILoginUser(); + boolean isSecureMode = user != null && UserGroupInformation.isSecurityEnabled(); + String relativeURL = RangerRESTUtils.REST_URL_SERVICE_REVOKE_ROLE + serviceNameUrlParam; + + if (isSecureMode) { + PrivilegedAction action = new PrivilegedAction() { + public ClientResponse run() { + ClientResponse clientResp = null; + try { + clientResp = restClient.put(relativeURL, null, request); + } catch (Exception e) { + LOG.error("Failed to get response, Error is : "+e.getMessage()); + } + return clientResp; + } + }; + if (LOG.isDebugEnabled()) { + LOG.debug("revoke role as user " + user); + } + response = user.doAs(action); + } else { + response = restClient.put(relativeURL, null, request); + } + if(response != null && response.getStatus() != HttpServletResponse.SC_OK) { + RESTResponse resp = RESTResponse.fromClientResponse(response); + LOG.error("revokeRole() failed: HTTP status=" + response.getStatus() + ", message=" + resp.getMessage() + ", isSecure=" + isSecureMode + (isSecureMode ? (", user=" + user) : "")); + + if(response.getStatus()==HttpServletResponse.SC_UNAUTHORIZED) { + throw new AccessControlException(); + } + + throw new Exception("HTTP " + response.getStatus() + " Error: " + resp.getMessage()); + } else if(response == null) { + throw new Exception("unknown error during revokeRole. serviceName=" + serviceName); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerAdminRESTClient.revokeRole(" + request + ")"); + } + } + + @Override + public void grantAccess(final GrantRevokeRequest request) throws Exception { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerAdminRESTClient.grantAccess(" + request + ")"); + } + + ClientResponse response = null; + UserGroupInformation user = MiscUtil.getUGILoginUser(); + boolean isSecureMode = user != null && UserGroupInformation.isSecurityEnabled(); + + Map queryParams = new HashMap(); + queryParams.put(RangerRESTUtils.REST_PARAM_PLUGIN_ID, pluginId); + + if (isSecureMode) { + PrivilegedAction action = new PrivilegedAction() { + public ClientResponse run() { + String relativeURL = RangerRESTUtils.REST_URL_SECURE_SERVICE_GRANT_ACCESS + serviceNameUrlParam; + ClientResponse clientResp = null; + try { + clientResp = restClient.post(relativeURL, queryParams, request); + } catch (Exception e) { + LOG.error("Failed to get response, Error is : "+e.getMessage()); + } + return clientResp; + } + }; + if (LOG.isDebugEnabled()) { + LOG.debug("grantAccess as user " + user); + } + response = user.doAs(action); + } else { + String relativeURL = RangerRESTUtils.REST_URL_SERVICE_GRANT_ACCESS + serviceNameUrlParam; + response = restClient.post(relativeURL, queryParams, request); + } + if(response != null && response.getStatus() != HttpServletResponse.SC_OK) { + RESTResponse resp = RESTResponse.fromClientResponse(response); + LOG.error("grantAccess() failed: HTTP status=" + response.getStatus() + ", message=" + resp.getMessage() + ", isSecure=" + isSecureMode + (isSecureMode ? (", user=" + user) : "")); + + if(response.getStatus()==HttpServletResponse.SC_UNAUTHORIZED) { + throw new AccessControlException(); + } + + throw new Exception("HTTP " + response.getStatus() + " Error: " + resp.getMessage()); + } else if(response == null) { + throw new Exception("unknown error during grantAccess. serviceName=" + serviceName); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerAdminRESTClient.grantAccess(" + request + ")"); + } + } + + @Override + public void revokeAccess(final GrantRevokeRequest request) throws Exception { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerAdminRESTClient.revokeAccess(" + request + ")"); + } + + ClientResponse response = null; + UserGroupInformation user = MiscUtil.getUGILoginUser(); + boolean isSecureMode = user != null && UserGroupInformation.isSecurityEnabled(); + + Map queryParams = new HashMap(); + queryParams.put(RangerRESTUtils.REST_PARAM_PLUGIN_ID, pluginId); + + if (isSecureMode) { + PrivilegedAction action = new PrivilegedAction() { + public ClientResponse run() { + String relativeURL = RangerRESTUtils.REST_URL_SECURE_SERVICE_REVOKE_ACCESS + serviceNameUrlParam; + ClientResponse clientResp = null; + try { + clientResp = restClient.post(relativeURL, queryParams, request); + } catch (Exception e) { + LOG.error("Failed to get response, Error is : "+e.getMessage()); + } + return clientResp; + } + }; + if (LOG.isDebugEnabled()) { + LOG.debug("revokeAccess as user " + user); + } + response = user.doAs(action); + } else { + String relativeURL = RangerRESTUtils.REST_URL_SERVICE_REVOKE_ACCESS + serviceNameUrlParam; + response = restClient.post(relativeURL, queryParams, request); + } + + if(response != null && response.getStatus() != HttpServletResponse.SC_OK) { + RESTResponse resp = RESTResponse.fromClientResponse(response); + LOG.error("revokeAccess() failed: HTTP status=" + response.getStatus() + ", message=" + resp.getMessage() + ", isSecure=" + isSecureMode + (isSecureMode ? (", user=" + user) : "")); + + if(response.getStatus() == HttpServletResponse.SC_UNAUTHORIZED) { + throw new AccessControlException(); + } + + throw new Exception("HTTP " + response.getStatus() + " Error: " + resp.getMessage()); + } else if(response == null) { + throw new Exception("unknown error. revokeAccess(). serviceName=" + serviceName); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerAdminRESTClient.revokeAccess(" + request + ")"); + } + } + + private void init(String url, String sslConfigFileName, int restClientConnTimeOutMs , int restClientReadTimeOutMs, Configuration config) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerAdminRESTClient.init(" + url + ", " + sslConfigFileName + ")"); + } + + restClient = new RangerRESTClient(url, sslConfigFileName, config); + restClient.setRestClientConnTimeOutMs(restClientConnTimeOutMs); + restClient.setRestClientReadTimeOutMs(restClientReadTimeOutMs); + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerAdminRESTClient.init(" + url + ", " + sslConfigFileName + ")"); + } + } + + @Override + public ServiceTags getServiceTagsIfUpdated(final long lastKnownVersion, final long lastActivationTimeInMillis) throws Exception { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerAdminRESTClient.getServiceTagsIfUpdated(" + lastKnownVersion + ", " + lastActivationTimeInMillis + "): "); + } + + final ServiceTags ret; + + if (isRangerCookieEnabled && tagDownloadSessionId != null && isValidTagDownloadSessionCookie) { + ret = getServiceTagsIfUpdatedWithCookie(lastKnownVersion, lastActivationTimeInMillis); + } else { + ret = getServiceTagsIfUpdatedWithCred(lastKnownVersion, lastActivationTimeInMillis); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerAdminRESTClient.getServiceTagsIfUpdated(" + lastKnownVersion + ", " + lastActivationTimeInMillis + "): "); + } + + return ret; + } + + @Override + public List getTagTypes(String pattern) throws Exception { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerAdminRESTClient.getTagTypes(" + pattern + "): "); + } + + List ret = null; + String emptyString = ""; + UserGroupInformation user = MiscUtil.getUGILoginUser(); + boolean isSecureMode = user != null && UserGroupInformation.isSecurityEnabled(); + + Map queryParams = new HashMap(); + queryParams.put(RangerRESTUtils.SERVICE_NAME_PARAM, serviceNameUrlParam); + queryParams.put(RangerRESTUtils.PATTERN_PARAM, pattern); + String relativeURL = RangerRESTUtils.REST_URL_LOOKUP_TAG_NAMES; + + ClientResponse response = null; + if (isSecureMode) { + PrivilegedAction action = new PrivilegedAction() { + public ClientResponse run() { + ClientResponse clientResp = null; + try { + clientResp = restClient.get(relativeURL, queryParams); + } catch (Exception e) { + LOG.error("Failed to get response, Error is : "+e.getMessage()); + } + return clientResp; + } + }; + if (LOG.isDebugEnabled()) { + LOG.debug("getTagTypes as user " + user); + } + response = user.doAs(action); + } else { + response = restClient.get(relativeURL, queryParams); + } + + if(response != null && response.getStatus() == HttpServletResponse.SC_OK) { + ret = response.getEntity(getGenericType(emptyString)); + } else { + RESTResponse resp = RESTResponse.fromClientResponse(response); + LOG.error("Error getting tags. response=" + resp + ", serviceName=" + serviceName + ", " + "pattern=" + pattern); + throw new Exception(resp.getMessage()); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerAdminRESTClient.getTagTypes(" + pattern + "): " + ret); + } + + return ret; + } + + @Override + public RangerUserStore getUserStoreIfUpdated(long lastKnownUserStoreVersion, long lastActivationTimeInMillis) throws Exception { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerAdminRESTClient.getUserStoreIfUpdated(" + lastKnownUserStoreVersion + ", " + lastActivationTimeInMillis + ")"); + } + + + final RangerUserStore ret; + + if (isRangerCookieEnabled && userStoreDownloadSessionId != null && isValidUserStoreDownloadSessionCookie) { + ret = getUserStoreIfUpdatedWithCookie(lastKnownUserStoreVersion, lastActivationTimeInMillis); + } else { + ret = getUserStoreIfUpdatedWithCred(lastKnownUserStoreVersion, lastActivationTimeInMillis); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerAdminRESTClient.getUserStoreIfUpdated(" + lastKnownUserStoreVersion + ", " + lastActivationTimeInMillis + "): "); + } + + return ret; + } + + /* Policies Download ranger admin rest call methods */ + private ServicePolicies getServicePoliciesIfUpdatedWithCred(final long lastKnownVersion, final long lastActivationTimeInMillis) throws Exception { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerAdminRESTClient.getServicePoliciesIfUpdatedWithCred(" + lastKnownVersion + ", " + lastActivationTimeInMillis + ")"); + } + + final ServicePolicies ret; + + final UserGroupInformation user = MiscUtil.getUGILoginUser(); + final boolean isSecureMode = user != null && UserGroupInformation.isSecurityEnabled(); + final ClientResponse response = getRangerAdminPolicyDownloadResponse(lastKnownVersion, lastActivationTimeInMillis, user, isSecureMode); + + if (response == null || response.getStatus() == HttpServletResponse.SC_NOT_MODIFIED || response.getStatus() == HttpServletResponse.SC_NO_CONTENT) { + if (response == null) { + policyDownloadSessionId = null; + LOG.error("Error getting policies; Received NULL response!!. secureMode=" + isSecureMode + ", user=" + user + ", serviceName=" + serviceName); + } else { + setCookieReceivedFromCredSession(response); + RESTResponse resp = RESTResponse.fromClientResponse(response); + if (LOG.isDebugEnabled()) { + LOG.debug("No change in policies. secureMode=" + isSecureMode + ", user=" + user + ", response=" + resp + ", serviceName=" + serviceName); + } + } + ret = null; + } else if (response.getStatus() == HttpServletResponse.SC_OK) { + setCookieReceivedFromCredSession(response); + ret = response.getEntity(ServicePolicies.class); + } else if (response.getStatus() == HttpServletResponse.SC_NOT_FOUND) { + policyDownloadSessionId = null; + ret = null; + LOG.error("Error getting policies; service not found. secureMode=" + isSecureMode + ", user=" + user + + ", response=" + response.getStatus() + ", serviceName=" + serviceName + + ", " + "lastKnownVersion=" + lastKnownVersion + + ", " + "lastActivationTimeInMillis=" + lastActivationTimeInMillis); + String exceptionMsg = response.hasEntity() ? response.getEntity(String.class) : null; + RangerServiceNotFoundException.throwExceptionIfServiceNotFound(serviceName, exceptionMsg); + LOG.warn("Received 404 error code with body:[" + exceptionMsg + "], Ignoring"); + } else { + policyDownloadSessionId = null; + ret = null; + RESTResponse resp = RESTResponse.fromClientResponse(response); + LOG.warn("Error getting policies. secureMode=" + isSecureMode + ", user=" + user + ", response=" + resp + ", serviceName=" + serviceName); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerAdminRESTClient.getServicePoliciesIfUpdatedWithCred(" + lastKnownVersion + ", " + lastActivationTimeInMillis + "): " + ret); + } + + return ret; + } + + private ServicePolicies getServicePoliciesIfUpdatedWithCookie(final long lastKnownVersion, final long lastActivationTimeInMillis) throws Exception { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerAdminRESTClient.getServicePoliciesIfUpdatedWithCookie(" + lastKnownVersion + ", " + lastActivationTimeInMillis + ")"); + } + + final ServicePolicies ret; + + final UserGroupInformation user = MiscUtil.getUGILoginUser(); + final boolean isSecureMode = user != null && UserGroupInformation.isSecurityEnabled(); + final ClientResponse response = getRangerAdminPolicyDownloadResponse(lastKnownVersion, lastActivationTimeInMillis, user, isSecureMode); + + if (response == null || response.getStatus() == HttpServletResponse.SC_NOT_MODIFIED || response.getStatus() == HttpServletResponse.SC_NO_CONTENT) { + if (response == null) { + policyDownloadSessionId = null; + isValidPolicyDownloadSessionCookie = false; + LOG.error("Error getting policies; Received NULL response!!. secureMode=" + isSecureMode + ", user=" + user + ", serviceName=" + serviceName); + } else { + checkAndResetSessionCookie(response); + RESTResponse resp = RESTResponse.fromClientResponse(response); + if (LOG.isDebugEnabled()) { + LOG.debug("No change in policies. secureMode=" + isSecureMode + ", user=" + user + ", response=" + resp + ", serviceName=" + serviceName); + } + } + ret = null; + } else if (response.getStatus() == HttpServletResponse.SC_OK) { + checkAndResetSessionCookie(response); + ret = response.getEntity(ServicePolicies.class); + } else if (response.getStatus() == HttpServletResponse.SC_NOT_FOUND) { + policyDownloadSessionId = null; + isValidPolicyDownloadSessionCookie = false; + ret = null; + LOG.error("Error getting policies; service not found. secureMode=" + isSecureMode + ", user=" + user + + ", response=" + response.getStatus() + ", serviceName=" + serviceName + + ", " + "lastKnownVersion=" + lastKnownVersion + + ", " + "lastActivationTimeInMillis=" + lastActivationTimeInMillis); + String exceptionMsg = response.hasEntity() ? response.getEntity(String.class) : null; + RangerServiceNotFoundException.throwExceptionIfServiceNotFound(serviceName, exceptionMsg); + LOG.warn("Received 404 error code with body:[" + exceptionMsg + "], Ignoring"); + } else { + policyDownloadSessionId = null; + isValidPolicyDownloadSessionCookie = false; + ret = null; + RESTResponse resp = RESTResponse.fromClientResponse(response); + LOG.warn("Error getting policies. secureMode=" + isSecureMode + ", user=" + user + ", response=" + resp + ", serviceName=" + serviceName); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerAdminRESTClient.getServicePoliciesIfUpdatedWithCookie(" + lastKnownVersion + ", " + lastActivationTimeInMillis + "): " + ret); + } + + return ret; + } + + private ClientResponse getRangerAdminPolicyDownloadResponse(final long lastKnownVersion, final long lastActivationTimeInMillis, final UserGroupInformation user, final boolean isSecureMode) throws Exception { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerAdminRESTClient.getRangerAdminPolicyDownloadResponse(" + lastKnownVersion + ", " + lastActivationTimeInMillis + ")"); + } + + final ClientResponse ret; + + Map queryParams = new HashMap(); + queryParams.put(RangerRESTUtils.REST_PARAM_LAST_KNOWN_POLICY_VERSION, Long.toString(lastKnownVersion)); + queryParams.put(RangerRESTUtils.REST_PARAM_LAST_ACTIVATION_TIME, Long.toString(lastActivationTimeInMillis)); + queryParams.put(RangerRESTUtils.REST_PARAM_PLUGIN_ID, pluginId); + queryParams.put(RangerRESTUtils.REST_PARAM_CLUSTER_NAME, clusterName); + queryParams.put(RangerRESTUtils.REST_PARAM_SUPPORTS_POLICY_DELTAS, Boolean.toString(supportsPolicyDeltas)); + queryParams.put(RangerRESTUtils.REST_PARAM_CAPABILITIES, pluginCapabilities); + + if (isSecureMode) { + if (LOG.isDebugEnabled()) { + LOG.debug("Checking Service policy if updated as user : " + user); + } + PrivilegedAction action = new PrivilegedAction() { + public ClientResponse run() { + String relativeURL = RangerRESTUtils.REST_URL_POLICY_GET_FOR_SECURE_SERVICE_IF_UPDATED + serviceNameUrlParam; + ClientResponse clientResp = null; + try { + clientResp = restClient.get(relativeURL, queryParams, policyDownloadSessionId); + } catch (Exception e) { + LOG.error("Failed to get response, Error is : "+e.getMessage()); + } + return clientResp; + } + }; + ret = user.doAs(action); + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Checking Service policy if updated with old api call"); + } + String relativeURL = RangerRESTUtils.REST_URL_POLICY_GET_FOR_SERVICE_IF_UPDATED + serviceNameUrlParam; + ret = restClient.get(relativeURL, queryParams, policyDownloadSessionId); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerAdminRESTClient.getRangerAdminPolicyDownloadResponse(" + lastKnownVersion + ", " + lastActivationTimeInMillis + "): " + ret); + } + + return ret; + } + + private void checkAndResetSessionCookie(ClientResponse response) { + List respCookieList = response.getCookies(); + for (NewCookie respCookie : respCookieList) { + if (respCookie.getName().equalsIgnoreCase(rangerAdminCookieName)) { + policyDownloadSessionId = respCookie; + isValidPolicyDownloadSessionCookie = (policyDownloadSessionId != null); + break; + } + } + } + + private void setCookieReceivedFromCredSession(ClientResponse clientResponse) { + if (isRangerCookieEnabled) { + Cookie sessionCookie = null; + List cookieList = clientResponse.getCookies(); + // save cookie received from credentials session login + for (NewCookie cookie : cookieList) { + if (cookie.getName().equalsIgnoreCase(rangerAdminCookieName)) { + sessionCookie = cookie.toCookie(); + break; + } + } + policyDownloadSessionId = sessionCookie; + isValidPolicyDownloadSessionCookie = (policyDownloadSessionId != null); + } + } + + /* Tags Download ranger admin rest call */ + private ServiceTags getServiceTagsIfUpdatedWithCred(final long lastKnownVersion, final long lastActivationTimeInMillis) throws Exception { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerAdminRESTClient.getServiceTagsIfUpdatedWithCred(" + lastKnownVersion + ", " + lastActivationTimeInMillis + ")"); + } + + final ServiceTags ret; + + final UserGroupInformation user = MiscUtil.getUGILoginUser(); + final boolean isSecureMode = user != null && UserGroupInformation.isSecurityEnabled(); + final ClientResponse response = getRangerAdminTagDownloadResponse(lastKnownVersion, lastActivationTimeInMillis, user, isSecureMode); + + if (response == null || response.getStatus() == HttpServletResponse.SC_NOT_MODIFIED) { + if (response == null) { + tagDownloadSessionId = null; + LOG.error("Error getting tags; Received NULL response!!. secureMode=" + isSecureMode + ", user=" + user + ", serviceName=" + serviceName); + } else { + setCookieReceivedFromTagDownloadSession(response); + RESTResponse resp = RESTResponse.fromClientResponse(response); + if (LOG.isDebugEnabled()) { + LOG.debug("No change in tags. secureMode=" + isSecureMode + ", user=" + user + + ", response=" + resp + ", serviceName=" + serviceName + + ", " + "lastKnownVersion=" + lastKnownVersion + + ", " + "lastActivationTimeInMillis=" + lastActivationTimeInMillis); + } + } + ret = null; + } else if (response.getStatus() == HttpServletResponse.SC_OK) { + setCookieReceivedFromTagDownloadSession(response); + ret = response.getEntity(ServiceTags.class); + } else if (response.getStatus() == HttpServletResponse.SC_NOT_FOUND) { + tagDownloadSessionId = null; + ret = null; + LOG.error("Error getting tags; service not found. secureMode=" + isSecureMode + ", user=" + user + + ", response=" + response.getStatus() + ", serviceName=" + serviceName + + ", " + "lastKnownVersion=" + lastKnownVersion + + ", " + "lastActivationTimeInMillis=" + lastActivationTimeInMillis); + + String exceptionMsg = response.hasEntity() ? response.getEntity(String.class) : null; + RangerServiceNotFoundException.throwExceptionIfServiceNotFound(serviceName, exceptionMsg); + LOG.warn("Received 404 error code with body:[" + exceptionMsg + "], Ignoring"); + } else { + RESTResponse resp = RESTResponse.fromClientResponse(response); + LOG.warn("Error getting tags. secureMode=" + isSecureMode + ", user=" + user + ", response=" + resp + ", serviceName=" + serviceName); + tagDownloadSessionId = null; + ret = null; + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerAdminRESTClient.getServiceTagsIfUpdatedWithCred(" + lastKnownVersion + ", " + lastActivationTimeInMillis + "): " + ret); + } + + return ret; + } + + private ServiceTags getServiceTagsIfUpdatedWithCookie(final long lastKnownVersion, final long lastActivationTimeInMillis) throws Exception { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerAdminRESTClient.getServiceTagsIfUpdatedWithCookie(" + lastKnownVersion + ", " + lastActivationTimeInMillis + ")"); + } + + final ServiceTags ret; + + final UserGroupInformation user = MiscUtil.getUGILoginUser(); + final boolean isSecureMode = user != null && UserGroupInformation.isSecurityEnabled(); + final ClientResponse response = getRangerAdminTagDownloadResponse(lastKnownVersion, lastActivationTimeInMillis, user, isSecureMode); + + if (response == null || response.getStatus() == HttpServletResponse.SC_NOT_MODIFIED) { + if (response == null) { + tagDownloadSessionId = null; + isValidTagDownloadSessionCookie = false; + LOG.error("Error getting tags; Received NULL response!!. secureMode=" + isSecureMode + ", user=" + user + ", serviceName=" + serviceName); + } else { + checkAndResetTagDownloadSessionCookie(response); + RESTResponse resp = RESTResponse.fromClientResponse(response); + if (LOG.isDebugEnabled()) { + LOG.debug("No change in tags. secureMode=" + isSecureMode + ", user=" + user + + ", response=" + resp + ", serviceName=" + serviceName + + ", " + "lastKnownVersion=" + lastKnownVersion + + ", " + "lastActivationTimeInMillis=" + lastActivationTimeInMillis); + } + } + ret = null; + } else if (response.getStatus() == HttpServletResponse.SC_OK) { + checkAndResetTagDownloadSessionCookie(response); + ret = response.getEntity(ServiceTags.class); + } else if (response.getStatus() == HttpServletResponse.SC_NOT_FOUND) { + tagDownloadSessionId = null; + isValidTagDownloadSessionCookie = false; + ret = null; + LOG.error("Error getting tags; service not found. secureMode=" + isSecureMode + ", user=" + user + + ", response=" + response.getStatus() + ", serviceName=" + serviceName + + ", " + "lastKnownVersion=" + lastKnownVersion + + ", " + "lastActivationTimeInMillis=" + lastActivationTimeInMillis); + + String exceptionMsg = response.hasEntity() ? response.getEntity(String.class) : null; + RangerServiceNotFoundException.throwExceptionIfServiceNotFound(serviceName, exceptionMsg); + LOG.warn("Received 404 error code with body:[" + exceptionMsg + "], Ignoring"); + } else { + RESTResponse resp = RESTResponse.fromClientResponse(response); + LOG.warn("Error getting tags. secureMode=" + isSecureMode + ", user=" + user + ", response=" + resp + ", serviceName=" + serviceName); + tagDownloadSessionId = null; + isValidTagDownloadSessionCookie = false; + ret = null; + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerAdminRESTClient.getServiceTagsIfUpdatedWithCookie(" + lastKnownVersion + ", " + lastActivationTimeInMillis + "): " + ret); + } + + return ret; + } + + private ClientResponse getRangerAdminTagDownloadResponse(final long lastKnownVersion, final long lastActivationTimeInMillis, final UserGroupInformation user, final boolean isSecureMode) throws Exception { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerAdminRESTClient.getRangerAdminTagDownloadResponse(" + lastKnownVersion + ", " + lastActivationTimeInMillis + ")"); + } + + final ClientResponse ret; + + Map queryParams = new HashMap(); + queryParams.put(RangerRESTUtils.LAST_KNOWN_TAG_VERSION_PARAM, Long.toString(lastKnownVersion)); + queryParams.put(RangerRESTUtils.REST_PARAM_LAST_ACTIVATION_TIME, Long.toString(lastActivationTimeInMillis)); + queryParams.put(RangerRESTUtils.REST_PARAM_PLUGIN_ID, pluginId); + queryParams.put(RangerRESTUtils.REST_PARAM_SUPPORTS_TAG_DELTAS, Boolean.toString(supportsTagDeltas)); + queryParams.put(RangerRESTUtils.REST_PARAM_CAPABILITIES, pluginCapabilities); + + if (isSecureMode) { + PrivilegedAction action = new PrivilegedAction() { + public ClientResponse run() { + String relativeURL = RangerRESTUtils.REST_URL_GET_SECURE_SERVICE_TAGS_IF_UPDATED + serviceNameUrlParam; + ClientResponse clientResp = null; + try { + clientResp = restClient.get(relativeURL, queryParams, tagDownloadSessionId); + } catch (Exception e) { + LOG.error("Failed to get response, Error is : "+e.getMessage()); + } + return clientResp; + } + }; + if (LOG.isDebugEnabled()) { + LOG.debug("getServiceTagsIfUpdated as user " + user); + } + ret = user.doAs(action); + } else { + String relativeURL = RangerRESTUtils.REST_URL_GET_SERVICE_TAGS_IF_UPDATED + serviceNameUrlParam; + ret = restClient.get(relativeURL, queryParams); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerAdminRESTClient.getRangerAdminTagDownloadResponse(" + lastKnownVersion + ", " + lastActivationTimeInMillis + "): " + ret); + } + + return ret; + } + + private void checkAndResetTagDownloadSessionCookie(ClientResponse response) { + List respCookieList = response.getCookies(); + for (NewCookie respCookie : respCookieList) { + if (respCookie.getName().equalsIgnoreCase(rangerAdminCookieName)) { + tagDownloadSessionId = respCookie; + isValidTagDownloadSessionCookie = (tagDownloadSessionId != null); + break; + } + } + } + + private void setCookieReceivedFromTagDownloadSession(ClientResponse clientResponse) { + if (isRangerCookieEnabled) { + Cookie sessionCookie = null; + List cookieList = clientResponse.getCookies(); + // save cookie received from credentials session login + for (NewCookie cookie : cookieList) { + if (cookie.getName().equalsIgnoreCase(rangerAdminCookieName)) { + sessionCookie = cookie.toCookie(); + break; + } + } + tagDownloadSessionId = sessionCookie; + isValidTagDownloadSessionCookie = (tagDownloadSessionId != null); + } + } + + /* Roles Download ranger admin rest call methods */ + private RangerRoles getRolesIfUpdatedWithCred(final long lastKnownRoleVersion, final long lastActivationTimeInMillis) throws Exception { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerAdminRESTClient.getRolesIfUpdatedWithCred(" + lastKnownRoleVersion + ", " + lastActivationTimeInMillis + ")"); + } + + final RangerRoles ret; + + final UserGroupInformation user = MiscUtil.getUGILoginUser(); + final boolean isSecureMode = user != null && UserGroupInformation.isSecurityEnabled(); + final ClientResponse response = getRangerRolesDownloadResponse(lastKnownRoleVersion, lastActivationTimeInMillis, user, isSecureMode); + + if (response == null || response.getStatus() == HttpServletResponse.SC_NOT_MODIFIED || response.getStatus() == HttpServletResponse.SC_NO_CONTENT) { + if (response == null) { + roleDownloadSessionId = null; + LOG.error("Error getting Roles; Received NULL response!!. secureMode=" + isSecureMode + ", user=" + user + ", serviceName=" + serviceName); + } else { + setCookieReceivedFromRoleDownloadSession(response); + RESTResponse resp = RESTResponse.fromClientResponse(response); + if (LOG.isDebugEnabled()) { + LOG.debug("No change in Roles. secureMode=" + isSecureMode + ", user=" + user + + ", response=" + resp + ", serviceName=" + serviceName + + ", " + "lastKnownRoleVersion=" + lastKnownRoleVersion + + ", " + "lastActivationTimeInMillis=" + lastActivationTimeInMillis); + } + } + ret = null; + } else if (response.getStatus() == HttpServletResponse.SC_OK) { + setCookieReceivedFromRoleDownloadSession(response); + ret = response.getEntity(RangerRoles.class); + } else if (response.getStatus() == HttpServletResponse.SC_NOT_FOUND) { + roleDownloadSessionId = null; + ret = null; + LOG.error("Error getting Roles; service not found. secureMode=" + isSecureMode + ", user=" + user + + ", response=" + response.getStatus() + ", serviceName=" + serviceName + + ", " + "lastKnownRoleVersion=" + lastKnownRoleVersion + + ", " + "lastActivationTimeInMillis=" + lastActivationTimeInMillis); + String exceptionMsg = response.hasEntity() ? response.getEntity(String.class) : null; + + RangerServiceNotFoundException.throwExceptionIfServiceNotFound(serviceName, exceptionMsg); + + LOG.warn("Received 404 error code with body:[" + exceptionMsg + "], Ignoring"); + } else { + RESTResponse resp = RESTResponse.fromClientResponse(response); + LOG.warn("Error getting Roles. secureMode=" + isSecureMode + ", user=" + user + ", response=" + resp + ", serviceName=" + serviceName); + roleDownloadSessionId = null; + ret = null; + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerAdminRESTClient.getRolesIfUpdatedWithCred(" + lastKnownRoleVersion + ", " + lastActivationTimeInMillis + "): " + ret); + } + + return ret; + } + + private RangerRoles getRolesIfUpdatedWithCookie(final long lastKnownRoleVersion, final long lastActivationTimeInMillis) throws Exception { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerAdminRESTClient.getRolesIfUpdatedWithCookie(" + lastKnownRoleVersion + ", " + lastActivationTimeInMillis + ")"); + } + + final RangerRoles ret; + + final UserGroupInformation user = MiscUtil.getUGILoginUser(); + final boolean isSecureMode = user != null && UserGroupInformation.isSecurityEnabled(); + final ClientResponse response = getRangerRolesDownloadResponse(lastKnownRoleVersion, lastActivationTimeInMillis, user, isSecureMode); + + if (response == null || response.getStatus() == HttpServletResponse.SC_NOT_MODIFIED || response.getStatus() == HttpServletResponse.SC_NO_CONTENT) { + if (response == null) { + roleDownloadSessionId = null; + isValidRoleDownloadSessionCookie = false; + LOG.error("Error getting Roles; Received NULL response!!. secureMode=" + isSecureMode + ", user=" + user + ", serviceName=" + serviceName); + } else { + checkAndResetRoleDownloadSessionCookie(response); + RESTResponse resp = RESTResponse.fromClientResponse(response); + if (LOG.isDebugEnabled()) { + LOG.debug("No change in Roles. secureMode=" + isSecureMode + ", user=" + user + + ", response=" + resp + ", serviceName=" + serviceName + + ", " + "lastKnownRoleVersion=" + lastKnownRoleVersion + + ", " + "lastActivationTimeInMillis=" + lastActivationTimeInMillis); + } + } + ret = null; + } else if (response.getStatus() == HttpServletResponse.SC_OK) { + checkAndResetRoleDownloadSessionCookie(response); + ret = response.getEntity(RangerRoles.class); + } else if (response.getStatus() == HttpServletResponse.SC_NOT_FOUND) { + roleDownloadSessionId = null; + isValidRoleDownloadSessionCookie = false; + ret = null; + LOG.error("Error getting Roles; service not found. secureMode=" + isSecureMode + ", user=" + user + + ", response=" + response.getStatus() + ", serviceName=" + serviceName + + ", " + "lastKnownRoleVersion=" + lastKnownRoleVersion + + ", " + "lastActivationTimeInMillis=" + lastActivationTimeInMillis); + String exceptionMsg = response.hasEntity() ? response.getEntity(String.class) : null; + RangerServiceNotFoundException.throwExceptionIfServiceNotFound(serviceName, exceptionMsg); + LOG.warn("Received 404 error code with body:[" + exceptionMsg + "], Ignoring"); + } else { + RESTResponse resp = RESTResponse.fromClientResponse(response); + LOG.warn("Error getting Roles. secureMode=" + isSecureMode + ", user=" + user + ", response=" + resp + ", serviceName=" + serviceName); + roleDownloadSessionId = null; + isValidRoleDownloadSessionCookie = false; + ret = null; + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerAdminRESTClient.getRolesIfUpdatedWithCookie(" + lastKnownRoleVersion + ", " + lastActivationTimeInMillis + "): " + ret); + } + + return ret; + } + + /* Roles Download ranger admin rest call methods */ + private RangerUserStore getUserStoreIfUpdatedWithCred(final long lastKnownUserStoreVersion, final long lastActivationTimeInMillis) throws Exception { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerAdminRESTClient.getUserStoreIfUpdatedWithCred(" + lastKnownUserStoreVersion + ", " + lastActivationTimeInMillis + ")"); + } + + final RangerUserStore ret; + + final UserGroupInformation user = MiscUtil.getUGILoginUser(); + final boolean isSecureMode = user != null && UserGroupInformation.isSecurityEnabled(); + final ClientResponse response = getUserStoreDownloadResponse(lastKnownUserStoreVersion, lastActivationTimeInMillis, user, isSecureMode); + + if (response == null || response.getStatus() == HttpServletResponse.SC_NOT_MODIFIED || response.getStatus() == HttpServletResponse.SC_NO_CONTENT) { + if (response == null) { + userStoreDownloadSessionId = null; + LOG.error("Error getting UserStore; Received NULL response!!. secureMode=" + isSecureMode + ", user=" + user + ", serviceName=" + serviceName); + } else { + setCookieReceivedFromUserStoreDownloadSession(response); + RESTResponse resp = RESTResponse.fromClientResponse(response); + if (LOG.isDebugEnabled()) { + LOG.debug("No change in UserStore. secureMode=" + isSecureMode + ", user=" + user + + ", response=" + resp + ", serviceName=" + serviceName + + ", " + "lastKnownUserStoreVersion=" + lastKnownUserStoreVersion + + ", " + "lastActivationTimeInMillis=" + lastActivationTimeInMillis); + } + } + ret = null; + } else if (response.getStatus() == HttpServletResponse.SC_OK) { + setCookieReceivedFromUserStoreDownloadSession(response); + ret = response.getEntity(RangerUserStore.class); + } else if (response.getStatus() == HttpServletResponse.SC_NOT_FOUND) { + userStoreDownloadSessionId = null; + ret = null; + LOG.error("Error getting UserStore; service not found. secureMode=" + isSecureMode + ", user=" + user + + ", response=" + response.getStatus() + ", serviceName=" + serviceName + + ", " + "lastKnownUserStoreVersion=" + lastKnownUserStoreVersion + + ", " + "lastActivationTimeInMillis=" + lastActivationTimeInMillis); + String exceptionMsg = response.hasEntity() ? response.getEntity(String.class) : null; + + RangerServiceNotFoundException.throwExceptionIfServiceNotFound(serviceName, exceptionMsg); + + LOG.warn("Received 404 error code with body:[" + exceptionMsg + "], Ignoring"); + } else { + RESTResponse resp = RESTResponse.fromClientResponse(response); + LOG.warn("Error getting UserStore. secureMode=" + isSecureMode + ", user=" + user + ", response=" + resp + ", serviceName=" + serviceName); + userStoreDownloadSessionId = null; + ret = null; + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerAdminRESTClient.getUserStoreIfUpdatedWithCred(" + lastKnownUserStoreVersion + ", " + lastActivationTimeInMillis + "): " + ret); + } + + return ret; + } + + private RangerUserStore getUserStoreIfUpdatedWithCookie(final long lastKnownUserStoreVersion, final long lastActivationTimeInMillis) throws Exception { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerAdminRESTClient.getUserStoreIfUpdatedWithCookie(" + lastKnownUserStoreVersion + ", " + lastActivationTimeInMillis + ")"); + } + + final RangerUserStore ret; + + final UserGroupInformation user = MiscUtil.getUGILoginUser(); + final boolean isSecureMode = user != null && UserGroupInformation.isSecurityEnabled(); + final ClientResponse response = getUserStoreDownloadResponse(lastKnownUserStoreVersion, lastActivationTimeInMillis, user, isSecureMode); + + if (response == null || response.getStatus() == HttpServletResponse.SC_NOT_MODIFIED || response.getStatus() == HttpServletResponse.SC_NO_CONTENT) { + if (response == null) { + userStoreDownloadSessionId = null; + isValidUserStoreDownloadSessionCookie = false; + LOG.error("Error getting Roles; Received NULL response!!. secureMode=" + isSecureMode + ", user=" + user + ", serviceName=" + serviceName); + } else { + checkAndResetUserStoreDownloadSessionCookie(response); + RESTResponse resp = RESTResponse.fromClientResponse(response); + if (LOG.isDebugEnabled()) { + LOG.debug("No change in Roles. secureMode=" + isSecureMode + ", user=" + user + + ", response=" + resp + ", serviceName=" + serviceName + + ", " + "lastKnownRoleVersion=" + lastKnownUserStoreVersion + + ", " + "lastActivationTimeInMillis=" + lastActivationTimeInMillis); + } + } + ret = null; + } else if (response.getStatus() == HttpServletResponse.SC_OK) { + checkAndResetUserStoreDownloadSessionCookie(response); + ret = response.getEntity(RangerUserStore.class); + } else if (response.getStatus() == HttpServletResponse.SC_NOT_FOUND) { + userStoreDownloadSessionId = null; + isValidUserStoreDownloadSessionCookie = false; + ret = null; + LOG.error("Error getting UserStore; service not found. secureMode=" + isSecureMode + ", user=" + user + + ", response=" + response.getStatus() + ", serviceName=" + serviceName + + ", " + "lastKnownUserStoreVersion=" + lastKnownUserStoreVersion + + ", " + "lastActivationTimeInMillis=" + lastActivationTimeInMillis); + String exceptionMsg = response.hasEntity() ? response.getEntity(String.class) : null; + RangerServiceNotFoundException.throwExceptionIfServiceNotFound(serviceName, exceptionMsg); + LOG.warn("Received 404 error code with body:[" + exceptionMsg + "], Ignoring"); + } else { + RESTResponse resp = RESTResponse.fromClientResponse(response); + LOG.warn("Error getting UserStore. secureMode=" + isSecureMode + ", user=" + user + ", response=" + resp + ", serviceName=" + serviceName); + userStoreDownloadSessionId = null; + isValidUserStoreDownloadSessionCookie = false; + ret = null; + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerAdminRESTClient.getUserStoreIfUpdatedWithCookie(" + lastKnownUserStoreVersion + ", " + lastActivationTimeInMillis + "): " + ret); + } + + return ret; + } + + private ClientResponse getRangerRolesDownloadResponse(final long lastKnownRoleVersion, final long lastActivationTimeInMillis, final UserGroupInformation user, final boolean isSecureMode) throws Exception { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerAdminRESTClient.getRangerRolesDownloadResponse(" + lastKnownRoleVersion + ", " + lastActivationTimeInMillis + ")"); + } + + final ClientResponse ret; + + Map queryParams = new HashMap(); + queryParams.put(RangerRESTUtils.REST_PARAM_LAST_KNOWN_ROLE_VERSION, Long.toString(lastKnownRoleVersion)); + queryParams.put(RangerRESTUtils.REST_PARAM_LAST_ACTIVATION_TIME, Long.toString(lastActivationTimeInMillis)); + queryParams.put(RangerRESTUtils.REST_PARAM_PLUGIN_ID, pluginId); + queryParams.put(RangerRESTUtils.REST_PARAM_CLUSTER_NAME, clusterName); + queryParams.put(RangerRESTUtils.REST_PARAM_CAPABILITIES, pluginCapabilities); + + if (isSecureMode) { + if (LOG.isDebugEnabled()) { + LOG.debug("Checking Roles updated as user : " + user + " isSecureMode :" + isSecureMode); + } + PrivilegedAction action = new PrivilegedAction() { + public ClientResponse run() { + ClientResponse clientRes = null; + String relativeURL = RangerRESTUtils.REST_URL_SERVICE_SERCURE_GET_USER_GROUP_ROLES + serviceNameUrlParam; + try { + clientRes = restClient.get(relativeURL, queryParams, roleDownloadSessionId); + } catch (Exception e) { + LOG.error("Failed to get response, Error is : "+e.getMessage()); + } + return clientRes; + } + }; + ret = user.doAs(action); + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Checking Roles updated as user : " + user + " isSecureMode :" + isSecureMode); + } + String relativeURL = RangerRESTUtils.REST_URL_SERVICE_GET_USER_GROUP_ROLES + serviceNameUrlParam; + ret = restClient.get(relativeURL, queryParams); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerAdminRESTClient.getRangerRolesDownloadResponse(" + lastKnownRoleVersion + ", " + lastActivationTimeInMillis + "): " + ret); + } + + return ret; + } + + private ClientResponse getUserStoreDownloadResponse(final long lastKnownUserStoreVersion, final long lastActivationTimeInMillis, final UserGroupInformation user, final boolean isSecureMode) throws Exception { + + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerAdminRESTClient.getUserStoreDownloadResponse(" + lastKnownUserStoreVersion + ", " + lastActivationTimeInMillis + ")"); + } + + final ClientResponse ret; + + Map queryParams = new HashMap(); + queryParams.put(RangerRESTUtils.REST_PARAM_LAST_KNOWN_USERSTORE_VERSION, Long.toString(lastKnownUserStoreVersion)); + queryParams.put(RangerRESTUtils.REST_PARAM_LAST_ACTIVATION_TIME, Long.toString(lastActivationTimeInMillis)); + queryParams.put(RangerRESTUtils.REST_PARAM_PLUGIN_ID, pluginId); + queryParams.put(RangerRESTUtils.REST_PARAM_CLUSTER_NAME, clusterName); + queryParams.put(RangerRESTUtils.REST_PARAM_CAPABILITIES, pluginCapabilities); + + if (isSecureMode) { + if (LOG.isDebugEnabled()) { + LOG.debug("Checking UserStore updated as user : " + user + " isSecureMode :" + isSecureMode); + } + PrivilegedAction action = new PrivilegedAction() { + public ClientResponse run() { + ClientResponse clientRes = null; + String relativeURL = RangerRESTUtils.REST_URL_SERVICE_SERCURE_GET_USERSTORE + serviceNameUrlParam; + try { + clientRes = restClient.get(relativeURL, queryParams, userStoreDownloadSessionId); + } catch (Exception e) { + LOG.error("Failed to get response, Error is : "+e.getMessage()); + } + return clientRes; + } + }; + ret = user.doAs(action); + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Checking UserStore updated as user : " + user + " isSecureMode :" + isSecureMode); + } + String relativeURL = RangerRESTUtils.REST_URL_SERVICE_GET_USERSTORE + serviceNameUrlParam; + ret = restClient.get(relativeURL, queryParams); + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerAdminRESTClient.getUserStoreDownloadResponse(" + restClient.getUsername() + ", " + restClient.getPassword() + "): " + ret); + } + + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerAdminRESTClient.getUserStoreDownloadResponse(" + lastKnownUserStoreVersion + ", " + lastActivationTimeInMillis + "): " + ret); + } + + return ret; + } + + private void checkAndResetRoleDownloadSessionCookie(ClientResponse response) { + List respCookieList = response.getCookies(); + for (NewCookie respCookie : respCookieList) { + if (respCookie.getName().equalsIgnoreCase(rangerAdminCookieName)) { + roleDownloadSessionId = respCookie; + isValidRoleDownloadSessionCookie = (roleDownloadSessionId != null); + break; + } + } + } + private void checkAndResetUserStoreDownloadSessionCookie(ClientResponse response) { + List respCookieList = response.getCookies(); + for (NewCookie respCookie : respCookieList) { + if (respCookie.getName().equalsIgnoreCase(rangerAdminCookieName)) { + userStoreDownloadSessionId = respCookie; + isValidUserStoreDownloadSessionCookie = (userStoreDownloadSessionId != null); + break; + } + } + } + + private void setCookieReceivedFromRoleDownloadSession(ClientResponse clientResponse) { + if (isRangerCookieEnabled) { + Cookie sessionCookie = null; + List cookieList = clientResponse.getCookies(); + // save cookie received from credentials session login + for (NewCookie cookie : cookieList) { + if (cookie.getName().equalsIgnoreCase(rangerAdminCookieName)) { + sessionCookie = cookie.toCookie(); + break; + } + } + roleDownloadSessionId = sessionCookie; + isValidRoleDownloadSessionCookie = (roleDownloadSessionId != null); + } + } + private void setCookieReceivedFromUserStoreDownloadSession(ClientResponse clientResponse) { + if (isRangerCookieEnabled) { + Cookie sessionCookie = null; + List cookieList = clientResponse.getCookies(); + // save cookie received from credentials session login + for (NewCookie cookie : cookieList) { + if (cookie.getName().equalsIgnoreCase(rangerAdminCookieName)) { + sessionCookie = cookie.toCookie(); + break; + } + } + userStoreDownloadSessionId = sessionCookie; + isValidUserStoreDownloadSessionCookie = (userStoreDownloadSessionId != null); + } + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/admin/client/datatype/GrantRevokeData.java b/auth-agents-common/src/main/java/org/apache/atlas/admin/client/datatype/GrantRevokeData.java new file mode 100644 index 00000000000..22c9470abc0 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/admin/client/datatype/GrantRevokeData.java @@ -0,0 +1,246 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + package org.apache.atlas.admin.client.datatype; + + +import org.apache.atlas.authorization.utils.StringUtil; +import org.apache.atlas.plugin.util.JsonUtilsV2; +import org.codehaus.jackson.annotate.JsonAutoDetect; +import org.codehaus.jackson.annotate.JsonAutoDetect.Visibility; +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +import java.util.ArrayList; +import java.util.List; + + +@JsonAutoDetect(getterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE, fieldVisibility = Visibility.ANY) +@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown = true) +public class GrantRevokeData implements java.io.Serializable { + private static final long serialVersionUID = 1L; + + private String grantor; + private String repositoryName; + private String repositoryType; + private String databases; + private String tables; + private String columns; + private String columnFamilies; + private List permMapList = new ArrayList<>(); + + private static String WILDCARD_ASTERISK = "*"; + + public GrantRevokeData() { + } + + public String getGrantor() { + return grantor; + } + + public void setGrantor(String grantor) { + this.grantor = grantor; + } + + public String getRepositoryName() { + return repositoryName; + } + + public void setRepositoryName(String repositoryName) { + this.repositoryName = repositoryName; + } + + public String getRepositoryType() { + return repositoryType; + } + + public void setRepositoryType(String repositoryType) { + this.repositoryType = repositoryType; + } + + public String getDatabases() { + return databases; + } + + public void setDatabases(String databases) { + this.databases = databases; + } + + public String getTables() { + return tables; + } + + public void setTables(String tables) { + this.tables = tables; + } + + public String getColumns() { + return columns; + } + + public void setColumns(String columns) { + this.columns = columns; + } + + public String getColumnFamilies() { + return columnFamilies; + } + + public void setColumnFamilies(String columnFamilies) { + this.columnFamilies = columnFamilies; + } + + public List getPermMapList() { + return permMapList; + } + + public void setPermMapList(List permMapList) { + this.permMapList = permMapList; + } + + + public void setHiveData(String grantor, + String repositoryName, + String databases, + String tables, + String columns, + PermMap permMap) { + this.grantor = grantor; + this.repositoryName = repositoryName; + this.repositoryType = "hive"; + this.databases = StringUtil.isEmpty(databases) ? WILDCARD_ASTERISK : databases; + this.tables = StringUtil.isEmpty(tables) ? WILDCARD_ASTERISK : tables; + this.columns = StringUtil.isEmpty(columns) ? WILDCARD_ASTERISK : columns; + this.permMapList.add(permMap); + } + + public void setHBaseData(String grantor, + String repositoryName, + String tables, + String columns, + String columnFamilies, + PermMap permMap) { + this.grantor = grantor; + this.repositoryName = repositoryName; + this.repositoryType = "hbase"; + this.tables = StringUtil.isEmpty(tables) ? WILDCARD_ASTERISK : tables; + this.columns = StringUtil.isEmpty(columns) ? WILDCARD_ASTERISK : columns; + this.columnFamilies = StringUtil.isEmpty(columnFamilies) ? WILDCARD_ASTERISK : columnFamilies; + this.permMapList.add(permMap); + } + + public String toJson() { + try { + return JsonUtilsV2.objToJson(this); + } catch (Exception e) { + e.printStackTrace(); + } + + return ""; + } + + @Override + public String toString() { + return toJson(); + } + + @JsonAutoDetect(getterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE, fieldVisibility = Visibility.ANY) + @JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) + @JsonIgnoreProperties(ignoreUnknown = true) + public static class PermMap implements java.io.Serializable { + private List userList = new ArrayList<>(); + private List groupList = new ArrayList<>(); + private List permList = new ArrayList<>(); + + public PermMap() { + } + + public PermMap(String user, String group, String perm) { + addUser(user); + addGroup(group); + addPerm(perm); + } + + public PermMap(List userList, List groupList, List permList) { + copyList(userList, this.userList); + copyList(groupList, this.groupList); + copyList(permList, this.permList); + } + + public List getUserList() { + return userList; + } + + public List getGroupList() { + return groupList; + } + + public List getPermList() { + return permList; + } + + public void addUser(String user) { + addToList(user, userList); + } + + public void addGroup(String group) { + addToList(group, groupList); + } + + public void addPerm(String perm) { + addToList(perm, permList); + } + + private void addToList(String str, List list) { + if(list != null && !StringUtil.isEmpty(str)) { + list.add(str); + } + } + + private void copyList(List fromList, List toList) { + if(fromList != null && toList != null) { + for(String str : fromList) { + addToList(str, toList); + } + } + } + + public String toJson() { + try { + return JsonUtilsV2.objToJson(this); + } catch (Exception e) { + e.printStackTrace(); + } + + return ""; + } + + @Override + public String toString() { + return toJson(); + } + } + + public static void main(String[] args) { + GrantRevokeData grData = new GrantRevokeData(); + + System.out.println(grData.toString()); + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/admin/client/datatype/RESTResponse.java b/auth-agents-common/src/main/java/org/apache/atlas/admin/client/datatype/RESTResponse.java new file mode 100644 index 00000000000..f48d1c56888 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/admin/client/datatype/RESTResponse.java @@ -0,0 +1,211 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.admin.client.datatype; + +import com.sun.jersey.api.client.ClientResponse; +import org.apache.atlas.authorization.utils.StringUtil; +import org.apache.atlas.plugin.util.JsonUtilsV2; +import org.codehaus.jackson.annotate.JsonAutoDetect; +import org.codehaus.jackson.annotate.JsonAutoDetect.Visibility; +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; + + +@JsonAutoDetect(getterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE, fieldVisibility = Visibility.ANY) +@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown = true) +public class RESTResponse implements java.io.Serializable { + private static final Logger LOG = LoggerFactory.getLogger(RESTResponse.class); + + /** + * values for statusCode + */ + public static final int STATUS_SUCCESS = 0; + public static final int STATUS_ERROR = 1; + public static final int STATUS_VALIDATION = 2; + public static final int STATUS_WARN = 3; + public static final int STATUS_INFO = 4; + public static final int STATUS_PARTIAL_SUCCESS = 5; + public static final int ResponseStatus_MAX = 5; + + private int httpStatusCode; + private int statusCode; + private String msgDesc; + private List messageList; + + + public int getHttpStatusCode() { + return httpStatusCode; + } + + public void setHttpStatusCode(int httpStatusCode) { + this.httpStatusCode = httpStatusCode; + } + + public int getStatusCode() { + return statusCode; + } + + public void setStatusCode(int statusCode) { + this.statusCode = statusCode; + } + + public String getMsgDesc() { + return msgDesc; + } + + public void setMsgDesc(String msgDesc) { + this.msgDesc = msgDesc; + } + + public List getMessageList() { + return messageList; + } + + public void setMessageList(List messageList) { + this.messageList = messageList; + } + + public String getMessage() { + return StringUtil.isEmpty(msgDesc) ? ("HTTP " + httpStatusCode) : msgDesc; + } + + public static RESTResponse fromClientResponse(ClientResponse response) { + RESTResponse ret = null; + + String jsonString = response == null ? null : response.getEntity(String.class); + int httpStatus = response == null ? 0 : response.getStatus(); + + if(! StringUtil.isEmpty(jsonString)) { + ret = RESTResponse.fromJson(jsonString); + } + + if(ret == null) { + ret = new RESTResponse(); + } + + ret.setHttpStatusCode(httpStatus); + + return ret; + } + + public String toJson() { + try { + return JsonUtilsV2.objToJson(this); + } catch (Exception e) { + if(LOG.isDebugEnabled()) { + LOG.debug("toJson() failed", e); + } + } + + return ""; + } + + public static RESTResponse fromJson(String jsonString) { + try { + return JsonUtilsV2.jsonToObj(jsonString, RESTResponse.class); + } catch (Exception e) { + if(LOG.isDebugEnabled()) { + LOG.debug("fromJson('" + jsonString + "') failed", e); + } + } + + return null; + } + + @Override + public String toString() { + return toJson(); + } + + @JsonAutoDetect(getterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE, fieldVisibility = Visibility.ANY) + @JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) + @JsonIgnoreProperties(ignoreUnknown = true) + public static class Message implements java.io.Serializable { + private String name; + private String rbKey; + private String message; + private Long objectId; + private String fieldName; + + public String getName() { + return name; + } + public void setName(String name) { + this.name = name; + } + public String getRbKey() { + return rbKey; + } + public void setRbKey(String rbKey) { + this.rbKey = rbKey; + } + public String getMessage() { + return message; + } + public void setMessage(String message) { + this.message = message; + } + public Long getObjectId() { + return objectId; + } + public void setObjectId(Long objectId) { + this.objectId = objectId; + } + public String getFieldName() { + return fieldName; + } + public void setFieldName(String fieldName) { + this.fieldName = fieldName; + } + + public String toJson() { + try { + return JsonUtilsV2.objToJson(this); + } catch (Exception e) { + if(LOG.isDebugEnabled()) { + LOG.debug("toJson() failed", e); + } + } + + return ""; + } + + public static RESTResponse fromJson(String jsonString) { + try { + return JsonUtilsV2.jsonToObj(jsonString, RESTResponse.class); + } catch (Exception e) { + if(LOG.isDebugEnabled()) { + LOG.debug("fromJson('" + jsonString + "') failed", e); + } + } + + return null; + } + + @Override + public String toString() { + return toJson(); + } + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/authorization/hadoop/config/RangerAdminConfig.java b/auth-agents-common/src/main/java/org/apache/atlas/authorization/hadoop/config/RangerAdminConfig.java new file mode 100644 index 00000000000..32e32cfe56b --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/authorization/hadoop/config/RangerAdminConfig.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.authorization.hadoop.config; + +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.security.KeyStore; + +public class RangerAdminConfig extends RangerConfiguration { + private static final Logger LOG = LoggerFactory.getLogger(RangerAdminConfig.class); + + private static volatile RangerAdminConfig sInstance = null; + private final boolean isFipsEnabled; + + public static RangerAdminConfig getInstance() { + RangerAdminConfig ret = RangerAdminConfig.sInstance; + + if (ret == null) { + synchronized (RangerAdminConfig.class) { + ret = RangerAdminConfig.sInstance; + + if (ret == null) { + ret = RangerAdminConfig.sInstance = new RangerAdminConfig(); + } + } + } + + return ret; + } + + private RangerAdminConfig() { + super(); + addAdminResources(); + String storeType = get(RangerConfigConstants.RANGER_KEYSTORE_TYPE, KeyStore.getDefaultType()); + isFipsEnabled = StringUtils.equalsIgnoreCase("bcfks", storeType) ? true : false; + + } + + private boolean addAdminResources() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> addAdminResources()"); + } + + String defaultCfg = "ranger-admin-default-site.xml"; + String addlCfg = "ranger-admin-site.xml"; + String coreCfg = "core-site.xml"; + + boolean ret = true; + + if (!addResourceIfReadable(defaultCfg)) { + ret = false; + } + + if (!addResourceIfReadable(addlCfg)) { + ret = false; + } + + if (!addResourceIfReadable(coreCfg)){ + ret = false; + } + + if (! ret) { + LOG.error("Could not add ranger-admin resources to RangerAdminConfig."); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== addAdminResources(), result=" + ret); + } + + return ret; + } + + public boolean isFipsEnabled() { + return isFipsEnabled; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/authorization/hadoop/config/RangerAuditConfig.java b/auth-agents-common/src/main/java/org/apache/atlas/authorization/hadoop/config/RangerAuditConfig.java new file mode 100644 index 00000000000..c8ee96dfbbf --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/authorization/hadoop/config/RangerAuditConfig.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.authorization.hadoop.config; + + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class RangerAuditConfig extends RangerConfiguration { + private static final Logger LOG = LoggerFactory.getLogger(RangerAuditConfig.class); + + private final boolean initSuccess; + + public RangerAuditConfig() { + this("standalone"); + } + + public RangerAuditConfig(String serviceName) { + super(); + + initSuccess = addAuditResources(serviceName); + } + + public boolean isInitSuccess() { return initSuccess; } + + private boolean addAuditResources(String serviceName) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> addAuditResources()"); + } + + String defaultCfg = "ranger-" + serviceName + "-audit.xml"; + + boolean ret = true; + + if (!addResourceIfReadable(defaultCfg)) { + LOG.error("Could not add " + defaultCfg + " to RangerAuditConfig."); + ret = false; + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== addAuditResources(), result=" + ret); + } + + return ret; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/authorization/hadoop/config/RangerChainedPluginConfig.java b/auth-agents-common/src/main/java/org/apache/atlas/authorization/hadoop/config/RangerChainedPluginConfig.java new file mode 100644 index 00000000000..a0e138c4976 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/authorization/hadoop/config/RangerChainedPluginConfig.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.authorization.hadoop.config; + +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +public class RangerChainedPluginConfig extends RangerPluginConfig { + + private static final Log LOG = LogFactory.getLog(RangerChainedPluginConfig.class); + + private final String[] legacySSLProperties = new String[] {"xasecure.policymgr.clientssl.keystore", "xasecure.policymgr.clientssl.keystore.type", "xasecure.policymgr.clientssl.keystore.credential.file","xasecure.policymgr.clientssl.truststore", "xasecure.policymgr.clientssl.truststore.credential.file", "hadoop.security.credential.provider.path"}; + private final String[] chainedPluginPropertyPrefixes = new String[] { ".chained.services"}; + + public RangerChainedPluginConfig(String serviceType, String serviceName, String appId, RangerPluginConfig sourcePluginConfig) { + super(serviceType, serviceName, appId, sourcePluginConfig); + + // Copy all of properties from sourcePluginConfig except chained properties but with converted propertyPrefix + copyProperties(sourcePluginConfig, sourcePluginConfig.getPropertyPrefix()); + + // Copy SSL configurations from sourcePluginConfig + copyLegacySSLProperties(sourcePluginConfig); + + // Override copied properties from those in sourcePluginConfig with getPropertyPrefix() + copyProperties(sourcePluginConfig, getPropertyPrefix()); + + // Copy chained properties + copyChainedProperties(sourcePluginConfig, getPropertyPrefix()); + + set(getPropertyPrefix() + ".service.name", serviceName); + } + + private void copyProperties(RangerPluginConfig sourcePluginConfig, String propertyPrefix) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> copyProperties: propertyPrefix:[" + propertyPrefix + "]"); + } + for (String propName : sourcePluginConfig.getProperties().stringPropertyNames()) { + String value = sourcePluginConfig.get(propName); + + if (value != null && propName.startsWith(propertyPrefix)) { + String suffix = propName.substring(propertyPrefix.length()); + if (!isExcludedSuffix(suffix)) { + set(getPropertyPrefix() + suffix, value); + if (LOG.isDebugEnabled()) { + LOG.debug("set property:[" + getPropertyPrefix() + suffix + "] to value:[" + value + "]"); + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Not copying property :[" + propName + "] value from sourcePluginConfig"); + } + } + } + } + if (LOG.isDebugEnabled()) { + LOG.debug("<== copyProperties: propertyPrefix:[" + propertyPrefix + "]"); + } + } + + private void copyLegacySSLProperties(RangerPluginConfig sourcePluginConfig) { + for (String legacyPropertyName : legacySSLProperties) { + String value = sourcePluginConfig.get(legacyPropertyName); + if (value != null) { + set(legacyPropertyName, value); + } + } + } + + private void copyChainedProperties(RangerPluginConfig sourcePluginConfig, String propertyPrefix) { + for (String propName : sourcePluginConfig.getProperties().stringPropertyNames()) { + String value = sourcePluginConfig.get(propName); + + if (value != null && propName.startsWith(propertyPrefix)) { + String suffix = propName.substring(propertyPrefix.length()); + for (String chainedPropertyPrefix : chainedPluginPropertyPrefixes) { + if (StringUtils.startsWith(suffix, chainedPropertyPrefix)) { + set(getPropertyPrefix() + suffix, value); + } + } + } + } + } + + private boolean isExcludedSuffix(String suffix) { + for (String excludedSuffix : chainedPluginPropertyPrefixes) { + if (StringUtils.startsWith(suffix, excludedSuffix)) { + return true; + } + } + return false; + } + + private String printProperties() { + StringBuilder sb = new StringBuilder(); + boolean seenOneProp = false; + for (String propName : this.getProperties().stringPropertyNames()) { + String value = this.get(propName); + if (!seenOneProp) { + seenOneProp = true; + } else { + sb.append(",\n"); + } + sb.append("{ propertyName:[").append(propName).append("], propertyValue:[").append(value).append("] }"); + } + return sb.toString(); + } + + @Override + public String toString() { + return this.getClass().getSimpleName() + " : { " + printProperties() + " }"; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/authorization/hadoop/config/RangerConfigConstants.java b/auth-agents-common/src/main/java/org/apache/atlas/authorization/hadoop/config/RangerConfigConstants.java new file mode 100644 index 00000000000..32b801ed88f --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/authorization/hadoop/config/RangerConfigConstants.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.authorization.hadoop.config; + +public class RangerConfigConstants { + //SECURITY CONFIG DEFAULTS + public static final String RANGER_SERVICE_NAME = "ranger.plugin..service.name"; + public static final String RANGER_PLUGIN_POLICY_SOURCE_IMPL = "ranger.plugin..policy.source.impl"; + public static final String RANGER_PLUGIN_POLICY_SOURCE_IMPL_DEFAULT = "org.apache.atlas.admin.client.RangerAdminRESTClient"; + public static final String RANGER_PLUGIN_POLICY_REST_URL = "ranger.plugin..policy.rest.url"; + public static final String RANGER_PLUGIN_REST_SSL_CONFIG_FILE = "ranger.plugin..policy.rest.ssl.config.file"; + public static final String RANGER_PLUGIN_POLICY_POLLINVETERVALMS = "ranger.plugin..policy.pollIntervalMs"; + public static final String RANGER_PLUGIN_POLICY_CACHE_DIR = "ranger.plugin..policy.cache.dir"; + public static final String RANGER_PLUGIN_ADD_HADDOOP_AUTHORIZATION = "xasecure.add-hadoop-authorization"; + public static final String RANGER_KEYSTORE_TYPE = "ranger.keystore.file.type"; + + //CHANGE MAP CONSTANTS + public static final String XASECURE_POLICYMGR_URL = "xasecure..policymgr.url"; + public static final String XASECURE_POLICYMGR_URL_LASTSTOREDFILE = "xasecure..policymgr.url.laststoredfile"; + public static final String XASECURE_POLICYMGR_GRL_RELOADINTERVALINMILLIS = "xasecure..policymgr.url.reloadIntervalInMillis"; + public static final String XASECURE_ADD_HADDOP_AUTHORZATION = "xasecure.add-hadoop-authorization"; + public static final String XASECURE_UPDATE_XAPOLICIES_ON_GRANT = "xasecure..update.xapolicies.on.grant.revoke"; + + //Legacy Files + public static final String XASECURE_AUDIT_FILE = "xasecure-audit.xml"; + public static final String XASECURE_SECURITY_FILE = "xasecure--security.xml"; + public static final String XASECURE_POLICYMGR_SSL_FILE = "/etc//conf/xasecure-policymgr-ssl.xml"; + + //KNOX + public static final String RANGER_KNOX_PLUGIN_POLICY_SOURCE_IMPL_DEFAULT = "org.apache.atlas.admin.client.RangerAdminJersey2RESTClient"; +} + diff --git a/auth-agents-common/src/main/java/org/apache/atlas/authorization/hadoop/config/RangerConfiguration.java b/auth-agents-common/src/main/java/org/apache/atlas/authorization/hadoop/config/RangerConfiguration.java new file mode 100644 index 00000000000..a00575ea848 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/authorization/hadoop/config/RangerConfiguration.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +package org.apache.atlas.authorization.hadoop.config; + +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.Properties; + +import static org.apache.atlas.ApplicationProperties.ATLAS_CONFIGURATION_DIRECTORY_PROPERTY; + + +public class RangerConfiguration extends Configuration { + private static final Logger LOG = LoggerFactory.getLogger(RangerConfiguration.class); + + protected RangerConfiguration() { + super(false); + } + + public boolean addResourceIfReadable(String aResourceName) { + boolean ret = false; + + if(LOG.isDebugEnabled()) { + LOG.debug("==> addResourceIfReadable(" + aResourceName + ")"); + } + + URL fUrl = getFileLocation(aResourceName); + if (fUrl != null) { + if(LOG.isInfoEnabled()) { + LOG.info("addResourceIfReadable(" + aResourceName + "): resource file is " + fUrl); + } + + try { + addResource(fUrl); + ret = true; + } catch (Exception e) { + LOG.error("Unable to load the resource name [" + aResourceName + "]. Ignoring the resource:" + fUrl); + if (LOG.isDebugEnabled()) { + LOG.debug("Resource loading failed for " + fUrl, e); + } + } + } else { + LOG.warn("addResourceIfReadable(" + aResourceName + "): couldn't find resource file location"); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== addResourceIfReadable(" + aResourceName + "), result=" + ret); + } + return ret; + } + + public Properties getProperties() { + return getProps(); + } + + private URL getFileLocation(String fileName) { + String confLocation = System.getProperty(ATLAS_CONFIGURATION_DIRECTORY_PROPERTY); + URL lurl = null; + if (!StringUtils.isEmpty(fileName)) { + lurl = RangerConfiguration.class.getClassLoader().getResource(fileName); + + if (lurl == null ) { + lurl = RangerConfiguration.class.getClassLoader().getResource("/" + fileName); + } + + if (lurl == null ) { + File f = null; + if (StringUtils.isNotEmpty(confLocation)) { + f = new File(confLocation, fileName); + } else { + f = new File(fileName); + } + + if (f.exists()) { + try { + lurl = f.toURI().toURL(); + } catch (MalformedURLException e) { + LOG.error("Unable to load the resource name [" + fileName + "]. Ignoring the resource:" + f.getPath()); + } + } else { + if(LOG.isDebugEnabled()) { + LOG.debug("Conf file path " + fileName + " does not exists"); + } + } + } + } + return lurl; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/authorization/hadoop/config/RangerLegacyConfigBuilder.java b/auth-agents-common/src/main/java/org/apache/atlas/authorization/hadoop/config/RangerLegacyConfigBuilder.java new file mode 100644 index 00000000000..bc71b56d399 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/authorization/hadoop/config/RangerLegacyConfigBuilder.java @@ -0,0 +1,245 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.authorization.hadoop.config; + +import org.apache.hadoop.conf.Configuration; +import org.apache.atlas.plugin.store.EmbeddedServiceDefsUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.net.URL; +import java.util.HashMap; +import java.util.Map; + +public class RangerLegacyConfigBuilder { + + private static final Logger LOG = LoggerFactory.getLogger(RangerLegacyConfigBuilder.class); + + static String serviceType; + static String legacyResource; + + public static Configuration getSecurityConfig(String serviceType) { + + RangerLegacyConfigBuilder.legacyResource = getPropertyName(RangerConfigConstants.XASECURE_SECURITY_FILE,serviceType); + RangerLegacyConfigBuilder.serviceType = serviceType; + + Configuration ret = null; + Configuration legacyConfig = new Configuration(); + URL legacyFileUrl = getFileURL(legacyResource); + + if(LOG.isDebugEnabled()) { + LOG.debug("==> getSecurityConfig() " + legacyResource + " FileName: " + legacyFileUrl); + } + + if ( legacyFileUrl != null) { + legacyConfig.addResource(legacyFileUrl); + Configuration rangerDefaultProp = buildRangerSecurityConf(serviceType); + ret = mapLegacyConfigToRanger(rangerDefaultProp,legacyConfig); + } + if(LOG.isDebugEnabled()) { + LOG.debug("<== getSecurityConfig() " + legacyResource + " FileName: " + legacyFileUrl); + } + + return ret; + } + + public static URL getAuditConfig(String serviceType) throws Throwable { + + RangerLegacyConfigBuilder.legacyResource = getPropertyName(RangerConfigConstants.XASECURE_AUDIT_FILE,serviceType); + RangerLegacyConfigBuilder.serviceType = serviceType; + + URL ret = null; + try { + ret = getAuditResource(legacyResource); + } catch (Throwable t) { + throw t; + } + return ret; + } + + private static Configuration mapLegacyConfigToRanger(Configuration rangerInConf, Configuration legacyConf) { + + Configuration ret = rangerInConf; + + HashMap chgMap = getConfigChangeMap(serviceType); + if(LOG.isDebugEnabled()) { + LOG.debug("<== mapLegacyConfigToRanger() MAP Size: " + chgMap.size()); + } + for(Map.Entry entry : chgMap.entrySet()) { + String legacyKey = entry.getKey(); + String rangerKey = entry.getValue(); + + String legacyConfVal = null; + + if ( rangerKey.equals(getPropertyName(RangerConfigConstants.RANGER_SERVICE_NAME,serviceType)) ) { + //For getting the service + String serviceURL = legacyConf.get(getPropertyName(RangerConfigConstants.XASECURE_POLICYMGR_URL,serviceType)); + legacyConfVal = fetchLegacyValue(serviceURL,rangerKey); + } else if ( rangerKey.equals(getPropertyName(RangerConfigConstants.RANGER_PLUGIN_POLICY_REST_URL,serviceType)) || + rangerKey.equals(getPropertyName(RangerConfigConstants.RANGER_PLUGIN_POLICY_CACHE_DIR,serviceType)) ) { + // For Getting Admin URL and CacheDir + legacyConfVal = fetchLegacyValue(legacyConf.get(legacyKey),rangerKey); + } else { + legacyConfVal = legacyConf.get(legacyKey); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== mapLegacyConfigToRanger() Ranger Key: " + rangerKey + "Legacy Key:" + legacyKey + "Legacy Value:" + legacyConfVal); + } + + ret.set(rangerKey, legacyConfVal); + } + return ret; + } + + + public static URL getAuditResource(String fName) throws Throwable { + URL ret = null; + + try { + for(String cfgFile : new String[] { "hive-site.xml", "hbase-site.xml", "hdfs-site.xml" } ) { + String loc = getFileLocation(cfgFile); + if (loc != null) { + File f = new File(loc); + if ( f.exists() && f.canRead()) { + File parentFile = new File(loc).getParentFile(); + ret = new File(parentFile, RangerConfigConstants.XASECURE_AUDIT_FILE).toURI().toURL(); + break; + } + } + } + } + catch(Throwable t) { + LOG.error("Missing Ranger Audit configuration files..."); + throw t; + } + return ret; + } + + public static Configuration buildRangerSecurityConf(String serviceType) { + Configuration rangerConf = new Configuration(); + + rangerConf.set(getPropertyName(RangerConfigConstants.RANGER_SERVICE_NAME,serviceType),""); + if (EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_KNOX_NAME.equals(serviceType) ) { + rangerConf.set(getPropertyName(RangerConfigConstants.RANGER_PLUGIN_POLICY_SOURCE_IMPL,serviceType),RangerConfigConstants.RANGER_KNOX_PLUGIN_POLICY_SOURCE_IMPL_DEFAULT); + } else { + rangerConf.set(getPropertyName(RangerConfigConstants.RANGER_PLUGIN_POLICY_SOURCE_IMPL,serviceType),""); + } + rangerConf.set(getPropertyName(RangerConfigConstants.RANGER_PLUGIN_POLICY_REST_URL,serviceType),""); + rangerConf.set(getPropertyName(RangerConfigConstants.RANGER_PLUGIN_REST_SSL_CONFIG_FILE,serviceType), getPropertyName(RangerConfigConstants.XASECURE_POLICYMGR_SSL_FILE,serviceType)); + rangerConf.set(getPropertyName(RangerConfigConstants.RANGER_PLUGIN_POLICY_POLLINVETERVALMS,serviceType), ""); + rangerConf.set(getPropertyName(RangerConfigConstants.RANGER_PLUGIN_POLICY_CACHE_DIR,serviceType), ""); + rangerConf.set(RangerConfigConstants.RANGER_PLUGIN_ADD_HADDOOP_AUTHORIZATION,""); + + return rangerConf; + } + + public static HashMap getConfigChangeMap(String serviceType) { + // ConfigMap for moving legacy Configuration to Ranger Configuration + HashMap changeMap = new HashMap<>(); + + changeMap.put(serviceType, + getPropertyName(RangerConfigConstants.RANGER_SERVICE_NAME,serviceType)); + changeMap.put(getPropertyName(RangerConfigConstants.XASECURE_POLICYMGR_URL,serviceType), + getPropertyName(RangerConfigConstants.RANGER_PLUGIN_POLICY_REST_URL,serviceType)); + changeMap.put(getPropertyName(RangerConfigConstants.XASECURE_POLICYMGR_GRL_RELOADINTERVALINMILLIS,serviceType), + getPropertyName(RangerConfigConstants.RANGER_PLUGIN_POLICY_POLLINVETERVALMS,serviceType)); + changeMap.put(getPropertyName(RangerConfigConstants.XASECURE_POLICYMGR_URL_LASTSTOREDFILE,serviceType), + getPropertyName(RangerConfigConstants.RANGER_PLUGIN_POLICY_CACHE_DIR,serviceType)); + + if (EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_HDFS_NAME.equals(serviceType)) { + changeMap.put(RangerConfigConstants.XASECURE_ADD_HADDOP_AUTHORZATION, + RangerConfigConstants.RANGER_PLUGIN_ADD_HADDOOP_AUTHORIZATION); + } + + if (EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_HBASE_NAME.equals(serviceType) || + EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_HIVE_NAME.equals(serviceType)) { + changeMap.put(getPropertyName(RangerConfigConstants.XASECURE_UPDATE_XAPOLICIES_ON_GRANT,serviceType), + getPropertyName(RangerConfigConstants.XASECURE_UPDATE_XAPOLICIES_ON_GRANT,serviceType)); + } + + if ( LOG.isDebugEnabled()) { + for(Map.Entry entry : changeMap.entrySet()) { + String legacyKey = entry.getKey(); + String rangerKey = entry.getValue(); + LOG.debug("<== getConfigChangeMap() RangerConfig Key: " + rangerKey + " Legacy Key: " + legacyKey); + } + } + + return changeMap; + } + + public static String getFileLocation(String fileName) { + String ret = null; + + URL lurl = RangerLegacyConfigBuilder.class.getClassLoader().getResource(fileName); + if (lurl == null ) { + lurl = RangerLegacyConfigBuilder.class.getClassLoader().getResource("/" + fileName); + } + if (lurl != null) { + ret = lurl.getFile(); + } + return ret; + } + + public static URL getFileURL(String fileName) { + return RangerLegacyConfigBuilder.class.getClassLoader().getResource(fileName); + } + + public static String getPropertyName(String rangerProp, String serviceType) { + return rangerProp.replace("", serviceType); + } + + public static String getPolicyMgrURL(String url) { + int index = url.indexOf("/",url.lastIndexOf(":")); + + return url.substring(0,index); + } + + public static String getServiceNameFromURL(String url) { + int index = url.lastIndexOf("/"); + + return url.substring(index+1); + } + + + public static String getCacheFileURL(String cacheFile) { + int index = cacheFile.lastIndexOf("/"); + + return cacheFile.substring(0,index); + } + + public static String fetchLegacyValue(String legacyVal, String rangerKey) { + String ret = null; + + if ( rangerKey.equals(getPropertyName(RangerConfigConstants.RANGER_SERVICE_NAME,serviceType)) ) { + // To Fetch ServiceName + ret = getServiceNameFromURL(legacyVal); + } else if ( rangerKey.equals(getPropertyName(RangerConfigConstants.RANGER_PLUGIN_POLICY_REST_URL,serviceType)) ) { + // To Fetch PolicyMgr URL + ret = getPolicyMgrURL(legacyVal); + } else if ( rangerKey.equals(getPropertyName(RangerConfigConstants.RANGER_PLUGIN_POLICY_CACHE_DIR,serviceType)) ) { + ret = getCacheFileURL(legacyVal); + } + + return ret; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/authorization/hadoop/config/RangerPluginConfig.java b/auth-agents-common/src/main/java/org/apache/atlas/authorization/hadoop/config/RangerPluginConfig.java new file mode 100644 index 00000000000..21e229500d5 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/authorization/hadoop/config/RangerPluginConfig.java @@ -0,0 +1,358 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.authorization.hadoop.config; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.atlas.authorization.utils.StringUtil; +import org.apache.atlas.plugin.policyengine.RangerPolicyEngineOptions; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.net.URL; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + + +public class RangerPluginConfig extends RangerConfiguration { + private static final Logger LOG = LoggerFactory.getLogger(RangerPluginConfig.class); + + private static final char RANGER_TRUSTED_PROXY_IPADDRESSES_SEPARATOR_CHAR = ','; + + private final String serviceType; + private final String serviceName; + private final String appId; + private final String clusterName; + private final String clusterType; + private final RangerPolicyEngineOptions policyEngineOptions; + private final boolean useForwardedIPAddress; + private final String[] trustedProxyAddresses; + private final String propertyPrefix; + private boolean isFallbackSupported; + private Set auditExcludedUsers = Collections.emptySet(); + private Set auditExcludedGroups = Collections.emptySet(); + private Set auditExcludedRoles = Collections.emptySet(); + private Set superUsers = Collections.emptySet(); + private Set superGroups = Collections.emptySet(); + private Set serviceAdmins = Collections.emptySet(); + + + public RangerPluginConfig(String serviceType, String serviceName, String appId, String clusterName, String clusterType, RangerPolicyEngineOptions policyEngineOptions) { + super(); + + addResourcesForServiceType(serviceType); + + this.serviceType = serviceType; + this.appId = StringUtils.isEmpty(appId) ? serviceType : appId; + this.propertyPrefix = "atlas.plugin." + serviceType; + this.serviceName = StringUtils.isEmpty(serviceName) ? this.get(propertyPrefix + ".service.name") : serviceName; + + addResourcesForServiceName(this.serviceType, this.serviceName); + + String trustedProxyAddressString = this.get(propertyPrefix + ".trusted.proxy.ipaddresses"); + + if (StringUtil.isEmpty(clusterName)) { + clusterName = this.get(propertyPrefix + ".access.cluster.name", ""); + + if (StringUtil.isEmpty(clusterName)) { + clusterName = this.get(propertyPrefix + ".ambari.cluster.name", ""); + } + } + + if (StringUtil.isEmpty(clusterType)) { + clusterType = this.get(propertyPrefix + ".access.cluster.type", ""); + + if (StringUtil.isEmpty(clusterType)) { + clusterType = this.get(propertyPrefix + ".ambari.cluster.type", ""); + } + } + + this.clusterName = clusterName; + this.clusterType = clusterType; + this.useForwardedIPAddress = this.getBoolean(propertyPrefix + ".use.x-forwarded-for.ipaddress", false); + this.trustedProxyAddresses = StringUtils.split(trustedProxyAddressString, RANGER_TRUSTED_PROXY_IPADDRESSES_SEPARATOR_CHAR); + + if (trustedProxyAddresses != null) { + for (int i = 0; i < trustedProxyAddresses.length; i++) { + trustedProxyAddresses[i] = trustedProxyAddresses[i].trim(); + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug(propertyPrefix + ".use.x-forwarded-for.ipaddress:" + useForwardedIPAddress); + LOG.debug(propertyPrefix + ".trusted.proxy.ipaddresses:[" + StringUtils.join(trustedProxyAddresses, ", ") + "]"); + } + + if (useForwardedIPAddress && StringUtils.isBlank(trustedProxyAddressString)) { + LOG.warn("Property " + propertyPrefix + ".use.x-forwarded-for.ipaddress" + " is set to true, and Property " + propertyPrefix + ".trusted.proxy.ipaddresses" + " is not set"); + LOG.warn("Ranger plugin will trust RemoteIPAddress and treat first X-Forwarded-Address in the access-request as the clientIPAddress"); + } + + if (policyEngineOptions == null) { + policyEngineOptions = new RangerPolicyEngineOptions(); + + policyEngineOptions.configureForPlugin(this, propertyPrefix); + } + + this.policyEngineOptions = policyEngineOptions; + + LOG.info(policyEngineOptions.toString()); + } + + protected RangerPluginConfig(String serviceType, String serviceName, String appId, RangerPluginConfig sourcePluginConfig) { + super(); + + this.serviceType = serviceType; + this.appId = StringUtils.isEmpty(appId) ? serviceType : appId; + this.propertyPrefix = "ranger.plugin." + serviceType; + this.serviceName = serviceName; + + this.clusterName = sourcePluginConfig.getClusterName(); + this.clusterType = sourcePluginConfig.getClusterType(); + this.useForwardedIPAddress = sourcePluginConfig.isUseForwardedIPAddress(); + this.trustedProxyAddresses = sourcePluginConfig.getTrustedProxyAddresses(); + this.isFallbackSupported = sourcePluginConfig.getIsFallbackSupported(); + + this.policyEngineOptions = sourcePluginConfig.getPolicyEngineOptions(); + + } + + public String getServiceType() { + return serviceType; + } + + public String getAppId() { + return appId; + } + + public String getServiceName() { + return serviceName; + } + + public String getClusterName() { + return clusterName; + } + + public String getClusterType() { + return clusterType; + } + + public boolean isUseForwardedIPAddress() { + return useForwardedIPAddress; + } + + public String[] getTrustedProxyAddresses() { + return trustedProxyAddresses; + } + + public String getPropertyPrefix() { + return propertyPrefix; + } + + public boolean getIsFallbackSupported() { + return isFallbackSupported; + } + + public void setIsFallbackSupported(boolean isFallbackSupported) { + this.isFallbackSupported = isFallbackSupported; + } + + public RangerPolicyEngineOptions getPolicyEngineOptions() { + return policyEngineOptions; + } + + public void setAuditExcludedUsersGroupsRoles(Set users, Set groups, Set roles) { + auditExcludedUsers = CollectionUtils.isEmpty(users) ? Collections.emptySet() : new HashSet<>(users); + auditExcludedGroups = CollectionUtils.isEmpty(groups) ? Collections.emptySet() : new HashSet<>(groups); + auditExcludedRoles = CollectionUtils.isEmpty(roles) ? Collections.emptySet() : new HashSet<>(roles); + + if (LOG.isDebugEnabled()) { + LOG.debug("auditExcludedUsers=" + auditExcludedUsers + ", auditExcludedGroups=" + auditExcludedGroups + ", auditExcludedRoles=" + auditExcludedRoles); + } + } + + public void setSuperUsersGroups(Set users, Set groups) { + superUsers = CollectionUtils.isEmpty(users) ? Collections.emptySet() : new HashSet<>(users); + superGroups = CollectionUtils.isEmpty(groups) ? Collections.emptySet() : new HashSet<>(groups); + + if (LOG.isDebugEnabled()) { + LOG.debug("superUsers=" + superUsers + ", superGroups=" + superGroups); + } + } + + public void setServiceAdmins(Set users) { + serviceAdmins = CollectionUtils.isEmpty(users) ? Collections.emptySet() : new HashSet<>(users); + } + + public boolean isAuditExcludedUser(String userName) { + return auditExcludedUsers.contains(userName); + } + + public boolean hasAuditExcludedGroup(Set userGroups) { + return userGroups != null && userGroups.size() > 0 && auditExcludedGroups.size() > 0 && CollectionUtils.containsAny(userGroups, auditExcludedGroups); + } + + public boolean hasAuditExcludedRole(Set userRoles) { + return userRoles != null && userRoles.size() > 0 && auditExcludedRoles.size() > 0 && CollectionUtils.containsAny(userRoles, auditExcludedRoles); + } + + public boolean isSuperUser(String userName) { + return superUsers.contains(userName); + } + + public boolean hasSuperGroup(Set userGroups) { + return userGroups != null && userGroups.size() > 0 && superGroups.size() > 0 && CollectionUtils.containsAny(userGroups, superGroups); + } + + public boolean isServiceAdmin(String userName) { + return serviceAdmins.contains(userName); + } + + private void addResourcesForServiceType(String serviceType) { + String auditCfg = "atlas-" + serviceType + "-audit.xml"; + String securityCfg = "atlas-" + serviceType + "-security.xml"; + String sslCfg = "atlas-" + serviceType + "-policymgr-ssl.xml"; + + if (!addResourceIfReadable(auditCfg)) { + addAuditResource(serviceType); + } + + if (!addResourceIfReadable(securityCfg)) { + addSecurityResource(serviceType); + } + + if (!addResourceIfReadable(sslCfg)) { + addSslConfigResource(serviceType); + } + } + + // load service specific config overrides, if config files are available + private void addResourcesForServiceName(String serviceType, String serviceName) { + if (StringUtils.isNotBlank(serviceType) && StringUtils.isNotBlank(serviceName)) { + String serviceAuditCfg = "atlas-" + serviceType + "-" + serviceName + "-audit.xml"; + String serviceSecurityCfg = "atlas-" + serviceType + "-" + serviceName + "-security.xml"; + String serviceSslCfg = "atlas-" + serviceType + "-" + serviceName + "-policymgr-ssl.xml"; + + addResourceIfReadable(serviceAuditCfg); + addResourceIfReadable(serviceSecurityCfg); + addResourceIfReadable(serviceSslCfg); + } + } + + private void addSecurityResource(String serviceType) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> addSecurityResource(Service Type: " + serviceType ); + } + + Configuration rangerConf = RangerLegacyConfigBuilder.getSecurityConfig(serviceType); + + if (rangerConf != null ) { + addResource(rangerConf); + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Unable to add the Security Config for " + serviceType + ". Plugin won't be enabled!"); + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<= addSecurityResource(Service Type: " + serviceType ); + } + } + + private void addAuditResource(String serviceType) { + + if (LOG.isDebugEnabled()) { + LOG.debug("==> addAuditResource(Service Type: " + serviceType ); + } + + try { + URL url = RangerLegacyConfigBuilder.getAuditConfig(serviceType); + + if (url != null) { + addResource(url); + + if (LOG.isDebugEnabled()) { + LOG.debug("==> addAuditResource() URL" + url.getPath()); + } + } + + } catch (Throwable t) { + LOG.warn("Unable to find Audit Config for " + serviceType + " Auditing not enabled !" ); + + if(LOG.isDebugEnabled()) { + LOG.debug("Unable to find Audit Config for " + serviceType + " Auditing not enabled !" + t); + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== addAuditResource(Service Type: " + serviceType + ")"); + } + } + + private void addSslConfigResource(String serviceType) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> addSslConfigResource(Service Type: " + serviceType); + } + + try { + String sslConfigFile = this.get(RangerLegacyConfigBuilder.getPropertyName(RangerConfigConstants.RANGER_PLUGIN_REST_SSL_CONFIG_FILE, serviceType)); + + URL url = getSSLConfigResource(sslConfigFile); + if (url != null) { + addResource(url); + if (LOG.isDebugEnabled()) { + LOG.debug("SSL config file URL:" + url.getPath()); + } + } + } catch (Throwable t) { + LOG.warn(" Unable to find SSL Configs"); + + if (LOG.isDebugEnabled()) { + LOG.debug(" Unable to find SSL Configs"); + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== addSslConfigResource(Service Type: " + serviceType + ")"); + } + } + + private URL getSSLConfigResource(String fileName) throws Throwable { + URL ret = null; + + try { + if (fileName != null) { + File f = new File(fileName); + if (f.exists() && f.canRead()) { + ret = f.toURI().toURL(); + } + } + } catch (Throwable t) { + LOG.error("Unable to read SSL configuration file:" + fileName); + + throw t; + } + + return ret; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/authorization/hadoop/constants/RangerHadoopConstants.java b/auth-agents-common/src/main/java/org/apache/atlas/authorization/hadoop/constants/RangerHadoopConstants.java new file mode 100644 index 00000000000..9d367ecf0c5 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/authorization/hadoop/constants/RangerHadoopConstants.java @@ -0,0 +1,89 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.authorization.hadoop.constants; + +public class RangerHadoopConstants { + + public static final String RANGER_ADD_HDFS_PERMISSION_PROP = "xasecure.add-hadoop-authorization"; + public static final String RANGER_OPTIMIZE_SUBACCESS_AUTHORIZATION_PROP = "ranger.optimize-subaccess-authorization" ; + public static final boolean RANGER_ADD_HDFS_PERMISSION_DEFAULT = false; + public static final boolean RANGER_OPTIMIZE_SUBACCESS_AUTHORIZATION_DEFAULT = false ; + public static final String READ_ACCCESS_TYPE = "read"; + public static final String WRITE_ACCCESS_TYPE = "write"; + public static final String EXECUTE_ACCCESS_TYPE = "execute"; + + public static final String READ_EXECUTE_PERM = "READ_EXECUTE"; + public static final String WRITE_EXECUTE_PERM = "WRITE_EXECUTE"; + public static final String READ_WRITE_PERM = "READ_WRITE"; + public static final String ALL_PERM = "ALL"; + + public static final String HDFS_ROOT_FOLDER_PATH_ALT = ""; + public static final String HDFS_ROOT_FOLDER_PATH = "/"; + + public static final String HIVE_UPDATE_RANGER_POLICIES_ON_GRANT_REVOKE_PROP = "xasecure.hive.update.xapolicies.on.grant.revoke"; + public static final boolean HIVE_UPDATE_RANGER_POLICIES_ON_GRANT_REVOKE_DEFAULT_VALUE = true; + public static final String HIVE_BLOCK_UPDATE_IF_ROWFILTER_COLUMNMASK_SPECIFIED_PROP = "xasecure.hive.block.update.if.rowfilter.columnmask.specified"; + public static final boolean HIVE_BLOCK_UPDATE_IF_ROWFILTER_COLUMNMASK_SPECIFIED_DEFAULT_VALUE = true; + public static final String HIVE_DESCRIBE_TABLE_SHOW_COLUMNS_AUTH_OPTION_PROP = "xasecure.hive.describetable.showcolumns.authorization.option"; + public static final String HIVE_DESCRIBE_TABLE_SHOW_COLUMNS_AUTH_OPTION_PROP_DEFAULT_VALUE = "NONE"; + + public static final String HBASE_UPDATE_RANGER_POLICIES_ON_GRANT_REVOKE_PROP = "xasecure.hbase.update.xapolicies.on.grant.revoke"; + public static final boolean HBASE_UPDATE_RANGER_POLICIES_ON_GRANT_REVOKE_DEFAULT_VALUE = true; + + public static final String KNOX_ACCESS_VERIFIER_CLASS_NAME_PROP = "knox.authorization.verifier.classname"; + public static final String KNOX_ACCESS_VERIFIER_CLASS_NAME_DEFAULT_VALUE = "org.apache.atlas.pdp.knox.RangerAuthorizer"; + + public static final String STORM_ACCESS_VERIFIER_CLASS_NAME_PROP = "storm.authorization.verifier.classname"; + public static final String STORM_ACCESS_VERIFIER_CLASS_NAME_DEFAULT_VALUE = "org.apache.atlas.pdp.storm.RangerAuthorizer"; + + public static final String RANGER_ADD_YARN_PERMISSION_PROP = "ranger.add-yarn-authorization"; + public static final boolean RANGER_ADD_YARN_PERMISSION_DEFAULT = true; + + // + // Logging constants + // + public static final String AUDITLOG_FIELD_DELIMITER_PROP = "xasecure.auditlog.fieldDelimiterString"; + public static final String AUDITLOG_RANGER_MODULE_ACL_NAME_PROP = "xasecure.auditlog.xasecureAcl.name"; + public static final String AUDITLOG_HADOOP_MODULE_ACL_NAME_PROP = "xasecure.auditlog.hadoopAcl.name"; + public static final String AUDITLOG_YARN_MODULE_ACL_NAME_PROP = "ranger.auditlog.yarnAcl.name"; + + public static final String DEFAULT_LOG_FIELD_DELIMITOR = "|"; + public static final String DEFAULT_XASECURE_MODULE_ACL_NAME = "xasecure-acl"; + public static final String DEFAULT_RANGER_MODULE_ACL_NAME = "ranger-acl"; + public static final String DEFAULT_HADOOP_MODULE_ACL_NAME = "hadoop-acl"; + public static final String DEFAULT_YARN_MODULE_ACL_NAME = "yarn-acl"; + + + public static final String AUDITLOG_FIELDINFO_VISIBLE_PROP = "xasecure.auditlog.fieldInfoVisible"; + public static final boolean DEFAULT_AUDITLOG_FIELDINFO_VISIBLE = false; + + public static final String AUDITLOG_ACCESS_GRANTED_TEXT_PROP = "xasecure.auditlog.accessgranted.text"; + public static final String AUDITLOG_ACCESS_DENIED_TEXT_PROP = "xasecure.auditlog.accessdenied.text"; + + public static final String DEFAULT_ACCESS_GRANTED_TEXT = "granted"; + public static final String DEFAULT_ACCESS_DENIED_TEXT = "denied"; + + public static final String AUDITLOG_EMPTY_STRING = ""; + + public static final String AUDITLOG_HDFS_EXCLUDE_LIST_PROP = "xasecure.auditlog.hdfs.excludeusers"; + public static final String AUDITLOG_REPOSITORY_NAME_PROP = "xasecure.audit.repository.name"; + public static final String AUDITLOG_IS_ENABLED_PROP = "xasecure.audit.is.enabled"; + + public static final String KEYMGR_URL_PROP = "hdfs.keymanager.url"; +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/authorization/utils/JsonUtils.java b/auth-agents-common/src/main/java/org/apache/atlas/authorization/utils/JsonUtils.java new file mode 100644 index 00000000000..ddc492908a0 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/authorization/utils/JsonUtils.java @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.authorization.utils; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.reflect.TypeToken; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.plugin.model.AuditFilter; +import org.apache.atlas.plugin.model.RangerValidityRecurrence; +import org.apache.atlas.plugin.model.RangerValiditySchedule; + +import java.lang.reflect.Type; +import java.util.List; +import java.util.Map; + +public class JsonUtils { + private static final Log LOG = LogFactory.getLog(JsonUtils.class); + + private static final ThreadLocal gson = new ThreadLocal() { + @Override + protected Gson initialValue() { + return new GsonBuilder().setDateFormat("yyyyMMdd-HH:mm:ss.SSS-Z").create(); + } + }; + + public static String mapToJson(Map map) { + String ret = null; + if (MapUtils.isNotEmpty(map)) { + try { + ret = gson.get().toJson(map); + } catch (Exception e) { + LOG.error("Invalid input data: ", e); + } + } + return ret; + } + + public static String listToJson(List list) { + String ret = null; + if (CollectionUtils.isNotEmpty(list)) { + try { + ret = gson.get().toJson(list); + } catch (Exception e) { + LOG.error("Invalid input data: ", e); + } + } + return ret; + } + + public static String objectToJson(Object object) { + String ret = null; + + if(object != null) { + try { + ret = gson.get().toJson(object); + } catch(Exception excp) { + LOG.warn("objectToJson() failed to convert object to Json", excp); + } + } + + return ret; + } + + public static T jsonToObject(String jsonStr, Class clz) { + T ret = null; + + if(StringUtils.isNotEmpty(jsonStr)) { + try { + ret = gson.get().fromJson(jsonStr, clz); + } catch(Exception excp) { + LOG.warn("jsonToObject() failed to convert json to object: " + jsonStr, excp); + } + } + + return ret; + } + + public static Map jsonToMapStringString(String jsonStr) { + Map ret = null; + + if(StringUtils.isNotEmpty(jsonStr)) { + try { + Type mapType = new TypeToken>() {}.getType(); + ret = gson.get().fromJson(jsonStr, mapType); + } catch(Exception excp) { + LOG.warn("jsonToObject() failed to convert json to object: " + jsonStr, excp); + } + } + + return ret; + } + + public static List jsonToRangerValiditySchedule(String jsonStr) { + try { + Type listType = new TypeToken>() {}.getType(); + return gson.get().fromJson(jsonStr, listType); + } catch (Exception e) { + LOG.error("Cannot get List from " + jsonStr, e); + return null; + } + } + + public static List jsonToAuditFilterList(String jsonStr) { + try { + Type listType = new TypeToken>() {}.getType(); + return gson.get().fromJson(jsonStr, listType); + } catch (Exception e) { + LOG.error("failed to create audit filters from: " + jsonStr, e); + return null; + } + } + + public static List jsonToRangerValidityRecurringSchedule(String jsonStr) { + try { + Type listType = new TypeToken>() { + }.getType(); + return gson.get().fromJson(jsonStr, listType); + } catch (Exception e) { + LOG.error("Cannot get List from " + jsonStr, e); + return null; + } + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/authorization/utils/StringUtil.java b/auth-agents-common/src/main/java/org/apache/atlas/authorization/utils/StringUtil.java new file mode 100644 index 00000000000..e685b96a90b --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/authorization/utils/StringUtil.java @@ -0,0 +1,341 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + package org.apache.atlas.authorization.utils; + +import org.apache.commons.lang.StringUtils; + +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.GregorianCalendar; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TimeZone; + +public class StringUtil { + + private static final TimeZone gmtTimeZone = TimeZone.getTimeZone("GMT+0"); + + public static boolean equals(String str1, String str2) { + boolean ret = false; + + if(str1 == null) { + ret = str2 == null; + } else if(str2 == null) { + ret = false; + } else { + ret = str1.equals(str2); + } + + return ret; + } + + public static boolean equalsIgnoreCase(String str1, String str2) { + boolean ret = false; + + if(str1 == null) { + ret = str2 == null; + } else if(str2 == null) { + ret = false; + } else { + ret = str1.equalsIgnoreCase(str2); + } + + return ret; + } + + public static boolean equals(Collection set1, Collection set2) { + boolean ret = false; + + if(set1 == null) { + ret = set2 == null; + } else if(set2 == null) { + ret = false; + } else if(set1.size() == set2.size()) { + ret = set1.containsAll(set2); + } + + return ret; + } + + public static boolean equalsIgnoreCase(Collection set1, Collection set2) { + boolean ret = false; + + if(set1 == null) { + ret = set2 == null; + } else if(set2 == null) { + ret = false; + } else if(set1.size() == set2.size()) { + int numFound = 0; + + for(String str1 : set1) { + boolean str1Found = false; + + for(String str2 : set2) { + if(equalsIgnoreCase(str1, str2)) { + str1Found = true; + + break; + } + } + + if(str1Found) { + numFound++; + } else { + break; + } + } + + ret = numFound == set1.size(); + } + + return ret; + } + + public static boolean matches(String pattern, String str) { + boolean ret = false; + + if(pattern == null || str == null || pattern.isEmpty() || str.isEmpty()) { + ret = true; + } else { + ret = str.matches(pattern); + } + + return ret; + } + + /* + public static boolean matches(Collection patternSet, Collection strSet) { + boolean ret = false; + + if(patternSet == null || strSet == null || patternSet.isEmpty() || strSet.isEmpty()) { + ret = true; + } else { + boolean foundUnmatched = false; + + for(String str : strSet) { + boolean isMatched = false; + for(String pattern : patternSet) { + isMatched = str.matches(pattern); + + if(isMatched) { + break; + } + } + + foundUnmatched = ! isMatched; + + if(foundUnmatched) { + break; + } + } + + ret = !foundUnmatched; + } + + return ret; + } + */ + + public static boolean contains(String str, String strToFind) { + return str != null && strToFind != null && str.contains(strToFind); + } + + public static boolean containsIgnoreCase(String str, String strToFind) { + return str != null && strToFind != null && str.toLowerCase().contains(strToFind.toLowerCase()); + } + + public static boolean contains(String[] strArr, String str) { + boolean ret = false; + + if(strArr != null && strArr.length > 0 && str != null) { + for(String s : strArr) { + ret = equals(s, str); + + if(ret) { + break; + } + } + } + + return ret; + } + + public static boolean containsIgnoreCase(String[] strArr, String str) { + boolean ret = false; + + if(strArr != null && strArr.length > 0 && str != null) { + for(String s : strArr) { + ret = equalsIgnoreCase(s, str); + + if(ret) { + break; + } + } + } + + return ret; + } + + public static String toString(Iterable iterable) { + String ret = ""; + + if(iterable != null) { + int count = 0; + for(String str : iterable) { + if(count == 0) + ret = str; + else + ret += (", " + str); + count++; + } + } + + return ret; + } + + public static String toString(String[] arr) { + String ret = ""; + + if(arr != null && arr.length > 0) { + ret = arr[0]; + for(int i = 1; i < arr.length; i++) { + ret += (", " + arr[i]); + } + } + + return ret; + } + + public static String toString(List arr) { + String ret = ""; + + if(arr != null && !arr.isEmpty()) { + ret = arr.get(0); + for(int i = 1; i < arr.size(); i++) { + ret += (", " + arr.get(i)); + } + } + + return ret; + } + + public static boolean isEmpty(String str) { + return str == null || str.trim().isEmpty(); + } + + public static boolean isEmpty(Collection set) { + return set == null || set.isEmpty(); + } + + public static String toLower(String str) { + return str == null ? null : str.toLowerCase(); + } + + public static byte[] getBytes(String str) { + return str == null ? null : str.getBytes(); + } + + public static Date getUTCDate() { + Calendar local = Calendar.getInstance(); + int offset = local.getTimeZone().getOffset(local.getTimeInMillis()); + + GregorianCalendar utc = new GregorianCalendar(gmtTimeZone); + + utc.setTimeInMillis(local.getTimeInMillis()); + utc.add(Calendar.MILLISECOND, -offset); + + return utc.getTime(); + } + + public static Date getUTCDateForLocalDate(Date date) { + Calendar local = Calendar.getInstance(); + int offset = local.getTimeZone().getOffset(local.getTimeInMillis()); + + GregorianCalendar utc = new GregorianCalendar(gmtTimeZone); + + utc.setTimeInMillis(date.getTime()); + utc.add(Calendar.MILLISECOND, -offset); + + return utc.getTime(); + } + + public static Map toStringObjectMap(Map map) { + Map ret = null; + + if (map != null) { + ret = new HashMap<>(map.size()); + + for (Map.Entry e : map.entrySet()) { + ret.put(e.getKey(), e.getValue()); + } + } + + return ret; + } + + public static Set toSet(String str) { + Set values = new HashSet(); + if (StringUtils.isNotBlank(str)) { + for (String item : str.split(",")) { + if (StringUtils.isNotBlank(item)) { + values.add(StringUtils.trim(item)); + } + } + } + return values; + } + + public static List toList(String str) { + List values; + if (StringUtils.isNotBlank(str)) { + values = new ArrayList<>(); + for (String item : str.split(",")) { + if (StringUtils.isNotBlank(item)) { + values.add(StringUtils.trim(item)); + } + } + } else { + values = Collections.emptyList(); + } + return values; + } + + public static List getURLs(String configURLs) { + List configuredURLs = new ArrayList<>(); + if(configURLs!=null) { + String[] urls = configURLs.split(","); + for (String strUrl : urls) { + if (StringUtils.isNotEmpty(StringUtils.trimToEmpty(strUrl))) { + if (strUrl.endsWith("/")) { + strUrl = strUrl.substring(0, strUrl.length() - 1); + } + configuredURLs.add(strUrl); + } + } + } + return configuredURLs; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/authz/admin/client/AtlasAuthAdminClient.java b/auth-agents-common/src/main/java/org/apache/atlas/authz/admin/client/AtlasAuthAdminClient.java new file mode 100644 index 00000000000..a79407ba9c6 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/authz/admin/client/AtlasAuthAdminClient.java @@ -0,0 +1,16 @@ +package org.apache.atlas.authz.admin.client; + +import org.apache.atlas.authorization.hadoop.config.RangerPluginConfig; +import org.apache.atlas.plugin.util.RangerRoles; +import org.apache.atlas.plugin.util.RangerUserStore; +import org.apache.atlas.plugin.util.ServicePolicies; + +public interface AtlasAuthAdminClient { + void init(RangerPluginConfig config); + + ServicePolicies getServicePoliciesIfUpdated(long lastUpdatedTimeInMillis) throws Exception; + + RangerRoles getRolesIfUpdated(long lastUpdatedTimeInMillis) throws Exception; + + RangerUserStore getUserStoreIfUpdated(long lastUpdateTimeInMillis) throws Exception; +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/authz/admin/client/AtlasAuthRESTClient.java b/auth-agents-common/src/main/java/org/apache/atlas/authz/admin/client/AtlasAuthRESTClient.java new file mode 100644 index 00000000000..10c97008c37 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/authz/admin/client/AtlasAuthRESTClient.java @@ -0,0 +1,141 @@ +package org.apache.atlas.authz.admin.client; + +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.Response; +import org.apache.atlas.authorization.hadoop.config.RangerPluginConfig; +import org.apache.atlas.authorization.utils.StringUtil; +import org.apache.atlas.plugin.util.RangerRoles; +import org.apache.atlas.plugin.util.RangerUserStore; +import org.apache.atlas.plugin.util.ServicePolicies; +import org.apache.atlas.type.AtlasType; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.http.client.utils.URIBuilder; + +import javax.servlet.http.HttpServletResponse; +import java.net.InetAddress; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.UnknownHostException; +import java.util.concurrent.TimeUnit; + +public class AtlasAuthRESTClient implements AtlasAuthAdminClient { + private static final Log LOG = LogFactory.getLog(AtlasAuthRESTClient.class); + private static final int MAX_PLUGIN_ID_LEN = 255; + private static final String SCHEME = "http"; + private String serviceName; + private String pluginId; + private OkHttpClient httpClient; + private String adminUrl; + + private static final String PARAM_LAST_UPDATED_TIME = "lastUpdatedTime"; + private static final String PARAM_PLUGIN_ID = "pluginId"; + + @Override + public void init(RangerPluginConfig config) { + this.serviceName = config.getServiceName(); + this.pluginId = getPluginId(serviceName, config.getAppId()); + adminUrl = getAdminUrl(config); + + initClient(config); + } + + public String getPluginId(String serviceName, String appId) { + String hostName = null; + + try { + hostName = InetAddress.getLocalHost().getHostName(); + } catch (UnknownHostException e) { + LOG.error("ERROR: Unable to find hostname for the agent ", e); + hostName = "unknownHost"; + } + + String ret = hostName + "-" + serviceName; + + if(StringUtils.isNotEmpty(appId)) { + ret = appId + "@" + ret; + } + + if (ret.length() > MAX_PLUGIN_ID_LEN ) { + ret = ret.substring(0,MAX_PLUGIN_ID_LEN); + } + + return ret ; + } + + @Override + public ServicePolicies getServicePoliciesIfUpdated(long lastUpdatedTimeInMillis) throws Exception { + URI uri = buildURI("/download/policies/" + serviceName, lastUpdatedTimeInMillis); + return sendRequestAndGetResponse(uri, ServicePolicies.class); + } + + @Override + public RangerRoles getRolesIfUpdated(long lastUpdatedTimeInMillis) throws Exception { + URI uri = buildURI("/download/roles/" + serviceName, lastUpdatedTimeInMillis); + return sendRequestAndGetResponse(uri, RangerRoles.class); + } + + @Override + public RangerUserStore getUserStoreIfUpdated(long lastUpdatedTimeInMillis) throws Exception { + URI uri = buildURI("/download/users/" + serviceName, lastUpdatedTimeInMillis); + return sendRequestAndGetResponse(uri, RangerUserStore.class); + } + + private void initClient(RangerPluginConfig config) { + this.httpClient = new OkHttpClient(); + + OkHttpClient.Builder builder = new OkHttpClient.Builder(); + builder.readTimeout(config.getLong(config.getPropertyPrefix() + ".policy.rest.client.read.timeoutMs", 20000), TimeUnit.MILLISECONDS); + this.httpClient = builder.build(); + } + + private T sendRequestAndGetResponse(URI uri, Class responseClass) throws Exception { + Request request = new Request.Builder().url(uri.toURL()).build(); + + try (Response response = httpClient.newCall(request).execute()) { + if (response.code() == HttpServletResponse.SC_OK) { + String responseBody = response.body().string(); + if (StringUtils.isNotEmpty(responseBody)) { + return AtlasType.fromJson(responseBody, responseClass); + } else { + LOG.warn("AtlasAuthRESTClient.sendRequestAndGetResponse(): Empty response from Atlas Auth"); + } + } else { + if (response.code() == HttpServletResponse.SC_NO_CONTENT || response.code() == HttpServletResponse.SC_NOT_MODIFIED) { + LOG.info("AtlasAuthRESTClient.sendRequestAndGetResponse(): " + response.code()); + } else { + LOG.error("AtlasAuthRESTClient.sendRequestAndGetResponse(): HTTP error: " + response.code()); + } + } + } + + return null; + } + + private URI buildURI(String path, long lastUpdatedTimeInMillis) throws URISyntaxException { + return new URIBuilder() + .setScheme(SCHEME) + .setHost(adminUrl) + .setPath(path) + .setParameter(PARAM_LAST_UPDATED_TIME, String.valueOf(lastUpdatedTimeInMillis)) + .setParameter(PARAM_PLUGIN_ID, pluginId) + .build(); + } + + private String getAdminUrl(RangerPluginConfig config) { + String url = ""; + String tmpUrl = config.get(config.getPropertyPrefix() + ".authz.rest.url"); + + if (!StringUtil.isEmpty(tmpUrl)) { + url = tmpUrl.trim(); + } + + if (url.endsWith("/")) { + url = url.substring(0, url.length() - 1); + } + + return url; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/audit/RangerDefaultAuditHandler.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/audit/RangerDefaultAuditHandler.java new file mode 100644 index 00000000000..4f760ee28eb --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/audit/RangerDefaultAuditHandler.java @@ -0,0 +1,313 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.audit; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.atlas.audit.model.AuthzAuditEvent; +import org.apache.atlas.audit.provider.AuditHandler; +import org.apache.atlas.audit.provider.MiscUtil; +import org.apache.atlas.authorization.hadoop.constants.RangerHadoopConstants; +import org.apache.atlas.plugin.contextenricher.RangerTagForEval; +import org.apache.atlas.plugin.policyengine.*; +import org.apache.atlas.plugin.service.RangerBasePlugin; +import org.apache.atlas.plugin.util.JsonUtilsV2; +import org.apache.atlas.plugin.util.RangerAccessRequestUtil; +import org.apache.atlas.plugin.util.RangerRESTUtils; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Date; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; + + +public class RangerDefaultAuditHandler implements RangerAccessResultProcessor { + private static final Log LOG = LogFactory.getLog(RangerDefaultAuditHandler.class); + + private static final String CONF_AUDIT_ID_STRICT_UUID = "xasecure.audit.auditid.strict.uuid"; + private static final boolean DEFAULT_AUDIT_ID_STRICT_UUID = false; + + + private final boolean auditIdStrictUUID; + protected final String moduleName; + private final RangerRESTUtils restUtils = new RangerRESTUtils(); + private long sequenceNumber = 0; + private String UUID = MiscUtil.generateUniqueId(); + private AtomicInteger counter = new AtomicInteger(0); + + + + public RangerDefaultAuditHandler() { + auditIdStrictUUID = DEFAULT_AUDIT_ID_STRICT_UUID; + moduleName = RangerHadoopConstants.DEFAULT_RANGER_MODULE_ACL_NAME; + } + + public RangerDefaultAuditHandler(Configuration config) { + auditIdStrictUUID = config.getBoolean(CONF_AUDIT_ID_STRICT_UUID, DEFAULT_AUDIT_ID_STRICT_UUID); + moduleName = config.get(RangerHadoopConstants.AUDITLOG_RANGER_MODULE_ACL_NAME_PROP , RangerHadoopConstants.DEFAULT_RANGER_MODULE_ACL_NAME); + } + + @Override + public void processResult(RangerAccessResult result) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> AtlasDefaultAccessAuditHandler.processResult(" + result + ")"); + } + + AuthzAuditEvent event = getAuthzEvents(result); + + logAuthzAudit(event); + + if(LOG.isDebugEnabled()) { + LOG.debug("<== AtlasDefaultAccessAuditHandler.processResult(" + result + ")"); + } + } + + @Override + public void processResults(Collection results) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> AtlasDefaultAccessAuditHandler.processResults(" + results + ")"); + } + + Collection events = getAuthzEvents(results); + + if (events != null) { + logAuthzAudits(events); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== AtlasDefaultAccessAuditHandler.processResults(" + results + ")"); + } + } + + public AuthzAuditEvent getAuthzEvents(RangerAccessResult result) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> AtlasDefaultAccessAuditHandler.getAuthzEvents(" + result + ")"); + } + + AuthzAuditEvent ret = null; + + RangerAccessRequest request = result != null ? result.getAccessRequest() : null; + + if(request != null && result != null && result.getIsAudited()) { + //RangerServiceDef serviceDef = result.getServiceDef(); + RangerAccessResource resource = request.getResource(); + String resourceType = resource == null ? null : resource.getLeafName(); + String resourcePath = resource == null ? null : resource.getAsString(); + + ret = createAuthzAuditEvent(); + + ret.setRepositoryName(result.getServiceName()); + ret.setRepositoryType(result.getServiceType()); + ret.setResourceType(resourceType); + ret.setResourcePath(resourcePath); + ret.setRequestData(request.getRequestData()); + ret.setEventTime(request.getAccessTime() != null ? request.getAccessTime() : new Date()); + ret.setUser(request.getUser()); + ret.setAction(request.getAccessType()); + ret.setAccessResult((short) (result.getIsAllowed() ? 1 : 0)); + ret.setPolicyId(result.getPolicyId()); + ret.setAccessType(request.getAction()); + ret.setClientIP(request.getClientIPAddress()); + ret.setClientType(request.getClientType()); + ret.setSessionId(request.getSessionId()); + ret.setAclEnforcer(moduleName); + Set tags = getTags(request); + if (tags != null) { + ret.setTags(tags); + } + ret.setAdditionalInfo(getAdditionalInfo(request)); + ret.setClusterName(request.getClusterName()); + ret.setZoneName(result.getZoneName()); + ret.setAgentHostname(restUtils.getAgentHostname()); + ret.setPolicyVersion(result.getPolicyVersion()); + + populateDefaults(ret); + + result.setAuditLogId(ret.getEventId()); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== AtlasDefaultAccessAuditHandler.getAuthzEvents(" + result + "): " + ret); + } + + return ret; + } + + public Collection getAuthzEvents(Collection results) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> AtlasDefaultAccessAuditHandler.getAuthzEvents(" + results + ")"); + } + + List ret = null; + + if(results != null) { + // TODO: optimize the number of audit logs created + for(RangerAccessResult result : results) { + AuthzAuditEvent event = getAuthzEvents(result); + + if(event == null) { + continue; + } + + if(ret == null) { + ret = new ArrayList<>(); + } + + ret.add(event); + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== AtlasDefaultAccessAuditHandler.getAuthzEvents(" + results + "): " + ret); + } + + return ret; + } + + public void logAuthzAudit(AuthzAuditEvent auditEvent) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> AtlasDefaultAccessAuditHandler.logAuthzAudit(" + auditEvent + ")"); + } + + if(auditEvent != null) { + populateDefaults(auditEvent); + + AuditHandler auditProvider = RangerBasePlugin.getAuditProvider(auditEvent.getRepositoryName()); + if (auditProvider == null || !auditProvider.log(auditEvent)) { + MiscUtil.logErrorMessageByInterval(LOG, "fail to log audit event " + auditEvent); + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== AtlasDefaultAccessAuditHandler.logAuthzAudit(" + auditEvent + ")"); + } + } + + private void populateDefaults(AuthzAuditEvent auditEvent) { + if( auditEvent.getAclEnforcer() == null || auditEvent.getAclEnforcer().isEmpty()) { + auditEvent.setAclEnforcer("ranger-acl"); + } + + if (auditEvent.getAgentHostname() == null || auditEvent.getAgentHostname().isEmpty()) { + auditEvent.setAgentHostname(MiscUtil.getHostname()); + } + + if (auditEvent.getLogType() == null || auditEvent.getLogType().isEmpty()) { + auditEvent.setLogType("AtlasAuthZAudit"); + } + + if (auditEvent.getEventId() == null || auditEvent.getEventId().isEmpty()) { + auditEvent.setEventId(generateNextAuditEventId()); + } + + if (auditEvent.getAgentId() == null) { + auditEvent.setAgentId(MiscUtil.getApplicationType()); + } + + auditEvent.setSeqNum(sequenceNumber++); + } + + public void logAuthzAudits(Collection auditEvents) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> AtlasDefaultAccessAuditHandler.logAuthzAudits(" + auditEvents + ")"); + } + + if(auditEvents != null) { + for(AuthzAuditEvent auditEvent : auditEvents) { + logAuthzAudit(auditEvent); + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== AtlasDefaultAccessAuditHandler.logAuthzAudits(" + auditEvents + ")"); + } + } + + public AuthzAuditEvent createAuthzAuditEvent() { + return new AuthzAuditEvent(); + } + + protected final Set getTags(RangerAccessRequest request) { + Set ret = null; + Set tags = RangerAccessRequestUtil.getRequestTagsFromContext(request.getContext()); + + if (CollectionUtils.isNotEmpty(tags)) { + ret = new HashSet<>(); + + for (RangerTagForEval tag : tags) { + ret.add(writeObjectAsString(tag)); + } + } + + return ret; + } + + public String getAdditionalInfo(RangerAccessRequest request) { + if (StringUtils.isBlank(request.getRemoteIPAddress()) && CollectionUtils.isEmpty(request.getForwardedAddresses())) { + return null; + } + StringBuilder sb = new StringBuilder(); + sb.append("{\"remote-ip-address\":").append(request.getRemoteIPAddress()) + .append(", \"forwarded-ip-addresses\":[").append(StringUtils.join(request.getForwardedAddresses(), ", ")).append("]"); + + return sb.toString(); + } + + private String generateNextAuditEventId() { + final String ret; + + if (auditIdStrictUUID) { + ret = MiscUtil.generateGuid(); + } else { + int nextId = counter.getAndIncrement(); + + if (nextId == Integer.MAX_VALUE) { + // reset UUID and counter + UUID = MiscUtil.generateUniqueId(); + counter = new AtomicInteger(0); + } + + ret = UUID + "-" + Integer.toString(nextId); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("generateNextAuditEventId(): " + ret); + } + + return ret; + } + + private String writeObjectAsString(Serializable obj) { + String jsonStr = StringUtils.EMPTY; + try { + jsonStr = JsonUtilsV2.objToJson(obj); + } catch (Exception e) { + LOG.error("Cannot create JSON string for object:[" + obj + "]", e); + } + return jsonStr; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/audit/RangerMultiResourceAuditHandler.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/audit/RangerMultiResourceAuditHandler.java new file mode 100644 index 00000000000..47809cf99e1 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/audit/RangerMultiResourceAuditHandler.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.audit; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.audit.model.AuthzAuditEvent; + +import java.util.ArrayList; +import java.util.Collection; + +public class RangerMultiResourceAuditHandler extends RangerDefaultAuditHandler { + private static final Log LOG = LogFactory.getLog(RangerMultiResourceAuditHandler.class); + + Collection auditEvents = new ArrayList<>(); + + public RangerMultiResourceAuditHandler() { + } + + + @Override + public void logAuthzAudit(AuthzAuditEvent auditEvent) { + auditEvents.add(auditEvent); + } + + @Override + public void logAuthzAudits(Collection auditEvents) { + this.auditEvents.addAll(auditEvents); + } + + public void flushAudit() { + try { + boolean deniedExists = false; + // First iterate to see if there are any denied + for (AuthzAuditEvent auditEvent : auditEvents) { + if (auditEvent.getAccessResult() == 0) { + deniedExists = true; + break; + } + } + + for (AuthzAuditEvent auditEvent : auditEvents) { + if (deniedExists && auditEvent.getAccessResult() != 0) { + continue; + } + + super.logAuthzAudit(auditEvent); + } + } catch (Throwable t) { + LOG.error("Error occured while writing audit log... ", t); + } finally { + // reset auditEvents once audits are logged + auditEvents = new ArrayList<>(); + } + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/classloader/RangerPluginClassLoader.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/classloader/RangerPluginClassLoader.java new file mode 100644 index 00000000000..d55983d66b2 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/classloader/RangerPluginClassLoader.java @@ -0,0 +1,362 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.classloader; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.script.ScriptEngine; +import javax.script.ScriptEngineFactory; +import javax.script.ScriptEngineManager; +import java.io.IOException; +import java.net.URL; +import java.net.URLClassLoader; +import java.security.AccessController; +import java.security.PrivilegedAction; +import java.security.PrivilegedExceptionAction; +import java.util.Enumeration; +import java.util.List; + +public class RangerPluginClassLoader extends URLClassLoader { + private static final Logger LOG = LoggerFactory.getLogger(RangerPluginClassLoader.class); + + ThreadLocal preActivateClassLoader = new ThreadLocal<>(); + + private static volatile RangerPluginClassLoader me = null; + private static MyClassLoader componentClassLoader = null; + + public RangerPluginClassLoader(String pluginType, Class pluginClass ) throws Exception { + super(RangerPluginClassLoaderUtil.getInstance().getPluginFilesForServiceTypeAndPluginclass(pluginType, pluginClass), null); + componentClassLoader = AccessController.doPrivileged( + new PrivilegedAction() { + public MyClassLoader run() { + return new MyClassLoader(Thread.currentThread().getContextClassLoader()); + } + } + ); + } + + public static RangerPluginClassLoader getInstance(final String pluginType, final Class pluginClass ) throws Exception { + RangerPluginClassLoader ret = me; + if ( ret == null) { + synchronized(RangerPluginClassLoader.class) { + ret = me; + if (ret == null && pluginClass != null) { + me = ret = AccessController.doPrivileged( + new PrivilegedExceptionAction(){ + public RangerPluginClassLoader run() throws Exception { + return new RangerPluginClassLoader(pluginType,pluginClass); + } + } + ); + } + } + } + return ret; + } + + @Override + public Class findClass(String name) throws ClassNotFoundException { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerPluginClassLoader.findClass(" + name + ")"); + } + + Class ret = null; + + try { + // first we try to find a class inside the child classloader + if(LOG.isDebugEnabled()) { + LOG.debug("RangerPluginClassLoader.findClass(" + name + "): calling childClassLoader().findClass() "); + } + + ret = super.findClass(name); + } catch( Throwable e ) { + // Use the Component ClassLoader findclass to load when childClassLoader fails to find + if(LOG.isDebugEnabled()) { + LOG.debug("RangerPluginClassLoader.findClass(" + name + "): calling componentClassLoader.findClass()"); + } + + MyClassLoader savedClassLoader = getComponentClassLoader(); + if (savedClassLoader != null) { + ret = savedClassLoader.findClass(name); + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerPluginClassLoader.findClass(" + name + "): " + ret); + } + return ret; + } + + @Override + public synchronized Class loadClass(String name) throws ClassNotFoundException { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerPluginClassLoader.loadClass(" + name + ")" ); + } + + Class ret = null; + + try { + // first we try to load a class inside the child classloader + if (LOG.isDebugEnabled()) { + LOG.debug("RangerPluginClassLoader.loadClass(" + name + "): calling childClassLoader.findClass()"); + } + ret = super.loadClass(name); + } catch(Throwable e) { + // Use the Component ClassLoader loadClass to load when childClassLoader fails to find + if (LOG.isDebugEnabled()) { + LOG.debug("RangerPluginClassLoader.loadClass(" + name + "): calling componentClassLoader.loadClass()"); + } + + MyClassLoader savedClassLoader = getComponentClassLoader(); + + if(savedClassLoader != null) { + ret = savedClassLoader.loadClass(name); + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerPluginClassLoader.loadClass(" + name + "): " + ret); + } + + return ret; + } + + @Override + public URL findResource(String name) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerPluginClassLoader.findResource(" + name + ") "); + } + + URL ret = super.findResource(name); + + if (ret == null) { + if(LOG.isDebugEnabled()) { + LOG.debug("RangerPluginClassLoader.findResource(" + name + "): calling componentClassLoader.getResources()"); + } + + MyClassLoader savedClassLoader = getComponentClassLoader(); + if (savedClassLoader != null) { + ret = savedClassLoader.getResource(name); + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerPluginClassLoader.findResource(" + name + "): " + ret); + } + + return ret; + } + + @Override + public Enumeration findResources(String name) throws IOException { + Enumeration ret = null; + + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerPluginClassLoader.findResources(" + name + ") "); + } + + ret = new MergeEnumeration(findResourcesUsingChildClassLoader(name),findResourcesUsingComponentClassLoader(name)); + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerPluginClassLoader.findResources(" + name + ") "); + } + + return ret; + } + + public Enumeration findResourcesUsingChildClassLoader(String name) { + + Enumeration ret = null; + + try { + if(LOG.isDebugEnabled()) { + LOG.debug("RangerPluginClassLoader.findResourcesUsingChildClassLoader(" + name + "): calling childClassLoader.findResources()"); + } + + ret = super.findResources(name); + + } catch ( Throwable t) { + //Ignore any exceptions. Null / Empty return is handle in following statements + if(LOG.isDebugEnabled()) { + LOG.debug("RangerPluginClassLoader.findResourcesUsingChildClassLoader(" + name + "): class not found in child. Falling back to componentClassLoader", t); + } + } + return ret; + } + + public Enumeration findResourcesUsingComponentClassLoader(String name) { + + Enumeration ret = null; + + try { + + if(LOG.isDebugEnabled()) { + LOG.debug("RangerPluginClassLoader.findResourcesUsingComponentClassLoader(" + name + "): calling componentClassLoader.getResources()"); + } + + MyClassLoader savedClassLoader = getComponentClassLoader(); + + if (savedClassLoader != null) { + ret = savedClassLoader.getResources(name); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerPluginClassLoader.findResourcesUsingComponentClassLoader(" + name + "): " + ret); + } + } catch( Throwable t) { + if(LOG.isDebugEnabled()) { + LOG.debug("RangerPluginClassLoader.findResourcesUsingComponentClassLoader(" + name + "): class not found in componentClassLoader.", t); + } + } + + return ret; + } + + public void activate() { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerPluginClassLoader.activate()"); + } + + //componentClassLoader.set(new MyClassLoader(Thread.currentThread().getContextClassLoader())); + + preActivateClassLoader.set(Thread.currentThread().getContextClassLoader()); + + Thread.currentThread().setContextClassLoader(this); + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerPluginClassLoader.activate()"); + } + } + + public void deactivate() { + + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerPluginClassLoader.deactivate()"); + } + + ClassLoader classLoader = preActivateClassLoader.get(); + + if (classLoader != null) { + preActivateClassLoader.remove(); + } else { + MyClassLoader savedClassLoader = getComponentClassLoader(); + if (savedClassLoader != null && savedClassLoader.getParent() != null) { + classLoader = savedClassLoader.getParent(); + } + } + + if (classLoader != null) { + Thread.currentThread().setContextClassLoader(classLoader); + } else { + LOG.warn("RangerPluginClassLoader.deactivate() was not successful. Couldn't not get the saved classLoader..."); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerPluginClassLoader.deactivate()"); + } + } + + private MyClassLoader getComponentClassLoader() { + return componentClassLoader; + //return componentClassLoader.get(); + } + + static class MyClassLoader extends ClassLoader { + public MyClassLoader(ClassLoader realClassLoader) { + super(realClassLoader); + } + + @Override + public Class findClass(String name) throws ClassNotFoundException { //NOPMD + return super.findClass(name); + } + } + + static class MergeEnumeration implements Enumeration { //NOPMD + + Enumeration e1 = null; + Enumeration e2 = null; + + public MergeEnumeration(Enumeration e1, Enumeration e2 ) { + this.e1 = e1; + this.e2 = e2; + } + + @Override + public boolean hasMoreElements() { + return ( (e1 != null && e1.hasMoreElements() ) || ( e2 != null && e2.hasMoreElements()) ); + } + + @Override + public URL nextElement() { + URL ret = null; + if (e1 != null && e1.hasMoreElements()) + ret = e1.nextElement(); + else if ( e2 != null && e2.hasMoreElements() ) { + ret = e2.nextElement(); + } + return ret; + } + } + + public ScriptEngine getScriptEngine(String engineName) { + + final ScriptEngine ret; + + ClassLoader classLoader = preActivateClassLoader.get(); + + if (classLoader == null) { + MyClassLoader savedClassLoader = getComponentClassLoader(); + if (savedClassLoader != null && savedClassLoader.getParent() != null) { + classLoader = savedClassLoader.getParent(); + } + } + + ScriptEngineManager manager = classLoader != null ? new ScriptEngineManager(classLoader) : new ScriptEngineManager(); + + if (LOG.isDebugEnabled()) { + + List factories = manager.getEngineFactories(); + + if (factories == null || factories.size() == 0) { + LOG.debug("List of scriptEngineFactories is empty!!"); + + } else { + for (ScriptEngineFactory factory : factories) { + LOG.debug("engineName=" + factory.getEngineName() + ", language=" + factory.getLanguageName()); + + } + } + } + + ret = manager.getEngineByName(engineName); + + if (ret == null) { + LOG.error("scriptEngine for JavaScript is null!!"); + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("scriptEngine for JavaScript:[" + ret + "]"); + } + } + + return ret; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/classloader/RangerPluginClassLoaderUtil.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/classloader/RangerPluginClassLoaderUtil.java new file mode 100644 index 00000000000..9167ff9fe71 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/classloader/RangerPluginClassLoaderUtil.java @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.classloader; + + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.net.URI; +import java.net.URL; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; + +public class RangerPluginClassLoaderUtil { + + private static final Logger LOG = LoggerFactory.getLogger(RangerPluginClassLoaderUtil.class); + + private static volatile RangerPluginClassLoaderUtil config = null; + private static String rangerPluginLibDir = "ranger-%-plugin-impl"; + + public static RangerPluginClassLoaderUtil getInstance() { + RangerPluginClassLoaderUtil result = config; + if (result == null) { + synchronized (RangerPluginClassLoaderUtil.class) { + result = config; + if (result == null) { + config = result = new RangerPluginClassLoaderUtil(); + } + } + } + return result; + } + + + public URL[] getPluginFilesForServiceTypeAndPluginclass( String serviceType, Class pluginClass) throws Exception { + + URL[] ret = null; + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerPluginClassLoaderUtil.getPluginFilesForServiceTypeAndPluginclass(" + serviceType + ")" + " Pluging Class :" + pluginClass.getName()); + } + + String[] libDirs = new String[] { getPluginImplLibPath(serviceType, pluginClass) }; + + ret = getPluginFiles(libDirs); + + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerPluginClassLoaderUtil.getPluginFilesForServiceTypeAndPluginclass(" + serviceType + ")" + " Pluging Class :" + pluginClass.getName()); + } + + return ret; + + } + + private URL[] getPluginFiles(String[] libDirs) throws Exception { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerPluginClassLoaderUtil.getPluginFiles()"); + } + + List ret = new ArrayList(); + for ( String libDir : libDirs) { + getFilesInDirectory(libDir,ret); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerPluginClassLoaderUtil.getPluginFilesForServiceType(): " + ret.size() + " files"); + } + + return ret.toArray(new URL[] { }); + } + + private void getFilesInDirectory(String dirPath, List files) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerPluginClassLoaderUtil.getPluginFiles()"); + } + + if ( dirPath != null) { + try { + + File[] dirFiles = new File(dirPath).listFiles(); + + if(dirFiles != null) { + for(File dirFile : dirFiles) { + try { + if (!dirFile.canRead()) { + LOG.error("getFilesInDirectory('" + dirPath + "'): " + dirFile.getAbsolutePath() + " is not readable!"); + } + + URL jarPath = dirFile.toURI().toURL(); + + LOG.info("getFilesInDirectory('" + dirPath + "'): adding " + dirFile.getAbsolutePath()); + + files.add(jarPath); + } catch(Exception excp) { + LOG.warn("getFilesInDirectory('" + dirPath + "'): failed to get URI for file " + dirFile.getAbsolutePath(), excp); + } + } + } + } catch(Exception excp) { + LOG.warn("getFilesInDirectory('" + dirPath + "'): error", excp); + } + } else { + LOG.warn("getFilesInDirectory('" + dirPath + "'): could not find directory in path " + dirPath); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerPluginClassLoaderUtil.getFilesInDirectory(" + dirPath + ")"); + } + } + + private String getPluginImplLibPath(String serviceType, Class pluginClass) throws Exception { + + String ret = null; + + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerPluginClassLoaderUtil.getPluginImplLibPath for Class (" + pluginClass.getName() + ")"); + } + + URI uri = pluginClass.getProtectionDomain().getCodeSource().getLocation().toURI(); + + Path path = Paths.get(URI.create(uri.toString())); + + ret = path.getParent().toString() + File.separatorChar + rangerPluginLibDir.replaceAll("%", serviceType); + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerPluginClassLoaderUtil.getPluginImplLibPath for Class (" + pluginClass.getName() + " PATH :" + ret + ")"); + } + + return ret; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/client/BaseClient.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/client/BaseClient.java new file mode 100644 index 00000000000..3047c05e55d --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/client/BaseClient.java @@ -0,0 +1,192 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + package org.apache.atlas.plugin.client; + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.security.SecureClientLogin; +import org.apache.atlas.plugin.util.PasswordUtils; + +import javax.security.auth.Subject; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public abstract class BaseClient { + private static final Log LOG = LogFactory.getLog(BaseClient.class); + + + private static final String DEFAULT_NAME_RULE = "DEFAULT"; + protected static final String DEFAULT_ERROR_MESSAGE = " You can still save the repository and start creating " + + "policies, but you would not be able to use autocomplete for " + + "resource names. Check ranger_admin.log for more info."; + + + private String serviceName; + private String defaultConfigFile; + private Subject loginSubject; + private HadoopConfigHolder configHolder; + + protected Map connectionProperties; + + public BaseClient(String svcName, Map connectionProperties) { + this(svcName, connectionProperties, null); + } + + public BaseClient(String serivceName, Map connectionProperties, String defaultConfigFile) { + this.serviceName = serivceName; + this.connectionProperties = connectionProperties; + this.defaultConfigFile = defaultConfigFile; + init(); + login(); + } + + + private void init() { + if (connectionProperties == null) { + configHolder = HadoopConfigHolder.getInstance(serviceName); + } + else { + configHolder = HadoopConfigHolder.getInstance(serviceName,connectionProperties, defaultConfigFile); + } + } + + + protected void login() { + ClassLoader prevCl = Thread.currentThread().getContextClassLoader(); + try { + //Thread.currentThread().setContextClassLoader(configHolder.getClassLoader()); + String lookupPrincipal = SecureClientLogin.getPrincipal(configHolder.getLookupPrincipal(), java.net.InetAddress.getLocalHost().getCanonicalHostName()); + String lookupKeytab = configHolder.getLookupKeytab(); + String nameRules = configHolder.getNameRules(); + if(StringUtils.isEmpty(nameRules)){ + if(LOG.isDebugEnabled()){ + LOG.debug("Name Rule is empty. Setting Name Rule as 'DEFAULT'"); + } + nameRules = DEFAULT_NAME_RULE; + } + String userName = configHolder.getUserName(); + if(StringUtils.isEmpty(lookupPrincipal) || StringUtils.isEmpty(lookupKeytab)){ + if (userName == null) { + throw createException("Unable to find login username for hadoop environment, [" + serviceName + "]", null); + } + String keyTabFile = configHolder.getKeyTabFile(); + if (keyTabFile != null) { + if ( configHolder.isKerberosAuthentication() ) { + LOG.info("Init Login: security enabled, using username/keytab"); + loginSubject = SecureClientLogin.loginUserFromKeytab(userName, keyTabFile, nameRules); + } + else { + LOG.info("Init Login: using username"); + loginSubject = SecureClientLogin.login(userName); + } + } + else { + String encryptedPwd = configHolder.getPassword(); + String password = null; + if (encryptedPwd != null) { + try { + password = PasswordUtils.decryptPassword(encryptedPwd); + } catch(Exception ex) { + LOG.info("Password decryption failed; trying connection with received password string"); + password = null; + } finally { + if (password == null) { + password = encryptedPwd; + } + } + } else { + LOG.info("Password decryption failed: no password was configured"); + } + if ( configHolder.isKerberosAuthentication() ) { + LOG.info("Init Login: using username/password"); + loginSubject = SecureClientLogin.loginUserWithPassword(userName, password); + } + else { + LOG.info("Init Login: security not enabled, using username"); + loginSubject = SecureClientLogin.login(userName); + } + } + }else{ + if ( configHolder.isKerberosAuthentication() ) { + LOG.info("Init Lookup Login: security enabled, using lookupPrincipal/lookupKeytab"); + loginSubject = SecureClientLogin.loginUserFromKeytab(lookupPrincipal, lookupKeytab, nameRules); + }else{ + LOG.info("Init Login: security not enabled, using username"); + loginSubject = SecureClientLogin.login(userName); + } + } + } catch (IOException ioe) { + throw createException(ioe); + } catch (SecurityException se) { + throw createException(se); + } finally { + Thread.currentThread().setContextClassLoader(prevCl); + } + } + + private HadoopException createException(Exception exp) { + return createException("Unable to login to Hadoop environment [" + serviceName + "]", exp); + } + + private HadoopException createException(String msgDesc, Exception exp) { + HadoopException hdpException = new HadoopException(msgDesc, exp); + final String fullDescription = exp != null ? getMessage(exp) : msgDesc; + hdpException.generateResponseDataMap(false, fullDescription + DEFAULT_ERROR_MESSAGE, + msgDesc + DEFAULT_ERROR_MESSAGE, null, null); + return hdpException; + } + + public String getSerivceName() { + return serviceName; + } + + protected Subject getLoginSubject() { + return loginSubject; + } + + protected HadoopConfigHolder getConfigHolder() { + return configHolder; + } + + public static void generateResponseDataMap(boolean connectivityStatus, + String message, String description, Long objectId, + String fieldName, Map responseData) { + responseData.put("connectivityStatus", connectivityStatus); + responseData.put("message", message); + responseData.put("description", description); + responseData.put("objectId", objectId); + responseData.put("fieldName", fieldName); + } + + public static String getMessage(Throwable excp) { + List errList = new ArrayList<>(); + while (excp != null) { + String message = excp.getMessage(); + if (StringUtils.isNotEmpty(message) && !errList.contains(message + ". \n")) { + errList.add(message + ". \n"); + } + excp = excp.getCause(); + } + return StringUtils.join(errList, ""); + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/client/HadoopConfigHolder.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/client/HadoopConfigHolder.java new file mode 100644 index 00000000000..26c585d3dc1 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/client/HadoopConfigHolder.java @@ -0,0 +1,476 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + package org.apache.atlas.plugin.client; + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.security.SecureClientLogin; + +import java.io.IOException; +import java.io.InputStream; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Properties; +import java.util.Set; + +public class HadoopConfigHolder { + private static final Log LOG = LogFactory.getLog(HadoopConfigHolder.class); + public static final String GLOBAL_LOGIN_PARAM_PROP_FILE = "hadoop-login.properties"; + public static final String DEFAULT_DATASOURCE_PARAM_PROP_FILE = "datasource.properties"; + public static final String RESOURCEMAP_PROP_FILE = "resourcenamemap.properties"; + public static final String DEFAULT_RESOURCE_NAME = "core-site.xml"; + public static final String RANGER_SECTION_NAME = "xalogin.xml"; + public static final String RANGER_LOGIN_USER_NAME_PROP = "username"; + public static final String RANGER_LOGIN_KEYTAB_FILE_PROP = "keytabfile"; + public static final String RANGER_LOGIN_PASSWORD = "password"; + public static final String RANGER_LOOKUP_PRINCIPAL = "lookupprincipal"; + public static final String RANGER_LOOKUP_KEYTAB = "lookupkeytab"; + public static final String RANGER_PRINCIPAL = "rangerprincipal"; + public static final String RANGER_KEYTAB = "rangerkeytab"; + public static final String RANGER_NAME_RULES = "namerules"; + public static final String RANGER_AUTH_TYPE = "authtype"; + public static final String HADOOP_SECURITY_AUTHENTICATION = "hadoop.security.authentication"; + public static final String HADOOP_NAME_RULES = "hadoop.security.auth_to_local"; + public static final String HADOOP_SECURITY_AUTHENTICATION_METHOD = "kerberos"; + public static final String HADOOP_RPC_PROTECTION = "hadoop.rpc.protection"; + + public static final String ENABLE_HIVE_METASTORE_LOOKUP = "enable.hive.metastore.lookup"; + public static final String HIVE_SITE_FILE_PATH = "hive.site.file.path"; + + private static boolean initialized; + private static Map> dataSource2ResourceListMap = new HashMap<>(); + private static Map dataSource2HadoopConfigHolder = new HashMap<>(); + private static Properties globalLoginProp = new Properties(); + private static Properties resourcemapProperties; + + private String datasourceName; + private String defaultConfigFile; + private String userName; + private String keyTabFile; + private String password; + private String lookupPrincipal; + private String lookupKeytab; + private String nameRules; + private String authType; + private String hiveSiteFilePath; + private boolean isKerberosAuth; + private boolean enableHiveMetastoreLookup; + + private Map connectionProperties; + + private static Set rangerInternalPropertyKeys = new HashSet<>(); + + public static HadoopConfigHolder getInstance(String aDatasourceName) { + HadoopConfigHolder ret = dataSource2HadoopConfigHolder.get(aDatasourceName); + if (ret == null) { + synchronized (HadoopConfigHolder.class) { + HadoopConfigHolder temp = ret; + if (temp == null) { + ret = new HadoopConfigHolder(aDatasourceName); + dataSource2HadoopConfigHolder.put(aDatasourceName, ret); + } + } + } + return ret; + } + + public static HadoopConfigHolder getInstance(String aDatasourceName, Map connectionProperties, + String defaultConfigFile) { + HadoopConfigHolder ret = dataSource2HadoopConfigHolder.get(aDatasourceName); + if (ret == null) { + synchronized (HadoopConfigHolder.class) { + HadoopConfigHolder temp = ret; + if (temp == null) { + ret = new HadoopConfigHolder(aDatasourceName,connectionProperties, defaultConfigFile); + dataSource2HadoopConfigHolder.put(aDatasourceName, ret); + } + } + } + else { + if (connectionProperties !=null && !connectionProperties.equals(ret.connectionProperties)) { + ret = new HadoopConfigHolder(aDatasourceName,connectionProperties); + dataSource2HadoopConfigHolder.remove(aDatasourceName); + dataSource2HadoopConfigHolder.put(aDatasourceName, ret); + } + } + + return ret; + } + + + + private HadoopConfigHolder(String aDatasourceName) { + datasourceName = aDatasourceName; + if (!initialized) { + init(); + } + initLoginInfo(); + } + + private HadoopConfigHolder(String aDatasourceName, + Map connectionProperties) { + this(aDatasourceName, connectionProperties, null); + } + + private HadoopConfigHolder(String aDatasourceName, Map connectionProperties, + String defaultConfigFile) { + datasourceName = aDatasourceName; + this.connectionProperties = connectionProperties; + this.defaultConfigFile = defaultConfigFile; + initConnectionProp(); + initLoginInfo(); + } + + private void initConnectionProp() { + if (!connectionProperties.containsKey(ENABLE_HIVE_METASTORE_LOOKUP)) { + connectionProperties.put(ENABLE_HIVE_METASTORE_LOOKUP, "false"); + } + if (!connectionProperties.containsKey(HIVE_SITE_FILE_PATH)) { + connectionProperties.put(HIVE_SITE_FILE_PATH, ""); + } + + for (Map.Entry entry : connectionProperties.entrySet()) { + String key = entry.getKey(); + String resourceName = getResourceName(key); + + if (resourceName == null) { + resourceName = RANGER_SECTION_NAME; + } + String val = entry.getValue(); + addConfiguration(datasourceName, resourceName, key, val ); + } + } + + private String getResourceName(String key) { + + if (resourcemapProperties == null) { + initResourceMap(); + } + + if (resourcemapProperties != null) { + String rn = resourcemapProperties.getProperty(key); + return ( rn != null) ? rn : defaultConfigFile; + } else { + return defaultConfigFile; + } + } + + private static void initResourceMap() { + Properties props = new Properties(); + InputStream in = HadoopConfigHolder.class.getClassLoader().getResourceAsStream(RESOURCEMAP_PROP_FILE); + if (in != null) { + try { + props.load(in); + for (Map.Entry entry : props + .entrySet()) { + String value = (String) entry.getValue(); + if (RANGER_SECTION_NAME.equals(value)) { + String key = (String) entry.getKey(); + rangerInternalPropertyKeys.add(key); + } + } + resourcemapProperties = props; + } catch (IOException e) { + throw new HadoopException("Unable to load resource map properties from [" + RESOURCEMAP_PROP_FILE + "]", e); + } + finally { + if (in != null) { + try { + in.close(); + } catch (IOException ioe) { + // Ignore IOException during close of stream + } + } + } + } else { + throw new HadoopException("Unable to locate resource map properties from [" + RESOURCEMAP_PROP_FILE + "] in the class path."); + } + } + + + + private static synchronized void init() { + + if (initialized) { + return; + } + + try { + InputStream in = HadoopConfigHolder.class.getClassLoader().getResourceAsStream(DEFAULT_DATASOURCE_PARAM_PROP_FILE); + if (in != null) { + Properties prop = new Properties(); + try { + prop.load(in); + } catch (IOException e) { + throw new HadoopException("Unable to get configuration information for Hadoop environments", e); + } + finally { + try { + in.close(); + } catch (IOException e) { + // Ignored exception when the stream is closed. + } + } + + if (prop.isEmpty()) { + return; + } + + for (Object keyobj : prop.keySet()) { + String key = (String)keyobj; + String val = prop.getProperty(key); + + int dotLocatedAt = key.indexOf("."); + + if (dotLocatedAt == -1) { + continue; + } + + String dataSource = key.substring(0,dotLocatedAt); + + String propKey = key.substring(dotLocatedAt+1); + int resourceFoundAt = propKey.indexOf("."); + if (resourceFoundAt > -1) { + String resourceName = propKey.substring(0, resourceFoundAt) + ".xml"; + propKey = propKey.substring(resourceFoundAt+1); + addConfiguration(dataSource, resourceName, propKey, val); + } + + } + } + + in = HadoopConfigHolder.class.getClassLoader().getResourceAsStream(GLOBAL_LOGIN_PARAM_PROP_FILE); + if (in != null) { + Properties tempLoginProp = new Properties(); + try { + tempLoginProp.load(in); + } catch (IOException e) { + throw new HadoopException("Unable to get login configuration information for Hadoop environments from file: [" + GLOBAL_LOGIN_PARAM_PROP_FILE + "]", e); + } + finally { + try { + in.close(); + } catch (IOException e) { + // Ignored exception when the stream is closed. + } + } + globalLoginProp = tempLoginProp; + } + } + finally { + initialized = true; + } + } + + + private void initLoginInfo() { + Properties prop = this.getRangerSection(); + if (prop != null) { + userName = prop.getProperty(RANGER_LOGIN_USER_NAME_PROP); + keyTabFile = prop.getProperty(RANGER_LOGIN_KEYTAB_FILE_PROP); + if (StringUtils.trimToNull(prop.getProperty(ENABLE_HIVE_METASTORE_LOOKUP)) != null) { + try { + enableHiveMetastoreLookup = Boolean.valueOf(prop.getProperty(ENABLE_HIVE_METASTORE_LOOKUP,"false").trim()); + } catch (Exception e) { + enableHiveMetastoreLookup = false; + LOG.error("Error while getting " + ENABLE_HIVE_METASTORE_LOOKUP + " : " + e.getMessage()); + } + } + if (StringUtils.trimToNull(prop.getProperty(HIVE_SITE_FILE_PATH)) != null) { + hiveSiteFilePath = prop.getProperty(HIVE_SITE_FILE_PATH).trim(); + } else { + hiveSiteFilePath = null; + } + + password = prop.getProperty(RANGER_LOGIN_PASSWORD); + lookupPrincipal = prop.getProperty(RANGER_LOOKUP_PRINCIPAL); + lookupKeytab = prop.getProperty(RANGER_LOOKUP_KEYTAB); + nameRules = prop.getProperty(RANGER_NAME_RULES); + authType = prop.getProperty(RANGER_AUTH_TYPE, "simple"); + + String hadoopSecurityAuthentication = getHadoopSecurityAuthentication(); + + if (hadoopSecurityAuthentication != null) { + isKerberosAuth = HADOOP_SECURITY_AUTHENTICATION_METHOD.equalsIgnoreCase(hadoopSecurityAuthentication); + } else { + isKerberosAuth = (((userName != null) && (userName.indexOf("@") > -1)) || (SecureClientLogin.isKerberosCredentialExists(lookupPrincipal, lookupKeytab))); + } + } + } + + + public Properties getRangerSection() { + Properties prop = this.getProperties(RANGER_SECTION_NAME); + if (prop == null) { + prop = globalLoginProp; + } + return prop; + } + + + + private static void addConfiguration(String dataSource, String resourceName, String propertyName, String value) { + + if (dataSource == null || dataSource.isEmpty()) { + return; + } + + if (propertyName == null || propertyName.isEmpty()) { + return; + } + + if (resourceName == null) { + resourceName = DEFAULT_RESOURCE_NAME; + } + + + HashMap resourceName2PropertiesMap = dataSource2ResourceListMap.get(dataSource); + + if (resourceName2PropertiesMap == null) { + resourceName2PropertiesMap = new HashMap<>(); + dataSource2ResourceListMap.put(dataSource, resourceName2PropertiesMap); + } + + Properties prop = resourceName2PropertiesMap.get(resourceName); + if (prop == null) { + prop = new Properties(); + resourceName2PropertiesMap.put(resourceName, prop); + } + if (value == null) { + prop.remove(propertyName); + } else { + prop.put(propertyName, value); + } + } + + + public String getDatasourceName() { + return datasourceName; + } + + public boolean hasResourceExists(String aResourceName) { // dilli + HashMap resourceName2PropertiesMap = dataSource2ResourceListMap.get(datasourceName); + return (resourceName2PropertiesMap != null && resourceName2PropertiesMap.containsKey(aResourceName)); + } + + public Properties getProperties(String aResourceName) { + Properties ret = null; + HashMap resourceName2PropertiesMap = dataSource2ResourceListMap.get(datasourceName); + if (resourceName2PropertiesMap != null) { + ret = resourceName2PropertiesMap.get(aResourceName); + } + return ret; + } + + public String getHadoopSecurityAuthentication() { + String ret = null; + String sectionName = RANGER_SECTION_NAME; + + if (defaultConfigFile != null) { + sectionName = defaultConfigFile; + } + + if (LOG.isDebugEnabled()) { + LOG.debug("==> HadoopConfigHolder.getHadoopSecurityAuthentication( " + " DataSource : " + sectionName + " Property : " + HADOOP_SECURITY_AUTHENTICATION + ")" ); + } + + ret = getProperties(sectionName,HADOOP_SECURITY_AUTHENTICATION); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== HadoopConfigHolder.getHadoopSecurityAuthentication(" + " DataSource : " + sectionName + " Property : " + HADOOP_SECURITY_AUTHENTICATION + " Value : " + ret + ")" ); + } + + return ret; + } + + public String getUserName() { + return userName; + } + + public String getKeyTabFile() { + return keyTabFile; + } + + public String getPassword() { + return password; + } + + public boolean isKerberosAuthentication() { + return isKerberosAuth; + } + + public String getLookupPrincipal() { + return lookupPrincipal; + } + + public String getLookupKeytab() { + return lookupKeytab; + } + + public String getNameRules() { + return nameRules; + } + + public String getAuthType() { + return authType; + } + + public boolean isEnableHiveMetastoreLookup() { + return enableHiveMetastoreLookup; + } + + public String getHiveSiteFilePath() { + return hiveSiteFilePath; + } + + public Set getRangerInternalPropertyKeys() { + return rangerInternalPropertyKeys; + } + + private String getProperties(String sectionName, String property) { + + if (LOG.isDebugEnabled()) { + LOG.debug("==> HadoopConfigHolder.getProperties( " + " DataSource : " + sectionName + " Property : " + property + ")" ); + } + + Properties repoParam = null; + String ret = null; + + HashMap resourceName2PropertiesMap = dataSource2ResourceListMap.get(this.getDatasourceName()); + + if (resourceName2PropertiesMap != null) { + repoParam=resourceName2PropertiesMap.get(sectionName); + } + + if (repoParam != null) { + ret = (String)repoParam.get(property); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== HadoopConfigHolder.getProperties( " + " DataSource : " + sectionName + " Property : " + property + " Value : " + ret); + } + + return ret; + } + + +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/client/HadoopException.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/client/HadoopException.java new file mode 100644 index 00000000000..7f8cc52d9e9 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/client/HadoopException.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + package org.apache.atlas.plugin.client; + +import org.apache.commons.lang.StringUtils; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + +public class HadoopException extends RuntimeException { + private static final long serialVersionUID = 8872734935128535649L; + + public HashMap responseData; + + public HadoopException() { + super(); + } + + public HadoopException(String message, Throwable cause) { + super(message, cause); + } + + public HadoopException(String message) { + super(message); + } + + public HadoopException(Throwable cause) { + super(cause); + } + + public void generateResponseDataMap(boolean connectivityStatus, + String message, String description, Long objectId, String fieldName) { + responseData = new HashMap<>(); + responseData.put("connectivityStatus", connectivityStatus); + responseData.put("message", message); + responseData.put("description", description); + responseData.put("objectId", objectId); + responseData.put("fieldName", fieldName); + } + + public String getMessage(Throwable excp) { + List errList = new ArrayList<>(); + while (excp != null) { + if (!errList.contains(excp.getMessage() + ". \n") && !errList.contains(excp.toString() + ". \n")) { + if (excp.getMessage() != null && !(excp.getMessage().equalsIgnoreCase(""))) { + errList.add(excp.getMessage() + ". \n"); + } + } + excp = excp.getCause(); + } + return StringUtils.join(errList, ""); + } + + public HashMap getResponseData() { + return responseData; + } + + public void setReponseData(HashMap responseData) { + this.responseData = responseData; + } + +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/conditionevaluator/RangerAbstractConditionEvaluator.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/conditionevaluator/RangerAbstractConditionEvaluator.java new file mode 100644 index 00000000000..8fa0aea9c53 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/conditionevaluator/RangerAbstractConditionEvaluator.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.atlas.plugin.conditionevaluator; + +import org.apache.atlas.plugin.model.RangerPolicy.RangerPolicyItemCondition; +import org.apache.atlas.plugin.model.RangerServiceDef; +import org.apache.atlas.plugin.model.RangerServiceDef.RangerPolicyConditionDef; + +public abstract class RangerAbstractConditionEvaluator implements RangerConditionEvaluator { + protected RangerServiceDef serviceDef; + protected RangerPolicyConditionDef conditionDef; + protected RangerPolicyItemCondition condition; + + @Override + public void setServiceDef(RangerServiceDef serviceDef) { + this.serviceDef = serviceDef; + } + + @Override + public void setConditionDef(RangerPolicyConditionDef conditionDef) { + this.conditionDef = conditionDef; + } + + @Override + public void setPolicyItemCondition(RangerPolicyItemCondition condition) { + this.condition = condition; + } + + @Override + public void init() { + } + + public RangerPolicyItemCondition getPolicyItemCondition() { return condition; } + +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/conditionevaluator/RangerAccessedFromClusterCondition.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/conditionevaluator/RangerAccessedFromClusterCondition.java new file mode 100644 index 00000000000..a1d29d03cdf --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/conditionevaluator/RangerAccessedFromClusterCondition.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.conditionevaluator; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.plugin.policyengine.RangerAccessRequest; + + +public class RangerAccessedFromClusterCondition extends RangerAbstractConditionEvaluator { + private static final Log LOG = LogFactory.getLog(RangerAccessedFromClusterCondition.class); + + private boolean isAlwaysTrue = false; + + @Override + public void init() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerAccessedFromClusterCondition.init(" + condition + ")"); + } + + super.init(); + + isAlwaysTrue = CollectionUtils.isEmpty(condition.getValues()); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerAccessedFromClusterCondition.init(" + condition + ")"); + } + } + + @Override + public boolean isMatched(RangerAccessRequest request) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerAccessedFromClusterCondition.isMatched(" + condition + ")"); + } + + final boolean ret; + + if (isAlwaysTrue || request.getClusterName() == null) { + ret = isAlwaysTrue; + } else { + ret = condition.getValues().contains(request.getClusterName()); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerAccessedFromClusterCondition.isMatched(" + condition + "): " + ret); + } + + return ret; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/conditionevaluator/RangerAccessedFromClusterTypeCondition.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/conditionevaluator/RangerAccessedFromClusterTypeCondition.java new file mode 100644 index 00000000000..cd5242ac516 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/conditionevaluator/RangerAccessedFromClusterTypeCondition.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.atlas.plugin.conditionevaluator; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.plugin.policyengine.RangerAccessRequest; + +public class RangerAccessedFromClusterTypeCondition extends RangerAbstractConditionEvaluator{ + private static final Log LOG = LogFactory.getLog(RangerAccessedFromClusterTypeCondition.class); + + private boolean isAlwaysTrue = false; + + @Override + public void init() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerAccessedFromClusterTypeCondition.init(" + condition + ")"); + } + + super.init(); + + isAlwaysTrue = CollectionUtils.isEmpty(condition.getValues()); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerAccessedFromClusterTypeCondition.init(" + condition + ")"); + } + } + @Override + public boolean isMatched(RangerAccessRequest request) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerAccessedFromClusterTypeCondition.isMatched(" + condition + ")"); + } + + final boolean ret; + + if (isAlwaysTrue || request.getClusterType() == null) { + ret = isAlwaysTrue; + } else { + ret = condition.getValues().contains(request.getClusterType()); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerAccessedFromClusterTypeCondition.isMatched(" + condition + "): " + ret); + } + + return ret; + } + +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/conditionevaluator/RangerAccessedNotFromClusterCondition.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/conditionevaluator/RangerAccessedNotFromClusterCondition.java new file mode 100644 index 00000000000..fea4eddfdca --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/conditionevaluator/RangerAccessedNotFromClusterCondition.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.conditionevaluator; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.plugin.policyengine.RangerAccessRequest; + + +public class RangerAccessedNotFromClusterCondition extends RangerAbstractConditionEvaluator { + private static final Log LOG = LogFactory.getLog(RangerAccessedNotFromClusterCondition.class); + + private boolean isAlwaysTrue = false; + + @Override + public void init() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerAccessedNotFromClusterCondition.init(" + condition + ")"); + } + + super.init(); + + isAlwaysTrue = CollectionUtils.isEmpty(condition.getValues()); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerAccessedNotFromClusterCondition.init(" + condition + ")"); + } + } + + @Override + public boolean isMatched(RangerAccessRequest request) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerAccessedNotFromClusterCondition.isMatched(" + condition + ")"); + } + + final boolean ret; + + if (isAlwaysTrue || request.getClusterName() == null) { + ret = true; + } else { + ret = !condition.getValues().contains(request.getClusterName()); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerAccessedNotFromClusterCondition.isMatched(" + condition + "): " + ret); + } + + return ret; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/conditionevaluator/RangerAccessedNotFromClusterTypeCondition.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/conditionevaluator/RangerAccessedNotFromClusterTypeCondition.java new file mode 100644 index 00000000000..0c3ff007423 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/conditionevaluator/RangerAccessedNotFromClusterTypeCondition.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.atlas.plugin.conditionevaluator; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.plugin.policyengine.RangerAccessRequest; + +public class RangerAccessedNotFromClusterTypeCondition extends RangerAbstractConditionEvaluator{ + private static final Log LOG = LogFactory.getLog(RangerAccessedNotFromClusterTypeCondition.class); + + private boolean isAlwaysTrue = false; + + @Override + public void init() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerAccessedNotFromClusterTypeCondition.init(" + condition + ")"); + } + + super.init(); + + isAlwaysTrue = CollectionUtils.isEmpty(condition.getValues()); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerAccessedNotFromClusterTypeCondition.init(" + condition + ")"); + } + } + + @Override + public boolean isMatched(RangerAccessRequest request) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerAccessedNotFromClusterTypeCondition.isMatched(" + condition + ")"); + } + + final boolean ret; + + if (isAlwaysTrue || request.getClusterType() == null) { + ret = true; + } else { + ret = !condition.getValues().contains(request.getClusterType()); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerAccessedNotFromClusterTypeCondition.isMatched(" + condition + "): " + ret); + } + + return ret; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/conditionevaluator/RangerAnyOfExpectedTagsPresentConditionEvaluator.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/conditionevaluator/RangerAnyOfExpectedTagsPresentConditionEvaluator.java new file mode 100644 index 00000000000..6e5b8311a0c --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/conditionevaluator/RangerAnyOfExpectedTagsPresentConditionEvaluator.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.conditionevaluator; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.plugin.policyengine.RangerAccessRequest; + +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +// Policy Condition to check if resource Tags does contain any of the policy Condition Tags + +public class RangerAnyOfExpectedTagsPresentConditionEvaluator extends RangerAbstractConditionEvaluator { + + private static final Log LOG = LogFactory.getLog(RangerAnyOfExpectedTagsPresentConditionEvaluator.class); + + private final Set policyConditionTags = new HashSet<>(); + + @Override + public void init() { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerAnyOfExpectedTagsPresentConditionEvaluator.init(" + condition + ")"); + } + + super.init(); + + if (condition != null ) { + for (String value : condition.getValues()) { + policyConditionTags.add(value.trim()); + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerAnyOfExpectedTagsPresentConditionEvaluator.init(" + condition + "): Tags[" + policyConditionTags + "]"); + } + } + + @Override + public boolean isMatched(RangerAccessRequest request) { + + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerAnyOfExpectedTagsPresentConditionEvaluator.isMatched(" + request + ")"); + } + + boolean matched = false; + + RangerAccessRequest readOnlyRequest = request.getReadOnlyCopy(); + RangerScriptExecutionContext context = new RangerScriptExecutionContext(readOnlyRequest); + Set resourceTags = context.getAllTagTypes(); + + if (resourceTags != null) { + // check if resource Tags does contain any of the policy Condition Tags + matched = (!Collections.disjoint(resourceTags, policyConditionTags)); + } + + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerAnyOfExpectedTagsPresentConditionEvaluator.isMatched(" + request+ "): " + matched); + } + + return matched; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/conditionevaluator/RangerConditionEvaluator.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/conditionevaluator/RangerConditionEvaluator.java new file mode 100644 index 00000000000..6160bd17b61 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/conditionevaluator/RangerConditionEvaluator.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.conditionevaluator; + +import org.apache.atlas.plugin.model.RangerPolicy.RangerPolicyItemCondition; +import org.apache.atlas.plugin.model.RangerServiceDef; +import org.apache.atlas.plugin.model.RangerServiceDef.RangerPolicyConditionDef; +import org.apache.atlas.plugin.policyengine.RangerAccessRequest; + +public interface RangerConditionEvaluator { + void setConditionDef(RangerPolicyConditionDef conditionDef); + + void setPolicyItemCondition(RangerPolicyItemCondition condition); + + void setServiceDef(RangerServiceDef serviceDef); + + void init(); + + boolean isMatched(RangerAccessRequest request); +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/conditionevaluator/RangerContextAttributeValueInCondition.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/conditionevaluator/RangerContextAttributeValueInCondition.java new file mode 100644 index 00000000000..9d06f0ca802 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/conditionevaluator/RangerContextAttributeValueInCondition.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.conditionevaluator; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.plugin.policyengine.RangerAccessRequest; + +import java.util.Map; + +public class RangerContextAttributeValueInCondition extends RangerAbstractConditionEvaluator { + private static final Log LOG = LogFactory.getLog(RangerContextAttributeValueInCondition.class); + + protected String attributeName; + + @Override + public void init() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerContextAttributeValueInCondition.init(" + condition + ")"); + } + + super.init(); + + Map evalOptions = conditionDef. getEvaluatorOptions(); + + if (MapUtils.isNotEmpty(evalOptions)) { + attributeName = evalOptions.get("attributeName"); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerContextAttributeValueInCondition.init(" + condition + ")"); + } + } + + @Override + public boolean isMatched(RangerAccessRequest request) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerContextAttributeValueInCondition.isMatched(" + condition + ")"); + } + + boolean ret = true; + + if(attributeName != null && condition != null && CollectionUtils.isNotEmpty(condition.getValues())) { + Object val = request.getContext().get(attributeName); + + if(val != null) { + ret = condition.getValues().contains(val); + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerContextAttributeValueInCondition.isMatched(" + condition + "): " + ret); + } + + return ret; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/conditionevaluator/RangerContextAttributeValueNotInCondition.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/conditionevaluator/RangerContextAttributeValueNotInCondition.java new file mode 100644 index 00000000000..3adfd97fd25 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/conditionevaluator/RangerContextAttributeValueNotInCondition.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.conditionevaluator; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.plugin.policyengine.RangerAccessRequest; + +import java.util.Map; + +public class RangerContextAttributeValueNotInCondition extends RangerAbstractConditionEvaluator { + private static final Log LOG = LogFactory.getLog(RangerContextAttributeValueNotInCondition.class); + + protected String attributeName; + + @Override + public void init() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerContextAttributeValueNotInCondition.init(" + condition + ")"); + } + + super.init(); + + Map evalOptions = conditionDef. getEvaluatorOptions(); + + if (MapUtils.isNotEmpty(evalOptions)) { + attributeName = evalOptions.get("attributeName"); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerContextAttributeValueNotInCondition.init(" + condition + ")"); + } + } + + @Override + public boolean isMatched(RangerAccessRequest request) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerContextAttributeValueNotInCondition.isMatched(" + condition + ")"); + } + + boolean ret = true; + + if(attributeName != null && condition != null && CollectionUtils.isNotEmpty(condition.getValues())) { + Object val = request.getContext().get(attributeName); + + if(val != null) { + ret = !condition.getValues().contains(val); + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerContextAttributeValueNotInCondition.isMatched(" + condition + "): " + ret); + } + + return ret; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/conditionevaluator/RangerHiveResourcesAccessedTogetherCondition.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/conditionevaluator/RangerHiveResourcesAccessedTogetherCondition.java new file mode 100644 index 00000000000..b1849dfac33 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/conditionevaluator/RangerHiveResourcesAccessedTogetherCondition.java @@ -0,0 +1,182 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.conditionevaluator; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.plugin.model.RangerPolicy; +import org.apache.atlas.plugin.policyengine.RangerAccessRequest; +import org.apache.atlas.plugin.policyresourcematcher.RangerDefaultPolicyResourceMatcher; +import org.apache.atlas.plugin.policyresourcematcher.RangerPolicyResourceMatcher; +import org.apache.atlas.plugin.store.EmbeddedServiceDefsUtil; +import org.apache.atlas.plugin.util.RangerAccessRequestUtil; +import org.apache.atlas.plugin.util.RangerRequestedResources; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class RangerHiveResourcesAccessedTogetherCondition extends RangerAbstractConditionEvaluator { + private static final Log LOG = LogFactory.getLog(RangerHiveResourcesAccessedTogetherCondition.class); + + private List matchers = new ArrayList<>(); + private boolean isInitialized; + + @Override + public void init() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerHiveResourcesAccessedTogetherCondition.init(" + condition + ")"); + } + + super.init(); + + if (serviceDef != null) { + doInitialize(); + } else { + LOG.error("RangerHiveResourcesAccessedTogetherCondition.init() - ServiceDef not set ... ERROR .."); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerHiveResourcesAccessedTogetherCondition.init(" + condition + ")"); + } + } + + @Override + public boolean isMatched(final RangerAccessRequest request) { + boolean ret = true; + + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerHiveResourcesAccessedTogetherCondition.isMatched(" + request + ")"); + } + + if (isInitialized && CollectionUtils.isNotEmpty(matchers)) { + RangerRequestedResources resources = RangerAccessRequestUtil.getRequestedResourcesFromContext(request.getContext()); + + ret = resources != null && !resources.isMutuallyExcluded(matchers, request.getContext()); + } else { + LOG.error("RangerHiveResourcesAccessedTogetherCondition.isMatched() - condition is not initialized correctly and will NOT be enforced"); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerHiveResourcesAccessedTogetherCondition.isMatched(" + request + ")" + ", result=" + ret); + } + + return ret; + } + + private void doInitialize() { + List mutuallyExclusiveResources = condition.getValues(); + + if (CollectionUtils.isNotEmpty(mutuallyExclusiveResources)) { + initializeMatchers(mutuallyExclusiveResources); + + if (CollectionUtils.isEmpty(matchers)) { + if (LOG.isDebugEnabled()) { + LOG.debug("RangerHiveResourcesAccessedTogetherCondition.doInitialize() - Cannot create matchers from values in MutualExclustionEnforcer"); + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("RangerHiveResourcesAccessedTogetherCondition.doInitialize() - Created " + matchers.size() + " matchers from values in MutualExclustionEnforcer"); + } + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("RangerHiveResourcesAccessedTogetherCondition.doInitialize() - No values in MutualExclustionEnforcer"); + } + } + + isInitialized = true; + } + + private void initializeMatchers(List mutuallyExclusiveResources) { + + for (String s : mutuallyExclusiveResources) { + + String policyResourceSpec = s.trim(); + + RangerPolicyResourceMatcher matcher = buildMatcher(policyResourceSpec); + + if (matcher != null) { + matchers.add(matcher); + } + } + } + + private RangerPolicyResourceMatcher buildMatcher(String policyResourceSpec) { + + RangerPolicyResourceMatcher matcher = null; + + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerHiveResourcesAccessedTogetherCondition.buildMatcher(" + policyResourceSpec + ")"); + } + + // Works only for Hive serviceDef for now + if (serviceDef != null && EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_HIVE_NAME.equals(serviceDef.getName())) { + //Parse policyResourceSpec + char separator = '.'; + String any = "*"; + + Map policyResources = new HashMap<>(); + + String[] elements = StringUtils.split(policyResourceSpec, separator); + + RangerPolicy.RangerPolicyResource policyResource; + + if (elements.length > 0 && elements.length < 4) { + if (elements.length == 3) { + policyResource = new RangerPolicy.RangerPolicyResource(elements[2]); + } else { + policyResource = new RangerPolicy.RangerPolicyResource(any); + } + policyResources.put("column", policyResource); + + if (elements.length >= 2) { + policyResource = new RangerPolicy.RangerPolicyResource(elements[1]); + } else { + policyResource = new RangerPolicy.RangerPolicyResource(any); + } + policyResources.put("table", policyResource); + + policyResource = new RangerPolicy.RangerPolicyResource(elements[0]); + policyResources.put("database", policyResource); + + matcher = new RangerDefaultPolicyResourceMatcher(); + matcher.setPolicyResources(policyResources); + matcher.setServiceDef(serviceDef); + matcher.init(); + + } else { + LOG.error("RangerHiveResourcesAccessedTogetherCondition.buildMatcher() - Incorrect elements in the hierarchy specified (" + + elements.length + ")"); + } + } else { + LOG.error("RangerHiveResourcesAccessedTogetherCondition.buildMatcher() - ServiceDef not set or ServiceDef is not for Hive"); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerHiveResourcesAccessedTogetherCondition.buildMatcher(" + policyResourceSpec + ")" + ", matcher=" + matcher); + } + + return matcher; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/conditionevaluator/RangerHiveResourcesNotAccessedTogetherCondition.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/conditionevaluator/RangerHiveResourcesNotAccessedTogetherCondition.java new file mode 100644 index 00000000000..aca442567ae --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/conditionevaluator/RangerHiveResourcesNotAccessedTogetherCondition.java @@ -0,0 +1,182 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.conditionevaluator; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.plugin.model.RangerPolicy; +import org.apache.atlas.plugin.policyengine.RangerAccessRequest; +import org.apache.atlas.plugin.policyresourcematcher.RangerDefaultPolicyResourceMatcher; +import org.apache.atlas.plugin.policyresourcematcher.RangerPolicyResourceMatcher; +import org.apache.atlas.plugin.store.EmbeddedServiceDefsUtil; +import org.apache.atlas.plugin.util.RangerAccessRequestUtil; +import org.apache.atlas.plugin.util.RangerRequestedResources; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class RangerHiveResourcesNotAccessedTogetherCondition extends RangerAbstractConditionEvaluator { + private static final Log LOG = LogFactory.getLog(RangerHiveResourcesNotAccessedTogetherCondition.class); + + private List matchers = new ArrayList<>(); + private boolean isInitialized; + + @Override + public void init() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerHiveResourcesNotAccessedTogetherCondition.init(" + condition + ")"); + } + + super.init(); + + if (serviceDef != null) { + doInitialize(); + } else { + LOG.error("RangerHiveResourcesNotAccessedTogetherCondition.init() - ServiceDef not set ... ERROR .."); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerHiveResourcesNotAccessedTogetherCondition.init(" + condition + ")"); + } + } + + @Override + public boolean isMatched(final RangerAccessRequest request) { + boolean ret = true; + + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerHiveResourcesNotAccessedTogetherCondition.isMatched(" + request + ")"); + } + + if (isInitialized && CollectionUtils.isNotEmpty(matchers)) { + RangerRequestedResources resources = RangerAccessRequestUtil.getRequestedResourcesFromContext(request.getContext()); + + ret = resources == null || resources.isMutuallyExcluded(matchers, request.getContext()); + } else { + LOG.error("RangerHiveResourcesNotAccessedTogetherCondition.isMatched() - Enforcer is not initialized correctly, Mutual Exclusion will NOT be enforced"); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerHiveResourcesNotAccessedTogetherCondition.isMatched(" + request + ")" + ", result=" + ret); + } + + return ret; + } + + private void doInitialize() { + List mutuallyExclusiveResources = condition.getValues(); + + if (CollectionUtils.isNotEmpty(mutuallyExclusiveResources)) { + initializeMatchers(mutuallyExclusiveResources); + + if (CollectionUtils.isEmpty(matchers)) { + if (LOG.isDebugEnabled()) { + LOG.debug("RangerHiveResourcesNotAccessedTogetherCondition.doInitialize() - Cannot create matchers from values in MutualExclustionEnforcer"); + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("RangerHiveResourcesNotAccessedTogetherCondition.doInitialize() - Created " + matchers.size() + " matchers from values in MutualExclustionEnforcer"); + } + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("RangerHiveResourcesNotAccessedTogetherCondition.doInitialize() - No values in MutualExclustionEnforcer"); + } + } + + isInitialized = true; + } + + private void initializeMatchers(List mutuallyExclusiveResources) { + + for (String s : mutuallyExclusiveResources) { + + String policyResourceSpec = s.trim(); + + RangerPolicyResourceMatcher matcher = buildMatcher(policyResourceSpec); + + if (matcher != null) { + matchers.add(matcher); + } + } + } + + private RangerPolicyResourceMatcher buildMatcher(String policyResourceSpec) { + + RangerPolicyResourceMatcher matcher = null; + + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerHiveResourcesNotAccessedTogetherCondition.buildMatcher(" + policyResourceSpec + ")"); + } + + // Works only for Hive serviceDef for now + if (serviceDef != null && EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_HIVE_NAME.equals(serviceDef.getName())) { + //Parse policyResourceSpec + char separator = '.'; + String any = "*"; + + Map policyResources = new HashMap<>(); + + String[] elements = StringUtils.split(policyResourceSpec, separator); + + RangerPolicy.RangerPolicyResource policyResource; + + if (elements.length > 0 && elements.length < 4) { + if (elements.length == 3) { + policyResource = new RangerPolicy.RangerPolicyResource(elements[2]); + } else { + policyResource = new RangerPolicy.RangerPolicyResource(any); + } + policyResources.put("column", policyResource); + + if (elements.length >= 2) { + policyResource = new RangerPolicy.RangerPolicyResource(elements[1]); + } else { + policyResource = new RangerPolicy.RangerPolicyResource(any); + } + policyResources.put("table", policyResource); + + policyResource = new RangerPolicy.RangerPolicyResource(elements[0]); + policyResources.put("database", policyResource); + + matcher = new RangerDefaultPolicyResourceMatcher(); + matcher.setPolicyResources(policyResources); + matcher.setServiceDef(serviceDef); + matcher.init(); + + } else { + LOG.error("RangerHiveResourcesNotAccessedTogetherCondition.buildMatcher() - Incorrect elements in the hierarchy specified (" + + elements.length + ")"); + } + } else { + LOG.error("RangerHiveResourcesNotAccessedTogetherCondition.buildMatcher() - ServiceDef not set or ServiceDef is not for Hive"); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerHiveResourcesNotAccessedTogetherCondition.buildMatcher(" + policyResourceSpec + ")" + ", matcher=" + matcher); + } + + return matcher; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/conditionevaluator/RangerIpMatcher.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/conditionevaluator/RangerIpMatcher.java new file mode 100644 index 00000000000..ed87b8a34d1 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/conditionevaluator/RangerIpMatcher.java @@ -0,0 +1,216 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +package org.apache.atlas.plugin.conditionevaluator; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.plugin.policyengine.RangerAccessRequest; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Credits: Large parts of this file have been lifted as is from org.apache.atlas.pdp.knox.URLBasedAuthDB. Credits for those are due to Dilli Arumugam. + * @author alal + */ +public class RangerIpMatcher extends RangerAbstractConditionEvaluator { + private static final Log LOG = LogFactory.getLog(RangerIpMatcher.class); + private List _exactIps = new ArrayList<>(); + private List _wildCardIps = new ArrayList<>(); + private boolean _allowAny; + + @Override + public void init() { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerIpMatcher.init(" + condition + ")"); + } + + super.init(); + + // NOTE: this evaluator does not use conditionDef! + if (condition == null) { + LOG.debug("init: null policy condition! Will match always!"); + _allowAny = true; + } else if (CollectionUtils.isEmpty(condition.getValues())) { + LOG.debug("init: empty conditions collection on policy condition! Will match always!"); + _allowAny = true; + } else if (condition.getValues().contains("*")) { + _allowAny = true; + LOG.debug("init: wildcard value found. Will match always."); + } else { + for (String ip : condition.getValues()) { + String digestedIp = digestPolicyIp(ip); + if (digestedIp.isEmpty()) { + LOG.debug("init: digested ip was empty! Will match always"); + _allowAny = true; + } else if (digestedIp.equals(ip)) { + _exactIps.add(ip); + } else { + _wildCardIps.add(digestedIp); + } + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerIpMatcher.init(" + condition + "): exact-ips[" + _exactIps + "], wildcard-ips[" + _wildCardIps + "]"); + } + } + + @Override + public boolean isMatched(final RangerAccessRequest request) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerIpMatcher.isMatched(" + request + ")"); + } + + boolean ipMatched = true; + if (_allowAny) { + LOG.debug("isMatched: allowAny flag is true. Matched!"); + } else { + String requestIp = extractIp(request); + if (requestIp == null) { + LOG.debug("isMatched: couldn't get ip address from request. Ok. Implicitly matched!"); + } else { + ipMatched = isWildcardMatched(_wildCardIps, requestIp) || isExactlyMatched(_exactIps, requestIp); + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerIpMatcher.isMatched(" + request+ "): " + ipMatched); + } + + return ipMatched; + } + + /** + * Pre-digests the policy ip address to drop any trailing wildcard specifiers such that a simple beginsWith match can be done to check for match during authorization calls + * @param ip + * @return + */ + static final Pattern allWildcards = Pattern.compile("^((\\*(\\.\\*)*)|(\\*(:\\*)*))$"); // "*", "*.*", "*.*.*", "*:*", "*:*:*", etc. + static final Pattern trailingWildcardsIp4 = Pattern.compile("(\\.\\*)+$"); // "blah.*", "blah.*.*", etc. + static final Pattern trailingWildcardsIp6 = Pattern.compile("(:\\*)+$"); // "blah:*", "blah:*:*", etc. + + String digestPolicyIp(final String policyIp) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerIpMatcher.digestPolicyIp(" + policyIp + ")"); + } + + String result; + Matcher matcher = allWildcards.matcher(policyIp); + if (matcher.matches()) { + if (LOG.isDebugEnabled()) { + LOG.debug("digestPolicyIp: policyIP[" + policyIp +"] all wildcards."); + } + result = ""; + } else if (policyIp.contains(".")) { + matcher = trailingWildcardsIp4.matcher(policyIp); + result = matcher.replaceFirst("."); + } else { + matcher = trailingWildcardsIp6.matcher(policyIp); + // also lower cases the ipv6 items + result = matcher.replaceFirst(":").toLowerCase(); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerIpMatcher.digestPolicyIp(" + policyIp + "): " + policyIp); + } + return result; + } + + boolean isWildcardMatched(final List ips, final String requestIp) { + + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerIpMatcher.isWildcardMatched(" + ips+ ", " + requestIp + ")"); + } + + boolean matchFound = false; + Iterator iterator = ips.iterator(); + while (iterator.hasNext() && !matchFound) { + String ip = iterator.next(); + if (requestIp.contains(".") && requestIp.startsWith(ip)) { + if (LOG.isDebugEnabled()) { + LOG.debug("Wildcard Policy IP[" + ip + "] matches request IPv4[" + requestIp + "]."); + } + matchFound = true; + } else if (requestIp.toLowerCase().startsWith(ip)) { + if (LOG.isDebugEnabled()) { + LOG.debug("Wildcard Policy IP[" + ip + "] matches request IPv6[" + requestIp + "]."); + } + matchFound = true; + } else { + LOG.debug("Wildcard policy IP[" + ip + "] did not match request IP[" + requestIp + "]."); + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerIpMatcher.isWildcardMatched(" + ips+ ", " + requestIp + "): " + matchFound); + } + return matchFound; + } + + boolean isExactlyMatched(final List ips, final String requestIp) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerIpMatcher.isExactlyMatched(" + ips+ ", " + requestIp + ")"); + } + + boolean matchFound = false; + if (requestIp.contains(".")) { + matchFound = ips.contains(requestIp); + } else { + matchFound = ips.contains(requestIp.toLowerCase()); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerIpMatcher.isExactlyMatched(" + ips+ ", " + requestIp + "): " + matchFound); + } + return matchFound; + } + + /** + * Extracts and returns the ip address from the request. Returns null if one can't be obtained out of the request. + * @param request + * @return + */ + String extractIp(final RangerAccessRequest request) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerIpMatcher.extractIp(" + request+ ")"); + } + + String ip = null; + if (request == null) { + LOG.debug("isMatched: Unexpected: null request object!"); + } else { + ip = request.getClientIPAddress(); + if (ip == null) { + LOG.debug("isMatched: Unexpected: Client ip in request object is null!"); + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerIpMatcher.extractIp(" + request+ "): " + ip); + } + return ip; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/conditionevaluator/RangerNoneOfExpectedTagsPresentConditionEvaluator.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/conditionevaluator/RangerNoneOfExpectedTagsPresentConditionEvaluator.java new file mode 100644 index 00000000000..8ab50d84a2e --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/conditionevaluator/RangerNoneOfExpectedTagsPresentConditionEvaluator.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.conditionevaluator; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.plugin.policyengine.RangerAccessRequest; + +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +//PolicyCondition to check if resource Tags does not contain any of the tags in the policy condition + +public class RangerNoneOfExpectedTagsPresentConditionEvaluator extends RangerAbstractConditionEvaluator { + + private static final Log LOG = LogFactory.getLog(RangerNoneOfExpectedTagsPresentConditionEvaluator.class); + + private final Set policyConditionTags = new HashSet<>(); + + @Override + public void init() { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerNoneOfExpectedTagsPresentConditionEvaluator.init(" + condition + ")"); + } + + super.init(); + + if (condition != null ) { + for (String value : condition.getValues()) { + policyConditionTags.add(value.trim()); + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerNoneOfExpectedTagsPresentConditionEvaluator.init(" + condition + "): Tags[" + policyConditionTags + "]"); + } + } + + @Override + public boolean isMatched(RangerAccessRequest request) { + + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerNoneOfExpectedTagsPresentConditionEvaluator.isMatched(" + request + ")"); + } + + boolean matched = true; + + RangerAccessRequest readOnlyRequest = request.getReadOnlyCopy(); + RangerScriptExecutionContext context = new RangerScriptExecutionContext(readOnlyRequest); + Set resourceTags = context.getAllTagTypes(); + + if (resourceTags != null) { + // check if resource Tags does not contain any tags in the policy condition + matched = (Collections.disjoint(resourceTags, policyConditionTags)); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerNoneOfExpectedTagsPresentConditionEvaluator.isMatched(" + request+ "): " + matched); + } + + return matched; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/conditionevaluator/RangerScriptConditionEvaluator.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/conditionevaluator/RangerScriptConditionEvaluator.java new file mode 100644 index 00000000000..f1413733f71 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/conditionevaluator/RangerScriptConditionEvaluator.java @@ -0,0 +1,232 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.conditionevaluator; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.plugin.classloader.RangerPluginClassLoader; +import org.apache.atlas.plugin.contextenricher.RangerTagForEval; +import org.apache.atlas.plugin.policyengine.RangerAccessRequest; +import org.apache.atlas.plugin.util.RangerPerfTracer; + +import javax.script.Bindings; +import javax.script.ScriptEngine; +import javax.script.ScriptEngineFactory; +import javax.script.ScriptEngineManager; +import javax.script.ScriptException; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import static org.apache.atlas.plugin.util.RangerCommonConstants.SCRIPT_OPTION_ENABLE_JSON_CTX; +import static org.apache.atlas.plugin.util.RangerCommonConstants.SCRIPT_VAR_CONTEXT; +import static org.apache.atlas.plugin.util.RangerCommonConstants.SCRIPT_VAR_CONTEXT_JSON; + +public class RangerScriptConditionEvaluator extends RangerAbstractConditionEvaluator { + private static final Log LOG = LogFactory.getLog(RangerScriptConditionEvaluator.class); + + private static final Log PERF_POLICY_CONDITION_SCRIPT_EVAL = RangerPerfTracer.getPerfLogger("policy.condition.script.eval"); + + private static final String SCRIPT_PREEXEC = SCRIPT_VAR_CONTEXT + "=JSON.parse(" + SCRIPT_VAR_CONTEXT_JSON + ");"; + + private ScriptEngine scriptEngine; + private boolean enableJsonCtx = false; + + @Override + public void init() { + + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerScriptConditionEvaluator.init(" + condition + ")"); + } + + super.init(); + + String engineName = "JavaScript"; + + Map evalOptions = conditionDef. getEvaluatorOptions(); + + if (MapUtils.isNotEmpty(evalOptions)) { + engineName = evalOptions.get("engineName"); + + enableJsonCtx = Boolean.parseBoolean(evalOptions.getOrDefault(SCRIPT_OPTION_ENABLE_JSON_CTX, Boolean.toString(enableJsonCtx))); + } + + if (StringUtils.isBlank(engineName)) { + engineName = "JavaScript"; + } + + if (LOG.isDebugEnabled()) { + LOG.debug("RangerScriptConditionEvaluator.init() - engineName=" + engineName); + } + + String conditionType = condition != null ? condition.getType() : null; + + try { + ScriptEngineManager manager = new ScriptEngineManager(); + + if (LOG.isDebugEnabled()) { + List factories = manager.getEngineFactories(); + + if (CollectionUtils.isEmpty(factories)) { + LOG.debug("List of scriptEngineFactories is empty!!"); + } else { + for (ScriptEngineFactory factory : factories) { + LOG.debug("engineName=" + factory.getEngineName() + ", language=" + factory.getLanguageName()); + } + } + } + + scriptEngine = manager.getEngineByName(engineName); + } catch (Exception exp) { + LOG.error("RangerScriptConditionEvaluator.init() failed with exception=" + exp); + } + + if (scriptEngine == null) { + LOG.warn("failed to initialize condition '" + conditionType + "': script engine '" + engineName + "' was not created in a default manner"); + LOG.info("Will try to get script-engine from plugin-class-loader"); + + + RangerPluginClassLoader pluginClassLoader; + + try { + + pluginClassLoader = RangerPluginClassLoader.getInstance(serviceDef.getName(), null); + + if (pluginClassLoader != null) { + scriptEngine = pluginClassLoader.getScriptEngine(engineName); + } else { + LOG.error("Cannot get script-engine from null pluginClassLoader"); + } + + } catch (Throwable exp) { + LOG.error("RangerScriptConditionEvaluator.init() failed with exception=", exp); + } + } + + if (scriptEngine == null) { + LOG.error("failed to initialize condition '" + conditionType + "': script engine '" + engineName + "' was not created"); + } else { + LOG.info("ScriptEngine for engineName=[" + engineName + "] is successfully created"); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerScriptConditionEvaluator.init(" + condition + ")"); + } + } + + @Override + public boolean isMatched(RangerAccessRequest request) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerScriptConditionEvaluator.isMatched()"); + } + boolean result = true; + + if (scriptEngine != null) { + + String script = getScript(); + + if (StringUtils.isNotBlank(script)) { + + RangerAccessRequest readOnlyRequest = request.getReadOnlyCopy(); + + RangerScriptExecutionContext context = new RangerScriptExecutionContext(readOnlyRequest); + RangerTagForEval currentTag = context.getCurrentTag(); + Map tagAttribs = currentTag != null ? currentTag.getAttributes() : Collections.emptyMap(); + + Bindings bindings = scriptEngine.createBindings(); + + bindings.put("ctx", context); + bindings.put("tag", currentTag); + bindings.put("tagAttr", tagAttribs); + + if (enableJsonCtx) { + bindings.put(SCRIPT_VAR_CONTEXT_JSON, context.toJson()); + + script = SCRIPT_PREEXEC + script; + } + + if (LOG.isDebugEnabled()) { + LOG.debug("RangerScriptConditionEvaluator.isMatched(): script={" + script + "}"); + } + + RangerPerfTracer perf = null; + + try { + long requestHash = request.hashCode(); + + if (RangerPerfTracer.isPerfTraceEnabled(PERF_POLICY_CONDITION_SCRIPT_EVAL)) { + perf = RangerPerfTracer.getPerfTracer(PERF_POLICY_CONDITION_SCRIPT_EVAL, "RangerScriptConditionEvaluator.isMatched(requestHash=" + requestHash + ")"); + } + + Object ret = scriptEngine.eval(script, bindings); + + if (ret == null) { + ret = context.getResult(); + } + if (ret instanceof Boolean) { + result = (Boolean) ret; + } + + } catch (NullPointerException nullp) { + LOG.error("RangerScriptConditionEvaluator.isMatched(): eval called with NULL argument(s)", nullp); + + } catch (ScriptException exception) { + LOG.error("RangerScriptConditionEvaluator.isMatched(): failed to evaluate script," + + " exception=" + exception); + } finally { + RangerPerfTracer.log(perf); + } + } else { + String conditionType = condition != null ? condition.getType() : null; + LOG.error("failed to evaluate condition '" + conditionType + "': script is empty"); + } + + } else { + String conditionType = condition != null ? condition.getType() : null; + LOG.error("failed to evaluate condition '" + conditionType + "': script engine not found"); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerScriptConditionEvaluator.isMatched(), result=" + result); + } + + return result; + + } + + protected String getScript() { + String ret = null; + + List values = condition.getValues(); + + if (CollectionUtils.isNotEmpty(values)) { + + String value = values.get(0); + if (StringUtils.isNotBlank(value)) { + ret = value.trim(); + } + } + + return ret; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/conditionevaluator/RangerScriptExecutionContext.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/conditionevaluator/RangerScriptExecutionContext.java new file mode 100644 index 00000000000..2b39df3b05f --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/conditionevaluator/RangerScriptExecutionContext.java @@ -0,0 +1,568 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.conditionevaluator; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.atlas.authorization.utils.JsonUtils; +import org.apache.atlas.authorization.utils.StringUtil; +import org.apache.atlas.plugin.contextenricher.RangerTagForEval; +import org.apache.atlas.plugin.policyengine.RangerAccessRequest; +import org.apache.atlas.plugin.policyengine.RangerAccessResource; +import org.apache.atlas.plugin.util.RangerAccessRequestUtil; +import org.apache.atlas.plugin.util.RangerPerfTracer; +import org.apache.atlas.plugin.util.RangerUserStore; + +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TimeZone; + +import static org.apache.atlas.plugin.util.RangerCommonConstants.SCRIPT_FIELD_ACCESS_TIME; +import static org.apache.atlas.plugin.util.RangerCommonConstants.SCRIPT_FIELD_ACCESS_TYPE; +import static org.apache.atlas.plugin.util.RangerCommonConstants.SCRIPT_FIELD_ACTION; +import static org.apache.atlas.plugin.util.RangerCommonConstants.SCRIPT_FIELD_CLIENT_IP_ADDRESS; +import static org.apache.atlas.plugin.util.RangerCommonConstants.SCRIPT_FIELD_CLIENT_TYPE; +import static org.apache.atlas.plugin.util.RangerCommonConstants.SCRIPT_FIELD_CLUSTER_NAME; +import static org.apache.atlas.plugin.util.RangerCommonConstants.SCRIPT_FIELD_CLUSTER_TYPE; +import static org.apache.atlas.plugin.util.RangerCommonConstants.SCRIPT_FIELD_FORWARDED_ADDRESSES; +import static org.apache.atlas.plugin.util.RangerCommonConstants.SCRIPT_FIELD_REMOTE_IP_ADDRESS; +import static org.apache.atlas.plugin.util.RangerCommonConstants.SCRIPT_FIELD_REQUEST; +import static org.apache.atlas.plugin.util.RangerCommonConstants.SCRIPT_FIELD_REQUEST_DATA; +import static org.apache.atlas.plugin.util.RangerCommonConstants.SCRIPT_FIELD_RESOURCE; +import static org.apache.atlas.plugin.util.RangerCommonConstants.SCRIPT_FIELD_RESOURCE_MATCHING_SCOPE; +import static org.apache.atlas.plugin.util.RangerCommonConstants.SCRIPT_FIELD_RESOURCE_OWNER_USER; +import static org.apache.atlas.plugin.util.RangerCommonConstants.SCRIPT_FIELD_TAG; +import static org.apache.atlas.plugin.util.RangerCommonConstants.SCRIPT_FIELD_TAGS; +import static org.apache.atlas.plugin.util.RangerCommonConstants.SCRIPT_FIELD_USER; +import static org.apache.atlas.plugin.util.RangerCommonConstants.SCRIPT_FIELD_USER_ATTRIBUTES; +import static org.apache.atlas.plugin.util.RangerCommonConstants.SCRIPT_FIELD_USER_GROUPS; +import static org.apache.atlas.plugin.util.RangerCommonConstants.SCRIPT_FIELD_USER_GROUP_ATTRIBUTES; +import static org.apache.atlas.plugin.util.RangerCommonConstants.SCRIPT_FIELD_USER_ROLES; + + +public final class RangerScriptExecutionContext { + private static final Log LOG = LogFactory.getLog(RangerScriptExecutionContext.class); + + private static final Log PERF_POLICY_CONDITION_SCRIPT_TOJSON = RangerPerfTracer.getPerfLogger("policy.condition.script.tojson"); + private static final String TAG_ATTR_DATE_FORMAT_PROP = "ranger.plugin.tag.attr.additional.date.formats"; + private static final String TAG_ATTR_DATE_FORMAT_SEPARATOR = "||"; + private static final String TAG_ATTR_DATE_FORMAT_SEPARATOR_REGEX = "\\|\\|"; + private static final String DEFAULT_RANGER_TAG_ATTRIBUTE_DATE_FORMAT = "yyyy/MM/dd"; + private static final String DEFAULT_ATLAS_TAG_ATTRIBUTE_DATE_FORMAT_NAME = "ATLAS_DATE_FORMAT"; + private static final String DEFAULT_ATLAS_TAG_ATTRIBUTE_DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"; + + private final RangerAccessRequest accessRequest; + private Boolean result = false; + + private static String[] dateFormatStrings = null; + + static { + init(null); + } + + private static final ThreadLocal> THREADLOCAL_DATE_FORMATS = + new ThreadLocal>() { + @Override protected List initialValue() { + List ret = new ArrayList<>(); + + for (String dateFormatString : dateFormatStrings) { + try { + if (StringUtils.isNotBlank(dateFormatString)) { + if (StringUtils.equalsIgnoreCase(dateFormatString, DEFAULT_ATLAS_TAG_ATTRIBUTE_DATE_FORMAT_NAME)) { + dateFormatString = DEFAULT_ATLAS_TAG_ATTRIBUTE_DATE_FORMAT; + } + SimpleDateFormat df = new SimpleDateFormat(dateFormatString); + df.setLenient(false); + ret.add(df); + } + } catch (Exception exception) { + // Ignore + } + } + + return ret; + } + }; + + RangerScriptExecutionContext(final RangerAccessRequest accessRequest) { + this.accessRequest = accessRequest; + } + + public String toJson() { + RangerPerfTracer perf = null; + + long requestHash = accessRequest.hashCode(); + + if (RangerPerfTracer.isPerfTraceEnabled(PERF_POLICY_CONDITION_SCRIPT_TOJSON)) { + perf = RangerPerfTracer.getPerfTracer(PERF_POLICY_CONDITION_SCRIPT_TOJSON, "RangerScriptExecutionContext.toJson(requestHash=" + requestHash + ")"); + } + + Map ret = new HashMap<>(); + Map request = new HashMap<>(); + RangerUserStore userStore = RangerAccessRequestUtil.getRequestUserStoreFromContext(accessRequest.getContext()); + + if (accessRequest.getAccessTime() != null) { + request.put(SCRIPT_FIELD_ACCESS_TIME, accessRequest.getAccessTime().getTime()); + } + + request.put(SCRIPT_FIELD_ACCESS_TYPE, accessRequest.getAccessType()); + request.put(SCRIPT_FIELD_ACTION, accessRequest.getAction()); + request.put(SCRIPT_FIELD_CLIENT_IP_ADDRESS, accessRequest.getClientIPAddress()); + request.put(SCRIPT_FIELD_CLIENT_TYPE, accessRequest.getClientType()); + request.put(SCRIPT_FIELD_CLUSTER_NAME, accessRequest.getClusterName()); + request.put(SCRIPT_FIELD_CLUSTER_TYPE, accessRequest.getClusterType()); + request.put(SCRIPT_FIELD_FORWARDED_ADDRESSES, accessRequest.getForwardedAddresses()); + request.put(SCRIPT_FIELD_REMOTE_IP_ADDRESS, accessRequest.getRemoteIPAddress()); + request.put(SCRIPT_FIELD_REQUEST_DATA, accessRequest.getRequestData()); + + if (accessRequest.getResource() != null) { + request.put(SCRIPT_FIELD_RESOURCE, accessRequest.getResource().getAsMap()); + request.put(SCRIPT_FIELD_RESOURCE_OWNER_USER, accessRequest.getResource().getOwnerUser()); + } + + request.put(SCRIPT_FIELD_RESOURCE_MATCHING_SCOPE, accessRequest.getResourceMatchingScope()); + + request.put(SCRIPT_FIELD_USER, accessRequest.getUser()); + request.put(SCRIPT_FIELD_USER_GROUPS, accessRequest.getUserGroups()); + request.put(SCRIPT_FIELD_USER_ROLES, accessRequest.getUserRoles()); + + if (userStore != null) { + Map> userAttrMapping = userStore.getUserAttrMapping(); + Map> groupAttrMapping = userStore.getGroupAttrMapping(); + + if (userAttrMapping != null) { + request.put(SCRIPT_FIELD_USER_ATTRIBUTES, userAttrMapping.get(accessRequest.getUser())); + } + + if (groupAttrMapping != null && accessRequest.getUserGroups() != null) { + Map> groupAttrs = new HashMap<>(); + + for (String groupName : accessRequest.getUserGroups()) { + groupAttrs.put(groupName, groupAttrMapping.get(groupName)); + } + + request.put(SCRIPT_FIELD_USER_GROUP_ATTRIBUTES, groupAttrs); + } + } + + ret.put(SCRIPT_FIELD_REQUEST, request); + + Set requestTags = RangerAccessRequestUtil.getRequestTagsFromContext(getRequestContext()); + + if (CollectionUtils.isNotEmpty(requestTags)) { + Set> tags = new HashSet<>(); + + for (RangerTagForEval tag : requestTags) { + tags.add(toMap(tag)); + } + + ret.put(SCRIPT_FIELD_TAGS, tags); + + RangerTagForEval currentTag = RangerAccessRequestUtil.getCurrentTagFromContext(getRequestContext()); + + if (currentTag != null) { + ret.put(SCRIPT_FIELD_TAG, toMap(currentTag)); + } + } + + RangerAccessResource resource = RangerAccessRequestUtil.getCurrentResourceFromContext(getRequestContext()); + + if (resource != null) { + ret.put(SCRIPT_FIELD_RESOURCE, resource.getAsMap()); + + if (resource.getOwnerUser() != null) { + ret.put(SCRIPT_FIELD_RESOURCE_OWNER_USER, resource.getOwnerUser()); + } + } + + String strRet = JsonUtils.objectToJson(ret); + + RangerPerfTracer.log(perf); + + return strRet; + } + + public static void init(Configuration config) { + StringBuilder sb = new StringBuilder(DEFAULT_RANGER_TAG_ATTRIBUTE_DATE_FORMAT); + + sb.append(TAG_ATTR_DATE_FORMAT_SEPARATOR).append(DEFAULT_ATLAS_TAG_ATTRIBUTE_DATE_FORMAT_NAME); + + String additionalDateFormatsValue = config != null ? config.get(TAG_ATTR_DATE_FORMAT_PROP) : null; + + if (StringUtils.isNotBlank(additionalDateFormatsValue)) { + sb.append(TAG_ATTR_DATE_FORMAT_SEPARATOR).append(additionalDateFormatsValue); + } + + String[] formatStrings = sb.toString().split(TAG_ATTR_DATE_FORMAT_SEPARATOR_REGEX); + + Arrays.sort(formatStrings, new Comparator() { + @Override + public int compare(String first, String second) { + return Integer.compare(second.length(), first.length()); + } + }); + + RangerScriptExecutionContext.dateFormatStrings = formatStrings; + } + + public String getResource() { + String ret = null; + RangerAccessResource val = RangerAccessRequestUtil.getCurrentResourceFromContext(getRequestContext()); + + if(val != null) { + ret = val.getAsString(); + } + + return ret; + } + + public Map getRequestContext() { + return accessRequest.getContext(); + } + + public String getRequestContextAttribute(String attributeName) { + String ret = null; + + if (StringUtils.isNotBlank(attributeName)) { + Object val = getRequestContext().get(attributeName); + + if(val != null) { + ret = val.toString(); + } + } + + return ret; + } + + public boolean isAccessTypeAny() { return accessRequest.isAccessTypeAny(); } + + public boolean isAccessTypeDelegatedAdmin() { return accessRequest.isAccessTypeDelegatedAdmin(); } + + public String getUser() { return accessRequest.getUser(); } + + public Set getUserGroups() { return accessRequest.getUserGroups(); } + + public Date getAccessTime() { return accessRequest.getAccessTime() != null ? accessRequest.getAccessTime() : new Date(); } + + public String getClientIPAddress() { return accessRequest.getClientIPAddress(); } + + public String getClientType() { return accessRequest.getClientType(); } + + public String getAction() { return accessRequest.getAction(); } + + public String getRequestData() { return accessRequest.getRequestData(); } + + public String getSessionId() { return accessRequest.getSessionId(); } + + public RangerTagForEval getCurrentTag() { + RangerTagForEval ret = RangerAccessRequestUtil.getCurrentTagFromContext(getRequestContext()); + + if(ret == null ) { + if (LOG.isDebugEnabled()) { + logDebug("RangerScriptExecutionContext.getCurrentTag() - No current TAG object. Script execution must be for resource-based policy."); + } + } + return ret; + } + + public String getCurrentTagType() { + RangerTagForEval tagObject = getCurrentTag(); + return (tagObject != null) ? tagObject.getType() : null; + } + + public Set getAllTagTypes() { + Set allTagTypes = null; + Set tagObjectList = getAllTags(); + + if (CollectionUtils.isNotEmpty(tagObjectList)) { + for (RangerTagForEval tag : tagObjectList) { + String tagType = tag.getType(); + if (allTagTypes == null) { + allTagTypes = new HashSet<>(); + } + allTagTypes.add(tagType); + } + } + + return allTagTypes; + } + + public Map getTagAttributes(final String tagType) { + Map ret = null; + + if (StringUtils.isNotBlank(tagType)) { + Set tagObjectList = getAllTags(); + + // Assumption: There is exactly one tag with given tagType in the list of tags - may not be true ***TODO*** + // This will get attributes of the first tagType that matches + if (CollectionUtils.isNotEmpty(tagObjectList)) { + for (RangerTagForEval tag : tagObjectList) { + if (tag.getType().equals(tagType)) { + ret = tag.getAttributes(); + break; + } + } + } + } + + return ret; + } + + public List> getTagAttributesForAllMatchingTags(final String tagType) { + List> ret = null; + + if (StringUtils.isNotBlank(tagType)) { + Set tagObjectList = getAllTags(); + + // Assumption: There is exactly one tag with given tagType in the list of tags - may not be true ***TODO*** + // This will get attributes of the first tagType that matches + if (CollectionUtils.isNotEmpty(tagObjectList)) { + for (RangerTagForEval tag : tagObjectList) { + if (tag.getType().equals(tagType)) { + Map tagAttributes = tag.getAttributes(); + if (tagAttributes != null) { + if (ret == null) { + ret = new ArrayList<>(); + } + ret.add(tagAttributes); + } + break; + } + } + } + } + + return ret; + } + + public Set getAttributeNames(final String tagType) { + Set ret = null; + Map attributes = getTagAttributes(tagType); + + if (attributes != null) { + ret = attributes.keySet(); + } + + return ret; + } + + public String getAttributeValue(final String tagType, final String attributeName) { + String ret = null; + + if (StringUtils.isNotBlank(tagType) || StringUtils.isNotBlank(attributeName)) { + Map attributes = getTagAttributes(tagType); + + if (attributes != null) { + ret = attributes.get(attributeName); + } + } + return ret; + } + + public List getAttributeValueForAllMatchingTags(final String tagType, final String attributeName) { + List ret = null; + + if (StringUtils.isNotBlank(tagType) || StringUtils.isNotBlank(attributeName)) { + Map attributes = getTagAttributes(tagType); + + if (attributes != null && attributes.get(attributeName) != null) { + if (ret == null) { + ret = new ArrayList<>(); + } + ret.add(attributes.get(attributeName)); + } + } + return ret; + } + + public String getAttributeValue(final String attributeName) { + String ret = null; + + if (StringUtils.isNotBlank(attributeName)) { + RangerTagForEval tag = getCurrentTag(); + Map attributes = null; + if (tag != null) { + attributes = tag.getAttributes(); + } + if (attributes != null) { + ret = attributes.get(attributeName); + } + } + + return ret; + } + + public boolean getResult() { + return result; + + } + + public void setResult(final boolean result) { + this.result = result; + } + + private Date getAsDate(String value, SimpleDateFormat df) { + Date ret = null; + + TimeZone savedTimeZone = df.getTimeZone(); + try { + ret = df.parse(value); + } catch (ParseException exception) { + // Ignore + } finally { + df.setTimeZone(savedTimeZone); + } + + return ret; + } + + public Date getAsDate(String value) { + Date ret = null; + + if (StringUtils.isNotBlank(value)) { + for (SimpleDateFormat simpleDateFormat : THREADLOCAL_DATE_FORMATS.get()) { + ret = getAsDate(value, simpleDateFormat); + if (ret != null) { + if (LOG.isDebugEnabled()) { + logDebug("RangerScriptExecutionContext.getAsDate() -The best match found for Format-String:[" + simpleDateFormat.toPattern() + "], date:[" + ret +"]"); + } + break; + } + } + } + + if (ret == null) { + logError("RangerScriptExecutionContext.getAsDate() - Could not convert [" + value + "] to Date using any of the Format-Strings: " + Arrays.toString(dateFormatStrings)); + } else { + ret = StringUtil.getUTCDateForLocalDate(ret); + } + + return ret; + } + + public Date getTagAttributeAsDate(String tagType, String attributeName) { + String attrValue = getAttributeValue(tagType, attributeName); + + return getAsDate(attrValue); + } + + public boolean isAccessedAfter(String tagType, String attributeName) { + boolean ret = false; + Date accessDate = getAccessTime(); + Date expiryDate = getTagAttributeAsDate(tagType, attributeName); + + if (expiryDate == null || accessDate.after(expiryDate) || accessDate.equals(expiryDate)) { + ret = true; + } + + return ret; + } + + public boolean isAccessedAfter(String attributeName) { + boolean ret = false; + Date accessDate = getAccessTime(); + Date expiryDate = getAsDate(getAttributeValue(attributeName)); + + if (expiryDate == null || accessDate.after(expiryDate) || accessDate.equals(expiryDate)) { + ret = true; + } + + return ret; + } + + public boolean isAccessedBefore(String tagType, String attributeName) { + boolean ret = true; + Date accessDate = getAccessTime(); + Date expiryDate = getTagAttributeAsDate(tagType, attributeName); + + if (expiryDate == null || accessDate.after(expiryDate)) { + ret = false; + } + + return ret; + } + + public boolean isAccessedBefore(String attributeName) { + boolean ret = true; + Date accessDate = getAccessTime(); + Date expiryDate = getAsDate(getAttributeValue(attributeName)); + + if (expiryDate == null || accessDate.after(expiryDate)) { + ret = false; + } + + return ret; + } + + private Set getAllTags() { + Set ret = RangerAccessRequestUtil.getRequestTagsFromContext(accessRequest.getContext()); + if(ret == null) { + if (LOG.isDebugEnabled()) { + String resource = accessRequest.getResource().getAsString(); + + logDebug("RangerScriptExecutionContext.getAllTags() - No TAGS. No TAGS for the RangerAccessResource=" + resource); + } + } + + return ret; + } + + private static Map toMap(RangerTagForEval tag) { + Map ret = new HashMap<>(); + + ret.put("type", tag.getType()); + ret.put("attributes", tag.getAttributes()); + ret.put("matchType", tag.getMatchType()); + + return ret; + } + + public void logDebug(Object msg) { + LOG.debug(msg); + } + + public void logInfo(Object msg) { + LOG.info(msg); + } + + public void logWarn(Object msg) { + LOG.warn(msg); + } + + public void logError(Object msg) { + LOG.error(msg); + } + + public void logFatal(Object msg) { + LOG.fatal(msg); + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/conditionevaluator/RangerScriptTemplateConditionEvaluator.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/conditionevaluator/RangerScriptTemplateConditionEvaluator.java new file mode 100644 index 00000000000..dc397c28959 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/conditionevaluator/RangerScriptTemplateConditionEvaluator.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.conditionevaluator; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.plugin.policyengine.RangerAccessRequest; + +public class RangerScriptTemplateConditionEvaluator extends RangerScriptConditionEvaluator { + private static final Log LOG = LogFactory.getLog(RangerScriptTemplateConditionEvaluator.class); + + protected String script; + private boolean reverseResult; + + @Override + public void init() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerScriptTemplateConditionEvaluator.init(" + condition + ")"); + } + + super.init(); + + if(CollectionUtils.isNotEmpty(condition.getValues())) { + String expectedScriptReturn = condition.getValues().get(0); + + if(StringUtils.isNotBlank(expectedScriptReturn)) { + if(StringUtils.equalsIgnoreCase(expectedScriptReturn, "false") || StringUtils.equalsIgnoreCase(expectedScriptReturn, "no")) { + reverseResult = true; + } + + script = MapUtils.getString(conditionDef.getEvaluatorOptions(), "scriptTemplate"); + + if(script != null) { + script = script.trim(); + } + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerScriptTemplateConditionEvaluator.init(" + condition + "): script=" + script + "; reverseResult=" + reverseResult); + } + } + + @Override + public boolean isMatched(RangerAccessRequest request) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerScriptTemplateConditionEvaluator.isMatched()"); + } + + boolean ret = super.isMatched(request); + + if(reverseResult) { + ret = !ret; + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerScriptTemplateConditionEvaluator.isMatched(): ret=" + ret); + } + + return ret; + } + + @Override + protected String getScript() { + return script; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/conditionevaluator/RangerTagsAllPresentConditionEvaluator.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/conditionevaluator/RangerTagsAllPresentConditionEvaluator.java new file mode 100644 index 00000000000..42de615de5e --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/conditionevaluator/RangerTagsAllPresentConditionEvaluator.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.conditionevaluator; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.plugin.policyengine.RangerAccessRequest; + +import java.util.HashSet; +import java.util.Set; + + +public class RangerTagsAllPresentConditionEvaluator extends RangerAbstractConditionEvaluator { + + private static final Log LOG = LogFactory.getLog(RangerTagsAllPresentConditionEvaluator.class); + + private final Set policyConditionTags = new HashSet<>(); + + @Override + public void init() { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerTagsAllPresentConditionEvaluator.init(" + condition + ")"); + } + + super.init(); + + if (condition != null ) { + for (String value : condition.getValues()) { + policyConditionTags.add(value.trim()); + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerTagsAllPresentConditionEvaluator.init(" + condition + "): Tags[" + policyConditionTags + "]"); + } + } + + @Override + public boolean isMatched(RangerAccessRequest request) { + + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerTagsAllPresentConditionEvaluator.isMatched(" + request + ")"); + } + + boolean matched = true; + + if (CollectionUtils.isNotEmpty(policyConditionTags)) { + RangerAccessRequest readOnlyRequest = request.getReadOnlyCopy(); + RangerScriptExecutionContext context = new RangerScriptExecutionContext(readOnlyRequest); + Set resourceTags = context.getAllTagTypes(); + + // check if resource Tags atleast have to have all the tags in policy Condition + matched = resourceTags != null && resourceTags.containsAll(policyConditionTags); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerTagsAllPresentConditionEvaluator.isMatched(" + request+ "): " + matched); + } + + return matched; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/conditionevaluator/RangerTimeOfDayMatcher.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/conditionevaluator/RangerTimeOfDayMatcher.java new file mode 100644 index 00000000000..d265c68670c --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/conditionevaluator/RangerTimeOfDayMatcher.java @@ -0,0 +1,203 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.conditionevaluator; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.plugin.policyengine.RangerAccessRequest; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Calendar; +import java.util.Date; +import java.util.GregorianCalendar; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class RangerTimeOfDayMatcher extends RangerAbstractConditionEvaluator { + private static final Log LOG = LogFactory.getLog(RangerTimeOfDayMatcher.class); + boolean _allowAny; + List _durations = new ArrayList<>(); + + @Override + public void init() { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerTimeOfDayMatcher.init(" + condition + ")"); + } + + super.init(); + + if (condition == null) { + LOG.debug("init: null policy condition! Will match always!"); + _allowAny = true; + } else if (CollectionUtils.isEmpty(condition.getValues())) { + LOG.debug("init: empty conditions collection on policy condition! Will match always!"); + _allowAny = true; + } else { + for (String value : condition.getValues()) { + if (StringUtils.isEmpty(value)) { + LOG.warn("init: Unexpected: one of the value in condition is null or empty!"); + } else { + int[] aDuration = extractDuration(value); + if (aDuration != null) { + _durations.add(aDuration); + } + } + } + } + + if (_durations.isEmpty()) { + LOG.debug("No valid durations found. Will always match!"); + _allowAny = true; + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerTimeOfDayMatcher.init(" + condition + "): durations[" + toString() + "]"); + } + } + + // match "9am-5pm", "9 Am - 5 PM", "9 am.- 5 P.M", "9:30 AM - 4:00p.m." etc. spaces around - and after digits are allowed and dots in am/pm string in mixed cases is allowed + static final Pattern _Pattern = Pattern.compile(" *(\\d{1,2})(:(\\d{1,2}))? *([aApP])\\.?[mM]\\.? *- *(\\d{1,2})(:(\\d{1,2}))? *([aApP])\\.?[mM]\\.? *"); + + int[] extractDuration(String value) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerTimeOfDayMatcher.extractDuration(" + value + ")"); + } + + int[] result = null; + if (value == null) { + LOG.warn("extractDuration: null input value!"); + } else { + Matcher m = _Pattern.matcher(value); + if (!m.matches()) { + LOG.warn("extractDuration: input[" + value + "] did not match pattern!"); + } else { + int startHour = Integer.parseInt(m.group(1)); + int startMin = 0; + if (m.group(3) != null) { + startMin = Integer.parseInt(m.group(3)); + } + String startType = m.group(4).toUpperCase(); + + int endHour = Integer.parseInt(m.group(5)); + int endMinute = 0; + if (m.group(7) != null) { + endMinute = Integer.parseInt(m.group(7)); + } + String endType = m.group(8).toUpperCase(); + if(startHour == 12) { + if("A".equals(startType)) { + startHour = 0; + } + } else if ("P".equals(startType)) { + startHour += 12; + } + + if(endHour == 12) { + if("A".equals(endType)) { + endHour = 0; + } + } else if ("P".equals(endType)) { + endHour += 12; + } + + result = new int[] { (startHour*60)+startMin, (endHour*60)+endMinute }; + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerTimeOfDayMatcher.extractDuration(" + value + "): duration:" + Arrays.toString(result)); + } + return result; + } + + @Override + public boolean isMatched(RangerAccessRequest request) { + + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerTimeOfDayMatcher.isMatched(" + request + ")"); + } + + boolean matched = true; + if (_allowAny) { + LOG.debug("isMatched: allowAny flag is true. Matched!"); + } else if (request == null) { + LOG.warn("isMatched: Unexpected: Request is null! Implicitly matched!"); + } else if (request.getAccessTime() == null) { + LOG.warn("isMatched: Unexpected: Accesstime on the request is null! Implicitly matched!"); + } else { + Date date = request.getAccessTime(); + Calendar calendar = GregorianCalendar.getInstance(); + calendar.setTime(date); + int hourOfDay = calendar.get(Calendar.HOUR_OF_DAY); + int minutes = calendar.get(Calendar.MINUTE); + if (! durationMatched(_durations, hourOfDay, minutes)) { + matched = false; + + if (LOG.isDebugEnabled()) { + LOG.debug("isMatched: None of the durations contains this hour of day[" + hourOfDay + "] and minutes[" + minutes + "]"); + } + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerTimeOfDayMatcher.isMatched(" + request+ "): " + matched); + } + + return matched; + } + + boolean durationMatched(List durations, int hourOfDay, int minutes) { + for (int[] aDuration : durations) { + int start = aDuration[0]; + int end = aDuration[1]; + int minutesOfDay = hourOfDay*60 + minutes; + if(start < end) { + if (start <= minutesOfDay && minutesOfDay <= end) { + return true; + } + } else { + if(start <= minutesOfDay || minutesOfDay <= end) { + return true; + } + } + } + return false; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + + sb.append("RangerTimeOfDayMatcher {"); + sb.append("_allowAny=").append(_allowAny).append(" "); + sb.append("_durations=["); + for(int[] duration : _durations) { + sb.append("{start=").append(duration[0]).append("; end=").append(duration[1]).append("} "); + } + sb.append("]"); + sb.append("}"); + + return sb.toString(); + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/contextenricher/RangerAbstractContextEnricher.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/contextenricher/RangerAbstractContextEnricher.java new file mode 100644 index 00000000000..3470ed1c7be --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/contextenricher/RangerAbstractContextEnricher.java @@ -0,0 +1,342 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.contextenricher; + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.atlas.authorization.hadoop.config.RangerPluginConfig; +import org.apache.atlas.plugin.model.RangerServiceDef; +import org.apache.atlas.plugin.model.RangerServiceDef.RangerContextEnricherDef; +import org.apache.atlas.plugin.policyengine.RangerAccessRequest; +import org.apache.atlas.plugin.policyengine.RangerPluginContext; +import org.apache.atlas.plugin.policyengine.RangerPolicyEngineOptions; +import org.apache.atlas.plugin.service.RangerAuthContext; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.InputStream; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.Map; +import java.util.Properties; + + +public abstract class RangerAbstractContextEnricher implements RangerContextEnricher { + private static final Log LOG = LogFactory.getLog(RangerAbstractContextEnricher.class); + + protected RangerContextEnricherDef enricherDef; + protected String serviceName; + protected String appId; + protected RangerServiceDef serviceDef; + private RangerPluginContext pluginContext; + protected RangerPolicyEngineOptions options = new RangerPolicyEngineOptions(); + + @Override + public void setEnricherDef(RangerContextEnricherDef enricherDef) { + this.enricherDef = enricherDef; + } + + @Override + public void setServiceName(String serviceName) { + this.serviceName = serviceName; + } + + @Override + public void setServiceDef(RangerServiceDef serviceDef) { + this.serviceDef = serviceDef; + } + + @Override + public void setAppId(String appId) { + this.appId = appId; + } + + @Override + public void init() { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerAbstractContextEnricher.init(" + enricherDef + ")"); + } + + RangerAuthContext authContext = getAuthContext(); + + if (authContext != null) { + authContext.addOrReplaceRequestContextEnricher(this, null); + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("authContext is null. This context-enricher is not added to authContext"); + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerAbstractContextEnricher.init(" + enricherDef + ")"); + } + } + + @Override + public void enrich(RangerAccessRequest request, Object dataStore) { + enrich(request); + } + + @Override + public boolean preCleanup() { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerAbstractContextEnricher.preCleanup(" + enricherDef + ")"); + } + + RangerAuthContext authContext = getAuthContext(); + + if (authContext != null) { + authContext.cleanupRequestContextEnricher(this); + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("authContext is null. AuthContext need not be cleaned."); + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerAbstractContextEnricher.preCleanup(" + enricherDef + ")"); + } + + return true; + } + + @Override + public void cleanup() { + preCleanup(); + } + + @Override + protected void finalize() throws Throwable { + try { + cleanup(); + } + finally { + super.finalize(); + } + } + + @Override + public String getName() { + return enricherDef == null ? null : enricherDef.getName(); + } + + public RangerContextEnricherDef getEnricherDef() { + return enricherDef; + } + + public String getServiceName() { + return serviceName; + } + + public RangerServiceDef getServiceDef() { + return serviceDef; + } + + public String getAppId() { + return appId; + } + + public String getOption(String name) { + String ret = null; + + Map options = enricherDef != null ? enricherDef.getEnricherOptions() : null; + + if(options != null && name != null) { + ret = options.get(name); + } + + return ret; + } + + public RangerAuthContext getAuthContext() { + RangerPluginContext pluginContext = this.pluginContext; + + return pluginContext != null ? pluginContext.getAuthContext() : null; + } + + final public void setPluginContext(RangerPluginContext pluginContext) { + this.pluginContext = pluginContext; + } + + public RangerPluginContext getPluginContext() { + return this.pluginContext; + } + + final public void setPolicyEngineOptions(RangerPolicyEngineOptions options) { + this.options = options; + } + + public RangerPluginConfig getPluginConfig() { + RangerPluginContext pluginContext = this.pluginContext; + + return pluginContext != null ? pluginContext.getConfig() : null; + } + + public RangerPolicyEngineOptions getPolicyEngineOptions() { + return options; + } + + public void notifyAuthContextChanged() { + RangerPluginContext pluginContext = this.pluginContext; + + if (pluginContext != null) { + pluginContext.notifyAuthContextChanged(); + } + } + + public String getConfig(String configName, String defaultValue) { + RangerPluginContext pluginContext = this.pluginContext; + String ret = defaultValue; + Configuration config = pluginContext != null ? pluginContext.getConfig() : null; + + if (config != null) { + ret = config.get(configName, defaultValue); + } + + return ret; + } + + public int getIntConfig(String configName, int defaultValue) { + RangerPluginContext pluginContext = this.pluginContext; + int ret = defaultValue; + Configuration config = pluginContext != null ? pluginContext.getConfig() : null; + + if (config != null) { + ret = config.getInt(configName, defaultValue); + } + + return ret; + } + + public boolean getBooleanConfig(String configName, boolean defaultValue) { + RangerPluginContext pluginContext = this.pluginContext; + boolean ret = defaultValue; + Configuration config = pluginContext != null ? pluginContext.getConfig() : null; + + if (config != null) { + ret = config.getBoolean(configName, defaultValue); + } + + return ret; + } + + public String getOption(String name, String defaultValue) { + String ret = defaultValue; + String val = getOption(name); + + if(val != null) { + ret = val; + } + + return ret; + } + + public boolean getBooleanOption(String name, boolean defaultValue) { + boolean ret = defaultValue; + String val = getOption(name); + + if(val != null) { + ret = Boolean.parseBoolean(val); + } + + return ret; + } + + public char getCharOption(String name, char defaultValue) { + char ret = defaultValue; + String val = getOption(name); + + if(! StringUtils.isEmpty(val)) { + ret = val.charAt(0); + } + + return ret; + } + + public long getLongOption(String name, long defaultValue) { + long ret = defaultValue; + String val = getOption(name); + + if(val != null) { + ret = Long.parseLong(val); + } + + return ret; + } + + public Properties readProperties(String fileName) { + Properties ret = null; + InputStream inStr = null; + URL fileURL = null; + File f = new File(fileName); + + if (f.exists() && f.isFile() && f.canRead()) { + try { + inStr = new FileInputStream(f); + fileURL = f.toURI().toURL(); + } catch (FileNotFoundException exception) { + LOG.error("Error processing input file:" + fileName + " or no privilege for reading file " + fileName, exception); + } catch (MalformedURLException malformedException) { + LOG.error("Error processing input file:" + fileName + " cannot be converted to URL " + fileName, malformedException); + } + } else { + fileURL = getClass().getResource(fileName); + + if (fileURL == null && !fileName.startsWith("/")) { + fileURL = getClass().getResource("/" + fileName); + } + + if (fileURL == null) { + fileURL = ClassLoader.getSystemClassLoader().getResource(fileName); + + if (fileURL == null && !fileName.startsWith("/")) { + fileURL = ClassLoader.getSystemClassLoader().getResource("/" + fileName); + } + } + } + + if (fileURL != null) { + try { + inStr = fileURL.openStream(); + + Properties prop = new Properties(); + + prop.load(inStr); + + ret = prop; + } catch (Exception excp) { + LOG.error("failed to load properties from file '" + fileName + "'", excp); + } finally { + if (inStr != null) { + try { + inStr.close(); + } catch (Exception excp) { + // ignore + } + } + } + } + + return ret; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/contextenricher/RangerAbstractGeolocationProvider.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/contextenricher/RangerAbstractGeolocationProvider.java new file mode 100644 index 00000000000..26c2595a853 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/contextenricher/RangerAbstractGeolocationProvider.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.contextenricher; + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.plugin.geo.RangerGeolocationData; +import org.apache.atlas.plugin.geo.RangerGeolocationDatabase; +import org.apache.atlas.plugin.policyengine.RangerAccessRequest; +import org.apache.atlas.plugin.store.GeolocationStore; + +import java.util.Map; + +public abstract class RangerAbstractGeolocationProvider extends RangerAbstractContextEnricher { + + private static final Log LOG = LogFactory.getLog(RangerAbstractGeolocationProvider.class); + + public static final String ENRICHER_OPTION_GEOLOCATION_META_PREFIX = "geolocation.meta.prefix"; + + public static final String KEY_CONTEXT_GEOLOCATION_PREFIX = "LOCATION_"; + private GeolocationStore store; + private String geoMetaPrefix; + + abstract public String getGeoSourceLoader(); + + @Override + public void init() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerAbstractGeolocationProvider.init()"); + } + + super.init(); + + geoMetaPrefix = getOption(ENRICHER_OPTION_GEOLOCATION_META_PREFIX); + if (geoMetaPrefix == null) { + geoMetaPrefix = ""; + } + + String geoSourceLoader = getGeoSourceLoader(); + + GeolocationStore geoStore = null; + Map context = enricherDef.getEnricherOptions(); + + if (context != null) { + try { + // Get the class definition and ensure it is of the correct type + @SuppressWarnings("unchecked") + Class geoSourceLoaderClass = (Class) Class.forName(geoSourceLoader); + // instantiate the loader class and initialize it with options + geoStore = geoSourceLoaderClass.newInstance(); + } + catch (ClassNotFoundException exception) { + LOG.error("RangerAbstractGeolocationProvider.init() - Class " + geoSourceLoader + " not found, exception=" + exception); + } + catch (ClassCastException exception) { + LOG.error("RangerAbstractGeolocationProvider.init() - Class " + geoSourceLoader + " is not a type of GeolocationStore, exception=" + exception); + } + catch (IllegalAccessException exception) { + LOG.error("RangerAbstractGeolocationProvider.init() - Class " + geoSourceLoader + " could not be instantiated, exception=" + exception); + } + catch (InstantiationException exception) { + LOG.error("RangerAbstractGeolocationProvider.init() - Class " + geoSourceLoader + " could not be instantiated, exception=" + exception); + } + + if (geoStore != null) { + try { + geoStore.init(context); + store = geoStore; + } catch (Exception exception) { + LOG.error("RangerAbstractGeolocationProvider.init() - geoLocation Store cannot be initialized, exception=" + exception); + } + } + } + + if (store == null) { + LOG.error("RangerAbstractGeolocationProvider.init() - is not initialized correctly."); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerAbstractGeolocationProvider.init()"); + } + } + + @Override + public void enrich(RangerAccessRequest request) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerAbstractGeolocationProvider.enrich(" + request + ")"); + } + + RangerGeolocationData geolocation = null; + + String clientIPAddress = request.getClientIPAddress(); + + if (LOG.isDebugEnabled()) { + LOG.debug("RangerAbstractGeolocationProvider.enrich() - clientIPAddress=" + clientIPAddress); + } + + if (StringUtils.isNotBlank(clientIPAddress) && store != null) { + geolocation = store.getGeoLocation(clientIPAddress); + + if (geolocation != null) { + if (LOG.isDebugEnabled()) { + LOG.debug("RangerAbstractGeolocationProvider.enrich() - Country=" + geolocation); + } + Map context = request.getContext(); + + String[] geoAttrValues = geolocation.getLocationData(); + + RangerGeolocationDatabase database = store.getGeoDatabase(); + String[] attributeNames = database.getMetadata().getLocationDataItemNames(); + + for (int i = 0; i < geoAttrValues.length && i < attributeNames.length; i++) { + String contextName = KEY_CONTEXT_GEOLOCATION_PREFIX + geoMetaPrefix + attributeNames[i]; + context.put(contextName, geoAttrValues[i]); + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("RangerAbstractGeolocationProvider.enrich() - clientIPAddress '" + clientIPAddress + "' not found."); + } + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("RangerAbstractGeolocationProvider.enrich() - clientIPAddress is null or blank, cannot get geolocation"); + } + } + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerAbstractGeolocationProvider.enrich(" + request + ")"); + } + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/contextenricher/RangerAdminTagRetriever.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/contextenricher/RangerAdminTagRetriever.java new file mode 100644 index 00000000000..55e909e3700 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/contextenricher/RangerAdminTagRetriever.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.contextenricher; + +import org.apache.atlas.authz.admin.client.AtlasAuthAdminClient; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.authorization.hadoop.config.RangerPluginConfig; +import org.apache.atlas.plugin.policyengine.RangerPluginContext; +import org.apache.atlas.plugin.util.ServiceTags; + +import java.util.Map; + +public class RangerAdminTagRetriever extends RangerTagRetriever { + private static final Log LOG = LogFactory.getLog(RangerAdminTagRetriever.class); + + private AtlasAuthAdminClient atlasAuthAdminClient; + + @Override + public void init(Map options) { + + if (StringUtils.isNotBlank(serviceName) && serviceDef != null && StringUtils.isNotBlank(appId)) { + RangerPluginConfig pluginConfig = super.pluginConfig; + + if (pluginConfig == null) { + pluginConfig = new RangerPluginConfig(serviceDef.getName(), serviceName, appId, null, null, null); + } + + RangerPluginContext pluginContext = getPluginContext(); + this.atlasAuthAdminClient = pluginContext.getAtlasAuthAdminClient(); + + } else { + LOG.error("FATAL: Cannot find service/serviceDef to use for retrieving tags. Will NOT be able to retrieve tags."); + } + } + + @Override + public ServiceTags retrieveTags(long lastKnownVersion, long lastActivationTimeInMillis) throws Exception { + + ServiceTags serviceTags = null; + + /*if (adminClient != null) { + try { + serviceTags = adminClient.getServiceTagsIfUpdated(lastKnownVersion, lastActivationTimeInMillis); + } catch (ClosedByInterruptException closedByInterruptException) { + LOG.error("Tag-retriever thread was interrupted while blocked on I/O"); + throw new InterruptedException(); + } catch (Exception e) { + LOG.error("Tag-retriever encounterd exception, exception=", e); + LOG.error("Returning null service tags"); + } + }*/ + return serviceTags; + } + +} + diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/contextenricher/RangerAdminUserStoreRetriever.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/contextenricher/RangerAdminUserStoreRetriever.java new file mode 100644 index 00000000000..237c9fc9605 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/contextenricher/RangerAdminUserStoreRetriever.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.contextenricher; + +import org.apache.atlas.authz.admin.client.AtlasAuthAdminClient; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.authorization.hadoop.config.RangerPluginConfig; +import org.apache.atlas.plugin.policyengine.RangerPluginContext; +import org.apache.atlas.plugin.util.RangerUserStore; + +import java.nio.channels.ClosedByInterruptException; +import java.util.Map; + +public class RangerAdminUserStoreRetriever extends RangerUserStoreRetriever { + private static final Log LOG = LogFactory.getLog(RangerAdminUserStoreRetriever.class); + + private AtlasAuthAdminClient atlasAuthAdminClient; + + @Override + public void init(Map options) { + + if (StringUtils.isNotBlank(serviceName) && serviceDef != null && StringUtils.isNotBlank(appId)) { + RangerPluginConfig pluginConfig = super.pluginConfig; + + if (pluginConfig == null) { + pluginConfig = new RangerPluginConfig(serviceDef.getName(), serviceName, appId, null, null, null); + } + + RangerPluginContext pluginContext = getPluginContext(); + this.atlasAuthAdminClient = pluginContext.getAtlasAuthAdminClient(); + + } else { + LOG.error("FATAL: Cannot find service/serviceDef to use for retrieving userstore. Will NOT be able to retrieve userstore."); + } + } + + @Override + public RangerUserStore retrieveUserStoreInfo(long lastKnownVersion, long lastActivationTimeInMillis) throws Exception { + + RangerUserStore rangerUserStore = null; + + if (atlasAuthAdminClient != null) { + try { + rangerUserStore = atlasAuthAdminClient.getUserStoreIfUpdated(lastActivationTimeInMillis); + } catch (ClosedByInterruptException closedByInterruptException) { + LOG.error("UserStore-retriever thread was interrupted while blocked on I/O"); + throw new InterruptedException(); + } catch (Exception e) { + LOG.error("UserStore-retriever encounterd exception, exception=", e); + LOG.error("Returning null userstore info"); + } + } + return rangerUserStore; + } + +} + diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/contextenricher/RangerContextEnricher.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/contextenricher/RangerContextEnricher.java new file mode 100644 index 00000000000..a427eb5e8bd --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/contextenricher/RangerContextEnricher.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.contextenricher; + + +import org.apache.atlas.plugin.model.RangerServiceDef; +import org.apache.atlas.plugin.model.RangerServiceDef.RangerContextEnricherDef; +import org.apache.atlas.plugin.policyengine.RangerAccessRequest; + +public interface RangerContextEnricher { + void setEnricherDef(RangerContextEnricherDef enricherDef); + + void setServiceName(String serviceName); + + void setServiceDef(RangerServiceDef serviceDef); + + void setAppId(String appId); + + void init(); + + void enrich(RangerAccessRequest request); + + void enrich(RangerAccessRequest request, Object dataStore); + + boolean preCleanup(); + + void cleanup(); + + String getName(); + +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/contextenricher/RangerFileBasedGeolocationProvider.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/contextenricher/RangerFileBasedGeolocationProvider.java new file mode 100644 index 00000000000..c71119731b9 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/contextenricher/RangerFileBasedGeolocationProvider.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.contextenricher; + +public class RangerFileBasedGeolocationProvider extends RangerAbstractGeolocationProvider { + + public static final String GEOLOCATION_SOURCE_LOADER_FILELOADER = "org.apache.atlas.plugin.store.file.GeolocationFileStore"; + + @Override + public String getGeoSourceLoader() { + return GEOLOCATION_SOURCE_LOADER_FILELOADER; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/contextenricher/RangerFileBasedTagRetriever.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/contextenricher/RangerFileBasedTagRetriever.java new file mode 100644 index 00000000000..79a4ea956bb --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/contextenricher/RangerFileBasedTagRetriever.java @@ -0,0 +1,164 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.contextenricher; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.plugin.util.ServiceTags; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.Reader; +import java.net.MalformedURLException; +import java.net.URL; +import java.nio.charset.Charset; +import java.util.Map; + +public class RangerFileBasedTagRetriever extends RangerTagRetriever { + private static final Log LOG = LogFactory.getLog(RangerFileBasedTagRetriever.class); + + + private URL serviceTagsFileURL; + private String serviceTagsFileName; + private Gson gsonBuilder; + + @Override + public void init(Map options) { + + if (LOG.isDebugEnabled()) { + LOG.debug("==> init()" ); + } + + gsonBuilder = new GsonBuilder().setDateFormat("yyyyMMdd-HH:mm:ss.SSS-Z") + .setPrettyPrinting() + .create(); + + String serviceTagsFileNameProperty = "serviceTagsFileName"; + String serviceTagsDefaultFileName = "/testdata/test_servicetags_hive.json"; + + if (StringUtils.isNotBlank(serviceName) && serviceDef != null && StringUtils.isNotBlank(appId)) { + InputStream serviceTagsFileStream = null; + + + // Open specified file from options- it should contain service-tags + + serviceTagsFileName = options != null? options.get(serviceTagsFileNameProperty) : null; + + serviceTagsFileName = serviceTagsFileName == null ? serviceTagsDefaultFileName : serviceTagsFileName; + + File f = new File(serviceTagsFileName); + + if (f.exists() && f.isFile() && f.canRead()) { + try { + serviceTagsFileStream = new FileInputStream(f); + serviceTagsFileURL = f.toURI().toURL(); + } catch (FileNotFoundException exception) { + LOG.error("Error processing input file:" + serviceTagsFileName + " or no privilege for reading file " + serviceTagsFileName, exception); + } catch (MalformedURLException malformedException) { + LOG.error("Error processing input file:" + serviceTagsFileName + " cannot be converted to URL " + serviceTagsFileName, malformedException); + } + } else { + URL fileURL = getClass().getResource(serviceTagsFileName); + if (fileURL == null && !serviceTagsFileName.startsWith("/")) { + fileURL = getClass().getResource("/" + serviceTagsFileName); + } + + if (fileURL == null) { + fileURL = ClassLoader.getSystemClassLoader().getResource(serviceTagsFileName); + if (fileURL == null && !serviceTagsFileName.startsWith("/")) { + fileURL = ClassLoader.getSystemClassLoader().getResource("/" + serviceTagsFileName); + } + } + + if (fileURL != null) { + try { + serviceTagsFileStream = fileURL.openStream(); + serviceTagsFileURL = fileURL; + } catch (Exception exception) { + LOG.error(serviceTagsFileName + " is not a file", exception); + } + } else { + LOG.warn("Error processing input file: URL not found for " + serviceTagsFileName + " or no privilege for reading file " + serviceTagsFileName); + } + } + + if (serviceTagsFileStream != null) { + try { + serviceTagsFileStream.close(); + } catch (Exception e) { + // Ignore + } + } + + } else { + LOG.error("FATAL: Cannot find service/serviceDef/serviceTagsFile to use for retrieving tags. Will NOT be able to retrieve tags."); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== init() : serviceTagsFileName=" + serviceTagsFileName); + } + } + + @Override + public ServiceTags retrieveTags(long lastKnownVersion, long lastActivationTimeInMillis) throws Exception { + + if (LOG.isDebugEnabled()) { + LOG.debug("==> retrieveTags(lastKnownVersion=" + lastKnownVersion + ", lastActivationTimeInMillis=" + lastActivationTimeInMillis + ", serviceTagsFilePath=" + serviceTagsFileName); + } + + ServiceTags serviceTags = null; + + if (serviceTagsFileURL != null) { + try ( + InputStream serviceTagsFileStream = serviceTagsFileURL.openStream(); + Reader reader = new InputStreamReader(serviceTagsFileStream, Charset.forName("UTF-8")) + ) { + + serviceTags = gsonBuilder.fromJson(reader, ServiceTags.class); + + if (serviceTags.getTagVersion() <= lastKnownVersion) { + // No change in serviceTags + serviceTags = null; + } + } catch (IOException e) { + LOG.warn("Error processing input file: or no privilege for reading file " + serviceTagsFileName); + throw e; + } + } else { + LOG.error("Error reading file: " + serviceTagsFileName); + throw new Exception("serviceTagsFileURL is null!"); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== retrieveTags(lastKnownVersion=" + lastKnownVersion + ", lastActivationTimeInMillis=" + lastActivationTimeInMillis); + } + + return serviceTags; + } + +} + diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/contextenricher/RangerServiceResourceMatcher.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/contextenricher/RangerServiceResourceMatcher.java new file mode 100644 index 00000000000..c7fb07bc6a2 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/contextenricher/RangerServiceResourceMatcher.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.contextenricher; + +import org.apache.atlas.plugin.model.RangerPolicy; +import org.apache.atlas.plugin.model.RangerServiceDef; +import org.apache.atlas.plugin.model.RangerServiceResource; +import org.apache.atlas.plugin.policyengine.RangerAccessResource; +import org.apache.atlas.plugin.policyresourcematcher.RangerPolicyResourceEvaluator; +import org.apache.atlas.plugin.policyresourcematcher.RangerPolicyResourceMatcher; +import org.apache.atlas.plugin.resourcematcher.RangerResourceMatcher; +import org.apache.atlas.plugin.util.ServiceDefUtil; + +import java.io.Serializable; +import java.util.Comparator; +import java.util.Map; + +public class RangerServiceResourceMatcher implements RangerPolicyResourceEvaluator { + public static final Comparator ID_COMPARATOR = new IdComparator(); + + private final RangerServiceResource serviceResource; + private final RangerPolicyResourceMatcher policyResourceMatcher; + private RangerServiceDef.RangerResourceDef leafResourceDef; + + public RangerServiceResourceMatcher(final RangerServiceResource serviceResource, RangerPolicyResourceMatcher policyResourceMatcher) { + this.serviceResource = serviceResource; + this.policyResourceMatcher = policyResourceMatcher; + this.leafResourceDef = ServiceDefUtil.getLeafResourceDef(policyResourceMatcher.getServiceDef(), getPolicyResource()); + } + + public RangerServiceResource getServiceResource() { return serviceResource; } + + @Override + public long getId() { + return serviceResource != null ? serviceResource.getId() :-1; + } + + @Override + public String getGuid() { + return serviceResource != null ? serviceResource.getGuid() : "-1"; + } + + @Override + public RangerPolicyResourceMatcher getPolicyResourceMatcher() { return policyResourceMatcher; } + + @Override + public Map getPolicyResource() { + return serviceResource != null ? serviceResource.getResourceElements() : null; + } + + @Override + public RangerResourceMatcher getResourceMatcher(String resourceName) { + return policyResourceMatcher != null ? policyResourceMatcher.getResourceMatcher(resourceName) : null; + } + + @Override + public boolean isAncestorOf(RangerServiceDef.RangerResourceDef resourceDef) { + return ServiceDefUtil.isAncestorOf(policyResourceMatcher.getServiceDef(), leafResourceDef, resourceDef); + } + + public RangerPolicyResourceMatcher.MatchType getMatchType(RangerAccessResource requestedResource, Map evalContext) { + return policyResourceMatcher != null ? policyResourceMatcher.getMatchType(requestedResource, evalContext) : RangerPolicyResourceMatcher.MatchType.NONE; + } + + static class IdComparator implements Comparator, Serializable { + @Override + public int compare(RangerServiceResourceMatcher me, RangerServiceResourceMatcher other) { + return Long.compare(me.getId(), other.getId()); + } + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/contextenricher/RangerTagEnricher.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/contextenricher/RangerTagEnricher.java new file mode 100644 index 00000000000..d5e7e9c9173 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/contextenricher/RangerTagEnricher.java @@ -0,0 +1,1191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.contextenricher; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.authorization.hadoop.config.RangerPluginConfig; +import org.apache.atlas.plugin.model.RangerPolicy; +import org.apache.atlas.plugin.model.RangerServiceDef; +import org.apache.atlas.plugin.model.RangerServiceResource; +import org.apache.atlas.plugin.model.RangerTag; +import org.apache.atlas.plugin.model.validation.RangerServiceDefHelper; +import org.apache.atlas.plugin.policyengine.RangerAccessRequest; +import org.apache.atlas.plugin.policyengine.RangerAccessRequestImpl; +import org.apache.atlas.plugin.policyengine.RangerAccessResource; +import org.apache.atlas.plugin.policyengine.RangerAccessResourceImpl; +import org.apache.atlas.plugin.policyengine.RangerResourceTrie; +import org.apache.atlas.plugin.policyresourcematcher.RangerDefaultPolicyResourceMatcher; +import org.apache.atlas.plugin.policyresourcematcher.RangerPolicyResourceMatcher; +import org.apache.atlas.plugin.service.RangerAuthContext; +import org.apache.atlas.plugin.util.DownloadTrigger; +import org.apache.atlas.plugin.util.DownloaderTask; +import org.apache.atlas.plugin.util.RangerAccessRequestUtil; +import org.apache.atlas.plugin.util.RangerCommonConstants; +import org.apache.atlas.plugin.util.RangerPerfTracer; +import org.apache.atlas.plugin.util.RangerReadWriteLock; +import org.apache.atlas.plugin.util.RangerServiceNotFoundException; +import org.apache.atlas.plugin.util.RangerServiceTagsDeltaUtil; +import org.apache.atlas.plugin.util.ServiceTags; + +import java.io.File; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.Reader; +import java.io.Writer; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.Timer; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; + +public class RangerTagEnricher extends RangerAbstractContextEnricher { + private static final Log LOG = LogFactory.getLog(RangerTagEnricher.class); + + private static final Log PERF_CONTEXTENRICHER_INIT_LOG = RangerPerfTracer.getPerfLogger("contextenricher.init"); + private static final Log PERF_TRIE_OP_LOG = RangerPerfTracer.getPerfLogger("resourcetrie.retrieval"); + private static final Log PERF_SET_SERVICETAGS_LOG = RangerPerfTracer.getPerfLogger("tagenricher.setservicetags"); + + + private static final String TAG_REFRESHER_POLLINGINTERVAL_OPTION = "tagRefresherPollingInterval"; + public static final String TAG_RETRIEVER_CLASSNAME_OPTION = "tagRetrieverClassName"; + private static final String TAG_DISABLE_TRIE_PREFILTER_OPTION = "disableTrieLookupPrefilter"; + + private RangerTagRefresher tagRefresher; + private RangerTagRetriever tagRetriever; + private boolean disableTrieLookupPrefilter; + private EnrichedServiceTags enrichedServiceTags; + private boolean disableCacheIfServiceNotFound = true; + + private final BlockingQueue tagDownloadQueue = new LinkedBlockingQueue<>(); + private Timer tagDownloadTimer; + + private RangerServiceDefHelper serviceDefHelper; + private RangerReadWriteLock lock = new RangerReadWriteLock(false); + + @Override + public void init() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerTagEnricher.init()"); + } + + super.init(); + + String tagRetrieverClassName = getOption(TAG_RETRIEVER_CLASSNAME_OPTION); + + long pollingIntervalMs = getLongOption(TAG_REFRESHER_POLLINGINTERVAL_OPTION, 60 * 1000); + + disableTrieLookupPrefilter = getBooleanOption(TAG_DISABLE_TRIE_PREFILTER_OPTION, false); + + serviceDefHelper = new RangerServiceDefHelper(serviceDef, false); + + if (StringUtils.isNotBlank(tagRetrieverClassName)) { + + try { + @SuppressWarnings("unchecked") + Class tagRetriverClass = (Class) Class.forName(tagRetrieverClassName); + + tagRetriever = tagRetriverClass.newInstance(); + + } catch (ClassNotFoundException exception) { + LOG.error("Class " + tagRetrieverClassName + " not found, exception=" + exception); + } catch (ClassCastException exception) { + LOG.error("Class " + tagRetrieverClassName + " is not a type of RangerTagRetriever, exception=" + exception); + } catch (IllegalAccessException exception) { + LOG.error("Class " + tagRetrieverClassName + " illegally accessed, exception=" + exception); + } catch (InstantiationException exception) { + LOG.error("Class " + tagRetrieverClassName + " could not be instantiated, exception=" + exception); + } + + if (tagRetriever != null) { + String propertyPrefix = "ranger.plugin." + serviceDef.getName(); + disableCacheIfServiceNotFound = getBooleanConfig(propertyPrefix + ".disable.cache.if.servicenotfound", true); + String cacheDir = getConfig(propertyPrefix + ".policy.cache.dir", null); + String cacheFilename = String.format("%s_%s_tag.json", appId, serviceName); + + cacheFilename = cacheFilename.replace(File.separatorChar, '_'); + cacheFilename = cacheFilename.replace(File.pathSeparatorChar, '_'); + + String cacheFile = cacheDir == null ? null : (cacheDir + File.separator + cacheFilename); + + createLock(); + + tagRetriever.setServiceName(serviceName); + tagRetriever.setServiceDef(serviceDef); + tagRetriever.setAppId(appId); + tagRetriever.setPluginConfig(getPluginConfig()); + tagRetriever.setPluginContext(getPluginContext()); + tagRetriever.init(enricherDef.getEnricherOptions()); + + tagRefresher = new RangerTagRefresher(tagRetriever, this, -1L, tagDownloadQueue, cacheFile); + LOG.info("Created RangerTagRefresher Thread(" + tagRefresher.getName() + ")"); + + try { + tagRefresher.populateTags(); + } catch (Throwable exception) { + LOG.error("Exception when retrieving tag for the first time for this enricher", exception); + } + tagRefresher.setDaemon(true); + tagRefresher.startRefresher(); + + tagDownloadTimer = new Timer("policyDownloadTimer", true); + + try { + tagDownloadTimer.schedule(new DownloaderTask(tagDownloadQueue), pollingIntervalMs, pollingIntervalMs); + if (LOG.isDebugEnabled()) { + LOG.debug("Scheduled tagDownloadRefresher to download tags every " + pollingIntervalMs + " milliseconds"); + } + } catch (IllegalStateException exception) { + LOG.error("Error scheduling tagDownloadTimer:", exception); + LOG.error("*** Tags will NOT be downloaded every " + pollingIntervalMs + " milliseconds ***"); + tagDownloadTimer = null; + } + } + } else { + LOG.warn("No value specified for " + TAG_RETRIEVER_CLASSNAME_OPTION + " in the RangerTagEnricher options"); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerTagEnricher.init()"); + } + } + + @Override + public void enrich(RangerAccessRequest request) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerTagEnricher.enrich(" + request + ")"); + } + + enrich(request, null); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerTagEnricher.enrich(" + request + ")"); + } + } + + @Override + public void enrich(RangerAccessRequest request, Object dataStore) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerTagEnricher.enrich(" + request + ") with dataStore:[" + dataStore + "]"); + } + + Set atlasClassificationSet = getMatchingAtlasClassificationsToAsset(request); + + final Set matchedTags; + + try (RangerReadWriteLock.RangerLock readLock = this.lock.getReadLock()) { + + if (LOG.isDebugEnabled()) { + if (readLock.isLockingEnabled()) { + LOG.debug("Acquired lock - " + readLock); + } + } + + final EnrichedServiceTags enrichedServiceTags; + + if (dataStore instanceof EnrichedServiceTags) { + enrichedServiceTags = (EnrichedServiceTags) dataStore; + } else { + enrichedServiceTags = this.enrichedServiceTags; + + if (dataStore != null) { + LOG.warn("Incorrect type of dataStore :[" + dataStore.getClass().getName() + "], falling back to original enrich"); + } + } + + if(atlasClassificationSet!=null && atlasClassificationSet.size() > 0) { + matchedTags = atlasClassificationSet; + } else { + matchedTags = enrichedServiceTags == null ? null : findMatchingTags(request, enrichedServiceTags); + } + + RangerAccessRequestUtil.setRequestTagsInContext(request.getContext(), matchedTags); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerTagEnricher.enrich(" + request + ") with dataStore:[" + dataStore + "]): tags count=" + (matchedTags == null ? 0 : matchedTags.size())); + } + } + + private Set getMatchingAtlasClassificationsToAsset(RangerAccessRequest request) { + + Map requestContext = request.getContext(); + + Set atlasClassificationSet = new HashSet<>(); + + if (requestContext != null && requestContext.get("CLASSIFICATIONS") != null) { + LOG.info("Classification found " + requestContext.get("CLASSIFICATIONS")); + atlasClassificationSet = (Set) requestContext.get("CLASSIFICATIONS"); + } + return atlasClassificationSet; + } + + /* + * This class implements a cache of result of look-up of keyset of policy-resources for each of the collections of hierarchies + * for policy types: access, datamask and rowfilter. If a keyset is examined for validity in a hierarchy of a policy-type, + * then that record is maintained in this cache for later look-up. + * + * The basic idea is that with a large number of tagged service-resources, this cache will speed up performance as well as put + * a cap on the upper bound because it is expected that the cardinality of set of all possible keysets for all resource-def + * combinations in a service-def will be much smaller than the number of service-resources. + */ + + static public class ResourceHierarchies { + private final Map, Boolean> accessHierarchies = new HashMap<>(); + private final Map, Boolean> dataMaskHierarchies = new HashMap<>(); + private final Map, Boolean> rowFilterHierarchies = new HashMap<>(); + + Boolean isValidHierarchy(String policyType, Collection resourceKeys) { + switch (policyType) { + case RangerPolicy.POLICY_TYPE_ACCESS: + return accessHierarchies.get(resourceKeys); + case RangerPolicy.POLICY_TYPE_DATAMASK: + return dataMaskHierarchies.get(resourceKeys); + case RangerPolicy.POLICY_TYPE_ROWFILTER: + return rowFilterHierarchies.get(resourceKeys); + default: + return null; + } + } + + void addHierarchy(String policyType, Collection resourceKeys, Boolean isValid) { + switch (policyType) { + case RangerPolicy.POLICY_TYPE_ACCESS: + accessHierarchies.put(resourceKeys, isValid); + break; + case RangerPolicy.POLICY_TYPE_DATAMASK: + dataMaskHierarchies.put(resourceKeys, isValid); + break; + case RangerPolicy.POLICY_TYPE_ROWFILTER: + rowFilterHierarchies.put(resourceKeys, isValid); + break; + default: + LOG.error("unknown policy-type " + policyType); + break; + } + } + } + + public void setServiceTags(final ServiceTags serviceTags) { + boolean rebuildOnlyIndex = false; + setServiceTags(serviceTags, rebuildOnlyIndex); + } + + protected void setServiceTags(final ServiceTags serviceTags, final boolean rebuildOnlyIndex) { + + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerTagEnricher.setServiceTags(serviceTags=" + serviceTags + ", rebuildOnlyIndex=" + rebuildOnlyIndex + ")"); + } + + try (RangerReadWriteLock.RangerLock writeLock = this.lock.getWriteLock()) { + + if (LOG.isDebugEnabled()) { + if (writeLock.isLockingEnabled()) { + LOG.debug("Acquired lock - " + writeLock); + } + } + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_SET_SERVICETAGS_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_SET_SERVICETAGS_LOG, "RangerTagEnricher.setServiceTags(newTagVersion=" + serviceTags.getTagVersion() + ",isDelta=" + serviceTags.getIsDelta() + ")"); + } + + if (serviceTags == null) { + LOG.info("ServiceTags is null for service " + serviceName); + enrichedServiceTags = null; + } else { + if (!serviceTags.getIsDelta()) { + processServiceTags(serviceTags); + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Received service-tag deltas:" + serviceTags); + } + ServiceTags oldServiceTags = enrichedServiceTags != null ? enrichedServiceTags.getServiceTags() : new ServiceTags(); + ServiceTags allServiceTags = rebuildOnlyIndex ? oldServiceTags : RangerServiceTagsDeltaUtil.applyDelta(oldServiceTags, serviceTags); + + if (serviceTags.getTagsChangeExtent() == ServiceTags.TagsChangeExtent.NONE) { + if (LOG.isDebugEnabled()) { + LOG.debug("No change to service-tags other than version change"); + } + } else { + if (serviceTags.getTagsChangeExtent() != ServiceTags.TagsChangeExtent.TAGS) { + Map> trieMap; + + if (enrichedServiceTags == null) { + trieMap = new HashMap<>(); + } else { + trieMap = writeLock.isLockingEnabled() ? enrichedServiceTags.getServiceResourceTrie() : copyServiceResourceTrie(); + } + + processServiceTagDeltas(serviceTags, allServiceTags, trieMap); + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Delta contains only tag attribute changes"); + } + List resourceMatchers = enrichedServiceTags != null ? enrichedServiceTags.getServiceResourceMatchers() : new ArrayList<>(); + Map> serviceResourceTrie = enrichedServiceTags != null ? enrichedServiceTags.getServiceResourceTrie() : new HashMap<>(); + enrichedServiceTags = new EnrichedServiceTags(allServiceTags, resourceMatchers, serviceResourceTrie); + } + } + } + + } + setEnrichedServiceTagsInPlugin(); + + RangerPerfTracer.logAlways(perf); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerTagEnricher.setServiceTags(serviceTags=" + serviceTags + ", rebuildOnlyIndex=" + rebuildOnlyIndex + ")"); + } + + } + + public Long getServiceTagsVersion() { + EnrichedServiceTags localEnrichedServiceTags = enrichedServiceTags; + return localEnrichedServiceTags != null ? localEnrichedServiceTags.getServiceTags().getTagVersion() : -1L; + } + + protected Long getResourceTrieVersion() { + EnrichedServiceTags localEnrichedServiceTags = enrichedServiceTags; + return localEnrichedServiceTags != null ? localEnrichedServiceTags.getResourceTrieVersion() : -1L; + } + + @Override + public boolean preCleanup() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerTagEnricher.preCleanup()"); + } + + super.preCleanup(); + + Timer tagDownloadTimer = this.tagDownloadTimer; + this.tagDownloadTimer = null; + + if (tagDownloadTimer != null) { + tagDownloadTimer.cancel(); + } + + RangerTagRefresher tagRefresher = this.tagRefresher; + this.tagRefresher = null; + + if (tagRefresher != null) { + tagRefresher.cleanup(); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerTagEnricher.preCleanup() : result=" + true); + } + return true; + } + + public void syncTagsWithAdmin(final DownloadTrigger token) throws InterruptedException { + tagDownloadQueue.put(token); + token.waitForCompletion(); + } + + public EnrichedServiceTags getEnrichedServiceTags() { + return enrichedServiceTags; + } + + protected RangerReadWriteLock createLock() { + String propertyPrefix = "ranger.plugin." + serviceDef.getName(); + RangerPluginConfig config = getPluginConfig(); + boolean deltasEnabled = config != null && config.getBoolean(propertyPrefix + RangerCommonConstants.PLUGIN_CONFIG_SUFFIX_TAG_DELTA, RangerCommonConstants.PLUGIN_CONFIG_SUFFIX_TAG_DELTA_DEFAULT); + boolean inPlaceUpdatesEnabled = config != null && config.getBoolean(propertyPrefix + RangerCommonConstants.PLUGIN_CONFIG_SUFFIX_IN_PLACE_TAG_UPDATES, RangerCommonConstants.PLUGIN_CONFIG_SUFFIX_IN_PLACE_TAG_UPDATES_DEFAULT); + boolean useReadWriteLock = deltasEnabled && inPlaceUpdatesEnabled; + + LOG.info("Policy-Engine will" + (useReadWriteLock ? " " : " not ") + "use read-write locking to update tags in place when tag-deltas are provided"); + + return new RangerReadWriteLock(useReadWriteLock); + + } + private void processServiceTags(ServiceTags serviceTags) { + if (LOG.isDebugEnabled()) { + LOG.debug("Processing all service-tags"); + } + + boolean isInError = false; + + if (CollectionUtils.isEmpty(serviceTags.getServiceResources())) { + LOG.info("There are no tagged resources for service " + serviceName); + enrichedServiceTags = null; + } else { + + ResourceHierarchies hierarchies = new ResourceHierarchies(); + + List resourceMatchers = new ArrayList<>(); + List serviceResources = serviceTags.getServiceResources(); + + for (RangerServiceResource serviceResource : serviceResources) { + RangerServiceResourceMatcher serviceResourceMatcher = createRangerServiceResourceMatcher(serviceResource, serviceDefHelper, hierarchies); + if (serviceResourceMatcher != null) { + resourceMatchers.add(serviceResourceMatcher); + } else { + LOG.error("Could not create service-resource-matcher for service-resource:[" + serviceResource + "]"); + isInError = true; + break; + } + } + + if (isInError) { + serviceTags.setTagVersion(-1L); + LOG.error("Error in processing tag-deltas. Will continue to use old tags"); + } else { + Map> serviceResourceTrie = null; + + if (!disableTrieLookupPrefilter) { + serviceResourceTrie = new HashMap<>(); + + for (RangerServiceDef.RangerResourceDef resourceDef : serviceDef.getResources()) { + serviceResourceTrie.put(resourceDef.getName(), new RangerResourceTrie(resourceDef, resourceMatchers, getPolicyEngineOptions().optimizeTrieForRetrieval, null)); + } + } + enrichedServiceTags = new EnrichedServiceTags(serviceTags, resourceMatchers, serviceResourceTrie); + } + } + } + + private void processServiceTagDeltas(ServiceTags deltas, ServiceTags allServiceTags, Map> serviceResourceTrie) { + if (LOG.isDebugEnabled()) { + LOG.debug("Delta contains changes other than tag attribute changes, [" + deltas.getTagsChangeExtent() + "]"); + } + + boolean isInError = false; + + ResourceHierarchies hierarchies = new ResourceHierarchies(); + List resourceMatchers = new ArrayList<>(); + + if (enrichedServiceTags != null) { + resourceMatchers.addAll(enrichedServiceTags.getServiceResourceMatchers()); + } + + List changedServiceResources = deltas.getServiceResources(); + + for (RangerServiceResource serviceResource : changedServiceResources) { + + final boolean removedOldServiceResource = MapUtils.isEmpty(serviceResource.getResourceElements()) || removeOldServiceResource(serviceResource, resourceMatchers, serviceResourceTrie); + if (removedOldServiceResource) { + + if (!StringUtils.isEmpty(serviceResource.getResourceSignature())) { + + RangerServiceResourceMatcher resourceMatcher = createRangerServiceResourceMatcher(serviceResource, serviceDefHelper, hierarchies); + + if (resourceMatcher != null) { + for (RangerServiceDef.RangerResourceDef resourceDef : serviceDef.getResources()) { + + RangerResourceTrie trie = serviceResourceTrie.get(resourceDef.getName()); + + if (trie != null) { + trie.add(serviceResource.getResourceElements().get(resourceDef.getName()), resourceMatcher); + if (LOG.isDebugEnabled()) { + LOG.debug("Added resource-matcher for service-resource:[" + serviceResource + "]"); + } + } else { + trie = new RangerResourceTrie<>(resourceDef, Collections.singletonList(resourceMatcher)); + serviceResourceTrie.put(resourceDef.getName(), trie); + } + } + resourceMatchers.add(resourceMatcher); + } else { + LOG.error("Could not create resource-matcher for resource: [" + serviceResource + "]. Should NOT happen!!"); + LOG.error("Setting tagVersion to -1 to ensure that in the next download all tags are downloaded"); + isInError = true; + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Service-resource:[id=" + serviceResource.getId() + "] is deleted as its resource-signature is empty. No need to create it!"); + } + } + } else { + isInError = true; + } + if (isInError) { + break; + } + } + if (isInError) { + LOG.error("Error in processing tag-deltas. Will continue to use old tags"); + deltas.setTagVersion(-1L); + } else { + for (Map.Entry> entry : serviceResourceTrie.entrySet()) { + entry.getValue().wrapUpUpdate(); + } + enrichedServiceTags = new EnrichedServiceTags(allServiceTags, resourceMatchers, serviceResourceTrie); + } + + } + + private boolean removeOldServiceResource(RangerServiceResource serviceResource, List resourceMatchers, Map> resourceTries) { + boolean ret = true; + + if (enrichedServiceTags != null) { + + if (LOG.isDebugEnabled()) { + LOG.debug("Removing service-resource:[" + serviceResource + "] from trie-map"); + } + + // Remove existing serviceResource from the copy + + RangerAccessResourceImpl accessResource = new RangerAccessResourceImpl(); + + for (Map.Entry entry : serviceResource.getResourceElements().entrySet()) { + accessResource.setValue(entry.getKey(), entry.getValue()); + } + if (LOG.isDebugEnabled()) { + LOG.debug("RangerAccessResource:[" + accessResource + "] created to represent service-resource[" + serviceResource + "] to find evaluators from trie-map"); + } + + RangerAccessRequestImpl request = new RangerAccessRequestImpl(); + request.setResource(accessResource); + + List oldMatchers = getEvaluators(request, enrichedServiceTags); + + if (LOG.isDebugEnabled()) { + LOG.debug("Found [" + oldMatchers.size() + "] matchers for service-resource[" + serviceResource + "]"); + } + + for (RangerServiceResourceMatcher matcher : oldMatchers) { + + for (RangerServiceDef.RangerResourceDef resourceDef : serviceDef.getResources()) { + String resourceDefName = resourceDef.getName(); + + RangerResourceTrie trie = resourceTries.get(resourceDefName); + + if (trie != null) { + trie.delete(serviceResource.getResourceElements().get(resourceDefName), matcher); + } else { + LOG.error("Cannot find resourceDef with name:[" + resourceDefName + "]. Should NOT happen!!"); + LOG.error("Setting tagVersion to -1 to ensure that in the next download all tags are downloaded"); + ret = false; + break; + } + } + } + + // Remove old resource matchers + if (ret) { + resourceMatchers.removeAll(oldMatchers); + + if (LOG.isDebugEnabled()) { + LOG.debug("Found and removed [" + oldMatchers.size() + "] matchers for service-resource[" + serviceResource + "] from trie-map"); + } + } + } + return ret; + } + + static public RangerServiceResourceMatcher createRangerServiceResourceMatcher(RangerServiceResource serviceResource, RangerServiceDefHelper serviceDefHelper, ResourceHierarchies hierarchies) { + + if (LOG.isDebugEnabled()) { + LOG.debug("==> createRangerServiceResourceMatcher(serviceResource=" + serviceResource + ")"); + } + + RangerServiceResourceMatcher ret = null; + + final Collection resourceKeys = serviceResource.getResourceElements().keySet(); + + for (String policyType : RangerPolicy.POLICY_TYPES) { + Boolean isValidHierarchy = hierarchies.isValidHierarchy(policyType, resourceKeys); + if (isValidHierarchy == null) { // hierarchy not yet validated + isValidHierarchy = Boolean.FALSE; + + for (List hierarchy : serviceDefHelper.getResourceHierarchies(policyType)) { + if (serviceDefHelper.hierarchyHasAllResources(hierarchy, resourceKeys)) { + isValidHierarchy = Boolean.TRUE; + + break; + } + } + + hierarchies.addHierarchy(policyType, resourceKeys, isValidHierarchy); + } + + if (isValidHierarchy) { + RangerDefaultPolicyResourceMatcher matcher = new RangerDefaultPolicyResourceMatcher(); + + matcher.setServiceDef(serviceDefHelper.getServiceDef()); + matcher.setPolicyResources(serviceResource.getResourceElements(), policyType); + + if (LOG.isDebugEnabled()) { + LOG.debug("RangerTagEnricher.setServiceTags() - Initializing matcher with (resource=" + serviceResource + + ", serviceDef=" + serviceDefHelper.getServiceDef() + ")"); + + } + matcher.setServiceDefHelper(serviceDefHelper); + matcher.init(); + + ret = new RangerServiceResourceMatcher(serviceResource, matcher); + break; + } + } + if (LOG.isDebugEnabled()) { + LOG.debug("<== createRangerServiceResourceMatcher(serviceResource=" + serviceResource + ") : [" + ret + "]"); + } + return ret; + + } + + private void setEnrichedServiceTagsInPlugin() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> setEnrichedServiceTagsInPlugin()"); + } + + RangerAuthContext authContext = getAuthContext(); + + if (authContext != null) { + authContext.addOrReplaceRequestContextEnricher(this, enrichedServiceTags); + + notifyAuthContextChanged(); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== setEnrichedServiceTagsInPlugin()"); + } + } + + private Set findMatchingTags(final RangerAccessRequest request, EnrichedServiceTags dataStore) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerTagEnricher.findMatchingTags(" + request + ")"); + } + + // To minimize chance for race condition between Tag-Refresher thread and access-evaluation thread + final EnrichedServiceTags enrichedServiceTags = dataStore != null ? dataStore : this.enrichedServiceTags; + + Set ret = null; + + RangerAccessResource resource = request.getResource(); + + if ((resource == null || resource.getKeys() == null || resource.getKeys().isEmpty()) && request.isAccessTypeAny()) { + ret = enrichedServiceTags.getTagsForEmptyResourceAndAnyAccess(); + } else { + + final List serviceResourceMatchers = getEvaluators(request, enrichedServiceTags); + + if (CollectionUtils.isNotEmpty(serviceResourceMatchers)) { + + for (RangerServiceResourceMatcher resourceMatcher : serviceResourceMatchers) { + + final RangerPolicyResourceMatcher.MatchType matchType = resourceMatcher.getMatchType(resource, request.getContext()); + + final boolean isMatched; + + if (request.isAccessTypeAny()) { + isMatched = matchType != RangerPolicyResourceMatcher.MatchType.NONE; + } else if (request.getResourceMatchingScope() == RangerAccessRequest.ResourceMatchingScope.SELF_OR_DESCENDANTS) { + isMatched = matchType != RangerPolicyResourceMatcher.MatchType.NONE; + } else { + isMatched = matchType == RangerPolicyResourceMatcher.MatchType.SELF || matchType == RangerPolicyResourceMatcher.MatchType.ANCESTOR; + } + + if (isMatched) { + if (ret == null) { + ret = new HashSet<>(); + } + ret.addAll(getTagsForServiceResource(request.getAccessTime(), enrichedServiceTags.getServiceTags(), resourceMatcher.getServiceResource(), matchType)); + } + + } + } + } + + if (CollectionUtils.isEmpty(ret)) { + if (LOG.isDebugEnabled()) { + LOG.debug("RangerTagEnricher.findMatchingTags(" + resource + ") - No tags Found "); + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("RangerTagEnricher.findMatchingTags(" + resource + ") - " + ret.size() + " tags Found "); + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerTagEnricher.findMatchingTags(" + request + ")"); + } + + return ret; + } + + private List getEvaluators(RangerAccessRequest request, EnrichedServiceTags enrichedServiceTags) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerTagEnricher.getEvaluators(request=" + request + ")"); + } + List ret = Collections.EMPTY_LIST; + + RangerAccessResource resource = request.getResource(); + + final Map> serviceResourceTrie = enrichedServiceTags.getServiceResourceTrie(); + + if (resource == null || resource.getKeys() == null || resource.getKeys().isEmpty() || serviceResourceTrie == null) { + ret = enrichedServiceTags.getServiceResourceMatchers(); + } else { + RangerPerfTracer perf = null; + + if (RangerPerfTracer.isPerfTraceEnabled(PERF_TRIE_OP_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_TRIE_OP_LOG, "RangerTagEnricher.getEvaluators(resource=" + resource.getAsString() + ")"); + } + + List resourceKeys = serviceDefHelper.getOrderedResourceNames(resource.getKeys()); + Set smallestList = null; + + if (CollectionUtils.isNotEmpty(resourceKeys)) { + + for (String resourceName : resourceKeys) { + RangerResourceTrie trie = serviceResourceTrie.get(resourceName); + + if (trie == null) { // if no trie exists for this resource level, ignore and continue to next level + continue; + } + + Set serviceResourceMatchersForResource = trie.getEvaluatorsForResource(resource.getValue(resourceName), request.getResourceMatchingScope()); + Set inheritedResourceMatchers = trie.getInheritedEvaluators(); + + if (smallestList != null) { + if (CollectionUtils.isEmpty(inheritedResourceMatchers) && CollectionUtils.isEmpty(serviceResourceMatchersForResource)) { + smallestList = null; + } else if (CollectionUtils.isEmpty(inheritedResourceMatchers)) { + smallestList.retainAll(serviceResourceMatchersForResource); + } else if (CollectionUtils.isEmpty(serviceResourceMatchersForResource)) { + smallestList.retainAll(inheritedResourceMatchers); + } else { + Set smaller, bigger; + if (serviceResourceMatchersForResource.size() < inheritedResourceMatchers.size()) { + smaller = serviceResourceMatchersForResource; + bigger = inheritedResourceMatchers; + } else { + smaller = inheritedResourceMatchers; + bigger = serviceResourceMatchersForResource; + } + Set tmp = new HashSet<>(); + if (smallestList.size() < smaller.size()) { + smallestList.stream().filter(smaller::contains).forEach(tmp::add); + smallestList.stream().filter(bigger::contains).forEach(tmp::add); + } else { + smaller.stream().filter(smallestList::contains).forEach(tmp::add); + if (smallestList.size() < bigger.size()) { + smallestList.stream().filter(bigger::contains).forEach(tmp::add); + } else { + bigger.stream().filter(smallestList::contains).forEach(tmp::add); + } + } + smallestList = tmp; + } + } else { + if (CollectionUtils.isEmpty(inheritedResourceMatchers) || CollectionUtils.isEmpty(serviceResourceMatchersForResource)) { + Set tmp = CollectionUtils.isEmpty(inheritedResourceMatchers) ? serviceResourceMatchersForResource : inheritedResourceMatchers; + smallestList = resourceKeys.size() == 1 || CollectionUtils.isEmpty(tmp) ? tmp : new HashSet<>(tmp); + } else { + smallestList = new HashSet<>(serviceResourceMatchersForResource); + smallestList.addAll(inheritedResourceMatchers); + } + } + + if (CollectionUtils.isEmpty(smallestList)) {// no tags for this resource, bail out + smallestList = null; + break; + } + } + } + + if (smallestList != null) { + ret = new ArrayList<>(smallestList); + } + + RangerPerfTracer.logAlways(perf); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerTagEnricher.getEvaluators(request=" + request + "): evaluatorCount=" + ret.size()); + } + + return ret; + } + + private static Set getTagsForServiceResource(Date accessTime, final ServiceTags serviceTags, final RangerServiceResource serviceResource, final RangerPolicyResourceMatcher.MatchType matchType) { + Set ret = new HashSet<>(); + + final Long resourceId = serviceResource.getId(); + final Map> resourceToTagIds = serviceTags.getResourceToTagIds(); + final Map tags = serviceTags.getTags(); + + if (LOG.isDebugEnabled()) { + LOG.debug("Looking for tags for resource-id:[" + resourceId + "] in serviceTags:[" + serviceTags + "]"); + } + + if (resourceId != null && MapUtils.isNotEmpty(resourceToTagIds) && MapUtils.isNotEmpty(tags)) { + + List tagIds = resourceToTagIds.get(resourceId); + + if (CollectionUtils.isNotEmpty(tagIds)) { + + accessTime = accessTime == null ? new Date() : accessTime; + + for (Long tagId : tagIds) { + + RangerTag tag = tags.get(tagId); + + if (tag != null) { + RangerTagForEval tagForEval = new RangerTagForEval(tag, matchType); + if (tagForEval.isApplicable(accessTime)) { + ret.add(tagForEval); + } + } + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("No tags mapping found for resource:[" + resourceId + "]"); + } + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("resourceId is null or resourceToTagTds mapping is null or tags mapping is null!"); + } + } + + return ret; + } + + private Map> copyServiceResourceTrie() { + Map> ret = new HashMap<>(); + + if (enrichedServiceTags != null) { + for (Map.Entry> entry : enrichedServiceTags.getServiceResourceTrie().entrySet()) { + RangerResourceTrie resourceTrie = new RangerResourceTrie<>(entry.getValue()); + ret.put(entry.getKey(), resourceTrie); + } + } + return ret; + } + + static public final class EnrichedServiceTags { + final private ServiceTags serviceTags; + final private List serviceResourceMatchers; + final private Map> serviceResourceTrie; + final private Set tagsForEmptyResourceAndAnyAccess; // Used only when accessed resource is empty and access type is 'any' + final private Long resourceTrieVersion; + + EnrichedServiceTags(ServiceTags serviceTags, List serviceResourceMatchers, Map> serviceResourceTrie) { + this.serviceTags = serviceTags; + this.serviceResourceMatchers = serviceResourceMatchers; + this.serviceResourceTrie = serviceResourceTrie; + this.tagsForEmptyResourceAndAnyAccess = createTagsForEmptyResourceAndAnyAccess(); + this.resourceTrieVersion = serviceTags.getTagVersion(); + } + public ServiceTags getServiceTags() {return serviceTags;} + public List getServiceResourceMatchers() { return serviceResourceMatchers;} + public Map> getServiceResourceTrie() { return serviceResourceTrie;} + public Long getResourceTrieVersion() { return resourceTrieVersion;} + public Set getTagsForEmptyResourceAndAnyAccess() { return tagsForEmptyResourceAndAnyAccess;} + + private Set createTagsForEmptyResourceAndAnyAccess() { + Set tagsForEmptyResourceAndAnyAccess = new HashSet<>(); + for (Map.Entry entry : serviceTags.getTags().entrySet()) { + tagsForEmptyResourceAndAnyAccess.add(new RangerTagForEval(entry.getValue(), RangerPolicyResourceMatcher.MatchType.DESCENDANT)); + } + return tagsForEmptyResourceAndAnyAccess; + } + } + + static class RangerTagRefresher extends Thread { + private static final Log LOG = LogFactory.getLog(RangerTagRefresher.class); + + private final RangerTagRetriever tagRetriever; + private final RangerTagEnricher tagEnricher; + private long lastKnownVersion; + private final BlockingQueue tagDownloadQueue; + private long lastActivationTimeInMillis; + + private final String cacheFile; + private boolean hasProvidedTagsToReceiver; + private Gson gson; + + RangerTagRefresher(RangerTagRetriever tagRetriever, RangerTagEnricher tagEnricher, long lastKnownVersion, BlockingQueue tagDownloadQueue, String cacheFile) { + this.tagRetriever = tagRetriever; + this.tagEnricher = tagEnricher; + this.lastKnownVersion = lastKnownVersion; + this.tagDownloadQueue = tagDownloadQueue; + this.cacheFile = cacheFile; + try { + gson = new GsonBuilder().setDateFormat("yyyyMMdd-HH:mm:ss.SSS-Z").create(); + } catch(Throwable excp) { + LOG.fatal("failed to create GsonBuilder object", excp); + } + setName("RangerTagRefresher(serviceName=" + tagRetriever.getServiceName() + ")-" + getId()); + } + + public long getLastActivationTimeInMillis() { + return lastActivationTimeInMillis; + } + + public void setLastActivationTimeInMillis(long lastActivationTimeInMillis) { + this.lastActivationTimeInMillis = lastActivationTimeInMillis; + } + + @Override + public void run() { + + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerTagRefresher().run()"); + } + + while (true) { + DownloadTrigger trigger = null; + + try { + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_CONTEXTENRICHER_INIT_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_CONTEXTENRICHER_INIT_LOG, "RangerTagRefresher(" + getName() + ").populateTags(lastKnownVersion=" + lastKnownVersion + ")"); + } + trigger = tagDownloadQueue.take(); + populateTags(); + + RangerPerfTracer.log(perf); + + } catch (InterruptedException excp) { + LOG.info("RangerTagRefresher(" + getName() + ").run(): Interrupted! Exiting thread", excp); + break; + } finally { + if (trigger != null) { + trigger.signalCompletion(); + } + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerTagRefresher().run()"); + } + } + + private void populateTags() throws InterruptedException { + + if (tagEnricher != null) { + ServiceTags serviceTags; + + try { + serviceTags = tagRetriever.retrieveTags(lastKnownVersion, lastActivationTimeInMillis); + + if (serviceTags == null) { + if (!hasProvidedTagsToReceiver) { + serviceTags = loadFromCache(); + } + } else if (!serviceTags.getIsDelta()) { + saveToCache(serviceTags); + } + + if (serviceTags != null) { + tagEnricher.setServiceTags(serviceTags); + if (serviceTags.getIsDelta() && serviceTags.getTagVersion() != -1L) { + saveToCache(tagEnricher.enrichedServiceTags.serviceTags); + } + LOG.info("RangerTagRefresher(serviceName=" + tagRetriever.getServiceName() + ").populateTags() - Updated tags-cache to new version of tags, lastKnownVersion=" + lastKnownVersion + "; newVersion=" + + (serviceTags.getTagVersion() == null ? -1L : serviceTags.getTagVersion())); + hasProvidedTagsToReceiver = true; + lastKnownVersion = serviceTags.getTagVersion() == null ? -1L : serviceTags.getTagVersion(); + setLastActivationTimeInMillis(System.currentTimeMillis()); + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("RangerTagRefresher(serviceName=" + tagRetriever.getServiceName() + ").populateTags() - No need to update tags-cache. lastKnownVersion=" + lastKnownVersion); + } + } + } catch (RangerServiceNotFoundException snfe) { + LOG.error("Caught ServiceNotFound exception :", snfe); + + // Need to clean up local tag cache + if (tagEnricher.disableCacheIfServiceNotFound) { + disableCache(); + tagEnricher.setServiceTags(null); + setLastActivationTimeInMillis(System.currentTimeMillis()); + lastKnownVersion = -1L; + } + } catch (InterruptedException interruptedException) { + throw interruptedException; + } catch (Exception e) { + LOG.error("RangerTagRefresher(serviceName=" + tagRetriever.getServiceName() + ").populateTags(): Encountered unexpected exception. Ignoring", e); + } + + } else { + LOG.error("RangerTagRefresher(serviceName=" + tagRetriever.getServiceName() + ".populateTags() - no tag receiver to update tag-cache"); + } + } + + void cleanup() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerTagRefresher.cleanup()"); + } + + stopRefresher(); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerTagRefresher.cleanup()"); + } + } + + final void startRefresher() { + try { + super.start(); + } catch (Exception excp) { + LOG.error("RangerTagRefresher(" + getName() + ").startRetriever(): Failed to start, exception=" + excp); + } + } + + private void stopRefresher() { + + if (super.isAlive()) { + super.interrupt(); + + boolean setInterrupted = false; + boolean isJoined = false; + + while (!isJoined) { + try { + super.join(); + isJoined = true; + } catch (InterruptedException excp) { + LOG.warn("RangerTagRefresher(" + getName() + ").stopRefresher(): Error while waiting for thread to exit", excp); + LOG.warn("Retrying Thread.join(). Current thread will be marked as 'interrupted' after Thread.join() returns"); + setInterrupted = true; + } + } + if (setInterrupted) { + Thread.currentThread().interrupt(); + } + } + } + + + final ServiceTags loadFromCache() { + ServiceTags serviceTags = null; + + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerTagRetriever(serviceName=" + tagEnricher.getServiceName() + ").loadFromCache()"); + } + + File cacheFile = StringUtils.isEmpty(this.cacheFile) ? null : new File(this.cacheFile); + + if (cacheFile != null && cacheFile.isFile() && cacheFile.canRead()) { + Reader reader = null; + + try { + reader = new FileReader(cacheFile); + + serviceTags = gson.fromJson(reader, ServiceTags.class); + + if (serviceTags != null && !StringUtils.equals(tagEnricher.getServiceName(), serviceTags.getServiceName())) { + LOG.warn("ignoring unexpected serviceName '" + serviceTags.getServiceName() + "' in cache file '" + cacheFile.getAbsolutePath() + "'"); + + serviceTags.setServiceName(tagEnricher.getServiceName()); + } + } catch (Exception excp) { + LOG.error("failed to load service-tags from cache file " + cacheFile.getAbsolutePath(), excp); + } finally { + if (reader != null) { + try { + reader.close(); + } catch (Exception excp) { + LOG.error("error while closing opened cache file " + cacheFile.getAbsolutePath(), excp); + } + } + } + } else { + LOG.warn("cache file does not exist or not readable '" + (cacheFile == null ? null : cacheFile.getAbsolutePath()) + "'"); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerTagRetriever(serviceName=" + tagEnricher.getServiceName() + ").loadFromCache()"); + } + + return serviceTags; + } + + final void saveToCache(ServiceTags serviceTags) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerTagRetriever(serviceName=" + tagEnricher.getServiceName() + ").saveToCache()"); + } + + if (serviceTags != null) { + File cacheFile = StringUtils.isEmpty(this.cacheFile) ? null : new File(this.cacheFile); + + if (cacheFile != null) { + Writer writer = null; + + try { + writer = new FileWriter(cacheFile); + + gson.toJson(serviceTags, writer); + } catch (Exception excp) { + LOG.error("failed to save service-tags to cache file '" + cacheFile.getAbsolutePath() + "'", excp); + } finally { + if (writer != null) { + try { + writer.close(); + } catch (Exception excp) { + LOG.error("error while closing opened cache file '" + cacheFile.getAbsolutePath() + "'", excp); + } + } + } + } + } else { + LOG.info("service-tags is null for service=" + tagRetriever.getServiceName() + ". Nothing to save in cache"); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerTagRetriever(serviceName=" + tagEnricher.getServiceName() + ").saveToCache()"); + } + } + + final void disableCache() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerTagRetriever.disableCache(serviceName=" + tagEnricher.getServiceName() + ")"); + } + + File cacheFile = StringUtils.isEmpty(this.cacheFile) ? null : new File(this.cacheFile); + if (cacheFile != null && cacheFile.isFile() && cacheFile.canRead()) { + LOG.warn("Cleaning up local tags cache"); + String renamedCacheFile = cacheFile.getAbsolutePath() + "_" + System.currentTimeMillis(); + if (!cacheFile.renameTo(new File(renamedCacheFile))) { + LOG.error("Failed to move " + cacheFile.getAbsolutePath() + " to " + renamedCacheFile); + } else { + LOG.warn("moved " + cacheFile.getAbsolutePath() + " to " + renamedCacheFile); + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("No local TAGS cache found. No need to disable it!"); + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerTagRetriever.disableCache(serviceName=" + tagEnricher.getServiceName() + ")"); + } + } + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/contextenricher/RangerTagForEval.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/contextenricher/RangerTagForEval.java new file mode 100644 index 00000000000..59ba56175e6 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/contextenricher/RangerTagForEval.java @@ -0,0 +1,239 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.contextenricher; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.authorization.utils.JsonUtils; +import org.apache.atlas.plugin.model.RangerTag; +import org.apache.atlas.plugin.model.RangerValiditySchedule; +import org.apache.atlas.plugin.policyevaluator.RangerValidityScheduleEvaluator; +import org.apache.atlas.plugin.policyresourcematcher.RangerPolicyResourceMatcher; +import org.codehaus.jackson.annotate.JsonAutoDetect; +import org.codehaus.jackson.annotate.JsonIgnore; +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.List; +import java.util.Map; + +@JsonAutoDetect(fieldVisibility=JsonAutoDetect.Visibility.ANY) +@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown=true) +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) + +// This class needs above annotations for policy-engine unit tests involving RangerTagForEval objects that are initialized +// from JSON specification + +public class RangerTagForEval implements Serializable { + private static final Log LOG = LogFactory.getLog(RangerTagForEval.class); + + private String type; + private Map attributes; + private Map options; + private RangerPolicyResourceMatcher.MatchType matchType = RangerPolicyResourceMatcher.MatchType.SELF; + @JsonIgnore + private List validityPeriods; + @JsonIgnore + private transient List validityPeriodEvaluators; + + + private RangerTagForEval() {} + + public RangerTagForEval(RangerTag tag, RangerPolicyResourceMatcher.MatchType matchType) { + this.type = tag.getType(); + this.attributes = tag.getAttributes(); + this.options = tag.getOptions(); + this.matchType = matchType; + this.validityPeriods = tag.getValidityPeriods(); + + this.validityPeriodEvaluators = createValidityPeriodEvaluators(); + } + + public String getType() { return type;} + + public Map getAttributes() { return attributes; } + + public Map getOptions() { return options; } + + public RangerPolicyResourceMatcher.MatchType getMatchType() { return matchType; } + + public List getValidityPeriods() { return validityPeriods; } + + public boolean isApplicable(Date accessTime) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerTagForEval.isApplicable(type=" + type + ", " + accessTime + ")"); + } + + boolean ret = false; + + List validityPeriodEvaluators = this.validityPeriodEvaluators; + + // Specifically for unit-testing using TestPolicyEngine + if (MapUtils.isNotEmpty(options) && CollectionUtils.isEmpty(validityPeriodEvaluators)) { + Object value = getOption(RangerTag.OPTION_TAG_VALIDITY_PERIODS); + + if (value != null && value instanceof String) { + this.validityPeriods = JsonUtils.jsonToRangerValiditySchedule((String) value); + + validityPeriodEvaluators = createValidityPeriodEvaluators(); + } else { + validityPeriodEvaluators = Collections.emptyList(); + } + + this.validityPeriodEvaluators = validityPeriodEvaluators; + } + + if (accessTime != null && CollectionUtils.isNotEmpty(validityPeriodEvaluators)) { + for (RangerValidityScheduleEvaluator evaluator : validityPeriodEvaluators) { + if (evaluator.isApplicable(accessTime.getTime())) { + ret = true; + break; + } + } + } else { + ret = true; + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerTagForEval.isApplicable(type=" + type + ", " + accessTime + ") : " + ret); + } + + return ret; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + + toString(sb); + + return sb.toString(); + } + + public StringBuilder toString(StringBuilder sb) { + sb.append("RangerTagForEval={ "); + sb.append("type=" ).append(type); + sb.append(", attributes={ "); + if (attributes != null) { + for (Map.Entry entry : attributes.entrySet()) { + sb.append('"').append(entry.getKey()).append("\":\"").append(entry.getValue()).append("\", "); + } + } + sb.append(" }"); + sb.append(", matchType=").append(matchType); + + if (options != null) { + sb.append(", options={").append(options).append("}"); + } + + if (validityPeriods != null) { + sb.append(", validityPeriods=").append(validityPeriods); + } + sb.append(" }"); + return sb; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + + ((type == null) ? 0 : type.hashCode()); + result = prime * result + + ((attributes == null) ? 0 : attributes.hashCode()); + result = prime * result + + ((matchType == null) ? 0 : matchType.hashCode()); + result = prime * result + + ((validityPeriods == null) ? 0 : validityPeriods.hashCode()); + result = prime * result + + ((options == null) ? 0 : options.hashCode()); + + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + RangerTagForEval other = (RangerTagForEval) obj; + if (type == null) { + if (other.type != null) + return false; + } else if (!type.equals(other.type)) + return false; + if (attributes == null) { + if (other.attributes != null) + return false; + } else if (!attributes.equals(other.attributes)) + return false; + if (matchType == null) { + if (other.matchType != null) + return false; + } else if (!matchType.equals(other.matchType)) + return false; + if (options == null) { + if (other.options != null) + return false; + } else if (!options.equals(other.options)) + return false; + if (validityPeriods == null) { + if (other.validityPeriods != null) + return false; + } else if (!validityPeriods.equals(other.validityPeriods)) + return false; + + return true; + } + + private Object getOption(String name) { + return options != null ? options.get(name) : null; + } + + private List createValidityPeriodEvaluators() { + final List ret; + + if (CollectionUtils.isNotEmpty(validityPeriods)) { + ret = new ArrayList<>(); + + for (RangerValiditySchedule schedule : validityPeriods) { + ret.add(new RangerValidityScheduleEvaluator(schedule)); + } + } else { + ret = Collections.emptyList(); + } + + return ret; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/contextenricher/RangerTagRetriever.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/contextenricher/RangerTagRetriever.java new file mode 100644 index 00000000000..880e820feb6 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/contextenricher/RangerTagRetriever.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.contextenricher; + +import org.apache.atlas.authorization.hadoop.config.RangerPluginConfig; +import org.apache.atlas.plugin.model.RangerServiceDef; +import org.apache.atlas.plugin.policyengine.RangerPluginContext; +import org.apache.atlas.plugin.util.ServiceTags; + +import java.util.Map; + +public abstract class RangerTagRetriever { + + protected String serviceName; + protected RangerServiceDef serviceDef; + protected String appId; + protected RangerPluginConfig pluginConfig; + protected RangerPluginContext pluginContext; + + public abstract void init(Map options); + + public abstract ServiceTags retrieveTags(long lastKnownVersion, long lastActivationTimeInMillis) throws Exception; + + public String getServiceName() { + return serviceName; + } + + public void setServiceName(String serviceName) { + this.serviceName = serviceName; + } + + public RangerServiceDef getServiceDef() { + return serviceDef; + } + + public void setServiceDef(RangerServiceDef serviceDef) { + this.serviceDef = serviceDef; + } + + public String getAppId() { + return appId; + } + + public void setAppId(String appId) { + this.appId = appId; + } + + public void setPluginConfig(RangerPluginConfig pluginConfig) { this.pluginConfig = pluginConfig; } + + public RangerPluginContext getPluginContext() { + return pluginContext; + } + + public void setPluginContext(RangerPluginContext pluginContext) { + this.pluginContext = pluginContext; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/contextenricher/RangerUserStoreEnricher.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/contextenricher/RangerUserStoreEnricher.java new file mode 100644 index 00000000000..87eebf0f8fe --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/contextenricher/RangerUserStoreEnricher.java @@ -0,0 +1,251 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.contextenricher; + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.plugin.policyengine.RangerAccessRequest; +import org.apache.atlas.plugin.service.RangerAuthContext; +import org.apache.atlas.plugin.util.DownloadTrigger; +import org.apache.atlas.plugin.util.DownloaderTask; +import org.apache.atlas.plugin.util.RangerAccessRequestUtil; +import org.apache.atlas.plugin.util.RangerPerfTracer; +import org.apache.atlas.plugin.util.RangerUserStore; + +import java.io.File; +import java.util.Timer; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; + +public class RangerUserStoreEnricher extends RangerAbstractContextEnricher { + private static final Log LOG = LogFactory.getLog(RangerUserStoreEnricher.class); + + private static final Log PERF_SET_USERSTORE_LOG = RangerPerfTracer.getPerfLogger("userstoreenricher.setuserstore"); + + + private static final String USERSTORE_REFRESHER_POLLINGINTERVAL_OPTION = "userStoreRefresherPollingInterval"; + private static final String USERSTORE_RETRIEVER_CLASSNAME_OPTION = "userStoreRetrieverClassName"; + + private RangerUserStoreRefresher userStoreRefresher; + private RangerUserStoreRetriever userStoreRetriever; + private RangerUserStore rangerUserStore; + private boolean disableCacheIfServiceNotFound = true; + + private final BlockingQueue userStoreDownloadQueue = new LinkedBlockingQueue<>(); + private Timer userStoreDownloadTimer; + + @Override + public void init() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerUserStoreEnricher.init()"); + } + + super.init(); + + String userStoreRetrieverClassName = getOption(USERSTORE_RETRIEVER_CLASSNAME_OPTION); + + long pollingIntervalMs = getLongOption(USERSTORE_REFRESHER_POLLINGINTERVAL_OPTION, 3600 * 1000); + + if (StringUtils.isNotBlank(userStoreRetrieverClassName)) { + + try { + @SuppressWarnings("unchecked") + Class userStoreRetriverClass = (Class) Class.forName(userStoreRetrieverClassName); + + userStoreRetriever = userStoreRetriverClass.newInstance(); + + } catch (ClassNotFoundException exception) { + LOG.error("Class " + userStoreRetrieverClassName + " not found, exception=" + exception); + } catch (ClassCastException exception) { + LOG.error("Class " + userStoreRetrieverClassName + " is not a type of RangerUserStoreRetriever, exception=" + exception); + } catch (IllegalAccessException exception) { + LOG.error("Class " + userStoreRetrieverClassName + " illegally accessed, exception=" + exception); + } catch (InstantiationException exception) { + LOG.error("Class " + userStoreRetrieverClassName + " could not be instantiated, exception=" + exception); + } + + if (userStoreRetriever != null) { + String propertyPrefix = "ranger.plugin." + serviceDef.getName(); + disableCacheIfServiceNotFound = getBooleanConfig(propertyPrefix + ".disable.cache.if.servicenotfound", true); + String cacheDir = getConfig(propertyPrefix + ".policy.cache.dir", null); + String cacheFilename = String.format("%s_%s_userstore.json", appId, serviceName); + + cacheFilename = cacheFilename.replace(File.separatorChar, '_'); + cacheFilename = cacheFilename.replace(File.pathSeparatorChar, '_'); + + String cacheFile = cacheDir == null ? null : (cacheDir + File.separator + cacheFilename); + + userStoreRetriever.setServiceName(serviceName); + userStoreRetriever.setServiceDef(serviceDef); + userStoreRetriever.setAppId(appId); + userStoreRetriever.setPluginConfig(getPluginConfig()); + userStoreRetriever.setPluginContext(getPluginContext()); + userStoreRetriever.init(enricherDef.getEnricherOptions()); + + userStoreRefresher = new RangerUserStoreRefresher(userStoreRetriever, this, null, -1L, userStoreDownloadQueue, cacheFile); + LOG.info("Created Thread(RangerUserStoreRefresher(" + getName() + ")"); + + try { + userStoreRefresher.populateUserStoreInfo(); + } catch (Throwable exception) { + LOG.error("Exception when retrieving userstore information for this enricher", exception); + } + + userStoreRefresher.setDaemon(true); + userStoreRefresher.startRefresher(); + + userStoreDownloadTimer = new Timer("userStoreDownloadTimer", true); + + try { + userStoreDownloadTimer.schedule(new DownloaderTask(userStoreDownloadQueue), pollingIntervalMs, pollingIntervalMs); + if (LOG.isDebugEnabled()) { + LOG.debug("Scheduled userStoreDownloadRefresher to download userstore every " + pollingIntervalMs + " milliseconds"); + } + } catch (IllegalStateException exception) { + LOG.error("Error scheduling userStoreDownloadTimer:", exception); + LOG.error("*** UserStore information will NOT be downloaded every " + pollingIntervalMs + " milliseconds ***"); + userStoreDownloadTimer = null; + } + } + } else { + LOG.error("No value specified for " + USERSTORE_RETRIEVER_CLASSNAME_OPTION + " in the RangerUserStoreEnricher options"); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerUserStoreEnricher.init()"); + } + } + + @Override + public void enrich(RangerAccessRequest request) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerUserStoreEnricher.enrich(" + request + ")"); + } + + enrich(request, null); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerUserStoreEnricher.enrich(" + request + ")"); + } + } + + @Override + public void enrich(RangerAccessRequest request, Object dataStore) { + + // Unused by Solr plugin as document level authorization gets RangerUserStore from AuthContext + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerUserStoreEnricher.enrich(" + request + ") with dataStore:[" + dataStore + "]"); + } + final RangerUserStore rangerUserStore; + + if (dataStore instanceof RangerUserStore) { + rangerUserStore = (RangerUserStore) dataStore; + } else { + rangerUserStore = this.rangerUserStore; + + if (dataStore != null) { + LOG.warn("Incorrect type of dataStore :[" + dataStore.getClass().getName() + "], falling back to original enrich"); + } + } + + RangerAccessRequestUtil.setRequestUserStoreInContext(request.getContext(), rangerUserStore); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerUserStoreEnricher.enrich(" + request + ") with dataStore:[" + dataStore + "])"); + } + } + + public boolean isDisableCacheIfServiceNotFound() { + return disableCacheIfServiceNotFound; + } + + public RangerUserStore getRangerUserStore() {return this.rangerUserStore;} + + public void setRangerUserStore(final RangerUserStore rangerUserStore) { + + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerUserStoreEnricher.setRangerUserStore(rangerUserStore=" + rangerUserStore + ")"); + } + + if (rangerUserStore == null) { + LOG.info("UserStore information is null for service " + serviceName); + this.rangerUserStore = null; + } else { + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_SET_USERSTORE_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_SET_USERSTORE_LOG, "RangerUserStoreEnricher.setRangerUserStore(newUserStoreVersion=" + rangerUserStore.getUserStoreVersion() + ")"); + } + + this.rangerUserStore = rangerUserStore; + RangerPerfTracer.logAlways(perf); + } + + setRangerUserStoreInPlugin(); + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerUserStoreEnricher.setRangerUserStore(rangerUserStore=" + rangerUserStore + ")"); + } + + } + + @Override + public boolean preCleanup() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerUserStoreEnricher.preCleanup()"); + } + + super.preCleanup(); + + if (userStoreDownloadTimer != null) { + userStoreDownloadTimer.cancel(); + userStoreDownloadTimer = null; + } + + if (userStoreRefresher != null) { + userStoreRefresher.cleanup(); + userStoreRefresher = null; + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerUserStoreEnricher.preCleanup() : result=" + true); + } + return true; + } + + private void setRangerUserStoreInPlugin() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> setRangerUserStoreInPlugin()"); + } + + RangerAuthContext authContext = getAuthContext(); + + if (authContext != null) { + authContext.addOrReplaceRequestContextEnricher(this, rangerUserStore); + + notifyAuthContextChanged(); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== setRangerUserStoreInPlugin()"); + } + } + +} \ No newline at end of file diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/contextenricher/RangerUserStoreRefresher.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/contextenricher/RangerUserStoreRefresher.java new file mode 100644 index 00000000000..e60e0a788cc --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/contextenricher/RangerUserStoreRefresher.java @@ -0,0 +1,443 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.atlas.plugin.contextenricher; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.sun.jersey.api.client.ClientResponse; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.atlas.admin.client.datatype.RESTResponse; +import org.apache.atlas.audit.provider.MiscUtil; +import org.apache.atlas.plugin.util.DownloadTrigger; +import org.apache.atlas.plugin.util.RangerPerfTracer; +import org.apache.atlas.plugin.util.RangerRESTClient; +import org.apache.atlas.plugin.util.RangerRESTUtils; +import org.apache.atlas.plugin.util.RangerServiceNotFoundException; +import org.apache.atlas.plugin.util.RangerUserStore; + +import javax.servlet.http.HttpServletResponse; +import java.io.File; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.Reader; +import java.io.Writer; +import java.nio.channels.ClosedByInterruptException; +import java.security.PrivilegedAction; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.BlockingQueue; + +public class RangerUserStoreRefresher extends Thread { + private static final Log LOG = LogFactory.getLog(RangerUserStoreRefresher.class); + private static final Log PERF_REFRESHER_INIT_LOG = RangerPerfTracer.getPerfLogger("userstore.init"); + + private final RangerUserStoreRetriever userStoreRetriever; + private final RangerUserStoreEnricher userStoreEnricher; + private long lastKnownVersion; + private final BlockingQueue userStoreDownloadQueue; + private long lastActivationTimeInMillis; + + private final String cacheFile; + private boolean hasProvidedUserStoreToReceiver; + private Gson gson; + private RangerRESTClient rangerRESTClient; + + public RangerUserStoreRefresher(RangerUserStoreRetriever userStoreRetriever, RangerUserStoreEnricher userStoreEnricher, + RangerRESTClient restClient, long lastKnownVersion, + BlockingQueue userStoreDownloadQueue, String cacheFile) { + this.userStoreRetriever = userStoreRetriever; + this.userStoreEnricher = userStoreEnricher; + this.rangerRESTClient = restClient; + this.lastKnownVersion = lastKnownVersion; + this.userStoreDownloadQueue = userStoreDownloadQueue; + this.cacheFile = cacheFile; + try { + gson = new GsonBuilder().setDateFormat("yyyyMMdd-HH:mm:ss.SSS-Z").create(); + } catch(Throwable excp) { + LOG.fatal("failed to create GsonBuilder object", excp); + } + setName("RangerUserStoreRefresher(serviceName=" + userStoreRetriever.getServiceName() + ")-" + getId()); + } + + public long getLastActivationTimeInMillis() { + return lastActivationTimeInMillis; + } + + public void setLastActivationTimeInMillis(long lastActivationTimeInMillis) { + this.lastActivationTimeInMillis = lastActivationTimeInMillis; + } + + @Override + public void run() { + + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerUserStoreRefresher().run()"); + } + + while (true) { + DownloadTrigger trigger = null; + + try { + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_REFRESHER_INIT_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_REFRESHER_INIT_LOG, + "RangerUserStoreRefresher.run(lastKnownVersion=" + lastKnownVersion + ")"); + } + trigger = userStoreDownloadQueue.take(); + populateUserStoreInfo(); + + RangerPerfTracer.log(perf); + + } catch (InterruptedException excp) { + LOG.debug("RangerUserStoreRefresher().run() : interrupted! Exiting thread", excp); + break; + } finally { + if (trigger != null) { + trigger.signalCompletion(); + } + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerUserStoreRefresher().run()"); + } + } + + public RangerUserStore populateUserStoreInfo() throws InterruptedException { + + RangerUserStore rangerUserStore = null; + if (userStoreEnricher != null && userStoreRetriever != null) { + try { + rangerUserStore = userStoreRetriever.retrieveUserStoreInfo(lastKnownVersion, lastActivationTimeInMillis); + + if (rangerUserStore == null) { + if (!hasProvidedUserStoreToReceiver) { + rangerUserStore = loadFromCache(); + } + } + + if (rangerUserStore != null) { + userStoreEnricher.setRangerUserStore(rangerUserStore); + if (rangerUserStore.getUserStoreVersion() != -1L) { + saveToCache(rangerUserStore); + } + LOG.info("RangerUserStoreRefresher.populateUserStoreInfo() - Updated userstore-cache to new version, lastKnownVersion=" + lastKnownVersion + "; newVersion=" + + (rangerUserStore.getUserStoreVersion() == null ? -1L : rangerUserStore.getUserStoreVersion())); + hasProvidedUserStoreToReceiver = true; + lastKnownVersion = rangerUserStore.getUserStoreVersion() == null ? -1L : rangerUserStore.getUserStoreVersion(); + setLastActivationTimeInMillis(System.currentTimeMillis()); + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("RangerUserStoreRefresher.populateUserStoreInfo() - No need to update userstore-cache. lastKnownVersion=" + lastKnownVersion); + } + } + } catch (RangerServiceNotFoundException snfe) { + LOG.error("Caught ServiceNotFound exception :", snfe); + + // Need to clean up local userstore cache + if (userStoreEnricher.isDisableCacheIfServiceNotFound()) { + disableCache(); + setLastActivationTimeInMillis(System.currentTimeMillis()); + lastKnownVersion = -1L; + } + } catch (InterruptedException interruptedException) { + throw interruptedException; + } catch (Exception e) { + LOG.error("Encountered unexpected exception. Ignoring", e); + } + } else if (rangerRESTClient != null) { + if (LOG.isDebugEnabled()) { + LOG.debug("RangerUserStoreRefresher.populateUserStoreInfo() for Ranger Raz"); + } + try { + rangerUserStore = retrieveUserStoreInfo(); + + if (rangerUserStore == null) { + if (!hasProvidedUserStoreToReceiver) { + rangerUserStore = loadFromCache(); + } + } + + if (rangerUserStore != null) { + + if (rangerUserStore.getUserStoreVersion() != -1L) { + saveToCache(rangerUserStore); + } + LOG.info("RangerUserStoreRefresher.populateUserStoreInfo() - Updated userstore-cache for raz to new version, lastKnownVersion=" + lastKnownVersion + "; newVersion=" + + (rangerUserStore.getUserStoreVersion() == null ? -1L : rangerUserStore.getUserStoreVersion())); + hasProvidedUserStoreToReceiver = true; + lastKnownVersion = rangerUserStore.getUserStoreVersion() == null ? -1L : rangerUserStore.getUserStoreVersion(); + setLastActivationTimeInMillis(System.currentTimeMillis()); + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("RangerUserStoreRefresher.populateUserStoreInfo() - No need to update userstore-cache for raz. lastKnownVersion=" + lastKnownVersion); + } + } + }catch (InterruptedException interruptedException) { + throw interruptedException; + } catch (Exception e) { + LOG.error("Encountered unexpected exception. Ignoring", e); + } + } + else { + LOG.error("RangerUserStoreRefresher.populateUserStoreInfo() - no userstore receiver to update userstore-cache"); + } + return rangerUserStore; + } + + public void cleanup() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerUserStoreRefresher.cleanup()"); + } + + stopRefresher(); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerUserStoreRefresher.cleanup()"); + } + } + + public void startRefresher() { + try { + super.start(); + } catch (Exception excp) { + LOG.error("RangerUserStoreRefresher.startRetriever() - failed to start, exception=" + excp); + } + } + + public void stopRefresher() { + + if (super.isAlive()) { + super.interrupt(); + + boolean setInterrupted = false; + boolean isJoined = false; + + while (!isJoined) { + try { + super.join(); + isJoined = true; + } catch (InterruptedException excp) { + LOG.warn("RangerUserStoreRefresher(): Error while waiting for thread to exit", excp); + LOG.warn("Retrying Thread.join(). Current thread will be marked as 'interrupted' after Thread.join() returns"); + setInterrupted = true; + } + } + if (setInterrupted) { + Thread.currentThread().interrupt(); + } + } + } + + + private RangerUserStore loadFromCache() { + RangerUserStore rangerUserStore = null; + + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerUserStoreRefreher.loadFromCache()"); + } + + File cacheFile = StringUtils.isEmpty(this.cacheFile) ? null : new File(this.cacheFile); + + if (cacheFile != null && cacheFile.isFile() && cacheFile.canRead()) { + Reader reader = null; + + try { + reader = new FileReader(cacheFile); + + rangerUserStore = gson.fromJson(reader, RangerUserStore.class); + + } catch (Exception excp) { + LOG.error("failed to load userstore information from cache file " + cacheFile.getAbsolutePath(), excp); + } finally { + if (reader != null) { + try { + reader.close(); + } catch (Exception excp) { + LOG.error("error while closing opened cache file " + cacheFile.getAbsolutePath(), excp); + } + } + } + } else { + LOG.warn("cache file does not exist or not readable '" + (cacheFile == null ? null : cacheFile.getAbsolutePath()) + "'"); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerUserStoreRefreher.loadFromCache()"); + } + + return rangerUserStore; + } + + public void saveToCache(RangerUserStore rangerUserStore) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerUserStoreRefreher.saveToCache()"); + } + + if (rangerUserStore != null) { + File cacheFile = StringUtils.isEmpty(this.cacheFile) ? null : new File(this.cacheFile); + + if (cacheFile != null) { + Writer writer = null; + + try { + writer = new FileWriter(cacheFile); + + gson.toJson(rangerUserStore, writer); + } catch (Exception excp) { + LOG.error("failed to save userstore information to cache file '" + cacheFile.getAbsolutePath() + "'", excp); + } finally { + if (writer != null) { + try { + writer.close(); + } catch (Exception excp) { + LOG.error("error while closing opened cache file '" + cacheFile.getAbsolutePath() + "'", excp); + } + } + } + } + } else { + LOG.info("userstore information is null. Nothing to save in cache"); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerUserStoreRefreher.saveToCache()"); + } + } + + private void disableCache() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerUserStoreRefreher.disableCache()"); + } + + File cacheFile = StringUtils.isEmpty(this.cacheFile) ? null : new File(this.cacheFile); + if (cacheFile != null && cacheFile.isFile() && cacheFile.canRead()) { + LOG.warn("Cleaning up local userstore cache"); + String renamedCacheFile = cacheFile.getAbsolutePath() + "_" + System.currentTimeMillis(); + if (!cacheFile.renameTo(new File(renamedCacheFile))) { + LOG.error("Failed to move " + cacheFile.getAbsolutePath() + " to " + renamedCacheFile); + } else { + LOG.warn("moved " + cacheFile.getAbsolutePath() + " to " + renamedCacheFile); + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("No local userstore cache found. No need to disable it!"); + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerUserStoreRefreher.disableCache()"); + } + } + + private RangerUserStore retrieveUserStoreInfo() throws Exception { + + RangerUserStore rangerUserStore = null; + + try { + rangerUserStore = getUserStoreIfUpdated(lastKnownVersion, lastActivationTimeInMillis); + } catch (ClosedByInterruptException closedByInterruptException) { + LOG.error("UserStore-retriever for raz thread was interrupted while blocked on I/O"); + throw new InterruptedException(); + } catch (Exception e) { + LOG.error("UserStore-retriever for raz encounterd exception, exception=", e); + LOG.error("Returning null userstore info"); + } + return rangerUserStore; + } + + private RangerUserStore getUserStoreIfUpdated(long lastKnownUserStoreVersion, long lastActivationTimeInMillis) throws Exception { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerUserStoreRefreher.getUserStoreIfUpdated(" + lastKnownUserStoreVersion + ", " + lastActivationTimeInMillis + ")"); + } + + final RangerUserStore ret; + final UserGroupInformation user = MiscUtil.getUGILoginUser(); + final boolean isSecureMode = user != null && UserGroupInformation.isSecurityEnabled(); + final ClientResponse response; + + Map queryParams = new HashMap(); + queryParams.put(RangerRESTUtils.REST_PARAM_LAST_KNOWN_USERSTORE_VERSION, Long.toString(lastKnownUserStoreVersion)); + queryParams.put(RangerRESTUtils.REST_PARAM_LAST_ACTIVATION_TIME, Long.toString(lastActivationTimeInMillis)); + + if (isSecureMode) { + if (LOG.isDebugEnabled()) { + LOG.debug("Checking UserStore updated as user : " + user); + } + PrivilegedAction action = new PrivilegedAction() { + public ClientResponse run() { + ClientResponse clientRes = null; + String relativeURL = RangerRESTUtils.REST_URL_SERVICE_SERCURE_GET_USERSTORE; + try { + clientRes = rangerRESTClient.get(relativeURL, queryParams); + } catch (Exception e) { + LOG.error("Failed to get response, Error is : "+e.getMessage()); + } + return clientRes; + } + }; + response = user.doAs(action); + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Checking UserStore updated as user : " + user); + } + String relativeURL = RangerRESTUtils.REST_URL_SERVICE_SERCURE_GET_USERSTORE; + response = rangerRESTClient.get(relativeURL, queryParams); + } + + if (response == null || response.getStatus() == HttpServletResponse.SC_NOT_MODIFIED) { + if (response == null) { + LOG.error("Error getting UserStore; Received NULL response!!. secureMode=" + isSecureMode + ", user=" + user); + } else { + RESTResponse resp = RESTResponse.fromClientResponse(response); + if (LOG.isDebugEnabled()) { + LOG.debug("No change in UserStore. secureMode=" + isSecureMode + ", user=" + user + + ", response=" + resp + + ", " + "lastKnownUserStoreVersion=" + lastKnownUserStoreVersion + + ", " + "lastActivationTimeInMillis=" + lastActivationTimeInMillis); + } + } + ret = null; + } else if (response.getStatus() == HttpServletResponse.SC_OK) { + ret = response.getEntity(RangerUserStore.class); + } else if (response.getStatus() == HttpServletResponse.SC_NOT_FOUND) { + ret = null; + LOG.error("Error getting UserStore; service not found. secureMode=" + isSecureMode + ", user=" + user + + ", response=" + response.getStatus() + + ", " + "lastKnownUserStoreVersion=" + lastKnownUserStoreVersion + + ", " + "lastActivationTimeInMillis=" + lastActivationTimeInMillis); + String exceptionMsg = response.hasEntity() ? response.getEntity(String.class) : null; + LOG.warn("Received 404 error code with body:[" + exceptionMsg + "], Ignoring"); + } else { + RESTResponse resp = RESTResponse.fromClientResponse(response); + LOG.warn("Error getting UserStore. secureMode=" + isSecureMode + ", user=" + user + ", response=" + resp); + ret = null; + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerUserStoreRefreher.getUserStoreIfUpdated(" + lastKnownUserStoreVersion + ", " + lastActivationTimeInMillis + "): "); + } + + return ret; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/contextenricher/RangerUserStoreRetriever.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/contextenricher/RangerUserStoreRetriever.java new file mode 100644 index 00000000000..94d4d250414 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/contextenricher/RangerUserStoreRetriever.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.contextenricher; + +import org.apache.atlas.authorization.hadoop.config.RangerPluginConfig; +import org.apache.atlas.plugin.model.RangerServiceDef; +import org.apache.atlas.plugin.policyengine.RangerPluginContext; +import org.apache.atlas.plugin.util.RangerUserStore; + +import java.util.Map; + +public abstract class RangerUserStoreRetriever { + + protected String serviceName; + protected RangerServiceDef serviceDef; + protected String appId; + protected RangerPluginConfig pluginConfig; + protected RangerPluginContext pluginContext; + + public abstract void init(Map options); + + public abstract RangerUserStore retrieveUserStoreInfo(long lastKnownVersion, long lastActivationTimeInMillis) throws Exception; + + public String getServiceName() { + return serviceName; + } + + public void setServiceName(String serviceName) { + this.serviceName = serviceName; + } + + public RangerServiceDef getServiceDef() { + return serviceDef; + } + + public void setServiceDef(RangerServiceDef serviceDef) { + this.serviceDef = serviceDef; + } + + public String getAppId() { + return appId; + } + + public void setAppId(String appId) { + this.appId = appId; + } + + public void setPluginConfig(RangerPluginConfig pluginConfig) { this.pluginConfig = pluginConfig; } + + public RangerPluginContext getPluginContext() { + return pluginContext; + } + + public void setPluginContext(RangerPluginContext pluginContext) { + this.pluginContext = pluginContext; + } +} \ No newline at end of file diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/errors/ValidationErrorCode.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/errors/ValidationErrorCode.java new file mode 100644 index 00000000000..3bcaf7332d4 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/errors/ValidationErrorCode.java @@ -0,0 +1,171 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.errors; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import java.text.MessageFormat; +import java.util.Arrays; + +public enum ValidationErrorCode { + // SERVICE VALIDATION + SERVICE_VALIDATION_ERR_UNSUPPORTED_ACTION(1001, "Internal error: unsupported action[{0}]; isValid(Long) is only supported for DELETE"), + SERVICE_VALIDATION_ERR_MISSING_FIELD(1002, "Internal error: missing field[{0}]"), + SERVICE_VALIDATION_ERR_NULL_SERVICE_OBJECT(1003, "Internal error: service object passed in was null"), + SERVICE_VALIDATION_ERR_EMPTY_SERVICE_ID(1004, "Internal error: service id was null/empty/blank"), + SERVICE_VALIDATION_ERR_INVALID_SERVICE_ID(1005, "No service found for id [{0}]"), + SERVICE_VALIDATION_ERR_INVALID_SERVICE_NAME(1006, "Missing service name"), + SERVICE_VALIDATION_ERR_SERVICE_NAME_CONFICT(1007, "Duplicate service name: name=[{0}]"), + SERVICE_VALIDATION_ERR_SERVICE_DISPLAY_NAME_CONFICT(3051,"Display name [{0}] is already used by service [{1}]"), + SERVICE_VALIDATION_ERR_SPECIAL_CHARACTERS_SERVICE_NAME(3031, "Invalid service name=[{0}]. It should not be longer than 256 characters and special characters are not allowed (except underscore and hyphen)"), + SERVICE_VALIDATION_ERR_SPECIAL_CHARACTERS_SERVICE_DISPLAY_NAME(3050, "Invalid service display name [{0}]. It should not be longer than 256 characters, should not start with space, and should not include special characters (except underscore, hyphen and space)"), + SERVICE_VALIDATION_ERR_ID_NAME_CONFLICT(1008, "Duplicate service name: name=[{0}], id=[{1}]"), + SERVICE_VALIDATION_ERR_MISSING_SERVICE_DEF(1009, "Missing service def"), + SERVICE_VALIDATION_ERR_INVALID_SERVICE_DEF(1010, "Service def not found: service-def-name=[{0}]"), + SERVICE_VALIDATION_ERR_REQUIRED_PARM_MISSING(1011, "Missing required configuration parameter(s): missing parameters={0}"), + + // SERVICE-DEF VALIDATION + SERVICE_DEF_VALIDATION_ERR_UNSUPPORTED_ACTION(2001, "Internal error: unsupported action[{0}]; isValid(Long) is only supported for DELETE"), + SERVICE_DEF_VALIDATION_ERR_MISSING_FIELD(2002, "Internal error: missing field[{0}]"), + SERVICE_DEF_VALIDATION_ERR_NULL_SERVICE_DEF_OBJECT(2003, "Internal error: service def object passed in was null"), + SERVICE_DEF_VALIDATION_ERR_EMPTY_SERVICE_DEF_ID(2004, "Internal error: service def id was null/empty/blank"), + SERVICE_DEF_VALIDATION_ERR_INVALID_SERVICE_DEF_ID(2005, "No service def found for id [{0}]"), + SERVICE_DEF_VALIDATION_ERR_INVALID_SERVICE_DEF_NAME(2006, "Service def name[{0}] was null/empty/blank"), + SERVICE_DEF_VALIDATION_ERR_INVALID_SERVICE_DEF_DISPLAY_NAME(2024, "Service def display name[{0}] was null/empty/blank"), + SERVICE_DEF_VALIDATION_ERR_SERVICE_DEF_NAME_CONFICT(2007, "service def with the name[{0}] already exists"), + SERVICE_DEF_VALIDATION_ERR_SERVICE_DEF__DISPLAY_NAME_CONFICT(2025, "Display name [{0}] is already used by service def [{1}]"), + SERVICE_DEF_VALIDATION_ERR_ID_NAME_CONFLICT(2008, "id/name conflict: another service def already exists with name[{0}], its id is [{1}]"), + SERVICE_DEF_VALIDATION_ERR_IMPLIED_GRANT_UNKNOWN_ACCESS_TYPE(2009, "implied grant[{0}] contains an unknown access types[{1}]"), + SERVICE_DEF_VALIDATION_ERR_IMPLIED_GRANT_IMPLIES_ITSELF(2010, "implied grants list [{0}] for access type[{1}] contains itself"), + SERVICE_DEF_VALIDATION_ERR_POLICY_CONDITION_NULL_EVALUATOR(2011, "evaluator on policy condition definition[{0}] was null/empty!"), + SERVICE_DEF_VALIDATION_ERR_CONFIG_DEF_UNKNOWN_ENUM(2012, "subtype[{0}] of service def config[{1}] was not among defined enums[{2}]"), + SERVICE_DEF_VALIDATION_ERR_CONFIG_DEF_UNKNOWN_ENUM_VALUE(2013, "default value[{0}] of service def config[{1}] was not among the valid values[{2}] of enums[{3}]"), + SERVICE_DEF_VALIDATION_ERR_CONFIG_DEF_MISSING_TYPE(2014, "type of service def config[{0}] was null/empty"), + SERVICE_DEF_VALIDATION_ERR_CONFIG_DEF_INVALID_TYPE(2015, "type[{0}] of service def config[{1}] is not among valid types: {2}"), + SERVICE_DEF_VALIDATION_ERR_RESOURCE_GRAPH_INVALID(2016, "Resource graph implied by various resources, e.g. parent value is invalid. Valid graph must forest (union of disjoint trees)."), + SERVICE_DEF_VALIDATION_ERR_ENUM_DEF_NULL_OBJECT(2017, "Internal error: An enum def in enums collection is null"), + SERVICE_DEF_VALIDATION_ERR_ENUM_DEF_NO_VALUES(2018, "enum [{0}] does not have any elements"), + SERVICE_DEF_VALIDATION_ERR_ENUM_DEF_INVALID_DEFAULT_INDEX(2019, "default index[{0}] for enum [{1}] is invalid"), + SERVICE_DEF_VALIDATION_ERR_ENUM_DEF_NULL_ENUM_ELEMENT(2020, "An enum element in enum element collection of enum [{0}] is null"), + SERVICE_DEF_VALIDATION_ERR_INVALID_SERVICE_RESOURCE_LEVELS(2021, "Resource-def levels are not in increasing order in an hierarchy"), + SERVICE_DEF_VALIDATION_ERR_NOT_LOWERCASE_NAME(2022, "{0}:[{1}] Invalid resource name. Resource name should consist of only lowercase, hyphen or underscore characters"), + SERVICE_DEF_VALIDATION_ERR_INVALID_MANADORY_VALUE_FOR_SERVICE_RESOURCE(2023, "{0} cannot be mandatory because {1}(parent) is not mandatory"), + + // POLICY VALIDATION + POLICY_VALIDATION_ERR_UNSUPPORTED_ACTION(3001, "Internal error: method signature isValid(Long) is only supported for DELETE"), + POLICY_VALIDATION_ERR_MISSING_FIELD(3002, "Internal error: missing field[{0}]"), + POLICY_VALIDATION_ERR_NULL_POLICY_OBJECT(3003, "Internal error: policy object passed in was null"), + POLICY_VALIDATION_ERR_INVALID_POLICY_ID(3004, "No policy found for id[{0}]"), + POLICY_VALIDATION_ERR_POLICY_NAME_MULTIPLE_POLICIES_WITH_SAME_NAME(3005, "Internal error: multiple policies found with the name[{0}]"), + POLICY_VALIDATION_ERR_POLICY_NAME_CONFLICT(3006, "Another policy already exists for this name: policy-id=[{0}], service=[{1}]"), + POLICY_VALIDATION_ERR_INVALID_SERVICE_NAME(3007, "no service found with name[{0}]"), + POLICY_VALIDATION_ERR_MISSING_POLICY_ITEMS(3008, "at least one policy item must be specified if audit isn't enabled"), + POLICY_VALIDATION_ERR_MISSING_SERVICE_DEF(3009, "Internal error: Service def[{0}] of policy's service[{1}] does not exist!"), + POLICY_VALIDATION_ERR_DUPLICATE_POLICY_RESOURCE(3010, "Another policy already exists for matching resource: policy-name=[{0}], service=[{1}]"), + POLICY_VALIDATION_ERR_INVALID_RESOURCE_NO_COMPATIBLE_HIERARCHY(3011, "Invalid resources specified. {0} policy can specify values for one of the following resource sets: {1}"), + POLICY_VALIDATION_ERR_INVALID_RESOURCE_MISSING_MANDATORY(3012, "Invalid resources specified. {0} policy must specify values for one of the following resource sets: {1}"), + POLICY_VALIDATION_ERR_NULL_RESOURCE_DEF(3013, "Internal error: a resource-def on resource def collection of service-def[{0}] was null"), + POLICY_VALIDATION_ERR_MISSING_RESOURCE_DEF_NAME(3014, "Internal error: name of a resource-def on resource def collection of service-def[{0}] was null"), + POLICY_VALIDATION_ERR_EXCLUDES_NOT_SUPPORTED(3015, "Excludes option not supported: resource-name=[{0}]"), + POLICY_VALIDATION_ERR_EXCLUDES_REQUIRES_ADMIN(3016, "Insufficient permissions to create excludes policy"), + POLICY_VALIDATION_ERR_RECURSIVE_NOT_SUPPORTED(3017, "Recursive option not supported: resource-name=[{0}]."), + POLICY_VALIDATION_ERR_INVALID_RESOURCE_VALUE_REGEX(3018, "Invalid resource specified. A value of [{0}] is not valid for resource [{1}]"), + POLICY_VALIDATION_ERR_NULL_POLICY_ITEM(3019, "policy items object was null"), + POLICY_VALIDATION_ERR_MISSING_USER_AND_GROUPS(3020, "All of users, user-groups and roles collections on the policy item were null/empty"), + POLICY_VALIDATION_ERR_NULL_POLICY_ITEM_ACCESS(3021, "policy items access object was null"), + POLICY_VALIDATION_ERR_POLICY_ITEM_ACCESS_TYPE_INVALID(3022, "Invalid access type: access type=[{0}], valid access types=[{1}]"), + POLICY_VALIDATION_ERR_POLICY_ITEM_ACCESS_TYPE_DENY(3023, "Currently deny access types are not supported. Access type is set to deny."), + POLICY_VALIDATION_ERR_INVALID_RESOURCE_NO_COMPATIBLE_HIERARCHY_SINGLE(3024, "Invalid resources specified. {0} policy can specify values for the following resources: {1}"), + POLICY_VALIDATION_ERR_INVALID_RESOURCE_MISSING_MANDATORY_SINGLE(3025, "Invalid resources specified. {0} policy must specify values for the following resources: {1}"), + POLICY_VALIDATION_ERR_MISSING_RESOURCE_LIST(3026, "Resource list was empty or contains null. At least one resource must be specified"), + POLICY_VALIDATION_ERR_POLICY_UPDATE_MOVE_SERVICE_NOT_ALLOWED(3027, "attempt to move policy id={0} from service={1} to service={2} is not allowed"), + POLICY_VALIDATION_ERR_POLICY_TYPE_CHANGE_NOT_ALLOWED(3028, "attempt to change type of policy id={0} from type={1} to type={2} is not allowed"), + POLICY_VALIDATION_ERR_POLICY_INVALID_VALIDITY_SCHEDULE(3029, "Invalid validity schedule specification"), + POLICY_VALIDATION_ERR_POLICY_INVALID_PRIORITY(3030, "Invalid priority value"), + POLICY_VALIDATION_ERR_UPDATE_ZONE_NAME_NOT_ALLOWED(3032, "Update of Zone name from={0} to={1} in policy is not supported"), + POLICY_VALIDATION_ERR_NONEXISTANT_ZONE_NAME(3033, "Non-existent Zone name={0} in policy create"), + POLICY_VALIDATION_ERR_SERVICE_NOT_ASSOCIATED_TO_ZONE(3048, "Service name = {0} is not associated to Zone name = {1}"), + POLICY_VALIDATION_ERR_UNSUPPORTED_POLICY_ITEM_TYPE(3049, "Deny or deny-exceptions are not supported if policy has isDenyAllElse flag set to true"), + POLICY_VALIDATION_ERR_INVALID_SERVICE_TYPE(4009," Invalid service type [{0}] provided for service [{1}]"), + + // SECURITY_ZONE Validations + SECURITY_ZONE_VALIDATION_ERR_UNSUPPORTED_ACTION(3034, "Internal error: unsupported action[{0}]; isValid() is only supported for DELETE"), + SECURITY_ZONE_VALIDATION_ERR_MISSING_FIELD(3035, "Internal error: missing field[{0}]"), + SECURITY_ZONE_VALIDATION_ERR_ZONE_NAME_CONFLICT(3036, "Another security zone already exists for this name: zone-id=[{0}]]"), + SECURITY_ZONE_VALIDATION_ERR_INVALID_ZONE_ID(3037, "No security zone found for [{0}]"), + SECURITY_ZONE_VALIDATION_ERR_MISSING_USER_AND_GROUPS(3038, "both users and user-groups collections for the security zone were null/empty"), + SECURITY_ZONE_VALIDATION_ERR_MISSING_RESOURCES(3039, "No resources specified for service [{0}]"), + SECURITY_ZONE_VALIDATION_ERR_INVALID_SERVICE_NAME(3040, "Invalid service [{0}]"), + SECURITY_ZONE_VALIDATION_ERR_INVALID_SERVICE_TYPE(3041, "Invalid service-type [{0}]"), + SECURITY_ZONE_VALIDATION_ERR_INVALID_RESOURCE_HIERARCHY(3042, "Invalid resource hierarchy specified for service:[{0}], resource-hierarchy:[{1}]"), + SECURITY_ZONE_VALIDATION_ERR_ALL_WILDCARD_RESOURCE_VALUES(3043, "All wildcard values specified for resources for service:[{0}]"), + SECURITY_ZONE_VALIDATION_ERR_MISSING_SERVICES(3044, "No services specified for security-zone:[{0}]"), + SECURITY_ZONE_VALIDATION_ERR_INTERNAL_ERROR(3045, "Internal Error:[{0}]"), + SECURITY_ZONE_VALIDATION_ERR_ZONE_RESOURCE_CONFLICT(3046, "Multiple zones:[{0}] match resource:[{1}]"), + SECURITY_ZONE_VALIDATION_ERR_UNEXPECTED_RESOURCES(3047, "Tag service [{0}], with non-empty resources, is associated with security zone"), + + //RANGER ROLE Validations + ROLE_VALIDATION_ERR_NULL_RANGER_ROLE_OBJECT(4001, "Internal error: RangerRole object passed in was null"), + ROLE_VALIDATION_ERR_MISSING_FIELD(4002, "Internal error: missing field[{0}]"), + ROLE_VALIDATION_ERR_NULL_RANGER_ROLE_NAME(4003, "Internal error: RangerRole name passed in was null/empty"), + ROLE_VALIDATION_ERR_MISSING_USER_OR_GROUPS_OR_ROLES(4004, "RangerRole should contain atleast one user or group or role"), + ROLE_VALIDATION_ERR_ROLE_NAME_CONFLICT(4005, "Another RangerRole already exists for this name: Role =[{0}]]"), + ROLE_VALIDATION_ERR_INVALID_ROLE_ID(4006, "No RangerRole found for id[{0}]"), + ROLE_VALIDATION_ERR_INVALID_ROLE_NAME(4007, "No RangerRole found for name[{0}]"), + ROLE_VALIDATION_ERR_UNSUPPORTED_ACTION(4008, "Internal error: method signature isValid(Long) is only supported for DELETE"), + + + ; + + + private static final Log LOG = LogFactory.getLog(ValidationErrorCode.class); + + final int _errorCode; + final String _template; + + ValidationErrorCode(int errorCode, String template) { + _errorCode = errorCode; + _template = template; + } + + public String getMessage(Object... items) { + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("<== ValidationErrorCode.getMessage(%s)", Arrays.toString(items))); + } + + MessageFormat mf = new MessageFormat(_template); + String result = mf.format(items); + + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("<== ValidationErrorCode.getMessage(%s): %s", Arrays.toString(items), result)); + } + return result; + } + + public int getErrorCode() { + return _errorCode; + } + + @Override + public String toString() { + return String.format("Code: %d, template: %s", _errorCode, _template); + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/geo/BinarySearchTree.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/geo/BinarySearchTree.java new file mode 100644 index 00000000000..bfd49526c9d --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/geo/BinarySearchTree.java @@ -0,0 +1,222 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.geo; + +public class BinarySearchTree & RangeChecker, V> { + private Node root; + private int size; + + public BinarySearchTree() { + root = null; + } + + public void insert(final T value) { + Node node = new Node(value); + + if (root == null) { + root = node; + return; + } + + Node focusNode = root; + Node parent; + + while (true) { + parent = focusNode; + + int comparison = focusNode.getValue().compareTo(value); + if (comparison == 0) { + return; + } + if (comparison < 0) { + focusNode = focusNode.getRight(); + if (focusNode == null) { + parent.setRight(node); + return; + } + } else { + focusNode = focusNode.getLeft(); + if (focusNode == null) { + parent.setLeft(node); + return; + } + } + } + } + + public T find(final V value) { + Node focusNode = root; + + int rangeCheck; + + while (focusNode != null) { + rangeCheck = focusNode.getValue().compareToRange(value); + if (rangeCheck == 0) { + break; + } else if (rangeCheck < 0) { + focusNode = focusNode.getRight(); + } else { + focusNode = focusNode.getLeft(); + } + } + + return focusNode == null ? null : focusNode.getValue(); + } + + final public void preOrderTraverseTree(final ValueProcessor processor) { + preOrderTraverseTree(getRoot(), processor); + } + + final public void inOrderTraverseTree(final ValueProcessor processor) { + inOrderTraverseTree(getRoot(), processor); + } + + Node getRoot() { + return root; + } + + void setRoot(final Node newRoot) { + root = newRoot; + } + + void rebalance() { + Node dummy = new Node(null); + dummy.setRight(root); + + setRoot(dummy); + + degenerate(); + reconstruct(); + + setRoot(getRoot().getRight()); + } + + final void inOrderTraverseTree(final Node focusNode, final ValueProcessor processor) { + if (focusNode != null) { + inOrderTraverseTree(focusNode.getLeft(), processor); + processor.process(focusNode.getValue()); + inOrderTraverseTree(focusNode.getRight(), processor); + } + } + + final void preOrderTraverseTree(final Node focusNode, final ValueProcessor processor) { + if (focusNode != null) { + processor.process(focusNode.getValue()); + + preOrderTraverseTree(focusNode.getLeft(), processor); + preOrderTraverseTree(focusNode.getRight(), processor); + } + } + + private void degenerate() { + + Node remainder, temp, sentinel; + + sentinel = getRoot(); + remainder = sentinel.getRight(); + + size = 0; + + while (remainder != null) { + if (remainder.getLeft() == null) { + sentinel = remainder; + remainder = remainder.getRight(); + size++; + } else { + temp = remainder.getLeft(); + remainder.setLeft(temp.getRight()); + temp.setRight(remainder); + remainder = temp; + sentinel.setRight(temp); + } + } + } + + private void reconstruct() { + + int sz = size; + Node node = getRoot(); + + int fullCount = fullSize(sz); + rotateLeft(node, sz - fullCount); + + for (sz = fullCount; sz > 1; sz /= 2) { + rotateLeft(node, sz / 2); + } + } + + private void rotateLeft(Node root, final int count) { + if (root == null) return; + + Node scanner = root; + + for (int i = 0; i < count; i++) { + //Leftward rotation + Node child = scanner.getRight(); + scanner.setRight(child.getRight()); + scanner = child.getRight(); + child.setRight(scanner.getLeft()); + scanner.setLeft(child); + } + } + + private int fullSize(final int size) { + // Full portion of a complete tree + int ret = 1; + while (ret <= size) { + ret = 2*ret + 1; + } + return ret / 2; + } + + static class Node { + private T value; + private Node left; + private Node right; + + public Node(final T value) { + this.value = value; + } + + public T getValue() { + return value; + } + + public void setValue(final T value) { + this.value = value; + } + + public Node getLeft() { + return left; + } + + public void setLeft(final Node left) { + this.left = left; + } + + public Node getRight() { + return right; + } + + public void setRight(final Node right) { + this.right = right; + } + } +} \ No newline at end of file diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/geo/GeolocationMetadata.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/geo/GeolocationMetadata.java new file mode 100644 index 00000000000..e8d64495ff7 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/geo/GeolocationMetadata.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.geo; + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +public class GeolocationMetadata { + private static final Log LOG = LogFactory.getLog(GeolocationMetadata.class); + + private String[] locationDataItemNames = new String[0]; + + public static GeolocationMetadata create(String fields[], int index) { + GeolocationMetadata ret = null; + + if (fields.length > 2) { + String[] metadataNames = new String[fields.length-2]; + + for (int i = 2; i < fields.length; i++) { + metadataNames[i-2] = fields[i]; + } + ret = new GeolocationMetadata(metadataNames); + } else { + LOG.error("GeolocationMetadata.createMetadata() - Not enough fields specified, need {start, end, location} at " + index); + } + + return ret; + } + + GeolocationMetadata() {} + + GeolocationMetadata(final String[] locationDataItemNames) { + this.locationDataItemNames = locationDataItemNames; + } + + public int getDataItemNameIndex(final String dataItemName) { + int ret = -1; + + if (!StringUtils.isBlank(dataItemName)) { + for (int i = 0; i < locationDataItemNames.length; i++) { + if (locationDataItemNames[i].equals(dataItemName)) { + ret = i; + break; + } + } + } + + return ret; + } + + public String[] getLocationDataItemNames() { + return locationDataItemNames; + } + + @Override + public String toString( ) { + StringBuilder sb = new StringBuilder(); + + toStringDump(sb); + + return sb.toString(); + } + + private StringBuilder toStringDump(StringBuilder sb) { + sb.append("FROM_IP,TO_IP,"); + + for (String locationDataItemName : locationDataItemNames) { + sb.append(locationDataItemName).append(", "); + } + + return sb; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/geo/RangeChecker.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/geo/RangeChecker.java new file mode 100644 index 00000000000..0cf9cb16e52 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/geo/RangeChecker.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.geo; + +public interface RangeChecker { + int compareToRange(V value); +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/geo/RangerGeolocationData.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/geo/RangerGeolocationData.java new file mode 100644 index 00000000000..db600a1768b --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/geo/RangerGeolocationData.java @@ -0,0 +1,222 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.geo; + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.Objects; + +public class RangerGeolocationData implements Comparable, RangeChecker { + private static final Log LOG = LogFactory.getLog(RangerGeolocationData.class); + + private static final Character IPSegmentsSeparator = '.'; + + private final long fromIPAddress; + private final long toIPAddress; + private final String[] locationData; + private int hash; + + public static RangerGeolocationData create(String fields[], int index, boolean useDotFormat) { + + RangerGeolocationData data = null; + + if (fields.length > 2) { + String startAddress = fields[0]; + String endAddress = fields[1]; + + if (RangerGeolocationData.validateAsIP(startAddress, useDotFormat) && RangerGeolocationData.validateAsIP(endAddress, useDotFormat)) { + + long startIP, endIP; + if (!useDotFormat) { + startAddress = RangerGeolocationData.unsignedIntToIPAddress(Long.valueOf(startAddress)); + endAddress = RangerGeolocationData.unsignedIntToIPAddress(Long.valueOf(endAddress)); + } + startIP = RangerGeolocationData.ipAddressToLong(startAddress); + endIP = RangerGeolocationData.ipAddressToLong(endAddress); + + if ((endIP - startIP) >= 0) { + + String[] locationData = new String[fields.length-2]; + for (int i = 2; i < fields.length; i++) { + locationData[i-2] = fields[i]; + } + data = new RangerGeolocationData(startIP, endIP, locationData); + } + } + + } else { + LOG.error("GeolocationMetadata.createMetadata() - Not enough fields specified, need {start, end, location} at " + index); + } + return data; + } + + private RangerGeolocationData(final long fromIPAddress, final long toIPAddress, final String[] locationData) { + this.fromIPAddress = fromIPAddress; + this.toIPAddress = toIPAddress; + this.locationData = locationData; + } + + public String[] getLocationData() { + return locationData; + } + + @Override + public int compareTo(final RangerGeolocationData other) { + int ret = (other == null) ? 1 : 0; + if (ret == 0) { + ret = Long.compare(fromIPAddress, other.fromIPAddress); + if (ret == 0) { + ret = Long.compare(toIPAddress, other.toIPAddress); + if (ret == 0) { + ret = Integer.compare(locationData.length, other.locationData.length); + for (int i = 0; ret == 0 && i < locationData.length; i++) { + ret = stringCompareTo(locationData[i], other.locationData[i]); + } + } + } + } + return ret; + } + + @Override + public boolean equals(Object other) { + boolean ret = false; + if (other != null && (other instanceof RangerGeolocationData)) { + ret = this == other || compareTo((RangerGeolocationData) other) == 0; + } + return ret; + } + + @Override + public int hashCode() { + if (hash == 0) { + hash = Objects.hash(fromIPAddress, toIPAddress, locationData); + } + return hash; + } + + @Override + public int compareToRange(final Long ip) { + int ret = Long.compare(fromIPAddress, ip.longValue()); + + if (ret < 0) { + ret = Long.compare(toIPAddress, ip.longValue()); + if (ret > 0) { + ret = 0; + } + } + + return ret; + } + + public static long ipAddressToLong(final String ipAddress) { + + long ret = 0L; + + try { + byte[] bytes = InetAddress.getByName(ipAddress).getAddress(); + + if (bytes != null && bytes.length <= 4) { + for (int i = 0; i < bytes.length; i++) { + int val = bytes[i] < 0 ? (256 + bytes[i]) : bytes[i]; + ret += (val << (8 * (3 - i))); + } + } + } + catch (UnknownHostException exception) { + LOG.error("RangerGeolocationData.ipAddressToLong() - Invalid IP address " + ipAddress); + } + + return ret; + } + + public static String unsignedIntToIPAddress(final long val) { + if (val <= 0) { + return ""; + } + long remaining = val; + String segments[] = new String[4]; + for (int i = 3; i >= 0; i--) { + long segment = remaining % 0x100; + remaining = remaining / 0x100; + segments[i] = String.valueOf(segment); + } + return StringUtils.join(segments, IPSegmentsSeparator); + } + + public static boolean validateAsIP(String ipAddress, boolean ipInDotNotation) { + if (!ipInDotNotation) { + return StringUtils.isNumeric(ipAddress); + } + + boolean ret = false; + + try { + // Only to validate to see if ipAddress is in correct format + InetAddress.getByName(ipAddress).getAddress(); + ret = true; + } + catch(UnknownHostException exception) { + LOG.error("RangerGeolocationData.validateAsIP() - Invalid address " + ipAddress); + } + + return ret; + } + + private static int stringCompareTo(String str1, String str2) { + if(str1 == str2) { + return 0; + } else if(str1 == null) { + return -1; + } else if(str2 == null) { + return 1; + } else { + return str1.compareTo(str2); + } + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + + toString(sb); + + return sb.toString(); + } + + private StringBuilder toString(StringBuilder sb) { + sb.append("{") + .append("from=") + .append(RangerGeolocationData.unsignedIntToIPAddress(fromIPAddress)) + .append(", to=") + .append(RangerGeolocationData.unsignedIntToIPAddress(toIPAddress)) + .append(", location={"); + for (String data : locationData) { + sb.append(data).append(", "); + } + sb.append("}"); + sb.append("}"); + return sb; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/geo/RangerGeolocationDatabase.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/geo/RangerGeolocationDatabase.java new file mode 100644 index 00000000000..c792397c311 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/geo/RangerGeolocationDatabase.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.geo; + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +public class RangerGeolocationDatabase { + private static final Log LOG = LogFactory.getLog(RangerGeolocationDatabase.class); + + private BinarySearchTree data = new BinarySearchTree<>(); + + private GeolocationMetadata metadata = new GeolocationMetadata(); + + public String getValue(final RangerGeolocationData geolocationData, final String attributeName) { + String value = null; + int index = -1; + + if (geolocationData != null && StringUtils.isNotBlank(attributeName)) { + if ((index = getMetadata().getDataItemNameIndex(attributeName)) != -1) { + String[] attrValues = geolocationData.getLocationData(); + if (index < attrValues.length) { + value = attrValues[index]; + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("RangerGeolocationDatabase.getValue() - No value specified attribute-name:" + attributeName); + } + } + } else { + LOG.error("RangerGeolocationDatabase.getValue() - RangerGeolocationDatabase not initialized or Invalid attribute-name:" + attributeName); + } + } + + return value; + } + + public RangerGeolocationData find(final String ipAddressStr) { + RangerGeolocationData ret = null; + + if (StringUtils.isNotBlank(ipAddressStr) && RangerGeolocationData.validateAsIP(ipAddressStr, true)) { + ret = data.find(RangerGeolocationData.ipAddressToLong(ipAddressStr)); + } + return ret; + } + + public void optimize() { + long start = 0L, end = 0L; + + start = System.currentTimeMillis(); + data.rebalance(); + end = System.currentTimeMillis(); + + if (LOG.isDebugEnabled()) { + LOG.debug("RangerGeolocationDatabase.optimize() - Time taken for optimizing database = " + (end - start) + " milliseconds"); + } + } + + public void setData(final BinarySearchTree dataArg) { data = dataArg != null ? dataArg : new BinarySearchTree();} + + public void setMetadata(final GeolocationMetadata metadataArg) { metadata = metadataArg != null ? metadataArg : new GeolocationMetadata();} + + public GeolocationMetadata getMetadata() { return metadata; } + + public BinarySearchTree getData() { return data; } + + public void dump(ValuePrinter processor) { + + BinarySearchTree geoDatabase = getData(); + GeolocationMetadata metadata = getMetadata(); + processor.build(); + + processor.print("#================== Geolocation metadata =================="); + processor.print(metadata.toString()); + + processor.print("#================== Dump of geoDatabase - START =================="); + geoDatabase.preOrderTraverseTree(processor); + processor.print("#================== Dump of geoDatabase - END =================="); + + processor.close(); + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/geo/ValuePrinter.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/geo/ValuePrinter.java new file mode 100644 index 00000000000..fe23f53bb65 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/geo/ValuePrinter.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.geo; + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import java.io.BufferedWriter; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.Writer; + +class ValuePrinter implements ValueProcessor { + private static final Log LOG = LogFactory.getLog(ValuePrinter.class); + + private Writer writer; + private String fileName; + + ValuePrinter(String fileName) { + this.fileName = fileName; + } + + public T process(T value) { + + if (value != null) { + if (writer == null) { + LOG.error("ValuePrinter.process() -" + value); + } else { + try { + writer.write(value.toString()); + writer.write('\n'); + } catch (IOException exception) { + LOG.error("ValuePrinter.process() - Cannot write '" + value + "' to " + fileName); + } + } + } + return value; + } + + public void print(String str) { + if (writer == null) { + LOG.error("ValuePrinter.print() -" + str); + } else { + try { + writer.write(str); + writer.write('\n'); + } catch (IOException exception) { + LOG.error("ValuePrinter.print() - Cannot write '" + str + "' to " + fileName ); + } + } + } + + void build() { + try { + if (StringUtils.isNotBlank(fileName)) { + writer = new BufferedWriter(new OutputStreamWriter( + new FileOutputStream(fileName))); + } + } catch(IOException exception) { + LOG.error("ValuePrinter.build() - Cannot open " + fileName + " for writing"); + } + } + + void close() { + try { + if (writer != null) { + writer.close(); + } + } catch (IOException exception) { + LOG.error("ValuePrinter.close() - Cannot close " + fileName); + } + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/geo/ValueProcessor.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/geo/ValueProcessor.java new file mode 100644 index 00000000000..713ffb3e25d --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/geo/ValueProcessor.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.geo; + +public interface ValueProcessor { + T process(T value); +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/AuditFilter.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/AuditFilter.java new file mode 100644 index 00000000000..4a1d9ecb766 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/AuditFilter.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.model; + +import org.apache.htrace.shaded.fasterxml.jackson.annotation.JsonInclude; +import org.apache.atlas.plugin.model.RangerPolicy.RangerPolicyResource; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.List; +import java.util.Map; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class AuditFilter { + public enum AccessResult { DENIED, ALLOWED, NOT_DETERMINED } + + private AccessResult accessResult; + private Map resources; + private List accessTypes; + private List actions; + private List users; + private List groups; + private List roles; + private Boolean isAudited; + + public AuditFilter() { } + + public AccessResult getAccessResult() { + return accessResult; + } + + public void setAccessResult(AccessResult accessResult) { + this.accessResult = accessResult; + } + + public Map getResources() { + return resources; + } + + public void setResources(Map resources) { + this.resources = resources; + } + + public List getAccessTypes() { + return accessTypes; + } + + public void setAccessTypes(List accessTypes) { + this.accessTypes = accessTypes; + } + + public List getActions() { + return actions; + } + + public void setActions(List actions) { + this.actions = actions; + } + + public List getUsers() { + return users; + } + + public void setUsers(List users) { + this.users = users; + } + + public List getGroups() { + return groups; + } + + public void setGroups(List groups) { + this.groups = groups; + } + + public List getRoles() { + return roles; + } + + public void setRoles(List roles) { + this.roles = roles; + } + + public Boolean getIsAudited() { + return isAudited; + } + + public void setAction(Boolean isAudited) { + this.isAudited = isAudited; + } + + @Override + public String toString() { + return "{accessResult=" + accessResult + + ", resources=" + resources + + ", accessTypes=" + accessTypes + + ", actions=" + actions + + ", users=" + users + + ", groups=" + groups + + ", roles=" + roles + + ", isAudited=" + isAudited + + "}"; + } +} \ No newline at end of file diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/GroupInfo.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/GroupInfo.java new file mode 100644 index 00000000000..ea5630744e7 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/GroupInfo.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.model; + +import org.apache.htrace.shaded.fasterxml.jackson.annotation.JsonInclude; +import org.apache.atlas.plugin.util.RangerUserStoreUtil; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.HashMap; +import java.util.Map; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class GroupInfo extends RangerBaseModelObject implements java.io.Serializable { + + private static final long serialVersionUID = 1L; + private String name; + private String description; + private Map otherAttributes; + + public GroupInfo() { + this(null, null, null); + } + + public GroupInfo(String name, String description, Map otherAttributes) { + setName(name); + setDescription(description); + setOtherAttributes(otherAttributes); + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public Map getOtherAttributes() { + return otherAttributes; + } + + public void setOtherAttributes(Map otherAttributes) { + this.otherAttributes = otherAttributes == null ? new HashMap<>() : otherAttributes; + } + + @Override + public String toString() { + return "{name=" + name + + ", description=" + description + + ", otherAttributes=" + RangerUserStoreUtil.getPrintableOptions(otherAttributes) + + "}"; + } + +} \ No newline at end of file diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerBaseModelObject.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerBaseModelObject.java new file mode 100644 index 00000000000..aba66614261 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerBaseModelObject.java @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.model; + +import org.apache.htrace.shaded.fasterxml.jackson.annotation.JsonInclude; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.Date; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class RangerBaseModelObject implements java.io.Serializable { + private static final long serialVersionUID = 1L; + + private Long id; + private String guid; + private Boolean isEnabled; + private String createdBy; + private String updatedBy; + private Date createTime; + private Date updateTime; + private Long version; + + public RangerBaseModelObject() { + setIsEnabled(null); + } + + public void updateFrom(RangerBaseModelObject other) { + setIsEnabled(other.getIsEnabled()); + } + + /** + * @return the id + */ + public Long getId() { + return id; + } + /** + * @param id the id to set + */ + public void setId(Long id) { + this.id = id; + } + /** + * @return the guid + */ + public String getGuid() { + return guid; + } + /** + * @param guid the guid to set + */ + public void setGuid(String guid) { + this.guid = guid; + } + /** + * @return the isEnabled + */ + public Boolean getIsEnabled() { + return isEnabled; + } + /** + * @param isEnabled the isEnabled to set + */ + public void setIsEnabled(Boolean isEnabled) { + this.isEnabled = isEnabled == null ? Boolean.TRUE : isEnabled; + } + /** + * @return the createdBy + */ + public String getCreatedBy() { + return createdBy; + } + /** + * @param createdBy the createdBy to set + */ + public void setCreatedBy(String createdBy) { + this.createdBy = createdBy; + } + /** + * @return the updatedBy + */ + public String getUpdatedBy() { + return updatedBy; + } + /** + * @param updatedBy the updatedBy to set + */ + public void setUpdatedBy(String updatedBy) { + this.updatedBy = updatedBy; + } + /** + * @return the createTime + */ + public Date getCreateTime() { + return createTime; + } + /** + * @param createTime the createTime to set + */ + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + /** + * @return the updateTime + */ + public Date getUpdateTime() { + return updateTime; + } + /** + * @param updateTime the updateTime to set + */ + public void setUpdateTime(Date updateTime) { + this.updateTime = updateTime; + } + /** + * @return the version + */ + public Long getVersion() { + return version; + } + /** + * @param version the version to set + */ + public void setVersion(Long version) { + this.version = version; + } + + @Override + public String toString( ) { + StringBuilder sb = new StringBuilder(); + + toString(sb); + + return sb.toString(); + } + + public StringBuilder toString(StringBuilder sb) { + sb.append("id={").append(id).append("} "); + sb.append("guid={").append(guid).append("} "); + sb.append("isEnabled={").append(isEnabled).append("} "); + sb.append("createdBy={").append(createdBy).append("} "); + sb.append("updatedBy={").append(updatedBy).append("} "); + sb.append("createTime={").append(createTime).append("} "); + sb.append("updateTime={").append(updateTime).append("} "); + sb.append("version={").append(version).append("} "); + + return sb; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerMetrics.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerMetrics.java new file mode 100644 index 00000000000..c6210df5e1e --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerMetrics.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.model; + + +import org.apache.htrace.shaded.fasterxml.jackson.annotation.JsonInclude; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.Map; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class RangerMetrics { + + private Map data; + + public RangerMetrics() { + setData(null); + } + public RangerMetrics(Map data) { + setData(data); + } + + public Map getData() { + return data; + } + + public void setData(Map data) { + this.data = data; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPluginInfo.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPluginInfo.java new file mode 100644 index 00000000000..0e671194537 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPluginInfo.java @@ -0,0 +1,452 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.model; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import org.apache.commons.lang.StringUtils; +import org.apache.htrace.shaded.fasterxml.jackson.annotation.JsonInclude; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import java.io.Serializable; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class RangerPluginInfo implements Serializable { + private static final long serialVersionUID = 1L; + + public static final int ENTITY_TYPE_POLICIES = 0; + public static final int ENTITY_TYPE_TAGS = 1; + public static final int ENTITY_TYPE_ROLES = 2; + public static final int ENTITY_TYPE_USERSTORE = 3; + + public static final String PLUGIN_INFO_POLICY_DOWNLOAD_TIME = "policyDownloadTime"; + public static final String PLUGIN_INFO_POLICY_DOWNLOADED_VERSION = "policyDownloadedVersion"; + public static final String PLUGIN_INFO_POLICY_ACTIVATION_TIME = "policyActivationTime"; + public static final String PLUGIN_INFO_POLICY_ACTIVE_VERSION = "policyActiveVersion"; + public static final String PLUGIN_INFO_TAG_DOWNLOAD_TIME = "tagDownloadTime"; + public static final String PLUGIN_INFO_TAG_DOWNLOADED_VERSION = "tagDownloadedVersion"; + public static final String PLUGIN_INFO_TAG_ACTIVATION_TIME = "tagActivationTime"; + public static final String PLUGIN_INFO_TAG_ACTIVE_VERSION = "tagActiveVersion"; + + public static final String PLUGIN_INFO_ROLE_DOWNLOAD_TIME = "roleDownloadTime"; + public static final String PLUGIN_INFO_ROLE_DOWNLOADED_VERSION = "roleDownloadedVersion"; + public static final String PLUGIN_INFO_ROLE_ACTIVATION_TIME = "roleActivationTime"; + public static final String PLUGIN_INFO_ROLE_ACTIVE_VERSION = "roleActiveVersion"; + + public static final String PLUGIN_INFO_USERSTORE_DOWNLOAD_TIME = "userstoreDownloadTime"; + public static final String PLUGIN_INFO_USERSTORE_DOWNLOADED_VERSION = "userstoreDownloadedVersion"; + public static final String PLUGIN_INFO_USERSTORE_ACTIVATION_TIME = "userstoreActivationTime"; + public static final String PLUGIN_INFO_USERSTORE_ACTIVE_VERSION = "userstoreActiveVersion"; + + public static final String RANGER_ADMIN_LAST_POLICY_UPDATE_TIME = "lastPolicyUpdateTime"; + public static final String RANGER_ADMIN_LATEST_POLICY_VERSION = "latestPolicyVersion"; + public static final String RANGER_ADMIN_LAST_TAG_UPDATE_TIME = "lastTagUpdateTime"; + public static final String RANGER_ADMIN_LATEST_TAG_VERSION = "latestTagVersion"; + + public static final String RANGER_ADMIN_CAPABILITIES = "adminCapabilities"; + public static final String PLUGIN_INFO_CAPABILITIES = "pluginCapabilities"; + + private Long id; + private Date createTime; + private Date updateTime; + + private String serviceName; + private String serviceDisplayName; + private String serviceType; + private String serviceTypeDisplayName; + private String hostName; + private String appType; + private String ipAddress; + private Map info; + + //FIXME UNUSED + public RangerPluginInfo(Long id, Date createTime, Date updateTime, String serviceName, String appType, String hostName, String ipAddress, Map info) { + super(); + + setId(id); + setCreateTime(createTime); + setUpdateTime(updateTime); + setServiceName(serviceName); + setAppType(appType); + setHostName(hostName); + setIpAddress(ipAddress); + setInfo(info); + } + + //FIXME UNUSED + public RangerPluginInfo() { + this(null, null, null, null, null, null, null, null); + } + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Date getCreateTime() { + return createTime; + } + + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + public Date getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(Date updateTime) { + this.updateTime = updateTime; + } + + public String getServiceType() { + return serviceType; + } + + public void setServiceType(String serviceType) { + this.serviceType = serviceType; + } + + public String getServiceTypeDisplayName() { + return serviceTypeDisplayName; + } + + public void setServiceTypeDisplayName(String serviceTypeDisplayName) { + this.serviceTypeDisplayName = serviceTypeDisplayName; + } + + public String getServiceName() { + return serviceName; + } + + public void setServiceName(String serviceName) { + this.serviceName = serviceName; + } + + public String getServiceDisplayName() { + return serviceDisplayName; + } + + public void setServiceDisplayName(String serviceDisplayName) { + this.serviceDisplayName = serviceDisplayName; + } + + public String getHostName() { + return hostName; + } + + public void setHostName(String hostName) { + this.hostName = hostName; + } + + public String getAppType() { + return appType; + } + + public void setAppType(String appType) { + this.appType = appType; + } + + public String getIpAddress() { + return ipAddress; + } + + public void setIpAddress(String ipAddress) { + this.ipAddress = ipAddress; + } + + public Map getInfo() { + return info; + } + + public void setInfo(Map info) { + this.info = info == null ? new HashMap() : info; + } + + @JsonIgnore + public void setPolicyDownloadTime(Long policyDownloadTime) { + getInfo().put(PLUGIN_INFO_POLICY_DOWNLOAD_TIME, policyDownloadTime == null ? null : Long.toString(policyDownloadTime)); + } + + @JsonIgnore + public Long getPolicyDownloadTime() { + String downloadTimeString = getInfo().get(PLUGIN_INFO_POLICY_DOWNLOAD_TIME); + return StringUtils.isNotBlank(downloadTimeString) ? Long.valueOf(downloadTimeString) : null; + } + + @JsonIgnore + public void setPolicyDownloadedVersion(Long policyDownloadedVersion) { + getInfo().put(PLUGIN_INFO_POLICY_DOWNLOADED_VERSION, policyDownloadedVersion == null ? null : Long.toString(policyDownloadedVersion)); + } + + @JsonIgnore + public Long getPolicyDownloadedVersion() { + String downloadedVersionString = getInfo().get(PLUGIN_INFO_POLICY_DOWNLOADED_VERSION); + return StringUtils.isNotBlank(downloadedVersionString) ? Long.valueOf(downloadedVersionString) : null; + } + + @JsonIgnore + public void setPolicyActivationTime(Long policyActivationTime) { + getInfo().put(PLUGIN_INFO_POLICY_ACTIVATION_TIME, policyActivationTime == null ? null : Long.toString(policyActivationTime)); + } + + @JsonIgnore + public Long getPolicyActivationTime() { + String activationTimeString = getInfo().get(PLUGIN_INFO_POLICY_ACTIVATION_TIME); + return StringUtils.isNotBlank(activationTimeString) ? Long.valueOf(activationTimeString) : null; + } + + @JsonIgnore + public void setPolicyActiveVersion(Long policyActiveVersion) { + getInfo().put(PLUGIN_INFO_POLICY_ACTIVE_VERSION, policyActiveVersion == null ? null : Long.toString(policyActiveVersion)); + } + + @JsonIgnore + public Long getPolicyActiveVersion() { + String activeVersionString = getInfo().get(PLUGIN_INFO_POLICY_ACTIVE_VERSION); + return StringUtils.isNotBlank(activeVersionString) ? Long.valueOf(activeVersionString) : null; + } + + @JsonIgnore + public void setTagDownloadTime(Long tagDownloadTime) { + getInfo().put(PLUGIN_INFO_TAG_DOWNLOAD_TIME, tagDownloadTime == null ? null : Long.toString(tagDownloadTime)); + } + + @JsonIgnore + public Long getTagDownloadTime() { + String downloadTimeString = getInfo().get(PLUGIN_INFO_TAG_DOWNLOAD_TIME); + return StringUtils.isNotBlank(downloadTimeString) ? Long.valueOf(downloadTimeString) : null; + } + + @JsonIgnore + public void setTagDownloadedVersion(Long tagDownloadedVersion) { + getInfo().put(PLUGIN_INFO_TAG_DOWNLOADED_VERSION, tagDownloadedVersion == null ? null : Long.toString(tagDownloadedVersion)); + } + + @JsonIgnore + public Long getTagDownloadedVersion() { + String downloadedVersion = getInfo().get(PLUGIN_INFO_TAG_DOWNLOADED_VERSION); + return StringUtils.isNotBlank(downloadedVersion) ? Long.valueOf(downloadedVersion) : null; + } + + @JsonIgnore + public void setTagActivationTime(Long tagActivationTime) { + getInfo().put(PLUGIN_INFO_TAG_ACTIVATION_TIME, tagActivationTime == null ? null : Long.toString(tagActivationTime)); + } + + @JsonIgnore + public Long getTagActivationTime() { + String activationTimeString = getInfo().get(PLUGIN_INFO_TAG_ACTIVATION_TIME); + return StringUtils.isNotBlank(activationTimeString) ? Long.valueOf(activationTimeString) : null; + } + + @JsonIgnore + public void setTagActiveVersion(Long tagActiveVersion) { + getInfo().put(PLUGIN_INFO_TAG_ACTIVE_VERSION, tagActiveVersion == null ? null : Long.toString(tagActiveVersion)); + } + + @JsonIgnore + public Long getTagActiveVersion() { + String activeVersionString = getInfo().get(PLUGIN_INFO_TAG_ACTIVE_VERSION); + return StringUtils.isNotBlank(activeVersionString) ? Long.valueOf(activeVersionString) : null; + } + + @JsonIgnore + public Long getLatestPolicyVersion() { + String latestPolicyVersionString = getInfo().get(RANGER_ADMIN_LATEST_POLICY_VERSION); + return StringUtils.isNotBlank(latestPolicyVersionString) ? Long.valueOf(latestPolicyVersionString) : null; + } + + @JsonIgnore + public Long getLastPolicyUpdateTime() { + String updateTimeString = getInfo().get(RANGER_ADMIN_LAST_POLICY_UPDATE_TIME); + return StringUtils.isNotBlank(updateTimeString) ? Long.valueOf(updateTimeString) : null; + } + + @JsonIgnore + public Long getLatestTagVersion() { + String latestTagVersionString = getInfo().get(RANGER_ADMIN_LATEST_TAG_VERSION); + return StringUtils.isNotBlank(latestTagVersionString) ? Long.valueOf(latestTagVersionString) : null; + } + + @JsonIgnore + public Long getLastTagUpdateTime() { + String updateTimeString = getInfo().get(RANGER_ADMIN_LAST_TAG_UPDATE_TIME); + return StringUtils.isNotBlank(updateTimeString) ? Long.valueOf(updateTimeString) : null; + } + + @JsonIgnore + public void setRoleDownloadTime(Long roleDownloadTime) { + getInfo().put(PLUGIN_INFO_ROLE_DOWNLOAD_TIME, roleDownloadTime == null ? null : Long.toString(roleDownloadTime)); + } + + @JsonIgnore + public Long getRoleDownloadTime() { + String downloadTimeString = getInfo().get(PLUGIN_INFO_ROLE_DOWNLOAD_TIME); + return StringUtils.isNotBlank(downloadTimeString) ? Long.valueOf(downloadTimeString) : null; + } + + @JsonIgnore + public void setRoleDownloadedVersion(Long roleDownloadedVersion) { + getInfo().put(PLUGIN_INFO_ROLE_DOWNLOADED_VERSION, roleDownloadedVersion == null ? null : Long.toString(roleDownloadedVersion)); + } + + @JsonIgnore + public Long getRoleDownloadedVersion() { + String downloadedVersionString = getInfo().get(PLUGIN_INFO_ROLE_DOWNLOADED_VERSION); + return StringUtils.isNotBlank(downloadedVersionString) ? Long.valueOf(downloadedVersionString) : null; + } + + @JsonIgnore + public void setRoleActivationTime(Long roleActivationTime) { + getInfo().put(PLUGIN_INFO_ROLE_ACTIVATION_TIME, roleActivationTime == null ? null : Long.toString(roleActivationTime)); + } + + @JsonIgnore + public Long getRoleActivationTime() { + String activationTimeString = getInfo().get(PLUGIN_INFO_ROLE_ACTIVATION_TIME); + return StringUtils.isNotBlank(activationTimeString) ? Long.valueOf(activationTimeString) : null; + } + + @JsonIgnore + public void setRoleActiveVersion(Long roleActiveVersion) { + getInfo().put(PLUGIN_INFO_ROLE_ACTIVE_VERSION, roleActiveVersion == null ? null : Long.toString(roleActiveVersion)); + } + + @JsonIgnore + public Long getRoleActiveVersion() { + String activeVersionString = getInfo().get(PLUGIN_INFO_POLICY_ACTIVE_VERSION); + return StringUtils.isNotBlank(activeVersionString) ? Long.valueOf(activeVersionString) : null; + } + + @JsonIgnore + public void setUserStoreDownloadTime(Long userstoreDownloadTime) { + getInfo().put(PLUGIN_INFO_USERSTORE_DOWNLOAD_TIME, userstoreDownloadTime == null ? null : Long.toString(userstoreDownloadTime)); + } + + @JsonIgnore + public Long getUserStoreDownloadTime() { + String downloadTimeString = getInfo().get(PLUGIN_INFO_USERSTORE_DOWNLOAD_TIME); + return StringUtils.isNotBlank(downloadTimeString) ? Long.valueOf(downloadTimeString) : null; + } + + @JsonIgnore + public void setUserStoreDownloadedVersion(Long userstoreDownloadedVersion) { + getInfo().put(PLUGIN_INFO_USERSTORE_DOWNLOADED_VERSION, userstoreDownloadedVersion == null ? null : Long.toString(userstoreDownloadedVersion)); + } + + @JsonIgnore + public Long getUserStoreDownloadedVersion() { + String downloadedVersionString = getInfo().get(PLUGIN_INFO_USERSTORE_DOWNLOADED_VERSION); + return StringUtils.isNotBlank(downloadedVersionString) ? Long.valueOf(downloadedVersionString) : null; + } + + @JsonIgnore + public void setUserStoreActivationTime(Long userstoreActivationTime) { + getInfo().put(PLUGIN_INFO_USERSTORE_ACTIVATION_TIME, userstoreActivationTime == null ? null : Long.toString(userstoreActivationTime)); + } + + @JsonIgnore + public Long getUserStoreActivationTime() { + String activationTimeString = getInfo().get(PLUGIN_INFO_USERSTORE_ACTIVATION_TIME); + return StringUtils.isNotBlank(activationTimeString) ? Long.valueOf(activationTimeString) : null; + } + + @JsonIgnore + public void setUserStoreActiveVersion(Long userstoreActiveVersion) { + getInfo().put(PLUGIN_INFO_USERSTORE_ACTIVE_VERSION, userstoreActiveVersion == null ? null : Long.toString(userstoreActiveVersion)); + } + + @JsonIgnore + public Long getUserStoreActiveVersion() { + String activeVersionString = getInfo().get(PLUGIN_INFO_USERSTORE_ACTIVE_VERSION); + return StringUtils.isNotBlank(activeVersionString) ? Long.valueOf(activeVersionString) : null; + } + + @JsonIgnore + public void setPluginCapabilities(String capabilities) { + setCapabilities(PLUGIN_INFO_CAPABILITIES, capabilities); + } + + @JsonIgnore + public String getPluginCapabilities() { + return getCapabilities(PLUGIN_INFO_CAPABILITIES); + } + + @JsonIgnore + public void setAdminCapabilities(String capabilities) { + setCapabilities(RANGER_ADMIN_CAPABILITIES, capabilities); + } + + @JsonIgnore + public String getAdminCapabilities() { + return getCapabilities(RANGER_ADMIN_CAPABILITIES); + } + + @JsonIgnore + private void setCapabilities(String optionName, String capabilities) { + getInfo().put(optionName, capabilities == null ? null : capabilities); + } + + @JsonIgnore + private String getCapabilities(String optionName) { + String capabilitiesString = getInfo().get(optionName); + return StringUtils.isNotBlank(capabilitiesString) ? capabilitiesString : null; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + + toString(sb); + + return sb.toString(); + } + + public StringBuilder toString(StringBuilder sb) { + sb.append("RangerPluginInfo={"); + + sb.append("id={").append(id).append("} "); + sb.append("createTime={").append(createTime).append("} "); + sb.append("updateTime={").append(updateTime).append("} "); + sb.append("serviceName={").append(serviceName).append("} "); + sb.append("serviceType={").append(serviceType).append("} "); + sb.append("serviceTypeDisplayName{").append(serviceTypeDisplayName).append("} "); + sb.append("serviceDisplayName={").append(serviceDisplayName).append("} "); + sb.append("hostName={").append(hostName).append("} "); + sb.append("appType={").append(appType).append("} "); + sb.append("ipAddress={").append(ipAddress).append("} "); + sb.append("info={").append(info).append("} "); + + sb.append(" }"); + + return sb; + } +} + diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicy.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicy.java new file mode 100644 index 00000000000..f81f8e28540 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicy.java @@ -0,0 +1,1703 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.model; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.htrace.shaded.fasterxml.jackson.annotation.JsonInclude; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + + +@JsonInclude(JsonInclude.Include.NON_NULL) +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class RangerPolicy extends RangerBaseModelObject implements java.io.Serializable { + public static final String POLICY_TYPE_ACCESS = "ACCESS"; + public static final String POLICY_TYPE_DATAMASK = "DATA_MASK"; + public static final String POLICY_TYPE_ROWFILTER = "ROW_FILTER"; + public static final String POLICY_TYPE_AUDIT = "AUDIT"; + + public static final String[] POLICY_TYPES = new String[] { + POLICY_TYPE_ACCESS, + POLICY_TYPE_DATAMASK, + POLICY_TYPE_ROWFILTER + }; + + public static final String MASK_TYPE_NULL = "MASK_NULL"; + public static final String MASK_TYPE_NONE = "MASK_NONE"; + public static final String MASK_TYPE_CUSTOM = "CUSTOM"; + + public static final int POLICY_PRIORITY_NORMAL = 0; + public static final int POLICY_PRIORITY_OVERRIDE = 1; + + public static final String POLICY_PRIORITY_NAME_NORMAL = "NORMAL"; + public static final String POLICY_PRIORITY_NAME_OVERRIDE = "OVERRIDE"; + + public static final Comparator POLICY_ID_COMPARATOR = new PolicyIdComparator(); + + // For future use + private static final long serialVersionUID = 1L; + + private String service; + private String name; + private String policyType; + private Integer policyPriority; + private String description; + private String resourceSignature; + private Boolean isAuditEnabled; + private Map resources; + private List conditions; + private List policyItems; + private List denyPolicyItems; + private List allowExceptions; + private List denyExceptions; + private List dataMaskPolicyItems; + private List rowFilterPolicyItems; + private String serviceType; + private Map options; + private List validitySchedules; + private List policyLabels; + private String zoneName; + private Boolean isDenyAllElse; + private Map attributes; + + public RangerPolicy() { + this(null, null, null, null, null, null, null, null, null, null, null); + } + + public RangerPolicy(String service, String name, String policyType, Integer policyPriority, String description, Map resources, List policyItems, String resourceSignature, Map options, List validitySchedules, List policyLables) { + this(service, name, policyType, policyPriority, description, resources, policyItems, resourceSignature, options, validitySchedules, policyLables, null); + } + + public RangerPolicy(String service, String name, String policyType, Integer policyPriority, String description, Map resources, List policyItems, String resourceSignature, Map options, List validitySchedules, List policyLables, String zoneName) { + this(service, name, policyType, policyPriority, description, resources, policyItems, resourceSignature, options, validitySchedules, policyLables, zoneName, null); + } + + public RangerPolicy(String service, String name, String policyType, Integer policyPriority, String description, Map resources, List policyItems, String resourceSignature, Map options, List validitySchedules, List policyLables, String zoneName, List conditions) { + this(service, name, policyType, policyPriority, description, resources, policyItems, resourceSignature, options, validitySchedules, policyLables, zoneName, conditions, null); + } + + /** + * @param service + * @param name + * @param policyType + * @param description + * @param resources + * @param policyItems + * @param resourceSignature TODO + */ + public RangerPolicy(String service, String name, String policyType, Integer policyPriority, String description, Map resources, List policyItems, String resourceSignature, Map options, List validitySchedules, List policyLables, String zoneName, List conditions, Boolean isDenyAllElse) { + super(); + + setService(service); + setName(name); + setPolicyType(policyType); + setPolicyPriority(policyPriority); + setDescription(description); + setResourceSignature(resourceSignature); + setIsAuditEnabled(null); + setResources(resources); + setPolicyItems(policyItems); + setDenyPolicyItems(null); + setAllowExceptions(null); + setDenyExceptions(null); + setDataMaskPolicyItems(null); + setRowFilterPolicyItems(null); + setOptions(options); + setValiditySchedules(validitySchedules); + setPolicyLabels(policyLables); + setZoneName(zoneName); + setConditions(conditions); + setIsDenyAllElse(isDenyAllElse); + + } + + /** + * @param other + */ + public void updateFrom(RangerPolicy other) { + super.updateFrom(other); + + setService(other.getService()); + setName(other.getName()); + setPolicyType(other.getPolicyType()); + setPolicyPriority(other.getPolicyPriority()); + setDescription(other.getDescription()); + setResourceSignature(other.getResourceSignature()); + setIsAuditEnabled(other.getIsAuditEnabled()); + setResources(other.getResources()); + setConditions(other.getConditions()); + setPolicyItems(other.getPolicyItems()); + setDenyPolicyItems(other.getDenyPolicyItems()); + setAllowExceptions(other.getAllowExceptions()); + setDenyExceptions(other.getDenyExceptions()); + setDataMaskPolicyItems(other.getDataMaskPolicyItems()); + setRowFilterPolicyItems(other.getRowFilterPolicyItems()); + setServiceType(other.getServiceType()); + setOptions(other.getOptions()); + setValiditySchedules(other.getValiditySchedules()); + setPolicyLabels(other.getPolicyLabels()); + setZoneName(other.getZoneName()); + setIsDenyAllElse(other.getIsDenyAllElse()); + } + + public Map getAttributes() { + return attributes; + } + + public void setAttributes(Map attributes) { + this.attributes = attributes; + } + + /** + * @return the type + */ + public String getService() { + return service; + } + + /** + * @param service the type to set + */ + public void setService(String service) { + this.service = service; + } + + /** + * @return the name + */ + public String getName() { + return name; + } + + /** + * @param name the name to set + */ + public void setName(String name) { + this.name = name; + } + + /** + * @return the policyType + */ + public String getPolicyType() { + return policyType; + } + + /** + * @param policyType the policyType to set + */ + public void setPolicyType(String policyType) { + this.policyType = policyType; + } + + /** + * @return the policyPriority + */ + public Integer getPolicyPriority() { + return policyPriority; + } + + /** + * @param policyPriority the policyPriority to set + */ + public void setPolicyPriority(Integer policyPriority) { + this.policyPriority = policyPriority == null ? RangerPolicy.POLICY_PRIORITY_NORMAL : policyPriority; + } + + /** + * @return the description + */ + public String getDescription() { + return description; + } + + /** + * @param description the description to set + */ + public void setDescription(String description) { + this.description = description; + } + + /** + * @return the resourceSignature + */ + public String getResourceSignature() { + return resourceSignature; + } + + /** + * @param resourceSignature the resourceSignature to set + */ + public void setResourceSignature(String resourceSignature) { + this.resourceSignature = resourceSignature; + } + + /** + * @return the isAuditEnabled + */ + public Boolean getIsAuditEnabled() { + return isAuditEnabled; + } + + /** + * @param isAuditEnabled the isEnabled to set + */ + public void setIsAuditEnabled(Boolean isAuditEnabled) { + this.isAuditEnabled = isAuditEnabled == null ? Boolean.TRUE : isAuditEnabled; + } + + public String getServiceType() { + return serviceType; + } + + public void setServiceType(String serviceType) { + this.serviceType = serviceType; + } + + public List getPolicyLabels() { + return policyLabels; + } + + public void setPolicyLabels(List policyLabels) { + if (this.policyLabels == null) { + this.policyLabels = new ArrayList<>(); + } + + if (this.policyLabels == policyLabels) { + return; + } + + this.policyLabels.clear(); + + if (policyLabels != null) { + this.policyLabels.addAll(policyLabels); + } + } + + /** + * @return the resources + */ + public Map getResources() { + return resources; + } + + /** + * @param resources the resources to set + */ + public void setResources(Map resources) { + if(this.resources == null) { + this.resources = new HashMap<>(); + } + + if(this.resources == resources) { + return; + } + + this.resources.clear(); + + if(resources != null) { + for(Map.Entry e : resources.entrySet()) { + this.resources.put(e.getKey(), e.getValue()); + } + } + } + + /** + * @return the policyItems + */ + public List getPolicyItems() { + return policyItems; + } + + /** + * @param policyItems the policyItems to set + */ + public void setPolicyItems(List policyItems) { + if(this.policyItems == null) { + this.policyItems = new ArrayList<>(); + } + + if(this.policyItems == policyItems) { + return; + } + + this.policyItems.clear(); + + if(policyItems != null) { + this.policyItems.addAll(policyItems); + } + } + + /** + * @return the denyPolicyItems + */ + public List getDenyPolicyItems() { + return denyPolicyItems; + } + + /** + * @param denyPolicyItems the denyPolicyItems to set + */ + public void setDenyPolicyItems(List denyPolicyItems) { + if(this.denyPolicyItems == null) { + this.denyPolicyItems = new ArrayList<>(); + } + + if(this.denyPolicyItems == denyPolicyItems) { + return; + } + + this.denyPolicyItems.clear(); + + if(denyPolicyItems != null) { + this.denyPolicyItems.addAll(denyPolicyItems); + } + } + + /** + * @return the allowExceptions + */ + public List getAllowExceptions() { + return allowExceptions; + } + + /** + * @param allowExceptions the allowExceptions to set + */ + public void setAllowExceptions(List allowExceptions) { + if(this.allowExceptions == null) { + this.allowExceptions = new ArrayList<>(); + } + + if(this.allowExceptions == allowExceptions) { + return; + } + + this.allowExceptions.clear(); + + if(allowExceptions != null) { + this.allowExceptions.addAll(allowExceptions); + } + } + + /** + * @return the denyExceptions + */ + public List getDenyExceptions() { + return denyExceptions; + } + + /** + * @param denyExceptions the denyExceptions to set + */ + public void setDenyExceptions(List denyExceptions) { + if(this.denyExceptions == null) { + this.denyExceptions = new ArrayList<>(); + } + + if(this.denyExceptions == denyExceptions) { + return; + } + + this.denyExceptions.clear(); + + if(denyExceptions != null) { + this.denyExceptions.addAll(denyExceptions); + } + } + + public List getDataMaskPolicyItems() { + return dataMaskPolicyItems; + } + + public void setDataMaskPolicyItems(List dataMaskPolicyItems) { + if(this.dataMaskPolicyItems == null) { + this.dataMaskPolicyItems = new ArrayList<>(); + } + + if(this.dataMaskPolicyItems == dataMaskPolicyItems) { + return; + } + + this.dataMaskPolicyItems.clear(); + + if(dataMaskPolicyItems != null) { + this.dataMaskPolicyItems.addAll(dataMaskPolicyItems); + } + } + + public List getRowFilterPolicyItems() { + return rowFilterPolicyItems; + } + + public void setRowFilterPolicyItems(List rowFilterPolicyItems) { + if(this.rowFilterPolicyItems == null) { + this.rowFilterPolicyItems = new ArrayList<>(); + } + + if(this.rowFilterPolicyItems == rowFilterPolicyItems) { + return; + } + + this.rowFilterPolicyItems.clear(); + + if(rowFilterPolicyItems != null) { + this.rowFilterPolicyItems.addAll(rowFilterPolicyItems); + } + } + + public Map getOptions() { return options; } + + public void setOptions(Map options) { + if (this.options == null) { + this.options = new HashMap<>(); + } + if (this.options == options) { + return; + } + this.options.clear(); + + if(options != null) { + for(Map.Entry e : options.entrySet()) { + this.options.put(e.getKey(), e.getValue()); + } + } + } + + public List getValiditySchedules() { return validitySchedules; } + + public void setValiditySchedules(List validitySchedules) { + if (this.validitySchedules == null) { + this.validitySchedules = new ArrayList<>(); + } + if (this.validitySchedules == validitySchedules) { + return; + } + this.validitySchedules.clear(); + + if(validitySchedules != null) { + this.validitySchedules.addAll(validitySchedules); + } + } + public String getZoneName() { return zoneName; } + + public void setZoneName(String zoneName) { + this.zoneName = zoneName; + } + + /** + * @return the conditions + */ + public List getConditions() { return conditions; } + /** + * @param conditions the conditions to set + */ + public void setConditions(List conditions) { + this.conditions = conditions; + } + + public Boolean getIsDenyAllElse() { + return isDenyAllElse; + } + + public void setIsDenyAllElse(Boolean isDenyAllElse) { + this.isDenyAllElse = isDenyAllElse == null ? Boolean.FALSE : isDenyAllElse; + } + + @Override + public String toString( ) { + StringBuilder sb = new StringBuilder(); + + toString(sb); + + return sb.toString(); + } + + public StringBuilder toString(StringBuilder sb) { + sb.append("RangerPolicy={"); + + super.toString(sb); + + sb.append("service={").append(service).append("} "); + sb.append("name={").append(name).append("} "); + sb.append("policyType={").append(policyType).append("} "); + sb.append("policyPriority={").append(policyPriority).append("} "); + sb.append("description={").append(description).append("} "); + sb.append("resourceSignature={").append(resourceSignature).append("} "); + sb.append("isAuditEnabled={").append(isAuditEnabled).append("} "); + sb.append("serviceType={").append(serviceType).append("} "); + + sb.append("resources={"); + if(resources != null) { + for(Map.Entry e : resources.entrySet()) { + sb.append(e.getKey()).append("={"); + e.getValue().toString(sb); + sb.append("} "); + } + } + sb.append("} "); + sb.append("policyLabels={"); + if(policyLabels != null) { + for(String policyLabel : policyLabels) { + if(policyLabel != null) { + sb.append(policyLabel).append(" "); + } + } + } + sb.append("} "); + + sb.append("policyConditions={"); + if(conditions != null) { + for(RangerPolicyItemCondition condition : conditions) { + if(condition != null) { + condition.toString(sb); + } + } + } + sb.append("} "); + + sb.append("policyItems={"); + if(policyItems != null) { + for(RangerPolicyItem policyItem : policyItems) { + if(policyItem != null) { + policyItem.toString(sb); + } + } + } + sb.append("} "); + + sb.append("denyPolicyItems={"); + if(denyPolicyItems != null) { + for(RangerPolicyItem policyItem : denyPolicyItems) { + if(policyItem != null) { + policyItem.toString(sb); + } + } + } + sb.append("} "); + + sb.append("allowExceptions={"); + if(allowExceptions != null) { + for(RangerPolicyItem policyItem : allowExceptions) { + if(policyItem != null) { + policyItem.toString(sb); + } + } + } + sb.append("} "); + + sb.append("denyExceptions={"); + if(denyExceptions != null) { + for(RangerPolicyItem policyItem : denyExceptions) { + if(policyItem != null) { + policyItem.toString(sb); + } + } + } + sb.append("} "); + + sb.append("dataMaskPolicyItems={"); + if(dataMaskPolicyItems != null) { + for(RangerDataMaskPolicyItem dataMaskPolicyItem : dataMaskPolicyItems) { + if(dataMaskPolicyItem != null) { + dataMaskPolicyItem.toString(sb); + } + } + } + sb.append("} "); + + sb.append("rowFilterPolicyItems={"); + if(rowFilterPolicyItems != null) { + for(RangerRowFilterPolicyItem rowFilterPolicyItem : rowFilterPolicyItems) { + if(rowFilterPolicyItem != null) { + rowFilterPolicyItem.toString(sb); + } + } + } + sb.append("} "); + + sb.append("options={"); + if(options != null) { + for(Map.Entry e : options.entrySet()) { + sb.append(e.getKey()).append("={"); + sb.append(e.getValue().toString()); + sb.append("} "); + } + } + sb.append("} "); + + //sb.append("validitySchedules={").append(validitySchedules).append("} "); + sb.append("validitySchedules={"); + if (CollectionUtils.isNotEmpty(validitySchedules)) { + for (RangerValiditySchedule schedule : validitySchedules) { + if (schedule != null) { + sb.append("schedule={").append(schedule).append("}"); + } + } + } + sb.append(", zoneName=").append(zoneName); + + sb.append(", isDenyAllElse={").append(isDenyAllElse).append("} "); + + sb.append("}"); + + sb.append("}"); + + return sb; + } + + static class PolicyIdComparator implements Comparator, java.io.Serializable { + @Override + public int compare(RangerPolicy me, RangerPolicy other) { + return Long.compare(me.getId(), other.getId()); + } + } + + @JsonInclude(JsonInclude.Include.NON_NULL) + @XmlRootElement + @XmlAccessorType(XmlAccessType.FIELD) + public static class RangerPolicyResource implements java.io.Serializable { + private static final long serialVersionUID = 1L; + + private List values; + private Boolean isExcludes; + private Boolean isRecursive; + + public RangerPolicyResource() { + this((List)null, null, null); + } + + public RangerPolicyResource(String value) { + setValue(value); + setIsExcludes(null); + setIsRecursive(null); + } + + public RangerPolicyResource(String value, Boolean isExcludes, Boolean isRecursive) { + setValue(value); + setIsExcludes(isExcludes); + setIsRecursive(isRecursive); + } + + public RangerPolicyResource(List values, Boolean isExcludes, Boolean isRecursive) { + setValues(values); + setIsExcludes(isExcludes); + setIsRecursive(isRecursive); + } + + /** + * @return the values + */ + public List getValues() { + return values; + } + + /** + * @param values the values to set + */ + public void setValues(List values) { + if(this.values == null) { + this.values = new ArrayList<>(); + } + + if(this.values == values) { + return; + } + + this.values.clear(); + + if(values != null) { + this.values.addAll(values); + } + } + + /** + * @param value the value to set + */ + public void setValue(String value) { + if(this.values == null) { + this.values = new ArrayList<>(); + } + + this.values.clear(); + + this.values.add(value); + } + + /** + * @return the isExcludes + */ + public Boolean getIsExcludes() { + return isExcludes; + } + + /** + * @param isExcludes the isExcludes to set + */ + public void setIsExcludes(Boolean isExcludes) { + this.isExcludes = isExcludes == null ? Boolean.FALSE : isExcludes; + } + + /** + * @return the isRecursive + */ + public Boolean getIsRecursive() { + return isRecursive; + } + + /** + * @param isRecursive the isRecursive to set + */ + public void setIsRecursive(Boolean isRecursive) { + this.isRecursive = isRecursive == null ? Boolean.FALSE : isRecursive; + } + + @Override + public String toString( ) { + StringBuilder sb = new StringBuilder(); + + toString(sb); + + return sb.toString(); + } + + public StringBuilder toString(StringBuilder sb) { + sb.append("RangerPolicyResource={"); + sb.append("values={"); + if(values != null) { + for(String value : values) { + sb.append(value).append(" "); + } + } + sb.append("} "); + sb.append("isExcludes={").append(isExcludes).append("} "); + sb.append("isRecursive={").append(isRecursive).append("} "); + sb.append("}"); + + return sb; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + + ((isExcludes == null) ? 0 : isExcludes.hashCode()); + result = prime * result + + ((isRecursive == null) ? 0 : isRecursive.hashCode()); + result = prime * result + + ((values == null) ? 0 : values.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + RangerPolicyResource other = (RangerPolicyResource) obj; + if (isExcludes == null) { + if (other.isExcludes != null) + return false; + } else if (!isExcludes.equals(other.isExcludes)) + return false; + if (isRecursive == null) { + if (other.isRecursive != null) + return false; + } else if (!isRecursive.equals(other.isRecursive)) + return false; + if (values == null) { + if (other.values != null) + return false; + } else if (!values.equals(other.values)) + return false; + return true; + } + + } + + @JsonInclude(JsonInclude.Include.NON_NULL) + @XmlRootElement + @XmlAccessorType(XmlAccessType.FIELD) + public static class RangerPolicyItem implements java.io.Serializable { + private static final long serialVersionUID = 1L; + + private List accesses; + private List users; + private List groups; + private List roles; + private List conditions; + private Boolean delegateAdmin; + + public RangerPolicyItem() { + this(null, null, null, null, null, null); + } + + public RangerPolicyItem(List accessTypes, List users, List groups, List roles, List conditions, Boolean delegateAdmin) { + setAccesses(accessTypes); + setUsers(users); + setGroups(groups); + setRoles(roles); + setConditions(conditions); + setDelegateAdmin(delegateAdmin); + } + + /** + * @return the accesses + */ + public List getAccesses() { + return accesses; + } + /** + * @param accesses the accesses to set + */ + public void setAccesses(List accesses) { + if(this.accesses == null) { + this.accesses = new ArrayList<>(); + } + + if(this.accesses == accesses) { + return; + } + + this.accesses.clear(); + + if(accesses != null) { + this.accesses.addAll(accesses); + } + } + /** + * @return the users + */ + public List getUsers() { + return users; + } + /** + * @param users the users to set + */ + public void setUsers(List users) { + if(this.users == null) { + this.users = new ArrayList<>(); + } + + if(this.users == users) { + return; + } + + this.users.clear(); + + if(users != null) { + this.users.addAll(users); + } + } + /** + * @return the groups + */ + public List getGroups() { + return groups; + } + /** + * @param groups the groups to set + */ + public void setGroups(List groups) { + if(this.groups == null) { + this.groups = new ArrayList<>(); + } + + if(this.groups == groups) { + return; + } + + this.groups.clear(); + + if(groups != null) { + this.groups.addAll(groups); + } + } + /** + * @return the roles + */ + public List getRoles() { + return roles; + } + /** + * @param roles the roles to set + */ + public void setRoles(List roles) { + if(this.roles == null) { + this.roles = new ArrayList<>(); + } + + if(this.roles == roles) { + return; + } + + this.roles.clear(); + + if(roles != null) { + this.roles.addAll(roles); + } + } + /** + * @return the conditions + */ + public List getConditions() { + return conditions; + } + /** + * @param conditions the conditions to set + */ + public void setConditions(List conditions) { + if(this.conditions == null) { + this.conditions = new ArrayList<>(); + } + + if(this.conditions == conditions) { + return; + } + + this.conditions.clear(); + + if(conditions != null) { + this.conditions.addAll(conditions); + } + } + + /** + * @return the delegateAdmin + */ + public Boolean getDelegateAdmin() { + return delegateAdmin; + } + + /** + * @param delegateAdmin the delegateAdmin to set + */ + public void setDelegateAdmin(Boolean delegateAdmin) { + this.delegateAdmin = delegateAdmin == null ? Boolean.FALSE : delegateAdmin; + } + + @Override + public String toString( ) { + StringBuilder sb = new StringBuilder(); + + toString(sb); + + return sb.toString(); + } + + public StringBuilder toString(StringBuilder sb) { + sb.append("RangerPolicyItem={"); + + sb.append("accessTypes={"); + if(accesses != null) { + for(RangerPolicyItemAccess access : accesses) { + if(access != null) { + access.toString(sb); + } + } + } + sb.append("} "); + + sb.append("users={"); + if(users != null) { + for(String user : users) { + if(user != null) { + sb.append(user).append(" "); + } + } + } + sb.append("} "); + + sb.append("groups={"); + if(groups != null) { + for(String group : groups) { + if(group != null) { + sb.append(group).append(" "); + } + } + } + sb.append("} "); + + sb.append("roles={"); + if(roles != null) { + for(String role : roles) { + if(role != null) { + sb.append(role).append(" "); + } + } + } + sb.append("} "); + + sb.append("conditions={"); + if(conditions != null) { + for(RangerPolicyItemCondition condition : conditions) { + if(condition != null) { + condition.toString(sb); + } + } + } + sb.append("} "); + + sb.append("delegateAdmin={").append(delegateAdmin).append("} "); + sb.append("}"); + + return sb; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + + ((accesses == null) ? 0 : accesses.hashCode()); + result = prime * result + + ((conditions == null) ? 0 : conditions.hashCode()); + result = prime * result + + ((delegateAdmin == null) ? 0 : delegateAdmin.hashCode()); + result = prime * result + + ((roles == null) ? 0 : roles.hashCode()); + result = prime * result + + ((groups == null) ? 0 : groups.hashCode()); + result = prime * result + ((users == null) ? 0 : users.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + RangerPolicyItem other = (RangerPolicyItem) obj; + if (accesses == null) { + if (other.accesses != null) + return false; + } else if (!accesses.equals(other.accesses)) + return false; + if (conditions == null) { + if (other.conditions != null) + return false; + } else if (!conditions.equals(other.conditions)) + return false; + if (delegateAdmin == null) { + if (other.delegateAdmin != null) + return false; + } else if (!delegateAdmin.equals(other.delegateAdmin)) + return false; + if (roles == null) { + if (other.roles != null) + return false; + } else if (!roles.equals(other.roles)) + return false; + if (groups == null) { + if (other.groups != null) + return false; + } else if (!groups.equals(other.groups)) + return false; + if (users == null) { + if (other.users != null) + return false; + } else if (!users.equals(other.users)) + return false; + return true; + + } + } + + @JsonInclude(JsonInclude.Include.NON_NULL) + @XmlRootElement + @XmlAccessorType(XmlAccessType.FIELD) + public static class RangerDataMaskPolicyItem extends RangerPolicyItem implements java.io.Serializable { + private static final long serialVersionUID = 1L; + + private RangerPolicyItemDataMaskInfo dataMaskInfo; + + public RangerDataMaskPolicyItem() { + this(null, null, null, null, null, null, null); + } + + public RangerDataMaskPolicyItem(List accesses, RangerPolicyItemDataMaskInfo dataMaskDetail, List users, List groups, List roles, List conditions, Boolean delegateAdmin) { + super(accesses, users, groups, roles, conditions, delegateAdmin); + + setDataMaskInfo(dataMaskDetail); + } + + /** + * @return the dataMaskInfo + */ + public RangerPolicyItemDataMaskInfo getDataMaskInfo() { + return dataMaskInfo; + } + + /** + * @param dataMaskInfo the dataMaskInfo to set + */ + public void setDataMaskInfo(RangerPolicyItemDataMaskInfo dataMaskInfo) { + this.dataMaskInfo = dataMaskInfo == null ? new RangerPolicyItemDataMaskInfo() : dataMaskInfo; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = super.hashCode(); + result = prime * result + ((dataMaskInfo == null) ? 0 : dataMaskInfo.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if(! super.equals(obj)) + return false; + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + RangerDataMaskPolicyItem other = (RangerDataMaskPolicyItem) obj; + if (dataMaskInfo == null) { + if (other.dataMaskInfo != null) + return false; + } else if (!dataMaskInfo.equals(other.dataMaskInfo)) + return false; + return true; + } + + @Override + public String toString( ) { + StringBuilder sb = new StringBuilder(); + + toString(sb); + + return sb.toString(); + } + + public StringBuilder toString(StringBuilder sb) { + sb.append("RangerDataMaskPolicyItem={"); + + super.toString(sb); + + sb.append("dataMaskInfo={"); + if(dataMaskInfo != null) { + dataMaskInfo.toString(sb); + } + sb.append("} "); + + sb.append("}"); + + return sb; + } + } + + @JsonInclude(JsonInclude.Include.NON_NULL) + @XmlRootElement + @XmlAccessorType(XmlAccessType.FIELD) + public static class RangerRowFilterPolicyItem extends RangerPolicyItem implements java.io.Serializable { + private static final long serialVersionUID = 1L; + + private RangerPolicyItemRowFilterInfo rowFilterInfo; + + public RangerRowFilterPolicyItem() { + this(null, null, null, null, null, null, null); + } + + public RangerRowFilterPolicyItem(RangerPolicyItemRowFilterInfo rowFilterInfo, List accesses, List users, List groups, List roles, List conditions, Boolean delegateAdmin) { + super(accesses, users, groups, roles, conditions, delegateAdmin); + + setRowFilterInfo(rowFilterInfo); + } + + /** + * @return the rowFilterInfo + */ + public RangerPolicyItemRowFilterInfo getRowFilterInfo() { + return rowFilterInfo; + } + + /** + * @param rowFilterInfo the rowFilterInfo to set + */ + public void setRowFilterInfo(RangerPolicyItemRowFilterInfo rowFilterInfo) { + this.rowFilterInfo = rowFilterInfo == null ? new RangerPolicyItemRowFilterInfo() : rowFilterInfo; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = super.hashCode(); + result = prime * result + ((rowFilterInfo == null) ? 0 : rowFilterInfo.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if(! super.equals(obj)) + return false; + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + RangerRowFilterPolicyItem other = (RangerRowFilterPolicyItem) obj; + if (rowFilterInfo == null) { + if (other.rowFilterInfo != null) + return false; + } else if (!rowFilterInfo.equals(other.rowFilterInfo)) + return false; + return true; + } + + @Override + public String toString( ) { + StringBuilder sb = new StringBuilder(); + + toString(sb); + + return sb.toString(); + } + + public StringBuilder toString(StringBuilder sb) { + sb.append("RangerRowFilterPolicyItem={"); + + super.toString(sb); + + sb.append("rowFilterInfo={"); + if(rowFilterInfo != null) { + rowFilterInfo.toString(sb); + } + sb.append("} "); + + sb.append("}"); + + return sb; + } + } + + @JsonInclude(JsonInclude.Include.NON_NULL) + @XmlRootElement + @XmlAccessorType(XmlAccessType.FIELD) + public static class RangerPolicyItemAccess implements java.io.Serializable { + private static final long serialVersionUID = 1L; + + private String type; + private Boolean isAllowed; + + public RangerPolicyItemAccess() { + this(null, null); + } + + public RangerPolicyItemAccess(String type) { + this(type, null); + } + + public RangerPolicyItemAccess(String type, Boolean isAllowed) { + setType(type); + setIsAllowed(isAllowed); + } + + /** + * @return the type + */ + public String getType() { + return type; + } + + /** + * @param type the type to set + */ + public void setType(String type) { + this.type = type; + } + + /** + * @return the isAllowed + */ + public Boolean getIsAllowed() { + return isAllowed; + } + + /** + * @param isAllowed the isAllowed to set + */ + public void setIsAllowed(Boolean isAllowed) { + this.isAllowed = isAllowed == null ? Boolean.TRUE : isAllowed; + } + + @Override + public String toString( ) { + StringBuilder sb = new StringBuilder(); + + toString(sb); + + return sb.toString(); + } + + public StringBuilder toString(StringBuilder sb) { + sb.append("RangerPolicyItemAccess={"); + sb.append("type={").append(type).append("} "); + sb.append("isAllowed={").append(isAllowed).append("} "); + sb.append("}"); + + return sb; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + + ((isAllowed == null) ? 0 : isAllowed.hashCode()); + result = prime * result + ((type == null) ? 0 : type.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + RangerPolicyItemAccess other = (RangerPolicyItemAccess) obj; + if (isAllowed == null) { + if (other.isAllowed != null) + return false; + } else if (!isAllowed.equals(other.isAllowed)) + return false; + if (type == null) { + if (other.type != null) + return false; + } else if (!type.equals(other.type)) + return false; + return true; + } + + } + + @JsonInclude(JsonInclude.Include.NON_NULL) + @XmlRootElement + @XmlAccessorType(XmlAccessType.FIELD) + public static class RangerPolicyItemCondition implements java.io.Serializable { + private static final long serialVersionUID = 1L; + + private String type; + private List values; + + public RangerPolicyItemCondition() { + this(null, null); + } + + public RangerPolicyItemCondition(String type, List values) { + setType(type); + setValues(values); + } + + /** + * @return the type + */ + public String getType() { + return type; + } + + /** + * @param type the type to set + */ + public void setType(String type) { + this.type = type; + } + + /** + * @return the value + */ + public List getValues() { + return values; + } + + /** + * @param values the value to set + */ + public void setValues(List values) { + if (this.values == null) { + this.values = new ArrayList<>(); + } + + if(this.values == values) { + return; + } + + this.values.clear(); + + if(values != null) { + this.values.addAll(values); + } + } + + @Override + public String toString( ) { + StringBuilder sb = new StringBuilder(); + + toString(sb); + + return sb.toString(); + } + + public StringBuilder toString(StringBuilder sb) { + sb.append("RangerPolicyCondition={"); + sb.append("type={").append(type).append("} "); + sb.append("values={"); + if(values != null) { + for(String value : values) { + sb.append(value).append(" "); + } + } + sb.append("} "); + sb.append("}"); + + return sb; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((type == null) ? 0 : type.hashCode()); + result = prime * result + + ((values == null) ? 0 : values.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + RangerPolicyItemCondition other = (RangerPolicyItemCondition) obj; + if (type == null) { + if (other.type != null) + return false; + } else if (!type.equals(other.type)) + return false; + if (values == null) { + if (other.values != null) + return false; + } else if (!values.equals(other.values)) + return false; + return true; + } + + } + + @JsonInclude(JsonInclude.Include.NON_NULL) + @XmlRootElement + @XmlAccessorType(XmlAccessType.FIELD) + public static class RangerPolicyItemDataMaskInfo implements java.io.Serializable { + private static final long serialVersionUID = 1L; + + private String dataMaskType; + private String conditionExpr; + private String valueExpr; + + public RangerPolicyItemDataMaskInfo() { } + + public RangerPolicyItemDataMaskInfo(String dataMaskType, String conditionExpr, String valueExpr) { + setDataMaskType(dataMaskType); + setConditionExpr(conditionExpr); + setValueExpr(valueExpr); + } + + public RangerPolicyItemDataMaskInfo(RangerPolicyItemDataMaskInfo that) { + this.dataMaskType = that.dataMaskType; + this.conditionExpr = that.conditionExpr; + this.valueExpr = that.valueExpr; + } + + public String getDataMaskType() { + return dataMaskType; + } + + public void setDataMaskType(String dataMaskType) { + this.dataMaskType = dataMaskType; + } + + public String getConditionExpr() { + return conditionExpr; + } + + public void setConditionExpr(String conditionExpr) { + this.conditionExpr = conditionExpr; + } + + public String getValueExpr() { + return valueExpr; + } + + public void setValueExpr(String valueExpr) { + this.valueExpr = valueExpr; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = super.hashCode(); + result = prime * result + ((dataMaskType == null) ? 0 : dataMaskType.hashCode()); + result = prime * result + ((conditionExpr == null) ? 0 : conditionExpr.hashCode()); + result = prime * result + ((valueExpr == null) ? 0 : valueExpr.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + RangerPolicyItemDataMaskInfo other = (RangerPolicyItemDataMaskInfo) obj; + if (dataMaskType == null) { + if (other.dataMaskType != null) + return false; + } else if (!dataMaskType.equals(other.dataMaskType)) + return false; + if (conditionExpr == null) { + if (other.conditionExpr != null) + return false; + } else if (!conditionExpr.equals(other.conditionExpr)) + return false; + if (valueExpr == null) { + if (other.valueExpr != null) + return false; + } else if (!valueExpr.equals(other.valueExpr)) + return false; + return true; + } + + @Override + public String toString( ) { + StringBuilder sb = new StringBuilder(); + + toString(sb); + + return sb.toString(); + } + + public StringBuilder toString(StringBuilder sb) { + sb.append("RangerPolicyItemDataMaskInfo={"); + + sb.append("dataMaskType={").append(dataMaskType).append("} "); + sb.append("conditionExpr={").append(conditionExpr).append("} "); + sb.append("valueExpr={").append(valueExpr).append("} "); + + sb.append("}"); + + return sb; + } + } + + @JsonInclude(JsonInclude.Include.NON_NULL) + @XmlRootElement + @XmlAccessorType(XmlAccessType.FIELD) + public static class RangerPolicyItemRowFilterInfo implements java.io.Serializable { + private static final long serialVersionUID = 1L; + + private String filterExpr; + + public RangerPolicyItemRowFilterInfo() { } + + public RangerPolicyItemRowFilterInfo(String filterExpr) { + setFilterExpr(filterExpr); + } + + public RangerPolicyItemRowFilterInfo(RangerPolicyItemRowFilterInfo that) { + this.filterExpr = that.filterExpr; + } + + public String getFilterExpr() { + return filterExpr; + } + + public void setFilterExpr(String filterExpr) { + this.filterExpr = filterExpr; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = super.hashCode(); + result = prime * result + ((filterExpr == null) ? 0 : filterExpr.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + RangerPolicyItemRowFilterInfo other = (RangerPolicyItemRowFilterInfo) obj; + if (filterExpr == null) { + if (other.filterExpr != null) + return false; + } else if (!filterExpr.equals(other.filterExpr)) + return false; + return true; + } + + @Override + public String toString( ) { + StringBuilder sb = new StringBuilder(); + + toString(sb); + + return sb.toString(); + } + + public StringBuilder toString(StringBuilder sb) { + sb.append("RangerPolicyItemRowFilterInfo={"); + + sb.append("filterExpr={").append(filterExpr).append("} "); + + sb.append("}"); + + return sb; + } + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicyDelta.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicyDelta.java new file mode 100644 index 00000000000..adfe24af1d6 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicyDelta.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.model; + +import org.apache.htrace.shaded.fasterxml.jackson.annotation.JsonIgnore; +import org.apache.htrace.shaded.fasterxml.jackson.annotation.JsonInclude; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class RangerPolicyDelta implements java.io.Serializable { + + public static final int CHANGE_TYPE_POLICY_CREATE = 0; + public static final int CHANGE_TYPE_POLICY_UPDATE = 1; + public static final int CHANGE_TYPE_POLICY_DELETE = 2; + public static final int CHANGE_TYPE_SERVICE_CHANGE = 3; + public static final int CHANGE_TYPE_SERVICE_DEF_CHANGE = 4; + public static final int CHANGE_TYPE_RANGER_ADMIN_START = 5; + public static final int CHANGE_TYPE_LOG_ERROR = 6; + public static final int CHANGE_TYPE_INVALIDATE_POLICY_DELTAS = 7; + public static final int CHANGE_TYPE_ROLE_UPDATE = 8; + + private static String[] changeTypeNames = { "POLICY_CREATE", "POLICY_UPDATE", "POLICY_DELETE", "SERVICE_CHANGE", "SERVICE_DEF_CHANGE", "RANGER_ADMIN_START", "LOG_ERROR", "INVALIDATE_POLICY_DELTAS", "ROLE_UPDATE" }; + + private Long id; + private Integer changeType; + private Long policiesVersion; + private RangerPolicy policy; + + public RangerPolicyDelta() { + this(null, null, null, null); + } + + public RangerPolicyDelta(final Long id, final Integer changeType, final Long policiesVersion, final RangerPolicy policy) { + setId(id); + setChangeType(changeType); + setPoliciesVersion(policiesVersion); + setPolicy(policy); + } + public Long getId() { return id; } + + public Integer getChangeType() { return changeType; } + + public Long getPoliciesVersion() { return policiesVersion; } + + @JsonIgnore + public String getServiceType() { return policy != null ? policy.getServiceType() : null; } + + @JsonIgnore + public String getPolicyType() { return policy != null ? policy.getPolicyType() : null; } + + @JsonIgnore + public Long getPolicyId() { return policy != null ? policy.getId() : null; } + + @JsonIgnore + public String getZoneName() { return policy != null ? policy.getZoneName() : null; } + + public RangerPolicy getPolicy() { return policy; } + + public void setId(Long id) { this.id = id;} + + private void setChangeType(Integer changeType) { this.changeType = changeType; } + + private void setPoliciesVersion(Long policiesVersion) { this.policiesVersion = policiesVersion; } + + public void setPolicy(RangerPolicy policy) { this.policy = policy; } + + @Override + public String toString() { + return "id:" + id + + ", changeType:" + changeTypeNames[changeType] + + ", policiesVersion:" + getPoliciesVersion() + + ", serviceType:" + getServiceType() + + ", policyType:" + getPolicyType() + + ", policyId:[" + getPolicyId() + "]" + + ", policy:[" + policy +"]"; + } + +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicyResourceSignature.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicyResourceSignature.java new file mode 100644 index 00000000000..5cfcdd1fedc --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicyResourceSignature.java @@ -0,0 +1,243 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.model; + +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.authorization.hadoop.config.RangerAdminConfig; +import org.apache.atlas.plugin.model.RangerPolicy.RangerPolicyItemCondition; +import org.apache.atlas.plugin.model.RangerPolicy.RangerPolicyResource; +import org.apache.commons.lang.StringUtils; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.TreeMap; + +public class RangerPolicyResourceSignature { + + static final int _SignatureVersion = 1; + private static final Log LOG = LogFactory.getLog(RangerPolicyResourceSignature.class); + static final RangerPolicyResourceSignature _EmptyResourceSignature = new RangerPolicyResourceSignature((RangerPolicy)null); + + private final String _string; + private final String _hash; + private final RangerPolicy _policy; + + public RangerPolicyResourceSignature(RangerPolicy policy) { + _policy = policy; + PolicySerializer serializer = new PolicySerializer(_policy); + _string = serializer.toString(); + if (RangerAdminConfig.getInstance().isFipsEnabled()) { + _hash = DigestUtils.sha512Hex(_string); + } else { + _hash = DigestUtils.sha256Hex(_string); + } + } + + /** + * Only added for testability. Do not make public + * @param string + */ + RangerPolicyResourceSignature(String string) { + _policy = null; + if (string == null) { + _string = ""; + } else { + _string = string; + } + if (RangerAdminConfig.getInstance().isFipsEnabled()) { + _hash = DigestUtils.sha384Hex(_string); + } else { + _hash = DigestUtils.sha256Hex(_string); + } + } + + String asString() { + return _string; + } + + public String getSignature() { + return _hash; + } + + @Override + public int hashCode() { + // we assume no collision + return Objects.hashCode(_hash); + } + + @Override + public boolean equals(Object object) { + if (object == null || !(object instanceof RangerPolicyResourceSignature)) { + return false; + } + RangerPolicyResourceSignature that = (RangerPolicyResourceSignature)object; + return Objects.equals(this._hash, that._hash); + } + + @Override + public String toString() { + return String.format("%s: %s", _hash, _string); + } + + static class PolicySerializer { + final RangerPolicy _policy; + PolicySerializer(RangerPolicy policy) { + _policy = policy; + } + + boolean isPolicyValidForResourceSignatureComputation() { + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("==> RangerPolicyResourceSignature.isPolicyValidForResourceSignatureComputation(%s)", _policy)); + } + + boolean valid = false; + if (_policy == null) { + LOG.debug("isPolicyValidForResourceSignatureComputation: policy was null!"); + } else if (_policy.getResources() == null) { + LOG.debug("isPolicyValidForResourceSignatureComputation: resources collection on policy was null!"); + } else if (_policy.getResources().containsKey(null)) { + LOG.debug("isPolicyValidForResourceSignatureComputation: resources collection has resource with null name!"); + } else { + valid = true; + } + + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("<== RangerPolicyResourceSignature.isPolicyValidForResourceSignatureComputation(%s): %s", _policy, valid)); + } + return valid; + } + + @Override + public String toString() { + // invalid/empty policy gets a deterministic signature as if it had an + // empty resource string + if (!isPolicyValidForResourceSignatureComputation()) { + return ""; + } + String type = RangerPolicy.POLICY_TYPE_ACCESS; + if (StringUtils.isEmpty(_policy.getPolicyType())) { + type = _policy.getPolicyType(); + } + Map resources = new TreeMap<>(); + for (Map.Entry entry : _policy.getResources().entrySet()) { + String resourceName = entry.getKey(); + ResourceSerializer resourceView = new ResourceSerializer(entry.getValue()); + resources.put(resourceName, resourceView); + } + String resource = resources.toString(); + if (CollectionUtils.isNotEmpty(_policy.getValiditySchedules())) { + resource += _policy.getValiditySchedules().toString(); + } + if (_policy.getPolicyPriority() != null && _policy.getPolicyPriority() != RangerPolicy.POLICY_PRIORITY_NORMAL) { + resource += _policy.getPolicyPriority(); + } + if (!StringUtils.isEmpty(_policy.getZoneName())) { + resource += _policy.getZoneName(); + } + + if (_policy.getConditions() != null) { + CustomConditionSerialiser customConditionSerialiser = new CustomConditionSerialiser(_policy.getConditions()); + resource += customConditionSerialiser.toString(); + } + + String result = String.format("{version=%d,type=%d,resource=%s}", _SignatureVersion, type, resource); + return result; + } + + } + + static class ResourceSerializer { + final RangerPolicyResource _policyResource; + + ResourceSerializer(RangerPolicyResource policyResource) { + _policyResource = policyResource; + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("{"); + if (_policyResource != null) { + builder.append("values="); + if (_policyResource.getValues() != null) { + List values = new ArrayList(_policyResource.getValues()); + Collections.sort(values); + builder.append(values); + } + builder.append(",excludes="); + if (_policyResource.getIsExcludes() == null) { // null is same as false + builder.append(Boolean.FALSE); + } else { + builder.append(_policyResource.getIsExcludes()); + } + builder.append(",recursive="); + if (_policyResource.getIsRecursive() == null) { // null is the same as false + builder.append(Boolean.FALSE); + } else { + builder.append(_policyResource.getIsRecursive()); + } + } + builder.append("}"); + return builder.toString(); + } + } + + static class CustomConditionSerialiser { + final List rangerPolicyConditions; + + CustomConditionSerialiser(List rangerPolicyConditions) { + this.rangerPolicyConditions = rangerPolicyConditions; + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + Map> conditionMap = new TreeMap<>(); + + for(RangerPolicyItemCondition rangerPolicyCondition : rangerPolicyConditions) { + if (rangerPolicyCondition.getType() != null) { + String type = rangerPolicyCondition.getType(); + List values = new ArrayList<>(); + if (rangerPolicyCondition.getValues() != null) { + values.addAll(rangerPolicyCondition.getValues()); + Collections.sort(values); + } + conditionMap.put(type, values); + } + } + + if (MapUtils.isNotEmpty(conditionMap)) { + builder.append("{"); + builder.append("RangerPolicyConditions="); + builder.append(conditionMap); + builder.append("}"); + } + + return builder.toString(); + } + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerRole.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerRole.java new file mode 100644 index 00000000000..1ae6970dbe4 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerRole.java @@ -0,0 +1,192 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.model; + +import org.apache.commons.collections.MapUtils; +import org.apache.htrace.shaded.fasterxml.jackson.annotation.JsonInclude; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class RangerRole extends RangerBaseModelObject implements java.io.Serializable { + public static final String KEY_USER = "user"; + public static final String KEY_GROUP = "group"; + + private static final long serialVersionUID = 1L; + private String name; + private String description; + private Map options; + private List users; + private List groups; + private List roles; + private String createdByUser; + + public RangerRole() { + this(null, null, null, null, null, null); + } + + public RangerRole(String name, String description, Map options, List users, List groups) { + this(name, description, options, users, groups, null); + } + + public RangerRole(String name, String description, Map options, List users, List groups, List roles) { + setName(name); + setDescription(description); + setOptions(options); + setUsers(users); + setGroups(groups); + setRoles(roles); + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public Map getOptions() { + return options; + } + + public void setOptions(Map options) { + this.options = options == null ? new HashMap<>() : options; + } + + public List getUsers() { + return users; + } + + public void setUsers(List users) { + this.users = users == null ? new ArrayList<>() : users; + } + + public List getGroups() { + return groups; + } + + public void setGroups(List groups) { + this.groups = groups == null ? new ArrayList<>() : groups; + } + + public List getRoles() { + return roles; + } + + public void setRoles(List roles) { + this.roles = roles == null ? new ArrayList<>() : roles; + } + + public String getCreatedByUser() { + return createdByUser; + } + + public void setCreatedByUser(String createdByUser) { + this.createdByUser = createdByUser; + } + + @Override + public String toString() { + return "{name=" + name + + ", description=" + description + + ", options=" + getPrintableOptions(options) + + ", users=" + users + + ", groups=" + groups + + ", roles=" + roles + + ", createdByUser=" + createdByUser + + "}"; + } + + @JsonInclude(JsonInclude.Include.NON_NULL) + @XmlRootElement + @XmlAccessorType(XmlAccessType.FIELD) + public static class RoleMember implements java.io.Serializable { + private static final long serialVersionUID = 1L; + + private String name; + private boolean isAdmin; + + public RoleMember() { + this(null, false); + } + public RoleMember(String name, boolean isAdmin) { + setName(name); + setIsAdmin(isAdmin); + } + public void setName(String name) { + this.name = name; + } + public void setIsAdmin(boolean isAdmin) { + this.isAdmin = isAdmin; + } + public String getName() { return name; } + public boolean getIsAdmin() { return isAdmin; } + + @Override + public String toString() { + return "{" + name + ", " + isAdmin + "}"; + } + @Override + public int hashCode() { + return Objects.hash(name, isAdmin); + } + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + RoleMember other = (RoleMember) obj; + return Objects.equals(name, other.name) && isAdmin == other.isAdmin; + } + } + + private String getPrintableOptions(Map options) { + if (MapUtils.isEmpty(options)) return "{}"; + StringBuilder ret = new StringBuilder(); + ret.append("{"); + for (Map.Entry entry : options.entrySet()) { + ret.append(entry.getKey()).append(", ").append("[").append(entry.getValue()).append("]").append(","); + } + ret.append("}"); + return ret.toString(); + } + +} \ No newline at end of file diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerSecurityZone.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerSecurityZone.java new file mode 100644 index 00000000000..336ba3d906d --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerSecurityZone.java @@ -0,0 +1,169 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.model; + +import org.codehaus.jackson.annotate.JsonAutoDetect; +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@JsonAutoDetect(fieldVisibility=JsonAutoDetect.Visibility.ANY) +@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown=true) +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class RangerSecurityZone extends RangerBaseModelObject implements java.io.Serializable { + public static final long RANGER_UNZONED_SECURITY_ZONE_ID = 1L; + private static final long serialVersionUID = 1L; + private String name; + private Map services; + private List tagServices; + private List adminUsers; + private List adminUserGroups; + private List auditUsers; + private List auditUserGroups; + private String description; + + public RangerSecurityZone() { + this(null, null, null, null, null, null, null,null); + } + + public RangerSecurityZone(String name, Map services,List tagServices, List adminUsers, List adminUserGroups, List auditUsers, List auditUserGroups, String description) { + setName(name); + setServices(services); + setAdminUsers(adminUsers); + setAdminUserGroups(adminUserGroups); + setAuditUsers(auditUsers); + setAuditUserGroups(auditUserGroups); + setDescription(description); + setTagServices(tagServices); + } + public String getName() { return name; } + + public void setName(String name) { + this.name = name; + } + + public String getDescription() { return description; } + + public void setDescription(String description) { + this.description = description; + } + + public Map getServices() { return services; } + + public void setServices(Map services) { + this.services = services == null ? new HashMap<>() : services; + } + + public List getAdminUsers() { return adminUsers; } + + public void setAdminUsers(List adminUsers) { + this.adminUsers = adminUsers == null ? new ArrayList<>() : adminUsers; + } + + public List getAdminUserGroups() { return adminUserGroups; } + + public void setAdminUserGroups(List adminUserGroups) { + this.adminUserGroups = adminUserGroups == null ? new ArrayList<>() : adminUserGroups; + } + + public List getAuditUsers() { return auditUsers; } + + public void setAuditUsers(List auditUsers) { + this.auditUsers = auditUsers == null ? new ArrayList<>() : auditUsers; + } + + public List getAuditUserGroups() { return auditUserGroups; } + + public void setAuditUserGroups(List auditUserGroups) { + this.auditUserGroups = auditUserGroups == null ? new ArrayList<>() : auditUserGroups; + } + + public List getTagServices() { + return tagServices; + } + + public void setTagServices(List tagServices) { + this.tagServices = (tagServices != null) ? tagServices : new ArrayList(); + } + + @Override + public String toString() { + return "{name=" + name + + ", services=" + services + + ", tagServices=" + tagServices + + ", adminUsers=" + adminUsers + + ", adminUserGroups=" + adminUserGroups + + ", auditUsers=" + auditUsers + + ", auditUserGroups=" + auditUserGroups + + ", description="+ description + +"}"; + } + + @JsonAutoDetect(fieldVisibility=JsonAutoDetect.Visibility.ANY) + @JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL) + @JsonIgnoreProperties(ignoreUnknown=true) + @XmlRootElement + @XmlAccessorType(XmlAccessType.FIELD) + public static class RangerSecurityZoneService implements java.io.Serializable { + private static final long serialVersionUID = 1L; + private List>> resources; + + public RangerSecurityZoneService() { + this(null); + } + + public RangerSecurityZoneService(List>> resources) { + setResources(resources); + } + + public List>> getResources() { return resources; } + + public void setResources(List>> resources) { + this.resources = resources == null ? new ArrayList<>() : resources; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("{resources={"); + for (Map> resource : resources) { + sb.append("[ "); + for (Map.Entry> entry : resource.entrySet()) { + sb.append("{resource-def-name=").append(entry.getKey()).append(", values=").append(entry.getValue()).append("},"); + } + sb.append(" ],"); + } + sb.append("}}"); + + return sb.toString(); + } + + } +} + diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerService.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerService.java new file mode 100644 index 00000000000..99e6de6e7d9 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerService.java @@ -0,0 +1,278 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.model; + +import org.apache.htrace.shaded.fasterxml.jackson.annotation.JsonInclude; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; + + +@JsonInclude(JsonInclude.Include.NON_NULL) +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class RangerService extends RangerBaseModelObject implements java.io.Serializable { + private static final long serialVersionUID = 1L; + + private String type; + private String name; + private String displayName; + private String description; + private String tagService; + private Map configs; + private Long policyVersion; + private Date policyUpdateTime; + private Long tagVersion; + private Date tagUpdateTime; + + + /** + * @param + */ + public RangerService() { + this(null, null, null, null, null); + } + + /** + * @param type + * @param name + * @param description + * @param configs + * @param tagService + */ + public RangerService(String type, String name, String description, String tagService, Map configs) { + super(); + + setType(type); + setName(name); + setDescription(description); + setTagService(tagService); + setConfigs(configs); + } + + /** + * @param other + */ + public void updateFrom(RangerService other) { + super.updateFrom(other); + + setType(other.getType()); + setName(other.getName()); + setDisplayName(other.getDisplayName()); + setDescription(other.getDescription()); + setTagService(other.getTagService()); + setConfigs(other.getConfigs()); + setPolicyVersion(other.getPolicyVersion()); + setPolicyUpdateTime(other.getPolicyUpdateTime()); + setTagVersion(other.getTagVersion()); + setTagUpdateTime(other.getTagUpdateTime()); + } + + /** + * @return the type + */ + public String getType() { + return type; + } + + /** + * @param type the type to set + */ + public void setType(String type) { + this.type = type; + } + + /** + * @return the name + */ + public String getName() { + return name; + } + + /** + * @param name the name to set + */ + public void setName(String name) { + this.name = name; + } + + public String getDisplayName() { + return displayName; + } + + public void setDisplayName(String displayName) { + this.displayName = displayName; + } + + /** + * @return the description + */ + public String getDescription() { + return description; + } + + /** + * @param description the description to set + */ + public void setDescription(String description) { + this.description = description; + } + + /** + * @return the tagService + */ + public String getTagService() { + return tagService; + } + + /** + * @param tagService the tagServiceName to set + */ + public void setTagService(String tagService) { + this.tagService = tagService; + } + + /** + * @return the configs + */ + public Map getConfigs() { + return configs; + } + + /** + * @param configs the configs to set + */ + public void setConfigs(Map configs) { + if(this.configs == null) { + this.configs = new HashMap<>(); + } + + if(this.configs == configs) { + return; + } + + this.configs.clear(); + + if(configs != null) { + for(Map.Entry e : configs.entrySet()) { + this.configs.put(e.getKey(), e.getValue()); + } + } + } + + /** + * @return the policyVersion + */ + public Long getPolicyVersion() { + return policyVersion; + } + + /** + * @param policyVersion the policyVersion to set + */ + public void setPolicyVersion(Long policyVersion) { + this.policyVersion = policyVersion; + } + + /** + * @return the policyUpdateTime + */ + public Date getPolicyUpdateTime() { + return policyUpdateTime; + } + + /** + * @param policyUpdateTime the policyUpdateTime to set + */ + public void setPolicyUpdateTime(Date policyUpdateTime) { + this.policyUpdateTime = policyUpdateTime; + } + + /** + * @return the tagVersion + */ + public Long getTagVersion() { + return tagVersion; + } + + /** + * @param tagVersion the tagVersion to set + */ + public void setTagVersion(Long tagVersion) { + this.tagVersion = tagVersion; + } + + + /** + * @return the tagUpdateTime + */ + public Date getTagUpdateTime() { + return tagUpdateTime; + } + + /** + * @param tagUpdateTime the policyUpdateTime to set + */ + public void setTagUpdateTime(Date tagUpdateTime) { + this.tagUpdateTime = tagUpdateTime; + } + + @Override + public String toString( ) { + StringBuilder sb = new StringBuilder(); + + toString(sb); + + return sb.toString(); + } + + public StringBuilder toString(StringBuilder sb) { + sb.append("RangerService={"); + + super.toString(sb); + sb.append("name={").append(name).append("} "); + sb.append("displayName={").append(displayName).append("} "); + sb.append("type={").append(type).append("} "); + sb.append("description={").append(description).append("} "); + sb.append("tagService={").append(tagService).append("} "); + + sb.append("configs={"); + if(configs != null) { + for(Map.Entry e : configs.entrySet()) { + sb.append(e.getKey()).append("={").append(e.getValue()).append("} "); + } + } + sb.append("} "); + + sb.append("policyVersion={").append(policyVersion).append("} "); + sb.append("policyUpdateTime={").append(policyUpdateTime).append("} "); + + sb.append("tagVersion={").append(tagVersion).append("} "); + sb.append("tagUpdateTime={").append(tagUpdateTime).append("} "); + + sb.append("}"); + + return sb; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerServiceDef.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerServiceDef.java new file mode 100644 index 00000000000..4c5d8988d0e --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerServiceDef.java @@ -0,0 +1,3064 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.model; + +import org.apache.htrace.shaded.fasterxml.jackson.annotation.JsonInclude; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + + +@JsonInclude(JsonInclude.Include.NON_NULL) +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class RangerServiceDef extends RangerBaseModelObject implements java.io.Serializable { + private static final long serialVersionUID = 1L; + + public static final String OPTION_ENABLE_DENY_AND_EXCEPTIONS_IN_POLICIES = "enableDenyAndExceptionsInPolicies"; + + private String name; + private String displayName; + private String implClass; + private String label; + private String description; + private String rbKeyLabel; + private String rbKeyDescription; + private Map options; + private List configs; + private List resources; + private List accessTypes; + private List policyConditions; + private List contextEnrichers; + private List enums; + private RangerDataMaskDef dataMaskDef; + private RangerRowFilterDef rowFilterDef; + + public RangerServiceDef() { + this(null, null, null, null, null, null, null, null, null, null, null, null, null); + } + + public RangerServiceDef(String name, String implClass, String label, String description, Map options, List configs, List resources, List accessTypes, List policyConditions, List contextEnrichers, List enums) { + this(name, implClass, label, description, options, configs, resources, accessTypes, policyConditions, contextEnrichers, enums, null, null); + } + + /** + * @param name + * @param implClass + * @param label + * @param description + * @param options + * @param configs + * @param resources + * @param accessTypes + * @param policyConditions + * @param contextEnrichers + * @param dataMaskDef + * @param enums + */ + public RangerServiceDef(String name, String implClass, String label, String description, Map options, List configs, List resources, List accessTypes, List policyConditions, List contextEnrichers, List enums, RangerDataMaskDef dataMaskDef, RangerRowFilterDef rowFilterDef) { + super(); + + setName(name); + setImplClass(implClass); + setLabel(label); + setDescription(description); + setConfigs(configs); + setOptions(options); + setResources(resources); + setAccessTypes(accessTypes); + setPolicyConditions(policyConditions); + setContextEnrichers(contextEnrichers); + setEnums(enums); + setDataMaskDef(dataMaskDef); + setRowFilterDef(rowFilterDef); + } + + public RangerServiceDef(String name, String displayName, String implClass, String label, String description, + Map options, List configs, + List modifiedResourceDefs, List accessTypes, + List policyConditions, List contextEnrichers, + List enums) { + this(name, implClass, label, description, options, configs, modifiedResourceDefs, accessTypes, policyConditions, contextEnrichers, enums); + this.setDisplayName(displayName); + } + + /** + * @param other + */ + public void updateFrom(RangerServiceDef other) { + super.updateFrom(other); + + setName(other.getName()); + setDisplayName(other.getDisplayName()); + setImplClass(other.getImplClass()); + setLabel(other.getLabel()); + setDescription(other.getDescription()); + setRbKeyLabel(other.getRbKeyLabel()); + setRbKeyDescription(other.getRbKeyDescription()); + setOptions(other.getOptions()); + setConfigs(other.getConfigs()); + setResources(other.getResources()); + setAccessTypes(other.getAccessTypes()); + setPolicyConditions(other.getPolicyConditions()); + setContextEnrichers(other.getContextEnrichers()); + setEnums(other.getEnums()); + setDataMaskDef(other.getDataMaskDef()); + setRowFilterDef(other.getRowFilterDef()); + } + + /** + * @return the name + */ + public String getName() { + return name; + } + + /** + * @param name the name to set + */ + public void setName(String name) { + this.name = name; + } + + /** + * @return the implClass + */ + public String getImplClass() { + return implClass; + } + + /** + * @param implClass the implClass to set + */ + public void setImplClass(String implClass) { + this.implClass = implClass; + } + + /** + * @return the label + */ + public String getLabel() { + return label; + } + + /** + * @param label the label to set + */ + public void setLabel(String label) { + this.label = label; + } + + /** + * @return the description + */ + public String getDescription() { + return description; + } + + /** + * @param description the description to set + */ + public void setDescription(String description) { + this.description = description; + } + + /** + * @return the rbKeyLabel + */ + public String getRbKeyLabel() { + return rbKeyLabel; + } + + /** + * @param rbKeyLabel the rbKeyLabel to set + */ + public void setRbKeyLabel(String rbKeyLabel) { + this.rbKeyLabel = rbKeyLabel; + } + + /** + * @return the rbKeyDescription + */ + public String getRbKeyDescription() { + return rbKeyDescription; + } + + /** + * @param rbKeyDescription the rbKeyDescription to set + */ + public void setRbKeyDescription(String rbKeyDescription) { + this.rbKeyDescription = rbKeyDescription; + } + + /** + * @return the configs + */ + public List getConfigs() { + return configs; + } + + /** + * @param configs the configs to set + */ + public void setConfigs(List configs) { + if(this.configs == null) { + this.configs = new ArrayList<>(); + } else + + if(this.configs == configs) { + return; + } + + this.configs.clear(); + + if(configs != null) { + this.configs.addAll(configs); + } + } + + /** + * @return the options + */ + public Map getOptions() { + return options; + } + + /** + * @param options the options to set + */ + public void setOptions(Map options) { + if(this.options == null) { + this.options = new HashMap<>(); + } else if(this.options == options) { + return; + } + + this.options.clear(); + + if(options != null) { + for(Map.Entry entry : options.entrySet()) { + this.options.put(entry.getKey(), entry.getValue()); + } + } + } + + /** + * @return the resources + */ + public List getResources() { + return resources; + } + + /** + * @param resources the resources to set + */ + public void setResources(List resources) { + if(this.resources == null) { + this.resources = new ArrayList<>(); + } + + if(this.resources == resources) { + return; + } + + this.resources.clear(); + + if(resources != null) { + this.resources.addAll(resources); + } + } + + /** + * @return the accessTypes + */ + public List getAccessTypes() { + return accessTypes; + } + + /** + * @param accessTypes the accessTypes to set + */ + public void setAccessTypes(List accessTypes) { + if(this.accessTypes == null) { + this.accessTypes = new ArrayList<>(); + } + + if(this.accessTypes == accessTypes) { + return; + } + + this.accessTypes.clear(); + + if(accessTypes != null) { + this.accessTypes.addAll(accessTypes); + } + } + + /** + * @return the policyConditions + */ + public List getPolicyConditions() { + return policyConditions; + } + + /** + * @param policyConditions the policyConditions to set + */ + public void setPolicyConditions(List policyConditions) { + if(this.policyConditions == null) { + this.policyConditions = new ArrayList<>(); + } + + if(this.policyConditions == policyConditions) { + return; + } + + this.policyConditions.clear(); + + if(policyConditions != null) { + this.policyConditions.addAll(policyConditions); + } + } + + /** + * @return the contextEnrichers + */ + public List getContextEnrichers() { + return contextEnrichers; + } + + /** + * @param contextEnrichers the contextEnrichers to set + */ + public void setContextEnrichers(List contextEnrichers) { + if(this.contextEnrichers == null) { + this.contextEnrichers = new ArrayList<>(); + } + + if(this.contextEnrichers == contextEnrichers) { + return; + } + + this.contextEnrichers.clear(); + + if(contextEnrichers != null) { + this.contextEnrichers.addAll(contextEnrichers); + } + } + + /** + * @return the enums + */ + public List getEnums() { + return enums; + } + + /** + * @param enums the enums to set + */ + public void setEnums(List enums) { + if(this.enums == null) { + this.enums = new ArrayList<>(); + } + + if(this.enums == enums) { + return; + } + + this.enums.clear(); + + if(enums != null) { + this.enums.addAll(enums); + } + } + + public RangerDataMaskDef getDataMaskDef() { + return dataMaskDef; + } + + public void setDataMaskDef(RangerDataMaskDef dataMaskDef) { + this.dataMaskDef = dataMaskDef == null ? new RangerDataMaskDef() : dataMaskDef; + } + + public RangerRowFilterDef getRowFilterDef() { + return rowFilterDef; + } + + public void setRowFilterDef(RangerRowFilterDef rowFilterDef) { + this.rowFilterDef = rowFilterDef == null ? new RangerRowFilterDef() : rowFilterDef; + } + + public String getDisplayName() { + return displayName; + } + + public void setDisplayName(String displayName) { + this.displayName = displayName; + } + + @Override + public String toString( ) { + StringBuilder sb = new StringBuilder(); + + toString(sb); + + return sb.toString(); + } + + public StringBuilder toString(StringBuilder sb) { + sb.append("RangerServiceDef={"); + + super.toString(sb); + + sb.append("name={").append(name).append("} "); + sb.append("displayName={").append(displayName).append("} "); + sb.append("implClass={").append(implClass).append("} "); + sb.append("label={").append(label).append("} "); + sb.append("description={").append(description).append("} "); + sb.append("rbKeyLabel={").append(rbKeyLabel).append("} "); + sb.append("rbKeyDescription={").append(rbKeyDescription).append("} "); + + sb.append("options={"); + if(options != null) { + for(Map.Entry entry : options.entrySet()) { + sb.append(entry.getKey()).append("=").append(entry.getValue()).append(" "); + } + } + sb.append("} "); + + sb.append("configs={"); + if(configs != null) { + for(RangerServiceConfigDef config : configs) { + if(config != null) { + config.toString(sb); + } + } + } + sb.append("} "); + + sb.append("resources={"); + if(resources != null) { + for(RangerResourceDef resource : resources) { + if(resource != null) { + resource.toString(sb); + } + } + } + sb.append("} "); + + sb.append("accessTypes={"); + if(accessTypes != null) { + for(RangerAccessTypeDef accessType : accessTypes) { + if(accessType != null) { + accessType.toString(sb); + } + } + } + sb.append("} "); + + sb.append("policyConditions={"); + if(policyConditions != null) { + for(RangerPolicyConditionDef policyCondition : policyConditions) { + if(policyCondition != null) { + policyCondition.toString(sb); + } + } + } + sb.append("} "); + + sb.append("contextEnrichers={"); + if(contextEnrichers != null) { + for(RangerContextEnricherDef contextEnricher : contextEnrichers) { + if(contextEnricher != null) { + contextEnricher.toString(sb); + } + } + } + sb.append("} "); + + sb.append("enums={"); + if(enums != null) { + for(RangerEnumDef e : enums) { + if(e != null) { + e.toString(sb); + } + } + } + sb.append("} "); + + sb.append("dataMaskDef={"); + if(dataMaskDef != null) { + dataMaskDef.toString(sb); + } + sb.append("} "); + + sb.append("rowFilterDef={"); + if(rowFilterDef != null) { + rowFilterDef.toString(sb); + } + sb.append("} "); + + sb.append("}"); + + return sb; + } + + + @JsonInclude(JsonInclude.Include.NON_NULL) + @XmlRootElement + @XmlAccessorType(XmlAccessType.FIELD) + public static class RangerEnumDef implements java.io.Serializable { + private static final long serialVersionUID = 1L; + + private Long itemId; + private String name; + private List elements; + private Integer defaultIndex; + + public RangerEnumDef() { + this(null, null, null, null); + } + + public RangerEnumDef(Long itemId, String name, List elements, Integer defaultIndex) { + setItemId(itemId); + setName(name); + setElements(elements); + setDefaultIndex(defaultIndex); + } + + /** + * @return the itemId + */ + public Long getItemId() { + return itemId; + } + + /** + * @param itemId the itemId to set + */ + public void setItemId(Long itemId) { + this.itemId = itemId; + } + + /** + * @return the name + */ + public String getName() { + return name; + } + + /** + * @param name the name to set + */ + public void setName(String name) { + this.name = name; + } + + /** + * @return the elements + */ + public List getElements() { + return elements; + } + + /** + * @param elements the elements to set + */ + public void setElements(List elements) { + if(this.elements == null) { + this.elements = new ArrayList<>(); + } + + if(this.elements == elements) { + return; + } + + this.elements.clear(); + + if(elements != null) { + this.elements.addAll(elements); + } + } + + /** + * @return the defaultIndex + */ + public Integer getDefaultIndex() { + return defaultIndex; + } + + /** + * @param defaultIndex the defaultIndex to set + */ + public void setDefaultIndex(Integer defaultIndex) { + this.defaultIndex = (defaultIndex != null && this.elements.size() > defaultIndex) ? defaultIndex : 0; + } + + @Override + public String toString( ) { + StringBuilder sb = new StringBuilder(); + + toString(sb); + + return sb.toString(); + } + + public StringBuilder toString(StringBuilder sb) { + sb.append("RangerEnumDef={"); + sb.append("itemId={").append(itemId).append("} "); + sb.append("name={").append(name).append("} "); + sb.append("elements={"); + if(elements != null) { + for(RangerEnumElementDef element : elements) { + if(element != null) { + element.toString(sb); + } + } + } + sb.append("} "); + sb.append("defaultIndex={").append(defaultIndex).append("} "); + sb.append("}"); + + return sb; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((itemId == null) ? 0 : itemId.hashCode()); + result = prime * result + + ((defaultIndex == null) ? 0 : defaultIndex.hashCode()); + result = prime * result + + ((elements == null) ? 0 : elements.hashCode()); + result = prime * result + ((name == null) ? 0 : name.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + RangerEnumDef other = (RangerEnumDef) obj; + if (itemId == null) { + if (other.itemId != null) + return false; + } else if (other.itemId == null || !itemId.equals(other.itemId)) + return false; + + if (defaultIndex == null) { + if (other.defaultIndex != null) + return false; + } else if (!defaultIndex.equals(other.defaultIndex)) + return false; + if (elements == null) { + if (other.elements != null) + return false; + } else if (!elements.equals(other.elements)) + return false; + if (name == null) { + if (other.name != null) + return false; + } else if (!name.equals(other.name)) + return false; + return true; + } + } + + + @JsonInclude(JsonInclude.Include.NON_NULL) + @XmlRootElement + @XmlAccessorType(XmlAccessType.FIELD) + public static class RangerEnumElementDef implements java.io.Serializable { + private static final long serialVersionUID = 1L; + + private Long itemId; + private String name; + private String label; + private String rbKeyLabel; + + public RangerEnumElementDef() { + this(null, null, null, null); + } + + public RangerEnumElementDef(Long itemId, String name, String label, String rbKeyLabel) { + setItemId(itemId); + setName(name); + setLabel(label); + setRbKeyLabel(rbKeyLabel); + } + + /** + * @return the itemId + */ + public Long getItemId() { + return itemId; + } + + /** + * @param itemId the itemId to set + */ + public void setItemId(Long itemId) { + this.itemId = itemId; + } + + /** + * @return the name + */ + public String getName() { + return name; + } + + /** + * @param name the name to set + */ + public void setName(String name) { + this.name = name; + } + + /** + * @return the label + */ + public String getLabel() { + return label; + } + + /** + * @param label the label to set + */ + public void setLabel(String label) { + this.label = label; + } + + /** + * @return the rbKeyLabel + */ + public String getRbKeyLabel() { + return rbKeyLabel; + } + + /** + * @param rbKeyLabel the rbKeyLabel to set + */ + public void setRbKeyLabel(String rbKeyLabel) { + this.rbKeyLabel = rbKeyLabel; + } + + @Override + public String toString( ) { + StringBuilder sb = new StringBuilder(); + + toString(sb); + + return sb.toString(); + } + + public StringBuilder toString(StringBuilder sb) { + sb.append("RangerEnumElementDef={"); + sb.append("itemId={").append(itemId).append("} "); + sb.append("name={").append(name).append("} "); + sb.append("label={").append(label).append("} "); + sb.append("rbKeyLabel={").append(rbKeyLabel).append("} "); + sb.append("}"); + + return sb; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((itemId == null) ? 0 : itemId.hashCode()); + result = prime * result + ((label == null) ? 0 : label.hashCode()); + result = prime * result + ((name == null) ? 0 : name.hashCode()); + result = prime * result + + ((rbKeyLabel == null) ? 0 : rbKeyLabel.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + RangerEnumElementDef other = (RangerEnumElementDef) obj; + if (itemId == null) { + if (other.itemId != null) { + return false; + } + } else if (other.itemId == null || !itemId.equals(other.itemId)) { + return false; + } + + if (label == null) { + if (other.label != null) + return false; + } else if (!label.equals(other.label)) + return false; + if (name == null) { + if (other.name != null) + return false; + } else if (!name.equals(other.name)) + return false; + if (rbKeyLabel == null) { + if (other.rbKeyLabel != null) + return false; + } else if (!rbKeyLabel.equals(other.rbKeyLabel)) + return false; + return true; + } + } + + + @JsonInclude(JsonInclude.Include.NON_NULL) + @XmlRootElement + @XmlAccessorType(XmlAccessType.FIELD) + public static class RangerServiceConfigDef implements java.io.Serializable { + private static final long serialVersionUID = 1L; + + private Long itemId; + private String name; + private String type; + private String subType; + private Boolean mandatory; + private String defaultValue; + private String validationRegEx; + private String validationMessage; + private String uiHint; + private String label; + private String description; + private String rbKeyLabel; + private String rbKeyDescription; + private String rbKeyValidationMessage; + + public RangerServiceConfigDef() { + this(null, null, null, null, null, null, null, null, null, null, null, null, null, null); + } + + public RangerServiceConfigDef(Long itemId, String name, String type, String subType, Boolean mandatory, String defaultValue, String validationRegEx, String validationMessage, String uiHint, String label, String description, String rbKeyLabel, String rbKeyDescription, String rbKeyValidationMessage) { + setItemId(itemId); + setName(name); + setType(type); + setSubType(subType); + setMandatory(mandatory); + setDefaultValue(defaultValue); + setValidationRegEx(validationRegEx); + setValidationMessage(validationMessage); + setUiHint(uiHint); + setLabel(label); + setDescription(description); + setRbKeyLabel(rbKeyLabel); + setRbKeyDescription(rbKeyDescription); + setRbKeyValidationMessage(rbKeyValidationMessage); + } + + /** + * @return the itemId + */ + public Long getItemId() { + return itemId; + } + + /** + * @param itemId the itemId to set + */ + public void setItemId(Long itemId) { + this.itemId = itemId; + } + + /** + * @return the name + */ + public String getName() { + return name; + } + + /** + * @param name the name to set + */ + public void setName(String name) { + this.name = name; + } + + /** + * @return the type + */ + public String getType() { + return type; + } + + /** + * @param type the type to set + */ + public void setType(String type) { + this.type = type; + } + + /** + * @return the subType + */ + public String getSubType() { + return subType; + } + + /** + * @param subType the subType to set + */ + public void setSubType(String subType) { + this.subType = subType; + } + + /** + * @return the mandatory + */ + public Boolean getMandatory() { + return mandatory; + } + + /** + * @param mandatory the mandatory to set + */ + public void setMandatory(Boolean mandatory) { + this.mandatory = mandatory == null ? Boolean.FALSE : mandatory; + } + + /** + * @return the defaultValue + */ + public String getDefaultValue() { + return defaultValue; + } + + /** + * @param defaultValue the defaultValue to set + */ + public void setDefaultValue(String defaultValue) { + this.defaultValue = defaultValue; + } + + /** + * @return the validationRegEx + */ + public String getValidationRegEx() { + return validationRegEx; + } + + /** + * @param validationRegEx the validationRegEx to set + */ + public void setValidationRegEx(String validationRegEx) { + this.validationRegEx = validationRegEx; + } + + /** + * @return the validationMessage + */ + public String getValidationMessage() { + return validationMessage; + } + + /** + * @param validationMessage the validationMessage to set + */ + public void setValidationMessage(String validationMessage) { + this.validationMessage = validationMessage; + } + + /** + * @return the uiHint + */ + public String getUiHint() { + return uiHint; + } + + /** + * @param uiHint the uiHint to set + */ + public void setUiHint(String uiHint) { + this.uiHint = uiHint; + } + + /** + * @return the label + */ + public String getLabel() { + return label; + } + + /** + * @param label the label to set + */ + public void setLabel(String label) { + this.label = label; + } + + /** + * @return the description + */ + public String getDescription() { + return description; + } + + /** + * @param description the description to set + */ + public void setDescription(String description) { + this.description = description; + } + + /** + * @return the rbKeyLabel + */ + public String getRbKeyLabel() { + return rbKeyLabel; + } + + /** + * @param rbKeyLabel the rbKeyLabel to set + */ + public void setRbKeyLabel(String rbKeyLabel) { + this.rbKeyLabel = rbKeyLabel; + } + + /** + * @return the rbKeyDescription + */ + public String getRbKeyDescription() { + return rbKeyDescription; + } + + /** + * @param rbKeyDescription the rbKeyDescription to set + */ + public void setRbKeyDescription(String rbKeyDescription) { + this.rbKeyDescription = rbKeyDescription; + } + + /** + * @return the rbKeyValidationMessage + */ + public String getRbKeyValidationMessage() { + return rbKeyValidationMessage; + } + + /** + * @param rbKeyValidationMessage the rbKeyValidationMessage to set + */ + public void setRbKeyValidationMessage(String rbKeyValidationMessage) { + this.rbKeyValidationMessage = rbKeyValidationMessage; + } + + @Override + public String toString( ) { + StringBuilder sb = new StringBuilder(); + + toString(sb); + + return sb.toString(); + } + + public StringBuilder toString(StringBuilder sb) { + sb.append("RangerServiceConfigDef={"); + sb.append("itemId={").append(name).append("} "); + sb.append("name={").append(name).append("} "); + sb.append("type={").append(type).append("} "); + sb.append("subType={").append(subType).append("} "); + sb.append("mandatory={").append(mandatory).append("} "); + sb.append("defaultValue={").append(defaultValue).append("} "); + sb.append("validationRegEx={").append(validationRegEx).append("} "); + sb.append("validationMessage={").append(validationMessage).append("} "); + sb.append("uiHint={").append(uiHint).append("} "); + sb.append("label={").append(label).append("} "); + sb.append("description={").append(description).append("} "); + sb.append("rbKeyLabel={").append(rbKeyLabel).append("} "); + sb.append("rbKeyDescription={").append(rbKeyDescription).append("} "); + sb.append("rbKeyValidationMessage={").append(rbKeyValidationMessage).append("} "); + sb.append("}"); + + return sb; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + + ((defaultValue == null) ? 0 : defaultValue.hashCode()); + result = prime * result + + ((description == null) ? 0 : description.hashCode()); + result = prime * result + ((label == null) ? 0 : label.hashCode()); + result = prime * result + + ((mandatory == null) ? 0 : mandatory.hashCode()); + result = prime * result + ((name == null) ? 0 : name.hashCode()); + result = prime + * result + + ((rbKeyDescription == null) ? 0 : rbKeyDescription + .hashCode()); + result = prime * result + + ((rbKeyLabel == null) ? 0 : rbKeyLabel.hashCode()); + result = prime + * result + + ((rbKeyValidationMessage == null) ? 0 + : rbKeyValidationMessage.hashCode()); + result = prime * result + + ((subType == null) ? 0 : subType.hashCode()); + result = prime * result + ((type == null) ? 0 : type.hashCode()); + result = prime * result + + ((uiHint == null) ? 0 : uiHint.hashCode()); + result = prime + * result + + ((validationMessage == null) ? 0 : validationMessage + .hashCode()); + result = prime + * result + + ((validationRegEx == null) ? 0 : validationRegEx + .hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + RangerServiceConfigDef other = (RangerServiceConfigDef) obj; + if (defaultValue == null) { + if (other.defaultValue != null) + return false; + } else if (!defaultValue.equals(other.defaultValue)) + return false; + if (description == null) { + if (other.description != null) + return false; + } else if (!description.equals(other.description)) + return false; + if (label == null) { + if (other.label != null) + return false; + } else if (!label.equals(other.label)) + return false; + if (mandatory == null) { + if (other.mandatory != null) + return false; + } else if (!mandatory.equals(other.mandatory)) + return false; + if (name == null) { + if (other.name != null) + return false; + } else if (!name.equals(other.name)) + return false; + if (rbKeyDescription == null) { + if (other.rbKeyDescription != null) + return false; + } else if (!rbKeyDescription.equals(other.rbKeyDescription)) + return false; + if (rbKeyLabel == null) { + if (other.rbKeyLabel != null) + return false; + } else if (!rbKeyLabel.equals(other.rbKeyLabel)) + return false; + if (rbKeyValidationMessage == null) { + if (other.rbKeyValidationMessage != null) + return false; + } else if (!rbKeyValidationMessage + .equals(other.rbKeyValidationMessage)) + return false; + if (subType == null) { + if (other.subType != null) + return false; + } else if (!subType.equals(other.subType)) + return false; + if (type == null) { + if (other.type != null) + return false; + } else if (!type.equals(other.type)) + return false; + if (uiHint == null) { + if (other.uiHint != null) + return false; + } else if (!uiHint.equals(other.uiHint)) + return false; + if (validationMessage == null) { + if (other.validationMessage != null) + return false; + } else if (!validationMessage.equals(other.validationMessage)) + return false; + if (validationRegEx == null) { + if (other.validationRegEx != null) + return false; + } else if (!validationRegEx.equals(other.validationRegEx)) + return false; + return true; + } + } + + + @JsonInclude(JsonInclude.Include.NON_NULL) + @XmlRootElement + @XmlAccessorType(XmlAccessType.FIELD) + public static class RangerResourceDef implements java.io.Serializable { + private static final long serialVersionUID = 1L; + + private Long itemId = null; + private String name = null; + private String type = null; + private Integer level = null; + private String parent = null; + private Boolean mandatory = null; + private Boolean lookupSupported = null; + private Boolean recursiveSupported = null; + private Boolean excludesSupported = null; + private String matcher = null; + private Map matcherOptions = null; + private String validationRegEx = null; + private String validationMessage = null; + private String uiHint = null; + private String label = null; + private String description = null; + private String rbKeyLabel = null; + private String rbKeyDescription = null; + private String rbKeyValidationMessage = null; + private Set accessTypeRestrictions = null; + private Boolean isValidLeaf = null; + + public RangerResourceDef() { + this(null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null); + } + + public RangerResourceDef(RangerResourceDef other) { + setItemId(other.getItemId()); + setName(other.getName()); + setType(other.getType()); + setLevel(other.getLevel()); + setParent(other.getParent()); + setMandatory(other.getMandatory()); + setLookupSupported(other.getLookupSupported()); + setRecursiveSupported(other.getRecursiveSupported()); + setExcludesSupported(other.getExcludesSupported()); + setMatcher(other.getMatcher()); + setMatcherOptions(other.getMatcherOptions()); + setValidationRegEx(other.getValidationRegEx()); + setValidationMessage(other.getValidationMessage()); + setUiHint(other.getUiHint()); + setLabel(other.getLabel()); + setDescription(other.getDescription()); + setRbKeyLabel(other.getRbKeyLabel()); + setRbKeyDescription(other.getRbKeyDescription()); + setRbKeyValidationMessage(other.getRbKeyValidationMessage()); + setAccessTypeRestrictions(other.getAccessTypeRestrictions()); + setIsValidLeaf(other.getIsValidLeaf()); + } + + public RangerResourceDef(Long itemId, String name, String type, Integer level, String parent, Boolean mandatory, Boolean lookupSupported, Boolean recursiveSupported, Boolean excludesSupported, String matcher, Map matcherOptions, String validationRegEx, String validationMessage, String uiHint, String label, String description, String rbKeyLabel, String rbKeyDescription, String rbKeyValidationMessage, Set accessTypeRestrictions, Boolean isValidLeaf) { + setItemId(itemId); + setName(name); + setType(type); + setLevel(level); + setParent(parent); + setMandatory(mandatory); + setLookupSupported(lookupSupported); + setRecursiveSupported(recursiveSupported); + setExcludesSupported(excludesSupported); + setMatcher(matcher); + setMatcherOptions(matcherOptions); + setValidationRegEx(validationRegEx); + setValidationMessage(validationMessage); + setUiHint(uiHint); + setLabel(label); + setDescription(description); + setRbKeyLabel(rbKeyLabel); + setRbKeyDescription(rbKeyDescription); + setRbKeyValidationMessage(rbKeyValidationMessage); + setAccessTypeRestrictions(accessTypeRestrictions); + setIsValidLeaf(isValidLeaf); + } + + /** + * @return the itemId + */ + public Long getItemId() { + return itemId; + } + + /** + * @param itemId the itemId to set + */ + public void setItemId(Long itemId) { + this.itemId = itemId; + } + + /** + * @return the name + */ + public String getName() { + return name; + } + + /** + * @param name the name to set + */ + public void setName(String name) { + this.name = name; + } + + /** + * @return the type + */ + public String getType() { + return type; + } + + /** + * @param type the type to set + */ + public void setType(String type) { + this.type = type; + } + + /** + * @return the level + */ + public Integer getLevel() { + return level; + } + + /** + * @param level the level to set + */ + public void setLevel(Integer level) { + this.level = level == null ? 1 : level; + } + + /** + * @return the parent + */ + public String getParent() { + return parent; + } + + /** + * @param parent the parent to set + */ + public void setParent(String parent) { + this.parent = parent; + } + + /** + * @return the mandatory + */ + public Boolean getMandatory() { + return mandatory; + } + + /** + * @param mandatory the mandatory to set + */ + public void setMandatory(Boolean mandatory) { + this.mandatory = mandatory == null ? Boolean.FALSE : mandatory; + } + + /** + * @return the lookupSupported + */ + public Boolean getLookupSupported() { + return lookupSupported; + } + + /** + * @param lookupSupported the lookupSupported to set + */ + public void setLookupSupported(Boolean lookupSupported) { + this.lookupSupported = lookupSupported == null ? Boolean.FALSE : lookupSupported; + } + + /** + * @return the recursiveSupported + */ + public Boolean getRecursiveSupported() { + return recursiveSupported; + } + + /** + * @param recursiveSupported the recursiveSupported to set + */ + public void setRecursiveSupported(Boolean recursiveSupported) { + this.recursiveSupported = recursiveSupported == null ? Boolean.FALSE : recursiveSupported; + } + + /** + * @return the excludesSupported + */ + public Boolean getExcludesSupported() { + return excludesSupported; + } + + /** + * @param excludesSupported the excludesSupported to set + */ + public void setExcludesSupported(Boolean excludesSupported) { + this.excludesSupported = excludesSupported == null ? Boolean.FALSE : excludesSupported; + } + + /** + * @return the matcher + */ + public String getMatcher() { + return matcher; + } + + /** + * @param matcher the matcher to set + */ + public void setMatcher(String matcher) { + this.matcher = matcher; + } + + /** + * @return the matcherOptions + */ + public Map getMatcherOptions() { + return matcherOptions; + } + + /** + * @param matcherOptions the matcherOptions to set + */ + public void setMatcherOptions(Map matcherOptions) { + this.matcherOptions = matcherOptions == null ? new HashMap() : new HashMap(matcherOptions); + } + + /** + * @return the validationRegEx + */ + public String getValidationRegEx() { + return validationRegEx; + } + + /** + * @param validationRegEx the validationRegEx to set + */ + public void setValidationRegEx(String validationRegEx) { + this.validationRegEx = validationRegEx; + } + + /** + * @return the validationMessage + */ + public String getValidationMessage() { + return validationMessage; + } + + /** + * @param validationMessage the validationMessage to set + */ + public void setValidationMessage(String validationMessage) { + this.validationMessage = validationMessage; + } + + /** + * @return the uiHint + */ + public String getUiHint() { + return uiHint; + } + + /** + * @param uiHint the uiHint to set + */ + public void setUiHint(String uiHint) { + this.uiHint = uiHint; + } + + /** + * @return the label + */ + public String getLabel() { + return label; + } + + /** + * @param label the label to set + */ + public void setLabel(String label) { + this.label = label; + } + + /** + * @return the description + */ + public String getDescription() { + return description; + } + + /** + * @param description the description to set + */ + public void setDescription(String description) { + this.description = description; + } + + /** + * @return the rbKeyLabel + */ + public String getRbKeyLabel() { + return rbKeyLabel; + } + + /** + * @param rbKeyLabel the rbKeyLabel to set + */ + public void setRbKeyLabel(String rbKeyLabel) { + this.rbKeyLabel = rbKeyLabel; + } + + /** + * @return the rbKeyDescription + */ + public String getRbKeyDescription() { + return rbKeyDescription; + } + + /** + * @param rbKeyDescription the rbKeyDescription to set + */ + public void setRbKeyDescription(String rbKeyDescription) { + this.rbKeyDescription = rbKeyDescription; + } + + /** + * @return the rbKeyValidationMessage + */ + public String getRbKeyValidationMessage() { + return rbKeyValidationMessage; + } + + /** + * @param rbKeyValidationMessage the rbKeyValidationMessage to set + */ + public void setRbKeyValidationMessage(String rbKeyValidationMessage) { + this.rbKeyValidationMessage = rbKeyValidationMessage; + } + + public Set getAccessTypeRestrictions() { + return accessTypeRestrictions; + } + + public void setAccessTypeRestrictions(Set accessTypeRestrictions) { + this.accessTypeRestrictions = accessTypeRestrictions == null ? new HashSet() : new HashSet(accessTypeRestrictions); + } + + public Boolean getIsValidLeaf() { return isValidLeaf; } + + public void setIsValidLeaf(Boolean isValidLeaf) { + this.isValidLeaf = isValidLeaf; + } + + @Override + public String toString( ) { + StringBuilder sb = new StringBuilder(); + + toString(sb); + + return sb.toString(); + } + + public StringBuilder toString(StringBuilder sb) { + sb.append("RangerResourceDef={"); + sb.append("itemId={").append(itemId).append("} "); + sb.append("name={").append(name).append("} "); + sb.append("type={").append(type).append("} "); + sb.append("level={").append(level).append("} "); + sb.append("parent={").append(parent).append("} "); + sb.append("mandatory={").append(mandatory).append("} "); + sb.append("lookupSupported={").append(lookupSupported).append("} "); + sb.append("recursiveSupported={").append(recursiveSupported).append("} "); + sb.append("excludesSupported={").append(excludesSupported).append("} "); + sb.append("matcher={").append(matcher).append("} "); + sb.append("matcherOptions={").append(matcherOptions).append("} "); + sb.append("validationRegEx={").append(validationRegEx).append("} "); + sb.append("validationMessage={").append(validationMessage).append("} "); + sb.append("uiHint={").append(uiHint).append("} "); + sb.append("label={").append(label).append("} "); + sb.append("description={").append(description).append("} "); + sb.append("rbKeyLabel={").append(rbKeyLabel).append("} "); + sb.append("rbKeyDescription={").append(rbKeyDescription).append("} "); + sb.append("rbKeyValidationMessage={").append(rbKeyValidationMessage).append("} "); + sb.append("accessTypeRestrictions={").append(accessTypeRestrictions == null ? "null" : accessTypeRestrictions.toString()).append("} "); + sb.append("isValidLeaf={").append(isValidLeaf == null ? "null" : isValidLeaf.toString()).append("} "); + sb.append("}"); + + return sb; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + + ((description == null) ? 0 : description.hashCode()); + result = prime + * result + + ((excludesSupported == null) ? 0 : excludesSupported + .hashCode()); + result = prime * result + ((label == null) ? 0 : label.hashCode()); + result = prime * result + ((level == null) ? 0 : level.hashCode()); + result = prime + * result + + ((lookupSupported == null) ? 0 : lookupSupported + .hashCode()); + result = prime * result + + ((mandatory == null) ? 0 : mandatory.hashCode()); + result = prime * result + + ((matcher == null) ? 0 : matcher.hashCode()); + result = prime + * result + + ((matcherOptions == null) ? 0 : matcherOptions.hashCode()); + result = prime * result + ((name == null) ? 0 : name.hashCode()); + result = prime * result + + ((parent == null) ? 0 : parent.hashCode()); + result = prime + * result + + ((rbKeyDescription == null) ? 0 : rbKeyDescription + .hashCode()); + result = prime * result + + ((rbKeyLabel == null) ? 0 : rbKeyLabel.hashCode()); + result = prime + * result + + ((rbKeyValidationMessage == null) ? 0 + : rbKeyValidationMessage.hashCode()); + result = prime + * result + + ((recursiveSupported == null) ? 0 : recursiveSupported + .hashCode()); + result = prime * result + ((type == null) ? 0 : type.hashCode()); + result = prime * result + + ((uiHint == null) ? 0 : uiHint.hashCode()); + result = prime + * result + + ((validationMessage == null) ? 0 : validationMessage + .hashCode()); + result = prime + * result + + ((validationRegEx == null) ? 0 : validationRegEx + .hashCode()); + result = prime + * result + + ((accessTypeRestrictions == null) ? 0 : accessTypeRestrictions.hashCode()); + result = prime + * result + + ((isValidLeaf == null) ? 0 : isValidLeaf.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + RangerResourceDef other = (RangerResourceDef) obj; + if (description == null) { + if (other.description != null) + return false; + } else if (!description.equals(other.description)) + return false; + if (excludesSupported == null) { + if (other.excludesSupported != null) + return false; + } else if (!excludesSupported.equals(other.excludesSupported)) + return false; + if (label == null) { + if (other.label != null) + return false; + } else if (!label.equals(other.label)) + return false; + if (level == null) { + if (other.level != null) + return false; + } else if (!level.equals(other.level)) + return false; + if (lookupSupported == null) { + if (other.lookupSupported != null) + return false; + } else if (!lookupSupported.equals(other.lookupSupported)) + return false; + if (mandatory == null) { + if (other.mandatory != null) + return false; + } else if (!mandatory.equals(other.mandatory)) + return false; + if (matcher == null) { + if (other.matcher != null) + return false; + } else if (!matcher.equals(other.matcher)) + return false; + if (matcherOptions == null) { + if (other.matcherOptions != null) + return false; + } else if (!matcherOptions.equals(other.matcherOptions)) + return false; + if (name == null) { + if (other.name != null) + return false; + } else if (!name.equals(other.name)) + return false; + if (parent == null) { + if (other.parent != null) + return false; + } else if (!parent.equals(other.parent)) + return false; + if (rbKeyDescription == null) { + if (other.rbKeyDescription != null) + return false; + } else if (!rbKeyDescription.equals(other.rbKeyDescription)) + return false; + if (rbKeyLabel == null) { + if (other.rbKeyLabel != null) + return false; + } else if (!rbKeyLabel.equals(other.rbKeyLabel)) + return false; + if (rbKeyValidationMessage == null) { + if (other.rbKeyValidationMessage != null) + return false; + } else if (!rbKeyValidationMessage + .equals(other.rbKeyValidationMessage)) + return false; + if (recursiveSupported == null) { + if (other.recursiveSupported != null) + return false; + } else if (!recursiveSupported.equals(other.recursiveSupported)) + return false; + if (type == null) { + if (other.type != null) + return false; + } else if (!type.equals(other.type)) + return false; + if (uiHint == null) { + if (other.uiHint != null) + return false; + } else if (!uiHint.equals(other.uiHint)) + return false; + if (validationMessage == null) { + if (other.validationMessage != null) + return false; + } else if (!validationMessage.equals(other.validationMessage)) + return false; + if (validationRegEx == null) { + if (other.validationRegEx != null) + return false; + } else if (!validationRegEx.equals(other.validationRegEx)) + return false; + if (accessTypeRestrictions == null) { + if (other.accessTypeRestrictions != null) + return false; + } else if (!accessTypeRestrictions.equals(other.accessTypeRestrictions)) + return false; + if (isValidLeaf == null) { + if (other.isValidLeaf != null) + return false; + } else if (!isValidLeaf.equals(other.isValidLeaf)) + return false; + return true; + } + + } + + + @JsonInclude(JsonInclude.Include.NON_NULL) + @XmlRootElement + @XmlAccessorType(XmlAccessType.FIELD) + public static class RangerAccessTypeDef implements java.io.Serializable { + private static final long serialVersionUID = 1L; + + private Long itemId; + private String name; + private String label; + private String rbKeyLabel; + private Collection impliedGrants; + + public RangerAccessTypeDef() { + this(null, null, null, null, null); + } + + public RangerAccessTypeDef(Long itemId, String name, String label, String rbKeyLabel, Collection impliedGrants) { + setItemId(itemId); + setName(name); + setLabel(label); + setRbKeyLabel(rbKeyLabel); + setImpliedGrants(impliedGrants); + } + + public RangerAccessTypeDef(RangerAccessTypeDef other) { + setItemId(other.getItemId()); + setName(other.getName()); + setLabel(other.getLabel()); + setRbKeyLabel(other.getRbKeyLabel()); + setImpliedGrants(other.getImpliedGrants()); + } + + /** + * @return the itemId + */ + public Long getItemId() { + return itemId; + } + + /** + * @param itemId the itemId to set + */ + public void setItemId(Long itemId) { + this.itemId = itemId; + } + + /** + * @return the name + */ + public String getName() { + return name; + } + + /** + * @param name the name to set + */ + public void setName(String name) { + this.name = name; + } + + /** + * @return the label + */ + public String getLabel() { + return label; + } + + /** + * @param label the label to set + */ + public void setLabel(String label) { + this.label = label; + } + + /** + * @return the rbKeyLabel + */ + public String getRbKeyLabel() { + return rbKeyLabel; + } + + /** + * @param rbKeyLabel the rbKeyLabel to set + */ + public void setRbKeyLabel(String rbKeyLabel) { + this.rbKeyLabel = rbKeyLabel; + } + + /** + * @return the impliedGrants + */ + public Collection getImpliedGrants() { + return impliedGrants; + } + + /** + * @param impliedGrants the impliedGrants to set + */ + public void setImpliedGrants(Collection impliedGrants) { + if(this.impliedGrants == null) { + this.impliedGrants = new ArrayList<>(); + } + + if(this.impliedGrants == impliedGrants) { + return; + } + + this.impliedGrants.clear(); + + if(impliedGrants != null) { + this.impliedGrants.addAll(impliedGrants); + } + } + + @Override + public String toString( ) { + StringBuilder sb = new StringBuilder(); + + toString(sb); + + return sb.toString(); + } + + public StringBuilder toString(StringBuilder sb) { + sb.append("RangerAccessTypeDef={"); + sb.append("itemId={").append(itemId).append("} "); + sb.append("name={").append(name).append("} "); + sb.append("label={").append(label).append("} "); + sb.append("rbKeyLabel={").append(rbKeyLabel).append("} "); + + sb.append("impliedGrants={"); + if(impliedGrants != null) { + for(String impliedGrant : impliedGrants) { + if(impliedGrant != null) { + sb.append(impliedGrant).append(" "); + } + } + } + sb.append("} "); + + sb.append("}"); + + return sb; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((itemId == null) ? 0 : itemId.hashCode()); + result = prime * result + + ((impliedGrants == null) ? 0 : impliedGrants.hashCode()); + result = prime * result + ((label == null) ? 0 : label.hashCode()); + result = prime * result + ((name == null) ? 0 : name.hashCode()); + result = prime * result + + ((rbKeyLabel == null) ? 0 : rbKeyLabel.hashCode()); + + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + RangerAccessTypeDef other = (RangerAccessTypeDef) obj; + if (itemId == null) { + if (other.itemId != null) + return false; + } else if (other.itemId == null || !itemId.equals(other.itemId)) + return false; + + if (impliedGrants == null) { + if (other.impliedGrants != null) + return false; + } else if (!impliedGrants.equals(other.impliedGrants)) + return false; + if (label == null) { + if (other.label != null) + return false; + } else if (!label.equals(other.label)) + return false; + if (name == null) { + if (other.name != null) + return false; + } else if (!name.equals(other.name)) + return false; + if (rbKeyLabel == null) { + if (other.rbKeyLabel != null) + return false; + } else if (!rbKeyLabel.equals(other.rbKeyLabel)) + return false; + return true; + } + } + + + @JsonInclude(JsonInclude.Include.NON_NULL) + @XmlRootElement + @XmlAccessorType(XmlAccessType.FIELD) + public static class RangerPolicyConditionDef implements java.io.Serializable { + private static final long serialVersionUID = 1L; + + private Long itemId; + private String name; + private String evaluator; + private Map evaluatorOptions; + private String validationRegEx; + private String validationMessage; + private String uiHint; + private String label; + private String description; + private String rbKeyLabel; + private String rbKeyDescription; + private String rbKeyValidationMessage; + + public RangerPolicyConditionDef() { + this(null, null, null, null, null, null, null, null, null, null, null, null); + } + + public RangerPolicyConditionDef(Long itemId, String name, String evaluator, Map evaluatorOptions) { + this(itemId, name, evaluator, evaluatorOptions, null, null, null, null, null, null, null, null); + } + + public RangerPolicyConditionDef(Long itemId, String name, String evaluator, Map evaluatorOptions, String validationRegEx, String vaidationMessage, String uiHint, String label, String description, String rbKeyLabel, String rbKeyDescription, String rbKeyValidationMessage) { //NOPMD + setItemId(itemId); + setName(name); + setEvaluator(evaluator); + setEvaluatorOptions(evaluatorOptions); + setValidationRegEx(validationRegEx); + setValidationMessage(validationMessage); + setUiHint(uiHint); + setLabel(label); + setDescription(description); + setRbKeyLabel(rbKeyLabel); + setRbKeyDescription(rbKeyDescription); + setRbKeyValidationMessage(rbKeyValidationMessage); + } + + /** + * @return the itemId + */ + public Long getItemId() { + return itemId; + } + + /** + * @param itemId the itemId to set + */ + public void setItemId(Long itemId) { + this.itemId = itemId; + } + + /** + * @return the name + */ + public String getName() { + return name; + } + + /** + * @param name the name to set + */ + public void setName(String name) { + this.name = name; + } + + /** + * @return the evaluator + */ + public String getEvaluator() { + return evaluator; + } + + /** + * @param evaluator the evaluator to set + */ + public void setEvaluator(String evaluator) { + this.evaluator = evaluator; + } + + /** + * @return the evaluatorOptions + */ + public Map getEvaluatorOptions() { + return evaluatorOptions; + } + + /** + * @param evaluatorOptions the evaluatorOptions to set + */ + public void setEvaluatorOptions(Map evaluatorOptions) { + this.evaluatorOptions = evaluatorOptions == null ? new HashMap() : evaluatorOptions; + } + + /** + * @return the validationRegEx + */ + public String getValidationRegEx() { + return validationRegEx; + } + + /** + * @param validationRegEx the validationRegEx to set + */ + public void setValidationRegEx(String validationRegEx) { + this.validationRegEx = validationRegEx; + } + + /** + * @return the validationMessage + */ + public String getValidationMessage() { + return validationMessage; + } + + /** + * @param validationMessage the validationMessage to set + */ + public void setValidationMessage(String validationMessage) { + this.validationMessage = validationMessage; + } + + /** + * @return the uiHint + */ + public String getUiHint() { + return uiHint; + } + + /** + * @param uiHint the uiHint to set + */ + public void setUiHint(String uiHint) { + this.uiHint = uiHint; + } + + /** + * @return the label + */ + public String getLabel() { + return label; + } + + /** + * @param label the label to set + */ + public void setLabel(String label) { + this.label = label; + } + + /** + * @return the description + */ + public String getDescription() { + return description; + } + + /** + * @param description the description to set + */ + public void setDescription(String description) { + this.description = description; + } + + /** + * @return the rbKeyLabel + */ + public String getRbKeyLabel() { + return rbKeyLabel; + } + + /** + * @param rbKeyLabel the rbKeyLabel to set + */ + public void setRbKeyLabel(String rbKeyLabel) { + this.rbKeyLabel = rbKeyLabel; + } + + /** + * @return the rbKeyDescription + */ + public String getRbKeyDescription() { + return rbKeyDescription; + } + + /** + * @param rbKeyDescription the rbKeyDescription to set + */ + public void setRbKeyDescription(String rbKeyDescription) { + this.rbKeyDescription = rbKeyDescription; + } + + /** + * @return the rbKeyValidationMessage + */ + public String getRbKeyValidationMessage() { + return rbKeyValidationMessage; + } + + /** + * @param rbKeyValidationMessage the rbKeyValidationMessage to set + */ + public void setRbKeyValidationMessage(String rbKeyValidationMessage) { + this.rbKeyValidationMessage = rbKeyValidationMessage; + } + + @Override + public String toString( ) { + StringBuilder sb = new StringBuilder(); + + toString(sb); + + return sb.toString(); + } + + public StringBuilder toString(StringBuilder sb) { + sb.append("RangerPolicyConditionDef={"); + sb.append("itemId={").append(itemId).append("} "); + sb.append("name={").append(name).append("} "); + sb.append("evaluator={").append(evaluator).append("} "); + sb.append("evaluatorOptions={").append(evaluatorOptions).append("} "); + sb.append("validationRegEx={").append(validationRegEx).append("} "); + sb.append("validationMessage={").append(validationMessage).append("} "); + sb.append("uiHint={").append(uiHint).append("} "); + sb.append("label={").append(label).append("} "); + sb.append("description={").append(description).append("} "); + sb.append("rbKeyLabel={").append(rbKeyLabel).append("} "); + sb.append("rbKeyDescription={").append(rbKeyDescription).append("} "); + sb.append("rbKeyValidationMessage={").append(rbKeyValidationMessage).append("} "); + sb.append("}"); + + return sb; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + + ((itemId == null) ? 0 : itemId.hashCode()); + result = prime * result + + ((description == null) ? 0 : description.hashCode()); + result = prime * result + + ((evaluator == null) ? 0 : evaluator.hashCode()); + result = prime + * result + + ((evaluatorOptions == null) ? 0 : evaluatorOptions + .hashCode()); + result = prime * result + ((label == null) ? 0 : label.hashCode()); + result = prime * result + ((name == null) ? 0 : name.hashCode()); + result = prime + * result + + ((rbKeyDescription == null) ? 0 : rbKeyDescription + .hashCode()); + result = prime * result + + ((rbKeyLabel == null) ? 0 : rbKeyLabel.hashCode()); + result = prime + * result + + ((rbKeyValidationMessage == null) ? 0 + : rbKeyValidationMessage.hashCode()); + result = prime * result + + ((uiHint == null) ? 0 : uiHint.hashCode()); + result = prime + * result + + ((validationMessage == null) ? 0 : validationMessage + .hashCode()); + result = prime + * result + + ((validationRegEx == null) ? 0 : validationRegEx + .hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + RangerPolicyConditionDef other = (RangerPolicyConditionDef) obj; + if (itemId == null) { + if (other.itemId != null) + return false; + } else if (other.itemId != null || !itemId.equals(other.itemId)) { + return false; + } + + if (description == null) { + if (other.description != null) + return false; + } else if (!description.equals(other.description)) + return false; + if (evaluator == null) { + if (other.evaluator != null) + return false; + } else if (!evaluator.equals(other.evaluator)) + return false; + if (evaluatorOptions == null) { + if (other.evaluatorOptions != null) + return false; + } else if (!evaluatorOptions.equals(other.evaluatorOptions)) + return false; + if (label == null) { + if (other.label != null) + return false; + } else if (!label.equals(other.label)) + return false; + if (name == null) { + if (other.name != null) + return false; + } else if (!name.equals(other.name)) + return false; + if (rbKeyDescription == null) { + if (other.rbKeyDescription != null) + return false; + } else if (!rbKeyDescription.equals(other.rbKeyDescription)) + return false; + if (rbKeyLabel == null) { + if (other.rbKeyLabel != null) + return false; + } else if (!rbKeyLabel.equals(other.rbKeyLabel)) + return false; + if (rbKeyValidationMessage == null) { + if (other.rbKeyValidationMessage != null) + return false; + } else if (!rbKeyValidationMessage + .equals(other.rbKeyValidationMessage)) + return false; + if (uiHint == null) { + if (other.uiHint != null) + return false; + } else if (!uiHint.equals(other.uiHint)) + return false; + if (validationMessage == null) { + if (other.validationMessage != null) + return false; + } else if (!validationMessage.equals(other.validationMessage)) + return false; + if (validationRegEx == null) { + if (other.validationRegEx != null) + return false; + } else if (!validationRegEx.equals(other.validationRegEx)) + return false; + return true; + } + } + + @JsonInclude(JsonInclude.Include.NON_NULL) + @XmlRootElement + @XmlAccessorType(XmlAccessType.FIELD) + public static class RangerContextEnricherDef implements java.io.Serializable { + private static final long serialVersionUID = 1L; + + private Long itemId; + private String name; + private String enricher; + private Map enricherOptions; + + public RangerContextEnricherDef() { + this(null, null, null, null); + } + + public RangerContextEnricherDef(Long itemId, String name, String enricher, Map enricherOptions) { + setItemId(itemId); + setName(name); + setEnricher(enricher); + setEnricherOptions(enricherOptions); + } + + /** + * @return the itemId + */ + public Long getItemId() { + return itemId; + } + + /** + * @param itemId the itemId to set + */ + public void setItemId(Long itemId) { + this.itemId = itemId; + } + + /** + * @return the name + */ + public String getName() { + return name; + } + + /** + * @param name the name to set + */ + public void setName(String name) { + this.name = name; + } + + /** + * @return the enricher + */ + public String getEnricher() { + return enricher; + } + + /** + * @param enricher the enricher to set + */ + public void setEnricher(String enricher) { + this.enricher = enricher; + } + + /** + * @return the enricherOptions + */ + public Map getEnricherOptions() { + return enricherOptions; + } + + /** + * @param enricherOptions the enricherOptions to set + */ + public void setEnricherOptions(Map enricherOptions) { + this.enricherOptions = enricherOptions == null ? new HashMap() : enricherOptions; + } + + @Override + public String toString( ) { + StringBuilder sb = new StringBuilder(); + + toString(sb); + + return sb.toString(); + } + + public StringBuilder toString(StringBuilder sb) { + sb.append("RangerContextEnricherDef={"); + sb.append("itemId={").append(itemId).append("} "); + sb.append("name={").append(name).append("} "); + sb.append("enricher={").append(enricher).append("} "); + sb.append("enricherOptions={").append(enricherOptions).append("} "); + sb.append("}"); + + return sb; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((itemId == null) ? 0 : itemId.hashCode()); + result = prime * result + + ((enricher == null) ? 0 : enricher.hashCode()); + result = prime + * result + + ((enricherOptions == null) ? 0 : enricherOptions + .hashCode()); + result = prime * result + ((name == null) ? 0 : name.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + RangerContextEnricherDef other = (RangerContextEnricherDef) obj; + if (itemId == null) { + if (other.itemId != null) + return false; + } else if (other.itemId == null || !itemId.equals(other.itemId)) + return false; + + if (enricher == null) { + if (other.enricher != null) + return false; + } else if (!enricher.equals(other.enricher)) + return false; + if (enricherOptions == null) { + if (other.enricherOptions != null) + return false; + } else if (!enricherOptions.equals(other.enricherOptions)) + return false; + if (name == null) { + if (other.name != null) + return false; + } else if (!name.equals(other.name)) + return false; + return true; + } + } + + + + @JsonInclude(JsonInclude.Include.NON_NULL) + @XmlRootElement + @XmlAccessorType(XmlAccessType.FIELD) + public static class RangerDataMaskDef implements java.io.Serializable { + private static final long serialVersionUID = 1L; + + private List maskTypes; + private List accessTypes; + private List resources; + + + public RangerDataMaskDef() { + setMaskTypes(null); + setAccessTypes(null); + setResources(null); + } + + public RangerDataMaskDef(List maskTypes, List accessTypes, List resources) { + setMaskTypes(maskTypes); + setAccessTypes(accessTypes); + setResources(resources); + } + + public RangerDataMaskDef(RangerDataMaskDef other) { + setMaskTypes(other.getMaskTypes()); + setAccessTypes(other.getAccessTypes()); + setResources(other.getResources()); + } + + public List getMaskTypes() { + return maskTypes; + } + + public void setMaskTypes(List maskTypes) { + if(this.maskTypes == null) { + this.maskTypes = new ArrayList<>(); + } + + if(this.maskTypes == maskTypes) { + return; + } + + this.maskTypes.clear(); + + if(maskTypes != null) { + this.maskTypes.addAll(maskTypes); + } + } + + public List getAccessTypes() { + return accessTypes; + } + + public void setAccessTypes(List accessTypes) { + if(this.accessTypes == null) { + this.accessTypes = new ArrayList<>(); + } + + if(this.accessTypes == accessTypes) { + return; + } + + this.accessTypes.clear(); + + if(accessTypes != null) { + this.accessTypes.addAll(accessTypes); + } + } + + public List getResources() { + return resources; + } + + public void setResources(List resources) { + if(this.resources == null) { + this.resources = new ArrayList<>(); + } + + if(this.resources == resources) { + return; + } + + this.resources.clear(); + + if(resources != null) { + this.resources.addAll(resources); + } + } + + @Override + public String toString( ) { + StringBuilder sb = new StringBuilder(); + + toString(sb); + + return sb.toString(); + } + + public StringBuilder toString(StringBuilder sb) { + sb.append("RangerDataMaskDef={"); + + sb.append("maskTypes={"); + if(maskTypes != null) { + for(RangerDataMaskTypeDef maskType : maskTypes) { + if(maskType != null) { + sb.append(maskType).append(" "); + } + } + } + sb.append("} "); + + sb.append("accessTypes={"); + if(accessTypes != null) { + for(RangerAccessTypeDef accessType : accessTypes) { + if(accessType != null) { + accessType.toString(sb).append(" "); + } + } + } + sb.append("} "); + + sb.append("resources={"); + if(resources != null) { + for(RangerResourceDef resource : resources) { + if(resource != null) { + resource.toString(sb).append(" "); + } + } + } + sb.append("} "); + + sb.append("}"); + + return sb; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((maskTypes == null) ? 0 : maskTypes.hashCode()); + result = prime * result + ((accessTypes == null) ? 0 : accessTypes.hashCode()); + result = prime * result + ((resources == null) ? 0 : resources.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + RangerDataMaskDef other = (RangerDataMaskDef) obj; + if (maskTypes == null) { + if (other.maskTypes != null) + return false; + } else if (other.maskTypes == null || !maskTypes.equals(other.maskTypes)) + return false; + + if (accessTypes == null) { + if (other.accessTypes != null) + return false; + } else if (!accessTypes.equals(other.accessTypes)) + return false; + if (resources == null) { + if (other.resources != null) + return false; + } else if (!resources.equals(other.resources)) + return false; + return true; + } + } + + @JsonInclude(JsonInclude.Include.NON_NULL) + @XmlRootElement + @XmlAccessorType(XmlAccessType.FIELD) + public static class RangerDataMaskTypeDef implements java.io.Serializable { + private static final long serialVersionUID = 1L; + + private Long itemId; + private String name; + private String label; + private String description; + private String transformer; + private Map dataMaskOptions; + private String rbKeyLabel; + private String rbKeyDescription; + + public RangerDataMaskTypeDef() { + this(null, null, null, null, null, null, null, null); + } + + public RangerDataMaskTypeDef(Long itemId, String name, String label, String description, String transformer, Map dataMaskOptions, String rbKeyLabel, String rbKeyDescription) { + setItemId(itemId); + setName(name); + setLabel(label); + setDescription(description); + setTransformer(transformer); + setDataMaskOptions(dataMaskOptions); + setRbKeyLabel(rbKeyLabel); + setRbKeyDescription(rbKeyDescription); + } + + public RangerDataMaskTypeDef(RangerDataMaskTypeDef other) { + setItemId(other.getItemId()); + setName(other.getName()); + setLabel(other.getLabel()); + setDescription(other.getDescription()); + setTransformer(other.getTransformer()); + setDataMaskOptions(other.getDataMaskOptions()); + setRbKeyLabel(other.getRbKeyLabel()); + setRbKeyDescription(other.getRbKeyDescription()); + } + + /** + * @return the itemId + */ + public Long getItemId() { + return itemId; + } + + /** + * @param itemId the itemId to set + */ + public void setItemId(Long itemId) { + this.itemId = itemId; + } + + /** + * @return the name + */ + public String getName() { + return name; + } + + /** + * @param name the name to set + */ + public void setName(String name) { + this.name = name; + } + + /** + * @return the label + */ + public String getLabel() { + return label; + } + + /** + * @param label the label to set + */ + public void setLabel(String label) { + this.label = label; + } + + /** + * @return the description + */ + public String getDescription() { + return description; + } + + /** + * @param description the description to set + */ + public void setDescription(String description) { + this.description = description; + } + + /** + * @return the transformer + */ + public String getTransformer() { + return transformer; + } + + /** + * @param transformer the transformer to set + */ + public void setTransformer(String transformer) { + this.transformer = transformer; + } + + /** + * @return the dataMaskOptions + */ + public Map getDataMaskOptions() { + return dataMaskOptions; + } + + /** + * @param dataMaskOptions the dataMaskOptions to set + */ + public void setDataMaskOptions(Map dataMaskOptions) { + this.dataMaskOptions = dataMaskOptions == null ? new HashMap() : dataMaskOptions; + } + + /** + * @return the rbKeyLabel + */ + public String getRbKeyLabel() { + return rbKeyLabel; + } + + /** + * @param rbKeyLabel the rbKeyLabel to set + */ + public void setRbKeyLabel(String rbKeyLabel) { + this.rbKeyLabel = rbKeyLabel; + } + + /** + * @return the rbKeyDescription + */ + public String getRbKeyDescription() { + return rbKeyDescription; + } + + /** + * @param rbKeyDescription the rbKeyDescription to set + */ + public void setRbKeyDescription(String rbKeyDescription) { + this.rbKeyDescription = rbKeyDescription; + } + + @Override + public String toString( ) { + StringBuilder sb = new StringBuilder(); + + toString(sb); + + return sb.toString(); + } + + public StringBuilder toString(StringBuilder sb) { + sb.append("RangerDataMaskTypeDef={"); + sb.append("itemId={").append(itemId).append("} "); + sb.append("name={").append(name).append("} "); + sb.append("label={").append(label).append("} "); + sb.append("description={").append(description).append("} "); + sb.append("transformer={").append(transformer).append("} "); + sb.append("dataMaskOptions={").append(dataMaskOptions).append("} "); + sb.append("rbKeyLabel={").append(rbKeyLabel).append("} "); + sb.append("rbKeyDescription={").append(rbKeyDescription).append("} "); + + sb.append("}"); + + return sb; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((itemId == null) ? 0 : itemId.hashCode()); + result = prime * result + + ((dataMaskOptions == null) ? 0 : dataMaskOptions.hashCode()); + result = prime * result + ((label == null) ? 0 : label.hashCode()); + result = prime * result + ((name == null) ? 0 : name.hashCode()); + result = prime * result + + ((rbKeyLabel == null) ? 0 : rbKeyLabel.hashCode()); + result = prime * result + ((description == null) ? 0 : description.hashCode()); + result = prime * result + ((transformer == null) ? 0 : transformer.hashCode()); + result = prime * result + ((rbKeyDescription == null) ? 0 : rbKeyDescription.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + RangerDataMaskTypeDef other = (RangerDataMaskTypeDef) obj; + if (itemId == null) { + if (other.itemId != null) + return false; + } else if (other.itemId == null || !itemId.equals(other.itemId)) + return false; + + if (dataMaskOptions == null) { + if (other.dataMaskOptions != null) + return false; + } else if (!dataMaskOptions.equals(other.dataMaskOptions)) + return false; + if (label == null) { + if (other.label != null) + return false; + } else if (!label.equals(other.label)) + return false; + if (name == null) { + if (other.name != null) + return false; + } else if (!name.equals(other.name)) + return false; + if (rbKeyLabel == null) { + if (other.rbKeyLabel != null) + return false; + } else if (!rbKeyLabel.equals(other.rbKeyLabel)) + return false; + if (description == null) { + if (other.description != null) + return false; + } else if (!description.equals(other.description)) + return false; + if (transformer == null) { + if (other.transformer != null) + return false; + } else if (!transformer.equals(other.transformer)) + return false; + if (rbKeyDescription == null) { + if (other.rbKeyDescription != null) + return false; + } else if (!rbKeyDescription.equals(other.rbKeyDescription)) + return false; + return true; + } + } + + @JsonInclude(JsonInclude.Include.NON_NULL) + @XmlRootElement + @XmlAccessorType(XmlAccessType.FIELD) + public static class RangerRowFilterDef implements java.io.Serializable { + private static final long serialVersionUID = 1L; + + private List accessTypes; + private List resources; + + + public RangerRowFilterDef() { + setAccessTypes(null); + setResources(null); + } + + public RangerRowFilterDef(List accessTypes, List resources) { + setAccessTypes(accessTypes); + setResources(resources); + } + + public RangerRowFilterDef(RangerRowFilterDef other) { + setAccessTypes(other.getAccessTypes()); + setResources(other.getResources()); + } + + public List getAccessTypes() { + return accessTypes; + } + + public void setAccessTypes(List accessTypes) { + if(this.accessTypes == null) { + this.accessTypes = new ArrayList<>(); + } + + if(this.accessTypes == accessTypes) { + return; + } + + this.accessTypes.clear(); + + if(accessTypes != null) { + this.accessTypes.addAll(accessTypes); + } + } + + public List getResources() { + return resources; + } + + public void setResources(List resources) { + if(this.resources == null) { + this.resources = new ArrayList<>(); + } + + if(this.resources == resources) { + return; + } + + this.resources.clear(); + + if(resources != null) { + this.resources.addAll(resources); + } + } + + @Override + public String toString( ) { + StringBuilder sb = new StringBuilder(); + + toString(sb); + + return sb.toString(); + } + + public StringBuilder toString(StringBuilder sb) { + sb.append("RangerRowFilterDef={"); + + sb.append("accessTypes={"); + if(accessTypes != null) { + for(RangerAccessTypeDef accessType : accessTypes) { + if(accessType != null) { + accessType.toString(sb).append(" "); + } + } + } + sb.append("} "); + + sb.append("resources={"); + if(resources != null) { + for(RangerResourceDef resource : resources) { + if(resource != null) { + resource.toString(sb).append(" "); + } + } + } + sb.append("} "); + + sb.append("}"); + + return sb; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((accessTypes == null) ? 0 : accessTypes.hashCode()); + result = prime * result + ((resources == null) ? 0 : resources.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + RangerRowFilterDef other = (RangerRowFilterDef) obj; + + if (accessTypes == null) { + if (other.accessTypes != null) + return false; + } else if (!accessTypes.equals(other.accessTypes)) + return false; + if (resources == null) { + if (other.resources != null) + return false; + } else if (!resources.equals(other.resources)) + return false; + return true; + } + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerServiceResource.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerServiceResource.java new file mode 100644 index 00000000000..4adf098fecf --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerServiceResource.java @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.model; + +import org.apache.htrace.shaded.fasterxml.jackson.annotation.JsonInclude; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.HashMap; +import java.util.Map; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class RangerServiceResource extends RangerBaseModelObject { + private static final long serialVersionUID = 1L; + + private String serviceName; + private Map resourceElements; + private String ownerUser; + private String resourceSignature; + private Map additionalInfo; + + public RangerServiceResource(String guid, String serviceName, Map resourceElements, String resourceSignature, String ownerUser, Map additionalInfo) { + super(); + setGuid(guid); + setServiceName(serviceName); + setResourceElements(resourceElements); + setResourceSignature(resourceSignature); + setOwnerUser(ownerUser); + setAdditionalInfo(additionalInfo); + } + public RangerServiceResource(String guid, String serviceName, Map resourceElements, String resourceSignature, String ownerUser) { + this(guid, serviceName, resourceElements, resourceSignature,ownerUser, null); + } + public RangerServiceResource(String guid, String serviceName, Map resourceElements, String resourceSignature) { + this(guid, serviceName, resourceElements, resourceSignature, null); + + } + + public RangerServiceResource(String guid, String serviceName, Map resourceElements) { + this(guid, serviceName, resourceElements, null, null); + } + public RangerServiceResource(String serviceName, Map resourceElements) { + this(null, serviceName, resourceElements, null, null); + } + + public RangerServiceResource() { + this(null, null, null, null, null); + } + + public String getServiceName() { return serviceName; } + + public Map getResourceElements() { return resourceElements; } + + public String getResourceSignature() { + return resourceSignature; + } + + public String getOwnerUser() { + return ownerUser; + } + + public Map getAdditionalInfo() { + return additionalInfo; + } + + public void setServiceName(String serviceName) { + this.serviceName = serviceName; + } + + public void setResourceElements(Map resource) { + this.resourceElements = resource == null ? new HashMap() : resource; + } + + public void setResourceSignature(String resourceSignature) { + this.resourceSignature = resourceSignature; + } + + public void setOwnerUser(String ownerUser) { + this.ownerUser = ownerUser; + } + + public void setAdditionalInfo(Map additionalInfo) { + this.additionalInfo = additionalInfo; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + + toString(sb); + + return sb.toString(); + } + + public StringBuilder toString(StringBuilder sb) { + + sb.append("RangerServiceResource={ "); + + super.toString(sb); + + sb.append("guid={").append(getGuid()).append("} "); + sb.append("serviceName={").append(serviceName).append("} "); + + sb.append("resourceElements={"); + if(resourceElements != null) { + for(Map.Entry e : resourceElements.entrySet()) { + sb.append(e.getKey()).append("={"); + e.getValue().toString(sb); + sb.append("} "); + } + } + sb.append("} "); + + sb.append("ownerUser={").append(ownerUser).append("} "); + + sb.append("additionalInfo={"); + if(additionalInfo != null) { + for(Map.Entry e : additionalInfo.entrySet()) { + sb.append(e.getKey()).append("={").append(e.getValue()).append("} "); + } + } + sb.append("} "); + + sb.append("resourceSignature={").append(resourceSignature).append("} "); + + sb.append(" }"); + + return sb; + } +} + diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerTag.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerTag.java new file mode 100644 index 00000000000..252d21d2d2e --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerTag.java @@ -0,0 +1,221 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.model; + +import org.apache.htrace.shaded.fasterxml.jackson.annotation.JsonInclude; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class RangerTag extends RangerBaseModelObject { + private static final long serialVersionUID = 1L; + + public static final short OWNER_SERVICERESOURCE = 0; + public static final short OWNER_GLOBAL = 1; + + public static final String OPTION_TAG_VALIDITY_PERIODS = "TAG_VALIDITY_PERIODS"; + + private String type; + private Short owner = OWNER_SERVICERESOURCE; + private Map attributes; + private Map options; + private List validityPeriods; + + public RangerTag(String guid, String type, Map attributes, Short owner, Map options, List validityPeriods) { + super(); + + setGuid(guid); + setType(type); + setOwner(owner); + setAttributes(attributes); + setOwner(owner); + setOptions(options); + setValidityPeriods(validityPeriods); + } + + public RangerTag(String guid, String type, Map attributes, Short owner) { + this(guid, type, attributes, owner, null, null); + } + + public RangerTag(String type, Map attributes) { + this(null, type, attributes, OWNER_SERVICERESOURCE, null, null); + } + + public RangerTag() { + this(null, null, null, OWNER_SERVICERESOURCE, null, null); + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public Map getAttributes() { + return attributes; + } + + public void setAttributes(Map attributes) { + this.attributes = attributes == null ? new HashMap() : attributes; + } + + public Short getOwner() { + return this.owner; + } + + public void setOwner(Short owner) { + this.owner = owner; + } + + public Map getOptions() { return options; } + + public void setOptions(Map options) { + if (this.options == null) { + this.options = new HashMap<>(); + } + if (this.options == options) { + return; + } + this.options.clear(); + + if(options != null) { + for(Map.Entry e : options.entrySet()) { + this.options.put(e.getKey(), e.getValue()); + } + } + } + + public List getValidityPeriods() { return validityPeriods; } + + public void setValidityPeriods(List validityPeriods) { + if (this.validityPeriods == null) { + this.validityPeriods = new ArrayList<>(); + } + if (this.validityPeriods == validityPeriods) { + return; + } + this.validityPeriods.clear(); + + if(validityPeriods != null) { + this.validityPeriods.addAll(validityPeriods); + } + } + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + + toString(sb); + + return sb.toString(); + } + + public StringBuilder toString(StringBuilder sb) { + sb.append("RangerTag={"); + + super.toString(sb); + + sb.append("type={").append(type).append("} "); + sb.append("owner={").append(owner).append("} "); + + sb.append("attributes={"); + if (attributes != null) { + for (Map.Entry e : attributes.entrySet()) { + sb.append(e.getKey()).append("={"); + sb.append(e.getValue()); + sb.append("} "); + } + } + sb.append("} "); + + if (validityPeriods != null) { + sb.append("validityPeriods={").append(validityPeriods).append("} "); + } + sb.append("options={").append(options).append("} "); + + sb.append(" }"); + + return sb; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + + ((type == null) ? 0 : type.hashCode()); + result = prime * result + + ((owner == null) ? 0 : owner.hashCode()); + result = prime * result + + ((attributes == null) ? 0 : attributes.hashCode()); + result = prime * result + + ((options == null) ? 0 : options.hashCode()); + result = prime * result + + ((validityPeriods == null) ? 0 : validityPeriods.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + RangerTag other = (RangerTag) obj; + if (type == null) { + if (other.type != null) + return false; + } else if (!type.equals(other.type)) + return false; + if (owner == null) { + if (other.owner != null) + return false; + } else if (!owner.equals(other.owner)) + return false; + if (attributes == null) { + if (other.attributes != null) + return false; + } else if (!attributes.equals(other.attributes)) + return false; + if (options == null) { + if (other.options != null) + return false; + } else if (!options.equals(other.options)) + return false; + if (validityPeriods == null) { + if (other.validityPeriods != null) + return false; + } else if (!validityPeriods.equals(other.validityPeriods)) + return false; + return true; + } +} + diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerTagDef.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerTagDef.java new file mode 100644 index 00000000000..849fac1a0a4 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerTagDef.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.model; + +import org.apache.htrace.shaded.fasterxml.jackson.annotation.JsonInclude; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.ArrayList; +import java.util.List; + + +/** + * Represents a TAG definition known to Ranger. In general, this will be provided + * by some external system identified by 'source'. + * + */ +@JsonInclude(JsonInclude.Include.NON_NULL) +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class RangerTagDef extends RangerBaseModelObject { + private static final long serialVersionUID = 1L; + + private String name; + private String source; + + private List attributeDefs; + + public RangerTagDef() { + this(null, "Internal"); + } + + public RangerTagDef(String name) { + this(name, "Internal"); + } + + public RangerTagDef(String name, String source) { + super(); + setName(name); + setSource(source); + setAttributeDefs(null); + } + + public String getName() { + return name; + } + + public void setName(String name) { + + this.name = name == null ? "" : name; + } + + public String getSource() { + return source; + } + + public void setSource(String source) { + this.source = source == null ? "" : source; + } + + public List getAttributeDefs() { + return attributeDefs; + } + + public void setAttributeDefs(List attributeDefs) { + this.attributeDefs = attributeDefs == null ? new ArrayList() : attributeDefs; + } + + /** + * Represents one attribute for a TAG. TAG-Attribute consists of a name and type. + * name provides a handle for possible specification of additional information + * associated with the TAG. + * Interpretation of type is up to the policy-engine. + */ + + @JsonInclude(JsonInclude.Include.NON_NULL) + @XmlRootElement + @XmlAccessorType(XmlAccessType.FIELD) + + public static class RangerTagAttributeDef implements java.io.Serializable { + private static final long serialVersionUID = 1L; + + private String name; + private String type; + + public RangerTagAttributeDef() { + this(null, null); + } + + public RangerTagAttributeDef(String name, String type) { + setName(name); + setType(type); + } + + public String getName() { + return name; + } + + public String getType() { + return type; + } + + public void setName(String name) { + this.name = name == null ? "" : name; + } + public void setType(String type) { + this.type = type == null ? "" : type; + } + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerTagResourceMap.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerTagResourceMap.java new file mode 100644 index 00000000000..a68e33e11bd --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerTagResourceMap.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.model; + +import org.apache.htrace.shaded.fasterxml.jackson.annotation.JsonInclude; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class RangerTagResourceMap extends RangerBaseModelObject { + private static final long serialVersionUID = 1L; + + private Long tagId; + private Long resourceId; + + public RangerTagResourceMap() { + } + + public Long getTagId() { + return tagId; + } + + public Long getResourceId() { + return resourceId; + } + + public void setTagId(Long tagId) { + this.tagId = tagId; + } + + public void setResourceId(Long resourceId) { + this.resourceId = resourceId; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + + toString(sb); + + return sb.toString(); + } + + public StringBuilder toString(StringBuilder sb) { + + sb.append("RangerTagResourceMap={ "); + + super.toString(sb); + + sb.append("resourceId=").append(resourceId).append(", "); + + sb.append("tagId=").append(tagId); + + sb.append(" }"); + + return sb; + }} + diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerValidityRecurrence.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerValidityRecurrence.java new file mode 100644 index 00000000000..449a3d356fd --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerValidityRecurrence.java @@ -0,0 +1,224 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.model; + +import org.apache.htrace.shaded.fasterxml.jackson.annotation.JsonInclude; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import java.io.Serializable; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) + +public class RangerValidityRecurrence implements Serializable { + + @JsonInclude(JsonInclude.Include.NON_NULL) + @XmlRootElement + @XmlAccessorType(XmlAccessType.FIELD) + public static class ValidityInterval { + private final int days; + private final int hours; + private final int minutes; + + public static int getValidityIntervalInMinutes(ValidityInterval interval) { + return interval != null ? + (interval.getDays()*24 + interval.getHours())*60 + interval.getMinutes() : 0; + } + + public ValidityInterval() { + this.days = 0; + this.hours = 0; + this.minutes = 0; + } + public ValidityInterval(int days, int hours, int minutes) { + this.days = days; + this.hours = hours; + this.minutes = minutes; + } + public int getDays() { return days; } + public int getHours() { return hours; } + public int getMinutes() { return minutes; } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("ValidityInterval={"); + + sb.append(" Interval={"); + sb.append("days=").append(days); + sb.append(", hours=").append(hours); + sb.append(", minutes=").append(minutes); + sb.append(" }"); + + return sb.toString(); + } + } + + @JsonInclude(JsonInclude.Include.NON_NULL) + @XmlRootElement + @XmlAccessorType(XmlAccessType.FIELD) + public static class RecurrenceSchedule { + static final String PERMITTED_SPECIAL_CHARACTERS = "*,-"; + static final String PERMITTED_SPECIAL_CHARACTERS_FOR_MINUTES = ","; + public static final String WILDCARD = "*"; + + public enum ScheduleFieldSpec { + minute(0, 59, PERMITTED_SPECIAL_CHARACTERS_FOR_MINUTES), + hour(0, 23, PERMITTED_SPECIAL_CHARACTERS), + dayOfMonth(1, 31, PERMITTED_SPECIAL_CHARACTERS), + dayOfWeek(1, 7, PERMITTED_SPECIAL_CHARACTERS), + month(0, 11, PERMITTED_SPECIAL_CHARACTERS), + year(2017, 2100, PERMITTED_SPECIAL_CHARACTERS), + ; + + public final int minimum; + public final int maximum; + public final String specialChars; + + ScheduleFieldSpec(int minimum, int maximum, String specialChars) { + this.minimum = minimum; + this.maximum = maximum; + this.specialChars = specialChars; + } + } + + + private String minute; + private String hour; + private String dayOfMonth; + private String dayOfWeek; + private String month; + private String year; + + public RecurrenceSchedule() {} + + public RecurrenceSchedule(String minute, String hour, String dayOfMonth, String dayOfWeek, String month, String year) { + setMinute(minute); + setHour(hour); + setDayOfMonth(dayOfMonth); + setDayOfWeek(dayOfWeek); + setMonth(month); + setYear(year); + } + public String getMinute() { return minute;} + public String getHour() { return hour;} + public String getDayOfMonth() { return dayOfMonth;} + public String getDayOfWeek() { return dayOfWeek;} + public String getMonth() { return month;} + public String getYear() { return year;} + + public void setMinute(String minute) { this.minute = minute;} + public void setHour(String hour) { this.hour = hour;} + public void setDayOfMonth(String dayOfMonth) { this.dayOfMonth = dayOfMonth;} + public void setDayOfWeek(String dayOfWeek) { this.dayOfWeek = dayOfWeek;} + public void setMonth(String month) { this.month = month;} + public void setYear(String year) { this.year = year;} + + public void setFieldValue(ScheduleFieldSpec field, String value) { + switch (field) { + case minute: + setMinute(value); + break; + case hour: + setHour(value); + break; + case dayOfMonth: + setDayOfMonth(value); + break; + case dayOfWeek: + setDayOfWeek(value); + break; + case month: + setMonth(value); + break; + case year: + setYear(value); + break; + default: + break; + } + } + + public String getFieldValue(ScheduleFieldSpec field) { + switch (field) { + case minute: + return getMinute(); + case hour: + return getHour(); + case dayOfMonth: + return getDayOfMonth(); + case dayOfWeek: + return getDayOfWeek(); + case month: + return getMonth(); + case year: + return getYear(); + default: + return null; + } + } + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(" Schedule={"); + sb.append(" minute=").append(minute); + sb.append(", hour=").append(hour); + sb.append(", dayOfMonth=").append(dayOfMonth); + sb.append(", dayOfWeek=").append(dayOfWeek); + sb.append(", month=").append(month); + sb.append(", year=").append(year); + sb.append(" }"); + + return sb.toString(); + } + } + + private RecurrenceSchedule schedule; + private ValidityInterval interval; + + public RangerValidityRecurrence() { + } + + public RangerValidityRecurrence(RecurrenceSchedule schedule, ValidityInterval interval) { + setSchedule(schedule); + setInterval(interval); + } + + public void setSchedule(RecurrenceSchedule schedule) { this.schedule = schedule;} + + public void setInterval(ValidityInterval interval) { this.interval = interval; } + + public RecurrenceSchedule getSchedule() { return schedule;} + + public ValidityInterval getInterval() { + return interval; + } + + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("{RangerValidityRecurrence= {"); + sb.append(schedule).append(interval); + sb.append(" }"); + return sb.toString(); + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerValiditySchedule.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerValiditySchedule.java new file mode 100644 index 00000000000..3bb3f078b53 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerValiditySchedule.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.model; + + +import org.codehaus.jackson.annotate.JsonAutoDetect; +import org.codehaus.jackson.annotate.JsonAutoDetect.Visibility; +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +@JsonAutoDetect(fieldVisibility=Visibility.ANY) +@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown=true) +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) + +public class RangerValiditySchedule implements Serializable { + + public static final String VALIDITY_SCHEDULE_DATE_STRING_SPECIFICATION = "yyyy/MM/dd HH:mm:ss"; + + private String startTime; + private String endTime; + private String timeZone; + + private List recurrences; + + public RangerValiditySchedule(String startTime, String endTime, String timeZone, List recurrences) { + setTimeZone(timeZone); + setStartTime(startTime); + setEndTime(endTime); + setRecurrences(recurrences); + } + + public RangerValiditySchedule() { + this(null, null, null, null); + } + + public String getTimeZone() { return timeZone; } + public String getStartTime() { return startTime;} + public String getEndTime() { return endTime;} + public List getRecurrences() { return recurrences;} + + public void setTimeZone(String timeZone) { this.timeZone = timeZone; } + public void setStartTime(String startTime) { this.startTime = startTime;} + public void setEndTime(String endTime) { this.endTime = endTime;} + public void setRecurrences(List recurrences) { this.recurrences = recurrences == null ? new ArrayList<>() : recurrences;} + + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("RangerValiditySchedule={"); + + sb.append(", startTime=").append(startTime); + sb.append(", endTime=").append(endTime); + sb.append(", timeZone=").append(timeZone); + + sb.append(", recurrences=").append(Arrays.toString(getRecurrences().toArray())); + + sb.append("}"); + + return sb.toString(); + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/ServiceDeleteResponse.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/ServiceDeleteResponse.java new file mode 100644 index 00000000000..ee574f49d9f --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/ServiceDeleteResponse.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.model; + +import org.apache.htrace.shaded.fasterxml.jackson.annotation.JsonInclude; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class ServiceDeleteResponse implements java.io.Serializable { + /** + * + */ + private static final long serialVersionUID = 1L; + + private String serviceName; + private Long serviceId; + private Boolean isDeleted; + private String errorMsg; + + public ServiceDeleteResponse(Long serviceId) { + this.serviceId = serviceId; + } + + public String getServiceName() { + return serviceName; + } + public void setServiceName(String serviceName) { + this.serviceName = serviceName; + } + public Long getServiceId() { + return serviceId; + } + public void setServiceId(Long serviceId) { + this.serviceId = serviceId; + } + public Boolean getIsDeleted() { + return isDeleted; + } + public void setIsDeleted(Boolean isDeleted) { + this.isDeleted = isDeleted; + } + public String getErrorMsg() { + return errorMsg; + } + public void setErrorMsg(String errorMsg) { + this.errorMsg = errorMsg; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/UserInfo.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/UserInfo.java new file mode 100644 index 00000000000..4a690a0a297 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/UserInfo.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.model; + +import org.apache.htrace.shaded.fasterxml.jackson.annotation.JsonInclude; +import org.apache.atlas.plugin.util.RangerUserStoreUtil; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class UserInfo extends RangerBaseModelObject implements java.io.Serializable { + + private static final long serialVersionUID = 1L; + private String name; + private String description; + private Map otherAttributes; + private Set groups; + + public UserInfo() { + this(null, null, null); + } + + public UserInfo(String name, String description, Map otherAttributes) { + setName(name); + setDescription(description); + setOtherAttributes(otherAttributes); + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public Map getOtherAttributes() { + return otherAttributes; + } + + public void setOtherAttributes(Map otherAttributes) { + this.otherAttributes = otherAttributes == null ? new HashMap<>() : otherAttributes; + } + + public Set getGroups(){ + return this.groups; + } + + public void setGroups(Set groups){ + this.groups = groups; + } + + @Override + public String toString() { + return "{name=" + name + + ", description=" + description + + ", otherAttributes=" + RangerUserStoreUtil.getPrintableOptions(otherAttributes) + + ", groups=" + groups + + "}"; + } +} \ No newline at end of file diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/validation/RangerPolicyValidator.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/validation/RangerPolicyValidator.java new file mode 100644 index 00000000000..a0594aa127a --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/validation/RangerPolicyValidator.java @@ -0,0 +1,1062 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.model.validation; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.plugin.errors.ValidationErrorCode; +import org.apache.atlas.plugin.model.RangerPolicy; +import org.apache.atlas.plugin.model.RangerPolicy.RangerDataMaskPolicyItem; +import org.apache.atlas.plugin.model.RangerPolicy.RangerPolicyItem; +import org.apache.atlas.plugin.model.RangerPolicy.RangerPolicyItemAccess; +import org.apache.atlas.plugin.model.RangerPolicy.RangerPolicyResource; +import org.apache.atlas.plugin.model.RangerPolicy.RangerRowFilterPolicyItem; +import org.apache.atlas.plugin.model.RangerPolicyResourceSignature; +import org.apache.atlas.plugin.model.RangerSecurityZone; +import org.apache.atlas.plugin.model.RangerService; +import org.apache.atlas.plugin.model.RangerServiceDef; +import org.apache.atlas.plugin.model.RangerServiceDef.RangerAccessTypeDef; +import org.apache.atlas.plugin.model.RangerServiceDef.RangerResourceDef; +import org.apache.atlas.plugin.model.RangerValiditySchedule; +import org.apache.atlas.plugin.store.ServiceStore; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class RangerPolicyValidator extends RangerValidator { + + private static final Log LOG = LogFactory.getLog(RangerPolicyValidator.class); + + public RangerPolicyValidator(ServiceStore store) { + super(store); + } + + public void validate(RangerPolicy policy, Action action, boolean isAdmin) throws Exception { + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("==> RangerPolicyValidator.validate(%s, %s, %s)", policy, action, isAdmin)); + } + + List failures = new ArrayList<>(); + boolean valid = isValid(policy, action, isAdmin, failures); + String message = ""; + try { + if (!valid) { + message = serializeFailures(failures); + throw new Exception(message); + } + } finally { + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("<== RangerPolicyValidator.validate(%s, %s, %s): %s, reason[%s]", policy, action, isAdmin, valid, message)); + } + } + } + + @Override + boolean isValid(Long id, Action action, List failures) { + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("==> RangerPolicyValidator.isValid(%s, %s, %s)", id, action, failures)); + } + + boolean valid = true; + if (action != Action.DELETE) { + ValidationErrorCode error = ValidationErrorCode.POLICY_VALIDATION_ERR_UNSUPPORTED_ACTION; + failures.add(new ValidationFailureDetailsBuilder() + .isAnInternalError() + .becauseOf(error.getMessage()) + .errorCode(error.getErrorCode()) + .build()); + valid = false; + } else if (id == null) { + ValidationErrorCode error = ValidationErrorCode.POLICY_VALIDATION_ERR_MISSING_FIELD; + failures.add(new ValidationFailureDetailsBuilder() + .becauseOf("policy id was null/missing") + .field("id") + .isMissing() + .errorCode(error.getErrorCode()) + .becauseOf(error.getMessage("id")) + .build()); + valid = false; + } else if (policyExists(id)) { + if (LOG.isDebugEnabled()) { + LOG.debug("No policy found for id[" + id + "]! ok!"); + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("<== RangerPolicyValidator.isValid(%s, %s, %s): %s", id, action, failures, valid)); + } + return valid; + } + + boolean isValid(RangerPolicy policy, Action action, boolean isAdmin, List failures) { + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("==> RangerPolicyValidator.isValid(%s, %s, %s, %s)", policy, action, isAdmin, failures)); + } + + if (!(action == Action.CREATE || action == Action.UPDATE)) { + throw new IllegalArgumentException("isValid(RangerPolicy, ...) is only supported for create/update"); + } + boolean valid = true; + if (policy == null) { + ValidationErrorCode error = ValidationErrorCode.POLICY_VALIDATION_ERR_NULL_POLICY_OBJECT; + failures.add(new ValidationFailureDetailsBuilder() + .field("policy") + .isMissing() + .becauseOf(error.getMessage()) + .errorCode(error.getErrorCode()) + .build()); + valid = false; + } else { + Integer priority = policy.getPolicyPriority(); + if (priority != null) { + if (priority < RangerPolicy.POLICY_PRIORITY_NORMAL || priority > RangerPolicy.POLICY_PRIORITY_OVERRIDE) { + ValidationErrorCode error = ValidationErrorCode.POLICY_VALIDATION_ERR_POLICY_INVALID_PRIORITY; + failures.add(new ValidationFailureDetailsBuilder() + .field("policyPriority") + .isSemanticallyIncorrect() + .becauseOf(error.getMessage("out of range")) + .errorCode(error.getErrorCode()) + .build()); + valid = false; + } + } + Long id = policy.getId(); + RangerPolicy existingPolicy = null; + + if (action == Action.UPDATE) { // id is ignored for CREATE + if (id == null) { + ValidationErrorCode error = ValidationErrorCode.POLICY_VALIDATION_ERR_MISSING_FIELD; + failures.add(new ValidationFailureDetailsBuilder() + .field("id") + .isMissing() + .becauseOf(error.getMessage("id")) + .errorCode(error.getErrorCode()) + .build()); + valid = false; + } + + existingPolicy = getPolicy(id); + + if (existingPolicy == null) { + ValidationErrorCode error = ValidationErrorCode.POLICY_VALIDATION_ERR_INVALID_POLICY_ID; + failures.add(new ValidationFailureDetailsBuilder() + .field("id") + .isSemanticallyIncorrect() + .becauseOf(error.getMessage(id)) + .errorCode(error.getErrorCode()) + .build()); + valid = false; + } + } + String policyName = policy.getName(); + String serviceName = policy.getService(); + String policyServicetype = policy.getServiceType(); + String zoneName = policy.getZoneName(); + + RangerService service = null; + RangerSecurityZone zone = null; + boolean serviceNameValid = false; + if (StringUtils.isBlank(serviceName)) { + ValidationErrorCode error = ValidationErrorCode.POLICY_VALIDATION_ERR_MISSING_FIELD; + failures.add(new ValidationFailureDetailsBuilder() + .field("service name") + .isMissing() + .becauseOf(error.getMessage("service name")) + .errorCode(error.getErrorCode()) + .build()); + valid = false; + } else { + service = getService(serviceName); + if (service == null) { + ValidationErrorCode error = ValidationErrorCode.POLICY_VALIDATION_ERR_INVALID_SERVICE_NAME; + failures.add(new ValidationFailureDetailsBuilder() + .field("service name") + .isSemanticallyIncorrect() + .becauseOf(error.getMessage(serviceName)) + .errorCode(error.getErrorCode()) + .build()); + valid = false; + } else { + serviceNameValid = true; + + String serviceType = service.getType(); + + if (StringUtils.isNotEmpty(serviceType) && StringUtils.isNotEmpty(policyServicetype)) { + if (!serviceType.equalsIgnoreCase(policyServicetype)) { + ValidationErrorCode error = ValidationErrorCode.POLICY_VALIDATION_ERR_INVALID_SERVICE_TYPE; + + failures.add(new ValidationFailureDetailsBuilder() + .field("service type") + .isSemanticallyIncorrect() + .becauseOf(error.getMessage(policyServicetype,serviceName)) + .errorCode(error.getErrorCode()) + .build()); + valid = false; + } + } + } + } + + if (StringUtils.isNotEmpty(zoneName)) { + zone = getSecurityZone(zoneName); + if (zone == null) { + ValidationErrorCode error = ValidationErrorCode.POLICY_VALIDATION_ERR_NONEXISTANT_ZONE_NAME; + failures.add(new ValidationFailureDetailsBuilder() + .field("zoneName") + .isSemanticallyIncorrect() + .becauseOf(error.getMessage(id, zoneName)) + .errorCode(error.getErrorCode()) + .build()); + valid = false; + } + List tagSvcList = zone.getTagServices(); + Set svcNameSet = zone.getServices().keySet(); + if(!svcNameSet.contains(serviceName) && !tagSvcList.contains(serviceName)){ + ValidationErrorCode error = ValidationErrorCode.POLICY_VALIDATION_ERR_SERVICE_NOT_ASSOCIATED_TO_ZONE; + failures.add(new ValidationFailureDetailsBuilder() + .field("zoneName") + .isSemanticallyIncorrect() + .becauseOf(error.getMessage(serviceName, zoneName)) + .errorCode(error.getErrorCode()) + .build()); + valid = false; + } + } + + if (StringUtils.isBlank(policyName)) { + ValidationErrorCode error = ValidationErrorCode.POLICY_VALIDATION_ERR_MISSING_FIELD; + failures.add(new ValidationFailureDetailsBuilder() + .field("name") + .isMissing() + .becauseOf(error.getMessage("name")) + .errorCode(error.getErrorCode()) + .build()); + valid = false; + } else { + if (service != null && (StringUtils.isEmpty(zoneName) || zone != null)) { + Long zoneId = zone != null ? zone.getId() : RangerSecurityZone.RANGER_UNZONED_SECURITY_ZONE_ID; + Long policyId = getPolicyId(service.getId(), policyName, zoneId); + + if (policyId != null) { + if (action == Action.CREATE) { + ValidationErrorCode error = ValidationErrorCode.POLICY_VALIDATION_ERR_POLICY_NAME_CONFLICT; + failures.add(new ValidationFailureDetailsBuilder() + .field("policy name") + .isSemanticallyIncorrect() + .becauseOf(error.getMessage(policyId, serviceName)) + .errorCode(error.getErrorCode()) + .build()); + valid = false; + } else if (!policyId.equals(id)) { // action == UPDATE + ValidationErrorCode error = ValidationErrorCode.POLICY_VALIDATION_ERR_POLICY_NAME_CONFLICT; + failures.add(new ValidationFailureDetailsBuilder() + .field("id/name") + .isSemanticallyIncorrect() + .becauseOf(error.getMessage(policyId, serviceName)) + .errorCode(error.getErrorCode()) + .build()); + valid = false; + } + } + } + } + + if(existingPolicy != null) { + if (!StringUtils.equalsIgnoreCase(existingPolicy.getService(), policy.getService())) { + ValidationErrorCode error = ValidationErrorCode.POLICY_VALIDATION_ERR_POLICY_UPDATE_MOVE_SERVICE_NOT_ALLOWED; + failures.add(new ValidationFailureDetailsBuilder() + .field("service name") + .isSemanticallyIncorrect() + .becauseOf(error.getMessage(policy.getId(), existingPolicy.getService(), policy.getService())) + .errorCode(error.getErrorCode()) + .build()); + valid = false; + } + + String existingPolicyType = StringUtils.isEmpty(existingPolicy.getPolicyType()) ? RangerPolicy.POLICY_TYPE_ACCESS : existingPolicy.getPolicyType(); + String policyType = StringUtils.isEmpty(policy.getPolicyType()) ? RangerPolicy.POLICY_TYPE_ACCESS : policy.getPolicyType(); + + if (existingPolicyType != policyType) { + ValidationErrorCode error = ValidationErrorCode.POLICY_VALIDATION_ERR_POLICY_TYPE_CHANGE_NOT_ALLOWED; + failures.add(new ValidationFailureDetailsBuilder() + .field("policy type") + .isSemanticallyIncorrect() + .becauseOf(error.getMessage(policy.getId(), existingPolicyType, policyType)) + .errorCode(error.getErrorCode()) + .build()); + valid = false; + } + + String existingZoneName = existingPolicy.getZoneName(); + + if (StringUtils.isNotEmpty(zoneName) || StringUtils.isNotEmpty(existingZoneName)) { + if (!StringUtils.equals(existingZoneName, zoneName)) { + ValidationErrorCode error = ValidationErrorCode.POLICY_VALIDATION_ERR_UPDATE_ZONE_NAME_NOT_ALLOWED; + failures.add(new ValidationFailureDetailsBuilder() + .field("zoneName") + .isSemanticallyIncorrect() + .becauseOf(error.getMessage(existingZoneName, zoneName)) + .errorCode(error.getErrorCode()) + .build()); + valid = false; + } + } + } + + boolean isAuditEnabled = getIsAuditEnabled(policy); + String serviceDefName = null; + RangerServiceDef serviceDef = null; + int policyItemsCount = 0; + + String policyType = StringUtils.isEmpty(policy.getPolicyType()) ? RangerPolicy.POLICY_TYPE_ACCESS : policy.getPolicyType(); + switch (policyType) { + case RangerPolicy.POLICY_TYPE_DATAMASK: + if (CollectionUtils.isNotEmpty(policy.getDataMaskPolicyItems())) { + policyItemsCount += policy.getDataMaskPolicyItems().size(); + } + break; + case RangerPolicy.POLICY_TYPE_ROWFILTER: + if (CollectionUtils.isNotEmpty(policy.getRowFilterPolicyItems())) { + policyItemsCount += policy.getRowFilterPolicyItems().size(); + } + break; + default: + if (CollectionUtils.isNotEmpty(policy.getPolicyItems())){ + policyItemsCount += policy.getPolicyItems().size(); + } + if(CollectionUtils.isNotEmpty(policy.getDenyPolicyItems())) { + policyItemsCount += policy.getDenyPolicyItems().size(); + } + break; + } + + if (policyItemsCount == 0 && !isAuditEnabled) { + ValidationErrorCode error = ValidationErrorCode.POLICY_VALIDATION_ERR_MISSING_POLICY_ITEMS; + failures.add(new ValidationFailureDetailsBuilder() + .field("policy items") + .isMissing() + .becauseOf(error.getMessage()) + .errorCode(error.getErrorCode()) + .build()); + valid = false; + } else if (service != null) { + serviceDefName = service.getType(); + serviceDef = getServiceDef(serviceDefName); + if (serviceDef == null) { + ValidationErrorCode error = ValidationErrorCode.POLICY_VALIDATION_ERR_MISSING_SERVICE_DEF; + failures.add(new ValidationFailureDetailsBuilder() + .field("policy service def") + .isAnInternalError() + .becauseOf(error.getMessage(serviceDefName, serviceName)) + .errorCode(error.getErrorCode()) + .build()); + valid = false; + } else { + if (Boolean.TRUE.equals(policy.getIsDenyAllElse())) { + if (CollectionUtils.isNotEmpty(policy.getDenyPolicyItems()) || CollectionUtils.isNotEmpty(policy.getDenyExceptions())) { + ValidationErrorCode error = ValidationErrorCode.POLICY_VALIDATION_ERR_UNSUPPORTED_POLICY_ITEM_TYPE; + failures.add(new ValidationFailureDetailsBuilder() + .field("policy items") + .becauseOf(error.getMessage()) + .errorCode(error.getErrorCode()) + .build()); + valid = false; + } + } + valid = isValidPolicyItems(policy.getPolicyItems(), failures, serviceDef) && valid; + valid = isValidPolicyItems(policy.getDenyPolicyItems(), failures, serviceDef) && valid; + valid = isValidPolicyItems(policy.getAllowExceptions(), failures, serviceDef) && valid; + valid = isValidPolicyItems(policy.getDenyExceptions(), failures, serviceDef) && valid; + } + } + + if (serviceNameValid) { // resource checks can't be done meaningfully otherwise + valid = isValidValiditySchedule(policy, failures, action) && valid; + valid = isValidResources(policy, failures, action, isAdmin, serviceDef) && valid; + valid = isValidAccessTypeDef(policy, failures, action, isAdmin, serviceDef) && valid; + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("<== RangerPolicyValidator.isValid(%s, %s, %s, %s): %s", policy, action, isAdmin, failures, valid)); + } + return valid; + } + + boolean isValidAccessTypeDef(RangerPolicy policy, final List failures, Action action,boolean isAdmin, final RangerServiceDef serviceDef) { + boolean valid = true; + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("==> RangerPolicyValidator.isValidAccessTypeDef(%s, %s, %s,%s,%s)", policy, failures, action,isAdmin,serviceDef)); + } + String policyType = StringUtils.isEmpty(policy.getPolicyType()) ? RangerPolicy.POLICY_TYPE_ACCESS : policy.getPolicyType(); + //row filter policy + if (policyType==RangerPolicy.POLICY_TYPE_ROWFILTER){ + List rowFilterAccessTypeDefNames=new ArrayList(); + if(serviceDef!=null && serviceDef.getRowFilterDef()!=null){ + if(!CollectionUtils.isEmpty(serviceDef.getRowFilterDef().getAccessTypes())){ + for(RangerAccessTypeDef rangerAccessTypeDef:serviceDef.getRowFilterDef().getAccessTypes()){ + rowFilterAccessTypeDefNames.add(rangerAccessTypeDef.getName().toLowerCase()); + } + } + } + + if(!CollectionUtils.isEmpty(policy.getRowFilterPolicyItems())){ + for(RangerRowFilterPolicyItem rangerRowFilterPolicyItem:policy.getRowFilterPolicyItems()){ + if(!CollectionUtils.isEmpty(rangerRowFilterPolicyItem.getAccesses())){ + for(RangerPolicyItemAccess rangerPolicyItemAccess : rangerRowFilterPolicyItem.getAccesses()){ + if(!rowFilterAccessTypeDefNames.contains(rangerPolicyItemAccess.getType().toLowerCase())){ + ValidationErrorCode error = ValidationErrorCode.POLICY_VALIDATION_ERR_POLICY_ITEM_ACCESS_TYPE_INVALID; + failures.add(new ValidationFailureDetailsBuilder() + .field("row filter policy item access type") + .isSemanticallyIncorrect() + .becauseOf(error.getMessage(rangerPolicyItemAccess.getType(), rowFilterAccessTypeDefNames)) + .errorCode(error.getErrorCode()) + .build()); + valid = false; + } + } + } + } + } + } + //data mask policy + if (policyType==RangerPolicy.POLICY_TYPE_DATAMASK){ + List dataMaskAccessTypeDefNames=new ArrayList(); + if(serviceDef!=null && serviceDef.getDataMaskDef()!=null){ + if(!CollectionUtils.isEmpty(serviceDef.getDataMaskDef().getAccessTypes())){ + for(RangerAccessTypeDef rangerAccessTypeDef:serviceDef.getDataMaskDef().getAccessTypes()){ + dataMaskAccessTypeDefNames.add(rangerAccessTypeDef.getName().toLowerCase()); + } + } + } + + if(!CollectionUtils.isEmpty(policy.getDataMaskPolicyItems())){ + for(RangerDataMaskPolicyItem rangerDataMaskPolicyItem:policy.getDataMaskPolicyItems()){ + if(!CollectionUtils.isEmpty(rangerDataMaskPolicyItem.getAccesses())){ + for(RangerPolicyItemAccess rangerPolicyItemAccess : rangerDataMaskPolicyItem.getAccesses()){ + if(!dataMaskAccessTypeDefNames.contains(rangerPolicyItemAccess.getType().toLowerCase())){ + ValidationErrorCode error = ValidationErrorCode.POLICY_VALIDATION_ERR_POLICY_ITEM_ACCESS_TYPE_INVALID; + failures.add(new ValidationFailureDetailsBuilder() + .field("data masking policy item access type") + .isSemanticallyIncorrect() + .becauseOf(error.getMessage(rangerPolicyItemAccess.getType(), dataMaskAccessTypeDefNames)) + .errorCode(error.getErrorCode()) + .build()); + valid = false; + } + } + } + } + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("<== RangerPolicyValidator.isValidAccessTypeDef(%s, %s, %s,%s,%s)", policy, failures, action,isAdmin,serviceDef)); + } + return valid; + } + + boolean isValidResources(RangerPolicy policy, final List failures, Action action, + boolean isAdmin, final RangerServiceDef serviceDef) { + + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("==> RangerPolicyValidator.isValidResources(%s, %s, %s, %s, %s)", policy, failures, action, isAdmin, serviceDef)); + } + + boolean valid = true; + Map resourceMap = policy.getResources(); + if (resourceMap != null) { // following checks can't be done meaningfully otherwise + valid = isPolicyResourceUnique(policy, failures, action) && valid; + if (serviceDef != null) { // following checks can't be done meaningfully otherwise + valid = isValidResourceNames(policy, failures, serviceDef) && valid; + valid = isValidResourceValues(resourceMap, failures, serviceDef) && valid; + valid = isValidResourceFlags(resourceMap, failures, serviceDef.getResources(), serviceDef.getName(), policy.getName(), isAdmin) && valid; + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("<== RangerPolicyValidator.isValidResources(%s, %s, %s, %s, %s): %s", policy, failures, action, isAdmin, serviceDef, valid)); + } + return valid; + } + + boolean isValidValiditySchedule(RangerPolicy policy, final List failures, Action action) { + + boolean valid = true; + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("==> RangerPolicyValidator.isValidValiditySchedule(%s, %s, %s)", policy, failures, action)); + } + List validitySchedules = policy.getValiditySchedules(); + List normalizedValiditySchedules = null; + + for (RangerValiditySchedule entry : validitySchedules) { + RangerValidityScheduleValidator validator = new RangerValidityScheduleValidator(entry); + + RangerValiditySchedule normalizedValiditySchedule = validator.validate(failures); + if (normalizedValiditySchedule == null) { + valid = false; + if (LOG.isDebugEnabled()) { + LOG.debug("Invalid Validity-Schedule:[" + entry +"]"); + } + } else { + if (normalizedValiditySchedules == null) { + normalizedValiditySchedules = new ArrayList<>(); + } + normalizedValiditySchedules.add(normalizedValiditySchedule); + } + } + if (valid && CollectionUtils.isNotEmpty(normalizedValiditySchedules)) { + policy.setValiditySchedules(normalizedValiditySchedules); + } + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("<== RangerPolicyValidator.isValidValiditySchedule(%s, %s, %s): %s", policy, failures, action, valid)); + } + return valid; + } + + boolean isPolicyResourceUnique(RangerPolicy policy, final List failures, Action action) { + + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("==> RangerPolicyValidator.isPolicyResourceUnique(%s, %s, %s)", policy, failures, action)); + } + + boolean valid = true; + if (!Boolean.TRUE.equals(policy.getIsEnabled())) { + LOG.debug("Policy is disabled. Skipping resource uniqueness validation."); + } else { + RangerPolicyResourceSignature policySignature = _factory.createPolicyResourceSignature(policy); + String signature = policySignature.getSignature(); + List policies = getPoliciesForResourceSignature(policy.getService(), signature); + if (CollectionUtils.isNotEmpty(policies)) { + ValidationErrorCode error = ValidationErrorCode.POLICY_VALIDATION_ERR_DUPLICATE_POLICY_RESOURCE; + RangerPolicy matchedPolicy = policies.iterator().next(); + // there shouldn't be a matching policy for create. During update only match should be to itself + if (action == Action.CREATE || (action == Action.UPDATE && (policies.size() > 1 || !matchedPolicy.getId().equals(policy.getId())))) { + failures.add(new ValidationFailureDetailsBuilder() + .field("resources") + .isSemanticallyIncorrect() + .becauseOf(error.getMessage(matchedPolicy.getName(), policy.getService())) + .errorCode(error.getErrorCode()) + .build()); + valid = false; + } + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("<== RangerPolicyValidator.isPolicyResourceUnique(%s, %s, %s): %s", policy, failures, action, valid)); + } + return valid; + } + + boolean isValidResourceNames(final RangerPolicy policy, final List failures, final RangerServiceDef serviceDef) { + + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("==> RangerPolicyValidator.isValidResourceNames(%s, %s, %s)", policy, failures, serviceDef)); + } + + boolean valid = true; + convertPolicyResourceNamesToLower(policy); + Set policyResources = policy.getResources().keySet(); + + RangerServiceDefHelper defHelper = new RangerServiceDefHelper(serviceDef); + Set> hierarchies = defHelper.getResourceHierarchies(policy.getPolicyType()); // this can be empty but not null! + if (hierarchies.isEmpty()) { + if (LOG.isDebugEnabled()) { + LOG.debug("RangerPolicyValidator.isValidResourceNames: serviceDef does not have any resource hierarchies, possibly due to invalid service def!!"); + } + ValidationErrorCode error = ValidationErrorCode.POLICY_VALIDATION_ERR_INVALID_RESOURCE_NO_COMPATIBLE_HIERARCHY; + failures.add(new ValidationFailureDetailsBuilder() + .field("service def resource hierarchies") + .subField("incompatible") + .isSemanticallyIncorrect() + .becauseOf(error.getMessage(serviceDef.getName(), " does not have any resource hierarchies")) + .errorCode(error.getErrorCode()) + .build()); + valid = false; + } else { + /* + * A policy is for a single hierarchy however, it doesn't specify which one. So we have to guess which hierarchy(s) it possibly be for. First, see if the policy could be for + * any of the known hierarchies? A candidate hierarchy is one whose resource levels are a superset of those in the policy. + * Why? What we want to catch at this stage is policies that straddles multiple hierarchies, e.g. db, udf and column for a hive policy. + * This has the side effect of catch spurious levels specified on the policy, e.g. having a level "blah" on a hive policy. + */ + Set> candidateHierarchies = filterHierarchies_hierarchyHasAllPolicyResources(policyResources, hierarchies, defHelper); + if (candidateHierarchies.isEmpty()) { + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("No compatible resource hierarchies found: resource[%s], service-def[%s], valid-resource-hierarchies[%s]", + policyResources.toString(), serviceDef.getName(), toStringHierarchies_all(hierarchies, defHelper))); + } + ValidationErrorCode error; + if (hierarchies.size() == 1) { // we can give a simpler message for single hierarchy service-defs which is the majority of cases + error = ValidationErrorCode.POLICY_VALIDATION_ERR_INVALID_RESOURCE_NO_COMPATIBLE_HIERARCHY_SINGLE; + } else { + error = ValidationErrorCode.POLICY_VALIDATION_ERR_INVALID_RESOURCE_NO_COMPATIBLE_HIERARCHY; + } + failures.add(new ValidationFailureDetailsBuilder() + .field("policy resources") + .subField("incompatible") + .isSemanticallyIncorrect() + .becauseOf(error.getMessage(serviceDef.getName(), toStringHierarchies_all(hierarchies, defHelper))) + .errorCode(error.getErrorCode()) + .build()); + valid = false; + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("isValidResourceNames: Found [" + candidateHierarchies.size() + "] compatible hierarchies: " + toStringHierarchies_all(candidateHierarchies, defHelper)); + } + /* + * Among the candidate hierarchies there should be at least one for which policy specifies all of the mandatory resources. Note that there could be multiple + * hierarchies that meet that criteria, e.g. a hive policy that specified only DB. It is not clear if it belongs to DB->UDF or DB->TBL->COL hierarchy. + * However, if both UDF and TBL were required then we can detect that policy does not specify mandatory levels for any of the candidate hierarchies. + */ + Set> validHierarchies = filterHierarchies_mandatoryResourcesSpecifiedInPolicy(policyResources, candidateHierarchies, defHelper); + if (validHierarchies.isEmpty()) { + ValidationErrorCode error; + if (candidateHierarchies.size() == 1) { // we can provide better message if there is a single candidate hierarchy + error = ValidationErrorCode.POLICY_VALIDATION_ERR_INVALID_RESOURCE_MISSING_MANDATORY_SINGLE; + } else { + error = ValidationErrorCode.POLICY_VALIDATION_ERR_INVALID_RESOURCE_MISSING_MANDATORY; + } + failures.add(new ValidationFailureDetailsBuilder() + .field("policy resources") + .subField("missing mandatory") + .isSemanticallyIncorrect() + .becauseOf(error.getMessage(serviceDef.getName(), toStringHierarchies_mandatory(candidateHierarchies, defHelper))) + .errorCode(error.getErrorCode()) + .build()); + valid = false; + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("isValidResourceNames: Found hierarchies with all mandatory fields specified: " + toStringHierarchies_mandatory(validHierarchies, defHelper)); + } + } + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("<== RangerPolicyValidator.isValidResourceNames(%s, %s, %s): %s", policy, failures, serviceDef, valid)); + } + return valid; + } + + /** + * String representation of mandatory resources of all the hierarchies suitable of showing to user. Mandatory resources within a hierarchy are not ordered per the hierarchy. + * @param hierarchies + * @param defHelper + * @return + */ + String toStringHierarchies_mandatory(Set> hierarchies, RangerServiceDefHelper defHelper) { + + // helper function skipping sanity checks of getting null arguments passed + StringBuilder builder = new StringBuilder(); + for (List aHierarchy : hierarchies) { + builder.append(defHelper.getMandatoryResourceNames(aHierarchy)); + builder.append(" "); + } + return builder.toString(); + } + + /** + * String representation of all resources of all hierarchies. Resources within a hierarchy are ordered per the hierarchy. + * @param hierarchies + * @param defHelper + * @return + */ + String toStringHierarchies_all(Set> hierarchies, RangerServiceDefHelper defHelper) { + + // helper function skipping sanity checks of getting null arguments passed + StringBuilder builder = new StringBuilder(); + for (List aHierarchy : hierarchies) { + builder.append(defHelper.getAllResourceNamesOrdered(aHierarchy)); + builder.append(" "); + } + return builder.toString(); + } + /** + * Returns the subset of all hierarchies that are a superset of the policy's resources. + * @param policyResources + * @param hierarchies + * @return + */ + Set> filterHierarchies_hierarchyHasAllPolicyResources(Set policyResources, Set> hierarchies, RangerServiceDefHelper defHelper) { + + // helper function skipping sanity checks of getting null arguments passed + Set> result = new HashSet>(hierarchies.size()); + for (List aHierarchy : hierarchies) { + if (defHelper.hierarchyHasAllResources(aHierarchy, policyResources)) { + result.add(aHierarchy); + } + } + return result; + } + + /** + * Returns the subset of hierarchies all of whose mandatory resources were found in policy's resource set. candidate hierarchies are expected to have passed + * filterHierarchies_hierarchyHasAllPolicyResources check first. + * @param policyResources + * @param hierarchies + * @param defHelper + * @return + */ + Set> filterHierarchies_mandatoryResourcesSpecifiedInPolicy(Set policyResources, Set> hierarchies, RangerServiceDefHelper defHelper) { + + // helper function skipping sanity checks of getting null arguments passed + Set> result = new HashSet>(hierarchies.size()); + for (List aHierarchy : hierarchies) { + Set mandatoryResources = defHelper.getMandatoryResourceNames(aHierarchy); + if (policyResources.containsAll(mandatoryResources)) { + result.add(aHierarchy); + } + } + return result; + } + + boolean isValidResourceFlags(final Map inputPolicyResources, final List failures, + final List resourceDefs, final String serviceDefName, final String policyName, boolean isAdmin) { + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("==> RangerPolicyValidator.isValidResourceFlags(%s, %s, %s, %s, %s, %s)", inputPolicyResources, failures, resourceDefs, serviceDefName, policyName, isAdmin)); + } + + boolean valid = true; + if (resourceDefs == null) { + LOG.debug("isValidResourceFlags: service Def is null"); + } else { + Map policyResources = getPolicyResourceWithLowerCaseKeys(inputPolicyResources); + for (RangerResourceDef resourceDef : resourceDefs) { + if (resourceDef == null) { + ValidationErrorCode error = ValidationErrorCode.POLICY_VALIDATION_ERR_NULL_RESOURCE_DEF; + failures.add(new ValidationFailureDetailsBuilder() + .field("resource-def") + .isAnInternalError() + .becauseOf(error.getMessage(serviceDefName)) + .errorCode(error.getErrorCode()) + .build()); + valid = false; + } else if (StringUtils.isBlank(resourceDef.getName())) { + ValidationErrorCode error = ValidationErrorCode.POLICY_VALIDATION_ERR_MISSING_RESOURCE_DEF_NAME; + failures.add(new ValidationFailureDetailsBuilder() + .field("resource-def-name") + .isAnInternalError() + .becauseOf(error.getMessage(serviceDefName)) + .errorCode(error.getErrorCode()) + .build()); + valid = false; + } else { + String resourceName = resourceDef.getName().toLowerCase(); + RangerPolicyResource policyResource = policyResources.get(resourceName); + if (policyResource == null) { + if (LOG.isDebugEnabled()) { + LOG.debug("a policy-resource object for resource[" + resourceName + "] on policy [" + policyName + "] was null"); + } + } else { + boolean excludesSupported = Boolean.TRUE.equals(resourceDef.getExcludesSupported()); // could be null + boolean policyResourceIsExcludes = Boolean.TRUE.equals(policyResource.getIsExcludes()); // could be null + if (policyResourceIsExcludes && !excludesSupported) { + ValidationErrorCode error = ValidationErrorCode.POLICY_VALIDATION_ERR_EXCLUDES_NOT_SUPPORTED; + failures.add(new ValidationFailureDetailsBuilder() + .field("isExcludes") + .subField(resourceName) + .isSemanticallyIncorrect() + .becauseOf(error.getMessage(resourceName)) + .errorCode(error.getErrorCode()) + .build()); + valid = false; + } + if (policyResourceIsExcludes && !isAdmin) { + ValidationErrorCode error = ValidationErrorCode.POLICY_VALIDATION_ERR_EXCLUDES_REQUIRES_ADMIN; + failures.add(new ValidationFailureDetailsBuilder() + .field("isExcludes") + .subField("isAdmin") + .isSemanticallyIncorrect() + .becauseOf(error.getMessage()) + .errorCode(error.getErrorCode()) + .build()); + valid = false; + } + boolean recursiveSupported = Boolean.TRUE.equals(resourceDef.getRecursiveSupported()); + boolean policyIsRecursive = Boolean.TRUE.equals(policyResource.getIsRecursive()); + if (policyIsRecursive && !recursiveSupported) { + ValidationErrorCode error = ValidationErrorCode.POLICY_VALIDATION_ERR_RECURSIVE_NOT_SUPPORTED; + failures.add(new ValidationFailureDetailsBuilder() + .field("isRecursive") + .subField(resourceName) + .isSemanticallyIncorrect() + .becauseOf(error.getMessage(resourceName)) + .errorCode(error.getErrorCode()) + .build()); + valid = false; + } + } + } + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("<== RangerPolicyValidator.isValidResourceFlags(%s, %s, %s, %s, %s, %s): %s", inputPolicyResources, failures, resourceDefs, serviceDefName, policyName, isAdmin, valid)); + } + return valid; + } + + boolean isValidResourceValues(Map resourceMap, List failures, RangerServiceDef serviceDef) { + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("==> RangerPolicyValidator.isValidResourceValues(%s, %s, %s)", resourceMap, failures, serviceDef)); + } + + boolean valid = true; + Map validationRegExMap = getValidationRegExes(serviceDef); + for (Map.Entry entry : resourceMap.entrySet()) { + String name = entry.getKey(); + RangerPolicyResource policyResource = entry.getValue(); + if(policyResource != null) { + if(CollectionUtils.isNotEmpty(policyResource.getValues())) { + Set resources = new HashSet<>(policyResource.getValues()); + for (String aValue : resources) { + if (StringUtils.isBlank(aValue)) { + policyResource.getValues().remove(aValue); + } + } + } + + if(CollectionUtils.isEmpty(policyResource.getValues())){ + ValidationErrorCode error = ValidationErrorCode.POLICY_VALIDATION_ERR_MISSING_RESOURCE_LIST; + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("Resource list was empty or contains null: value[%s], resource-name[%s], service-def-name[%s]", policyResource.getValues(), name, serviceDef.getName())); + } + failures.add(new ValidationFailureDetailsBuilder() + .field("resource-values") + .subField(name) + .isMissing() + .becauseOf(error.getMessage(name)) + .errorCode(error.getErrorCode()) + .build()); + valid=false; + } + + if (validationRegExMap.containsKey(name) && CollectionUtils.isNotEmpty(policyResource.getValues())) { + String regEx = validationRegExMap.get(name); + for (String aValue : policyResource.getValues()) { + if (!aValue.matches(regEx)) { + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("Resource failed regex check: value[%s], resource-name[%s], regEx[%s], service-def-name[%s]", aValue, name, regEx, serviceDef.getName())); + } + ValidationErrorCode error = ValidationErrorCode.POLICY_VALIDATION_ERR_INVALID_RESOURCE_VALUE_REGEX; + failures.add(new ValidationFailureDetailsBuilder() + .field("resource-values") + .subField(name) + .isSemanticallyIncorrect() + .becauseOf(error.getMessage(aValue, name)) + .errorCode(error.getErrorCode()) + .build()); + valid = false; + } + } + } + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("<== RangerPolicyValidator.isValidResourceValues(%s, %s, %s): %s", resourceMap, failures, serviceDef, valid)); + } + return valid; + } + + boolean isValidPolicyItems(List policyItems, List failures, RangerServiceDef serviceDef) { + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("==> RangerPolicyValidator.isValid(%s, %s, %s)", policyItems, failures, serviceDef)); + } + + boolean valid = true; + if (CollectionUtils.isEmpty(policyItems)) { + LOG.debug("policy items collection was null/empty"); + } else { + for (RangerPolicyItem policyItem : policyItems) { + if (policyItem == null) { + ValidationErrorCode error = ValidationErrorCode.POLICY_VALIDATION_ERR_NULL_POLICY_ITEM; + failures.add(new ValidationFailureDetailsBuilder() + .field("policy item") + .isMissing() + .becauseOf(error.getMessage()) + .errorCode(error.getErrorCode()) + .build()); + valid = false; + } else { + // we want to go through all elements even though one may be bad so all failures are captured + valid = isValidPolicyItem(policyItem, failures, serviceDef) && valid; + } + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("<== RangerPolicyValidator.isValid(%s, %s, %s): %s", policyItems, failures, serviceDef, valid)); + } + return valid; + } + + boolean isValidPolicyItem(RangerPolicyItem policyItem, List failures, RangerServiceDef serviceDef) { + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("==> RangerPolicyValidator.isValid(%s, %s, %s)", policyItem, failures, serviceDef)); + } + + boolean valid = true; + if (policyItem == null) { + LOG.debug("policy item was null!"); + } else { + // access items collection can't be empty (unless delegated admin is true) and should be otherwise valid + if (CollectionUtils.isEmpty(policyItem.getAccesses())) { + if (!Boolean.TRUE.equals(policyItem.getDelegateAdmin())) { + ValidationErrorCode error = ValidationErrorCode.POLICY_VALIDATION_ERR_MISSING_FIELD; + failures.add(new ValidationFailureDetailsBuilder() + .field("policy item accesses") + .isMissing() + .becauseOf(error.getMessage("policy item accesses")) + .errorCode(error.getErrorCode()) + .build()); + valid = false; + } else { + LOG.debug("policy item collection was null but delegated admin is true. Ok"); + } + } else { + valid = isValidItemAccesses(policyItem.getAccesses(), failures, serviceDef) && valid; + } + // both users and user-groups collections can't be empty + if (CollectionUtils.isEmpty(policyItem.getUsers()) && CollectionUtils.isEmpty(policyItem.getGroups()) && CollectionUtils.isEmpty(policyItem.getRoles())) { + ValidationErrorCode error = ValidationErrorCode.POLICY_VALIDATION_ERR_MISSING_USER_AND_GROUPS; + failures.add(new ValidationFailureDetailsBuilder() + .field("policy item users/user-groups/roles") + .isMissing() + .becauseOf(error.getMessage()) + .errorCode(error.getErrorCode()) + .build()); + valid = false; + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("<== RangerPolicyValidator.isValid(%s, %s, %s): %s", policyItem, failures, serviceDef, valid)); + } + return valid; + } + + boolean isValidItemAccesses(List accesses, List failures, RangerServiceDef serviceDef) { + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("==> RangerPolicyValidator.isValid(%s, %s, %s)", accesses, failures, serviceDef)); + } + + boolean valid = true; + if (CollectionUtils.isEmpty(accesses)) { + LOG.debug("policy item accesses collection was null/empty!"); + } else { + Set accessTypes = getAccessTypes(serviceDef); + for (RangerPolicyItemAccess access : accesses) { + if (access == null) { + ValidationErrorCode error = ValidationErrorCode.POLICY_VALIDATION_ERR_NULL_POLICY_ITEM_ACCESS; + failures.add(new ValidationFailureDetailsBuilder() + .field("policy item access") + .isMissing() + .becauseOf(error.getMessage()) + .errorCode(error.getErrorCode()) + .build()); + valid = false; + } else { + // we want to go through all elements even though one may be bad so all failures are captured + valid = isValidPolicyItemAccess(access, failures, accessTypes) && valid; + } + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("<== RangerPolicyValidator.isValid(%s, %s, %s): %b", accesses, failures, serviceDef, valid)); + } + return valid; + } + + boolean isValidPolicyItemAccess(RangerPolicyItemAccess access, List failures, Set accessTypes) { + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("==> RangerPolicyValidator.isValidPolicyItemAccess(%s, %s, %s)", access, failures, accessTypes)); + } + + boolean valid = true; + if (CollectionUtils.isEmpty(accessTypes)) { // caller should firewall this argument! + LOG.debug("isValidPolicyItemAccess: accessTypes was null!"); + } else if (access == null) { + LOG.debug("isValidPolicyItemAccess: policy item access was null!"); + } else { + String accessType = access.getType(); + if (StringUtils.isBlank(accessType)) { + ValidationErrorCode error = ValidationErrorCode.POLICY_VALIDATION_ERR_MISSING_FIELD; + failures.add(new ValidationFailureDetailsBuilder() + .field("policy item access type") + .isMissing() + .becauseOf(error.getMessage("policy item access type")) + .errorCode(error.getErrorCode()) + .build()); + valid = false; + } else { + String matchedAccessType = getMatchedAccessType(accessType, accessTypes); + if (StringUtils.isEmpty(matchedAccessType)) { + ValidationErrorCode error = ValidationErrorCode.POLICY_VALIDATION_ERR_POLICY_ITEM_ACCESS_TYPE_INVALID; + failures.add(new ValidationFailureDetailsBuilder() + .field("policy item access type") + .isSemanticallyIncorrect() + .becauseOf(error.getMessage(accessType, accessTypes)) + .errorCode(error.getErrorCode()) + .build()); + valid = false; + } else { + access.setType(matchedAccessType); + } + } + Boolean isAllowed = access.getIsAllowed(); + // it can be null (which is treated as allowed) but not false + if (isAllowed != null && isAllowed == false) { + ValidationErrorCode error = ValidationErrorCode.POLICY_VALIDATION_ERR_POLICY_ITEM_ACCESS_TYPE_DENY; + failures.add(new ValidationFailureDetailsBuilder() + .field("policy item access type allowed") + .isSemanticallyIncorrect() + .becauseOf(error.getMessage()) + .errorCode(error.getErrorCode()) + .build()); + valid = false; + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("<== RangerPolicyValidator.isValidPolicyItemAccess(%s, %s, %s): %s", access, failures, accessTypes, valid)); + } + return valid; + } + + String getMatchedAccessType(String accessType, Set validAccessTypes) { + String ret = null; + for (String validType : validAccessTypes) { + if (StringUtils.equalsIgnoreCase(accessType, validType)) { + ret = validType; + break; + } + } + return ret; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/validation/RangerRoleValidator.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/validation/RangerRoleValidator.java new file mode 100644 index 00000000000..de9dfd55896 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/validation/RangerRoleValidator.java @@ -0,0 +1,224 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.model.validation; + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.plugin.errors.ValidationErrorCode; +import org.apache.atlas.plugin.model.RangerRole; +import org.apache.atlas.plugin.store.RoleStore; + +import java.util.ArrayList; +import java.util.List; + +public class RangerRoleValidator extends RangerValidator { + private static final Log LOG = LogFactory.getLog(RangerRoleValidator.class); + + public RangerRoleValidator(RoleStore store) { + super(store); + } + + public void validate(RangerRole rangeRole, Action action) throws Exception { + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("==> RangerRoleValidator.validate(%s, %s)", rangeRole, action)); + } + + List failures = new ArrayList<>(); + boolean valid = isValid(rangeRole, action, failures); + String message = ""; + try { + if (!valid) { + message = serializeFailures(failures); + throw new Exception(message); + } + } finally { + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("<== RangerRoleValidator.validate(%s, %s): %s, reason[%s]", rangeRole, action, valid, message)); + } + } + } + + @Override + boolean isValid(Long id, Action action, List failures) { + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("==> RangerRoleValidator.isValid(%s, %s, %s)", id, action, failures)); + } + + boolean valid = true; + if (action != Action.DELETE) { + ValidationErrorCode error = ValidationErrorCode.ROLE_VALIDATION_ERR_UNSUPPORTED_ACTION; + failures.add(new ValidationFailureDetailsBuilder() + .isAnInternalError() + .becauseOf(error.getMessage()) + .errorCode(error.getErrorCode()) + .build()); + valid = false; + } else if (id == null) { + ValidationErrorCode error = ValidationErrorCode.ROLE_VALIDATION_ERR_MISSING_FIELD; + failures.add(new ValidationFailureDetailsBuilder() + .becauseOf("Role id was null/missing") + .field("id") + .isMissing() + .errorCode(error.getErrorCode()) + .becauseOf(error.getMessage(id)) + .build()); + valid = false; + } else if (!roleExists(id)) { + ValidationErrorCode error = ValidationErrorCode.ROLE_VALIDATION_ERR_INVALID_ROLE_ID; + failures.add(new ValidationFailureDetailsBuilder() + .becauseOf("Role with id[{0}] does not exist") + .field("id") + .isMissing() + .errorCode(error.getErrorCode()) + .becauseOf(error.getMessage(id)) + .build()); + valid = false; + } + + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("<== RangerRoleValidator.isValid(%s, %s, %s): %s", id, action, failures, valid)); + } + return valid; + } + + + @Override + boolean isValid(String name, Action action, List failures) { + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("==> RangerRoleValidator.isValid(%s, %s, %s)", name, action, failures)); + } + + boolean valid = true; + if (action != Action.DELETE) { + ValidationErrorCode error = ValidationErrorCode.ROLE_VALIDATION_ERR_UNSUPPORTED_ACTION; + failures.add(new ValidationFailureDetailsBuilder() + .isAnInternalError() + .becauseOf(error.getMessage()) + .errorCode(error.getErrorCode()) + .build()); + valid = false; + } else if (name == null) { + ValidationErrorCode error = ValidationErrorCode.ROLE_VALIDATION_ERR_MISSING_FIELD; + failures.add(new ValidationFailureDetailsBuilder() + .becauseOf("Role name was null/missing") + .field("id") + .isMissing() + .errorCode(error.getErrorCode()) + .becauseOf(error.getMessage(name)) + .build()); + valid = false; + } else if (!roleExists(name)) { + ValidationErrorCode error = ValidationErrorCode.ROLE_VALIDATION_ERR_INVALID_ROLE_NAME; + failures.add(new ValidationFailureDetailsBuilder() + .becauseOf("Role with name[{0}] does not exist") + .field("name") + .isMissing() + .errorCode(error.getErrorCode()) + .becauseOf(error.getMessage(name)) + .build()); + valid = false; + } + + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("<== RangerRoleValidator.isValid(%s, %s, %s): %s", name, action, failures, valid)); + } + return valid; + } + + boolean isValid(RangerRole rangerRole, Action action, List failures) { + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("==> RangerRoleValidator.isValid(%s, %s, %s)", rangerRole, action, failures)); + } + + boolean valid = true; + if (rangerRole == null) { + ValidationErrorCode error = ValidationErrorCode.ROLE_VALIDATION_ERR_NULL_RANGER_ROLE_OBJECT; + failures.add(new ValidationFailureDetailsBuilder() + .isAnInternalError() + .isMissing() + .becauseOf(error.getMessage()) + .errorCode(error.getErrorCode()) + .build()); + valid = false; + } else { + String roleName = rangerRole.getName(); + if (StringUtils.isEmpty(roleName)) { + ValidationErrorCode error = ValidationErrorCode.ROLE_VALIDATION_ERR_NULL_RANGER_ROLE_NAME; + failures.add(new ValidationFailureDetailsBuilder() + .field("name") + .isMissing() + .becauseOf(error.getMessage()) + .errorCode(error.getErrorCode()) + .build()); + valid = false; + } + + Long id = rangerRole.getId(); + RangerRole existingRangerRole = null; + if (null != id) { + existingRangerRole = getRangerRole(id); + } + + if (action == Action.CREATE) { + if (existingRangerRole != null) { + String existingRoleName = existingRangerRole.getName(); + if (roleName.equals(existingRoleName)) { + ValidationErrorCode error = ValidationErrorCode.ROLE_VALIDATION_ERR_ROLE_NAME_CONFLICT; + failures.add(new ValidationFailureDetailsBuilder() + .field("name") + .isSemanticallyIncorrect() + .becauseOf(error.getMessage(existingRoleName)) + .errorCode(error.getErrorCode()) + .build()); + valid = false; + } + } + } else if (action == Action.UPDATE) { // id is ignored for CREATE + if (id == null) { + ValidationErrorCode error = ValidationErrorCode.ROLE_VALIDATION_ERR_MISSING_FIELD; + failures.add(new ValidationFailureDetailsBuilder() + .field("id") + .isMissing() + .becauseOf(error.getMessage(id)) + .errorCode(error.getErrorCode()) + .build()); + valid = false; + } + if (existingRangerRole == null) { + ValidationErrorCode error = ValidationErrorCode.ROLE_VALIDATION_ERR_INVALID_ROLE_ID; + failures.add(new ValidationFailureDetailsBuilder() + .field("id") + .isSemanticallyIncorrect() + .becauseOf(error.getMessage(id)) + .errorCode(error.getErrorCode()) + .build()); + valid = false; + } + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("<== RangerRoleValidator.isValid(%s, %s, %s): %s", rangerRole, action, failures, valid)); + } + + return valid; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/validation/RangerSecurityZoneValidator.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/validation/RangerSecurityZoneValidator.java new file mode 100644 index 00000000000..a9d5df52af6 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/validation/RangerSecurityZoneValidator.java @@ -0,0 +1,655 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.model.validation; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.plugin.errors.ValidationErrorCode; +import org.apache.atlas.plugin.model.RangerPolicy; +import org.apache.atlas.plugin.model.RangerSecurityZone; +import org.apache.atlas.plugin.model.RangerSecurityZone.RangerSecurityZoneService; +import org.apache.atlas.plugin.model.RangerService; +import org.apache.atlas.plugin.model.RangerServiceDef; +import org.apache.atlas.plugin.policyengine.RangerAccessResourceImpl; +import org.apache.atlas.plugin.policyengine.RangerResourceTrie; +import org.apache.atlas.plugin.policyresourcematcher.RangerDefaultPolicyResourceMatcher; +import org.apache.atlas.plugin.policyresourcematcher.RangerPolicyResourceMatcher; +import org.apache.atlas.plugin.store.EmbeddedServiceDefsUtil; +import org.apache.atlas.plugin.store.SecurityZoneStore; +import org.apache.atlas.plugin.store.ServiceStore; +import org.apache.atlas.plugin.util.SearchFilter; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class RangerSecurityZoneValidator extends RangerValidator { + private static final Log LOG = LogFactory.getLog(RangerSecurityZoneValidator.class); + + private final SecurityZoneStore securityZoneStore; + + public RangerSecurityZoneValidator(ServiceStore store, SecurityZoneStore securityZoneStore) { + super(store); + this.securityZoneStore = securityZoneStore; + } + + public void validate(RangerSecurityZone securityZone, Action action) throws Exception { + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("==> RangerPolicyValidator.validate(%s, %s)", securityZone, action)); + } + + List failures = new ArrayList<>(); + + boolean valid = isValid(securityZone, action, failures); + + String message; + try { + if (!valid) { + message = serializeFailures(failures); + throw new Exception(message); + } + + } finally { + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("<== RangerPolicyValidator.validate(%s, %s)", securityZone, action)); + } + } + } + + @Override + boolean isValid(String name, Action action, List failures) { + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("==> RangerPolicyValidator.isValid(%s, %s, %s)", name, action, failures)); + } + + boolean ret = true; + + if (action != Action.DELETE) { + ValidationErrorCode error = ValidationErrorCode.SECURITY_ZONE_VALIDATION_ERR_UNSUPPORTED_ACTION; + + failures.add(new ValidationFailureDetailsBuilder().isAnInternalError().becauseOf(error.getMessage()).errorCode(error.getErrorCode()).build()); + ret = false; + } else { + if (StringUtils.isEmpty(name)) { + ValidationErrorCode error = ValidationErrorCode.SECURITY_ZONE_VALIDATION_ERR_MISSING_FIELD; + + failures.add(new ValidationFailureDetailsBuilder().becauseOf("security zone name was null/missing").field("name").isMissing().errorCode(error.getErrorCode()).becauseOf(error.getMessage("name")).build()); + ret = false; + } else { + if (getSecurityZone(name) == null) { + ValidationErrorCode error = ValidationErrorCode.SECURITY_ZONE_VALIDATION_ERR_INVALID_ZONE_ID; + + failures.add(new ValidationFailureDetailsBuilder().becauseOf("security zone does not exist").field("name").errorCode(error.getErrorCode()).becauseOf(error.getMessage(name)).build()); + ret = false; + } + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("<== RangerPolicyValidator.isValid(%s, %s, %s) : %s", name, action, failures, ret)); + } + + return ret; + } + + @Override + boolean isValid(Long id, Action action, List failures) { + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("==> RangerPolicyValidator.isValid(%s, %s, %s)", id, action, failures)); + } + + boolean ret = true; + + if (action != Action.DELETE) { + ValidationErrorCode error = ValidationErrorCode.SECURITY_ZONE_VALIDATION_ERR_UNSUPPORTED_ACTION; + + failures.add(new ValidationFailureDetailsBuilder().isAnInternalError().becauseOf(error.getMessage()).errorCode(error.getErrorCode()).build()); + ret = false; + } else if (id == null) { + ValidationErrorCode error = ValidationErrorCode.SECURITY_ZONE_VALIDATION_ERR_MISSING_FIELD; + + failures.add(new ValidationFailureDetailsBuilder().becauseOf("security zone id was null/missing").field("id").isMissing().errorCode(error.getErrorCode()).becauseOf(error.getMessage("id")).build()); + ret = false; + } else if (getSecurityZone(id) == null) { + ValidationErrorCode error = ValidationErrorCode.SECURITY_ZONE_VALIDATION_ERR_INVALID_ZONE_ID; + + failures.add(new ValidationFailureDetailsBuilder().becauseOf("security zone id does not exist").field("id").errorCode(error.getErrorCode()).becauseOf(error.getMessage(id)).build()); + ret = false; + } + + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("<== RangerPolicyValidator.isValid(%s, %s, %s) : %s", id, action, failures, ret)); + } + + return ret; + } + + boolean isValid(RangerSecurityZone securityZone, Action action, List failures) { + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("==> RangerPolicyValidator.isValid(%s, %s, %s)", securityZone, action, failures)); + } + + if (!(action == Action.CREATE || action == Action.UPDATE)) { + throw new IllegalArgumentException("isValid(RangerPolicy, ...) is only supported for create/update"); + } + + boolean ret = true; + + RangerSecurityZone existingZone; + final String zoneName = securityZone.getName(); + if (StringUtils.isEmpty(StringUtils.trim(zoneName))) { + ValidationErrorCode error = ValidationErrorCode.SECURITY_ZONE_VALIDATION_ERR_MISSING_FIELD; + + failures.add(new ValidationFailureDetailsBuilder().becauseOf("security zone name was null/missing").field("name").isMissing().errorCode(error.getErrorCode()).becauseOf(error.getMessage("name")).build()); + ret = false; + } + + if (action == Action.CREATE) { + securityZone.setId(-1L); + existingZone = getSecurityZone(zoneName); + if (existingZone != null) { + ValidationErrorCode error = ValidationErrorCode.SECURITY_ZONE_VALIDATION_ERR_ZONE_NAME_CONFLICT; + + failures.add(new ValidationFailureDetailsBuilder().becauseOf("security zone name exists").field("name").errorCode(error.getErrorCode()).becauseOf(error.getMessage(existingZone.getId())).build()); + ret = false; + } + } else { + Long zoneId = securityZone.getId(); + existingZone = getSecurityZone(zoneId); + + if (existingZone == null) { + ValidationErrorCode error = ValidationErrorCode.SECURITY_ZONE_VALIDATION_ERR_INVALID_ZONE_ID; + + failures.add(new ValidationFailureDetailsBuilder().becauseOf("security zone with id does not exist").field("id").errorCode(error.getErrorCode()).becauseOf(error.getMessage(zoneId)).build()); + ret = false; + } else if (StringUtils.isNotEmpty(StringUtils.trim(zoneName)) && !StringUtils.equals(zoneName, existingZone.getName())) { + existingZone = getSecurityZone(zoneName); + + if (existingZone != null) { + if (!StringUtils.equals(existingZone.getName(), zoneName)) { + ValidationErrorCode error = ValidationErrorCode.SECURITY_ZONE_VALIDATION_ERR_ZONE_NAME_CONFLICT; + + failures.add(new ValidationFailureDetailsBuilder().becauseOf("security zone name").field("name").errorCode(error.getErrorCode()).becauseOf(error.getMessage(existingZone.getId())).build()); + ret = false; + } + } + } + } + + ret = ret && validateWithinSecurityZone(securityZone, action, failures); + + ret = ret && validateAgainstAllSecurityZones(securityZone, action, failures); + + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("<== RangerPolicyValidator.isValid(%s, %s, %s) : %s", securityZone, action, failures, ret)); + } + + return ret; + } + + private boolean validateWithinSecurityZone(RangerSecurityZone securityZone, Action action, List failures) { + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("==> RangerPolicyValidator.validateWithinSecurityZone(%s, %s, %s)", securityZone, action, failures)); + } + + boolean ret = true; + + // Validate each service for existence, not being tag-service and each resource-spec for validity + if (MapUtils.isNotEmpty(securityZone.getServices())) { + for (Map.Entry serviceSpecification : securityZone.getServices().entrySet()) { + String serviceName = serviceSpecification.getKey(); + RangerSecurityZone.RangerSecurityZoneService securityZoneService = serviceSpecification.getValue(); + + ret = ret && validateSecurityZoneService(serviceName, securityZoneService, failures); + } + } else { + ValidationErrorCode error = ValidationErrorCode.SECURITY_ZONE_VALIDATION_ERR_MISSING_SERVICES; + + failures.add(new ValidationFailureDetailsBuilder().becauseOf("security zone services").isMissing().field("services").errorCode(error.getErrorCode()).becauseOf(error.getMessage(securityZone.getName())).build()); + ret = false; + } + // both admin users and user-groups collections can't be empty + if (CollectionUtils.isEmpty(securityZone.getAdminUsers()) && CollectionUtils.isEmpty(securityZone.getAdminUserGroups())) { + ValidationErrorCode error = ValidationErrorCode.SECURITY_ZONE_VALIDATION_ERR_MISSING_USER_AND_GROUPS; + + failures.add(new ValidationFailureDetailsBuilder().field("security zone admin users/user-groups").isMissing().becauseOf(error.getMessage()).errorCode(error.getErrorCode()).build()); + ret = false; + } + // both audit users and user-groups collections can't be empty + if (CollectionUtils.isEmpty(securityZone.getAuditUsers()) && CollectionUtils.isEmpty(securityZone.getAuditUserGroups())) { + ValidationErrorCode error = ValidationErrorCode.SECURITY_ZONE_VALIDATION_ERR_MISSING_USER_AND_GROUPS; + + failures.add(new ValidationFailureDetailsBuilder().field("security zone audit users/user-groups").isMissing().becauseOf(error.getMessage()).errorCode(error.getErrorCode()).build()); + ret = false; + } + + if (securityZone.getServices() != null) { + for (Map.Entry serviceResourceMapEntry : securityZone.getServices() + .entrySet()) { + if (serviceResourceMapEntry.getValue().getResources() != null) { + for (Map> resource : serviceResourceMapEntry.getValue().getResources()) { + if (resource != null) { + for (Map.Entry> entry : resource.entrySet()) { + if (CollectionUtils.isEmpty(entry.getValue())) { + ValidationErrorCode error = ValidationErrorCode.SECURITY_ZONE_VALIDATION_ERR_MISSING_RESOURCES; + failures.add(new ValidationFailureDetailsBuilder().field("security zone resources") + .subField("resources").isMissing() + .becauseOf(error.getMessage(serviceResourceMapEntry.getKey())) + .errorCode(error.getErrorCode()).build()); + ret = false; + } + } + } + } + } + } + } + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("<== RangerPolicyValidator.validateWithinSecurityZone(%s, %s, %s) : %s", securityZone, action, failures, ret)); + } + return ret; + } + + private boolean validateAgainstAllSecurityZones(RangerSecurityZone securityZone, Action action, List failures) { + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("==> RangerPolicyValidator.validateAgainstAllSecurityZones(%s, %s, %s)", securityZone, action, failures)); + } + + boolean ret = true; + + final String zoneName; + + if (securityZone.getId() != -1L) { + RangerSecurityZone existingZone = getSecurityZone(securityZone.getId()); + zoneName = existingZone.getName(); + } else { + zoneName = securityZone.getName(); + } + + for (Map.Entry entry: securityZone.getServices().entrySet()) { + String serviceName = entry.getKey(); + RangerSecurityZone.RangerSecurityZoneService serviceResources = entry.getValue(); + + if (CollectionUtils.isNotEmpty(serviceResources.getResources())) { + SearchFilter filter = new SearchFilter(); + List zones = null; + + filter.setParam(SearchFilter.SERVICE_NAME, serviceName); + filter.setParam(SearchFilter.ZONE_NAME, zoneName); + + try { + zones = securityZoneStore.getSecurityZones(filter); + } catch (Exception excp) { + LOG.error("Failed to get Security-Zones", excp); + ValidationErrorCode error = ValidationErrorCode.SECURITY_ZONE_VALIDATION_ERR_INTERNAL_ERROR; + + failures.add(new ValidationFailureDetailsBuilder().becauseOf(error.getMessage(excp.getMessage())).errorCode(error.getErrorCode()).build()); + ret = false; + } + + if (CollectionUtils.isNotEmpty(zones)) { + RangerService service = getService(serviceName); + RangerServiceDef serviceDef = service != null ? getServiceDef(service.getType()) : null; + + if (serviceDef == null) { + ValidationErrorCode error = ValidationErrorCode.SECURITY_ZONE_VALIDATION_ERR_INTERNAL_ERROR; + + failures.add(new ValidationFailureDetailsBuilder().becauseOf(error.getMessage(serviceName)).errorCode(error.getErrorCode()).build()); + ret = false; + + } else { + zones.add(securityZone); + ret = ret && validateZoneServiceInAllZones(zones, serviceName, serviceDef, failures); + } + } + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("<== RangerPolicyValidator.validateAgainstAllSecurityZones(%s, %s, %s) : %s", securityZone, action, failures, ret)); + } + + return ret; + } + + private boolean validateZoneServiceInAllZones(List zones, String serviceName, RangerServiceDef serviceDef, List failures) { + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("==> RangerPolicyValidator.validateZoneServiceInAllZones(%s, %s, %s, %s)", zones, serviceName, serviceDef, failures)); + } + + boolean ret = true; + + // For each zone, get list-of-resources corresponding to serviceName. + // For each list-of-resources: + // get one resource (this is a map of >); convert it into map of . excludes is always false, recursive true only for HDFS + // build a subclass of RangerPolicyResourceEvaluator with id of zone, zoneName as a member, and RangerDefaultResourceMatcher as matcher. + // add this to list-of-evaluators + + Map> matchersForResourceDef = new HashMap<>(); + + for (RangerSecurityZone zone : zones) { + List>> resources = zone.getServices().get(serviceName).getResources(); + + for (Map> resource : resources) { + Map policyResources = new HashMap<>(); + + for (Map.Entry> entry : resource.entrySet()) { + String resourceDefName = entry.getKey(); + List resourceValues = entry.getValue(); + + RangerPolicy.RangerPolicyResource policyResource = new RangerPolicy.RangerPolicyResource(); + + policyResource.setIsExcludes(false); + policyResource.setIsRecursive(EmbeddedServiceDefsUtil.isRecursiveEnabled(serviceDef, resourceDefName)); + policyResource.setValues(resourceValues); + policyResources.put(resourceDefName, policyResource); + + if (matchersForResourceDef.get(resourceDefName) == null) { + matchersForResourceDef.put(resourceDefName, new ArrayList<>()); + } + } + + RangerZoneResourceMatcher matcher = new RangerZoneResourceMatcher(zone.getName(), policyResources, serviceDef); + + for (String resourceDefName : resource.keySet()) { + matchersForResourceDef.get(resourceDefName).add(matcher); + } + } + } + + // Build a map of trie with list-of-evaluators with one entry corresponds to one resource-def if it exists in the list-of-resources + + Map> trieMap = new HashMap<>(); + List resourceDefs = serviceDef.getResources(); + + for (Map.Entry> entry : matchersForResourceDef.entrySet()) { + String resourceDefName = entry.getKey(); + List matchers = entry.getValue(); + RangerServiceDef.RangerResourceDef resourceDef = null; + + for (RangerServiceDef.RangerResourceDef element : resourceDefs) { + if (StringUtils.equals(element.getName(), resourceDefName)) { + resourceDef = element; + break; + } + } + + trieMap.put(entry.getKey(), new RangerResourceTrie<>(resourceDef, matchers)); + } + + // For each zone, get list-of-resources corresponding to serviceName + // For each list-of-resources: + // get one resource; for each level in the resource, run it through map of trie and get possible evaluators. + // check each evaluator to see if the resource-match actually happens. If yes then add the zone-evaluator to matching evaluators. + // flag error if there are more than one matching evaluators with different zone-ids. + // + + RangerServiceDefHelper serviceDefHelper = new RangerServiceDefHelper(serviceDef, true); + + for (RangerSecurityZone zone : zones) { + List>> resources = zone.getServices().get(serviceName).getResources(); + + for (Map> resource : resources) { + + Set smallestList = null; + + List resourceKeys = serviceDefHelper.getOrderedResourceNames(resource.keySet()); + + for (String resourceDefName : resourceKeys) { + List resourceValues = resource.get(resourceDefName); + + RangerResourceTrie trie = trieMap.get(resourceDefName); + + Set zoneMatchersForResource = trie.getEvaluatorsForResource(resourceValues); + Set inheritedZoneMatchers = trie.getInheritedEvaluators(); + + if (LOG.isDebugEnabled()) { + LOG.debug("ResourceDefName:[" + resourceDefName + "], values:[" + resourceValues + "], matched-zones:[" + zoneMatchersForResource + "], inherited-zones:[" + inheritedZoneMatchers + "]"); + } + + if (smallestList != null) { + if (CollectionUtils.isEmpty(inheritedZoneMatchers) && CollectionUtils.isEmpty(zoneMatchersForResource)) { + smallestList = null; + } else if (CollectionUtils.isEmpty(inheritedZoneMatchers)) { + smallestList.retainAll(zoneMatchersForResource); + } else if (CollectionUtils.isEmpty(zoneMatchersForResource)) { + smallestList.retainAll(inheritedZoneMatchers); + } else { + Set smaller, bigger; + if (zoneMatchersForResource.size() < inheritedZoneMatchers.size()) { + smaller = zoneMatchersForResource; + bigger = inheritedZoneMatchers; + } else { + smaller = inheritedZoneMatchers; + bigger = zoneMatchersForResource; + } + Set tmp = new HashSet<>(); + if (smallestList.size() < smaller.size()) { + smallestList.stream().filter(smaller::contains).forEach(tmp::add); + smallestList.stream().filter(bigger::contains).forEach(tmp::add); + } else { + smaller.stream().filter(smallestList::contains).forEach(tmp::add); + if (smallestList.size() < bigger.size()) { + smallestList.stream().filter(bigger::contains).forEach(tmp::add); + } else { + bigger.stream().filter(smallestList::contains).forEach(tmp::add); + } + } + smallestList = tmp; + } + } else { + if (CollectionUtils.isEmpty(inheritedZoneMatchers) || CollectionUtils.isEmpty(zoneMatchersForResource)) { + Set tmp = CollectionUtils.isEmpty(inheritedZoneMatchers) ? zoneMatchersForResource : inheritedZoneMatchers; + smallestList = resourceKeys.size() == 1 || CollectionUtils.isEmpty(tmp) ? tmp : new HashSet<>(tmp); + } else { + smallestList = new HashSet<>(zoneMatchersForResource); + smallestList.addAll(inheritedZoneMatchers); + } + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("Resource:[" + resource +"], matched-zones:[" + smallestList +"]"); + } + + if (CollectionUtils.isEmpty(smallestList) || smallestList.size() == 1) { + continue; + } + + final Set intersection = smallestList; + + RangerAccessResourceImpl accessResource = new RangerAccessResourceImpl(); + + accessResource.setServiceDef(serviceDef); + + for (Map.Entry> entry : resource.entrySet()) { + accessResource.setValue(entry.getKey(), entry.getValue()); + } + + Set matchedZoneNames = new HashSet<>(); + + for (RangerZoneResourceMatcher zoneMatcher : intersection) { + if (LOG.isDebugEnabled()) { + LOG.debug("Trying to match resource:[" + accessResource +"] using zoneMatcher:[" + zoneMatcher + "]"); + } + // These are potential matches. Try to really match them + if (zoneMatcher.getPolicyResourceMatcher().isMatch(accessResource, RangerPolicyResourceMatcher.MatchScope.ANY, null)) { + if (LOG.isDebugEnabled()) { + LOG.debug("Matched resource:[" + accessResource +"] using zoneMatcher:[" + zoneMatcher + "]"); + } + // Actual match happened + matchedZoneNames.add(zoneMatcher.getSecurityZoneName()); + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Did not match resource:[" + accessResource +"] using zoneMatcher:[" + zoneMatcher + "]"); + } + } + } + LOG.info("The following zone-names matched resource:[" + resource + "]: " + matchedZoneNames); + + if (matchedZoneNames.size() > 1) { + ValidationErrorCode error = ValidationErrorCode.SECURITY_ZONE_VALIDATION_ERR_ZONE_RESOURCE_CONFLICT; + + failures.add(new ValidationFailureDetailsBuilder().becauseOf(error.getMessage(matchedZoneNames, resource)).errorCode(error.getErrorCode()).build()); + ret = false; + break; + } + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("<== RangerPolicyValidator.validateZoneServiceInAllZones(%s, %s, %s, %s) : %s", zones, serviceName, serviceDef, failures, ret)); + } + return ret; + } + + private boolean validateSecurityZoneService(String serviceName, RangerSecurityZone.RangerSecurityZoneService securityZoneService, List failures) { + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("==> RangerPolicyValidator.validateSecurityZoneService(%s, %s, %s)", serviceName, securityZoneService, failures)); + } + + boolean ret = true; + + // Verify service with serviceName exists - get the service-type + RangerService service = getService(serviceName); + + if (service == null) { + ValidationErrorCode error = ValidationErrorCode.SECURITY_ZONE_VALIDATION_ERR_INVALID_SERVICE_NAME; + + failures.add(new ValidationFailureDetailsBuilder().field("security zone resource service-name").becauseOf(error.getMessage(serviceName)).errorCode(error.getErrorCode()).build()); + ret = false; + } else { + RangerServiceDef serviceDef = getServiceDef(service.getType()); + + if (serviceDef == null) { + ValidationErrorCode error = ValidationErrorCode.SECURITY_ZONE_VALIDATION_ERR_INVALID_SERVICE_TYPE; + failures.add(new ValidationFailureDetailsBuilder().field("security zone resource service-type").becauseOf(error.getMessage(service.getType())).errorCode(error.getErrorCode()).build()); + ret = false; + } else { + String serviceType = serviceDef.getName(); + + if (StringUtils.equals(serviceType, EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_TAG_NAME)) { + if (CollectionUtils.isNotEmpty(securityZoneService.getResources())) { + ValidationErrorCode error = ValidationErrorCode.SECURITY_ZONE_VALIDATION_ERR_UNEXPECTED_RESOURCES; + failures.add(new ValidationFailureDetailsBuilder().field("security zone resources").becauseOf(error.getMessage(serviceName)).errorCode(error.getErrorCode()).build()); + ret = false; + } + } else { + if (CollectionUtils.isEmpty(securityZoneService.getResources())) { + ValidationErrorCode error = ValidationErrorCode.SECURITY_ZONE_VALIDATION_ERR_MISSING_RESOURCES; + failures.add(new ValidationFailureDetailsBuilder().field("security zone resources").isMissing().becauseOf(error.getMessage(serviceName)).errorCode(error.getErrorCode()).build()); + ret = false; + } else { + // For each resource-spec, verify that it forms valid hierarchy for some policy-type + for (Map> resource : securityZoneService.getResources()) { + Set resourceDefNames = resource.keySet(); + RangerServiceDefHelper serviceDefHelper = new RangerServiceDefHelper(serviceDef); + boolean isValidHierarchy = false; + + for (String policyType : RangerPolicy.POLICY_TYPES) { + Set> resourceHierarchies = serviceDefHelper.getResourceHierarchies(policyType, resourceDefNames); + + if (LOG.isDebugEnabled()) { + LOG.debug("Size of resourceHierarchies for resourceDefNames:[" + resourceDefNames + ", policyType=" + policyType + "] = " + resourceHierarchies.size()); + } + + for (List resourceHierarchy : resourceHierarchies) { + + if (RangerDefaultPolicyResourceMatcher.isHierarchyValidForResources(resourceHierarchy, resource)) { + isValidHierarchy = true; + break; + } else { + LOG.info("gaps found in resource, skipping hierarchy:[" + resourceHierarchies + "]"); + } + } + } + + if (!isValidHierarchy) { + ValidationErrorCode error = ValidationErrorCode.SECURITY_ZONE_VALIDATION_ERR_INVALID_RESOURCE_HIERARCHY; + + failures.add(new ValidationFailureDetailsBuilder().field("security zone resource hierarchy").becauseOf(error.getMessage(serviceName, resourceDefNames)).errorCode(error.getErrorCode()).build()); + ret = false; + } + + /* + * Ignore this check. It should be possible to have all wildcard resource in a zone if zone-admin so desires + * + boolean isValidResourceSpec = isAnyNonWildcardResource(resource, failures); + + if (!isValidResourceSpec) { + ValidationErrorCode error = ValidationErrorCode.SECURITY_ZONE_VALIDATION_ERR_ALL_WILDCARD_RESOURCE_VALUES; + + failures.add(new ValidationFailureDetailsBuilder().field("security zone resource values").becauseOf(error.getMessage(serviceName)).errorCode(error.getErrorCode()).build()); + ret = false; + LOG.warn("RangerPolicyValidator.validateSecurityZoneService() : All wildcard resource-values specified for service :[" + serviceName + "]"); + } + */ + + } + } + } + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("<== RangerPolicyValidator.validateSecurityZoneService(%s, %s, %s) : %s", serviceName, securityZoneService, failures, ret)); + } + + return ret; + } + + /* + private boolean isAnyNonWildcardResource(Map> resource, List failures) { + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("==> RangerPolicyValidator.isAnyNonWildcardResource(%s, %s)", resource, failures)); + } + + boolean ret = false; + + for (Map.Entry> resourceDefValue : resource.entrySet()) { + boolean wildCardResourceFound = false; + List resourceValues = resourceDefValue.getValue(); + + for (String resourceValue : resourceValues) { + if (StringUtils.equals(resourceValue, RangerDefaultResourceMatcher.WILDCARD_ASTERISK)) { + wildCardResourceFound = true; + break; + } + } + + if (!wildCardResourceFound) { + ret = true; + break; + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("<== RangerPolicyValidator.isAnyNonWildcardResource(%s, %s) : %s", resource, failures, ret)); + } + return ret; + } + */ +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/validation/RangerServiceDefHelper.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/validation/RangerServiceDefHelper.java new file mode 100644 index 00000000000..ddc359b3cd4 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/validation/RangerServiceDefHelper.java @@ -0,0 +1,797 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.model.validation; + +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.plugin.model.RangerPolicy; +import org.apache.atlas.plugin.model.RangerServiceDef; +import org.apache.atlas.plugin.model.RangerServiceDef.RangerResourceDef; +import org.apache.atlas.plugin.resourcematcher.RangerAbstractResourceMatcher; +import org.apache.atlas.plugin.resourcematcher.RangerPathResourceMatcher; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +public class RangerServiceDefHelper { + private static final Log LOG = LogFactory.getLog(RangerServiceDefHelper.class); + + static final Map _Cache = new ConcurrentHashMap<>(); + final Delegate _delegate; + + static public RangerServiceDef getServiceDefForPolicyFiltering(RangerServiceDef serviceDef) { + + List modifiedResourceDefs = new ArrayList(); + + for (RangerResourceDef resourceDef : serviceDef.getResources()) { + + final RangerResourceDef modifiedResourceDef; + + String matcherClassName = resourceDef.getMatcher(); + + if (RangerPathResourceMatcher.class.getName().equals(matcherClassName)) { + + Map modifiedMatcherOptions = new HashMap(resourceDef.getMatcherOptions()); + + modifiedMatcherOptions.put(RangerAbstractResourceMatcher.OPTION_WILD_CARD, "false"); + + modifiedResourceDef = new RangerResourceDef(resourceDef); + modifiedResourceDef.setMatcherOptions(modifiedMatcherOptions); + modifiedResourceDef.setRecursiveSupported(false); + + } else { + modifiedResourceDef = resourceDef; + } + + modifiedResourceDefs.add(modifiedResourceDef); + } + + return new RangerServiceDef(serviceDef.getName(), serviceDef.getDisplayName(), serviceDef.getImplClass(), serviceDef.getLabel(), + serviceDef.getDescription(), serviceDef.getOptions(), serviceDef.getConfigs(), modifiedResourceDefs, serviceDef.getAccessTypes(), + serviceDef.getPolicyConditions(), serviceDef.getContextEnrichers(), serviceDef.getEnums()); + } + + public static Map getFilterResourcesForAncestorPolicyFiltering(RangerServiceDef serviceDef, Map filterResources) { + + Map ret = null; + + for (RangerResourceDef resourceDef : serviceDef.getResources()) { + + String matcherClassName = resourceDef.getMatcher(); + + if (RangerPathResourceMatcher.class.getName().equals(matcherClassName)) { + + String resourceDefName = resourceDef.getName(); + + final Map resourceMatcherOptions = resourceDef.getMatcherOptions(); + + String delimiter = resourceMatcherOptions.get(RangerPathResourceMatcher.OPTION_PATH_SEPARATOR); + if (StringUtils.isBlank(delimiter)) { + delimiter = Character.toString(RangerPathResourceMatcher.DEFAULT_PATH_SEPARATOR_CHAR); + } + + String resourceValue = filterResources.get(resourceDefName); + if (StringUtils.isNotBlank(resourceValue)) { + if (!resourceValue.endsWith(delimiter)) { + resourceValue += delimiter; + } + resourceValue += RangerAbstractResourceMatcher.WILDCARD_ASTERISK; + + if (ret == null) { + ret = new HashMap(); + } + ret.put(resourceDefName, resourceValue); + } + } + } + + return ret; + } + + public RangerServiceDefHelper(RangerServiceDef serviceDef) { + this(serviceDef, true, false); + } + + public RangerServiceDefHelper(RangerServiceDef serviceDef, boolean useCache) { + this(serviceDef, useCache, false); + } + + /** + * Intended for use when serviceDef object is not-trusted, e.g. when service-def is being created or updated. + * @param serviceDef + * @param useCache + */ + public RangerServiceDefHelper(RangerServiceDef serviceDef, boolean useCache, boolean checkForCycles) { + // NOTE: we assume serviceDef, its name and update time are can never by null. + + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("==> RangerServiceDefHelper(). The RangerServiceDef: %s", serviceDef)); + } + + String serviceName = serviceDef.getName(); + Date serviceDefFreshnessDate = serviceDef.getUpdateTime(); + + Delegate delegate = null; + if (useCache && _Cache.containsKey(serviceName)) { + LOG.debug("RangerServiceDefHelper(): found delegate in cache with matching serviceName. Need to check date"); + Delegate that = _Cache.get(serviceName); + if (Objects.equals(that.getServiceFreshnessDate(), serviceDefFreshnessDate)) { + delegate = that; + LOG.debug("RangerServiceDefHelper(): cached delegate matched in date, too! Will use it now."); + } else { + LOG.debug("RangerServiceDefHelper(): cached delegate date mismatch!"); + } + } + if (delegate == null) { // either not found in cache or date didn't match + delegate = new Delegate(serviceDef, checkForCycles); + if (useCache) { + LOG.debug("RangerServiceDefHelper(): Created new delegate and put in delegate cache!"); + _Cache.put(serviceName, delegate); + } + } + _delegate = delegate; + } + + public RangerServiceDef getServiceDef() { + return _delegate._serviceDef; + } + + public void patchServiceDefWithDefaultValues() { + _delegate.patchServiceDefWithDefaultValues(); + } + + /** + * for a resource definition as follows: + * + * /-> E -> F + * A -> B -> C -> D + * \-> G -> H + * + * It would return a set with following ordered entries in it + * { [A B C D], [A E F], [A B G H] } + * + * @return + */ + public Set> getResourceHierarchies(String policyType) { + return _delegate.getResourceHierarchies(policyType); + } + + public Set> filterHierarchies_containsOnlyMandatoryResources(String policyType) { + Set> hierarchies = getResourceHierarchies(policyType); + Set> result = new HashSet>(hierarchies.size()); + for (List aHierarchy : hierarchies) { + Set mandatoryResources = getMandatoryResourceNames(aHierarchy); + if (aHierarchy.size() == mandatoryResources.size()) { + result.add(aHierarchy); + } + } + return result; + } + + public Set> getResourceHierarchies(String policyType, Collection keys) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> getResourceHierarchies(policyType=" + policyType + ", keys=" + StringUtils.join(keys, ",") + ")"); + } + + Set> ret = new HashSet>(); + + if (StringUtils.isEmpty(policyType)) { + policyType = RangerPolicy.POLICY_TYPE_ACCESS; + } + + for (List hierarchy : getResourceHierarchies(policyType)) { + if (hierarchyHasAllResources(hierarchy, keys)) { + ret.add(hierarchy); + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== getResourceHierarchies(policyType=" + policyType + ", keys=" + StringUtils.join(keys, ",") + ") : " + StringUtils.join(ret, ",")); + } + return ret; + } + + public boolean hierarchyHasAllResources(List hierarchy, Collection resourceNames) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> hierarchyHasAllResources(hierarchy=" + StringUtils.join(hierarchy, ",") + ", resourceNames=" + StringUtils.join(resourceNames, ",") + ")"); + } + boolean foundAllResourceKeys = true; + + for (String resourceKey : resourceNames) { + boolean found = false; + + for (RangerResourceDef resourceDef : hierarchy) { + if (resourceDef.getName().equals(resourceKey)) { + found = true; + break; + } + } + + if (!found) { + foundAllResourceKeys = false; + break; + } + } + if (LOG.isDebugEnabled()) { + LOG.debug("<== hierarchyHasAllResources(hierarchy=" + StringUtils.join(hierarchy, ",") + ", resourceNames=" + StringUtils.join(resourceNames, ",") + "): " + foundAllResourceKeys); + } + return foundAllResourceKeys; + } + + public Set getMandatoryResourceNames(List hierarchy) { + Set result = new HashSet(hierarchy.size()); + for (RangerResourceDef resourceDef : hierarchy) { + if (Boolean.TRUE.equals(resourceDef.getMandatory())) { + result.add(resourceDef.getName()); + } + } + return result; + } + + /** + * Set view of a hierarchy's resource names for efficient searching + * @param hierarchy + * @return + */ + public Set getAllResourceNames(List hierarchy) { + Set result = new HashSet(hierarchy.size()); + for (RangerResourceDef resourceDef : hierarchy) { + result.add(resourceDef.getName()); + } + return result; + } + + /** + * Resources names matching the order of list of resource defs passed in. + * @param hierarchy + * @return + */ + public List getAllResourceNamesOrdered(List hierarchy) { + List result = new ArrayList(hierarchy.size()); + for (RangerResourceDef resourceDef : hierarchy) { + result.add(resourceDef.getName()); + } + return result; + } + + public boolean isResourceGraphValid() { + return _delegate.isResourceGraphValid(); + } + + public List getOrderedResourceNames(Collection resourceNames) { + final List ret; + if (resourceNames != null) { + ret = new ArrayList<>(); + for (String orderedName : _delegate.getAllOrderedResourceNames()) { + for (String resourceName : resourceNames) { + if (StringUtils.equals(orderedName, resourceName) && !ret.contains(resourceName)) { + ret.add(resourceName); + break; + } + } + } + } else { + ret = Collections.EMPTY_LIST; + } + return ret; + } + + /** + * Not designed for public access. Package level only for testability. + */ + static class Delegate { + final RangerServiceDef _serviceDef; + final Map>> _hierarchies = new HashMap<>(); + final Date _serviceDefFreshnessDate; + final String _serviceName; + final boolean _checkForCycles; + final boolean _valid; + final List _orderedResourceNames; + final static Set> EMPTY_RESOURCE_HIERARCHY = Collections.unmodifiableSet(new HashSet>()); + + + public Delegate(RangerServiceDef serviceDef, boolean checkForCycles) { + // NOTE: we assume serviceDef, its name and update time are can never by null. + _serviceDef = serviceDef; + _serviceName = serviceDef.getName(); + _serviceDefFreshnessDate = serviceDef.getUpdateTime(); + _checkForCycles = checkForCycles; + + boolean isValid = true; + for(String policyType : RangerPolicy.POLICY_TYPES) { + List resources = getResourceDefs(serviceDef, policyType); + DirectedGraph graph = createGraph(resources); + + if(graph != null) { + Map resourceDefMap = getResourcesAsMap(resources); + if (isValid(graph, resourceDefMap)) { + Set> hierarchies = getHierarchies(graph, resourceDefMap); + _hierarchies.put(policyType, Collections.unmodifiableSet(hierarchies)); + } else { + isValid = false; + _hierarchies.put(policyType, EMPTY_RESOURCE_HIERARCHY); + } + } else { + _hierarchies.put(policyType, EMPTY_RESOURCE_HIERARCHY); + } + } + + if (isValid) { + _orderedResourceNames = buildSortedResourceNames(); + } else { + _orderedResourceNames = new ArrayList<>(); + } + + _valid = isValid; + if (LOG.isDebugEnabled()) { + String message = String.format("Found [%d] resource hierarchies for service [%s] update-date[%s]: %s", _hierarchies.size(), _serviceName, + _serviceDefFreshnessDate == null ? null : _serviceDefFreshnessDate.toString(), _hierarchies); + LOG.debug(message); + } + } + + public void patchServiceDefWithDefaultValues() { + for(String policyType : RangerPolicy.POLICY_TYPES) { + Set> resourceHierarchies = getResourceHierarchies(policyType); + for (List resourceHierarchy : resourceHierarchies) { + for (int index = 0; index < resourceHierarchy.size(); index++) { + RangerResourceDef resourceDef = resourceHierarchy.get(index); + if (!Boolean.TRUE.equals(resourceDef.getIsValidLeaf())) { + resourceDef.setIsValidLeaf(index == resourceHierarchy.size()-1); + } + } + } + } + } + + public Set> getResourceHierarchies(String policyType) { + if(StringUtils.isEmpty(policyType)) { + policyType = RangerPolicy.POLICY_TYPE_ACCESS; + } + + Set> ret = _hierarchies.get(policyType); + + if(ret == null) { + ret = EMPTY_RESOURCE_HIERARCHY; + } + + return ret; + } + + public String getServiceName() { + return _serviceName; + } + + public Date getServiceFreshnessDate() { + return _serviceDefFreshnessDate; + } + + public boolean isResourceGraphValid() { + return _valid; + } + /** + * Builds a directed graph where each resource is node and arc goes from parent level to child level + * + * @param resourceDefs + * @return + */ + DirectedGraph createGraph(List resourceDefs) { + DirectedGraph graph = null; + + if(CollectionUtils.isNotEmpty(resourceDefs)) { + graph = new DirectedGraph(); + + for (RangerResourceDef resourceDef : resourceDefs) { + String name = resourceDef.getName(); + + graph.add(name); + String parent = resourceDef.getParent(); + if (StringUtils.isNotEmpty(parent)) { + graph.addArc(parent, name); + } + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("Created graph for resources: " + graph); + } + return graph; + } + + List getResourceDefs(RangerServiceDef serviceDef, String policyType) { + final List resourceDefs; + + if(StringUtils.isEmpty(policyType) || RangerPolicy.POLICY_TYPE_ACCESS.equals(policyType)) { + resourceDefs = serviceDef.getResources(); + } else if(RangerPolicy.POLICY_TYPE_DATAMASK.equals(policyType)) { + if(serviceDef.getDataMaskDef() != null) { + resourceDefs = serviceDef.getDataMaskDef().getResources(); + } else { + resourceDefs = null; + } + } else if(RangerPolicy.POLICY_TYPE_ROWFILTER.equals(policyType)) { + if(serviceDef.getRowFilterDef() != null) { + resourceDefs = serviceDef.getRowFilterDef().getResources(); + } else { + resourceDefs = null; + } + } else { // unknown policyType; use all resources + resourceDefs = serviceDef.getResources(); + } + + return resourceDefs; + } + /** + * A valid resource graph is a forest, i.e. a disjoint union of trees. In our case, given that node can have only one "parent" node, we can detect this validity simply by ensuring that + * the resource graph has: + * - at least one sink AND + * - and least one source. + * + * A more direct method would have been ensure that the resulting graph does not have any cycles. + * + * @param graph + * + * @return + */ + boolean isValid(DirectedGraph graph, Map resourceDefMap) { + boolean ret = true; + Set sources = graph.getSources(); + Set sinks = graph.getSinks(); + + if (CollectionUtils.isEmpty(sources) || CollectionUtils.isEmpty(sinks)) { + ret = false; + } else { + List cycle = _checkForCycles ? graph.getACycle(sources, sinks) : null; + if (cycle == null) { + for (String sink : sinks) { + RangerResourceDef sinkResourceDef = resourceDefMap.get(sink); + if (Boolean.FALSE.equals(sinkResourceDef.getIsValidLeaf())) { + LOG.error("Error in path: sink node:[" + sink + "] is not leaf node"); + ret = false; + break; + } + } + } else { + LOG.error("Graph contains cycle! - " + cycle); + ret = false; + } + } + + return ret; + } + + /** + * Returns all valid resource hierarchies for the configured resource-defs. Behavior is undefined if it is called on and invalid graph. Use isValid to check validation first. + * + * @param graph + * @param resourceMap + * @return + */ + Set> getHierarchies(DirectedGraph graph, Map resourceMap) { + Set> hierarchies = new HashSet<>(); + Set sources = graph.getSources(); + Set sinks = graph.getSinks(); + + for (String source : sources) { + /* + * A disconnected node, i.e. one that does not have any arc coming into or out of it is a hierarchy in itself! + * A source by definition does not have any arcs coming into it. So if it also doesn't have any neighbors then we know + * it is a disconnected node. + */ + if (!graph.hasNeighbors(source)) { + hierarchies.add(Lists.newArrayList(source)); + } else { + for (String sink : sinks) { + List path = graph.getAPath(source, sink, new HashSet()); + + if (!path.isEmpty()) { + hierarchies.add(path); + + List workingPath = new ArrayList(); + + for (int index = 0, pathSize = path.size(); index < pathSize -1; index++) { + String node = path.get(index); + + workingPath.add(node); + + if (Boolean.TRUE.equals(resourceMap.get(node).getIsValidLeaf())) { + hierarchies.add(new ArrayList(workingPath)); + } + } + } + } + } + } + + return convertHierarchies(hierarchies, resourceMap); + } + + Set> convertHierarchies(Set> hierarchies, Map resourceMap) { + Set> result = new HashSet>(hierarchies.size()); + for (List hierarchy : hierarchies) { + List resourceList = new ArrayList(hierarchy.size()); + for (String name : hierarchy) { + RangerResourceDef def = resourceMap.get(name); + resourceList.add(def); + } + result.add(resourceList); + } + return result; + } + + /** + * Converts resource list to resource map for efficient querying + * + * @param resourceList - is guaranteed to be non-null and non-empty + * @return + */ + Map getResourcesAsMap(List resourceList) { + Map map = new HashMap(resourceList.size()); + for (RangerResourceDef resourceDef : resourceList) { + map.put(resourceDef.getName(), resourceDef); + } + return map; + } + + List getAllOrderedResourceNames() { + return this._orderedResourceNames; + } + + private static class ResourceNameLevel implements Comparable { + private String resourceName; + private int level; + + ResourceNameLevel(String resourceName, int level) { + this.resourceName = resourceName; + this.level = level; + } + + @Override + public int compareTo(ResourceNameLevel other) { + return Integer.compare(this.level, other.level); + } + } + + private List buildSortedResourceNames() { + final List ret = new ArrayList<>(); + + boolean isValid = true; + List resourceNameLevels = new ArrayList<>(); + for (RangerResourceDef resourceDef : _serviceDef.getResources()) { + String name = resourceDef.getName(); + Integer level = resourceDef.getLevel(); + if (name != null && level != null) { + ResourceNameLevel resourceNameLevel = new ResourceNameLevel(name, level); + resourceNameLevels.add(resourceNameLevel); + } else { + LOG.error("Incorrect resourceDef:[name=" + name + "]"); + isValid = false; + break; + } + } + + if (isValid) { + Collections.sort(resourceNameLevels); + + for (ResourceNameLevel resourceNameLevel : resourceNameLevels) { + ret.add(resourceNameLevel.resourceName); + } + } + return ret; + } + } + + /** + * Limited DAG implementation to analyze resource graph for a service. Not designed for public access. Package level only for testability. + */ + static class DirectedGraph { + Map> _nodes = new HashMap<>(); + + /** + * Add a node to the graph + * + * @param node + */ + void add(String node) { + if (node == null) { + throw new IllegalArgumentException("Node can't be null!"); + } else if (!_nodes.containsKey(node)) { // don't mess with a node's neighbors if it already exists in the graph + _nodes.put(node, new HashSet()); + } + } + + /** + * Connects node "from" to node "to". Being a directed graph, after this call "to" will be in the list of neighbor's of "from". While the converse need not be true. + * + * @param from + * @param to + */ + void addArc(String from, String to) { + // connecting two nodes, implicitly adds nodes to the graph if they aren't already in it + if (!_nodes.containsKey(from)) { + add(from); + } + if (!_nodes.containsKey(to)) { + add(to); + } + _nodes.get(from).add(to); + } + + /** + * Returns true if "to" is in the list of neighbors of "from" + * + * @param from + * @param to + * @return + */ + boolean hasArc(String from, String to) { + return _nodes.containsKey(from) && _nodes.containsKey(to) && _nodes.get(from).contains(to); + } + + /** + * Returns true if the node "from" has any neighbor. + * @param from + * @return + */ + boolean hasNeighbors(String from) { + return _nodes.containsKey(from) && !_nodes.get(from).isEmpty(); + } + /** + * Return the set of nodes with in degree of 0, i.e. those that are not in any other nodes' list of neighbors + * + * @return + */ + Set getSources() { + Set sources = new HashSet<>(_nodes.keySet()); + for (Map.Entry> entry : _nodes.entrySet()) { + Set nbrs = entry.getValue(); // can never be null + sources.removeAll(nbrs); // A source in a DAG can't be a neighbor of any other node + } + if (LOG.isDebugEnabled()) { + LOG.debug("Returning sources: " + sources); + } + return sources; + } + + /** + * Returns the set of nodes with out-degree of 0, i.e. those nodes whose list of neighbors is empty + * + * @return + */ + Set getSinks() { + Set sinks = new HashSet<>(); + for (Map.Entry> entry : _nodes.entrySet()) { + Set nbrs = entry.getValue(); + if (nbrs.isEmpty()) { // A sink in a DAG doesn't have any neighbor + String node = entry.getKey(); + sinks.add(node); + } + } + if (LOG.isDebugEnabled()) { + LOG.debug("Returning sinks: " + sinks); + } + return sinks; + } + + List getACycle(Set sources, Set sinks) { + List ret = null; + Set nonSourceOrSinkNodes = Sets.difference(_nodes.keySet(), Sets.union(sources, sinks)); + for (String node : nonSourceOrSinkNodes) { + List seenNodes = new ArrayList<>(); + seenNodes.add(node); + ret = findCycle(node, seenNodes); + if (ret != null) { + break; + } + } + return ret; + } + + /** + * Does a depth first traversal of a graph starting from given node. Returns a sequence of nodes that form first cycle or null if no cycle is found. + * + * @param node Start node + * @param seenNodes List of nodes seen thus far + * @return list of nodes comprising first cycle in graph; null if no cycle was found + */ + List findCycle(String node, List seenNodes) { + List ret = null; + Set nbrs = _nodes.get(node); + for (String nbr : nbrs) { + boolean foundCycle = seenNodes.contains(nbr); + seenNodes.add(nbr); + if (foundCycle) { + ret = seenNodes; + break; + } else { + ret = findCycle(nbr, seenNodes); + if (ret != null) { + break; + } + } + } + return ret; + } + /** + * Attempts to do a depth first traversal of a graph and returns the resulting path. Note that there could be several paths that connect node "from" to node "to". + * + * @param from + * @param to + * @return + */ + List getAPath(String from, String to, Set alreadyVisited) { + List path = new ArrayList(_nodes.size()); + if (_nodes.containsKey(from) && _nodes.containsKey(to)) { // one can never reach non-existent nodes + if (hasArc(from, to)) { + path.add(from); + path.add(to); + } else { + alreadyVisited.add(from); + Set nbrs = _nodes.get(from); + for (String nbr : nbrs) { + if (!alreadyVisited.contains(nbr)) { + List subPath = getAPath(nbr, to, alreadyVisited); + if (!subPath.isEmpty()) { + path.add(from); + path.addAll(subPath); + } + } + } + } + } + return path; + } + + @Override + public boolean equals(Object object) { + if (object == this) { + return true; + } + if (object == null || object.getClass() != this.getClass()) { + return false; + } + DirectedGraph that = (DirectedGraph)object; + return Objects.equals(this._nodes, that._nodes); + } + + @Override + public int hashCode() { + return Objects.hashCode(_nodes); + } + + @Override + public String toString() { + return "_nodes=" + Objects.toString(_nodes); + } + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/validation/RangerServiceDefValidator.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/validation/RangerServiceDefValidator.java new file mode 100644 index 00000000000..8d7a3f4acda --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/validation/RangerServiceDefValidator.java @@ -0,0 +1,835 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.model.validation; + +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.plugin.errors.ValidationErrorCode; +import org.apache.atlas.plugin.model.RangerPolicy; +import org.apache.atlas.plugin.model.RangerServiceDef; +import org.apache.atlas.plugin.model.RangerServiceDef.RangerAccessTypeDef; +import org.apache.atlas.plugin.model.RangerServiceDef.RangerDataMaskTypeDef; +import org.apache.atlas.plugin.model.RangerServiceDef.RangerEnumDef; +import org.apache.atlas.plugin.model.RangerServiceDef.RangerEnumElementDef; +import org.apache.atlas.plugin.model.RangerServiceDef.RangerPolicyConditionDef; +import org.apache.atlas.plugin.model.RangerServiceDef.RangerResourceDef; +import org.apache.atlas.plugin.model.RangerServiceDef.RangerServiceConfigDef; +import org.apache.atlas.plugin.store.ServiceStore; +import org.apache.atlas.plugin.util.ServiceDefUtil; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +public class RangerServiceDefValidator extends RangerValidator { + + private static final Log LOG = LogFactory.getLog(RangerServiceDefValidator.class); + + public RangerServiceDefValidator(ServiceStore store) { + super(store); + } + + public void validate(final RangerServiceDef serviceDef, final Action action) throws Exception { + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("==> RangerServiceDefValidator.validate(%s, %s)", serviceDef, action)); + } + + RangerServiceDef normalizedServiceDef = ServiceDefUtil.normalize(serviceDef); + + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("Normalized Service Definition being validated: (%s, %s)", serviceDef, action)); + } + + List failures = new ArrayList<>(); + boolean valid = isValid(normalizedServiceDef, action, failures); + String message = ""; + try { + if (!valid) { + message = serializeFailures(failures); + throw new Exception(message); + } + } finally { + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("<== RangerServiceDefValidator.validate(%s, %s): %s, reason[%s]", normalizedServiceDef, action, valid, message)); + } + } + } + + boolean isValid(final Long id, final Action action, final List failures) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerServiceDefValidator.isValid(" + id + ")"); + } + + boolean valid = true; + if (action != Action.DELETE) { + ValidationErrorCode error = ValidationErrorCode.SERVICE_DEF_VALIDATION_ERR_UNSUPPORTED_ACTION; + failures.add(new ValidationFailureDetailsBuilder() + .isAnInternalError() + .errorCode(error.getErrorCode()) + .becauseOf(error.getMessage(action)) + .build()); + valid = false; + } else if (id == null) { + ValidationErrorCode error = ValidationErrorCode.SERVICE_DEF_VALIDATION_ERR_MISSING_FIELD; + failures.add(new ValidationFailureDetailsBuilder() + .field("id") + .isMissing() + .errorCode(error.getErrorCode()) + .becauseOf(error.getMessage("id")) + .build()); + valid = false; + } else if (getServiceDef(id) == null) { + if (LOG.isDebugEnabled()) { + LOG.debug("No service found for id[" + id + "]! ok!"); + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerServiceDefValidator.isValid(" + id + "): " + valid); + } + return valid; + } + + boolean isValid(final RangerServiceDef serviceDef, final Action action, final List failures) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerServiceDefValidator.isValid(" + serviceDef + ")"); + } + + if (!(action == Action.CREATE || action == Action.UPDATE)) { + throw new IllegalArgumentException("isValid(RangerServiceDef, ...) is only supported for CREATE/UPDATE"); + } + boolean valid = true; + if (serviceDef == null) { + ValidationErrorCode error = ValidationErrorCode.SERVICE_DEF_VALIDATION_ERR_NULL_SERVICE_DEF_OBJECT; + failures.add(new ValidationFailureDetailsBuilder() + .field("service def") + .isMissing() + .errorCode(error.getErrorCode()) + .becauseOf(error.getMessage(action)) + .build()); + valid = false; + } else { + Long id = serviceDef.getId(); + valid = isValidServiceDefId(id, action, failures) && valid; + valid = isValidServiceDefName(serviceDef.getName(), id, action, failures) && valid; + valid = isValidServiceDefDisplayName(serviceDef.getDisplayName(), id, action, failures) && valid; + valid = isValidAccessTypes(serviceDef.getId(), serviceDef.getAccessTypes(), failures, action) && valid; + if (isValidResources(serviceDef, failures, action)) { + // Semantic check of resource graph can only be done if resources are "syntactically" valid + valid = isValidResourceGraph(serviceDef, failures) && valid; + } else { + valid = false; + } + List enumDefs = serviceDef.getEnums(); + if (isValidEnums(enumDefs, failures)) { + // config def validation requires valid enums + valid = isValidConfigs(serviceDef.getConfigs(), enumDefs, failures) && valid; + } else { + valid = false; + } + valid = isValidPolicyConditions(serviceDef.getId(), serviceDef.getPolicyConditions(), failures, action) && valid; + valid = isValidDataMaskTypes(serviceDef.getId(), serviceDef.getDataMaskDef().getMaskTypes(), failures, action) && valid; + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerServiceDefValidator.isValid(" + serviceDef + "): " + valid); + } + return valid; + } + + boolean isValidServiceDefId(Long id, final Action action, final List failures) { + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("==> RangerServiceDefValidator.isValidServiceDefId(%s, %s, %s)", id, action, failures)); + } + boolean valid = true; + + if (action == Action.UPDATE) { // id is ignored for CREATE + if (id == null) { + ValidationErrorCode error = ValidationErrorCode.SERVICE_DEF_VALIDATION_ERR_EMPTY_SERVICE_DEF_ID; + failures.add(new ValidationFailureDetailsBuilder() + .field("id") + .isMissing() + .errorCode(error.getErrorCode()) + .becauseOf(error.getMessage()) + .build()); + valid = false; + } else if (getServiceDef(id) == null) { + ValidationErrorCode error = ValidationErrorCode.SERVICE_DEF_VALIDATION_ERR_INVALID_SERVICE_DEF_ID; + failures.add(new ValidationFailureDetailsBuilder() + .field("id") + .isSemanticallyIncorrect() + .errorCode(error.getErrorCode()) + .becauseOf(error.getMessage(id)) + .build()); + valid = false; + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("<== RangerServiceDefValidator.isValidServiceDefId(%s, %s, %s): %s", id, action, failures, valid)); + } + return valid; + } + + boolean isValidServiceDefName(String name, Long id, final Action action, final List failures) { + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("==> RangerServiceDefValidator.isValidServiceDefName(%s, %s, %s, %s)", name, id, action, failures)); + } + boolean valid = true; + + if (StringUtils.isBlank(name)) { + ValidationErrorCode error = ValidationErrorCode.SERVICE_DEF_VALIDATION_ERR_INVALID_SERVICE_DEF_NAME; + failures.add(new ValidationFailureDetailsBuilder() + .field("name") + .isMissing() + .errorCode(error.getErrorCode()) + .becauseOf(error.getMessage(name)) + .build()); + valid = false; + } else { + RangerServiceDef otherServiceDef = getServiceDef(name); + if (otherServiceDef != null && action == Action.CREATE) { + ValidationErrorCode error = ValidationErrorCode.SERVICE_DEF_VALIDATION_ERR_SERVICE_DEF_NAME_CONFICT; + failures.add(new ValidationFailureDetailsBuilder() + .field("name") + .isSemanticallyIncorrect() + .errorCode(error.getErrorCode()) + .becauseOf(error.getMessage(name)) + .build()); + valid = false; + } else if (otherServiceDef != null && !Objects.equals(id, otherServiceDef.getId())) { + ValidationErrorCode error = ValidationErrorCode.SERVICE_DEF_VALIDATION_ERR_ID_NAME_CONFLICT; + failures.add(new ValidationFailureDetailsBuilder() + .field("id/name") + .isSemanticallyIncorrect() + .errorCode(error.getErrorCode()) + .becauseOf(error.getMessage(name, otherServiceDef.getId())) + .build()); + valid = false; + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("<== RangerServiceDefValidator.isValidServiceDefName(%s, %s, %s, %s): %s", name, id, action, failures, valid)); + } + return valid; + } + + /** + * Performs all validations related to ServiceDef displayName. + * @param displayName + * @param id + * @param action + * @param failures + * @return + */ + boolean isValidServiceDefDisplayName(final String displayName, Long id, final Action action, final List failures) { + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("==> RangerServiceDefValidator.isValidServiceDefDisplayName(%s, %s, %s, %s)", displayName, id, action, failures)); + } + boolean valid = true; + + if (StringUtils.isBlank(displayName)) { + ValidationErrorCode error = ValidationErrorCode.SERVICE_DEF_VALIDATION_ERR_INVALID_SERVICE_DEF_DISPLAY_NAME; + failures.add(new ValidationFailureDetailsBuilder() + .field("displayName") + .isMissing() + .errorCode(error.getErrorCode()) + .becauseOf(error.getMessage(displayName)) + .build()); + valid = false; + } else { + RangerServiceDef otherServiceDef = getServiceDefByDisplayName(displayName); + if (otherServiceDef != null && action == Action.CREATE) { + ValidationErrorCode error = ValidationErrorCode.SERVICE_DEF_VALIDATION_ERR_SERVICE_DEF__DISPLAY_NAME_CONFICT; + failures.add(new ValidationFailureDetailsBuilder() + .field("displayName") + .isSemanticallyIncorrect() + .errorCode(error.getErrorCode()) + .becauseOf(error.getMessage(displayName, otherServiceDef.getName())) + .build()); + valid = false; + } else if (otherServiceDef != null && !Objects.equals(id, otherServiceDef.getId())) { + ValidationErrorCode error = ValidationErrorCode.SERVICE_DEF_VALIDATION_ERR_SERVICE_DEF__DISPLAY_NAME_CONFICT; + failures.add(new ValidationFailureDetailsBuilder() + .field("id/displayName") + .isSemanticallyIncorrect() + .errorCode(error.getErrorCode()) + .becauseOf(error.getMessage(displayName, otherServiceDef.getName())) + .build()); + valid = false; + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("<== RangerServiceDefValidator.isValidServiceDefName(%s, %s, %s, %s): %s", displayName, id, action, failures, valid)); + } + return valid; + } + + boolean isValidAccessTypes(final Long serviceDefId, final List accessTypeDefs, + final List failures, final Action action) { + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("==> RangerServiceDefValidator.isValidAccessTypes(%s, %s)", accessTypeDefs, failures)); + } + + boolean valid = true; + if (CollectionUtils.isEmpty(accessTypeDefs)) { + ValidationErrorCode error = ValidationErrorCode.SERVICE_DEF_VALIDATION_ERR_MISSING_FIELD; + failures.add(new ValidationFailureDetailsBuilder() + .field("access types") + .isMissing() + .errorCode(error.getErrorCode()) + .becauseOf(error.getMessage("access types")) + .build()); + valid = false; + } else { + Map existingAccessTypeIDNameMap = new HashMap<>(); + if (action == Action.UPDATE) { + List existingAccessTypes = this.getServiceDef(serviceDefId).getAccessTypes(); + for (RangerAccessTypeDef existingAccessType : existingAccessTypes) { + existingAccessTypeIDNameMap.put(existingAccessType.getItemId(), existingAccessType.getName()); + } + } + if(LOG.isDebugEnabled()) { + LOG.debug("accessType names from db = " + existingAccessTypeIDNameMap.values()); + } + List defsWithImpliedGrants = new ArrayList<>(); + Set accessNames = new HashSet<>(); + Set ids = new HashSet<>(); + for (RangerAccessTypeDef def : accessTypeDefs) { + String name = def.getName(); + Long itemId = def.getItemId(); + valid = isUnique(name, accessNames, "access type name", "access types", failures) && valid; + valid = isUnique(def.getItemId(), ids, "access type itemId", "access types", failures) && valid; + if (action == Action.UPDATE) { + if (existingAccessTypeIDNameMap.get(itemId) != null && !existingAccessTypeIDNameMap.get(itemId).equals(name)) { + ValidationErrorCode error; + error = ValidationErrorCode.SERVICE_DEF_VALIDATION_ERR_SERVICE_DEF_NAME_CONFICT; + failures.add((new ValidationFailureDetailsBuilder()).field("access type name").isSemanticallyIncorrect().errorCode(error.getErrorCode()).becauseOf(String.format("changing %s[%s] in %s is not supported", "access type name", name, "access types")).build()); + valid = false; + } + } + if (CollectionUtils.isNotEmpty(def.getImpliedGrants())) { + defsWithImpliedGrants.add(def); + } + } + // validate implied grants + for (RangerAccessTypeDef def : defsWithImpliedGrants) { + Collection impliedGrants = getImpliedGrants(def); + Set unknownAccessTypes = Sets.difference(Sets.newHashSet(impliedGrants), accessNames); + if (!unknownAccessTypes.isEmpty()) { + ValidationErrorCode error = ValidationErrorCode.SERVICE_DEF_VALIDATION_ERR_IMPLIED_GRANT_UNKNOWN_ACCESS_TYPE; + failures.add(new ValidationFailureDetailsBuilder() + .field("implied grants") + .subField(unknownAccessTypes.iterator().next()) // we return just on item here. Message has all unknow items + .isSemanticallyIncorrect() + .errorCode(error.getErrorCode()) + .becauseOf(error.getMessage(impliedGrants, unknownAccessTypes)) + .build()); + valid = false; + } + // implied grant should not imply itself! + String name = def.getName(); // note: this name could be null/blank/empty! + if (impliedGrants.contains(name)) { + ValidationErrorCode error = ValidationErrorCode.SERVICE_DEF_VALIDATION_ERR_IMPLIED_GRANT_IMPLIES_ITSELF; + failures.add(new ValidationFailureDetailsBuilder() + .field("implied grants") + .subField(name) + .isSemanticallyIncorrect() + .errorCode(error.getErrorCode()) + .becauseOf(error.getMessage(impliedGrants, name)) + .build()); + valid = false; + } + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("<== RangerServiceDefValidator.isValidAccessTypes(%s, %s): %s", accessTypeDefs, failures, valid)); + } + return valid; + } + + boolean isValidPolicyConditions(Long serviceDefId, List policyConditions, + List failures, final Action action) { + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("==> RangerServiceDefValidator.isValidPolicyConditions(%s, %s)", policyConditions, failures)); + } + boolean valid = true; + + if (CollectionUtils.isEmpty(policyConditions)) { + LOG.debug("Configs collection was null/empty! ok"); + } else { + Map existingPolicyCondIDNameMap = new HashMap<>(); + if (action == Action.UPDATE) { + List existingPolicyConditions = this.getServiceDef(serviceDefId).getPolicyConditions(); + for (RangerPolicyConditionDef existingPolicyCondition : existingPolicyConditions) { + existingPolicyCondIDNameMap.put(existingPolicyCondition.getItemId(), existingPolicyCondition.getName()); + } + } + if(LOG.isDebugEnabled()) { + LOG.debug("policy condition names from db = " + existingPolicyCondIDNameMap.values()); + } + Set ids = new HashSet<>(); + Set names = new HashSet<>(); + for (RangerPolicyConditionDef conditionDef : policyConditions) { + Long itemId = conditionDef.getItemId(); + valid = isUnique(itemId, ids, "policy condition def itemId", "policy condition defs", failures) && valid; + String name = conditionDef.getName(); + valid = isUnique(name, names, "policy condition def name", "policy condition defs", failures) && valid; + if (action == Action.UPDATE) { + if (existingPolicyCondIDNameMap.get(itemId) != null && !existingPolicyCondIDNameMap.get(itemId).equals(name)) { + ValidationErrorCode error; + error = ValidationErrorCode.SERVICE_DEF_VALIDATION_ERR_SERVICE_DEF_NAME_CONFICT; + failures.add((new ValidationFailureDetailsBuilder()).field("policy condition def name").isSemanticallyIncorrect().errorCode(error.getErrorCode()).becauseOf(String.format("changing %s[%s] in %s is not supported", "policy condition def name", name, "policy condition defs")).build()); + valid = false; + } + } + if (StringUtils.isBlank(conditionDef.getEvaluator())) { + ValidationErrorCode error = ValidationErrorCode.SERVICE_DEF_VALIDATION_ERR_POLICY_CONDITION_NULL_EVALUATOR; + failures.add(new ValidationFailureDetailsBuilder() + .field("policy condition def evaluator") + .subField(name) + .isMissing() + .errorCode(error.getErrorCode()) + .becauseOf(error.getMessage(name)) + .build()); + valid = false; + } + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("<== RangerServiceDefValidator.isValidPolicyConditions(%s, %s): %s", policyConditions, failures, valid)); + } + return valid; + } + + boolean isValidConfigs(List configs, List enumDefs, List failures) { + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("==> RangerServiceDefValidator.isValidConfigs(%s, %s, %s)", configs, enumDefs, failures)); + } + boolean valid = true; + + if (CollectionUtils.isEmpty(configs)) { + LOG.debug("Configs collection was null/empty! ok"); + } else { + Set ids = new HashSet(configs.size()); + Set names = new HashSet(configs.size()); + for (RangerServiceConfigDef aConfig : configs) { + valid = isUnique(aConfig.getItemId(), ids, "config def itemId", "config defs", failures) && valid; + String configName = aConfig.getName(); + valid = isUnique(configName, names, "config def name", "config defs", failures) && valid; + String type = aConfig.getType(); + valid = isValidConfigType(type, configName, failures) && valid; + if ("enum".equals(type)) { + valid = isValidConfigOfEnumType(aConfig, enumDefs, failures) && valid; + } + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("<== RangerServiceDefValidator.isValidConfigs(%s, %s, %s): %s", configs, enumDefs, failures, valid)); + } + return valid; + } + + boolean isValidConfigOfEnumType(RangerServiceConfigDef configDef, List enumDefs, List failures) { + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("==> RangerServiceDefValidator.isValidConfigOfEnumType(%s, %s, %s)", configDef, enumDefs, failures)); + } + boolean valid = true; + + if (!"enum".equals(configDef.getType())) { + LOG.debug("ConfigDef wasn't of enum type!"); + } else { + Map enumDefsMap = getEnumDefMap(enumDefs); + Set enumTypes = enumDefsMap.keySet(); + String subType = configDef.getSubType(); + String configName = configDef.getName(); + + if (!enumTypes.contains(subType)) { + ValidationErrorCode error = ValidationErrorCode.SERVICE_DEF_VALIDATION_ERR_CONFIG_DEF_UNKNOWN_ENUM; + failures.add(new ValidationFailureDetailsBuilder() + .field("config def subtype") + .subField(configName) + .isSemanticallyIncorrect() + .errorCode(error.getErrorCode()) + .becauseOf(error.getMessage(subType, configName, enumTypes)) + .build()); + valid = false; + } else { + // default value check is possible only if sub-type is correctly configured + String defaultValue = configDef.getDefaultValue(); + if (StringUtils.isNotBlank(defaultValue)) { + RangerEnumDef enumDef = enumDefsMap.get(subType); + Set enumValues = getEnumValues(enumDef); + if (!enumValues.contains(defaultValue)) { + ValidationErrorCode error = ValidationErrorCode.SERVICE_DEF_VALIDATION_ERR_CONFIG_DEF_UNKNOWN_ENUM_VALUE; + failures.add(new ValidationFailureDetailsBuilder() + .field("config def default value") + .subField(configName) + .isSemanticallyIncorrect() + .errorCode(error.getErrorCode()) + .becauseOf(error.getMessage(defaultValue, configName, enumValues, subType)) + .build()); + valid = false; + } + } + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("<== RangerServiceDefValidator.isValidConfigOfEnumType(%s, %s, %s): %s", configDef, enumDefs, failures, valid)); + } + return valid; + } + + boolean isValidConfigType(String type, String configName, List failures) { + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("==> RangerServiceDefValidator.isValidConfigType(%s, %s, %s)", type, configName, failures)); + } + boolean valid = true; + + Set validTypes = ImmutableSet.of("bool", "enum", "int", "string", "password", "path"); + if (StringUtils.isBlank(type)) { + ValidationErrorCode error = ValidationErrorCode.SERVICE_DEF_VALIDATION_ERR_CONFIG_DEF_MISSING_TYPE; + failures.add(new ValidationFailureDetailsBuilder() + .field("config def type") + .subField(configName) + .isMissing() + .errorCode(error.getErrorCode()) + .becauseOf(error.getMessage(configName)) + .build()); + valid = false; + } else if (!validTypes.contains(type)) { + ValidationErrorCode error = ValidationErrorCode.SERVICE_DEF_VALIDATION_ERR_CONFIG_DEF_INVALID_TYPE; + failures.add(new ValidationFailureDetailsBuilder() + .field("config def type") + .subField(configName) + .isSemanticallyIncorrect() + .errorCode(error.getErrorCode()) + .becauseOf(error.getMessage(type, configName, validTypes)) + .build()); + valid = false; + } + + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("<== RangerServiceDefValidator.isValidConfigType(%s, %s, %s): %s", type, configName, failures, valid)); + } + return valid; + } + + public boolean isValidResources(RangerServiceDef serviceDef, List failures, final Action action) { + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("==> RangerServiceDefValidator.isValidResources(%s, %s)", serviceDef, failures)); + } + boolean valid = true; + + List resources = serviceDef.getResources(); + if (CollectionUtils.isEmpty(resources)) { + ValidationErrorCode error = ValidationErrorCode.SERVICE_DEF_VALIDATION_ERR_MISSING_FIELD; + failures.add(new ValidationFailureDetailsBuilder() + .field("resources") + .isMissing() + .errorCode(error.getErrorCode()) + .becauseOf(error.getMessage("resources")) + .build()); + valid = false; + } else { + Map existingResourceIDNameMap = new HashMap<>(); + if (action == Action.UPDATE) { + List existingResources = this.getServiceDef(serviceDef.getId()).getResources(); + for (RangerResourceDef existingResource : existingResources) { + existingResourceIDNameMap.put(existingResource.getItemId(), existingResource.getName()); + } + } + if(LOG.isDebugEnabled()) { + LOG.debug("resource names from db = " + existingResourceIDNameMap.values()); + } + + Set names = new HashSet(resources.size()); + Set ids = new HashSet(resources.size()); + for (RangerResourceDef resource : resources) { + valid = isValidResourceName(resource.getName(), "resource type name", failures) && valid; + + /* + * While id is the natural key, name is a surrogate key. At several places code expects resource name to be unique within a service. + */ + String name = resource.getName(); + Long itemId = resource.getItemId(); + valid = isUnique(name, names, "resource name", "resources", failures) && valid; + valid = isUnique(itemId, ids, "resource itemId", "resources", failures) && valid; + if (action == Action.UPDATE) { + if (existingResourceIDNameMap.get(itemId) != null && !existingResourceIDNameMap.get(itemId).equals(name)) { + ValidationErrorCode error; + error = ValidationErrorCode.SERVICE_DEF_VALIDATION_ERR_SERVICE_DEF_NAME_CONFICT; + failures.add((new ValidationFailureDetailsBuilder()).field("resource name").isSemanticallyIncorrect().errorCode(error.getErrorCode()).becauseOf(String.format("changing %s[%s] in %s is not supported", "resource name", name, "resources")).build()); + valid = false; + } + } + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("<== RangerServiceDefValidator.isValidResources(%s, %s): %s", serviceDef, failures, valid)); + } + return valid; + } + + boolean isValidResourceGraph(RangerServiceDef serviceDef, List failures) { + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("==> RangerServiceDefValidator.isValidResourceGraph(%s, %s)", serviceDef, failures)); + } + boolean valid = true; + // We don't want this helper to get into the cache or to use what is in the cache!! + RangerServiceDefHelper defHelper = _factory.createServiceDefHelper(serviceDef, false); + if (!defHelper.isResourceGraphValid()) { + ValidationErrorCode error = ValidationErrorCode.SERVICE_DEF_VALIDATION_ERR_RESOURCE_GRAPH_INVALID; + failures.add(new ValidationFailureDetailsBuilder() + .field("resource graph") + .isSemanticallyIncorrect() + .errorCode(error.getErrorCode()) + .becauseOf(error.getMessage()) + .build()); + valid = false; + } + // resource level should be unique within a hierarchy + for(String policyType : RangerPolicy.POLICY_TYPES) { + Set> hierarchies = defHelper.getResourceHierarchies(policyType); + for (List aHierarchy : hierarchies) { + Set levels = new HashSet(aHierarchy.size()); + for (RangerResourceDef resourceDef : aHierarchy) { + valid = isUnique(resourceDef.getLevel(), levels, "resource level", "resources", failures) && valid; + } + + // Ensure that aHierarchy contains resource-defs with increasing level values + int lastResourceLevel = Integer.MIN_VALUE; + for (RangerResourceDef resourceDef : aHierarchy) { + Integer resourceDefLevel = resourceDef.getLevel(); + if (resourceDefLevel == null || resourceDefLevel < lastResourceLevel) { + ValidationErrorCode error = ValidationErrorCode.SERVICE_DEF_VALIDATION_ERR_INVALID_SERVICE_RESOURCE_LEVELS; + failures.add(new ValidationFailureDetailsBuilder() + .field("resource level") + .subField(String.valueOf(resourceDefLevel)) + .isSemanticallyIncorrect() + .errorCode(error.getErrorCode()) + .becauseOf(error.getMessage()) + .build()); + valid = false; + break; + } else { + lastResourceLevel = resourceDef.getLevel(); + } + } + } + } + // If a resource is not mandatory, then it cannot be non-leaf in any hierarchy (RANGER-2207) + List resources = serviceDef.getResources(); + List resourceNames = new ArrayList<>(resources.size()); + for (RangerResourceDef resourceDef : resources) { + resourceNames.add(resourceDef.getName()); + } + for (String resourceName : resourceNames) { + for (String policyType : RangerPolicy.POLICY_TYPES) { + Set> hierarchies = defHelper.getResourceHierarchies(policyType); + for (List aHierarchy : hierarchies) { + boolean foundOptionalResource = false; + for (RangerResourceDef resourceDef : aHierarchy) { + if (!foundOptionalResource) { + if (resourceDef.getName().equalsIgnoreCase(resourceName) && !Boolean.TRUE.equals(resourceDef.getMandatory())) { + foundOptionalResource = true; + } + } else { + if (Boolean.TRUE.equals(resourceDef.getMandatory())) { + valid = false; + ValidationErrorCode error = ValidationErrorCode.SERVICE_DEF_VALIDATION_ERR_INVALID_MANADORY_VALUE_FOR_SERVICE_RESOURCE; + failures.add(new ValidationFailureDetailsBuilder() + .field(resourceDef.getName()) + .isSemanticallyIncorrect() + .errorCode(error.getErrorCode()) + .becauseOf(error.getMessage(resourceDef.getName(), resourceName)) + .build()); + } + } + } + } + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("<== RangerServiceDefValidator.isValidResourceGraph(%s, %s): %s", serviceDef, failures, valid)); + } + return valid; + } + + boolean isValidEnums(List enumDefs, List failures) { + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("==> RangerServiceDefValidator.isValidEnums(%s, %s)", enumDefs, failures)); + } + + boolean valid = true; + if (CollectionUtils.isEmpty(enumDefs)) { + LOG.debug("enum def collection passed in was null/empty. Ok."); + } else { + Set names = new HashSet<>(); + Set ids = new HashSet<>(); + for (RangerEnumDef enumDef : enumDefs) { + if (enumDef == null) { + ValidationErrorCode error = ValidationErrorCode.SERVICE_DEF_VALIDATION_ERR_ENUM_DEF_NULL_OBJECT; + failures.add(new ValidationFailureDetailsBuilder() + .field("enum def") + .isMissing() + .errorCode(error.getErrorCode()) + .becauseOf(error.getMessage()) + .build()); + valid = false; + } else { + // enum-names and ids must non-blank and be unique to a service definition + String enumName = enumDef.getName(); + valid = isUnique(enumName, names, "enum def name", "enum defs", failures) && valid; + valid = isUnique(enumDef.getItemId(), ids, "enum def itemId", "enum defs", failures) && valid; + // enum must contain at least one valid value and those values should be non-blank and distinct + if (CollectionUtils.isEmpty(enumDef.getElements())) { + ValidationErrorCode error = ValidationErrorCode.SERVICE_DEF_VALIDATION_ERR_ENUM_DEF_NO_VALUES; + failures.add(new ValidationFailureDetailsBuilder() + .field("enum values") + .subField(enumName) + .isMissing() + .errorCode(error.getErrorCode()) + .becauseOf(error.getMessage(enumName)) + .build()); + valid = false; + } else { + valid = isValidEnumElements(enumDef.getElements(), failures, enumName) && valid; + // default index should be valid + int defaultIndex = getEnumDefaultIndex(enumDef); + if (defaultIndex < 0 || defaultIndex >= enumDef.getElements().size()) { // max index is one less than the size of the elements list + ValidationErrorCode error = ValidationErrorCode.SERVICE_DEF_VALIDATION_ERR_ENUM_DEF_INVALID_DEFAULT_INDEX; + failures.add(new ValidationFailureDetailsBuilder() + .field("enum default index") + .subField(enumName) + .isSemanticallyIncorrect() + .errorCode(error.getErrorCode()) + .becauseOf(error.getMessage(defaultIndex, enumName)) + .build()); + valid = false; + } + } + } + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("<== RangerServiceDefValidator.isValidEnums(%s, %s): %s", enumDefs, failures, valid)); + } + return valid; + } + + boolean isValidEnumElements(List enumElementsDefs, List failures, String enumName) { + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("==> RangerServiceDefValidator.isValidEnumElements(%s, %s)", enumElementsDefs, failures)); + } + + boolean valid = true; + if (CollectionUtils.isEmpty(enumElementsDefs)) { + LOG.debug("Enum elements list passed in was null/empty!"); + } else { + // enum element names should be valid and distinct + Set elementNames = new HashSet<>(); + Set ids = new HashSet<>(); + for (RangerEnumElementDef elementDef : enumElementsDefs) { + if (elementDef == null) { + ValidationErrorCode error = ValidationErrorCode.SERVICE_DEF_VALIDATION_ERR_ENUM_DEF_NULL_ENUM_ELEMENT; + failures.add(new ValidationFailureDetailsBuilder() + .field("enum element") + .subField(enumName) + .isMissing() + .errorCode(error.getErrorCode()) + .becauseOf(error.getMessage(enumName)) + .build()); + valid = false; + } else { + valid = isUnique(elementDef.getName(), enumName, elementNames, "enum element name", "enum elements", failures) && valid; + valid = isUnique(elementDef.getItemId(), enumName, ids, "enum element itemId", "enum elements", failures) && valid; + } + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("<== RangerServiceDefValidator.isValidEnumElements(%s, %s): %s", enumElementsDefs, failures, valid)); + } + return valid; + } + + boolean isValidDataMaskTypes(Long serviceDefId, List dataMaskTypes, List failures, final Action action) { + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("==> RangerServiceDefValidator.isValidDataMaskTypes(%s, %s)", dataMaskTypes, failures)); + } + boolean valid = true; + + if (CollectionUtils.isEmpty(dataMaskTypes)) { + LOG.debug("Configs collection was null/empty! ok"); + } else { + Map existingDataMaskTypeIDNameMap = new HashMap<>(); + if (action == Action.UPDATE) { + List existingDataMaskTypes = this.getServiceDef(serviceDefId).getDataMaskDef().getMaskTypes(); + for (RangerDataMaskTypeDef existingDataMaskType : existingDataMaskTypes) { + existingDataMaskTypeIDNameMap.put(existingDataMaskType.getItemId(), existingDataMaskType.getName()); + } + } + if(LOG.isDebugEnabled()) { + LOG.debug("data mask type names from db = " + existingDataMaskTypeIDNameMap.values()); + } + + Set ids = new HashSet(); + Set names = new HashSet(); + for (RangerDataMaskTypeDef dataMaskType : dataMaskTypes) { + String name = dataMaskType.getName(); + Long itemId = dataMaskType.getItemId(); + valid = isUnique(itemId, ids, "data mask type def itemId", "data mask type defs", failures) && valid; + valid = isUnique(name, names, "data mask type def name", "data mask type defs", failures) && valid; + if (action == Action.UPDATE) { + if (existingDataMaskTypeIDNameMap.get(itemId) != null && !existingDataMaskTypeIDNameMap.get(itemId).equals(name)) { + ValidationErrorCode error; + error = ValidationErrorCode.SERVICE_DEF_VALIDATION_ERR_SERVICE_DEF_NAME_CONFICT; + failures.add((new ValidationFailureDetailsBuilder()).field("data mask type def name").isSemanticallyIncorrect().errorCode(error.getErrorCode()).becauseOf(String.format("changing %s[%s] in %s is not supported", "data mask type def name", name, "data mask type defs")).build()); + valid = false; + } + } + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("<== RangerServiceDefValidator.isValidDataMaskTypes(%s, %s): %s", dataMaskTypes, failures, valid)); + } + return valid; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/validation/RangerServiceValidator.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/validation/RangerServiceValidator.java new file mode 100644 index 00000000000..5a1e88d54ae --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/validation/RangerServiceValidator.java @@ -0,0 +1,318 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.model.validation; + +import com.google.common.collect.Sets; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.plugin.errors.ValidationErrorCode; +import org.apache.atlas.plugin.model.RangerService; +import org.apache.atlas.plugin.model.RangerServiceDef; +import org.apache.atlas.plugin.store.EmbeddedServiceDefsUtil; +import org.apache.atlas.plugin.store.ServiceStore; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.regex.Pattern; + +public class RangerServiceValidator extends RangerValidator { + private static final Log LOG = LogFactory.getLog(RangerServiceValidator.class); + + private static final Pattern SERVICE_NAME_VALIDATION_REGEX = Pattern.compile("^[a-zA-Z0-9_-][a-zA-Z0-9_-]{0,254}", Pattern.CASE_INSENSITIVE); + private static final Pattern LEGACY_SERVICE_NAME_VALIDATION_REGEX = Pattern.compile("^[a-zA-Z0-9_-][a-zA-Z0-9\\s_-]{0,254}", Pattern.CASE_INSENSITIVE); + private static final Pattern SERVICE_DISPLAY_NAME_VALIDATION_REGEX = Pattern.compile("^[a-zA-Z0-9_-][a-zA-Z0-9\\s_-]{0,254}", Pattern.CASE_INSENSITIVE); + + public RangerServiceValidator(ServiceStore store) { + super(store); + } + + public void validate(RangerService service, Action action) throws Exception { + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("==> RangerServiceValidator.validate(%s, %s)", service, action)); + } + List failures = new ArrayList<>(); + boolean valid = isValid(service, action, failures); + String message = ""; + try { + if (!valid) { + message = serializeFailures(failures); + throw new Exception(message); + } + } finally { + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("<== RangerServiceValidator.validate(%s, %s): %s, reason[%s]", service, action, valid, message)); + } + } + } + + boolean isValid(Long id, Action action, List failures) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerServiceValidator.isValid(" + id + ")"); + } + + boolean valid = true; + if (action != Action.DELETE) { + ValidationErrorCode error = ValidationErrorCode.SERVICE_VALIDATION_ERR_UNSUPPORTED_ACTION; + failures.add(new ValidationFailureDetailsBuilder() + .isAnInternalError() + .errorCode(error.getErrorCode()) + .becauseOf(error.getMessage(action)) + .build()); + valid = false; + } else if (id == null) { + ValidationErrorCode error = ValidationErrorCode.SERVICE_VALIDATION_ERR_MISSING_FIELD; + failures.add(new ValidationFailureDetailsBuilder() + .field("id") + .isMissing() + .errorCode(error.getErrorCode()) + .becauseOf(error.getMessage(id)) + .build()); + valid = false; + } else if (getService(id) == null) { + if (LOG.isDebugEnabled()) { + LOG.debug("No service found for id[" + id + "]! ok!"); + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerServiceValidator.isValid(" + id + "): " + valid); + } + return valid; + } + + boolean isValid(RangerService service, Action action, List failures) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerServiceValidator.isValid(" + service + ")"); + } + if (!(action == Action.CREATE || action == Action.UPDATE)) { + throw new IllegalArgumentException("isValid(RangerService, ...) is only supported for CREATE/UPDATE"); + } + boolean valid = true; + if (service == null) { + ValidationErrorCode error = ValidationErrorCode.SERVICE_VALIDATION_ERR_NULL_SERVICE_OBJECT; + failures.add(new ValidationFailureDetailsBuilder() + .field("service") + .isMissing() + .errorCode(error.getErrorCode()) + .becauseOf(error.getMessage()) + .build()); + valid = false; + } else { + Long id = service.getId(); + if (action == Action.UPDATE) { // id is ignored for CREATE + if (id == null) { + ValidationErrorCode error = ValidationErrorCode.SERVICE_VALIDATION_ERR_EMPTY_SERVICE_ID; + failures.add(new ValidationFailureDetailsBuilder() + .field("id") + .isMissing() + .errorCode(error.getErrorCode()) + .becauseOf(error.getMessage()) + .build()); + valid = false; + } else if (getService(id) == null) { + ValidationErrorCode error = ValidationErrorCode.SERVICE_VALIDATION_ERR_INVALID_SERVICE_ID; + failures.add(new ValidationFailureDetailsBuilder() + .field("id") + .isSemanticallyIncorrect() + .errorCode(error.getErrorCode()) + .becauseOf(error.getMessage(id)) + .build()); + valid = false; + } + } + String name = service.getName(); + boolean nameSpecified = StringUtils.isNotBlank(name); + RangerServiceDef serviceDef = null; + if (!nameSpecified) { + ValidationErrorCode error = ValidationErrorCode.SERVICE_VALIDATION_ERR_INVALID_SERVICE_NAME; + failures.add(new ValidationFailureDetailsBuilder() + .field("name") + .isMissing() + .errorCode(error.getErrorCode()) + .becauseOf(error.getMessage(name)) + .build()); + valid = false; + } else { + Pattern serviceNameRegex = SERVICE_NAME_VALIDATION_REGEX; + if (action == Action.UPDATE) { + RangerService rangerService = getService(service.getId()); + if (rangerService != null && StringUtils.isNotBlank(rangerService.getName()) && rangerService.getName().contains(" ")) { + //RANGER-2808 Support for space in services created with space in earlier version + serviceNameRegex = LEGACY_SERVICE_NAME_VALIDATION_REGEX; + } + } + + if(!isValidString(serviceNameRegex, name)){ + ValidationErrorCode error = ValidationErrorCode.SERVICE_VALIDATION_ERR_SPECIAL_CHARACTERS_SERVICE_NAME; + failures.add(new ValidationFailureDetailsBuilder() + .field("name") + .isSemanticallyIncorrect() + .errorCode(error.getErrorCode()) + .becauseOf(error.getMessage(name)) + .build()); + valid = false; + }else{ + RangerService otherService = getService(name); + if (otherService != null && action == Action.CREATE) { + ValidationErrorCode error = ValidationErrorCode.SERVICE_VALIDATION_ERR_SERVICE_NAME_CONFICT; + failures.add(new ValidationFailureDetailsBuilder() + .field("name") + .isSemanticallyIncorrect() + .errorCode(error.getErrorCode()) + .becauseOf(error.getMessage(name)) + .build()); + valid = false; + } else if (otherService != null && otherService.getId() !=null && !otherService.getId().equals(id)) { + ValidationErrorCode error = ValidationErrorCode.SERVICE_VALIDATION_ERR_ID_NAME_CONFLICT; + failures.add(new ValidationFailureDetailsBuilder() + .field("id/name") + .isSemanticallyIncorrect() + .errorCode(error.getErrorCode()) + .becauseOf(error.getMessage(name, otherService.getId())) + .build()); + valid = false; + } + } + } + // Display name + String displayName = service.getDisplayName(); + if(!isValidString(SERVICE_DISPLAY_NAME_VALIDATION_REGEX, displayName)){ + ValidationErrorCode error = ValidationErrorCode.SERVICE_VALIDATION_ERR_SPECIAL_CHARACTERS_SERVICE_DISPLAY_NAME; + failures.add(new ValidationFailureDetailsBuilder() + .field("displayName") + .isSemanticallyIncorrect() + .errorCode(error.getErrorCode()) + .becauseOf(error.getMessage(displayName)) + .build()); + valid = false; + }else{ + RangerService otherService = getServiceByDisplayName(displayName); + if (otherService != null && action == Action.CREATE) { + ValidationErrorCode error = ValidationErrorCode.SERVICE_VALIDATION_ERR_SERVICE_DISPLAY_NAME_CONFICT; + failures.add(new ValidationFailureDetailsBuilder() + .field("displayName") + .isSemanticallyIncorrect() + .errorCode(error.getErrorCode()) + .becauseOf(error.getMessage(displayName, otherService.getName())) + .build()); + valid = false; + } else if (otherService != null && otherService.getId() !=null && !otherService.getId().equals(id)) { + ValidationErrorCode error = ValidationErrorCode.SERVICE_VALIDATION_ERR_SERVICE_DISPLAY_NAME_CONFICT; + failures.add(new ValidationFailureDetailsBuilder() + .field("id/displayName") + .isSemanticallyIncorrect() + .errorCode(error.getErrorCode()) + .becauseOf(error.getMessage(displayName, otherService.getName())) + .build()); + valid = false; + } + } + String type = service.getType(); + boolean typeSpecified = StringUtils.isNotBlank(type); + if (!typeSpecified) { + ValidationErrorCode error = ValidationErrorCode.SERVICE_VALIDATION_ERR_MISSING_SERVICE_DEF; + failures.add(new ValidationFailureDetailsBuilder() + .field("type") + .isMissing() + .errorCode(error.getErrorCode()) + .becauseOf(error.getMessage(type)) + .build()); + valid = false; + } else { + serviceDef = getServiceDef(type); + if (serviceDef == null) { + ValidationErrorCode error = ValidationErrorCode.SERVICE_VALIDATION_ERR_INVALID_SERVICE_DEF; + failures.add(new ValidationFailureDetailsBuilder() + .field("type") + .isSemanticallyIncorrect() + .errorCode(error.getErrorCode()) + .becauseOf(error.getMessage(type)) + .build()); + valid = false; + } + } + if (nameSpecified && serviceDef != null) { + // check if required parameters were specified + Set reqiredParameters = getRequiredParameters(serviceDef); + Set inputParameters = getServiceConfigParameters(service); + Set missingParameters = Sets.difference(reqiredParameters, inputParameters); + if (!missingParameters.isEmpty()) { + ValidationErrorCode error = ValidationErrorCode.SERVICE_VALIDATION_ERR_REQUIRED_PARM_MISSING; + failures.add(new ValidationFailureDetailsBuilder() + .field("configuration") + .subField(missingParameters.iterator().next()) // we return any one parameter! + .isMissing() + .errorCode(error.getErrorCode()) + .becauseOf(error.getMessage(missingParameters)) + .build()); + valid = false; + } + } + String tagServiceName = service.getTagService(); + + if (StringUtils.isNotBlank(tagServiceName) && StringUtils.equals(type, EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_TAG_NAME)) { + failures.add(new ValidationFailureDetailsBuilder() + .field("tag_service") + .isSemanticallyIncorrect() + .becauseOf("tag service cannot be part of any other service") + .build()); + valid = false; + } + + boolean needToEnsureServiceType = false; + if (action == Action.UPDATE) { + RangerService otherService = getService(name); + String otherTagServiceName = otherService == null ? null : otherService.getTagService(); + + if (StringUtils.isNotBlank(tagServiceName)) { + if (!StringUtils.equals(tagServiceName, otherTagServiceName)) { + needToEnsureServiceType = true; + } + } + } else { // action == Action.CREATE + if (StringUtils.isNotBlank(tagServiceName)) { + needToEnsureServiceType = true; + } + } + if (needToEnsureServiceType) { + RangerService maybeTagService = getService(tagServiceName); + if (maybeTagService == null || !StringUtils.equals(maybeTagService.getType(), EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_TAG_NAME)) { + failures.add(new ValidationFailureDetailsBuilder() + .field("tag_service") + .isSemanticallyIncorrect() + .becauseOf("tag service name does not refer to existing tag service:" + tagServiceName) + .build()); + valid = false; + } + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerServiceValidator.isValid(" + service + "): " + valid); + } + return valid; + } + + public boolean isValidString(final Pattern pattern, final String name) { + return pattern != null && StringUtils.isNotBlank(name) && pattern.matcher(name).matches(); + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/validation/RangerValidator.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/validation/RangerValidator.java new file mode 100644 index 00000000000..b19af0cddd3 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/validation/RangerValidator.java @@ -0,0 +1,866 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.model.validation; + + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.plugin.errors.ValidationErrorCode; +import org.apache.atlas.plugin.model.RangerPolicy; +import org.apache.atlas.plugin.model.RangerPolicy.RangerPolicyResource; +import org.apache.atlas.plugin.model.RangerRole; +import org.apache.atlas.plugin.model.RangerSecurityZone; +import org.apache.atlas.plugin.model.RangerService; +import org.apache.atlas.plugin.model.RangerServiceDef; +import org.apache.atlas.plugin.model.RangerServiceDef.RangerAccessTypeDef; +import org.apache.atlas.plugin.model.RangerServiceDef.RangerEnumDef; +import org.apache.atlas.plugin.model.RangerServiceDef.RangerEnumElementDef; +import org.apache.atlas.plugin.model.RangerServiceDef.RangerResourceDef; +import org.apache.atlas.plugin.model.RangerServiceDef.RangerServiceConfigDef; +import org.apache.atlas.plugin.store.RoleStore; +import org.apache.atlas.plugin.store.ServiceStore; +import org.apache.atlas.plugin.util.RangerObjectFactory; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public abstract class RangerValidator { + + private static final Log LOG = LogFactory.getLog(RangerValidator.class); + + RoleStore _roleStore; + ServiceStore _store; + RangerObjectFactory _factory = new RangerObjectFactory(); + + public enum Action { + CREATE, UPDATE, DELETE; + }; + + protected RangerValidator(ServiceStore store) { + if (store == null) { + throw new IllegalArgumentException("ServiceValidator(): store is null!"); + } + _store = store; + } + + protected RangerValidator(RoleStore roleStore) { + if (roleStore == null) { + throw new IllegalArgumentException("ServiceValidator(): store is null!"); + } + _roleStore = roleStore; + } + + public void validate(Long id, Action action) throws Exception { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerValidator.validate(" + id + ")"); + } + + List failures = new ArrayList<>(); + if (isValid(id, action, failures)) { + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerValidator.validate(" + id + "): valid"); + } + } else { + String message = serializeFailures(failures); + LOG.debug("<== RangerValidator.validate(" + id + "): invalid, reason[" + message + "]"); + throw new Exception(message); + } + } + public void validate(String name, Action action) throws Exception { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerValidator.validate(" + name + ")"); + } + + List failures = new ArrayList<>(); + if (isValid(name, action, failures)) { + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerValidator.validate(" + name + "): valid"); + } + } else { + String message = serializeFailures(failures); + LOG.debug("<== RangerValidator.validate(" + name + "): invalid, reason[" + message + "]"); + throw new Exception(message); + } + } + + /** + * This method is expected to be overridden by sub-classes. Default implementation provided to not burden implementers from having to implement methods that they know would never be called. + * @param id + * @param action + * @param failures + * @return + */ + boolean isValid(Long id, Action action, List failures) { + failures.add(new ValidationFailureDetailsBuilder() + .isAnInternalError() + .becauseOf("unimplemented method called") + .build()); + return false; + } + + boolean isValid(String name, Action action, List failures) { + failures.add(new ValidationFailureDetailsBuilder() + .isAnInternalError() + .becauseOf("unimplemented method called") + .build()); + return false; + } + + public static String serializeFailures(List failures) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerValidator.getFailureMessage()"); + } + + String message = null; + if (CollectionUtils.isEmpty(failures)) { + LOG.warn("serializeFailures: called while list of failures is null/empty!"); + } else { + StringBuilder builder = new StringBuilder(); + for (int i = 0; i < failures.size(); i++) { + builder.append(String.format("(%d)", i)); + builder.append(failures.get(i)); + builder.append(" "); + } + message = builder.toString(); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerValidator.serializeFailures(): " + message); + } + return message; + } + + Set getServiceConfigParameters(RangerService service) { + if (service == null || service.getConfigs() == null) { + return new HashSet<>(); + } else { + return service.getConfigs().keySet(); + } + } + + Set getRequiredParameters(RangerServiceDef serviceDef) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerValidator.getRequiredParameters(" + serviceDef + ")"); + } + + Set result; + if (serviceDef == null) { + result = Collections.emptySet(); + } else { + List configs = serviceDef.getConfigs(); + if (CollectionUtils.isEmpty(configs)) { + result = Collections.emptySet(); + } else { + result = new HashSet(configs.size()); // at worse all of the config items are required! + for (RangerServiceConfigDef configDef : configs) { + if (configDef.getMandatory()) { + result.add(configDef.getName()); + } + } + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerValidator.getRequiredParameters(" + serviceDef + "): " + result); + } + return result; + } + + RangerServiceDef getServiceDef(Long id) { + + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerValidator.getServiceDef(" + id + ")"); + } + RangerServiceDef result = null; + try { + result = _store.getServiceDef(id); + } catch (Exception e) { + LOG.debug("Encountred exception while retrieving service def from service store!", e); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerValidator.getServiceDef(" + id + "): " + result); + } + return result; + } + + RangerServiceDef getServiceDef(String type) { + + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerValidator.getServiceDef(" + type + ")"); + } + RangerServiceDef result = null; + try { + result = _store.getServiceDefByName(type); + } catch (Exception e) { + LOG.debug("Encountred exception while retrieving service definition from service store!", e); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerValidator.getServiceDef(" + type + "): " + result); + } + return result; + } + + /** + * @param displayName + * @return {@link RangerServiceDef} - service using display name if present, null otherwise. + */ + RangerServiceDef getServiceDefByDisplayName(final String displayName) { + + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerValidator.getServiceDefByDisplayName(" + displayName + ")"); + } + RangerServiceDef result = null; + try { + result = _store.getServiceDefByDisplayName(displayName); + } catch (Exception e) { + LOG.debug("Encountered exception while retrieving service definition from service store!", e); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerValidator.getServiceDefByDisplayName(" + displayName + "): " + result); + } + return result; + } + + RangerService getService(Long id) { + + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerValidator.getService(" + id + ")"); + } + RangerService result = null; + try { + result = _store.getService(id); + } catch (Exception e) { + LOG.debug("Encountred exception while retrieving service from service store!", e); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerValidator.getService(" + id + "): " + result); + } + return result; + } + + RangerService getService(String name) { + + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerValidator.getService(" + name + ")"); + } + RangerService result = null; + try { + result = _store.getServiceByName(name); + } catch (Exception e) { + LOG.debug("Encountred exception while retrieving service from service store!", e); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerValidator.getService(" + name + "): " + result); + } + return result; + } + + RangerService getServiceByDisplayName(final String displayName) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerValidator.getService(" + displayName + ")"); + } + RangerService result = null; + try { + result = _store.getServiceByDisplayName(displayName); + } catch (Exception e) { + LOG.debug("Encountred exception while retrieving service from service store!", e); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerValidator.getService(" + displayName + "): " + result); + } + return result; + } + + boolean policyExists(Long id) { + try { + return _store.policyExists(id); + } catch (Exception e) { + LOG.debug("Encountred exception while retrieving policy from service store!", e); + return false; + } + } + + RangerPolicy getPolicy(Long id) { + + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerValidator.getPolicy(" + id + ")"); + } + RangerPolicy result = null; + try { + result = _store.getPolicy(id); + } catch (Exception e) { + LOG.debug("Encountred exception while retrieving policy from service store!", e); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerValidator.getPolicy(" + id + "): " + result); + } + return result; + } + + Long getPolicyId(final Long serviceId, final String policyName, final Long zoneId) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerValidator.getPolicyId(" + serviceId + ", " + policyName + ", " + zoneId + ")"); + } + + Long policyId = _store.getPolicyId(serviceId, policyName, zoneId); + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerValidator.getPolicyId(" + serviceId + ", " + policyName + ", " + zoneId + "): policy-id[" + policyId + "]"); + } + return policyId; + } + + List getPoliciesForResourceSignature(String serviceName, String policySignature) { + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("==> RangerValidator.getPoliciesForResourceSignature(%s, %s)", serviceName, policySignature)); + } + + List policies = null; + try { + policies = _store.getPoliciesByResourceSignature(serviceName, policySignature, true); // only look for enabled policies + } catch (Exception e) { + LOG.debug("Encountred exception while retrieving policies from service store!", e); + } + + if(LOG.isDebugEnabled()) { + int count = policies == null ? 0 : policies.size(); + LOG.debug(String.format("<== RangerValidator.getPoliciesForResourceSignature(%s, %s): count[%d], %s", serviceName, policySignature, count, policies)); + } + return policies; + } + + RangerSecurityZone getSecurityZone(Long id) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerValidator.getSecurityZone(" + id + ")"); + } + RangerSecurityZone result = null; + + if (id != null) { + try { + result = _store.getSecurityZone(id); + } catch (Exception e) { + LOG.debug("Encountred exception while retrieving security zone from service store!", e); + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerValidator.getSecurityZone(" + id + "): " + result); + } + return result; + } + + RangerSecurityZone getSecurityZone(String name) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerValidator.getSecurityZone(" + name + ")"); + } + RangerSecurityZone result = null; + + if (StringUtils.isNotEmpty(name)) { + try { + result = _store.getSecurityZone(name); + } catch (Exception e) { + LOG.debug("Encountred exception while retrieving security zone from service store!", e); + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerValidator.getSecurityZone(" + name + "): " + result); + } + return result; + } + Set getAccessTypes(RangerServiceDef serviceDef) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerValidator.getAccessTypes(" + serviceDef + ")"); + } + + Set accessTypes = new HashSet<>(); + if (serviceDef == null) { + LOG.warn("serviceDef passed in was null!"); + } else if (CollectionUtils.isEmpty(serviceDef.getAccessTypes())) { + LOG.warn("AccessTypeDef collection on serviceDef was null!"); + } else { + for (RangerAccessTypeDef accessTypeDef : serviceDef.getAccessTypes()) { + if (accessTypeDef == null) { + LOG.warn("Access type def was null!"); + } else { + String accessType = accessTypeDef.getName(); + if (StringUtils.isBlank(accessType)) { + LOG.warn("Access type def name was null/empty/blank!"); + } else { + accessTypes.add(accessType); + } + } + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerValidator.getAccessTypes(" + serviceDef + "): " + accessTypes); + } + return accessTypes; + } + + /** + * This function exists to encapsulates the current behavior of code which treats and unspecified audit preference to mean audit is enabled. + * @param policy + * @return + */ + boolean getIsAuditEnabled(RangerPolicy policy) { + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerValidator.getIsAuditEnabled(" + policy + ")"); + } + + boolean isEnabled = false; + if (policy == null) { + LOG.warn("policy was null!"); + } else if (policy.getIsAuditEnabled() == null) { + isEnabled = true; + } else { + isEnabled = policy.getIsAuditEnabled(); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerValidator.getIsAuditEnabled(" + policy + "): " + isEnabled); + } + return isEnabled; + } + + /** + * Returns names of resource types set to lower-case to allow for case-insensitive comparison. + * @param serviceDef + * @return + */ + Set getMandatoryResourceNames(RangerServiceDef serviceDef) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerValidator.getMandatoryResourceNames(" + serviceDef + ")"); + } + + Set resourceNames = new HashSet<>(); + if (serviceDef == null) { + LOG.warn("serviceDef passed in was null!"); + } else if (CollectionUtils.isEmpty(serviceDef.getResources())) { + LOG.warn("ResourceDef collection on serviceDef was null!"); + } else { + for (RangerResourceDef resourceTypeDef : serviceDef.getResources()) { + if (resourceTypeDef == null) { + LOG.warn("resource type def was null!"); + } else { + Boolean mandatory = resourceTypeDef.getMandatory(); + if (mandatory != null && mandatory == true) { + String resourceName = resourceTypeDef.getName(); + if (StringUtils.isBlank(resourceName)) { + LOG.warn("Resource def name was null/empty/blank!"); + } else { + resourceNames.add(resourceName.toLowerCase()); + } + } + } + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerValidator.getMandatoryResourceNames(" + serviceDef + "): " + resourceNames); + } + return resourceNames; + } + + Set getAllResourceNames(RangerServiceDef serviceDef) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerValidator.getAllResourceNames(" + serviceDef + ")"); + } + + Set resourceNames = new HashSet<>(); + if (serviceDef == null) { + LOG.warn("serviceDef passed in was null!"); + } else if (CollectionUtils.isEmpty(serviceDef.getResources())) { + LOG.warn("ResourceDef collection on serviceDef was null!"); + } else { + for (RangerResourceDef resourceTypeDef : serviceDef.getResources()) { + if (resourceTypeDef == null) { + LOG.warn("resource type def was null!"); + } else { + String resourceName = resourceTypeDef.getName(); + if (StringUtils.isBlank(resourceName)) { + LOG.warn("Resource def name was null/empty/blank!"); + } else { + resourceNames.add(resourceName.toLowerCase()); + } + } + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerValidator.getAllResourceNames(" + serviceDef + "): " + resourceNames); + } + return resourceNames; + } + + /** + * Converts, in place, the resources defined in the policy to have lower-case resource-def-names + * @param policy + * @return + */ + + void convertPolicyResourceNamesToLower(RangerPolicy policy) { + Map lowerCasePolicyResources = new HashMap<>(); + if (policy.getResources() != null) { + for (Map.Entry entry : policy.getResources().entrySet()) { + String lowerCasekey = entry.getKey().toLowerCase(); + lowerCasePolicyResources.put(lowerCasekey, entry.getValue()); + } + } + policy.setResources(lowerCasePolicyResources); + } + + Map getValidationRegExes(RangerServiceDef serviceDef) { + if (serviceDef == null || CollectionUtils.isEmpty(serviceDef.getResources())) { + return new HashMap<>(); + } else { + Map result = new HashMap<>(); + for (RangerResourceDef resourceDef : serviceDef.getResources()) { + if (resourceDef == null) { + LOG.warn("A resource def in resource def collection is null"); + } else { + String name = resourceDef.getName(); + String regEx = resourceDef.getValidationRegEx(); + if (StringUtils.isBlank(name)) { + LOG.warn("resource name is null/empty/blank"); + } else if (StringUtils.isBlank(regEx)) { + LOG.debug("validation regex is null/empty/blank"); + } else { + result.put(name, regEx); + } + } + } + return result; + } + } + + int getEnumDefaultIndex(RangerEnumDef enumDef) { + int index; + if (enumDef == null) { + index = -1; + } else if (enumDef.getDefaultIndex() == null) { + index = 0; + } else { + index = enumDef.getDefaultIndex(); + } + return index; + } + + Collection getImpliedGrants(RangerAccessTypeDef def) { + if (def == null) { + return null; + } else if (CollectionUtils.isEmpty(def.getImpliedGrants())) { + return new ArrayList<>(); + } else { + List result = new ArrayList(def.getImpliedGrants().size()); + for (String name : def.getImpliedGrants()) { + if (StringUtils.isBlank(name)) { + result.add(name); // could be null! + } else { + result.add(name.toLowerCase()); + } + } + return result; + } + } + + /** + * Returns a copy of the policy resource map where all keys (resource-names) are lowercase + * @param input + * @return + */ + Map getPolicyResourceWithLowerCaseKeys(Map input) { + if (input == null) { + return null; + } + Map output = new HashMap(input.size()); + for (Map.Entry entry : input.entrySet()) { + output.put(entry.getKey().toLowerCase(), entry.getValue()); + } + return output; + } + + boolean isUnique(Long value, Set alreadySeen, String valueName, String collectionName, List failures) { + return isUnique(value, null, alreadySeen, valueName, collectionName, failures); + } + + /** + * NOTE: alreadySeen collection passed in gets updated. + * @param value + * @param alreadySeen + * @param valueName - use-friendly name of the value that would be used when generating failure message + * @param collectionName - use-friendly name of the value collection that would be used when generating failure message + * @param failures + * @return + */ + boolean isUnique(Long value, String valueContext, Set alreadySeen, String valueName, String collectionName, List failures) { + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("==> RangerServiceDefValidator.isValueUnique(%s, %s, %s, %s, %s)", value, alreadySeen, valueName, collectionName, failures)); + } + boolean valid = true; + + if (value == null) { // null/empty/blank value is an error + failures.add(new ValidationFailureDetailsBuilder() + .field(valueName) + .subField(valueContext) + .isMissing() + .becauseOf(String.format("%s[%s] is null/empty", valueName, value)) + .build()); + valid = false; + } else if (alreadySeen.contains(value)) { // it shouldn't have been seen already + failures.add(new ValidationFailureDetailsBuilder() + .field(valueName) + .subField(value.toString()) + .isSemanticallyIncorrect() + .becauseOf(String.format("duplicate %s[%s] in %s", valueName, value, collectionName)) + .build()); + valid = false; + } else { + alreadySeen.add(value); // we have a new unique access type + } + + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("==> RangerServiceDefValidator.isValueUnique(%s, %s, %s, %s, %s): %s", value, alreadySeen, valueName, collectionName, failures, valid)); + } + return valid; + } + + /** + * NOTE: alreadySeen collection passed in gets updated. + * @param value + * @param alreadySeen + * @param valueName - use-friendly name of the value that would be used when generating failure message + * @param collectionName - use-friendly name of the value collection that would be used when generating failure message + * @param failures + * @return + */ + boolean isUnique(Integer value, Set alreadySeen, String valueName, String collectionName, List failures) { + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("==> RangerServiceDefValidator.isValueUnique(%s, %s, %s, %s, %s)", value, alreadySeen, valueName, collectionName, failures)); + } + boolean valid = true; + + if (value == null) { // null/empty/blank value is an error + failures.add(new ValidationFailureDetailsBuilder() + .field(valueName) + .isMissing() + .becauseOf(String.format("%s[%s] is null/empty", valueName, value)) + .build()); + valid = false; + } else if (alreadySeen.contains(value)) { // it shouldn't have been seen already + failures.add(new ValidationFailureDetailsBuilder() + .field(valueName) + .subField(value.toString()) + .isSemanticallyIncorrect() + .becauseOf(String.format("duplicate %s[%s] in %s", valueName, value, collectionName)) + .build()); + valid = false; + } else { + alreadySeen.add(value); // we have a new unique access type + } + + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("==> RangerServiceDefValidator.isValueUnique(%s, %s, %s, %s, %s): %s", value, alreadySeen, valueName, collectionName, failures, valid)); + } + return valid; + } + + /* + * Important: Resource-names are required to be lowercase. This is used in validating policy create/update operations. + * Ref: RANGER-2272 + */ + boolean isValidResourceName(final String value, final String valueContext, final List failures) { + boolean ret = true; + + if (value != null && !StringUtils.isEmpty(value)) { + int sz = value.length(); + + for(int i = 0; i < sz; ++i) { + char c = value.charAt(i); + if (!(Character.isLowerCase(c) || c == '-' || c == '_')) { // Allow only lowercase, hyphen or underscore characters + ret = false; + break; + } + } + } else { + ret = false; + } + if (!ret) { + ValidationErrorCode errorCode = ValidationErrorCode.SERVICE_DEF_VALIDATION_ERR_NOT_LOWERCASE_NAME; + failures.add(new ValidationFailureDetailsBuilder() + .errorCode(errorCode.getErrorCode()) + .field(value) + .becauseOf(errorCode.getMessage(valueContext, value)) + .build()); + } + return ret; + } + + boolean isUnique(final String value, final Set alreadySeen, final String valueName, final String collectionName, final List failures) { + return isUnique(value, null, alreadySeen, valueName, collectionName, failures); + } + /** + * NOTE: alreadySeen collection passed in gets updated. + * @param value + * @param alreadySeen + * @param valueName - use-friendly name of the value that would be used when generating failure message + * @param collectionName - use-friendly name of the value collection that would be used when generating failure message + * @param failures + * @return + */ + boolean isUnique(final String value, final String valueContext, final Set alreadySeen, final String valueName, final String collectionName, final List failures) { + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("==> RangerServiceDefValidator.isValueUnique(%s, %s, %s, %s, %s)", value, alreadySeen, valueName, collectionName, failures)); + } + boolean valid = true; + + if (StringUtils.isBlank(value)) { // null/empty/blank value is an error + failures.add(new ValidationFailureDetailsBuilder() + .field(valueName) + .subField(valueContext) + .isMissing() + .becauseOf(String.format("%s[%s] is null/empty", valueName, value)) + .build()); + valid = false; + } else if (alreadySeen.contains(value.toLowerCase())) { // it shouldn't have been seen already + failures.add(new ValidationFailureDetailsBuilder() + .field(valueName) + .subField(value) + .isSemanticallyIncorrect() + .becauseOf(String.format("duplicate %s[%s] in %s", valueName, value, collectionName)) + .build()); + valid = false; + } else { + alreadySeen.add(value.toLowerCase()); // we have a new unique access type + } + + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("==> RangerServiceDefValidator.isValueUnique(%s, %s, %s, %s, %s): %s", value, alreadySeen, valueName, collectionName, failures, valid)); + } + return valid; + } + + Map getEnumDefMap(List enumDefs) { + Map result = new HashMap<>(); + if (enumDefs != null) { + for (RangerEnumDef enumDef : enumDefs) { + result.put(enumDef.getName(), enumDef); + } + } + return result; + } + + Set getEnumValues(RangerEnumDef enumDef) { + Set result = new HashSet<>(); + if (enumDef != null) { + for (RangerEnumElementDef element : enumDef.getElements()) { + result.add(element.getName()); + } + } + return result; + } + + static Map createMap(int[][] data) { + Map result = new HashMap<>(); + if (data != null) { + for (int[] row : data) { + Integer key = row[0]; + Integer value = row[1]; + if (result.containsKey(key)) { + LOG.warn("createMap: Internal error: duplicate key: multiple rows found for [" + key + "]. Skipped"); + } else { + result.put(key, value); + } + } + } + return result; + } + + static Map createMap(Object[][] data) { + Map result = new HashMap<>(); + if (data != null) { + for (Object[] row : data) { + Integer key = (Integer)row[0]; + String value = (String)row[1]; + if (key == null) { + LOG.warn("createMap: error converting key[" + row[0] + "] to Integer! Sipped!"); + } else if (StringUtils.isEmpty(value)) { + LOG.warn("createMap: empty/null value. Skipped!"); + } else if (result.containsKey(key)) { + LOG.warn("createMap: Internal error: duplicate key. Multiple rows found for [" + key + "]. Skipped"); + } else { + result.put(key, value); + } + } + } + return result; + } + + RangerRole getRangerRole(Long id) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerValidator.getRangerRole(" + id + ")"); + } + RangerRole result = null; + try { + result = _roleStore.getRole(id); + } catch (Exception e) { + LOG.debug("Encountred exception while retrieving RangerRole from RoleStore store!", e); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerValidator.getRangerRole(" + id + "): " + result); + } + return result; + } + + boolean roleExists(Long id) { + try { + return _roleStore.roleExists(id); + } catch (Exception e) { + LOG.debug("Encountred exception while retrieving RangerRole from role store!", e); + return false; + } + } + + boolean roleExists(String name) { + try { + return _roleStore.roleExists(name); + } catch (Exception e) { + LOG.debug("Encountred exception while retrieving RangerRole from role store!", e); + return false; + } + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/validation/RangerValidityScheduleValidator.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/validation/RangerValidityScheduleValidator.java new file mode 100644 index 00000000000..c2f7adc1677 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/validation/RangerValidityScheduleValidator.java @@ -0,0 +1,482 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.model.validation; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.plugin.model.RangerValidityRecurrence; +import org.apache.atlas.plugin.model.RangerValidityRecurrence.RecurrenceSchedule; +import org.apache.atlas.plugin.model.RangerValiditySchedule; + +import javax.annotation.Nonnull; +import java.text.DateFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.Date; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.TimeZone; + +public class RangerValidityScheduleValidator { + private static final Log LOG = LogFactory.getLog(RangerValidityScheduleValidator.class); + + private static final ThreadLocal DATE_FORMATTER = new ThreadLocal() { + @Override + protected DateFormat initialValue() { + return new SimpleDateFormat(RangerValiditySchedule.VALIDITY_SCHEDULE_DATE_STRING_SPECIFICATION); + } + }; + + private static final Set validTimeZoneIds = new HashSet<>(Arrays.asList(TimeZone.getAvailableIDs())); + + private final RangerValiditySchedule validitySchedule; + private Date startTime; + private Date endTime; + private RecurrenceSchedule validityPeriodEstimator; + private RangerValiditySchedule normalizedValiditySchedule; + + public RangerValidityScheduleValidator(@Nonnull RangerValiditySchedule validitySchedule) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerValidityScheduleValidator:: " + validitySchedule); + } + + this.validitySchedule = validitySchedule; + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerValidityScheduleValidator:: " + validitySchedule); + } + } + + public RangerValiditySchedule validate(List validationFailures) { + RangerValiditySchedule ret = null; + + if (StringUtils.isEmpty(validitySchedule.getStartTime()) && StringUtils.isEmpty(validitySchedule.getEndTime()) && CollectionUtils.isEmpty(validitySchedule.getRecurrences())) { + validationFailures.add(new ValidationFailureDetails(0, "startTime,endTime,recurrences", "", true, true, false, "empty values")); + } else { + + if (StringUtils.isNotEmpty(validitySchedule.getStartTime())) { + try { + startTime = DATE_FORMATTER.get().parse(validitySchedule.getStartTime()); + } catch (ParseException exception) { + LOG.error("Error parsing startTime:[" + validitySchedule.getStartTime() + "]", exception); + validationFailures.add(new ValidationFailureDetails(0, "startTime", "", true, true, false, "invalid value")); + } + } else { + startTime = new Date(); + } + + if (StringUtils.isNotEmpty(validitySchedule.getEndTime())) { + try { + endTime = DATE_FORMATTER.get().parse(validitySchedule.getEndTime()); + } catch (ParseException exception) { + LOG.error("Error parsing endTime:[" + validitySchedule.getEndTime() + "]", exception); + validationFailures.add(new ValidationFailureDetails(0, "endTime", "", true, true, false, "invalid value")); + } + } else { + endTime = new Date(Long.MAX_VALUE); + } + + if (startTime != null && endTime != null) { + validityPeriodEstimator = new RangerValidityRecurrence.RecurrenceSchedule(); + normalizedValiditySchedule = new RangerValiditySchedule(); + + boolean isValid = validateTimeRangeSpec(validationFailures); + + if (isValid) { + if (LOG.isDebugEnabled()) { + LOG.debug("validityPeriodEstimator:[" + validityPeriodEstimator + "]"); + } + + normalizedValiditySchedule.setStartTime(validitySchedule.getStartTime()); + normalizedValiditySchedule.setEndTime(validitySchedule.getEndTime()); + normalizedValiditySchedule.setTimeZone(validitySchedule.getTimeZone()); + + ret = normalizedValiditySchedule; + } else { + normalizedValiditySchedule = null; + } + } + } + + return ret; + } + + private boolean validateTimeRangeSpec(List validationFailures) { + boolean ret; + if (startTime.getTime() >= endTime.getTime()) { + validationFailures.add(new ValidationFailureDetails(0, "startTime", "", false, true, false, "endTime is not later than startTime")); + ret = false; + } else { + ret = true; + } + ret = validateTimeZone(validitySchedule.getTimeZone(), validationFailures) && ret; + + for (RangerValidityRecurrence recurrence : validitySchedule.getRecurrences()) { + ret = validateValidityInterval(recurrence, validationFailures) && ret; + + if (ret) { + ret = validateFieldSpec(recurrence, RangerValidityRecurrence.RecurrenceSchedule.ScheduleFieldSpec.minute, validationFailures) && ret; + ret = validateFieldSpec(recurrence, RangerValidityRecurrence.RecurrenceSchedule.ScheduleFieldSpec.hour, validationFailures) && ret; + ret = validateFieldSpec(recurrence, RangerValidityRecurrence.RecurrenceSchedule.ScheduleFieldSpec.dayOfMonth, validationFailures) && ret; + ret = validateFieldSpec(recurrence, RangerValidityRecurrence.RecurrenceSchedule.ScheduleFieldSpec.dayOfWeek, validationFailures) && ret; + ret = validateFieldSpec(recurrence, RangerValidityRecurrence.RecurrenceSchedule.ScheduleFieldSpec.month, validationFailures) && ret; + ret = validateFieldSpec(recurrence, RangerValidityRecurrence.RecurrenceSchedule.ScheduleFieldSpec.year, validationFailures) && ret; + ret = ret && validateIntervalDuration(recurrence, validationFailures); + + if (ret) { + RangerValidityRecurrence.RecurrenceSchedule schedule = new RangerValidityRecurrence.RecurrenceSchedule(getNormalizedValue(recurrence, RangerValidityRecurrence.RecurrenceSchedule.ScheduleFieldSpec.minute), getNormalizedValue(recurrence, RangerValidityRecurrence.RecurrenceSchedule.ScheduleFieldSpec.hour), + getNormalizedValue(recurrence, RangerValidityRecurrence.RecurrenceSchedule.ScheduleFieldSpec.dayOfMonth), getNormalizedValue(recurrence, RangerValidityRecurrence.RecurrenceSchedule.ScheduleFieldSpec.dayOfWeek), + getNormalizedValue(recurrence, RangerValidityRecurrence.RecurrenceSchedule.ScheduleFieldSpec.month), getNormalizedValue(recurrence, RangerValidityRecurrence.RecurrenceSchedule.ScheduleFieldSpec.year)); + RangerValidityRecurrence normalizedRecurrence = new RangerValidityRecurrence(schedule, recurrence.getInterval()); + normalizedValiditySchedule.getRecurrences().add(normalizedRecurrence); + } + } + } + + return ret; + } + + private boolean validateTimeZone(String timeZone, List validationFailures) { + validTimeZoneIds.removeAll(Arrays.asList("JST", "IST", "BET", "ACT", "AET", "AGT", "VST", "SystemV/AST4", "CNT", + "NET", "SystemV/MST7", "PLT", "CST", "SST", "SystemV/CST6", "CTT", "PNT", "BST", "SystemV/YST9", "MIT", + "ART", "AST", "PRT", "SystemV/HST10", "PST", "SystemV/EST5", "IET", "SystemV/PST8", "SystemV/CST6CDT", + "NST", "EAT", "ECT", "SystemV/MST7MDT", "SystemV/YST9YDT", "CAT", "SystemV/PST8PDT", "SystemV/AST4ADT", + "SystemV/EST5EDT")); + boolean ret = !StringUtils.isNotBlank(timeZone) || validTimeZoneIds.contains(timeZone); + if (!ret) { + validationFailures.add(new ValidationFailureDetails(0, "timeZone", "", false, true, false, "invalid timeZone")); + } + return ret; + } + + private boolean validateValidityInterval(RangerValidityRecurrence recurrence, List validationFailures) { + boolean ret = recurrence.getInterval() != null && recurrence.getSchedule() != null; + + if (ret) { + RangerValidityRecurrence.ValidityInterval validityInterval = recurrence.getInterval(); + + if (validityInterval.getDays() < 0 + || (validityInterval.getHours() < 0 || validityInterval.getHours() > 23) + || (validityInterval.getMinutes() < 0 || validityInterval.getMinutes() > 59)) { + validationFailures.add(new ValidationFailureDetails(0, "interval", "", false, true, false, "invalid interval")); + ret = false; + } + + if (StringUtils.isBlank(recurrence.getSchedule().getDayOfMonth()) && StringUtils.isBlank(recurrence.getSchedule().getDayOfWeek())) { + validationFailures.add(new ValidationFailureDetails(0, "validitySchedule", "", false, true, false, "empty dayOfMonth and dayOfWeek")); + ret = false; + } + } else { + validationFailures.add(new ValidationFailureDetails(0, "recurrence", "schedule/interval", true, true, false, "empty schedule/interval in recurrence spec")); + } + return ret; + } + + private boolean validateFieldSpec(RangerValidityRecurrence recurrence, RangerValidityRecurrence.RecurrenceSchedule.ScheduleFieldSpec field, List validationFailures) { + boolean ret = true; + + String fieldValue = recurrence.getSchedule().getFieldValue(field); + if (StringUtils.isBlank(fieldValue)) { + if (LOG.isDebugEnabled()) { + LOG.debug("No value provided for [" + field + "]"); + } + if (StringUtils.equals(field.name(), RangerValidityRecurrence.RecurrenceSchedule.ScheduleFieldSpec.dayOfWeek.name()) + || StringUtils.equals(field.name(), RangerValidityRecurrence.RecurrenceSchedule.ScheduleFieldSpec.dayOfMonth.name())) { + if (LOG.isDebugEnabled()) { + LOG.debug("Allow blank value for dayOfWeek or dayOfMonth here. Check for both being null is done elsewhere."); + } + } else { + validationFailures.add(new ValidationFailureDetails(0, field.toString(), "", false, true, false, "No value provided")); + } + } + ret = validateCharacters(fieldValue, field.specialChars); + + if (!ret) { + validationFailures.add(new ValidationFailureDetails(0, field.toString(), "", false, true, false, "invalid character(s)")); + } else { + // Valid month values in java.util.Date are from 1-12, in java.util.Calendar from 0 to 11 + // Internally we use Calendar values for validation and evaluation + int minimum = field == RangerValidityRecurrence.RecurrenceSchedule.ScheduleFieldSpec.month ? field.minimum + 1 : field.minimum; + int maximum = field == RangerValidityRecurrence.RecurrenceSchedule.ScheduleFieldSpec.month ? field.maximum + 1 : field.maximum; + ret = validateRanges(recurrence, field, minimum, maximum, validationFailures); + } + return ret; + } + + private boolean validateCharacters(String str, String permittedCharacters) { + boolean ret = true; + if (StringUtils.isNotBlank(str)) { + char[] chars = str.toCharArray(); + for (char c : chars) { + if (!(Character.isDigit(c) || Character.isWhitespace(c) || StringUtils.contains(permittedCharacters, c))) { + ret = false; + break; + } + } + } + return ret; + } + + private boolean validateIntervalDuration(RangerValidityRecurrence recurrence, List validationFailures) { + boolean ret = true; + + if (!validationFailures.isEmpty() || validityPeriodEstimator == null) { + ret = false; + } else { + int minSchedulingInterval = 1; // In minutes + + String minutes = validityPeriodEstimator.getFieldValue(RangerValidityRecurrence.RecurrenceSchedule.ScheduleFieldSpec.minute); + if (!StringUtils.equals(minutes, RangerValidityRecurrence.RecurrenceSchedule.WILDCARD)) { + minSchedulingInterval = StringUtils.isBlank(minutes) ? RangerValidityRecurrence.RecurrenceSchedule.ScheduleFieldSpec.minute.maximum + 1 : Integer.valueOf(minutes); + + if (minSchedulingInterval == RangerValidityRecurrence.RecurrenceSchedule.ScheduleFieldSpec.minute.maximum + 1) { + String hours = validityPeriodEstimator.getFieldValue(RangerValidityRecurrence.RecurrenceSchedule.ScheduleFieldSpec.hour); + if (!StringUtils.equals(hours, RangerValidityRecurrence.RecurrenceSchedule.WILDCARD)) { + int hour = StringUtils.isBlank(hours) ? RangerValidityRecurrence.RecurrenceSchedule.ScheduleFieldSpec.hour.maximum + 1 :Integer.valueOf(hours); + minSchedulingInterval = hour * (RangerValidityRecurrence.RecurrenceSchedule.ScheduleFieldSpec.minute.maximum+1); + + if (hour == RangerValidityRecurrence.RecurrenceSchedule.ScheduleFieldSpec.hour.maximum + 1) { + String dayOfMonths = validityPeriodEstimator.getFieldValue(RangerValidityRecurrence.RecurrenceSchedule.ScheduleFieldSpec.dayOfMonth); + String dayOfWeeks = validityPeriodEstimator.getFieldValue(RangerValidityRecurrence.RecurrenceSchedule.ScheduleFieldSpec.dayOfWeek); + + int dayOfMonth = 1, dayOfWeek = 1; + if (!StringUtils.equals(dayOfMonths, RangerValidityRecurrence.RecurrenceSchedule.WILDCARD)) { + dayOfMonth = StringUtils.isBlank(dayOfMonths) ? RangerValidityRecurrence.RecurrenceSchedule.ScheduleFieldSpec.dayOfMonth.maximum + 1 : Integer.valueOf(dayOfMonths); + } + if (!StringUtils.equals(dayOfWeeks, RangerValidityRecurrence.RecurrenceSchedule.WILDCARD)) { + dayOfWeek = StringUtils.isBlank(dayOfWeeks) ? RangerValidityRecurrence.RecurrenceSchedule.ScheduleFieldSpec.dayOfWeek.maximum + 1 : Integer.valueOf(dayOfWeeks); + } + if (!StringUtils.equals(dayOfMonths, RangerValidityRecurrence.RecurrenceSchedule.WILDCARD) || !StringUtils.equals(dayOfWeeks, RangerValidityRecurrence.RecurrenceSchedule.WILDCARD)) { + int minDays = dayOfMonth > dayOfWeek ? dayOfWeek : dayOfMonth; + minSchedulingInterval = minDays*(RangerValidityRecurrence.RecurrenceSchedule.ScheduleFieldSpec.hour.maximum+1)*(RangerValidityRecurrence.RecurrenceSchedule.ScheduleFieldSpec.minute.maximum+1); + + if (dayOfMonth == (RangerValidityRecurrence.RecurrenceSchedule.ScheduleFieldSpec.dayOfMonth.maximum+1) && dayOfWeek == (RangerValidityRecurrence.RecurrenceSchedule.ScheduleFieldSpec.dayOfWeek.maximum+1)) { + String months = validityPeriodEstimator.getFieldValue(RangerValidityRecurrence.RecurrenceSchedule.ScheduleFieldSpec.month); + if (!StringUtils.equals(months, RangerValidityRecurrence.RecurrenceSchedule.WILDCARD)) { + int month = StringUtils.isBlank(months) ? RangerValidityRecurrence.RecurrenceSchedule.ScheduleFieldSpec.month.maximum + 1 :Integer.valueOf(months); + minSchedulingInterval = month * 28 * (RangerValidityRecurrence.RecurrenceSchedule.ScheduleFieldSpec.hour.maximum + 1) * (RangerValidityRecurrence.RecurrenceSchedule.ScheduleFieldSpec.minute.maximum + 1); + + if (month == RangerValidityRecurrence.RecurrenceSchedule.ScheduleFieldSpec.month.maximum + 1) { + // Maximum interval is 1 year + minSchedulingInterval = 365 * (RangerValidityRecurrence.RecurrenceSchedule.ScheduleFieldSpec.hour.maximum + 1) * (RangerValidityRecurrence.RecurrenceSchedule.ScheduleFieldSpec.minute.maximum + 1); + } + } + } + } + } + } + } + } + if (RangerValidityRecurrence.ValidityInterval.getValidityIntervalInMinutes(recurrence.getInterval()) > minSchedulingInterval) { + if (LOG.isDebugEnabled()) { + LOG.warn("Specified scheduling interval:" + RangerValidityRecurrence.ValidityInterval.getValidityIntervalInMinutes(recurrence.getInterval()) + " minutes] is more than minimum possible scheduling interval:[" + minSchedulingInterval + " minutes]."); + LOG.warn("This may turn this (expected to be temporary) policy into effectively permanent policy."); + } + } + } + return ret; + } + + private boolean validateRanges(RangerValidityRecurrence recurrence, RangerValidityRecurrence.RecurrenceSchedule.ScheduleFieldSpec field, int minValidValue, int maxValidValue, List validationFailures) { + boolean ret = true; + + String value = null; + String fieldName = field.toString(); + + String noWhiteSpace = StringUtils.deleteWhitespace(recurrence.getSchedule().getFieldValue(field)); + String[] specs = StringUtils.split(noWhiteSpace, ","); + class Range { + private int lower; + private int upper; + private Range(int lower, int upper) { + this.lower = lower; + this.upper = upper; + } + } + class RangeComparator implements Comparator { + @Override + public int compare(Range me, Range other) { + int result; + result = Integer.compare(me.lower, other.lower); + if (result == 0) { + result = Integer.compare(me.upper, other.upper); + } + return result; + } + } + + List rangeOfValues = new ArrayList<>(); + + List values = new ArrayList<>(); + + for (String spec : specs) { + + if (StringUtils.isNotEmpty(spec)) { + // Range + if (spec.startsWith("-") || spec.endsWith("-")) { + validationFailures.add(new ValidationFailureDetails(0, fieldName, "", false, true, false, "incorrect range spec")); + ret = false; + } else { + String[] ranges = StringUtils.split(spec, "-"); + if (ranges.length > 2) { + validationFailures.add(new ValidationFailureDetails(0, fieldName, "", false, true, false, "incorrect range spec")); + ret = false; + } else if (ranges.length == 2) { + int val1 = minValidValue, val2 = maxValidValue; + if (!StringUtils.equals(ranges[0], RangerValidityRecurrence.RecurrenceSchedule.WILDCARD)) { + val1 = Integer.valueOf(ranges[0]); + if (val1 < minValidValue || val1 > maxValidValue) { + validationFailures.add(new ValidationFailureDetails(0, fieldName, "", false, true, false, "incorrect lower range value")); + ret = false; + } + } else { + value = RangerValidityRecurrence.RecurrenceSchedule.WILDCARD; + } + if (!StringUtils.equals(ranges[1], RangerValidityRecurrence.RecurrenceSchedule.WILDCARD)) { + val2 = Integer.valueOf(ranges[1]); + if (val1 < minValidValue || val2 > maxValidValue) { + validationFailures.add(new ValidationFailureDetails(0, fieldName, "", false, true, false, "incorrect upper range value")); + ret = false; + } + } else { + value = RangerValidityRecurrence.RecurrenceSchedule.WILDCARD; + } + if (ret) { + if (val1 >= val2) { + validationFailures.add(new ValidationFailureDetails(0, fieldName, "", false, true, false, "incorrect range")); + ret = false; + } else { + value = RangerValidityRecurrence.RecurrenceSchedule.WILDCARD; + for (Range range : rangeOfValues) { + if (range.lower == val1 || range.upper == val2) { + validationFailures.add(new ValidationFailureDetails(0, fieldName, "", false, true, false, "duplicate range")); + ret = false; + break; + } + } + if (ret) { + rangeOfValues.add(new Range(val1, val2)); + } + } + } + } else if (ranges.length == 1) { + if (!StringUtils.equals(ranges[0], RangerValidityRecurrence.RecurrenceSchedule.WILDCARD)) { + int val = Integer.valueOf(ranges[0]); + if (val < minValidValue || val > maxValidValue) { + validationFailures.add(new ValidationFailureDetails(0, fieldName, "", false, true, false, "incorrect value")); + ret = false; + } else { + if (!StringUtils.equals(value, RangerValidityRecurrence.RecurrenceSchedule.WILDCARD)) { + values.add(Integer.valueOf(ranges[0])); + } + } + } else { + value = RangerValidityRecurrence.RecurrenceSchedule.WILDCARD; + } + } else { + ret = false; + } + } + } + } + //if (ret) { + if (CollectionUtils.isNotEmpty(rangeOfValues)) { + rangeOfValues.sort( new RangeComparator()); + } + for (int i = 0; i < rangeOfValues.size(); i++) { + Range range = rangeOfValues.get(i); + int upper = range.upper; + for (int j = i+1; j < rangeOfValues.size(); j++) { + Range r = rangeOfValues.get(j); + if (upper < r.upper) { + validationFailures.add(new ValidationFailureDetails(0, fieldName, "", false, true, false, "overlapping range value")); + ret = false; + } + } + } + //} + if (ret) { + if (!StringUtils.equals(value, RangerValidityRecurrence.RecurrenceSchedule.WILDCARD)) { + + int minDiff = (values.size() <= 1) ? maxValidValue + 1 : Integer.MAX_VALUE; + + if (values.size() > 1) { + Collections.sort(values); + for (int i = 0; i < values.size() - 1; i++) { + int diff = values.get(i + 1) - values.get(i); + if (diff < minDiff) { + minDiff = diff; + } + int firstLastDiff = values.get(0) + (maxValidValue - minValidValue + 1) - values.get(values.size() - 1); + + if (minDiff > firstLastDiff) { + minDiff = firstLastDiff; + } + } + } + if (values.size() > 0) { + value = Integer.toString(minDiff); + } + } + validityPeriodEstimator.setFieldValue(field, value); + if (LOG.isDebugEnabled()) { + LOG.debug("Set " + field + " to " + value); + } + } + return ret; + } + + private String getNormalizedValue(RangerValidityRecurrence recurrence, RangerValidityRecurrence.RecurrenceSchedule.ScheduleFieldSpec field) { + String ret = null; + + if (RangerValidityRecurrence.ValidityInterval.getValidityIntervalInMinutes(recurrence.getInterval()) > 0) { + String noWhiteSpace = StringUtils.deleteWhitespace(recurrence.getSchedule().getFieldValue(field)); + String[] specs = StringUtils.split(noWhiteSpace, ","); + + List values = new ArrayList<>(); + + for (String spec : specs) { + if (StringUtils.isNotBlank(spec)) { + values.add(spec); + } + } + if (values.size() > 0) { + Collections.sort(values); + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < values.size(); i++) { + if (i != 0) { + sb.append(","); + } + sb.append(values.get(i)); + } + ret = sb.toString(); + } + } + return ret; + } + +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/validation/RangerZoneResourceMatcher.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/validation/RangerZoneResourceMatcher.java new file mode 100644 index 00000000000..1a56657ae2d --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/validation/RangerZoneResourceMatcher.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.model.validation; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.plugin.model.RangerPolicy; +import org.apache.atlas.plugin.model.RangerServiceDef; +import org.apache.atlas.plugin.policyresourcematcher.RangerDefaultPolicyResourceMatcher; +import org.apache.atlas.plugin.policyresourcematcher.RangerPolicyResourceEvaluator; +import org.apache.atlas.plugin.policyresourcematcher.RangerPolicyResourceMatcher; +import org.apache.atlas.plugin.resourcematcher.RangerResourceMatcher; +import org.apache.atlas.plugin.util.ServiceDefUtil; + +import java.util.Collection; +import java.util.List; +import java.util.Map; + +public class RangerZoneResourceMatcher implements RangerPolicyResourceEvaluator { + private static final Log LOG = LogFactory.getLog(RangerZoneResourceMatcher.class); + + private final String securityZoneName; + private final Map policyResource; + private final RangerPolicyResourceMatcher policyResourceMatcher; + private RangerServiceDef.RangerResourceDef leafResourceDef; + + public RangerZoneResourceMatcher(final String securityZoneName, final Map policyResource, final RangerServiceDef serviceDef) { + + RangerServiceDefHelper serviceDefHelper = new RangerServiceDefHelper(serviceDef); + final Collection resourceKeys = policyResource.keySet(); + + RangerDefaultPolicyResourceMatcher matcher = new RangerDefaultPolicyResourceMatcher(); + + matcher.setServiceDef(serviceDef); + matcher.setServiceDefHelper(serviceDefHelper); + + boolean found = false; + + for (String policyType : RangerPolicy.POLICY_TYPES) { + for (List hierarchy : serviceDefHelper.getResourceHierarchies(policyType)) { + if (serviceDefHelper.hierarchyHasAllResources(hierarchy, resourceKeys)) { + if (LOG.isDebugEnabled()) { + LOG.debug("Found hierarchy for resource-keys:[" + resourceKeys + "], policy-type:[" + policyType + "]"); + } + matcher.setPolicyResources(policyResource, policyType); + found = true; + break; + } + } + if (found) { + break; + } + } + if (found) { + matcher.init(); + } else { + LOG.error("Cannot initialize matcher for RangerZoneResourceMatcher"); + } + + this.securityZoneName = securityZoneName; + this.policyResourceMatcher = matcher; + this.policyResource = policyResource; + this.leafResourceDef = ServiceDefUtil.getLeafResourceDef(serviceDef, getPolicyResource()); + } + + public String getSecurityZoneName() { return securityZoneName; } + + @Override + public long getId() { + return securityZoneName.hashCode(); + } + + @Override + public String getGuid() { + return null; + } + + @Override + public RangerPolicyResourceMatcher getPolicyResourceMatcher() { return policyResourceMatcher; } + + @Override + public Map getPolicyResource() { + return policyResource; + } + + @Override + public RangerResourceMatcher getResourceMatcher(String resourceName) { + return policyResourceMatcher != null ? policyResourceMatcher.getResourceMatcher(resourceName) : null; + } + + @Override + public boolean isAncestorOf(RangerServiceDef.RangerResourceDef resourceDef) { + return ServiceDefUtil.isAncestorOf(policyResourceMatcher.getServiceDef(), leafResourceDef, resourceDef); + } + + @Override + public String toString() { + return "{security-zone-name:[" + securityZoneName + "], policyResource=[" + policyResource +"]}"; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/validation/ValidationFailureDetails.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/validation/ValidationFailureDetails.java new file mode 100644 index 00000000000..925925eb0cf --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/validation/ValidationFailureDetails.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.model.validation; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import java.util.Objects; + +public class ValidationFailureDetails { + + private static final Log LOG = LogFactory.getLog(ValidationFailureDetails.class); + + final String _fieldName; + final String _subFieldName; + final boolean _missing; + final boolean _semanticError; + final boolean _internalError; + final String _reason; + final int _errorCode; + + public ValidationFailureDetails(int errorCode, String fieldName, String subFieldName, boolean missing, boolean semanticError, boolean internalError, String reason) { + _errorCode = errorCode; + _missing = missing; + _semanticError = semanticError; + _internalError = internalError; + _fieldName = fieldName; + _subFieldName = subFieldName; + _reason = reason; + } + + public String getFieldName() { + return _fieldName; + } + + public boolean isMissingRequiredValue() { + return _missing; + } + + public boolean isSemanticallyIncorrect() { + return _semanticError; + } + + String getType() { + if (_missing) return "missing"; + if (_semanticError) return "semantically incorrect"; + if (_internalError) return "internal error"; + return ""; + } + + public String getSubFieldName() { + return _subFieldName; + } + + @Override + public String toString() { + LOG.debug("ValidationFailureDetails.toString()"); + return String.format(" %s: error code[%d], reason[%s], field[%s], subfield[%s], type[%s]", "Validation failure", + _errorCode, _reason, _fieldName, _subFieldName, getType()); + } + + @Override + public int hashCode() { + return Objects.hash(_fieldName, _subFieldName, _missing, _semanticError, _internalError, _reason, _errorCode); + } + + @Override + public boolean equals(Object obj) { + if (obj == null || !(obj instanceof ValidationFailureDetails)) { + return false; + } + ValidationFailureDetails that = (ValidationFailureDetails)obj; + return Objects.equals(_fieldName, that._fieldName) && + Objects.equals(_subFieldName, that._subFieldName) && + Objects.equals(_reason, that._reason) && + _internalError == that._internalError && + _missing == that._missing && + _semanticError == that._semanticError && + _errorCode == that._errorCode; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/validation/ValidationFailureDetailsBuilder.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/validation/ValidationFailureDetailsBuilder.java new file mode 100644 index 00000000000..5f040a1e590 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/validation/ValidationFailureDetailsBuilder.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.model.validation; + +public class ValidationFailureDetailsBuilder { + protected String _fieldName; + protected boolean _missing; + protected boolean _semanticError; + protected String _reason; + protected String _subFieldName; + protected boolean _internalError; + protected int _errorCode; + + ValidationFailureDetailsBuilder becauseOf(String aReason) { + _reason = aReason; + return this; + } + + ValidationFailureDetailsBuilder isMissing() { + _missing = true; + return this; + } + + ValidationFailureDetailsBuilder isSemanticallyIncorrect() { + _semanticError = true; + return this; + } + + ValidationFailureDetailsBuilder field(String fieldName) { + _fieldName = fieldName; + return this; + } + + ValidationFailureDetails build() { + return new ValidationFailureDetails(_errorCode, _fieldName, _subFieldName, _missing, _semanticError, _internalError, _reason); + } + + ValidationFailureDetailsBuilder subField(String missingParameter) { + _subFieldName = missingParameter; + return this; + } + + ValidationFailureDetailsBuilder isAnInternalError() { + _internalError = true; + return this; + } + + ValidationFailureDetailsBuilder errorCode(int errorCode) { + _errorCode = errorCode; + return this; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/CacheMap.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/CacheMap.java new file mode 100644 index 00000000000..e44bd0fba16 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/CacheMap.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.atlas.plugin.policyengine; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import java.util.LinkedHashMap; +import java.util.Map; + +public class CacheMap extends LinkedHashMap { + private static final long serialVersionUID = 1L; + private static final Log LOG = LogFactory.getLog(CacheMap.class); + + + private static final float RANGER_CACHE_DEFAULT_LOAD_FACTOR = 0.75f; + + protected int initialCapacity; + + public CacheMap(int initialCapacity) { + super(initialCapacity, CacheMap.RANGER_CACHE_DEFAULT_LOAD_FACTOR, true); // true for access-order + + this.initialCapacity = initialCapacity; + } + + @Override + protected boolean removeEldestEntry(Map.Entry eldest) { + boolean result = size() > initialCapacity; + + if (LOG.isDebugEnabled()) { + LOG.debug("CacheMap.removeEldestEntry(), result:"+ result); + } + + return result; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/PolicyEngine.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/PolicyEngine.java new file mode 100644 index 00000000000..6cd52d2692b --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/PolicyEngine.java @@ -0,0 +1,930 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.policyengine; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.ListUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.plugin.contextenricher.RangerContextEnricher; +import org.apache.atlas.plugin.model.RangerPolicy; +import org.apache.atlas.plugin.model.RangerPolicyDelta; +import org.apache.atlas.plugin.model.RangerServiceDef; +import org.apache.atlas.plugin.model.validation.RangerServiceDefHelper; +import org.apache.atlas.plugin.model.validation.RangerZoneResourceMatcher; +import org.apache.atlas.plugin.policyevaluator.RangerPolicyEvaluator; +import org.apache.atlas.plugin.policyresourcematcher.RangerPolicyResourceMatcher; +import org.apache.atlas.plugin.resourcematcher.RangerAbstractResourceMatcher; +import org.apache.atlas.plugin.service.RangerAuthContext; +import org.apache.atlas.plugin.store.EmbeddedServiceDefsUtil; +import org.apache.atlas.plugin.util.RangerPerfTracer; +import org.apache.atlas.plugin.util.RangerPolicyDeltaUtil; +import org.apache.atlas.plugin.util.RangerReadWriteLock; +import org.apache.atlas.plugin.util.RangerRoles; +import org.apache.atlas.plugin.util.ServicePolicies; +import org.apache.atlas.plugin.util.StringTokenReplacer; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class PolicyEngine { + private static final Log LOG = LogFactory.getLog(PolicyEngine.class); + + private static final Log PERF_POLICYENGINE_INIT_LOG = RangerPerfTracer.getPerfLogger("policyengine.init"); + private static final Log PERF_POLICYENGINE_REBALANCE_LOG = RangerPerfTracer.getPerfLogger("policyengine.rebalance"); + + private final RangerPolicyRepository policyRepository; + private final RangerPolicyRepository tagPolicyRepository; + private final List allContextEnrichers; + private final RangerPluginContext pluginContext; + private final Map zonePolicyRepositories = new HashMap<>(); + private final Map resourceZoneTrie = new HashMap<>(); + private final Map zoneTagServiceMap = new HashMap<>(); + private boolean useForwardedIPAddress; + private String[] trustedProxyAddresses; + private final Map tokenReplacers = new HashMap<>(); + + private final RangerReadWriteLock lock; + + public RangerReadWriteLock.RangerLock getReadLock() { + return lock.getReadLock(); + } + + public RangerReadWriteLock.RangerLock getWriteLock() { + return lock.getWriteLock(); + } + + public boolean getUseForwardedIPAddress() { + return useForwardedIPAddress; + } + + public void setUseForwardedIPAddress(boolean useForwardedIPAddress) { + this.useForwardedIPAddress = useForwardedIPAddress; + } + + public String[] getTrustedProxyAddresses() { + return trustedProxyAddresses; + } + + public void setTrustedProxyAddresses(String[] trustedProxyAddresses) { + this.trustedProxyAddresses = trustedProxyAddresses; + } + + public long getRoleVersion() { return this.pluginContext.getAuthContext().getRoleVersion(); } + + public void setRoles(RangerRoles roles) { this.pluginContext.getAuthContext().setRoles(roles); } + + public String getServiceName() { + return policyRepository.getServiceName(); + } + + public RangerServiceDef getServiceDef() { + return policyRepository.getServiceDef(); + } + + public long getPolicyVersion() { + return policyRepository.getPolicyVersion(); + } + + public RangerPolicyRepository getPolicyRepository() { + return policyRepository; + } + + public RangerPolicyRepository getTagPolicyRepository() { + return tagPolicyRepository; + } + + public Map getZonePolicyRepositories() { return zonePolicyRepositories; } + + public List getAllContextEnrichers() { return allContextEnrichers; } + + public RangerPluginContext getPluginContext() { return pluginContext; } + + public StringTokenReplacer getStringTokenReplacer(String resourceName) { + return tokenReplacers.get(resourceName); + } + + @Override + public String toString() { + return toString(new StringBuilder()).toString(); + } + + @Override + protected void finalize() throws Throwable { + try { + cleanup(); + } finally { + super.finalize(); + } + } + + public StringBuilder toString(StringBuilder sb) { + if (sb == null) { + sb = new StringBuilder(); + } + + sb.append("PolicyEngine={"); + + sb.append("serviceName={").append(this.getServiceName()).append("} "); + + sb.append("policyRepository={"); + if (policyRepository != null) { + policyRepository.toString(sb); + } + sb.append("} "); + + sb.append("tagPolicyRepository={"); + if (tagPolicyRepository != null) { + tagPolicyRepository.toString(sb); + } + sb.append("} "); + + sb.append(lock.toString()); + + sb.append("}"); + + return sb; + } + + public List getResourcePolicies(String zoneName) { + RangerPolicyRepository zoneResourceRepository = zonePolicyRepositories.get(zoneName); + + return zoneResourceRepository == null ? ListUtils.EMPTY_LIST : zoneResourceRepository.getPolicies(); + } + + Map getResourceZoneTrie() { + return resourceZoneTrie; + } + + public PolicyEngine(ServicePolicies servicePolicies, RangerPluginContext pluginContext, RangerRoles roles, boolean isUseReadWriteLock) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> PolicyEngine(" + ", " + servicePolicies + ", " + pluginContext + ")"); + } + + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_POLICYENGINE_INIT_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_POLICYENGINE_INIT_LOG, "RangerPolicyEngine.init(hashCode=" + Integer.toHexString(System.identityHashCode(this)) + ")"); + + long freeMemory = Runtime.getRuntime().freeMemory(); + long totalMemory = Runtime.getRuntime().totalMemory(); + + PERF_POLICYENGINE_INIT_LOG.debug("In-Use memory: " + (totalMemory - freeMemory) + ", Free memory:" + freeMemory); + } + + this.pluginContext = pluginContext; + this.lock = new RangerReadWriteLock(isUseReadWriteLock); + + LOG.info("Policy engine will" + (isUseReadWriteLock ? " " : " not ") + "perform in place update while processing policy-deltas."); + + this.pluginContext.setAuthContext(new RangerAuthContext(null, roles)); + + RangerPolicyEngineOptions options = pluginContext.getConfig().getPolicyEngineOptions(); + + if(StringUtils.isBlank(options.evaluatorType) || StringUtils.equalsIgnoreCase(options.evaluatorType, RangerPolicyEvaluator.EVALUATOR_TYPE_AUTO)) { + options.evaluatorType = RangerPolicyEvaluator.EVALUATOR_TYPE_OPTIMIZED; + } + + policyRepository = new RangerPolicyRepository(servicePolicies, this.pluginContext); + + ServicePolicies.TagPolicies tagPolicies = servicePolicies.getTagPolicies(); + + if (!options.disableTagPolicyEvaluation + && tagPolicies != null + && !StringUtils.isEmpty(tagPolicies.getServiceName()) + && tagPolicies.getServiceDef() != null) { + if (LOG.isDebugEnabled()) { + LOG.debug("PolicyEngine : Building tag-policy-repository for tag-service " + tagPolicies.getServiceName()); + } + + tagPolicyRepository = new RangerPolicyRepository(tagPolicies, this.pluginContext, servicePolicies.getServiceDef(), servicePolicies.getServiceName()); + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("PolicyEngine : No tag-policy-repository for service " + servicePolicies.getServiceName()); + } + + tagPolicyRepository = null; + } + + List tmpList; + List tagContextEnrichers = tagPolicyRepository == null ? null :tagPolicyRepository.getContextEnrichers(); + List resourceContextEnrichers = policyRepository.getContextEnrichers(); + + if (CollectionUtils.isEmpty(tagContextEnrichers)) { + tmpList = resourceContextEnrichers; + } else if (CollectionUtils.isEmpty(resourceContextEnrichers)) { + tmpList = tagContextEnrichers; + } else { + tmpList = new ArrayList<>(tagContextEnrichers); + + tmpList.addAll(resourceContextEnrichers); + } + + this.allContextEnrichers = tmpList; + + if (MapUtils.isNotEmpty(servicePolicies.getSecurityZones())) { + buildZoneTrie(servicePolicies); + + for (Map.Entry zone : servicePolicies.getSecurityZones().entrySet()) { + RangerPolicyRepository policyRepository = new RangerPolicyRepository(servicePolicies, this.pluginContext, zone.getKey()); + + zonePolicyRepositories.put(zone.getKey(), policyRepository); + } + } + + for (RangerServiceDef.RangerResourceDef resourceDef : getServiceDef().getResources()) { + Map matchOptions = resourceDef.getMatcherOptions(); + + if (RangerAbstractResourceMatcher.getOptionReplaceTokens(matchOptions)) { + String delimiterPrefix = RangerAbstractResourceMatcher.getOptionDelimiterPrefix(matchOptions); + char delimiterStart = RangerAbstractResourceMatcher.getOptionDelimiterStart(matchOptions); + char delimiterEnd = RangerAbstractResourceMatcher.getOptionDelimiterEnd(matchOptions); + char escapeChar = RangerAbstractResourceMatcher.getOptionDelimiterEscape(matchOptions); + + StringTokenReplacer tokenReplacer = new StringTokenReplacer(delimiterStart, delimiterEnd, escapeChar, delimiterPrefix); + tokenReplacers.put(resourceDef.getName(), tokenReplacer); + } + } + + RangerPerfTracer.log(perf); + + if (PERF_POLICYENGINE_INIT_LOG.isDebugEnabled()) { + long freeMemory = Runtime.getRuntime().freeMemory(); + long totalMemory = Runtime.getRuntime().totalMemory(); + + PERF_POLICYENGINE_INIT_LOG.debug("In-Use memory: " + (totalMemory - freeMemory) + ", Free memory:" + freeMemory); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== PolicyEngine()"); + } + } + + public PolicyEngine cloneWithDelta(ServicePolicies servicePolicies) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> cloneWithDelta(" + Arrays.toString(servicePolicies.getPolicyDeltas().toArray()) + ", " + servicePolicies.getPolicyVersion() + ")"); + } + + final PolicyEngine ret; + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_POLICYENGINE_INIT_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_POLICYENGINE_INIT_LOG, "RangerPolicyEngine.cloneWithDelta()"); + } + + try (RangerReadWriteLock.RangerLock writeLock = getWriteLock()) { + if (LOG.isDebugEnabled()) { + if (writeLock.isLockingEnabled()) { + LOG.debug("Acquired lock - " + writeLock); + } + } + + RangerServiceDef serviceDef = this.getServiceDef(); + String serviceType = (serviceDef != null) ? serviceDef.getName() : ""; + boolean isValidDeltas = false; + + if (CollectionUtils.isNotEmpty(servicePolicies.getPolicyDeltas()) || MapUtils.isNotEmpty(servicePolicies.getSecurityZones())) { + isValidDeltas = CollectionUtils.isEmpty(servicePolicies.getPolicyDeltas()) || RangerPolicyDeltaUtil.isValidDeltas(servicePolicies.getPolicyDeltas(), serviceType); + + if (isValidDeltas) { + if (MapUtils.isNotEmpty(servicePolicies.getSecurityZones())) { + for (Map.Entry entry : servicePolicies.getSecurityZones().entrySet()) { + if (!RangerPolicyDeltaUtil.isValidDeltas(entry.getValue().getPolicyDeltas(), serviceType)) { + if (LOG.isDebugEnabled()) { + LOG.debug("Invalid policy-deltas for security zone:[" + entry.getKey() + "]"); + } + + isValidDeltas = false; + break; + } + } + } + } + } + + if (isValidDeltas) { + if (writeLock.isLockingEnabled()) { + updatePolicyEngine(servicePolicies); + ret = this; + } else { + ret = new PolicyEngine(this, servicePolicies); + } + } else { + ret = null; + } + } + + RangerPerfTracer.log(perf); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== cloneWithDelta(" + Arrays.toString(servicePolicies.getPolicyDeltas().toArray()) + ", " + servicePolicies.getPolicyVersion() + ")"); + } + return ret; + } + + public RangerPolicyRepository getRepositoryForMatchedZone(RangerPolicy policy) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> PolicyEngine.getRepositoryForMatchedZone(" + policy + ")"); + } + + String zoneName = policy.getZoneName(); + final RangerPolicyRepository ret = getRepositoryForZone(zoneName); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== PolicyEngine.getRepositoryForMatchedZone(" + policy + ")"); + } + + return ret; + } + + public Set getMatchedZonesForResourceAndChildren(RangerAccessResource accessResource) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> PolicyEngine.getMatchedZonesForResourceAndChildren(" + accessResource + ")"); + } + + Set ret = null; + + if (MapUtils.isNotEmpty(this.resourceZoneTrie)) { + ret = getMatchedZonesForResourceAndChildren(accessResource.getAsMap(), accessResource); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== PolicyEngine.getMatchedZonesForResourceAndChildren(" + accessResource + ") : " + ret); + } + + return ret; + } + + public String getUniquelyMatchedZoneName(Map resourceAsMap) { + String ret = null; + Set matchedZones = getMatchedZonesForResourceAndChildren(resourceAsMap, convertToAccessResource(resourceAsMap)); + if (CollectionUtils.isNotEmpty(matchedZones) && matchedZones.size() == 1) { + String[] matchedZonesArray = new String[1]; + matchedZones.toArray(matchedZonesArray); + ret = matchedZonesArray[0]; + } + return ret; + } + + public RangerPolicyRepository getRepositoryForZone(String zoneName) { + final RangerPolicyRepository ret; + + if (LOG.isDebugEnabled()) { + LOG.debug("zoneName:[" + zoneName + "]"); + } + + if (StringUtils.isNotEmpty(zoneName)) { + ret = getZonePolicyRepositories().get(zoneName); + } else { + ret = getPolicyRepository(); + } + + if (ret == null) { + LOG.error("policyRepository for zoneName:[" + zoneName + "], serviceName:[" + getServiceName() + "], policyVersion:[" + getPolicyVersion() + "] is null!! ERROR!"); + } + + return ret; + } + + public boolean hasTagPolicies(RangerPolicyRepository tagPolicyRepository) { + return tagPolicyRepository != null && CollectionUtils.isNotEmpty(tagPolicyRepository.getPolicies()); + } + + public boolean hasResourcePolicies(RangerPolicyRepository policyRepository) { + return policyRepository != null && CollectionUtils.isNotEmpty(policyRepository.getPolicies()); + } + + public boolean isResourceZoneAssociatedWithTagService(String resourceZoneName) { + final boolean ret; + + if (StringUtils.isNotEmpty(resourceZoneName) && tagPolicyRepository != null && zoneTagServiceMap.get(resourceZoneName) != null) { + if (LOG.isDebugEnabled()) { + LOG.debug("Accessed resource is in a zone:[" + resourceZoneName + "] which is associated with the tag-service:[" + tagPolicyRepository.getServiceName() + "]"); + } + + ret = true; + } else { + ret = false; + } + + return ret; + } + + public void preCleanup(boolean isForced) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> PolicyEngine.preCleanup(isForced=" + isForced + ")"); + } + + if (policyRepository != null) { + policyRepository.preCleanup(isForced); + } + + if (tagPolicyRepository != null) { + tagPolicyRepository.preCleanup(isForced); + } + + if (MapUtils.isNotEmpty(this.zonePolicyRepositories)) { + for (Map.Entry entry : this.zonePolicyRepositories.entrySet()) { + entry.getValue().preCleanup(isForced); + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== PolicyEngine.preCleanup(isForced=" + isForced + ")"); + } + } + + private Set getMatchedZonesForResourceAndChildren(Map resource, RangerAccessResource accessResource) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> PolicyEngine.getMatchedZonesForResourceAndChildren(" + resource + ", " + accessResource + ")"); + } + + Set ret = null; + + if (MapUtils.isNotEmpty(this.resourceZoneTrie)) { + Set smallestList = null; + RangerServiceDefHelper serviceDefHelper = policyRepository.getOptions().getServiceDefHelper(); + + List resourceKeys = resource == null ? new ArrayList<>() : serviceDefHelper.getOrderedResourceNames(resource.keySet()); + + for (String resourceDefName : resourceKeys) { + RangerResourceTrie trie = resourceZoneTrie.get(resourceDefName); + + if (trie == null) { + continue; + } + + Object resourceValues = resource.get(resourceDefName); + + Set zoneMatchersForResource = trie.getEvaluatorsForResource(resourceValues); + Set inheritedZoneMatchers = trie.getInheritedEvaluators(); + + if (LOG.isDebugEnabled()) { + LOG.debug("ResourceDefName:[" + resourceDefName + "], values:[" + resourceValues + "], matched-zones:[" + zoneMatchersForResource + "], inherited-zones:[" + inheritedZoneMatchers + "]"); + } + + if (smallestList != null) { + if (CollectionUtils.isEmpty(inheritedZoneMatchers) && CollectionUtils.isEmpty(zoneMatchersForResource)) { + smallestList = null; + } else if (CollectionUtils.isEmpty(inheritedZoneMatchers)) { + smallestList.retainAll(zoneMatchersForResource); + } else if (CollectionUtils.isEmpty(zoneMatchersForResource)) { + smallestList.retainAll(inheritedZoneMatchers); + } else { + Set smaller, bigger; + if (zoneMatchersForResource.size() < inheritedZoneMatchers.size()) { + smaller = zoneMatchersForResource; + bigger = inheritedZoneMatchers; + } else { + smaller = inheritedZoneMatchers; + bigger = zoneMatchersForResource; + } + Set tmp = new HashSet<>(); + if (smallestList.size() < smaller.size()) { + smallestList.stream().filter(smaller::contains).forEach(tmp::add); + smallestList.stream().filter(bigger::contains).forEach(tmp::add); + } else { + smaller.stream().filter(smallestList::contains).forEach(tmp::add); + if (smallestList.size() < bigger.size()) { + smallestList.stream().filter(bigger::contains).forEach(tmp::add); + } else { + bigger.stream().filter(smallestList::contains).forEach(tmp::add); + } + } + smallestList = tmp; + } + } else { + if (CollectionUtils.isEmpty(inheritedZoneMatchers) || CollectionUtils.isEmpty(zoneMatchersForResource)) { + Set tmp = CollectionUtils.isEmpty(inheritedZoneMatchers) ? zoneMatchersForResource : inheritedZoneMatchers; + smallestList = resourceKeys.size() == 1 || CollectionUtils.isEmpty(tmp) ? tmp : new HashSet<>(tmp); + } else { + smallestList = new HashSet<>(zoneMatchersForResource); + smallestList.addAll(inheritedZoneMatchers); + } + } + } + + if (CollectionUtils.isNotEmpty(smallestList)) { + final Set intersection = smallestList; + + if (LOG.isDebugEnabled()) { + LOG.debug("Resource:[" + resource + "], matched-zones:[" + intersection + "]"); + } + + if (intersection.size() > 0) { + ret = new HashSet<>(); + + for (RangerZoneResourceMatcher zoneMatcher : intersection) { + if (LOG.isDebugEnabled()) { + LOG.debug("Trying to match resource:[" + accessResource + "] using zoneMatcher:[" + zoneMatcher + "]"); + } + + // These are potential matches. Try to really match them + if (zoneMatcher.getPolicyResourceMatcher().isMatch(accessResource, RangerPolicyResourceMatcher.MatchScope.ANY, null)) { + if (LOG.isDebugEnabled()) { + LOG.debug("Matched resource:[" + accessResource + "] using zoneMatcher:[" + zoneMatcher + "]"); + } + + // Actual match happened + ret.add(zoneMatcher.getSecurityZoneName()); + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Did not match resource:[" + accessResource + "] using zoneMatcher:[" + zoneMatcher + "]"); + } + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("The following zone-names matched resource:[" + accessResource + "]: " + ret); + } + } + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== PolicyEngine.getMatchedZonesForResourceAndChildren(" + resource + ", " + accessResource + ") : " + ret); + } + + return ret; + } + + private RangerAccessResource convertToAccessResource(Map resource) { + RangerAccessResourceImpl ret = new RangerAccessResourceImpl(); + + ret.setServiceDef(getServiceDef()); + + for (Map.Entry entry : resource.entrySet()) { + ret.setValue(entry.getKey(), entry.getValue()); + } + + return ret; + } + + private PolicyEngine(final PolicyEngine other, ServicePolicies servicePolicies) { + this.useForwardedIPAddress = other.useForwardedIPAddress; + this.trustedProxyAddresses = other.trustedProxyAddresses; + this.pluginContext = other.pluginContext; + this.lock = other.lock; + + long policyVersion = servicePolicies.getPolicyVersion() != null ? servicePolicies.getPolicyVersion() : -1L; + List defaultZoneDeltas = new ArrayList<>(); + List defaultZoneDeltasForTagPolicies = new ArrayList<>(); + + getDeltasSortedByZones(other, servicePolicies, defaultZoneDeltas, defaultZoneDeltasForTagPolicies); + + if (other.policyRepository != null && CollectionUtils.isNotEmpty(defaultZoneDeltas)) { + this.policyRepository = new RangerPolicyRepository(other.policyRepository, defaultZoneDeltas, policyVersion); + } else { + this.policyRepository = shareWith(other.policyRepository); + } + + if (MapUtils.isEmpty(zonePolicyRepositories) && MapUtils.isNotEmpty(other.zonePolicyRepositories)) { + if (LOG.isDebugEnabled()) { + LOG.debug("Existing engine contains some zonePolicyRepositories and new engine contains no zonePolicyRepositories"); + } + for (Map.Entry entry : other.zonePolicyRepositories.entrySet()) { + if (LOG.isDebugEnabled()) { + LOG.debug("Copying over zoneRepository for zone :[" + entry.getKey() + "]"); + } + RangerPolicyRepository otherZonePolicyRepository = entry.getValue(); + RangerPolicyRepository zonePolicyRepository = shareWith(otherZonePolicyRepository); + this.zonePolicyRepositories.put(entry.getKey(), zonePolicyRepository); + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Existing engine contains no zonePolicyRepositories or new engine contains some zonePolicyRepositories"); + LOG.debug("Not copying zoneRepositories from existing engine, as they are already copied or modified"); + } + } + + if (servicePolicies.getTagPolicies() != null && CollectionUtils.isNotEmpty(defaultZoneDeltasForTagPolicies)) { + if (other.tagPolicyRepository == null) { + + if (LOG.isDebugEnabled()) { + LOG.debug("Current policy-engine does not have any tagPolicyRepository"); + } + // Only creates are expected + List tagPolicies = new ArrayList<>(); + + for (RangerPolicyDelta delta : defaultZoneDeltasForTagPolicies) { + if (delta.getChangeType() == RangerPolicyDelta.CHANGE_TYPE_POLICY_CREATE) { + tagPolicies.add(delta.getPolicy()); + } else { + LOG.warn("Expected changeType:[" + RangerPolicyDelta.CHANGE_TYPE_POLICY_CREATE + "], found policy-change-delta:[" + delta + "]"); + } + } + + servicePolicies.getTagPolicies().setPolicies(tagPolicies); + + this.tagPolicyRepository = new RangerPolicyRepository(servicePolicies.getTagPolicies(), this.pluginContext, servicePolicies.getServiceDef(), servicePolicies.getServiceName()); + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Current policy-engine has a tagPolicyRepository"); + } + this.tagPolicyRepository = new RangerPolicyRepository(other.tagPolicyRepository, defaultZoneDeltasForTagPolicies, policyVersion); + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Either no associated tag repository or no changes to tag policies"); + } + this.tagPolicyRepository = shareWith(other.tagPolicyRepository); + } + + List tmpList; + List tagContextEnrichers = tagPolicyRepository == null ? null :tagPolicyRepository.getContextEnrichers(); + List resourceContextEnrichers = policyRepository == null ? null : policyRepository.getContextEnrichers(); + + if (CollectionUtils.isEmpty(tagContextEnrichers)) { + tmpList = resourceContextEnrichers; + } else if (CollectionUtils.isEmpty(resourceContextEnrichers)) { + tmpList = tagContextEnrichers; + } else { + tmpList = new ArrayList<>(tagContextEnrichers); + + tmpList.addAll(resourceContextEnrichers); + } + + this.allContextEnrichers = tmpList; + + reorderPolicyEvaluators(); + } + + private void buildZoneTrie(ServicePolicies servicePolicies) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> PolicyEngine.buildZoneTrie()"); + } + + Map securityZones = servicePolicies.getSecurityZones(); + + if (MapUtils.isNotEmpty(securityZones)) { + RangerServiceDef serviceDef = servicePolicies.getServiceDef(); + List matchers = new ArrayList<>(); + + for (Map.Entry securityZone : securityZones.entrySet()) { + String zoneName = securityZone.getKey(); + ServicePolicies.SecurityZoneInfo zoneDetails = securityZone.getValue(); + + if (LOG.isDebugEnabled()) { + LOG.debug("Building matchers for zone:[" + zoneName +"]"); + } + + for (Map> resource : zoneDetails.getResources()) { + if (LOG.isDebugEnabled()) { + LOG.debug("Building matcher for resource:[" + resource + "] in zone:[" + zoneName +"]"); + } + + Map policyResources = new HashMap<>(); + + for (Map.Entry> entry : resource.entrySet()) { + String resourceDefName = entry.getKey(); + List resourceValues = entry.getValue(); + RangerPolicy.RangerPolicyResource policyResource = new RangerPolicy.RangerPolicyResource(); + policyResource.setIsExcludes(false); + policyResource.setIsRecursive(EmbeddedServiceDefsUtil.isRecursiveEnabled(serviceDef, resourceDefName)); + policyResource.setValues(resourceValues); + policyResources.put(resourceDefName, policyResource); + } + + matchers.add(new RangerZoneResourceMatcher(zoneName, policyResources, serviceDef)); + + if (LOG.isDebugEnabled()) { + LOG.debug("Built matcher for resource:[" + resource +"] in zone:[" + zoneName + "]"); + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("Built all matchers for zone:[" + zoneName +"]"); + } + + if (zoneDetails.getContainsAssociatedTagService()) { + zoneTagServiceMap.put(zoneName, zoneName); + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("Built matchers for all Zones"); + } + + for (RangerServiceDef.RangerResourceDef resourceDef : serviceDef.getResources()) { + resourceZoneTrie.put(resourceDef.getName(), new RangerResourceTrie<>(resourceDef, matchers)); + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== PolicyEngine.buildZoneTrie()"); + } + } + + private RangerPolicyRepository shareWith(RangerPolicyRepository other) { + if (other != null) { + other.setShared(); + } + + return other; + } + private void reorderPolicyEvaluators() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> reorderEvaluators()"); + } + + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_POLICYENGINE_REBALANCE_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_POLICYENGINE_REBALANCE_LOG, "RangerPolicyEngine.reorderEvaluators()"); + } + + if (tagPolicyRepository != null) { + tagPolicyRepository.reorderPolicyEvaluators(); + } + if (policyRepository != null) { + policyRepository.reorderPolicyEvaluators(); + } + + RangerPerfTracer.log(perf); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== reorderEvaluators()"); + } + } + + private void cleanup() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> PolicyEngine.cleanup()"); + } + + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_POLICYENGINE_INIT_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_POLICYENGINE_INIT_LOG, "RangerPolicyEngine.cleanUp(hashCode=" + Integer.toHexString(System.identityHashCode(this)) + ")"); + } + + preCleanup(false); + + if (policyRepository != null) { + policyRepository.cleanup(); + } + + if (tagPolicyRepository != null) { + tagPolicyRepository.cleanup(); + } + + if (MapUtils.isNotEmpty(this.zonePolicyRepositories)) { + for (Map.Entry entry : this.zonePolicyRepositories.entrySet()) { + entry.getValue().cleanup(); + } + } + + RangerPerfTracer.log(perf); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== PolicyEngine.cleanup()"); + } + } + + void updatePolicyEngine(ServicePolicies servicePolicies) { + List defaultZoneDeltas = new ArrayList<>(); + List defaultZoneDeltasForTagPolicies = new ArrayList<>(); + + getDeltasSortedByZones(this, servicePolicies, defaultZoneDeltas, defaultZoneDeltasForTagPolicies); + + if (this.policyRepository != null && CollectionUtils.isNotEmpty(defaultZoneDeltas)) { + this.policyRepository.reinit(defaultZoneDeltas); + } + + if (servicePolicies.getTagPolicies() != null && CollectionUtils.isNotEmpty(defaultZoneDeltasForTagPolicies)) { + if (this.tagPolicyRepository != null) { + this.tagPolicyRepository.reinit(defaultZoneDeltasForTagPolicies); + } else { + LOG.error("No previous tagPolicyRepository to update! Should not have come here!!"); + } + } + + reorderPolicyEvaluators(); + } + + private void getDeltasSortedByZones(PolicyEngine current, ServicePolicies servicePolicies, List defaultZoneDeltas, List defaultZoneDeltasForTagPolicies) { + + if (LOG.isDebugEnabled()) { + LOG.debug("==> getDeltasSortedByZones()"); + } + + long policyVersion = servicePolicies.getPolicyVersion() != null ? servicePolicies.getPolicyVersion() : -1L; + + if (CollectionUtils.isNotEmpty(defaultZoneDeltas)) { + LOG.warn("Emptying out defaultZoneDeltas!"); + defaultZoneDeltas.clear(); + } + if (CollectionUtils.isNotEmpty(defaultZoneDeltasForTagPolicies)) { + LOG.warn("Emptying out defaultZoneDeltasForTagPolicies!"); + defaultZoneDeltasForTagPolicies.clear(); + } + + if (MapUtils.isNotEmpty(servicePolicies.getSecurityZones())) { + buildZoneTrie(servicePolicies); + + Map> zoneDeltasMap = new HashMap<>(); + + for (Map.Entry zone : servicePolicies.getSecurityZones().entrySet()) { + String zoneName = zone.getKey(); + List deltas = zone.getValue().getPolicyDeltas(); + List zoneDeltas = new ArrayList<>(); + + if (StringUtils.isNotEmpty(zoneName)) { + zoneDeltasMap.put(zoneName, zoneDeltas); + + for (RangerPolicyDelta delta : deltas) { + zoneDeltas = zoneDeltasMap.get(zoneName); + zoneDeltas.add(delta); + } + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("Security zones found in the service-policies:[" + zoneDeltasMap.keySet() + "]"); + } + + for (Map.Entry> entry : zoneDeltasMap.entrySet()) { + final String zoneName = entry.getKey(); + final List zoneDeltas = entry.getValue(); + final RangerPolicyRepository otherRepository = current.zonePolicyRepositories.get(zoneName); + final RangerPolicyRepository policyRepository; + + if (LOG.isDebugEnabled()) { + LOG.debug("zoneName:[" + zoneName + "], zoneDeltas:[" + Arrays.toString(zoneDeltas.toArray()) + "], doesOtherRepositoryExist:[" + (otherRepository != null) + "]"); + } + + if (CollectionUtils.isNotEmpty(zoneDeltas)) { + if (otherRepository == null) { + List policies = new ArrayList<>(); + + for (RangerPolicyDelta delta : zoneDeltas) { + if (delta.getChangeType() == RangerPolicyDelta.CHANGE_TYPE_POLICY_CREATE) { + policies.add(delta.getPolicy()); + } else { + LOG.warn("Expected changeType:[" + RangerPolicyDelta.CHANGE_TYPE_POLICY_CREATE + "], found policy-change-delta:[" + delta +"]"); + } + } + + servicePolicies.getSecurityZones().get(zoneName).setPolicies(policies); + + policyRepository = new RangerPolicyRepository(servicePolicies, current.pluginContext, zoneName); + } else { + policyRepository = new RangerPolicyRepository(otherRepository, zoneDeltas, policyVersion); + } + } else { + policyRepository = shareWith(otherRepository); + } + + zonePolicyRepositories.put(zoneName, policyRepository); + } + } + + List unzonedDeltas = servicePolicies.getPolicyDeltas(); + + if (LOG.isDebugEnabled()) { + LOG.debug("ServicePolicies.policyDeltas:[" + Arrays.toString(servicePolicies.getPolicyDeltas().toArray()) + "]"); + } + + for (RangerPolicyDelta delta : unzonedDeltas) { + if (servicePolicies.getServiceDef().getName().equals(delta.getServiceType())) { + defaultZoneDeltas.add(delta); + } else { + defaultZoneDeltasForTagPolicies.add(delta); + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("defaultZoneDeltas:[" + Arrays.toString(defaultZoneDeltas.toArray()) + "]"); + LOG.debug("defaultZoneDeltasForTagPolicies:[" + Arrays.toString(defaultZoneDeltasForTagPolicies.toArray()) + "]"); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== getDeltasSortedByZones()"); + } + } +} + diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/PolicyEvaluatorForTag.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/PolicyEvaluatorForTag.java new file mode 100644 index 00000000000..4f6dad2992d --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/PolicyEvaluatorForTag.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.policyengine; + +import org.apache.atlas.plugin.contextenricher.RangerTagForEval; +import org.apache.atlas.plugin.policyevaluator.RangerPolicyEvaluator; + +import java.io.Serializable; +import java.util.Comparator; + +public class PolicyEvaluatorForTag { + public static final Comparator EVAL_ORDER_COMPARATOR = new PolicyEvalOrderComparator(); + public static final Comparator NAME_COMPARATOR = new PolicyNameComparator(); + + private final RangerPolicyEvaluator evaluator; + private final RangerTagForEval tag; + + PolicyEvaluatorForTag(RangerPolicyEvaluator evaluator, RangerTagForEval tag) { + this.evaluator = evaluator; + this.tag = tag; + } + + RangerPolicyEvaluator getEvaluator() { + return evaluator; + } + + RangerTagForEval getTag() { + return tag; + } + + static class PolicyNameComparator implements Comparator, Serializable { + @Override + public int compare(PolicyEvaluatorForTag me, PolicyEvaluatorForTag other) { + return RangerPolicyEvaluator.NAME_COMPARATOR.compare(me.getEvaluator(), other.getEvaluator()); + } + } + + static class PolicyEvalOrderComparator implements Comparator, Serializable { + @Override + public int compare(PolicyEvaluatorForTag me, PolicyEvaluatorForTag other) { + return RangerPolicyEvaluator.EVAL_ORDER_COMPARATOR.compare(me.getEvaluator(), other.getEvaluator()); + } + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerAccessRequest.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerAccessRequest.java new file mode 100644 index 00000000000..31a32a01232 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerAccessRequest.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.policyengine; + +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public interface RangerAccessRequest { + String RANGER_ACCESS_REQUEST_SCOPE_STRING = "Scope"; + + RangerAccessResource getResource(); + + String getAccessType(); + + boolean isAccessorsRequested(); + + boolean isAccessTypeAny(); + + boolean isAccessTypeDelegatedAdmin(); + + String getUser(); + + Set getUserGroups(); + + Set getUserRoles(); + + Date getAccessTime(); + + String getClientIPAddress(); + + String getRemoteIPAddress(); + + List getForwardedAddresses(); + + String getClientType(); + + String getAction(); + + String getRequestData(); + + String getSessionId(); + + String getClusterName(); + + String getClusterType(); + + Map getContext(); + + RangerAccessRequest getReadOnlyCopy(); + + ResourceMatchingScope getResourceMatchingScope(); + + enum ResourceMatchingScope {SELF, SELF_OR_DESCENDANTS, SELF_OR_CHILD} +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerAccessRequestImpl.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerAccessRequestImpl.java new file mode 100644 index 00000000000..52bf384fabb --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerAccessRequestImpl.java @@ -0,0 +1,373 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.policyengine; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class RangerAccessRequestImpl implements RangerAccessRequest { + private static final Logger LOG = LoggerFactory.getLogger(RangerAccessRequestImpl.class); + + private RangerAccessResource resource; + private String accessType; + private String user; + private Set userGroups; + private Set userRoles; + private Date accessTime; + private String clientIPAddress; + private List forwardedAddresses; + private String remoteIPAddress; + private String clientType; + private String action; + private String requestData; + private String sessionId; + private Map context; + private String clusterName; + private String clusterType; + + private boolean isAccessorsRequested; + private boolean isAccessTypeAny; + private boolean isAccessTypeDelegatedAdmin; + private ResourceMatchingScope resourceMatchingScope = ResourceMatchingScope.SELF; + + public RangerAccessRequestImpl() { + this(null, null, null, null, null); + } + + public RangerAccessRequestImpl(RangerAccessResource resource, String accessType, String user, Set userGroups, Set userRoles) { + setResource(resource); + setAccessType(accessType); + setUser(user); + setUserGroups(userGroups); + setUserRoles(userRoles); + setForwardedAddresses(null); + + // set remaining fields to default value + setAccessTime(null); + setRemoteIPAddress(null); + setClientType(null); + setAction(null); + setRequestData(null); + setSessionId(null); + setContext(null); + setClusterName(null); + } + + public RangerAccessRequestImpl(RangerAccessRequest request) { + setResource(request.getResource()); + setAccessType(request.getAccessType()); + setAccessorsRequested(request.isAccessorsRequested()); + setUser(request.getUser()); + setUserGroups(request.getUserGroups()); + setUserRoles(request.getUserRoles()); + setForwardedAddresses(request.getForwardedAddresses()); + setAccessTime(request.getAccessTime()); + setRemoteIPAddress(request.getRemoteIPAddress()); + setClientType(request.getClientType()); + setAction(request.getAction()); + setRequestData(request.getRequestData()); + setSessionId(request.getSessionId()); + setContext(request.getContext()); + setClusterName(request.getClusterName()); + setResourceMatchingScope(request.getResourceMatchingScope()); + setClientIPAddress(request.getClientIPAddress()); + setClusterType(request.getClusterType()); + } + + @Override + public RangerAccessResource getResource() { + return resource; + } + + @Override + public String getAccessType() { + return accessType; + } + + @Override + public String getUser() { + return user; + } + + @Override + public Set getUserGroups() { + return userGroups; + } + + @Override + public Set getUserRoles() { + return userRoles; + } + + @Override + public Date getAccessTime() { + return accessTime; + } + + @Override + public String getClientIPAddress() { return clientIPAddress;} + + @Override + public String getRemoteIPAddress() { + return remoteIPAddress; + } + + @Override + public List getForwardedAddresses() { return forwardedAddresses; } + + @Override + public String getClientType() { + return clientType; + } + + @Override + public String getAction() { + return action; + } + + @Override + public String getRequestData() { + return requestData; + } + + @Override + public String getSessionId() { + return sessionId; + } + + @Override + public Map getContext() { + return context; + } + + @Override + public ResourceMatchingScope getResourceMatchingScope() { + return resourceMatchingScope; + } + + @Override + public boolean isAccessTypeAny() { + return isAccessTypeAny; + } + + @Override + public boolean isAccessTypeDelegatedAdmin() { + return isAccessTypeDelegatedAdmin; + } + + @Override + public boolean isAccessorsRequested() { + return isAccessorsRequested; + } + + public void setAccessorsRequested(boolean accessorsRequested) { + isAccessorsRequested = accessorsRequested; + } + + public void setResource(RangerAccessResource resource) { + this.resource = resource; + } + + public void setAccessType(String accessType) { + if (StringUtils.isEmpty(accessType)) { + accessType = RangerPolicyEngine.ANY_ACCESS; + } + + this.accessType = accessType; + isAccessTypeAny = StringUtils.equals(accessType, RangerPolicyEngine.ANY_ACCESS); + isAccessTypeDelegatedAdmin = StringUtils.equals(accessType, RangerPolicyEngine.ADMIN_ACCESS); + } + + public void setUser(String user) { + this.user = user; + } + + public void setUserGroups(Set userGroups) { + this.userGroups = (userGroups == null) ? new HashSet() : userGroups; + } + + public void setUserRoles(Set userRoles) { + this.userRoles = (userRoles == null) ? new HashSet() : userRoles; + } + + public void setAccessTime(Date accessTime) { + this.accessTime = accessTime; + } + + public void setClientIPAddress(String ipAddress) { + this.clientIPAddress = ipAddress; + } + + public void setForwardedAddresses(List forwardedAddresses) { + this.forwardedAddresses = (forwardedAddresses == null) ? new ArrayList() : forwardedAddresses; + } + + public void setRemoteIPAddress(String remoteIPAddress) { + this.remoteIPAddress = remoteIPAddress; + } + + public void setClientType(String clientType) { + this.clientType = clientType; + } + + public void setAction(String action) { + this.action = action; + } + + public void setRequestData(String requestData) { + this.requestData = requestData; + } + + public void setSessionId(String sessionId) { + this.sessionId = sessionId; + } + + public String getClusterName() { + return clusterName; + } + + public void setClusterName(String clusterName) { + this.clusterName = clusterName; + } + + public String getClusterType() { + return clusterType; + } + + public void setClusterType(String clusterType) { + this.clusterType = clusterType; + } + + public void setResourceMatchingScope(ResourceMatchingScope scope) { this.resourceMatchingScope = scope; } + + public void setContext(Map context) { + this.context = (context == null) ? new HashMap() : context; + } + + public void extractAndSetClientIPAddress(boolean useForwardedIPAddress, String[]trustedProxyAddresses) { + String ip = getRemoteIPAddress(); + if (ip == null) { + ip = getClientIPAddress(); + } + + String newIp = ip; + + if (useForwardedIPAddress) { + if (LOG.isDebugEnabled()) { + LOG.debug("Using X-Forward-For..."); + } + if (CollectionUtils.isNotEmpty(getForwardedAddresses())) { + if (trustedProxyAddresses != null && trustedProxyAddresses.length > 0) { + if (StringUtils.isNotEmpty(ip)) { + for (String trustedProxyAddress : trustedProxyAddresses) { + if (StringUtils.equals(ip, trustedProxyAddress)) { + newIp = getForwardedAddresses().get(0); + break; + } + } + } + } else { + newIp = getForwardedAddresses().get(0); + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("No X-Forwarded-For addresses in the access-request"); + } + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("Old Remote/Client IP Address=" + ip + ", new IP Address=" + newIp); + } + setClientIPAddress(newIp); + } + + @Override + public String toString( ) { + StringBuilder sb = new StringBuilder(); + + toString(sb); + + return sb.toString(); + } + + public StringBuilder toString(StringBuilder sb) { + sb.append("RangerAccessRequestImpl={"); + + sb.append("resource={").append(resource).append("} "); + sb.append("accessType={").append(accessType).append("} "); + sb.append("isAccessorsRequested=").append(isAccessorsRequested).append(","); + sb.append("user={").append(user).append("} "); + + sb.append("userGroups={"); + if(userGroups != null) { + for(String userGroup : userGroups) { + sb.append(userGroup).append(" "); + } + } + sb.append("} "); + + sb.append("userRoles={"); + if(userRoles != null) { + for(String role : userRoles) { + sb.append(role).append(" "); + } + } + sb.append("} "); + + sb.append("accessTime={").append(accessTime).append("} "); + sb.append("clientIPAddress={").append(getClientIPAddress()).append("} "); + sb.append("forwardedAddresses={").append(StringUtils.join(forwardedAddresses, " ")).append("} "); + sb.append("remoteIPAddress={").append(remoteIPAddress).append("} "); + sb.append("clientType={").append(clientType).append("} "); + sb.append("action={").append(action).append("} "); + sb.append("requestData={").append(requestData).append("} "); + sb.append("sessionId={").append(sessionId).append("} "); + sb.append("resourceMatchingScope={").append(resourceMatchingScope).append("} "); + sb.append("clusterName={").append(clusterName).append("} "); + sb.append("clusterType={").append(clusterType).append("} "); + + sb.append("context={"); + if(context != null) { + for(Map.Entry e : context.entrySet()) { + sb.append(e.getKey()).append("={").append(e.getValue()).append("} "); + } + } + sb.append("} "); + + sb.append("}"); + + return sb; + } + @Override + public RangerAccessRequest getReadOnlyCopy() { + return new RangerAccessRequestReadOnly(this); + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerAccessRequestProcessor.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerAccessRequestProcessor.java new file mode 100644 index 00000000000..e1ff23e711f --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerAccessRequestProcessor.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.policyengine; + +public interface RangerAccessRequestProcessor { + void preProcess(RangerAccessRequest request); + + default void enrich(RangerAccessRequest request) {} +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerAccessRequestReadOnly.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerAccessRequestReadOnly.java new file mode 100644 index 00000000000..e7fd1324cab --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerAccessRequestReadOnly.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.policyengine; + +import java.util.Collections; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class RangerAccessRequestReadOnly implements RangerAccessRequest { + private final RangerAccessRequest source; + + // Cached here for reducing access overhead + private final RangerAccessResource resource; + private final Set userGroups; + private final Set userRoles; + private final List forwardedAddresses; + private final Map context; + + RangerAccessRequestReadOnly(final RangerAccessRequest source) { + this.source = source; + this.resource = source.getResource().getReadOnlyCopy(); + this.userGroups = Collections.unmodifiableSet(source.getUserGroups()); + this.userRoles = Collections.unmodifiableSet(source.getUserRoles()); + this.context = Collections.unmodifiableMap(source.getContext()); + this.forwardedAddresses = Collections.unmodifiableList(source.getForwardedAddresses()); + } + + @Override + public RangerAccessResource getResource() { return resource; } + + @Override + public String getAccessType() { return source.getAccessType(); } + + @Override + public boolean isAccessorsRequested() { + return source.isAccessorsRequested(); + } + + @Override + public boolean isAccessTypeAny() { return source.isAccessTypeAny(); } + + @Override + public boolean isAccessTypeDelegatedAdmin() { return source.isAccessTypeDelegatedAdmin(); } + + @Override + public String getUser() { return source.getUser(); } + + @Override + public Set getUserGroups() { return userGroups; } + + @Override + public Set getUserRoles() { return userRoles; } + + @Override + public Date getAccessTime() { return source.getAccessTime(); } + + @Override + public String getClientIPAddress() { return source.getClientIPAddress(); } + + @Override + public String getRemoteIPAddress() { return source.getRemoteIPAddress(); } + + @Override + public List getForwardedAddresses() { return forwardedAddresses; } + + @Override + public String getClientType() { return source.getClientType(); } + + @Override + public String getAction() { return source.getAction(); } + + @Override + public String getRequestData() { return source.getRequestData(); } + + @Override + public String getSessionId() { return source.getSessionId(); } + + @Override + public Map getContext() { return context; } + + @Override + public RangerAccessRequest getReadOnlyCopy() { return this; } + + @Override + public ResourceMatchingScope getResourceMatchingScope() { return source.getResourceMatchingScope(); } + + @Override + public String getClusterName() { return source.getClusterName(); } + + @Override + public String getClusterType() { return source.getClusterType(); } + +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerAccessResource.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerAccessResource.java new file mode 100644 index 00000000000..dec7ee8bf17 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerAccessResource.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.policyengine; + +import org.apache.atlas.plugin.model.RangerServiceDef; + +import java.util.Map; +import java.util.Set; + + +public interface RangerAccessResource { + String RESOURCE_SEP = "/"; + String RESOURCE_NAME_VAL_SEP = "="; + + String getOwnerUser(); + + boolean exists(String name); + + Object getValue(String name); + + RangerServiceDef getServiceDef(); + + Set getKeys(); + + String getLeafName(); + + String getAsString(); + + String getCacheKey(); + + Map getAsMap(); + + RangerAccessResource getReadOnlyCopy(); +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerAccessResourceImpl.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerAccessResourceImpl.java new file mode 100644 index 00000000000..6ac4787c65b --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerAccessResourceImpl.java @@ -0,0 +1,268 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.policyengine; + +import org.apache.commons.lang.ObjectUtils; +import org.apache.atlas.plugin.model.RangerServiceDef; +import org.apache.atlas.plugin.model.RangerServiceDef.RangerResourceDef; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class RangerAccessResourceImpl implements RangerMutableResource { + private String ownerUser; + private Map elements; + private String stringifiedValue; + private String stringifiedCacheKeyValue; + private String leafName; + private RangerServiceDef serviceDef; + + public RangerAccessResourceImpl() { + this(null, null); + } + + public RangerAccessResourceImpl(Map elements) { + this(elements, null); + } + + public RangerAccessResourceImpl(Map elements, String ownerUser) { + this.elements = elements; + this.ownerUser = ownerUser; + } + + @Override + public String getOwnerUser() { + return ownerUser; + } + + @Override + public boolean exists(String name) { + return elements != null && elements.containsKey(name); + } + + @Override + public Object getValue(String name) { + Object ret = null; + + if(elements != null && elements.containsKey(name)) { + ret = elements.get(name); + } + + return ret; + } + + @Override + public Set getKeys() { + Set ret = null; + + if(elements != null) { + ret = elements.keySet(); + } + + return ret; + } + + @Override + public void setOwnerUser(String ownerUser) { + this.ownerUser = ownerUser; + } + + @Override + public void setValue(String name, Object value) { + if(value == null) { + if(elements != null) { + elements.remove(name); + + if(elements.isEmpty()) { + elements = null; + } + } + } else { + if(elements == null) { + elements = new HashMap<>(); + } + elements.put(name, value); + } + + // reset, so that these will be computed again with updated elements + stringifiedValue = stringifiedCacheKeyValue = leafName = null; + } + + @Override + public void setServiceDef(final RangerServiceDef serviceDef) { + this.serviceDef = serviceDef; + this.stringifiedValue = this.stringifiedCacheKeyValue = this.leafName = null; + } + + @Override + public RangerServiceDef getServiceDef() { + return this.serviceDef; + } + + @Override + public String getLeafName() { + String ret = leafName; + + if(ret == null) { + if(serviceDef != null && serviceDef.getResources() != null) { + List resourceDefs = serviceDef.getResources(); + + for(int idx = resourceDefs.size() - 1; idx >= 0; idx--) { + RangerResourceDef resourceDef = resourceDefs.get(idx); + + if(resourceDef != null && exists(resourceDef.getName())) { + ret = leafName = resourceDef.getName(); + break; + } + } + } + } + + return ret; + } + + @Override + public String getAsString() { + String ret = stringifiedValue; + + if(ret == null) { + if(serviceDef != null && serviceDef.getResources() != null) { + StringBuilder sb = new StringBuilder(); + + for(RangerResourceDef resourceDef : serviceDef.getResources()) { + if(resourceDef == null || !exists(resourceDef.getName())) { + continue; + } + + if(sb.length() > 0) { + sb.append(RESOURCE_SEP); + } + + sb.append(getValue(resourceDef.getName())); + } + + if(sb.length() > 0) { + ret = stringifiedValue = sb.toString(); + } + } + } + + return ret; + } + + @Override + public String getCacheKey() { + String ret = stringifiedCacheKeyValue; + + if(ret == null) { + if(serviceDef != null && serviceDef.getResources() != null) { + StringBuilder sb = new StringBuilder(); + + for(RangerResourceDef resourceDef : serviceDef.getResources()) { + if(resourceDef == null || !exists(resourceDef.getName())) { + continue; + } + + if(sb.length() > 0) { + sb.append(RESOURCE_SEP); + } + + sb.append(resourceDef.getName()).append(RESOURCE_NAME_VAL_SEP).append(getValue(resourceDef.getName())); + } + + if(sb.length() > 0) { + ret = stringifiedCacheKeyValue = sb.toString(); + } + } + } + + return ret; + } + + @Override + public Map getAsMap() { + return elements == null ? Collections.EMPTY_MAP : Collections.unmodifiableMap(elements); + } + + @Override + public RangerAccessResource getReadOnlyCopy() { + return new RangerAccessResourceReadOnly(this); + } + + @Override + public boolean equals(Object obj) { + if(obj == null || !(obj instanceof RangerAccessResourceImpl)) { + return false; + } + + if(this == obj) { + return true; + } + + RangerAccessResourceImpl other = (RangerAccessResourceImpl) obj; + + return ObjectUtils.equals(ownerUser, other.ownerUser) && + ObjectUtils.equals(elements, other.elements); + } + + @Override + public int hashCode() { + int ret = 7; + + ret = 31 * ret + ObjectUtils.hashCode(ownerUser); + ret = 31 * ret + ObjectUtils.hashCode(elements); + + return ret; + } + + @Override + public String toString( ) { + StringBuilder sb = new StringBuilder(); + + toString(sb); + + return sb.toString(); + } + + public StringBuilder toString(StringBuilder sb) { + sb.append("RangerResourceImpl={"); + + sb.append("ownerUser={").append(ownerUser).append("} "); + + sb.append("elements={"); + if(elements != null) { + for(Map.Entry e : elements.entrySet()) { + sb.append(e.getKey()).append("=").append(e.getValue()).append("; "); + } + } + sb.append("} "); + + sb.append("}"); + + return sb; + } + + protected String getStringifiedValue() { return stringifiedValue; } + + protected void setStringifiedValue(String val) { this.stringifiedValue = val; } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerAccessResourceReadOnly.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerAccessResourceReadOnly.java new file mode 100644 index 00000000000..1556d15dc0f --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerAccessResourceReadOnly.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.policyengine; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.atlas.plugin.model.RangerServiceDef; + +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +public class RangerAccessResourceReadOnly implements RangerAccessResource { + + private final RangerAccessResource source; + private final Set keys; + private final Map map; + + public RangerAccessResourceReadOnly(final RangerAccessResource source) { + this.source = source; + + // Cached here for reducing access overhead + Set sourceKeys = source.getKeys(); + + if (CollectionUtils.isEmpty(sourceKeys)) { + sourceKeys = new HashSet<>(); + } + this.keys = Collections.unmodifiableSet(sourceKeys); + + Map sourceMap = source.getAsMap(); + + if (MapUtils.isEmpty(sourceMap)) { + sourceMap = new HashMap<>(); + } + this.map = Collections.unmodifiableMap(sourceMap); + } + + public String getOwnerUser() { return source.getOwnerUser(); } + + public boolean exists(String name) { return source.exists(name); } + + public Object getValue(String name) { return source.getValue(name); } + + public RangerServiceDef getServiceDef() { return source.getServiceDef(); } + + public Set getKeys() { return keys; } + + public String getLeafName() { return source.getLeafName(); } + + public String getAsString() { return source.getAsString(); } + + public String getCacheKey() { return source.getCacheKey(); } + + public Map getAsMap() { return map; } + + public RangerAccessResource getReadOnlyCopy() { return this; } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerAccessResult.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerAccessResult.java new file mode 100644 index 00000000000..ab6d5f037b1 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerAccessResult.java @@ -0,0 +1,382 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.atlas.plugin.policyengine; + +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.atlas.plugin.model.RangerPolicy; +import org.apache.atlas.plugin.model.RangerServiceDef; +import org.apache.atlas.plugin.policyevaluator.RangerPolicyItemEvaluator; +import org.apache.atlas.plugin.util.ServiceDefUtil; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class RangerAccessResult { + public final static String KEY_MASK_TYPE = "maskType"; + public final static String KEY_MASK_CONDITION = "maskCondition"; + public final static String KEY_MASKED_VALUE = "maskedValue"; + + private static String KEY_FILTER_EXPR = "filterExpr"; + + private final String serviceName; + private final RangerServiceDef serviceDef; + private final RangerAccessRequest request; + + private final String policyType; // Really, policy-type for audit purpose + private boolean isAccessDetermined; + private boolean isAllowed; + private boolean isAuditedDetermined; + private boolean isAudited; + private String auditPolicyId = "-1"; + private String auditLogId; + private String policyId = "-1"; + private int policyPriority; + private String zoneName; + private Long policyVersion; + private long evaluatedPoliciesCount; + private String reason; + private Map additionalInfo; + private List matchedItemEvaluators = new ArrayList<>(); + + public RangerAccessResult(final String policyType, final String serviceName, final RangerServiceDef serviceDef, final RangerAccessRequest request) { + this.serviceName = serviceName; + this.serviceDef = serviceDef; + this.request = request; + this.policyType = policyType; + this.isAccessDetermined = false; + this.isAllowed = false; + this.isAuditedDetermined = false; + this.isAudited = false; + this.auditPolicyId = "-1"; + this.policyId = "-1"; + this.zoneName = null; + this.policyVersion = null; + this.policyPriority = RangerPolicy.POLICY_PRIORITY_NORMAL; + this.evaluatedPoliciesCount = 0; + this.reason = null; + } + + public void setAccessResultFrom(final RangerAccessResult other) { + this.isAccessDetermined = other.getIsAccessDetermined(); + this.isAllowed = other.getIsAllowed(); + this.policyId = other.getPolicyId(); + this.policyPriority = other.getPolicyPriority(); + this.zoneName = other.zoneName; + this.policyVersion = other.policyVersion; + this.evaluatedPoliciesCount = other.evaluatedPoliciesCount; + this.reason = other.getReason(); + this.additionalInfo = other.additionalInfo == null ? new HashMap() : new HashMap<>(other.additionalInfo); + this.matchedItemEvaluators = new ArrayList<>(other.matchedItemEvaluators); + } + + public void setAuditResultFrom(final RangerAccessResult other) { + this.isAuditedDetermined = other.getIsAuditedDetermined(); + this.isAudited = other.getIsAudited(); + this.auditPolicyId = other.getAuditPolicyId(); + this.policyVersion = other.policyVersion; + } + + /** + * @return the serviceName + */ + public String getServiceName() { + return serviceName; + } + + /** + * @return the serviceDef + */ + public RangerServiceDef getServiceDef() { + return serviceDef; + } + + /** + * @return the request + */ + public RangerAccessRequest getAccessRequest() { + return request; + } + + public String getPolicyType() { return policyType; } + + public boolean getIsAccessDetermined() { return isAccessDetermined; } + + public void setIsAccessDetermined(boolean value) { isAccessDetermined = value; } + + /** + * @return the isAllowed + */ + public boolean getIsAllowed() { + return isAllowed; + } + + /** + * @param isAllowed the isAllowed to set + */ + public void setIsAllowed(boolean isAllowed) { + if(! isAllowed) { + setIsAccessDetermined(true); + } + + this.isAllowed = isAllowed; + } + public int getPolicyPriority() { return policyPriority;} + + public void setPolicyPriority(int policyPriority) { this.policyPriority = policyPriority; } + + public String getZoneName() { return zoneName; } + + public void setZoneName(String zoneName) { this.zoneName = zoneName; } + + public Long getPolicyVersion() { return policyVersion; } + + public void setPolicyVersion(Long policyVersion) { this.policyVersion = policyVersion; } + + /** + * @param reason the reason to set + */ + public void setReason(String reason) { + this.reason = reason; + } + + public boolean getIsAuditedDetermined() { return isAuditedDetermined; } + + public void setIsAuditedDetermined(boolean value) { isAuditedDetermined = value; } + + /** + * @return the isAudited + */ + public boolean getIsAudited() { + return isAudited; + } + + + + /** + * @param isAudited the isAudited to set + */ + public void setIsAudited(boolean isAudited) { + setIsAuditedDetermined(true); + this.isAudited = isAudited; + } + + /** + * @return the reason + */ + public String getReason() { + return reason; + } + + /** + * @return the policyId + */ + public String getPolicyId() { + return policyId; + } + + /** + * @return the auditPolicyId + */ + public String getAuditPolicyId() { + return auditPolicyId; + } + + public long getEvaluatedPoliciesCount() { return this.evaluatedPoliciesCount; } + /** + * @param policyId the policyId to set + */ + public void setPolicyId(String policyId) { + this.policyId = policyId; + } + + public String getAuditLogId() { + return auditLogId; + } + + public void setAuditLogId(String auditLogId) { + this.auditLogId = auditLogId; + } + + + /** + * @param policyId the auditPolicyId to set + */ + public void setAuditPolicyId(String policyId) { + this.auditPolicyId = policyId; + } + + public void incrementEvaluatedPoliciesCount() { this.evaluatedPoliciesCount++; } + + public int getServiceType() { + int ret = -1; + + if(serviceDef != null && serviceDef.getId() != null) { + ret = serviceDef.getId().intValue(); + } + + return ret; + } + + public void setAdditionalInfo(Map additionalInfo) { + this.additionalInfo = additionalInfo; + } + + public Map getAdditionalInfo() { + return additionalInfo; + } + + public void addAdditionalInfo(String key, Object value) { + if (additionalInfo == null) { + additionalInfo = new HashMap(); + } + additionalInfo.put(key, value); + } + + public void removeAdditionalInfo(String key) { + if (MapUtils.isNotEmpty(additionalInfo)) { + additionalInfo.remove(key); + } + } + + public List getMatchedItemEvaluators() { + return matchedItemEvaluators; + } + + public void setMatchedItemEvaluators(List matchedItemEvaluators) { + this.matchedItemEvaluators = matchedItemEvaluators; + } + + public void addMatchedItemEvaluator(RangerPolicyItemEvaluator matchItemEvaluator) { + this.matchedItemEvaluators.add(matchItemEvaluator); + } + + /** + * @return the maskType + */ + public String getMaskType() { + return additionalInfo == null ? null : (String) additionalInfo.get(KEY_MASK_TYPE); + } + + /** + * @param maskType the maskType to set + */ + public void setMaskType(String maskType) { + addAdditionalInfo(KEY_MASK_TYPE, maskType); + } + + /** + * @return the maskCondition + */ + public String getMaskCondition() { + return additionalInfo == null ? null : (String) additionalInfo.get(KEY_MASK_CONDITION); + } + + /** + * @param maskCondition the maskCondition to set + */ + public void setMaskCondition(String maskCondition) { + addAdditionalInfo(KEY_MASK_CONDITION, maskCondition); + } + + /** + * @return the maskedValue + */ + public String getMaskedValue() { + return additionalInfo == null ? null : (String) additionalInfo.get(KEY_MASKED_VALUE); + } + /** + * @param maskedValue the maskedValue to set + */ + public void setMaskedValue(String maskedValue) { + addAdditionalInfo(KEY_MASKED_VALUE, maskedValue); + } + + public boolean isMaskEnabled() { + return StringUtils.isNotEmpty(this.getMaskType()) && !StringUtils.equalsIgnoreCase(this.getMaskType(), RangerPolicy.MASK_TYPE_NONE); + } + + public RangerServiceDef.RangerDataMaskTypeDef getMaskTypeDef() { + RangerServiceDef.RangerDataMaskTypeDef ret = null; + + String maskType = getMaskType(); + if(StringUtils.isNotEmpty(maskType)) { + ret = ServiceDefUtil.getDataMaskType(getServiceDef(), maskType); + } + + return ret; + } + + /** + * @return the filterExpr + */ + public String getFilterExpr() { + return additionalInfo == null ? null : (String) additionalInfo.get(KEY_FILTER_EXPR); + } + + /** + * @param filterExpr the filterExpr to set + */ + public void setFilterExpr(String filterExpr) { + addAdditionalInfo(KEY_FILTER_EXPR, filterExpr); + } + + public boolean isRowFilterEnabled() { + return StringUtils.isNotEmpty(getFilterExpr()); + } + + @Override + public String toString( ) { + StringBuilder sb = new StringBuilder(); + + toString(sb); + + return sb.toString(); + } + + public StringBuilder toString(StringBuilder sb) { + sb.append("RangerAccessResult={"); + + sb.append("isAccessDetermined={").append(isAccessDetermined).append("} "); + sb.append("isAllowed={").append(isAllowed).append("} "); + sb.append("isAuditedDetermined={").append(isAuditedDetermined).append("} "); + sb.append("isAudited={").append(isAudited).append("} "); + sb.append("auditLogId={").append(auditLogId).append("} "); + sb.append("policyType={").append(policyType).append("} "); + sb.append("policyId={").append(policyId).append("} "); + sb.append("zoneName={").append(zoneName).append("} "); + sb.append("auditPolicyId={").append(auditPolicyId).append("} "); + sb.append("policyVersion={").append(policyVersion).append("} "); + sb.append("evaluatedPoliciesCount={").append(evaluatedPoliciesCount).append("} "); + sb.append("reason={").append(reason).append("} "); + sb.append("additionalInfo={"); + if (additionalInfo != null) { + for (Map.Entry entry : additionalInfo.entrySet()) { + sb.append(entry.getKey()).append("=").append(entry.getValue()).append(", "); + } + } + sb.append("}"); + + sb.append("}"); + + return sb; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerAccessResultProcessor.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerAccessResultProcessor.java new file mode 100644 index 00000000000..7b1374106ea --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerAccessResultProcessor.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.policyengine; + +import java.util.Collection; + + +public interface RangerAccessResultProcessor { + void processResult(RangerAccessResult result); + + void processResults(Collection results); +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerMutableResource.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerMutableResource.java new file mode 100644 index 00000000000..7b7e038b376 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerMutableResource.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.policyengine; + + +import org.apache.atlas.plugin.model.RangerServiceDef; + +public interface RangerMutableResource extends RangerAccessResource { + void setOwnerUser(String ownerUser); + + void setValue(String type, Object value); + void setServiceDef(RangerServiceDef serviceDef); +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerPluginContext.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerPluginContext.java new file mode 100644 index 00000000000..9abcb8bd95e --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerPluginContext.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.policyengine; + +import org.apache.atlas.authz.admin.client.AtlasAuthAdminClient; +import org.apache.atlas.authz.admin.client.AtlasAuthRESTClient; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.admin.client.RangerAdminRESTClient; +import org.apache.atlas.authorization.hadoop.config.RangerPluginConfig; +import org.apache.atlas.plugin.service.RangerAuthContext; +import org.apache.atlas.plugin.service.RangerAuthContextListener; + +public class RangerPluginContext { + private static final Log LOG = LogFactory.getLog(RangerPluginContext.class); + + private final RangerPluginConfig config; + private RangerAuthContext authContext; + private RangerAuthContextListener authContextListener; + private AtlasAuthAdminClient atlasAdminClient; + + + public RangerPluginContext(RangerPluginConfig config) { + this.config = config; + } + + public RangerPluginConfig getConfig() { return config; } + + public String getClusterName() { + return config.getClusterName(); + } + + public String getClusterType() { + return config.getClusterType(); + } + + public RangerAuthContext getAuthContext() { return authContext; } + + public void setAuthContext(RangerAuthContext authContext) { this.authContext = authContext; } + + public void setAuthContextListener(RangerAuthContextListener authContextListener) { this.authContextListener = authContextListener; } + + public void notifyAuthContextChanged() { + RangerAuthContextListener authContextListener = this.authContextListener; + + if (authContextListener != null) { + authContextListener.contextChanged(); + } + } + + public AtlasAuthAdminClient getAtlasAuthAdminClient() { + if (atlasAdminClient == null) { + atlasAdminClient = initAtlasAuthAdminClient(); + } + + return atlasAdminClient; + } + + private AtlasAuthAdminClient initAtlasAuthAdminClient() { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerBasePlugin.createAdminClient(" + config.getServiceName() + ", " + config.getAppId() + ", " + config.getPropertyPrefix() + ")"); + } + + AtlasAuthAdminClient ret = null; + String propertyName = config.getPropertyPrefix() + ".policy.source.impl"; + String policySourceImpl = config.get(propertyName); + + if(StringUtils.isEmpty(policySourceImpl)) { + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("Value for property[%s] was null or empty. Unexpected! Will use policy source of type[%s]", propertyName, RangerAdminRESTClient.class.getName())); + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("Value for property[%s] was [%s].", propertyName, policySourceImpl)); + } + + try { + @SuppressWarnings("unchecked") + Class adminClass = (Class)Class.forName(policySourceImpl); + + ret = adminClass.newInstance(); + } catch (Exception excp) { + LOG.error("failed to instantiate policy source of type '" + policySourceImpl + "'. Will use policy source of type '" + RangerAdminRESTClient.class.getName() + "'", excp); + } + } + + if(ret == null) { + ret = new AtlasAuthRESTClient(); + } + + ret.init(config); + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerBasePlugin.createAdminClient(" + config.getServiceName() + ", " + config.getAppId() + ", " + config.getPropertyPrefix() + "): policySourceImpl=" + policySourceImpl + ", client=" + ret); + } + + return ret; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerPolicyEngine.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerPolicyEngine.java new file mode 100644 index 00000000000..98852f910d1 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerPolicyEngine.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.policyengine; + +import org.apache.atlas.plugin.model.RangerPolicy; +import org.apache.atlas.plugin.model.RangerServiceDef; +import org.apache.atlas.plugin.util.GrantRevokeRequest; +import org.apache.atlas.plugin.util.RangerAccessRequestUtil; +import org.apache.atlas.plugin.util.RangerRoles; + +import java.util.Collection; +import java.util.List; +import java.util.Set; + +public interface RangerPolicyEngine { + String GROUP_PUBLIC = "public"; + String ANY_ACCESS = "_any"; + String ADMIN_ACCESS = "_admin"; + String SUPER_USER_ACCESS = "_super_user"; + + String AUDIT_ALL = "audit-all"; + String AUDIT_NONE = "audit-none"; + String AUDIT_DEFAULT = "audit-default"; + + String PLUGIN_AUDIT_EXCLUDE_USERS = "ranger.plugin.audit.exclude.users"; + String PLUGIN_AUDIT_EXCLUDE_GROUPS = "ranger.plugin.audit.exclude.groups"; + String PLUGIN_AUDIT_EXCLUDE_ROLES = "ranger.plugin.audit.exclude.roles"; + String PLUGIN_SUPER_USERS = "ranger.plugin.super.users"; + String PLUGIN_SUPER_GROUPS = "ranger.plugin.super.groups"; + String PLUGIN_AUDIT_FILTER = "ranger.plugin.audit.filters"; + String PLUGIN_SERVICE_ADMINS = "ranger.plugin.service.admins"; + + String USER_CURRENT = "{" + RangerAccessRequestUtil.KEY_USER + "}"; + String RESOURCE_OWNER = "{OWNER}"; + + void setUseForwardedIPAddress(boolean useForwardedIPAddress); + + void setTrustedProxyAddresses(String[] trustedProxyAddresses); + + RangerServiceDef getServiceDef(); + + long getPolicyVersion(); + + long getRoleVersion(); + + void setRoles(RangerRoles roles); + + RangerAccessResult evaluatePolicies(RangerAccessRequest request, String policyType, RangerAccessResultProcessor resultProcessor); + + Collection evaluatePolicies(Collection requests, String policyType, RangerAccessResultProcessor resultProcessor); + + void evaluateAuditPolicies(RangerAccessResult result); + + RangerResourceACLs getResourceACLs(RangerAccessRequest request); + + RangerResourceACLs getResourceACLs(RangerAccessRequest request, String requestedPolicyType); + + Set getRolesFromUserAndGroups(String user, Set groups); + + RangerRoles getRangerRoles(); + + RangerPluginContext getPluginContext(); + + String getUniquelyMatchedZoneName(GrantRevokeRequest grantRevokeRequest); + + // Helpers + + List getResourcePolicies(String zoneName); + + List getResourcePolicies(); + + List getTagPolicies(); + + // This API is used only used by test code + RangerResourceAccessInfo getResourceAccessInfo(RangerAccessRequest request); +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerPolicyEngineImpl.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerPolicyEngineImpl.java new file mode 100644 index 00000000000..90d18bf752f --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerPolicyEngineImpl.java @@ -0,0 +1,1351 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.policyengine; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.ListUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.atlas.authorization.hadoop.config.RangerPluginConfig; +import org.apache.atlas.authorization.utils.StringUtil; +import org.apache.atlas.plugin.contextenricher.RangerTagForEval; +import org.apache.atlas.plugin.model.RangerPolicy; +import org.apache.atlas.plugin.model.RangerPolicy.RangerPolicyItemDataMaskInfo; +import org.apache.atlas.plugin.model.RangerPolicy.RangerPolicyItemRowFilterInfo; +import org.apache.atlas.plugin.model.RangerServiceDef; +import org.apache.atlas.plugin.policyengine.RangerResourceACLs.DataMaskResult; +import org.apache.atlas.plugin.policyengine.RangerResourceACLs.RowFilterResult; +import org.apache.atlas.plugin.policyevaluator.RangerPolicyEvaluator; +import org.apache.atlas.plugin.policyevaluator.RangerPolicyEvaluator.PolicyACLSummary; +import org.apache.atlas.plugin.policyresourcematcher.RangerPolicyResourceMatcher.MatchType; +import org.apache.atlas.plugin.service.RangerDefaultRequestProcessor; +import org.apache.atlas.plugin.util.GrantRevokeRequest; +import org.apache.atlas.plugin.util.RangerAccessRequestUtil; +import org.apache.atlas.plugin.util.RangerCommonConstants; +import org.apache.atlas.plugin.util.RangerPerfTracer; +import org.apache.atlas.plugin.util.RangerReadWriteLock; +import org.apache.atlas.plugin.util.RangerRoles; +import org.apache.atlas.plugin.util.ServicePolicies; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.apache.atlas.plugin.policyevaluator.RangerPolicyEvaluator.ACCESS_CONDITIONAL; + +public class RangerPolicyEngineImpl implements RangerPolicyEngine { + private static final Log LOG = LogFactory.getLog(RangerPolicyEngineImpl.class); + + private static final Log PERF_POLICYENGINE_REQUEST_LOG = RangerPerfTracer.getPerfLogger("policyengine.request"); + private static final Log PERF_POLICYENGINE_AUDIT_LOG = RangerPerfTracer.getPerfLogger("policyengine.audit"); + private static final Log PERF_POLICYENGINE_GET_ACLS_LOG = RangerPerfTracer.getPerfLogger("policyengine.getResourceACLs"); + + private final PolicyEngine policyEngine; + private final RangerAccessRequestProcessor requestProcessor; + private final ServiceConfig serviceConfig; + + + static public RangerPolicyEngine getPolicyEngine(final RangerPolicyEngineImpl other, final ServicePolicies servicePolicies) { + RangerPolicyEngine ret = null; + + if (other != null && servicePolicies != null) { + PolicyEngine policyEngine = other.policyEngine.cloneWithDelta(servicePolicies); + + if (policyEngine != null) { + if (policyEngine == other.policyEngine) { + ret = other; + } else { + ret = new RangerPolicyEngineImpl(policyEngine, other); + } + } + } + + return ret; + } + + public RangerPolicyEngineImpl(ServicePolicies servicePolicies, RangerPluginContext pluginContext, RangerRoles roles) { + final boolean isUseReadWriteLock; + + Configuration config = pluginContext != null ? pluginContext.getConfig() : null; + + if (config != null) { + boolean isDeltasSupported = config.getBoolean(pluginContext.getConfig().getPropertyPrefix() + RangerCommonConstants.PLUGIN_CONFIG_SUFFIX_POLICY_DELTA, RangerCommonConstants.PLUGIN_CONFIG_SUFFIX_POLICY_DELTA_DEFAULT); + isUseReadWriteLock = isDeltasSupported && config.getBoolean(pluginContext.getConfig().getPropertyPrefix() + RangerCommonConstants.PLUGIN_CONFIG_SUFFIX_IN_PLACE_POLICY_UPDATES, RangerCommonConstants.PLUGIN_CONFIG_SUFFIX_IN_PLACE_POLICY_UPDATES_DEFAULT); + } else { + isUseReadWriteLock = false; + } + + policyEngine = new PolicyEngine(servicePolicies, pluginContext, roles, isUseReadWriteLock); + serviceConfig = new ServiceConfig(servicePolicies.getServiceConfig()); + requestProcessor = new RangerDefaultRequestProcessor(policyEngine); + } + + @Override + public String toString() { + return policyEngine.toString(); + } + + @Override + public RangerAccessResult evaluatePolicies(RangerAccessRequest request, String policyType, RangerAccessResultProcessor resultProcessor) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerPolicyEngineImpl.evaluatePolicies(" + request + ", policyType=" + policyType + ")"); + } + + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_POLICYENGINE_REQUEST_LOG)) { + String requestHashCode = Integer.toHexString(System.identityHashCode(request)) + "_" + policyType; + + perf = RangerPerfTracer.getPerfTracer(PERF_POLICYENGINE_REQUEST_LOG, "RangerPolicyEngine.evaluatePolicies(requestHashCode=" + requestHashCode + ")"); + } + + RangerAccessResult ret; + + try (RangerReadWriteLock.RangerLock readLock = policyEngine.getReadLock()) { + if (LOG.isDebugEnabled()) { + if (readLock.isLockingEnabled()) { + LOG.debug("Acquired lock - " + readLock); + } + } + + requestProcessor.preProcess(request); + + ret = zoneAwareAccessEvaluationWithNoAudit(request, policyType); + + if (resultProcessor != null) { + RangerPerfTracer perfAuditTracer = null; + + if (RangerPerfTracer.isPerfTraceEnabled(PERF_POLICYENGINE_AUDIT_LOG)) { + String requestHashCode = Integer.toHexString(System.identityHashCode(request)) + "_" + policyType; + + perfAuditTracer = RangerPerfTracer.getPerfTracer(PERF_POLICYENGINE_AUDIT_LOG, "RangerPolicyEngine.processAudit(requestHashCode=" + requestHashCode + ")"); + } + + resultProcessor.processResult(ret); + + RangerPerfTracer.log(perfAuditTracer); + } + } + + RangerPerfTracer.log(perf); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerPolicyEngineImpl.evaluatePolicies(" + request + ", policyType=" + policyType + "): " + ret); + } + + return ret; + } + + @Override + public Collection evaluatePolicies(Collection requests, String policyType, RangerAccessResultProcessor resultProcessor) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerPolicyEngineImpl.evaluatePolicies(" + requests + ", policyType=" + policyType + ")"); + } + + Collection ret = new ArrayList<>(); + + try (RangerReadWriteLock.RangerLock readLock = policyEngine.getReadLock()) { + if (LOG.isDebugEnabled()) { + if (readLock.isLockingEnabled()) { + LOG.debug("Acquired lock - " + readLock); + } + } + if (requests != null) { + for (RangerAccessRequest request : requests) { + requestProcessor.preProcess(request); + + RangerAccessResult result = zoneAwareAccessEvaluationWithNoAudit(request, policyType); + + ret.add(result); + } + } + + if (resultProcessor != null) { + resultProcessor.processResults(ret); + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerPolicyEngineImpl.evaluatePolicies(" + requests + ", policyType=" + policyType + "): " + ret); + } + + return ret; + } + + @Override + public void evaluateAuditPolicies(RangerAccessResult result) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerPolicyEngineImpl.evaluateAuditPolicies(result=" + result + ")"); + } + + try (RangerReadWriteLock.RangerLock readLock = policyEngine.getReadLock()) { + if (LOG.isDebugEnabled()) { + if (readLock.isLockingEnabled()) { + LOG.debug("Acquired lock - " + readLock); + } + } + + RangerPolicyRepository tagPolicyRepository = policyEngine.getTagPolicyRepository(); + RangerPolicyRepository policyRepository = policyEngine.getPolicyRepository(); + RangerAccessRequest request = result.getAccessRequest(); + boolean savedIsAuditedDetermined = result.getIsAuditedDetermined(); + boolean savedIsAudited = result.getIsAudited(); + + result.setIsAudited(false); + result.setIsAuditedDetermined(false); + + try { + if (tagPolicyRepository != null) { + evaluateTagAuditPolicies(request, result, tagPolicyRepository); + } + + if (!result.getIsAuditedDetermined() && policyRepository != null) { + evaluateResourceAuditPolicies(request, result, policyRepository); + } + } finally { + if (!result.getIsAuditedDetermined()) { + result.setIsAudited(savedIsAudited); + result.setIsAuditedDetermined(savedIsAuditedDetermined); + } + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerPolicyEngineImpl.evaluateAuditPolicies(result=" + result + ")"); + } + } + + @Override + public RangerResourceACLs getResourceACLs(RangerAccessRequest request) { + return getResourceACLs(request, null); + } + + @Override + public RangerResourceACLs getResourceACLs(RangerAccessRequest request, String requestedPolicyType) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerPolicyEngineImpl.getResourceACLs(request=" + request + ", policyType=" + requestedPolicyType + ")"); + } + + RangerResourceACLs ret = new RangerResourceACLs(); + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_POLICYENGINE_GET_ACLS_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_POLICYENGINE_GET_ACLS_LOG, "RangerPolicyEngine.getResourceACLs(requestHashCode=" + request.getResource().getAsString() + ")"); + } + + try (RangerReadWriteLock.RangerLock readLock = policyEngine.getReadLock()) { + if (LOG.isDebugEnabled()) { + if (readLock.isLockingEnabled()) { + LOG.debug("Acquired lock - " + readLock); + } + } + + requestProcessor.preProcess(request); + + String zoneName = policyEngine.getUniquelyMatchedZoneName(request.getResource().getAsMap()); + + if (LOG.isDebugEnabled()) { + LOG.debug("zoneName:[" + zoneName + "]"); + } + + String[] policyTypes = StringUtils.isEmpty(requestedPolicyType) ? RangerPolicy.POLICY_TYPES : new String[] { requestedPolicyType }; + + + for (String policyType : policyTypes) { + List allEvaluators = new ArrayList<>(); + Map tagMatchTypeMap = new HashMap<>(); + Set policyIdForTemporalTags = new HashSet<>(); + + getResourceACLEvaluatorsForZone(request, zoneName, policyType, allEvaluators, tagMatchTypeMap, policyIdForTemporalTags); + + allEvaluators.sort(RangerPolicyEvaluator.EVAL_ORDER_COMPARATOR); + + if (CollectionUtils.isEmpty(allEvaluators)) { + continue; + } + + Integer policyPriority = null; + + for (RangerPolicyEvaluator evaluator : allEvaluators) { + if (policyPriority == null) { + policyPriority = evaluator.getPolicyPriority(); + } + + if (policyPriority != evaluator.getPolicyPriority()) { + if (RangerPolicy.POLICY_TYPE_ACCESS.equals(policyType)) { + ret.finalizeAcls(); + } + + policyPriority = evaluator.getPolicyPriority(); + } + + MatchType matchType = tagMatchTypeMap.get(evaluator.getId()); + + if (matchType == null) { + matchType = evaluator.getPolicyResourceMatcher().getMatchType(request.getResource(), request.getContext()); + } + + final boolean isMatched; + + if (request.getResourceMatchingScope() == RangerAccessRequest.ResourceMatchingScope.SELF_OR_DESCENDANTS) { + isMatched = matchType != MatchType.NONE; + } else { + isMatched = matchType == MatchType.SELF || matchType == MatchType.SELF_AND_ALL_DESCENDANTS; + } + + if (!isMatched) { + continue; + } + + if (RangerPolicy.POLICY_TYPE_ACCESS.equals(policyType)) { + updateFromPolicyACLs(evaluator, policyIdForTemporalTags, ret); + } else if (RangerPolicy.POLICY_TYPE_ROWFILTER.equals(policyType)) { + updateRowFiltersFromPolicy(evaluator, policyIdForTemporalTags, ret); + } else if (RangerPolicy.POLICY_TYPE_DATAMASK.equals(policyType)) { + updateDataMasksFromPolicy(evaluator, policyIdForTemporalTags, ret); + } + } + + ret.finalizeAcls(); + } + } + + RangerPerfTracer.logAlways(perf); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerPolicyEngineImpl.getResourceACLs(request=" + request + ", policyType=" + requestedPolicyType + ") : ret=" + ret); + } + + return ret; + } + + @Override + public void setUseForwardedIPAddress(boolean useForwardedIPAddress) { + try (RangerReadWriteLock.RangerLock writeLock = policyEngine.getWriteLock()) { + if (LOG.isDebugEnabled()) { + if (writeLock.isLockingEnabled()) { + LOG.debug("Acquired lock - " + writeLock); + } + } + policyEngine.setUseForwardedIPAddress(useForwardedIPAddress); + } + } + + @Override + public void setTrustedProxyAddresses(String[] trustedProxyAddresses) { + try (RangerReadWriteLock.RangerLock writeLock = policyEngine.getWriteLock()) { + if (LOG.isDebugEnabled()) { + if (writeLock.isLockingEnabled()) { + LOG.debug("Acquired lock - " + writeLock); + } + } + policyEngine.setTrustedProxyAddresses(trustedProxyAddresses); + } + } + + @Override + public RangerServiceDef getServiceDef() { + final RangerServiceDef ret; + + try (RangerReadWriteLock.RangerLock readLock = policyEngine.getReadLock()) { + if (LOG.isDebugEnabled()) { + if (readLock.isLockingEnabled()) { + LOG.debug("Acquired lock - " + readLock); + } + } + ret = policyEngine.getServiceDef(); + } + return ret; + } + + @Override + public long getPolicyVersion() { + long ret; + + try (RangerReadWriteLock.RangerLock readLock = policyEngine.getReadLock()) { + if (LOG.isDebugEnabled()) { + if (readLock.isLockingEnabled()) { + LOG.debug("Acquired lock - " + readLock); + } + } + ret = policyEngine.getPolicyVersion(); + } + return ret; + } + + @Override + public long getRoleVersion() { return policyEngine.getRoleVersion(); } + + @Override + public void setRoles(RangerRoles roles) { + try (RangerReadWriteLock.RangerLock writeLock = policyEngine.getWriteLock()) { + if (LOG.isDebugEnabled()) { + if (writeLock.isLockingEnabled()) { + LOG.debug("Acquired lock - " + writeLock); + } + } + policyEngine.setRoles(roles); + } + + } + + @Override + public Set getRolesFromUserAndGroups(String user, Set groups) { + Set ret; + try (RangerReadWriteLock.RangerLock readLock = policyEngine.getReadLock()) { + if (LOG.isDebugEnabled()) { + if (readLock.isLockingEnabled()) { + LOG.debug("Acquired lock - " + readLock); + } + } + ret = policyEngine.getPluginContext().getAuthContext().getRolesForUserAndGroups(user, groups); + } + return ret; + } + + @Override + public RangerRoles getRangerRoles() { + return policyEngine.getPluginContext().getAuthContext().getRangerRolesUtil().getRoles(); + } + + @Override + public RangerPluginContext getPluginContext() { + return policyEngine.getPluginContext(); + } + + @Override + public String getUniquelyMatchedZoneName(GrantRevokeRequest grantRevokeRequest) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerPolicyEngineImpl.getUniquelyMatchedZoneName(" + grantRevokeRequest + ")"); + } + + String ret; + + try (RangerReadWriteLock.RangerLock readLock = policyEngine.getReadLock()) { + if (LOG.isDebugEnabled()) { + if (readLock.isLockingEnabled()) { + LOG.debug("Acquired lock - " + readLock); + } + } + ret = policyEngine.getUniquelyMatchedZoneName(grantRevokeRequest.getResource()); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerPolicyEngineImpl.getUniquelyMatchedZoneName(" + grantRevokeRequest + ") : " + ret); + } + + return ret; + } + + @Override + public List getResourcePolicies(String zoneName) { + List ret; + + try (RangerReadWriteLock.RangerLock readLock = policyEngine.getReadLock()) { + if (LOG.isDebugEnabled()) { + if (readLock.isLockingEnabled()) { + LOG.debug("Acquired lock - " + readLock); + } + } + List oldPolicies = policyEngine.getResourcePolicies(zoneName); + ret = CollectionUtils.isNotEmpty(oldPolicies) ? new ArrayList<>(oldPolicies) : oldPolicies; + } + + return ret; + } + + @Override + public List getResourcePolicies() { + List ret; + + try (RangerReadWriteLock.RangerLock readLock = policyEngine.getReadLock()) { + if (LOG.isDebugEnabled()) { + if (readLock.isLockingEnabled()) { + LOG.debug("Acquired lock - " + readLock); + } + } + RangerPolicyRepository policyRepository = policyEngine.getPolicyRepository(); + List oldPolicies = policyRepository == null ? ListUtils.EMPTY_LIST : policyRepository.getPolicies(); + ret = CollectionUtils.isNotEmpty(oldPolicies) ? new ArrayList<>(oldPolicies) : oldPolicies; + } + return ret; + } + + @Override + public List getTagPolicies() { + List ret; + + try (RangerReadWriteLock.RangerLock readLock = policyEngine.getReadLock()) { + if (LOG.isDebugEnabled()) { + if (readLock.isLockingEnabled()) { + LOG.debug("Acquired lock - " + readLock); + } + } + RangerPolicyRepository tagPolicyRepository = policyEngine.getTagPolicyRepository(); + List oldPolicies = tagPolicyRepository == null ? ListUtils.EMPTY_LIST : tagPolicyRepository.getPolicies(); + ret = CollectionUtils.isNotEmpty(oldPolicies) ? new ArrayList<>(oldPolicies) : oldPolicies; + } + return ret; + } + + // This API is used only used by test code + @Override + public RangerResourceAccessInfo getResourceAccessInfo(RangerAccessRequest request) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerPolicyEngineImpl.getResourceAccessInfo(" + request + ")"); + } + + requestProcessor.preProcess(request); + + RangerResourceAccessInfo ret = new RangerResourceAccessInfo(request); + Set zoneNames = policyEngine.getMatchedZonesForResourceAndChildren(request.getResource()); + + if (LOG.isDebugEnabled()) { + LOG.debug("zoneNames:[" + zoneNames + "]"); + } + + if (CollectionUtils.isEmpty(zoneNames)) { + getResourceAccessInfoForZone(request, ret, null); + } else { + for (String zoneName : zoneNames) { + getResourceAccessInfoForZone(request, ret, zoneName); + + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerPolicyEngineImpl.getResourceAccessInfo(" + request + "): " + ret); + } + + return ret; + } + + public void releaseResources(boolean isForced) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerPolicyEngineImpl.releaseResources(isForced=" + isForced + ")"); + } + + PolicyEngine policyEngine = this.policyEngine; + + if (policyEngine != null) { + policyEngine.preCleanup(isForced); + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Cannot preCleanup policy-engine as it is null!"); + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerPolicyEngineImpl.releaseResources(isForced=" + isForced + ")"); + } + } + + public boolean isServiceAdmin(String userName) { + boolean ret = serviceConfig.isServiceAdmin(userName); + + if (!ret) { + + RangerPluginConfig pluginConfig = policyEngine.getPluginContext().getConfig(); + + ret = pluginConfig.isServiceAdmin(userName); + } + + return ret; + } + + PolicyEngine getPolicyEngine() { + return policyEngine; + } + + private RangerPolicyEngineImpl(final PolicyEngine policyEngine, RangerPolicyEngineImpl other) { + this.policyEngine = policyEngine; + this.requestProcessor = new RangerDefaultRequestProcessor(policyEngine); + this.serviceConfig = new ServiceConfig(other.serviceConfig); + } + + private RangerAccessResult zoneAwareAccessEvaluationWithNoAudit(RangerAccessRequest request, String policyType) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerPolicyEngineImpl.zoneAwareAccessEvaluationWithNoAudit(" + request + ", policyType =" + policyType + ")"); + } + + RangerAccessResult ret = null; + RangerPolicyRepository policyRepository = policyEngine.getPolicyRepository(); + RangerPolicyRepository tagPolicyRepository = policyEngine.getTagPolicyRepository(); + Set zoneNames = policyEngine.getMatchedZonesForResourceAndChildren(request.getResource()); // Evaluate zone-name from request + + if (LOG.isDebugEnabled()) { + LOG.debug("zoneNames:[" + zoneNames + "]"); + } + + if (CollectionUtils.isEmpty(zoneNames) || (zoneNames.size() > 1 && !request.isAccessTypeAny())) { + // Evaluate default policies + policyRepository = policyEngine.getRepositoryForZone(null); + + ret = evaluatePoliciesNoAudit(request, policyType, null, policyRepository, tagPolicyRepository); + + ret.setZoneName(null); + } else if (zoneNames.size() == 1 || request.isAccessTypeAny()) { + // Evaluate zone specific policies + for (String zoneName : zoneNames) { + policyRepository = policyEngine.getRepositoryForZone(zoneName); + + ret = evaluatePoliciesNoAudit(request, policyType, zoneName, policyRepository, tagPolicyRepository); + ret.setZoneName(zoneName); + + if (ret.getIsAllowed()) { + if (LOG.isDebugEnabled()) { + LOG.debug("Zone:[" + zoneName + "] allowed access. Completed processing other zones"); + } + break; + } + } + } + + if (request.isAccessTypeAny() && (request.getResource() == null || CollectionUtils.isEmpty(request.getResource().getKeys())) && ret != null && !ret.getIsAllowed() && MapUtils.isNotEmpty(policyEngine.getZonePolicyRepositories())) { + // resource is empty and access is ANY + if (LOG.isDebugEnabled()) { + LOG.debug("Process all security-zones"); + } + RangerAccessResult accessResult; + + for (Map.Entry entry : policyEngine.getZonePolicyRepositories().entrySet()) { + String someZone = entry.getKey(); + policyRepository = entry.getValue(); + + if (LOG.isDebugEnabled()) { + LOG.debug("Evaluating policies for zone:[" + someZone + "]"); + } + + if (policyRepository != null) { + accessResult = evaluatePoliciesNoAudit(request, policyType, someZone, policyRepository, tagPolicyRepository); + + if (accessResult.getIsAllowed()) { + if (LOG.isDebugEnabled()) { + LOG.debug("Zone:[" + someZone + "] allowed access. Completed processing other zones"); + } + accessResult.setZoneName(someZone); + ret = accessResult; + break; + } + } + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerPolicyEngineImpl.zoneAwareAccessEvaluationWithNoAudit(" + request + ", policyType =" + policyType + "): " + ret); + } + + return ret; + } + + private RangerAccessResult evaluatePoliciesNoAudit(RangerAccessRequest request, String policyType, String zoneName, RangerPolicyRepository policyRepository, RangerPolicyRepository tagPolicyRepository) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerPolicyEngineImpl.evaluatePoliciesNoAudit(" + request + ", policyType =" + policyType + ", zoneName=" + zoneName + ")"); + } + + final RangerAccessResult ret; + + if (request.isAccessTypeAny()) { + RangerAccessResult denyResult = null; + RangerAccessResult allowResult = null; + + ret = createAccessResult(request, policyType); + + List allAccessDefs = getServiceDef().getAccessTypes(); + + for (RangerServiceDef.RangerAccessTypeDef accessTypeDef : allAccessDefs) { + RangerAccessRequestImpl requestForOneAccessType = new RangerAccessRequestImpl(request); + RangerAccessRequestUtil.setIsAnyAccessInContext(requestForOneAccessType.getContext(), Boolean.TRUE); + + requestForOneAccessType.setAccessType(accessTypeDef.getName()); + + RangerAccessResult resultForOneAccessType = evaluatePoliciesForOneAccessTypeNoAudit(requestForOneAccessType, policyType, zoneName, policyRepository, tagPolicyRepository); + + ret.setAuditResultFrom(resultForOneAccessType); + + if (resultForOneAccessType.getIsAccessDetermined()) { + if (resultForOneAccessType.getIsAllowed()) { + allowResult = resultForOneAccessType; + break; + } else if (denyResult == null) { + denyResult = resultForOneAccessType; + } + } + } + + if (allowResult != null) { + ret.setAccessResultFrom(allowResult); + } else if (denyResult != null) { + ret.setAccessResultFrom(denyResult); + } + } else { + ret = evaluatePoliciesForOneAccessTypeNoAudit(request, policyType, zoneName, policyRepository, tagPolicyRepository); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerPolicyEngineImpl.evaluatePoliciesNoAudit(" + request + ", policyType =" + policyType + ", zoneName=" + zoneName + "): " + ret); + } + + return ret; + } + + private RangerAccessResult evaluatePoliciesForOneAccessTypeNoAudit(RangerAccessRequest request, String policyType, String zoneName, RangerPolicyRepository policyRepository, RangerPolicyRepository tagPolicyRepository) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerPolicyEngineImpl.evaluatePoliciesForOneAccessTypeNoAudit(" + request + ", policyType =" + policyType + ", zoneName=" + zoneName + ")"); + } + + final boolean isSuperUser = isSuperUser(request.getUser(), request.getUserGroups()); + final Date accessTime = request.getAccessTime() != null ? request.getAccessTime() : new Date(); + final RangerAccessResult ret = createAccessResult(request, policyType); + + if (isSuperUser || StringUtils.equals(request.getAccessType(), RangerPolicyEngine.SUPER_USER_ACCESS)) { + ret.setIsAllowed(isSuperUser); + ret.setIsAccessDetermined(true); + ret.setPolicyId("-1"); + ret.setPolicyPriority(Integer.MAX_VALUE); + ret.setReason("superuser"); + } + + evaluateTagPolicies(request, policyType, zoneName, tagPolicyRepository, ret); + + if (LOG.isDebugEnabled()) { + if (ret.getIsAccessDetermined() && ret.getIsAuditedDetermined()) { + if (!ret.getIsAllowed()) { + LOG.debug("RangerPolicyEngineImpl.evaluatePoliciesNoAudit() - audit determined and access denied by a tag policy. Higher priority resource policies will be evaluated to check for allow, request=" + request + ", result=" + ret); + } else { + LOG.debug("RangerPolicyEngineImpl.evaluatePoliciesNoAudit() - audit determined and access allowed by a tag policy. Same or higher priority resource policies will be evaluated to check for deny, request=" + request + ", result=" + ret); + } + } + } + + boolean isAllowedByTags = ret.getIsAccessDetermined() && ret.getIsAllowed(); + boolean isDeniedByTags = ret.getIsAccessDetermined() && !ret.getIsAllowed(); + boolean evaluateResourcePolicies = policyEngine.hasResourcePolicies(policyRepository); + + if (evaluateResourcePolicies) { + boolean findAuditByResource = !ret.getIsAuditedDetermined(); + boolean foundInCache = findAuditByResource && policyRepository.setAuditEnabledFromCache(request, ret); + + ret.setIsAccessDetermined(false); // discard result by tag-policies, to evaluate resource policies for possible override + + List evaluators = policyRepository.getLikelyMatchPolicyEvaluators(request, policyType); + + for (RangerPolicyEvaluator evaluator : evaluators) { + if (!evaluator.isApplicable(accessTime)) { + continue; + } + + if (isDeniedByTags) { + if (ret.getPolicyPriority() >= evaluator.getPolicyPriority()) { + ret.setIsAccessDetermined(true); + } + } else if (ret.getIsAllowed()) { + if (ret.getPolicyPriority() > evaluator.getPolicyPriority()) { + ret.setIsAccessDetermined(true); + } + } + + ret.incrementEvaluatedPoliciesCount(); + evaluator.evaluate(request, ret); + + if (ret.getIsAllowed()) { + if (!evaluator.hasDeny()) { // No more deny policies left + ret.setIsAccessDetermined(true); + } + } + + if (ret.getIsAuditedDetermined() && ret.getIsAccessDetermined() && !request.isAccessorsRequested()) { + break; + } + } + + if (!ret.getIsAccessDetermined()) { + if (isDeniedByTags) { + ret.setIsAllowed(false); + } else if (isAllowedByTags) { + ret.setIsAllowed(true); + } + if (!ret.getIsAllowed() && + !getIsFallbackSupported()) { + ret.setIsAccessDetermined(true); + } + } + + if (ret.getIsAllowed()) { + ret.setIsAccessDetermined(true); + } + + if (findAuditByResource && !foundInCache) { + policyRepository.storeAuditEnabledInCache(request, ret); + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerPolicyEngineImpl.evaluatePoliciesForOneAccessTypeNoAudit(" + request + ", policyType =" + policyType + ", zoneName=" + zoneName + "): " + ret); + } + + return ret; + } + + private void evaluateTagPolicies(final RangerAccessRequest request, String policyType, String zoneName, RangerPolicyRepository tagPolicyRepository, RangerAccessResult result) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerPolicyEngineImpl.evaluateTagPolicies(" + request + ", policyType =" + policyType + ", zoneName=" + zoneName + ", " + result + ")"); + } + + Date accessTime = request.getAccessTime() != null ? request.getAccessTime() : new Date(); + Set tags = RangerAccessRequestUtil.getRequestTagsFromContext(request.getContext()); + List policyEvaluators = tagPolicyRepository == null ? null : tagPolicyRepository.getLikelyMatchPolicyEvaluators(request, tags, policyType, accessTime); + + if (CollectionUtils.isNotEmpty(policyEvaluators)) { + final boolean useTagPoliciesFromDefaultZone = !policyEngine.isResourceZoneAssociatedWithTagService(zoneName); + + for (PolicyEvaluatorForTag policyEvaluator : policyEvaluators) { + RangerPolicyEvaluator evaluator = policyEvaluator.getEvaluator(); + String policyZoneName = evaluator.getPolicy().getZoneName(); + + if (useTagPoliciesFromDefaultZone) { + if (StringUtils.isNotEmpty(policyZoneName)) { + if (LOG.isDebugEnabled()) { + LOG.debug("Tag policy [zone:" + policyZoneName + "] does not belong to default zone. Not evaluating this policy:[" + evaluator.getPolicy() + "]"); + } + + continue; + } + } else { + if (!StringUtils.equals(zoneName, policyZoneName)) { + if (LOG.isDebugEnabled()) { + LOG.debug("Tag policy [zone:" + policyZoneName + "] does not belong to the zone:[" + zoneName + "] of the accessed resource. Not evaluating this policy:[" + evaluator.getPolicy() + "]"); + } + + continue; + } + } + + RangerTagForEval tag = policyEvaluator.getTag(); + RangerAccessRequest tagEvalRequest = new RangerTagAccessRequest(tag, tagPolicyRepository.getServiceDef(), request); + RangerAccessResult tagEvalResult = createAccessResult(tagEvalRequest, policyType); + + if (LOG.isDebugEnabled()) { + LOG.debug("RangerPolicyEngineImpl.evaluateTagPolicies: Evaluating policies for tag (" + tag.getType() + ")"); + } + + tagEvalResult.setAccessResultFrom(result); + tagEvalResult.setAuditResultFrom(result); + + result.incrementEvaluatedPoliciesCount(); + + evaluator.evaluate(tagEvalRequest, tagEvalResult); + + if (tagEvalResult.getIsAllowed()) { + if (!evaluator.hasDeny()) { // No Deny policies left now + tagEvalResult.setIsAccessDetermined(true); + } + } + + if (tagEvalResult.getIsAudited()) { + result.setAuditResultFrom(tagEvalResult); + } + + if (!result.getIsAccessDetermined()) { + if (tagEvalResult.getIsAccessDetermined()) { + result.setAccessResultFrom(tagEvalResult); + } else { + if (!result.getIsAllowed() && tagEvalResult.getIsAllowed()) { + result.setAccessResultFrom(tagEvalResult); + } + } + } + + if (result.getIsAuditedDetermined() && result.getIsAccessDetermined() && !request.isAccessorsRequested()) { + break; // Break out of policy-evaluation loop + } + } + } + + if (result.getIsAllowed()) { + result.setIsAccessDetermined(true); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerPolicyEngineImpl.evaluateTagPolicies(" + request + ", policyType =" + policyType + ", zoneName=" + zoneName + ", " + result + ")"); + } + } + + private RangerAccessResult createAccessResult(RangerAccessRequest request, String policyType) { + RangerPolicyRepository repository = policyEngine.getPolicyRepository(); + RangerAccessResult ret = new RangerAccessResult(policyType, repository.getServiceName(), repository.getServiceDef(), request); + + switch (repository.getAuditModeEnum()) { + case AUDIT_ALL: + ret.setIsAudited(true); + break; + + case AUDIT_NONE: + ret.setIsAudited(false); + break; + + default: + if (CollectionUtils.isEmpty(repository.getPolicies()) && policyEngine.getTagPolicyRepository() == null) { + ret.setIsAudited(true); + } + + break; + } + + if (isAuditExcludedUser(request.getUser(), request.getUserGroups(), RangerAccessRequestUtil.getCurrentUserRolesFromContext(request.getContext()))) { + ret.setIsAudited(false); + } + + return ret; + } + + private boolean isAuditExcludedUser(String userName, Set userGroups, Set userRoles) { + boolean ret = serviceConfig.isAuditExcludedUser(userName); + + if (!ret) { + RangerPluginConfig pluginConfig = policyEngine.getPluginContext().getConfig(); + + ret = pluginConfig.isAuditExcludedUser(userName); + + if (!ret && userGroups != null && userGroups.size() > 0) { + ret = serviceConfig.hasAuditExcludedGroup(userGroups) || pluginConfig.hasAuditExcludedGroup(userGroups); + } + + if (!ret && userRoles != null && userRoles.size() > 0) { + ret = serviceConfig.hasAuditExcludedRole(userRoles) || pluginConfig.hasAuditExcludedRole(userRoles); + } + } + + return ret; + } + + private boolean isSuperUser(String userName, Set userGroups) { + boolean ret = serviceConfig.isSuperUser(userName); + + if (!ret) { + RangerPluginConfig pluginConfig = policyEngine.getPluginContext().getConfig(); + + ret = pluginConfig.isSuperUser(userName); + + if (!ret && userGroups != null && userGroups.size() > 0) { + ret = serviceConfig.hasSuperGroup(userGroups) || pluginConfig.hasSuperGroup(userGroups); + } + } + + return ret; + } + + private void getResourceACLEvaluatorsForZone(RangerAccessRequest request, String zoneName, String policyType, List allEvaluators, Map tagMatchTypeMap, Set policyIdForTemporalTags) { + final RangerPolicyRepository matchedRepository = policyEngine.getRepositoryForZone(zoneName); + + if (matchedRepository == null) { + LOG.error("policyRepository for zoneName:[" + zoneName + "], serviceName:[" + policyEngine.getPolicyRepository().getServiceName() + "], policyVersion:[" + getPolicyVersion() + "] is null!! ERROR!"); + } else { + Set tags = RangerAccessRequestUtil.getRequestTagsFromContext(request.getContext()); + List tagPolicyEvaluators = policyEngine.getTagPolicyRepository() == null ? null : policyEngine.getTagPolicyRepository().getLikelyMatchPolicyEvaluators(request, tags, policyType, null); + + if (CollectionUtils.isNotEmpty(tagPolicyEvaluators)) { + + final boolean useTagPoliciesFromDefaultZone = !policyEngine.isResourceZoneAssociatedWithTagService(zoneName); + + for (PolicyEvaluatorForTag tagEvaluator : tagPolicyEvaluators) { + RangerPolicyEvaluator evaluator = tagEvaluator.getEvaluator(); + String policyZoneName = evaluator.getPolicy().getZoneName(); + + if (useTagPoliciesFromDefaultZone) { + if (StringUtils.isNotEmpty(policyZoneName)) { + if (LOG.isDebugEnabled()) { + LOG.debug("Tag policy [zone:" + policyZoneName + "] does not belong to default zone. Not evaluating this policy:[" + evaluator.getPolicy() + "]"); + } + + continue; + } + } else { + if (!StringUtils.equals(zoneName, policyZoneName)) { + if (LOG.isDebugEnabled()) { + LOG.debug("Tag policy [zone:" + policyZoneName + "] does not belong to the zone:[" + zoneName + "] of the accessed resource. Not evaluating this policy:[" + evaluator.getPolicy() + "]"); + } + + continue; + } + } + + RangerTagForEval tag = tagEvaluator.getTag(); + + allEvaluators.add(evaluator); + tagMatchTypeMap.put(evaluator.getId(), tag.getMatchType()); + + if (CollectionUtils.isNotEmpty(tag.getValidityPeriods())) { + policyIdForTemporalTags.add(evaluator.getId()); + } + } + } + + List resourcePolicyEvaluators = matchedRepository.getLikelyMatchPolicyEvaluators(request, policyType); + + allEvaluators.addAll(resourcePolicyEvaluators); + } + } + + private void getResourceAccessInfoForZone(RangerAccessRequest request, RangerResourceAccessInfo ret, String zoneName) { + final RangerPolicyRepository matchedRepository = policyEngine.getRepositoryForZone(zoneName); + + if (matchedRepository == null) { + LOG.error("policyRepository for zoneName:[" + zoneName + "], serviceName:[" + policyEngine.getPolicyRepository().getServiceName() + "], policyVersion:[" + getPolicyVersion() + "] is null!! ERROR!"); + } else { + List tagPolicyEvaluators = policyEngine.getTagPolicyRepository() == null ? null : policyEngine.getTagPolicyRepository().getPolicyEvaluators(); + + if (CollectionUtils.isNotEmpty(tagPolicyEvaluators)) { + Set tags = RangerAccessRequestUtil.getRequestTagsFromContext(request.getContext()); + + if (CollectionUtils.isNotEmpty(tags)) { + final boolean useTagPoliciesFromDefaultZone = !policyEngine.isResourceZoneAssociatedWithTagService(zoneName); + + for (RangerTagForEval tag : tags) { + RangerAccessRequest tagEvalRequest = new RangerTagAccessRequest(tag, policyEngine.getTagPolicyRepository().getServiceDef(), request); + List evaluators = policyEngine.getTagPolicyRepository().getLikelyMatchPolicyEvaluators(tagEvalRequest, RangerPolicy.POLICY_TYPE_ACCESS); + + for (RangerPolicyEvaluator evaluator : evaluators) { + String policyZoneName = evaluator.getPolicy().getZoneName(); + + if (useTagPoliciesFromDefaultZone) { + if (StringUtils.isNotEmpty(policyZoneName)) { + if (LOG.isDebugEnabled()) { + LOG.debug("Tag policy [zone:" + policyZoneName + "] does not belong to default zone. Not evaluating this policy:[" + evaluator.getPolicy() + "]"); + } + + continue; + } + } else { + if (!StringUtils.equals(zoneName, policyZoneName)) { + if (LOG.isDebugEnabled()) { + LOG.debug("Tag policy [zone:" + policyZoneName + "] does not belong to the zone:[" + zoneName + "] of the accessed resource. Not evaluating this policy:[" + evaluator.getPolicy() + "]"); + } + + continue; + } + } + + evaluator.getResourceAccessInfo(tagEvalRequest, ret); + } + } + } + } + + List resPolicyEvaluators = matchedRepository.getLikelyMatchPolicyEvaluators(request, RangerPolicy.POLICY_TYPE_ACCESS); + + if (CollectionUtils.isNotEmpty(resPolicyEvaluators)) { + for (RangerPolicyEvaluator evaluator : resPolicyEvaluators) { + evaluator.getResourceAccessInfo(request, ret); + } + } + + ret.getAllowedUsers().removeAll(ret.getDeniedUsers()); + ret.getAllowedGroups().removeAll(ret.getDeniedGroups()); + } + } + + private void evaluateTagAuditPolicies(RangerAccessRequest request, RangerAccessResult result, RangerPolicyRepository tagPolicyRepository) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerPolicyEngineImpl.evaluateTagAuditPolicies(request=" + request + ", result=" + result + ")"); + } + + Set tags = RangerAccessRequestUtil.getRequestTagsFromContext(request.getContext()); + + if (CollectionUtils.isNotEmpty(tags)) { + Date accessTime = request.getAccessTime() != null ? request.getAccessTime() : new Date(); + List evaluators = tagPolicyRepository.getLikelyMatchPolicyEvaluators(request, tags, RangerPolicy.POLICY_TYPE_AUDIT, accessTime); + + if (CollectionUtils.isNotEmpty(evaluators)) { + for (PolicyEvaluatorForTag policyEvaluator : evaluators) { + RangerPolicyEvaluator evaluator = policyEvaluator.getEvaluator(); + RangerTagForEval tag = policyEvaluator.getTag(); + RangerAccessRequest tagEvalRequest = new RangerTagAccessRequest(tag, tagPolicyRepository.getServiceDef(), request); + RangerAccessResult tagEvalResult = createAccessResult(tagEvalRequest, RangerPolicy.POLICY_TYPE_AUDIT); + + if (LOG.isDebugEnabled()) { + LOG.debug("RangerPolicyEngineImpl.evaluateTagAuditPolicies: Evaluating Audit policies for tag (" + tag.getType() + ")" + "Tag Evaluator: " + policyEvaluator); + } + + tagEvalResult.setAccessResultFrom(result); + + result.incrementEvaluatedPoliciesCount(); + + evaluator.evaluate(tagEvalRequest, tagEvalResult); + + if (tagEvalResult.getIsAuditedDetermined()) { + result.setIsAudited(tagEvalResult.getIsAudited()); + break; + } + } + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerPolicyEngineImpl.evaluateTagAuditPolicies(request=" + request + ", result=" + result + ")"); + } + } + + private boolean evaluateResourceAuditPolicies(RangerAccessRequest request, RangerAccessResult result, RangerPolicyRepository policyRepository) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerPolicyEngineImpl.evaluateResourceAuditPolicies(request=" + request + ", result=" + result + ")"); + } + + boolean ret = false; + List evaluators = policyRepository.getLikelyMatchAuditPolicyEvaluators(request); + + if (CollectionUtils.isNotEmpty(evaluators)) { + for (RangerPolicyEvaluator evaluator : evaluators) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerPolicyEngineImpl.evaluateResourceAuditPolicies(): Evaluating RangerPolicyEvaluator...: " + evaluator); + } + + result.incrementEvaluatedPoliciesCount(); + + evaluator.evaluate(request, result); + + if (result.getIsAuditedDetermined()) { + ret = true; + + break; + } + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerPolicyEngineImpl.evaluateResourceAuditPolicies(request=" + request + ", result=" + result + "): ret=" + ret); + } + + return ret; + } + + private boolean getIsFallbackSupported() { + return policyEngine.getPluginContext().getConfig().getIsFallbackSupported(); + } + + private void updateFromPolicyACLs(RangerPolicyEvaluator evaluator, Set policyIdForTemporalTags, RangerResourceACLs resourceACLs) { + PolicyACLSummary aclSummary = evaluator.getPolicyACLSummary(); + + if (aclSummary == null) { + return; + } + + boolean isConditional = policyIdForTemporalTags.contains(evaluator.getId()) || evaluator.getValidityScheduleEvaluatorsCount() != 0; + + for (Map.Entry> userAccessInfo : aclSummary.getUsersAccessInfo().entrySet()) { + final String userName = userAccessInfo.getKey(); + + for (Map.Entry accessInfo : userAccessInfo.getValue().entrySet()) { + Integer accessResult; + + if (isConditional) { + accessResult = ACCESS_CONDITIONAL; + } else { + accessResult = accessInfo.getValue().getResult(); + + if (accessResult.equals(RangerPolicyEvaluator.ACCESS_UNDETERMINED)) { + accessResult = RangerPolicyEvaluator.ACCESS_DENIED; + } + } + + RangerPolicy policy = evaluator.getPolicy(); + + resourceACLs.setUserAccessInfo(userName, accessInfo.getKey(), accessResult, policy); + } + } + + for (Map.Entry> groupAccessInfo : aclSummary.getGroupsAccessInfo().entrySet()) { + final String groupName = groupAccessInfo.getKey(); + + for (Map.Entry accessInfo : groupAccessInfo.getValue().entrySet()) { + Integer accessResult; + + if (isConditional) { + accessResult = ACCESS_CONDITIONAL; + } else { + accessResult = accessInfo.getValue().getResult(); + + if (accessResult.equals(RangerPolicyEvaluator.ACCESS_UNDETERMINED)) { + accessResult = RangerPolicyEvaluator.ACCESS_DENIED; + } + } + + RangerPolicy policy = evaluator.getPolicy(); + + resourceACLs.setGroupAccessInfo(groupName, accessInfo.getKey(), accessResult, policy); + } + } + + for (Map.Entry> roleAccessInfo : aclSummary.getRolesAccessInfo().entrySet()) { + final String roleName = roleAccessInfo.getKey(); + + for (Map.Entry accessInfo : roleAccessInfo.getValue().entrySet()) { + Integer accessResult; + + if (isConditional) { + accessResult = ACCESS_CONDITIONAL; + } else { + accessResult = accessInfo.getValue().getResult(); + + if (accessResult.equals(RangerPolicyEvaluator.ACCESS_UNDETERMINED)) { + accessResult = RangerPolicyEvaluator.ACCESS_DENIED; + } + } + + RangerPolicy policy = evaluator.getPolicy(); + + resourceACLs.setRoleAccessInfo(roleName, accessInfo.getKey(), accessResult, policy); + } + } + } + + private void updateRowFiltersFromPolicy(RangerPolicyEvaluator evaluator, Set policyIdForTemporalTags, RangerResourceACLs resourceACLs) { + PolicyACLSummary aclSummary = evaluator.getPolicyACLSummary(); + + if (aclSummary != null) { + boolean isConditional = policyIdForTemporalTags.contains(evaluator.getId()) || evaluator.getValidityScheduleEvaluatorsCount() != 0; + + for (RowFilterResult rowFilterResult : aclSummary.getRowFilters()) { + rowFilterResult = copyRowFilter(rowFilterResult); + + if (isConditional) { + rowFilterResult.setIsConditional(true); + } + + resourceACLs.getRowFilters().add(rowFilterResult); + } + } + } + + private void updateDataMasksFromPolicy(RangerPolicyEvaluator evaluator, Set policyIdForTemporalTags, RangerResourceACLs resourceACLs) { + PolicyACLSummary aclSummary = evaluator.getPolicyACLSummary(); + + if (aclSummary != null) { + boolean isConditional = policyIdForTemporalTags.contains(evaluator.getId()) || evaluator.getValidityScheduleEvaluatorsCount() != 0; + + for (DataMaskResult dataMaskResult : aclSummary.getDataMasks()) { + dataMaskResult = copyDataMask(dataMaskResult); + + if (isConditional) { + dataMaskResult.setIsConditional(true); + } + + resourceACLs.getDataMasks().add(dataMaskResult); + } + } + } + + private DataMaskResult copyDataMask(DataMaskResult dataMask) { + DataMaskResult ret = new DataMaskResult(copyStrings(dataMask.getUsers()), + copyStrings(dataMask.getGroups()), + copyStrings(dataMask.getRoles()), + copyStrings(dataMask.getAccessTypes()), + new RangerPolicyItemDataMaskInfo(dataMask.getMaskInfo())); + + ret.setIsConditional(dataMask.getIsConditional()); + + return ret; + } + + private RowFilterResult copyRowFilter(RowFilterResult rowFilter) { + RowFilterResult ret = new RowFilterResult(copyStrings(rowFilter.getUsers()), + copyStrings(rowFilter.getGroups()), + copyStrings(rowFilter.getRoles()), + copyStrings(rowFilter.getAccessTypes()), + new RangerPolicyItemRowFilterInfo(rowFilter.getFilterInfo())); + + ret.setIsConditional(rowFilter.getIsConditional()); + + return ret; + } + + private Set copyStrings(Set values) { + return values != null ? new HashSet<>(values) : null; + } + + private static class ServiceConfig { + private final Set auditExcludedUsers; + private final Set auditExcludedGroups; + private final Set auditExcludedRoles; + private final Set superUsers; + private final Set superGroups; + private final Set serviceAdmins; + + public ServiceConfig(Map svcConfig) { + if (svcConfig != null) { + auditExcludedUsers = StringUtil.toSet(svcConfig.get(RangerPolicyEngine.PLUGIN_AUDIT_EXCLUDE_USERS)); + auditExcludedGroups = StringUtil.toSet(svcConfig.get(RangerPolicyEngine.PLUGIN_AUDIT_EXCLUDE_GROUPS)); + auditExcludedRoles = StringUtil.toSet(svcConfig.get(RangerPolicyEngine.PLUGIN_AUDIT_EXCLUDE_ROLES)); + superUsers = StringUtil.toSet(svcConfig.get(RangerPolicyEngine.PLUGIN_SUPER_USERS)); + superGroups = StringUtil.toSet(svcConfig.get(RangerPolicyEngine.PLUGIN_SUPER_GROUPS)); + serviceAdmins = StringUtil.toSet(svcConfig.get(RangerPolicyEngine.PLUGIN_SERVICE_ADMINS)); + } else { + auditExcludedUsers = Collections.emptySet(); + auditExcludedGroups = Collections.emptySet(); + auditExcludedRoles = Collections.emptySet(); + superUsers = Collections.emptySet(); + superGroups = Collections.emptySet(); + serviceAdmins = Collections.emptySet(); + } + } + + public ServiceConfig(ServiceConfig other) { + auditExcludedUsers = other == null || CollectionUtils.isEmpty(other.auditExcludedUsers) ? Collections.emptySet() : new HashSet<>(other.auditExcludedUsers); + auditExcludedGroups = other == null || CollectionUtils.isEmpty(other.auditExcludedGroups) ? Collections.emptySet() : new HashSet<>(other.auditExcludedGroups); + auditExcludedRoles = other == null || CollectionUtils.isEmpty(other.auditExcludedRoles) ? Collections.emptySet() : new HashSet<>(other.auditExcludedRoles); + superUsers = other == null || CollectionUtils.isEmpty(other.superUsers) ? Collections.emptySet() : new HashSet<>(other.superUsers); + superGroups = other == null || CollectionUtils.isEmpty(other.superGroups) ? Collections.emptySet() : new HashSet<>(other.superGroups); + serviceAdmins = other == null || CollectionUtils.isEmpty(other.serviceAdmins) ? Collections.emptySet() : new HashSet<>(other.serviceAdmins); + } + + public boolean isAuditExcludedUser(String userName) { + return auditExcludedUsers.contains(userName); + } + + public boolean hasAuditExcludedGroup(Set userGroups) { + return userGroups != null && userGroups.size() > 0 && auditExcludedGroups.size() > 0 && CollectionUtils.containsAny(userGroups, auditExcludedGroups); + } + + public boolean hasAuditExcludedRole(Set userRoles) { + return userRoles != null && userRoles.size() > 0 && auditExcludedRoles.size() > 0 && CollectionUtils.containsAny(userRoles, auditExcludedRoles); + } + + public boolean isSuperUser(String userName) { + return superUsers.contains(userName); + } + + public boolean hasSuperGroup(Set userGroups) { + return userGroups != null && userGroups.size() > 0 && superGroups.size() > 0 && CollectionUtils.containsAny(userGroups, superGroups); + } + + public boolean isServiceAdmin(String userName) { + return serviceAdmins.contains(userName); + } + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerPolicyEngineOptions.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerPolicyEngineOptions.java new file mode 100644 index 00000000000..8cd7df91b51 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerPolicyEngineOptions.java @@ -0,0 +1,208 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.atlas.plugin.policyengine; + +import org.apache.hadoop.conf.Configuration; +import org.apache.atlas.plugin.model.validation.RangerServiceDefHelper; +import org.apache.atlas.plugin.policyevaluator.RangerPolicyEvaluator; + +public class RangerPolicyEngineOptions { + public String evaluatorType = RangerPolicyEvaluator.EVALUATOR_TYPE_AUTO; + + public boolean disableContextEnrichers = false; + public boolean disableCustomConditions = false; + public boolean disableTagPolicyEvaluation = false; + public boolean disableTrieLookupPrefilter = false; + public boolean disablePolicyRefresher = false; + public boolean disableTagRetriever = false; + public boolean cacheAuditResults = true; + public boolean evaluateDelegateAdminOnly = false; + public boolean enableTagEnricherWithLocalRefresher = false; + public boolean disableAccessEvaluationWithPolicyACLSummary = true; + public boolean optimizeTrieForRetrieval = false; + public boolean disableRoleResolution = true; + + private RangerServiceDefHelper serviceDefHelper; + + public RangerPolicyEngineOptions() {} + + public RangerPolicyEngineOptions(final RangerPolicyEngineOptions other) { + this.disableContextEnrichers = other.disableContextEnrichers; + this.disableCustomConditions = other.disableCustomConditions; + this.disableTagPolicyEvaluation = other.disableTagPolicyEvaluation; + this.disableTrieLookupPrefilter = other.disableTrieLookupPrefilter; + this.disablePolicyRefresher = other.disablePolicyRefresher; + this.disableTagRetriever = other.disableTagRetriever; + this.cacheAuditResults = other.cacheAuditResults; + this.evaluateDelegateAdminOnly = other.evaluateDelegateAdminOnly; + this.enableTagEnricherWithLocalRefresher = other.enableTagEnricherWithLocalRefresher; + this.disableAccessEvaluationWithPolicyACLSummary = other.disableAccessEvaluationWithPolicyACLSummary; + this.optimizeTrieForRetrieval = other.optimizeTrieForRetrieval; + this.disableRoleResolution = other.disableRoleResolution; + this.serviceDefHelper = null; + } + + public void configureForPlugin(Configuration conf, String propertyPrefix) { + disableContextEnrichers = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.context.enrichers", false); + disableCustomConditions = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.custom.conditions", false); + disableTagPolicyEvaluation = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.tagpolicy.evaluation", false); + disableTrieLookupPrefilter = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.trie.lookup.prefilter", false); + disablePolicyRefresher = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.policy.refresher", false); + disableTagRetriever = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.tag.retriever", false); + + cacheAuditResults = conf.getBoolean(propertyPrefix + ".policyengine.option.cache.audit.results", true); + + if (!disableTrieLookupPrefilter) { + cacheAuditResults = false; + } + evaluateDelegateAdminOnly = false; + enableTagEnricherWithLocalRefresher = false; + disableAccessEvaluationWithPolicyACLSummary = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.access.evaluation.with.policy.acl.summary", true); + optimizeTrieForRetrieval = conf.getBoolean(propertyPrefix + ".policyengine.option.optimize.trie.for.retrieval", false); + disableRoleResolution = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.role.resolution", true); + + } + + public void configureDefaultRangerAdmin(Configuration conf, String propertyPrefix) { + disableContextEnrichers = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.context.enrichers", true); + disableCustomConditions = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.custom.conditions", true); + disableTagPolicyEvaluation = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.tagpolicy.evaluation", true); + disableTrieLookupPrefilter = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.trie.lookup.prefilter", false); + disablePolicyRefresher = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.policy.refresher", true); + disableTagRetriever = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.tag.retriever", true); + + cacheAuditResults = false; + evaluateDelegateAdminOnly = false; + enableTagEnricherWithLocalRefresher = false; + disableAccessEvaluationWithPolicyACLSummary = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.access.evaluation.with.policy.acl.summary", true); + optimizeTrieForRetrieval = conf.getBoolean(propertyPrefix + ".policyengine.option.optimize.trie.for.retrieval", false); + disableRoleResolution = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.role.resolution", true); + + } + + public void configureDelegateAdmin(Configuration conf, String propertyPrefix) { + disableContextEnrichers = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.context.enrichers", true); + disableCustomConditions = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.custom.conditions", true); + disableTagPolicyEvaluation = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.tagpolicy.evaluation", true); + disableTrieLookupPrefilter = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.trie.lookup.prefilter", false); + disablePolicyRefresher = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.policy.refresher", true); + disableTagRetriever = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.tag.retriever", true); + optimizeTrieForRetrieval = conf.getBoolean(propertyPrefix + ".policyengine.option.optimize.trie.for.retrieval", false); + + + cacheAuditResults = false; + evaluateDelegateAdminOnly = true; + enableTagEnricherWithLocalRefresher = false; + + } + + public void configureRangerAdminForPolicySearch(Configuration conf, String propertyPrefix) { + disableContextEnrichers = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.context.enrichers", true); + disableCustomConditions = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.custom.conditions", true); + disableTagPolicyEvaluation = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.tagpolicy.evaluation", false); + disableTrieLookupPrefilter = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.trie.lookup.prefilter", false); + disablePolicyRefresher = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.policy.refresher", true); + disableTagRetriever = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.tag.retriever", false); + optimizeTrieForRetrieval = conf.getBoolean(propertyPrefix + ".policyengine.option.optimize.trie.for.retrieval", false); + + + cacheAuditResults = false; + evaluateDelegateAdminOnly = false; + enableTagEnricherWithLocalRefresher = true; + } + + public RangerServiceDefHelper getServiceDefHelper() { + return serviceDefHelper; + } + + void setServiceDefHelper(RangerServiceDefHelper serviceDefHelper) { + this.serviceDefHelper = serviceDefHelper; + } + + /* + * There is no need to implement these, as the options are predefined in a component ServiceREST and hence + * guaranteed to be unique objects. That implies that the default equals and hashCode should suffice. + */ + + @Override + public boolean equals(Object other) { + boolean ret = false; + if (other instanceof RangerPolicyEngineOptions) { + RangerPolicyEngineOptions that = (RangerPolicyEngineOptions) other; + ret = this.disableContextEnrichers == that.disableContextEnrichers + && this.disableCustomConditions == that.disableCustomConditions + && this.disableTagPolicyEvaluation == that.disableTagPolicyEvaluation + && this.disableTrieLookupPrefilter == that.disableTrieLookupPrefilter + && this.disablePolicyRefresher == that.disablePolicyRefresher + && this.disableTagRetriever == that.disableTagRetriever + && this.cacheAuditResults == that.cacheAuditResults + && this.evaluateDelegateAdminOnly == that.evaluateDelegateAdminOnly + && this.enableTagEnricherWithLocalRefresher == that.enableTagEnricherWithLocalRefresher + && this.optimizeTrieForRetrieval == that.optimizeTrieForRetrieval + && this.disableRoleResolution == that.disableRoleResolution; + } + return ret; + } + + @Override + public int hashCode() { + int ret = 0; + ret += disableContextEnrichers ? 1 : 0; + ret *= 2; + ret += disableCustomConditions ? 1 : 0; + ret *= 2; + ret += disableTagPolicyEvaluation ? 1 : 0; + ret *= 2; + ret += disableTrieLookupPrefilter ? 1 : 0; + ret *= 2; + ret += disablePolicyRefresher ? 1 : 0; + ret *= 2; + ret += disableTagRetriever ? 1 : 0; + ret *= 2; + ret += cacheAuditResults ? 1 : 0; + ret *= 2; + ret += evaluateDelegateAdminOnly ? 1 : 0; + ret *= 2; + ret += enableTagEnricherWithLocalRefresher ? 1 : 0; + ret *= 2; + ret += optimizeTrieForRetrieval ? 1 : 0; + ret *= 2; + ret += disableRoleResolution ? 1 : 0; + ret *= 2; return ret; + } + + @Override + public String toString() { + return "PolicyEngineOptions: {" + + " evaluatorType: " + evaluatorType + + ", evaluateDelegateAdminOnly: " + evaluateDelegateAdminOnly + + ", disableContextEnrichers: " + disableContextEnrichers + + ", disableCustomConditions: " + disableContextEnrichers + + ", disableTagPolicyEvaluation: " + disableTagPolicyEvaluation + + ", disablePolicyRefresher: " + disablePolicyRefresher + + ", disableTagRetriever: " + disableTagRetriever + + ", enableTagEnricherWithLocalRefresher: " + enableTagEnricherWithLocalRefresher + + ", disableTrieLookupPrefilter: " + disableTrieLookupPrefilter + + ", optimizeTrieForRetrieval: " + optimizeTrieForRetrieval + + ", cacheAuditResult: " + cacheAuditResults + + ", disableRoleResolution: " + disableRoleResolution + + " }"; + + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerPolicyRepository.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerPolicyRepository.java new file mode 100644 index 00000000000..5b3b469fb02 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerPolicyRepository.java @@ -0,0 +1,1557 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.policyengine; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.authorization.utils.JsonUtils; +import org.apache.atlas.plugin.contextenricher.RangerAbstractContextEnricher; +import org.apache.atlas.plugin.contextenricher.RangerContextEnricher; +import org.apache.atlas.plugin.contextenricher.RangerTagEnricher; +import org.apache.atlas.plugin.contextenricher.RangerTagForEval; +import org.apache.atlas.plugin.model.AuditFilter; +import org.apache.atlas.plugin.model.RangerPolicy; +import org.apache.atlas.plugin.model.RangerPolicy.RangerPolicyItemDataMaskInfo; +import org.apache.atlas.plugin.model.RangerPolicyDelta; +import org.apache.atlas.plugin.model.RangerServiceDef; +import org.apache.atlas.plugin.model.validation.RangerServiceDefHelper; +import org.apache.atlas.plugin.policyevaluator.RangerAbstractPolicyEvaluator; +import org.apache.atlas.plugin.policyevaluator.RangerAuditPolicyEvaluator; +import org.apache.atlas.plugin.policyevaluator.RangerCachedPolicyEvaluator; +import org.apache.atlas.plugin.policyevaluator.RangerOptimizedPolicyEvaluator; +import org.apache.atlas.plugin.policyevaluator.RangerPolicyEvaluator; +import org.apache.atlas.plugin.store.AbstractServiceStore; +import org.apache.atlas.plugin.util.RangerPerfTracer; +import org.apache.atlas.plugin.util.ServiceDefUtil; +import org.apache.atlas.plugin.util.ServicePolicies; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.apache.atlas.plugin.contextenricher.RangerTagEnricher.TAG_RETRIEVER_CLASSNAME_OPTION; +import static org.apache.atlas.plugin.policyengine.RangerPolicyEngine.PLUGIN_AUDIT_FILTER; + +public class RangerPolicyRepository { + private static final Log LOG = LogFactory.getLog(RangerPolicyRepository.class); + + private static final Log PERF_CONTEXTENRICHER_INIT_LOG = RangerPerfTracer.getPerfLogger("contextenricher.init"); + private static final Log PERF_TRIE_OP_LOG = RangerPerfTracer.getPerfLogger("resourcetrie.retrieval"); + + enum AuditModeEnum { + AUDIT_ALL, AUDIT_NONE, AUDIT_DEFAULT + } + + private final String serviceName; + private final String zoneName; + private final String appId; + private final RangerPolicyEngineOptions options; + private final RangerPluginContext pluginContext; + private final RangerServiceDef serviceDef; + private /*final*/ List policies; + private final long policyVersion; + private /*final*/ List contextEnrichers; + private final AuditModeEnum auditModeEnum; + private final Map accessAuditCache; + private final String componentServiceName; + private final RangerServiceDef componentServiceDef; + private final Map policyResourceTrie; + private final Map dataMaskResourceTrie; + private final Map rowFilterResourceTrie; + private final Map auditFilterResourceTrie; + private List policyEvaluators; + private List dataMaskPolicyEvaluators; + private List rowFilterPolicyEvaluators; + private final List auditPolicyEvaluators; + private Map policyEvaluatorsMap; + private boolean isContextEnrichersShared = false; + private boolean isPreCleaned = false; + + RangerPolicyRepository(final RangerPolicyRepository other, final List deltas, long policyVersion) { + this.serviceName = other.serviceName; + this.zoneName = other.zoneName; + this.appId = other.appId; + this.options = other.options; + this.pluginContext = other.pluginContext; + this.serviceDef = other.serviceDef; + this.policies = new ArrayList<>(other.policies); + this.policyEvaluators = new ArrayList<>(other.policyEvaluators); + this.dataMaskPolicyEvaluators = new ArrayList<>(other.dataMaskPolicyEvaluators); + this.rowFilterPolicyEvaluators = new ArrayList<>(other.rowFilterPolicyEvaluators); + this.auditPolicyEvaluators = new ArrayList<>(other.auditPolicyEvaluators); + this.auditModeEnum = other.auditModeEnum; + this.componentServiceName = other.componentServiceName; + this.componentServiceDef = other.componentServiceDef; + this.policyEvaluatorsMap = new HashMap<>(other.policyEvaluatorsMap); + + if (other.policyResourceTrie != null) { + this.policyResourceTrie = new HashMap<>(); + + for (Map.Entry entry : other.policyResourceTrie.entrySet()) { + policyResourceTrie.put(entry.getKey(), new RangerResourceTrie(entry.getValue())); + } + } else { + this.policyResourceTrie = null; + } + + if (other.dataMaskResourceTrie != null) { + this.dataMaskResourceTrie = new HashMap<>(); + + for (Map.Entry entry : other.dataMaskResourceTrie.entrySet()) { + dataMaskResourceTrie.put(entry.getKey(), new RangerResourceTrie(entry.getValue())); + } + } else { + this.dataMaskResourceTrie = null; + } + + if (other.rowFilterResourceTrie != null) { + this.rowFilterResourceTrie = new HashMap<>(); + + for (Map.Entry entry : other.rowFilterResourceTrie.entrySet()) { + rowFilterResourceTrie.put(entry.getKey(), new RangerResourceTrie(entry.getValue())); + } + } else { + this.rowFilterResourceTrie = null; + } + + if (other.auditFilterResourceTrie != null) { + this.auditFilterResourceTrie = new HashMap<>(); + + for (Map.Entry entry : other.auditFilterResourceTrie.entrySet()) { + auditFilterResourceTrie.put(entry.getKey(), new RangerResourceTrie(entry.getValue())); + } + } else { + this.auditFilterResourceTrie = null; + } + + if (other.accessAuditCache != null) { + int auditResultCacheSize = other.accessAuditCache.size(); + + this.accessAuditCache = Collections.synchronizedMap(new CacheMap(auditResultCacheSize)); + } else { + this.accessAuditCache = null; + } + + final boolean isExistingPolicies = CollectionUtils.isNotEmpty(this.policies); + + updateResourceTrie(deltas); + + if (CollectionUtils.isNotEmpty(this.policies)) { + this.contextEnrichers = isExistingPolicies ? shareWith(other) : buildContextEnrichers(options); + } else { + this.contextEnrichers = null; + } + + if (LOG.isDebugEnabled()) { + LOG.debug("other.serviceName=" + other.serviceName + ", other.isContextEnrichersShared=" + other.isContextEnrichersShared + ", Context-enrichers are " + (CollectionUtils.isNotEmpty(contextEnrichers) ? " not empty " : "empty ")); + } + + this.policyVersion = policyVersion; + } + + public RangerPolicyRepository(ServicePolicies servicePolicies, RangerPluginContext pluginContext) { + this(servicePolicies, pluginContext, null); + } + + RangerPolicyRepository(ServicePolicies servicePolicies, RangerPluginContext pluginContext, String zoneName) { + super(); + + this.componentServiceName = this.serviceName = servicePolicies.getServiceName(); + this.componentServiceDef = this.serviceDef = ServiceDefUtil.normalize(servicePolicies.getServiceDef()); + this.zoneName = zoneName; + this.appId = pluginContext.getConfig().getAppId(); + this.options = new RangerPolicyEngineOptions(pluginContext.getConfig().getPolicyEngineOptions()); + this.pluginContext = pluginContext; + + if (StringUtils.isEmpty(zoneName)) { + this.policies = servicePolicies.getPolicies(); + } else { + this.policies = servicePolicies.getSecurityZones().get(zoneName).getPolicies(); + } + this.policyVersion = servicePolicies.getPolicyVersion() != null ? servicePolicies.getPolicyVersion() : -1; + + String auditMode = servicePolicies.getAuditMode(); + + if (StringUtils.equals(auditMode, RangerPolicyEngine.AUDIT_ALL)) { + auditModeEnum = AuditModeEnum.AUDIT_ALL; + } else if (StringUtils.equals(auditMode, RangerPolicyEngine.AUDIT_NONE)) { + auditModeEnum = AuditModeEnum.AUDIT_NONE; + } else { + auditModeEnum = AuditModeEnum.AUDIT_DEFAULT; + } + + if (auditModeEnum == AuditModeEnum.AUDIT_DEFAULT) { + String propertyName = "ranger.plugin." + serviceName + ".policyengine.auditcachesize"; + + if (options.cacheAuditResults) { + final int RANGER_POLICYENGINE_AUDITRESULT_CACHE_SIZE = 64 * 1024; + + int auditResultCacheSize = pluginContext.getConfig().getInt(propertyName, RANGER_POLICYENGINE_AUDITRESULT_CACHE_SIZE); + accessAuditCache = Collections.synchronizedMap(new CacheMap(auditResultCacheSize)); + } else { + accessAuditCache = null; + } + } else { + this.accessAuditCache = null; + } + + if (LOG.isDebugEnabled()) { + LOG.debug("RangerPolicyRepository : building policy-repository for service[" + serviceName + "], and zone:[" + zoneName + "] with auditMode[" + auditModeEnum + "]"); + } + + init(options); + + if (StringUtils.isEmpty(zoneName)) { + this.contextEnrichers = buildContextEnrichers(options); + this.auditPolicyEvaluators = buildAuditPolicyEvaluators(servicePolicies.getServiceConfig()); + } else { + this.contextEnrichers = null; + this.auditPolicyEvaluators = Collections.emptyList(); + } + + if (options.disableTrieLookupPrefilter) { + policyResourceTrie = null; + dataMaskResourceTrie = null; + rowFilterResourceTrie = null; + auditFilterResourceTrie = null; + } else { + policyResourceTrie = createResourceTrieMap(policyEvaluators, options.optimizeTrieForRetrieval); + dataMaskResourceTrie = createResourceTrieMap(dataMaskPolicyEvaluators, options.optimizeTrieForRetrieval); + rowFilterResourceTrie = createResourceTrieMap(rowFilterPolicyEvaluators, options.optimizeTrieForRetrieval); + auditFilterResourceTrie = createResourceTrieMap(auditPolicyEvaluators, options.optimizeTrieForRetrieval); + } + } + + RangerPolicyRepository(ServicePolicies.TagPolicies tagPolicies, RangerPluginContext pluginContext, + RangerServiceDef componentServiceDef, String componentServiceName) { + super(); + + this.serviceName = tagPolicies.getServiceName(); + this.componentServiceName = componentServiceName; + this.zoneName = null; + this.serviceDef = ServiceDefUtil.normalizeAccessTypeDefs(ServiceDefUtil.normalize(tagPolicies.getServiceDef()), componentServiceDef.getName()); + this.componentServiceDef = componentServiceDef; + this.appId = pluginContext.getConfig().getAppId(); + this.options = new RangerPolicyEngineOptions(pluginContext.getConfig().getPolicyEngineOptions()); + this.pluginContext = pluginContext; + this.policies = normalizeAndPrunePolicies(tagPolicies.getPolicies(), componentServiceDef.getName()); + this.policyVersion = tagPolicies.getPolicyVersion() != null ? tagPolicies.getPolicyVersion() : -1; + + String auditMode = tagPolicies.getAuditMode(); + + if (StringUtils.equals(auditMode, RangerPolicyEngine.AUDIT_ALL)) { + auditModeEnum = AuditModeEnum.AUDIT_ALL; + } else if (StringUtils.equals(auditMode, RangerPolicyEngine.AUDIT_NONE)) { + auditModeEnum = AuditModeEnum.AUDIT_NONE; + } else { + auditModeEnum = AuditModeEnum.AUDIT_DEFAULT; + } + + this.accessAuditCache = null; + + if (LOG.isDebugEnabled()) { + LOG.debug("RangerPolicyRepository : building tag-policy-repository for tag service:[" + serviceName +"], with auditMode[" + auditModeEnum +"]"); + } + + init(options); + + if (StringUtils.isEmpty(zoneName)) { + this.contextEnrichers = buildContextEnrichers(options); + this.auditPolicyEvaluators = buildAuditPolicyEvaluators(tagPolicies.getServiceConfig()); + } else { + this.contextEnrichers = null; + this.auditPolicyEvaluators = Collections.emptyList(); + } + + if (options.disableTrieLookupPrefilter) { + policyResourceTrie = null; + dataMaskResourceTrie = null; + rowFilterResourceTrie = null; + auditFilterResourceTrie = null; + } else { + policyResourceTrie = createResourceTrieMap(policyEvaluators, options.optimizeTrieForRetrieval); + dataMaskResourceTrie = createResourceTrieMap(dataMaskPolicyEvaluators, options.optimizeTrieForRetrieval); + rowFilterResourceTrie = createResourceTrieMap(rowFilterPolicyEvaluators, options.optimizeTrieForRetrieval); + auditFilterResourceTrie = createResourceTrieMap(auditPolicyEvaluators, options.optimizeTrieForRetrieval); + } + } + + private List buildAuditPolicyEvaluators(Map svcConfigs) { + List ret = Collections.emptyList(); + String jsonStr = svcConfigs != null ? svcConfigs.get(PLUGIN_AUDIT_FILTER) : null; + + if (StringUtils.isNotBlank(jsonStr)) { + List auditFilters = JsonUtils.jsonToAuditFilterList(jsonStr); + int filterCount = auditFilters != null ? auditFilters.size() : 0; + + if (filterCount > 0) { + ret = new ArrayList<>(filterCount); + + // set priority so that policies will be evaluated in the same order as in the list + int policyPriority = filterCount; + + for (AuditFilter auditFilter : auditFilters) { + RangerAuditPolicyEvaluator evaluator = new RangerAuditPolicyEvaluator(auditFilter, policyPriority--); + + evaluator.init(evaluator.getAuditPolicy(), serviceDef, options); + + ret.add(evaluator); + } + } + } + + return ret; + } + + @Override + public String toString( ) { + StringBuilder sb = new StringBuilder(); + + toString(sb); + + return sb.toString(); + } + + public StringBuilder toString(StringBuilder sb) { + if (sb == null) { + sb = new StringBuilder(); + } + + sb.append("RangerPolicyRepository={"); + + sb.append("serviceName={").append(serviceName).append("} "); + sb.append("zoneName={").append(zoneName).append("} "); + sb.append("serviceDef={").append(serviceDef).append("} "); + sb.append("appId={").append(appId).append("} "); + + sb.append("policyEvaluators={"); + if (policyEvaluators != null) { + for (RangerPolicyEvaluator policyEvaluator : policyEvaluators) { + if (policyEvaluator != null) { + sb.append(policyEvaluator).append(" "); + } + } + } + sb.append("} "); + + sb.append("dataMaskPolicyEvaluators={"); + if (this.dataMaskPolicyEvaluators != null) { + for (RangerPolicyEvaluator policyEvaluator : dataMaskPolicyEvaluators) { + if (policyEvaluator != null) { + sb.append(policyEvaluator).append(" "); + } + } + } + sb.append("} "); + + sb.append("rowFilterPolicyEvaluators={"); + if (this.rowFilterPolicyEvaluators != null) { + for (RangerPolicyEvaluator policyEvaluator : rowFilterPolicyEvaluators) { + if (policyEvaluator != null) { + sb.append(policyEvaluator).append(" "); + } + } + } + sb.append("} "); + + sb.append("auditPolicyEvaluators={"); + if (this.auditPolicyEvaluators != null) { + for (RangerPolicyEvaluator policyEvaluator : auditPolicyEvaluators) { + if (policyEvaluator != null) { + sb.append(policyEvaluator).append(" "); + } + } + } + sb.append("} "); + + sb.append("contextEnrichers={"); + if (contextEnrichers != null) { + for (RangerContextEnricher contextEnricher : contextEnrichers) { + if (contextEnricher != null) { + sb.append(contextEnricher).append(" "); + } + } + } + sb.append("} "); + + sb.append("} "); + + return sb; + } + + List shareWith(RangerPolicyRepository other) { + if (other != null && other.contextEnrichers != null) { + other.setShared(); + } + return other == null ? null : other.contextEnrichers; + } + + void setShared() { + isContextEnrichersShared = true; + } + + void preCleanup(boolean isForced) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> preCleanup(isForced=" + isForced + " )"); + LOG.debug("Repository holds [" + (CollectionUtils.isEmpty(this.contextEnrichers) ? 0 : this.contextEnrichers.size()) + "] enrichers. isPreCleaned=" + isPreCleaned); + } + if (!isPreCleaned) { + if (CollectionUtils.isNotEmpty(this.contextEnrichers) && (!isContextEnrichersShared || isForced)) { + isPreCleaned = true; + if (LOG.isDebugEnabled()) { + LOG.debug("preCleaning context-enrichers"); + } + for (RangerContextEnricher enricher : this.contextEnrichers) { + enricher.preCleanup(); + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("No preCleaning of context-enrichers; Context-enrichers are " + (CollectionUtils.isNotEmpty(contextEnrichers) ? " not empty " : "empty ") + ", isContextEnrichersShared=" + isContextEnrichersShared + ", isForced=" + isForced + ")"); + } + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("preCleanup() already done. No need to do it again"); + } + } + if (LOG.isDebugEnabled()) { + LOG.debug("<== preCleanup(isForced=" + isForced + " )"); + } + } + + void cleanup() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> cleanup()"); + } + preCleanup(false); + + if (CollectionUtils.isNotEmpty(this.contextEnrichers) && !isContextEnrichersShared) { + for (RangerContextEnricher enricher : this.contextEnrichers) { + enricher.cleanup(); + } + } + if (LOG.isDebugEnabled()) { + LOG.debug("<== cleanup()"); + } + } + + @Override + protected void finalize() throws Throwable { + try { + cleanup(); + } finally { + super.finalize(); + } + } + + void reorderPolicyEvaluators() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> reorderEvaluators()"); + } + + if (policyResourceTrie == null) { + policyEvaluators = getReorderedPolicyEvaluators(policyEvaluators); + } + + if (dataMaskResourceTrie == null) { + dataMaskPolicyEvaluators = getReorderedPolicyEvaluators(dataMaskPolicyEvaluators); + } + + if (rowFilterResourceTrie == null) { + rowFilterPolicyEvaluators = getReorderedPolicyEvaluators(rowFilterPolicyEvaluators); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== reorderEvaluators()"); + } + } + + public String getServiceName() { return serviceName; } + + String getZoneName() { return zoneName; } + + public RangerServiceDef getServiceDef() { + return serviceDef; + } + + List getPolicies() { + return policies; + } + + long getPolicyVersion() { + return policyVersion; + } + + AuditModeEnum getAuditModeEnum() { return auditModeEnum; } + + boolean setAuditEnabledFromCache(RangerAccessRequest request, RangerAccessResult result) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerPolicyRepository.setAuditEnabledFromCache()"); + } + + final AuditInfo auditInfo = accessAuditCache != null ? accessAuditCache.get(request.getResource().getAsString()) : null; + + if (auditInfo != null) { + result.setIsAudited(auditInfo.getIsAudited()); + result.setAuditPolicyId(auditInfo.getAuditPolicyId()); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerPolicyRepository.setAuditEnabledFromCache():" + (auditInfo != null)); + } + + return auditInfo != null; + } + + void storeAuditEnabledInCache(RangerAccessRequest request, RangerAccessResult result) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerPolicyRepository.storeAuditEnabledInCache()"); + } + + if (accessAuditCache != null && result.getIsAuditedDetermined()) { + accessAuditCache.put(request.getResource().getAsString(), new AuditInfo(result.getIsAudited(), result.getAuditPolicyId())); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerPolicyRepository.storeAuditEnabledInCache()"); + } + } + + List getContextEnrichers() { return contextEnrichers; } + + List getPolicyEvaluators(String policyType) { + switch (policyType) { + case RangerPolicy.POLICY_TYPE_ACCESS: + return getPolicyEvaluators(); + case RangerPolicy.POLICY_TYPE_DATAMASK: + return getDataMaskPolicyEvaluators(); + case RangerPolicy.POLICY_TYPE_ROWFILTER: + return getRowFilterPolicyEvaluators(); + default: + return getPolicyEvaluators(); + } + } + + public List getPolicyEvaluators() { + return policyEvaluators; + } + + List getDataMaskPolicyEvaluators() { + return dataMaskPolicyEvaluators; + } + + List getRowFilterPolicyEvaluators() { + return rowFilterPolicyEvaluators; + } + + List getAuditPolicyEvaluators() { + return auditPolicyEvaluators; + } + + String getAppId() { return appId; } + + RangerPolicyEngineOptions getOptions() { return options; } + + List getLikelyMatchPolicyEvaluators(RangerAccessRequest request, Set tags, String policyType, Date accessTime) { + List ret = Collections.EMPTY_LIST; + + if (CollectionUtils.isNotEmpty(tags) && getServiceDef() != null) { + + ret = new ArrayList(); + + for (RangerTagForEval tag : tags) { + if (tag.isApplicable(accessTime)) { + RangerAccessRequest tagRequest = new RangerTagAccessRequest(tag, getServiceDef(), request); + List evaluators = getLikelyMatchPolicyEvaluators(tagRequest, policyType); + + if (CollectionUtils.isNotEmpty(evaluators)) { + for (RangerPolicyEvaluator evaluator : evaluators) { + if (evaluator.isApplicable(accessTime)) { + ret.add(new PolicyEvaluatorForTag(evaluator, tag)); + } + } + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Tag:[" + tag.getType() + "] is not applicable at accessTime:[" + accessTime +"]"); + } + } + } + + if (CollectionUtils.isNotEmpty(ret)) { + switch (policyType) { + case RangerPolicy.POLICY_TYPE_ACCESS: + case RangerPolicy.POLICY_TYPE_AUDIT: + Collections.sort(ret, PolicyEvaluatorForTag.EVAL_ORDER_COMPARATOR); + break; + case RangerPolicy.POLICY_TYPE_DATAMASK: + Collections.sort(ret, PolicyEvaluatorForTag.NAME_COMPARATOR); + break; + case RangerPolicy.POLICY_TYPE_ROWFILTER: + Collections.sort(ret, PolicyEvaluatorForTag.NAME_COMPARATOR); + break; + default: + LOG.warn("Unknown policy-type:[" + policyType + "]. Ignoring.."); + break; + } + } + } + return ret; + } + + public List getLikelyMatchPolicyEvaluators(RangerAccessRequest request) { + List ret = new ArrayList<>(); + + for (String policyType : RangerPolicy.POLICY_TYPES) { + List evaluators = getLikelyMatchPolicyEvaluators(request, policyType); + if (CollectionUtils.isNotEmpty(evaluators)) { + ret.addAll(evaluators); + } + } + return ret; + } + + public List getLikelyMatchPolicyEvaluators(RangerAccessRequest request, String policyType) { + switch (policyType) { + case RangerPolicy.POLICY_TYPE_ACCESS: + return getLikelyMatchAccessPolicyEvaluators(request); + case RangerPolicy.POLICY_TYPE_DATAMASK: + return getLikelyMatchDataMaskPolicyEvaluators(request); + case RangerPolicy.POLICY_TYPE_ROWFILTER: + return getLikelyMatchRowFilterPolicyEvaluators(request); + case RangerPolicy.POLICY_TYPE_AUDIT: + return getLikelyMatchAuditPolicyEvaluators(request); + default: + return Collections.EMPTY_LIST; + } + } + + + public Map getPolicyEvaluatorsMap() { return policyEvaluatorsMap; } + + RangerPolicyEvaluator getPolicyEvaluator(Long id) { + return policyEvaluatorsMap.get(id); + } + + private List getLikelyMatchAccessPolicyEvaluators(RangerAccessRequest request) { + RangerAccessResource resource = request.getResource(); + String resourceStr = resource == null ? null : resource.getAsString(); + + return policyResourceTrie == null || StringUtils.isEmpty(resourceStr) ? getPolicyEvaluators() : getLikelyMatchPolicyEvaluators(policyResourceTrie, request); + } + + private List getLikelyMatchDataMaskPolicyEvaluators(RangerAccessRequest request) { + RangerAccessResource resource = request.getResource(); + String resourceStr = resource == null ? null : resource.getAsString(); + + return dataMaskResourceTrie == null || StringUtils.isEmpty(resourceStr) ? getDataMaskPolicyEvaluators() : getLikelyMatchPolicyEvaluators(dataMaskResourceTrie, request); + } + + private List getLikelyMatchRowFilterPolicyEvaluators(RangerAccessRequest request) { + RangerAccessResource resource = request.getResource(); + String resourceStr = resource == null ? null : resource.getAsString(); + + return rowFilterResourceTrie == null || StringUtils.isEmpty(resourceStr) ? getRowFilterPolicyEvaluators() : getLikelyMatchPolicyEvaluators(rowFilterResourceTrie, request); + } + + List getLikelyMatchAuditPolicyEvaluators(RangerAccessRequest request) { + RangerAccessResource resource = request.getResource(); + String resourceStr = resource == null ? null : resource.getAsString(); + + return auditFilterResourceTrie == null || StringUtils.isEmpty(resourceStr) ? getAuditPolicyEvaluators() : getLikelyMatchPolicyEvaluators(auditFilterResourceTrie, request); + } + + private List getLikelyMatchPolicyEvaluators(Map resourceTrie, RangerAccessRequest request) { + List ret = Collections.EMPTY_LIST; + + RangerAccessResource resource = request.getResource(); + + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_TRIE_OP_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_TRIE_OP_LOG, "RangerPolicyRepository.getLikelyMatchEvaluators(resource=" + resource.getAsString() + ")"); + } + + List resourceKeys = resource == null ? null : options.getServiceDefHelper().getOrderedResourceNames(resource.getKeys()); + Set smallestList = null; + + if (CollectionUtils.isNotEmpty(resourceKeys)) { + + for (String resourceName : resourceKeys) { + RangerResourceTrie trie = resourceTrie.get(resourceName); + + if (trie == null) { // if no trie exists for this resource level, ignore and continue to next level + continue; + } + + Set serviceResourceMatchersForResource = trie.getEvaluatorsForResource(resource.getValue(resourceName), request.getResourceMatchingScope()); + Set inheritedResourceMatchers = trie.getInheritedEvaluators(); + + if (smallestList != null) { + if (CollectionUtils.isEmpty(inheritedResourceMatchers) && CollectionUtils.isEmpty(serviceResourceMatchersForResource)) { + smallestList = null; + } else if (CollectionUtils.isEmpty(inheritedResourceMatchers)) { + smallestList.retainAll(serviceResourceMatchersForResource); + } else if (CollectionUtils.isEmpty(serviceResourceMatchersForResource)) { + smallestList.retainAll(inheritedResourceMatchers); + } else { + Set smaller, bigger; + if (serviceResourceMatchersForResource.size() < inheritedResourceMatchers.size()) { + smaller = serviceResourceMatchersForResource; + bigger = inheritedResourceMatchers; + } else { + smaller = inheritedResourceMatchers; + bigger = serviceResourceMatchersForResource; + } + Set tmp = new HashSet<>(); + if (smallestList.size() < smaller.size()) { + smallestList.stream().filter(smaller::contains).forEach(tmp::add); + smallestList.stream().filter(bigger::contains).forEach(tmp::add); + } else { + smaller.stream().filter(smallestList::contains).forEach(tmp::add); + if (smallestList.size() < bigger.size()) { + smallestList.stream().filter(bigger::contains).forEach(tmp::add); + } else { + bigger.stream().filter(smallestList::contains).forEach(tmp::add); + } + } + smallestList = tmp; + } + } else { + if (CollectionUtils.isEmpty(inheritedResourceMatchers) || CollectionUtils.isEmpty(serviceResourceMatchersForResource)) { + Set tmp = CollectionUtils.isEmpty(inheritedResourceMatchers) ? serviceResourceMatchersForResource : inheritedResourceMatchers; + smallestList = resourceKeys.size() == 1 || CollectionUtils.isEmpty(tmp) ? tmp : new HashSet<>(tmp); + } else { + smallestList = new HashSet<>(serviceResourceMatchersForResource); + smallestList.addAll(inheritedResourceMatchers); + } + } + + if (CollectionUtils.isEmpty(smallestList)) {// no tags for this resource, bail out + smallestList = null; + break; + } + } + } + + if (smallestList != null) { + ret = new ArrayList<>(smallestList); + ret.sort(RangerPolicyEvaluator.EVAL_ORDER_COMPARATOR); + } + + RangerPerfTracer.logAlways(perf); + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerPolicyRepository.getLikelyMatchPolicyEvaluators(" + resource.getAsString() + "): evaluatorCount=" + ret.size()); + } + + return ret; + } + + private List normalizeAndPrunePolicies(List rangerPolicies, final String componentType) { + if (CollectionUtils.isNotEmpty(rangerPolicies) && StringUtils.isNotBlank(componentType)) { + List policiesToPrune = null; + + for (RangerPolicy policy : rangerPolicies) { + if (isPolicyNeedsPruning(policy, componentType)) { + + if(policiesToPrune == null) { + policiesToPrune = new ArrayList<>(); + } + + policiesToPrune.add(policy); + } + } + + if(policiesToPrune != null) { + rangerPolicies.removeAll(policiesToPrune); + } + } + + return rangerPolicies == null ? new ArrayList<>() : rangerPolicies; + } + + private boolean isPolicyNeedsPruning(RangerPolicy policy, final String componentType) { + + normalizeAndPrunePolicyItems(policy.getPolicyItems(), componentType); + normalizeAndPrunePolicyItems(policy.getDenyPolicyItems(), componentType); + normalizeAndPrunePolicyItems(policy.getAllowExceptions(), componentType); + normalizeAndPrunePolicyItems(policy.getDenyExceptions(), componentType); + normalizeAndPrunePolicyItems(policy.getDataMaskPolicyItems(), componentType); + normalizeAndPrunePolicyItems(policy.getRowFilterPolicyItems(), componentType); + + if (!policy.getIsAuditEnabled() && + CollectionUtils.isEmpty(policy.getPolicyItems()) && + CollectionUtils.isEmpty(policy.getDenyPolicyItems()) && + CollectionUtils.isEmpty(policy.getAllowExceptions()) && + CollectionUtils.isEmpty(policy.getDenyExceptions()) && + CollectionUtils.isEmpty(policy.getDataMaskPolicyItems()) && + CollectionUtils.isEmpty(policy.getRowFilterPolicyItems())) { + return true; + } else { + return false; + } + } + + private List normalizeAndPrunePolicyItems(List policyItems, final String componentType) { + if(CollectionUtils.isNotEmpty(policyItems)) { + final String prefix = componentType + AbstractServiceStore.COMPONENT_ACCESSTYPE_SEPARATOR; + List itemsToPrune = null; + + for (RangerPolicy.RangerPolicyItem policyItem : policyItems) { + List policyItemAccesses = policyItem.getAccesses(); + + if (CollectionUtils.isNotEmpty(policyItemAccesses)) { + List accessesToPrune = null; + + for (RangerPolicy.RangerPolicyItemAccess access : policyItemAccesses) { + String accessType = access.getType(); + + if (StringUtils.startsWith(accessType, prefix)) { + String newAccessType = StringUtils.removeStart(accessType, prefix); + + access.setType(newAccessType); + } else if (accessType.contains(AbstractServiceStore.COMPONENT_ACCESSTYPE_SEPARATOR)) { + if(accessesToPrune == null) { + accessesToPrune = new ArrayList<>(); + } + + accessesToPrune.add(access); + } + } + + if(accessesToPrune != null) { + policyItemAccesses.removeAll(accessesToPrune); + } + + if (policyItemAccesses.isEmpty() && !policyItem.getDelegateAdmin()) { + if(itemsToPrune == null) { + itemsToPrune = new ArrayList<>(); + } + + itemsToPrune.add(policyItem); + + continue; + } + } + + if (policyItem instanceof RangerPolicy.RangerDataMaskPolicyItem) { + RangerPolicyItemDataMaskInfo dataMaskInfo = ((RangerPolicy.RangerDataMaskPolicyItem) policyItem).getDataMaskInfo(); + String maskType = dataMaskInfo.getDataMaskType(); + + if (StringUtils.startsWith(maskType, prefix)) { + dataMaskInfo.setDataMaskType(StringUtils.removeStart(maskType, prefix)); + } else if (maskType.contains(AbstractServiceStore.COMPONENT_ACCESSTYPE_SEPARATOR)) { + if (itemsToPrune == null) { + itemsToPrune = new ArrayList<>(); + } + + itemsToPrune.add(policyItem); + } + } + } + + if(itemsToPrune != null) { + policyItems.removeAll(itemsToPrune); + } + } + + return policyItems; + } + + private static boolean isDelegateAdminPolicy(RangerPolicy policy) { + boolean ret = hasDelegateAdminItems(policy.getPolicyItems()) + || hasDelegateAdminItems(policy.getDenyPolicyItems()) + || hasDelegateAdminItems(policy.getAllowExceptions()) + || hasDelegateAdminItems(policy.getDenyExceptions()); + + return ret; + } + + private static boolean hasDelegateAdminItems(List items) { + boolean ret = false; + + if (CollectionUtils.isNotEmpty(items)) { + for (RangerPolicy.RangerPolicyItem item : items) { + if(item.getDelegateAdmin()) { + ret = true; + + break; + } + } + } + return ret; + } + + private static boolean skipBuildingPolicyEvaluator(RangerPolicy policy, RangerPolicyEngineOptions options) { + boolean ret = false; + if (!policy.getIsEnabled()) { + ret = true; + } else if (options.evaluateDelegateAdminOnly && !isDelegateAdminPolicy(policy)) { + ret = true; + } + return ret; + } + + private void init(RangerPolicyEngineOptions options) { + RangerServiceDefHelper serviceDefHelper = new RangerServiceDefHelper(serviceDef, false); + options.setServiceDefHelper(serviceDefHelper); + + List policyEvaluators = new ArrayList<>(); + List dataMaskPolicyEvaluators = new ArrayList<>(); + List rowFilterPolicyEvaluators = new ArrayList<>(); + + for (RangerPolicy policy : policies) { + if (skipBuildingPolicyEvaluator(policy, options)) { + continue; + } + + RangerPolicyEvaluator evaluator = buildPolicyEvaluator(policy, serviceDef, options); + + if (evaluator != null) { + if(StringUtils.isEmpty(policy.getPolicyType()) || RangerPolicy.POLICY_TYPE_ACCESS.equals(policy.getPolicyType())) { + policyEvaluators.add(evaluator); + } else if(RangerPolicy.POLICY_TYPE_DATAMASK.equals(policy.getPolicyType())) { + dataMaskPolicyEvaluators.add(evaluator); + } else if(RangerPolicy.POLICY_TYPE_ROWFILTER.equals(policy.getPolicyType())) { + rowFilterPolicyEvaluators.add(evaluator); + } else { + LOG.warn("RangerPolicyEngine: ignoring policy id=" + policy.getId() + " - invalid policyType '" + policy.getPolicyType() + "'"); + } + } + } + if (LOG.isInfoEnabled()) { + LOG.info("This policy engine contains " + (policyEvaluators.size()+dataMaskPolicyEvaluators.size()+rowFilterPolicyEvaluators.size()) + " policy evaluators"); + } + RangerPolicyEvaluator.PolicyEvalOrderComparator comparator = new RangerPolicyEvaluator.PolicyEvalOrderComparator(); + Collections.sort(policyEvaluators, comparator); + this.policyEvaluators = policyEvaluators; + + Collections.sort(dataMaskPolicyEvaluators, comparator); + this.dataMaskPolicyEvaluators = dataMaskPolicyEvaluators; + + Collections.sort(rowFilterPolicyEvaluators, comparator); + this.rowFilterPolicyEvaluators = rowFilterPolicyEvaluators; + + this.policyEvaluatorsMap = createPolicyEvaluatorsMap(); + + if(LOG.isDebugEnabled()) { + LOG.debug("policy evaluation order: " + this.policyEvaluators.size() + " policies"); + + int order = 0; + for(RangerPolicyEvaluator policyEvaluator : this.policyEvaluators) { + RangerPolicy policy = policyEvaluator.getPolicy(); + + LOG.debug("policy evaluation order: #" + (++order) + " - policy id=" + policy.getId() + "; name=" + policy.getName() + "; evalOrder=" + policyEvaluator.getEvalOrder()); + } + + LOG.debug("dataMask policy evaluation order: " + this.dataMaskPolicyEvaluators.size() + " policies"); + order = 0; + for(RangerPolicyEvaluator policyEvaluator : this.dataMaskPolicyEvaluators) { + RangerPolicy policy = policyEvaluator.getPolicy(); + + LOG.debug("dataMask policy evaluation order: #" + (++order) + " - policy id=" + policy.getId() + "; name=" + policy.getName() + "; evalOrder=" + policyEvaluator.getEvalOrder()); + } + + LOG.debug("rowFilter policy evaluation order: " + this.rowFilterPolicyEvaluators.size() + " policies"); + order = 0; + for(RangerPolicyEvaluator policyEvaluator : this.rowFilterPolicyEvaluators) { + RangerPolicy policy = policyEvaluator.getPolicy(); + + LOG.debug("rowFilter policy evaluation order: #" + (++order) + " - policy id=" + policy.getId() + "; name=" + policy.getName() + "; evalOrder=" + policyEvaluator.getEvalOrder()); + } + + LOG.debug("audit policy evaluation order: " + this.auditPolicyEvaluators.size() + " policies"); + order = 0; + for(RangerPolicyEvaluator policyEvaluator : this.auditPolicyEvaluators) { + RangerPolicy policy = policyEvaluator.getPolicy(); + + LOG.debug("rowFilter policy evaluation order: #" + (++order) + " - policy id=" + policy.getId() + "; name=" + policy.getName() + "; evalOrder=" + policyEvaluator.getEvalOrder()); + } + } + } + + private List buildContextEnrichers(RangerPolicyEngineOptions options) { + List contextEnrichers = new ArrayList<>(); + + if (StringUtils.isEmpty(zoneName) && CollectionUtils.isNotEmpty(serviceDef.getContextEnrichers())) { + for (RangerServiceDef.RangerContextEnricherDef enricherDef : serviceDef.getContextEnrichers()) { + if (enricherDef == null) { + continue; + } + + if (options.disableTagRetriever && StringUtils.equals(enricherDef.getEnricher(), RangerTagEnricher.class.getName())) { + if (MapUtils.isNotEmpty(enricherDef.getEnricherOptions())) { + Map enricherOptions = new HashMap<>(enricherDef.getEnricherOptions()); + + enricherOptions.remove(TAG_RETRIEVER_CLASSNAME_OPTION); + + enricherDef = new RangerServiceDef.RangerContextEnricherDef(enricherDef.getItemId(), enricherDef.getName(), enricherDef.getEnricher(), enricherOptions); + } + } + + if (!options.disableContextEnrichers || options.enableTagEnricherWithLocalRefresher && StringUtils.equals(enricherDef.getEnricher(), RangerTagEnricher.class.getName())) { + // This will be true only if the engine is initialized within ranger-admin + RangerServiceDef.RangerContextEnricherDef contextEnricherDef = enricherDef; + + if (options.enableTagEnricherWithLocalRefresher && StringUtils.equals(enricherDef.getEnricher(), RangerTagEnricher.class.getName())) { + contextEnricherDef = new RangerServiceDef.RangerContextEnricherDef(enricherDef.getItemId(), enricherDef.getName(), "org.apache.atlas.common.RangerAdminTagEnricher", null); + } + + RangerContextEnricher contextEnricher = buildContextEnricher(contextEnricherDef, options); + + if (contextEnricher != null) { + contextEnrichers.add(contextEnricher); + } + } + } + } + return contextEnrichers; + } + + private RangerContextEnricher buildContextEnricher(RangerServiceDef.RangerContextEnricherDef enricherDef, RangerPolicyEngineOptions options) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerPolicyRepository.buildContextEnricher(" + enricherDef + ")"); + } + + RangerContextEnricher ret = null; + + Map enricherDefOptions = enricherDef.getEnricherOptions(); + + String isEnabledAsString = enricherDefOptions.get("IsEnabled"); + + if (!StringUtils.equalsIgnoreCase(isEnabledAsString, "false")) { + RangerPerfTracer perf = null; + + if (RangerPerfTracer.isPerfTraceEnabled(PERF_CONTEXTENRICHER_INIT_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_CONTEXTENRICHER_INIT_LOG, "RangerContextEnricher.init(appId=" + appId + ",name=" + enricherDef.getName() + ")"); + } + + String name = enricherDef != null ? enricherDef.getName() : null; + String clsName = enricherDef != null ? enricherDef.getEnricher() : null; + + if (!StringUtils.isEmpty(clsName)) { + try { + @SuppressWarnings("unchecked") + Class enricherClass = (Class) Class.forName(clsName); + + ret = enricherClass.newInstance(); + } catch (Exception excp) { + LOG.error("failed to instantiate context enricher '" + clsName + "' for '" + name + "'", excp); + } + } + + if (ret != null) { + ret.setEnricherDef(enricherDef); + ret.setServiceName(componentServiceName); + ret.setServiceDef(componentServiceDef); + ret.setAppId(appId); + if (ret instanceof RangerAbstractContextEnricher) { + RangerAbstractContextEnricher abstractContextEnricher = (RangerAbstractContextEnricher) ret; + abstractContextEnricher.setPluginContext(pluginContext); + abstractContextEnricher.setPolicyEngineOptions(options); + } + ret.init(); + } + + RangerPerfTracer.log(perf); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerPolicyRepository.buildContextEnricher(" + enricherDef + "): " + ret); + } + return ret; + } + + private RangerPolicyEvaluator buildPolicyEvaluator(RangerPolicy policy, RangerServiceDef serviceDef, RangerPolicyEngineOptions options) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerPolicyRepository.buildPolicyEvaluator(" + policy + "," + serviceDef + ", " + options + ")"); + } + + scrubPolicy(policy); + RangerAbstractPolicyEvaluator ret; + + if(StringUtils.equalsIgnoreCase(options.evaluatorType, RangerPolicyEvaluator.EVALUATOR_TYPE_CACHED)) { + ret = new RangerCachedPolicyEvaluator(); + } else { + ret = new RangerOptimizedPolicyEvaluator(); + } + + ret.setPluginContext(pluginContext); + ret.init(policy, serviceDef, options); + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerPolicyRepository.buildPolicyEvaluator(" + policy + "," + serviceDef + "): " + ret); + } + + return ret; + } + + private boolean scrubPolicy(RangerPolicy policy) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerPolicyRepository.scrubPolicy(" + policy + ")"); + } + boolean altered = false; + Long policyId = policy.getId(); + Map resourceMap = policy.getResources(); + for (Map.Entry entry : resourceMap.entrySet()) { + String resourceName = entry.getKey(); + RangerPolicy.RangerPolicyResource resource = entry.getValue(); + Iterator iterator = resource.getValues().iterator(); + while (iterator.hasNext()) { + String value = iterator.next(); + if (value == null) { + LOG.warn("RangerPolicyRepository.scrubPolicyResource: found null resource value for " + resourceName + " in policy " + policyId + "! Removing..."); + iterator.remove(); + altered = true; + } + } + } + + scrubPolicyItems(policyId, policy.getPolicyItems()); + scrubPolicyItems(policyId, policy.getAllowExceptions()); + scrubPolicyItems(policyId, policy.getDenyPolicyItems()); + scrubPolicyItems(policyId, policy.getDenyExceptions()); + scrubPolicyItems(policyId, policy.getRowFilterPolicyItems()); + scrubPolicyItems(policyId, policy.getDataMaskPolicyItems()); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerPolicyRepository.scrubPolicy(" + policy + "): " + altered); + } + return altered; + } + + private void scrubPolicyItems(final Long policyId, final List policyItems) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerPolicyRepository.scrubPolicyItems(" + policyId + "): "); + } + + for (RangerPolicy.RangerPolicyItem policyItem : policyItems) { + removeNulls(policyItem.getUsers(), policyId, policyItem); + removeNulls(policyItem.getGroups(), policyId, policyItem); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerPolicyRepository.scrubPolicyItems(" + policyId + "): "); + } + } + + private void removeNulls(Collection strings, final Long policyId, final RangerPolicy.RangerPolicyItem policyItem) { + Iterator iterator = strings.iterator(); + + while (iterator.hasNext()) { + String value = iterator.next(); + if (value == null) { + LOG.warn("RangerPolicyRepository.removeNulls: found null user/group in policyItem '" + policyItem + "' in policy " + policyId + "! Removing..."); + iterator.remove(); + } + } + } + + private List getReorderedPolicyEvaluators(List evaluators) { + List ret = evaluators; + + if (CollectionUtils.isNotEmpty(evaluators)) { + ret = new ArrayList<>(evaluators); + Collections.sort(ret, new RangerPolicyEvaluator.PolicyEvalOrderComparator()); + } + + return ret; + } + + private Map createResourceTrieMap(List evaluators, boolean optimizeTrieForRetrieval) { + final Map ret; + + if (serviceDef != null && CollectionUtils.isNotEmpty(serviceDef.getResources())) { + ret = new HashMap<>(); + + for (RangerServiceDef.RangerResourceDef resourceDef : serviceDef.getResources()) { + ret.put(resourceDef.getName(), new RangerResourceTrie(resourceDef, evaluators, optimizeTrieForRetrieval, pluginContext)); + } + } else { + ret = null; + } + + return ret; + } + + private void updateTrie(Map trieMap, Integer policyDeltaType, RangerPolicyEvaluator oldEvaluator, RangerPolicyEvaluator newEvaluator) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerPolicyRepository.updateTrie(policyDeltaType=" + policyDeltaType + "): "); + } + for (RangerServiceDef.RangerResourceDef resourceDef : serviceDef.getResources()) { + + String resourceDefName = resourceDef.getName(); + + RangerResourceTrie trie = trieMap.get(resourceDefName); + + if (trie == null) { + if (RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE == policyDeltaType || RangerPolicyDelta.CHANGE_TYPE_POLICY_UPDATE == policyDeltaType) { + LOG.warn("policyDeltaType is not for POLICY_CREATE and trie for resourceDef:[" + resourceDefName + "] was null! Should not have happened!!"); + } + trie = new RangerResourceTrie<>(resourceDef, new ArrayList<>(), true, pluginContext); + trieMap.put(resourceDefName, trie); + } + + if (policyDeltaType == RangerPolicyDelta.CHANGE_TYPE_POLICY_CREATE) { + removeEvaluatorFromTrie(oldEvaluator, trie, resourceDefName); + addEvaluatorToTrie(newEvaluator, trie, resourceDefName); + } else if (policyDeltaType == RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE) { + removeEvaluatorFromTrie(oldEvaluator, trie, resourceDefName); + } else if (policyDeltaType == RangerPolicyDelta.CHANGE_TYPE_POLICY_UPDATE) { + removeEvaluatorFromTrie(oldEvaluator, trie, resourceDefName); + addEvaluatorToTrie(newEvaluator, trie, resourceDefName); + } else { + LOG.error("policyDeltaType:" + policyDeltaType + " is currently not handled, policy-id:[" + oldEvaluator.getPolicy().getId() +"]"); + } + } + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerPolicyRepository.updateTrie(policyDeltaType=" + policyDeltaType + "): "); + } + } + + private void addEvaluatorToTrie(RangerPolicyEvaluator newEvaluator, RangerResourceTrie trie, String resourceDefName) { + if (newEvaluator != null) { + RangerPolicy.RangerPolicyResource resource = newEvaluator.getPolicyResource().get(resourceDefName); + trie.add(resource, newEvaluator); + } else { + LOG.warn("Unexpected: newPolicyEvaluator is null for resource:[" + resourceDefName + "]"); + } + } + + private void removeEvaluatorFromTrie(RangerPolicyEvaluator oldEvaluator, RangerResourceTrie trie, String resourceDefName) { + if (oldEvaluator != null) { + trie.delete(oldEvaluator.getPolicyResource().get(resourceDefName), oldEvaluator); + } + } + + private Map createPolicyEvaluatorsMap() { + Map tmpPolicyEvaluatorMap = new HashMap<>(); + + for (RangerPolicyEvaluator evaluator : getPolicyEvaluators()) { + tmpPolicyEvaluatorMap.put(evaluator.getPolicy().getId(), evaluator); + } + for (RangerPolicyEvaluator evaluator : getDataMaskPolicyEvaluators()) { + tmpPolicyEvaluatorMap.put(evaluator.getPolicy().getId(), evaluator); + } + for (RangerPolicyEvaluator evaluator : getRowFilterPolicyEvaluators()) { + tmpPolicyEvaluatorMap.put(evaluator.getPolicy().getId(), evaluator); + } + + return tmpPolicyEvaluatorMap; + } + + + private RangerPolicyEvaluator addPolicy(RangerPolicy policy) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerPolicyRepository.addPolicy(" + policy +")"); + } + RangerPolicyEvaluator ret = null; + + if (StringUtils.equals(this.serviceDef.getName(), this.componentServiceDef.getName()) || !isPolicyNeedsPruning(policy, this.componentServiceDef.getName())) { + policies.add(policy); + + if (!skipBuildingPolicyEvaluator(policy, options)) { + + ret = buildPolicyEvaluator(policy, serviceDef, options); + + if (ret != null) { + if (StringUtils.isEmpty(policy.getPolicyType()) || RangerPolicy.POLICY_TYPE_ACCESS.equals(policy.getPolicyType())) { + policyEvaluators.add(ret); + } else if (RangerPolicy.POLICY_TYPE_DATAMASK.equals(policy.getPolicyType())) { + dataMaskPolicyEvaluators.add(ret); + } else if (RangerPolicy.POLICY_TYPE_ROWFILTER.equals(policy.getPolicyType())) { + rowFilterPolicyEvaluators.add(ret); + } else { + LOG.warn("RangerPolicyEngine: ignoring policy id=" + policy.getId() + " - invalid policyType '" + policy.getPolicyType() + "'"); + } + + if (!RangerPolicy.POLICY_TYPE_AUDIT.equals(policy.getPolicyType())) { + policyEvaluatorsMap.put(policy.getId(), ret); + } + } + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerPolicyRepository.addPolicy(" + policy +"): " + ret); + } + return ret; + } + + private void removePolicy(Long id) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerPolicyRepository.removePolicy(" + id +")"); + } + Iterator iterator = policies.iterator(); + while (iterator.hasNext()) { + if (id.equals(iterator.next().getId())) { + iterator.remove(); + //break; + } + } + + policyEvaluatorsMap.remove(id); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerPolicyRepository.removePolicy(" + id +")"); + } + } + + private void deletePolicyEvaluator(RangerPolicyEvaluator evaluator) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerPolicyRepository.deletePolicyEvaluator(" + evaluator.getPolicy() + ")"); + } + String policyType = evaluator.getPolicy().getPolicyType(); + if (StringUtils.isEmpty(policyType)) { + policyType = RangerPolicy.POLICY_TYPE_ACCESS; + } + + List evaluators = null; + + if (RangerPolicy.POLICY_TYPE_ACCESS.equals(policyType)) { + evaluators = this.policyEvaluators; + } else if (RangerPolicy.POLICY_TYPE_DATAMASK.equals(policyType)) { + evaluators = this.dataMaskPolicyEvaluators; + } else if (RangerPolicy.POLICY_TYPE_ROWFILTER.equals(policyType)) { + evaluators = this.rowFilterPolicyEvaluators; + } else { + LOG.error("Unknown policyType:[" + policyType +"]"); + } + if (evaluators != null) { + evaluators.remove(evaluator); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerPolicyRepository.deletePolicyEvaluator(" + evaluator.getPolicy() + ")"); + } + } + + private RangerPolicyEvaluator update(final RangerPolicyDelta delta, final RangerPolicyEvaluator currentEvaluator) { + + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerPolicyRepository.update(delta=" + delta + ", currentEvaluator=" + (currentEvaluator == null ? null : currentEvaluator.getPolicy()) + ")"); + } + Integer changeType = delta.getChangeType(); + String policyType = delta.getPolicyType(); + Long policyId = delta.getPolicyId(); + + RangerPolicy policy = delta.getPolicy(); + + RangerPolicyEvaluator newEvaluator = null; + + switch (changeType) { + case RangerPolicyDelta.CHANGE_TYPE_POLICY_CREATE: + if (currentEvaluator != null) { + removePolicy(policyId); + } + if (policy != null) { + newEvaluator = addPolicy(policy); + } + break; + case RangerPolicyDelta.CHANGE_TYPE_POLICY_UPDATE: { + if (currentEvaluator != null) { + removePolicy(policyId); + } + if (policy != null) { + newEvaluator = addPolicy(policy); + } + } + break; + case RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE: { + if (currentEvaluator != null) { + removePolicy(policyId); + } + } + break; + } + + Map trieMap = getTrie(policyType); + + if (trieMap != null) { + updateTrie(trieMap, changeType, currentEvaluator, newEvaluator); + } + + if (changeType == RangerPolicyDelta.CHANGE_TYPE_POLICY_UPDATE || changeType == RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE) { + if (currentEvaluator != null) { + deletePolicyEvaluator(currentEvaluator); + } + } + + RangerPolicyEvaluator ret = changeType == RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE ? currentEvaluator : newEvaluator; + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerPolicyRepository.update(delta=" + delta + ", currentEvaluator=" + (currentEvaluator == null ? null : currentEvaluator.getPolicy()) + ")"); + } + + return ret; + } + + Map getTrie(final String policyType) { + final Map ret; + switch (policyType) { + case RangerPolicy.POLICY_TYPE_ACCESS: + ret = policyResourceTrie; + break; + case RangerPolicy.POLICY_TYPE_DATAMASK: + ret = dataMaskResourceTrie; + break; + case RangerPolicy.POLICY_TYPE_ROWFILTER: + ret = rowFilterResourceTrie; + break; + case RangerPolicy.POLICY_TYPE_AUDIT: + ret = auditFilterResourceTrie; + break; + default: + ret = null; + } + return ret; + } + + static private final class AuditInfo { + final boolean isAudited; + final String auditPolicyId; + + AuditInfo(boolean isAudited, String auditPolicyId) { + this.isAudited = isAudited; + this.auditPolicyId = auditPolicyId; + } + String getAuditPolicyId() { + return this.auditPolicyId; + } + boolean getIsAudited() { + return isAudited; + } + } + + void reinit(List deltas) { + final boolean isExistingPolicies = CollectionUtils.isNotEmpty(this.policies); + + updateResourceTrie(deltas); + + if (StringUtils.isEmpty(zoneName) && CollectionUtils.isNotEmpty(this.policies)) { + if (!isExistingPolicies) { + this.contextEnrichers = buildContextEnrichers(options); + } + } else { + this.contextEnrichers = null; + } + } + + private void updateResourceTrie(List deltas) { + + Set flags = new HashSet<>(); + + for (RangerPolicyDelta delta : deltas) { + final Integer changeType = delta.getChangeType(); + final String serviceType = delta.getServiceType(); + final Long policyId = delta.getPolicyId(); + final String policyType = delta.getPolicyType(); + + if (!serviceType.equals(this.serviceDef.getName())) { + continue; + } + + RangerPolicyEvaluator evaluator = null; + + switch (changeType) { + case RangerPolicyDelta.CHANGE_TYPE_POLICY_CREATE: + if (delta.getPolicy() == null) { + LOG.warn("Could not find policy for policy-id:[" + policyId + "]"); + continue; + } + evaluator = getPolicyEvaluator(policyId); + if (evaluator != null) { + LOG.warn("Unexpected: Found evaluator for policy-id:[" + policyId + "], changeType=CHANGE_TYPE_POLICY_CREATE"); + } + + break; + + case RangerPolicyDelta.CHANGE_TYPE_POLICY_UPDATE: + evaluator = getPolicyEvaluator(policyId); + + if (evaluator == null) { + LOG.warn("Unexpected: Did not find evaluator for policy-id:[" + policyId + "], changeType=CHANGE_TYPE_POLICY_UPDATE"); + } + break; + + case RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE: + evaluator = getPolicyEvaluator(policyId); + if (evaluator == null) { + LOG.warn("Unexpected: Did not find evaluator for policy-id:[" + policyId + "], changeType=CHANGE_TYPE_POLICY_DELETE"); + } + break; + + default: + LOG.error("Unknown changeType:[" + changeType + "], Ignoring"); + break; + } + + evaluator = update(delta, evaluator); + + if (evaluator != null) { + switch (changeType) { + case RangerPolicyDelta.CHANGE_TYPE_POLICY_CREATE: + policyEvaluatorsMap.put(policyId, evaluator); + break; + + case RangerPolicyDelta.CHANGE_TYPE_POLICY_UPDATE: + policyEvaluatorsMap.put(policyId, evaluator); + break; + + case RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE: + policyEvaluatorsMap.remove(policyId); + break; + + default: + break; + } + + flags.add(policyType); + } + } + + for (String policyType : flags) { + Map trie = getTrie(policyType); + + if (trie != null) { + for (Map.Entry entry : trie.entrySet()) { + entry.getValue().wrapUpUpdate(); + } + } + } + + if (auditFilterResourceTrie != null) { + for (Map.Entry entry : auditFilterResourceTrie.entrySet()) { + entry.getValue().wrapUpUpdate(); + } + } + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerResourceACLs.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerResourceACLs.java new file mode 100644 index 00000000000..e4a34c917ba --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerResourceACLs.java @@ -0,0 +1,548 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.policyengine; + +import org.apache.commons.lang.StringUtils; +import org.apache.atlas.plugin.model.RangerPolicy; +import org.apache.atlas.plugin.model.RangerPolicy.RangerPolicyItemDataMaskInfo; +import org.apache.atlas.plugin.model.RangerPolicy.RangerPolicyItemRowFilterInfo; +import org.codehaus.jackson.annotate.JsonAutoDetect; +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +import static org.apache.atlas.plugin.policyevaluator.RangerPolicyEvaluator.ACCESS_ALLOWED; +import static org.apache.atlas.plugin.policyevaluator.RangerPolicyEvaluator.ACCESS_DENIED; + +public class RangerResourceACLs { + final private Map> userACLs = new HashMap<>(); + final private Map> groupACLs = new HashMap<>(); + final private Map> roleACLs = new HashMap<>(); + final private List rowFilters = new ArrayList<>(); + final private List dataMasks = new ArrayList<>(); + + public RangerResourceACLs() { + } + + public Map> getUserACLs() { + return userACLs; + } + + public Map> getGroupACLs() { + return groupACLs; + } + + public Map> getRoleACLs() { return roleACLs; } + + public List getRowFilters() { return rowFilters; } + + public List getDataMasks() { return dataMasks; } + + public void finalizeAcls() { + Map publicGroupAccessInfo = groupACLs.get(RangerPolicyEngine.GROUP_PUBLIC); + if (publicGroupAccessInfo != null) { + + for (Map.Entry entry : publicGroupAccessInfo.entrySet()) { + String accessType = entry.getKey(); + AccessResult accessResult = entry.getValue(); + int access = accessResult.getResult(); + + if (access == ACCESS_DENIED || access == ACCESS_ALLOWED) { + for (Map.Entry> mapEntry : userACLs.entrySet()) { + Map mapValue = mapEntry.getValue(); + AccessResult savedAccessResult = mapValue.get(accessType); + if (savedAccessResult != null && !savedAccessResult.getIsFinal()) { + mapValue.remove(accessType); + } + } + + for (Map.Entry> mapEntry : groupACLs.entrySet()) { + if (!StringUtils.equals(mapEntry.getKey(), RangerPolicyEngine.GROUP_PUBLIC)) { + Map mapValue = mapEntry.getValue(); + AccessResult savedAccessResult = mapValue.get(accessType); + if (savedAccessResult != null && !savedAccessResult.getIsFinal()) { + mapValue.remove(accessType); + } + } + } + } + } + } + finalizeAcls(userACLs); + finalizeAcls(groupACLs); + finalizeAcls(roleACLs); + } + + public void setUserAccessInfo(String userName, String accessType, Integer access, RangerPolicy policy) { + Map userAccessInfo = userACLs.get(userName); + + if (userAccessInfo == null) { + userAccessInfo = new HashMap<>(); + + userACLs.put(userName, userAccessInfo); + } + + AccessResult accessResult = userAccessInfo.get(accessType); + + if (accessResult == null) { + accessResult = new AccessResult(access, policy); + + userAccessInfo.put(accessType, accessResult); + } else { + accessResult.setResult(access); + accessResult.setPolicy(policy); + } + } + + public void setGroupAccessInfo(String groupName, String accessType, Integer access, RangerPolicy policy) { + Map groupAccessInfo = groupACLs.get(groupName); + + if (groupAccessInfo == null) { + groupAccessInfo = new HashMap<>(); + + groupACLs.put(groupName, groupAccessInfo); + } + + AccessResult accessResult = groupAccessInfo.get(accessType); + + if (accessResult == null) { + accessResult = new AccessResult(access, policy); + + groupAccessInfo.put(accessType, accessResult); + } else { + accessResult.setResult(access); + accessResult.setPolicy(policy); + } + } + + public void setRoleAccessInfo(String roleName, String accessType, Integer access, RangerPolicy policy) { + Map roleAccessInfo = roleACLs.get(roleName); + + if (roleAccessInfo == null) { + roleAccessInfo = new HashMap<>(); + + roleACLs.put(roleName, roleAccessInfo); + } + + AccessResult accessResult = roleAccessInfo.get(accessType); + + if (accessResult == null) { + accessResult = new AccessResult(access, policy); + + roleAccessInfo.put(accessType, accessResult); + } else { + accessResult.setResult(access); + accessResult.setPolicy(policy); + } + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + + sb.append("{"); + + sb.append("UserACLs={"); + for (Map.Entry> entry : userACLs.entrySet()) { + sb.append("user=").append(entry.getKey()).append(":"); + sb.append("permissions={"); + for (Map.Entry permission : entry.getValue().entrySet()) { + sb.append("{Permission=").append(permission.getKey()).append(", value=").append(permission.getValue()).append("},"); + sb.append("{RangerPolicyID=").append(permission.getValue().getPolicy() == null ? null : permission.getValue().getPolicy().getId()).append("},"); + } + sb.append("},"); + } + sb.append("}"); + + sb.append(", GroupACLs={"); + for (Map.Entry> entry : groupACLs.entrySet()) { + sb.append("group=").append(entry.getKey()).append(":"); + sb.append("permissions={"); + for (Map.Entry permission : entry.getValue().entrySet()) { + sb.append("{Permission=").append(permission.getKey()).append(", value=").append(permission.getValue()).append("}, "); + sb.append("{RangerPolicy ID=").append(permission.getValue().getPolicy() == null ? null : permission.getValue().getPolicy().getId()).append("},"); + } + sb.append("},"); + } + sb.append("}"); + + sb.append(", RoleACLs={"); + for (Map.Entry> entry : roleACLs.entrySet()) { + sb.append("role=").append(entry.getKey()).append(":"); + sb.append("permissions={"); + for (Map.Entry permission : entry.getValue().entrySet()) { + sb.append("{Permission=").append(permission.getKey()).append(", value=").append(permission.getValue()).append("}, "); + sb.append("{RangerPolicy ID=").append(permission.getValue().getPolicy() == null ? null : permission.getValue().getPolicy().getId()).append("},"); + } + sb.append("},"); + } + sb.append("}"); + + sb.append("}"); + + sb.append(", rowFilters=["); + for (RowFilterResult rowFilter : rowFilters) { + rowFilter.toString(sb); + sb.append(" "); + } + sb.append("]"); + + sb.append(", dataMasks=["); + for (DataMaskResult dataMask : dataMasks) { + dataMask.toString(sb); + sb.append(" "); + } + sb.append("]"); + + return sb.toString(); + } + + private void finalizeAcls(Map> acls) { + List keysToRemove = new ArrayList<>(); + for (Map.Entry> entry : acls.entrySet()) { + if (entry.getValue().isEmpty()) { + keysToRemove.add(entry.getKey()); + } else { + for (Map.Entry permission : entry.getValue().entrySet()) { + permission.getValue().setIsFinal(true); + } + } + + } + for (String keyToRemove : keysToRemove) { + acls.remove(keyToRemove); + } + } + + @JsonAutoDetect(fieldVisibility=JsonAutoDetect.Visibility.ANY) + @JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL) + @JsonIgnoreProperties(ignoreUnknown=true) + @XmlRootElement + @XmlAccessorType(XmlAccessType.FIELD) + public static class AccessResult { + private int result; + private boolean isFinal; + private RangerPolicy policy; + + public AccessResult() { + this(-1, null); + } + + public AccessResult(int result, RangerPolicy policy) { + this(result, false, policy); + } + + public AccessResult(int result, boolean isFinal, RangerPolicy policy) { + setIsFinal(isFinal); + setResult(result); + setPolicy(policy); + } + + public int getResult() { return result; } + + public void setResult(int result) { + if (!isFinal) { + this.result = result; + + if (this.result == ACCESS_DENIED) { + isFinal = true; + } + } + } + + public boolean getIsFinal() { return isFinal; } + + public void setIsFinal(boolean isFinal) { this.isFinal = isFinal; } + + public RangerPolicy getPolicy() { + return policy; + } + + public void setPolicy(RangerPolicy policy){ + this.policy = policy; + } + + @Override + public boolean equals(Object other) { + if (other == null) + return false; + if (other instanceof AccessResult) { + AccessResult otherObject = (AccessResult)other; + return result == otherObject.result && isFinal == otherObject.isFinal; + } else + return false; + + } + @Override + public String toString() { + if (result == ACCESS_ALLOWED) { + return "ALLOWED, final=" + isFinal; + } + if (result == ACCESS_DENIED) { + return "NOT_ALLOWED, final=" + isFinal; + } + return "CONDITIONAL_ALLOWED, final=" + isFinal; + } + } + + @JsonAutoDetect(fieldVisibility=JsonAutoDetect.Visibility.ANY) + @JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL) + @JsonIgnoreProperties(ignoreUnknown=true) + @XmlRootElement + @XmlAccessorType(XmlAccessType.FIELD) + public static class DataMaskResult implements Serializable { + private static final long serialVersionUID = 1L; + + private final Set users; + private final Set groups; + private final Set roles; + private final Set accessTypes; + private final RangerPolicyItemDataMaskInfo maskInfo; + private boolean isConditional = false; + + public DataMaskResult(Set users, Set groups, Set roles, Set accessTypes, RangerPolicyItemDataMaskInfo maskInfo) { + this.users = users; + this.groups = groups; + this.roles = roles; + this.accessTypes = accessTypes; + this.maskInfo = maskInfo; + } + + public DataMaskResult(DataMaskResult that) { + this.users = that.users; + this.groups = that.groups; + this.roles = that.roles; + this.accessTypes = that.accessTypes; + this.maskInfo = that.maskInfo; + this.isConditional = that.isConditional; + } + + public Set getUsers() { return users; } + + public Set getGroups() { return groups; } + + public Set getRoles() { return roles; } + + public Set getAccessTypes() { return accessTypes; } + + public RangerPolicyItemDataMaskInfo getMaskInfo() { return maskInfo; } + + public boolean getIsConditional() { return isConditional; } + + public void setIsConditional(boolean isConditional) { this.isConditional = isConditional; } + + @Override + public int hashCode() { return Objects.hash(users, groups, roles, accessTypes, maskInfo, isConditional); } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } else if (other == null || getClass() != other.getClass()) { + return false; + } else { + DataMaskResult that = (DataMaskResult) other; + + return Objects.equals(users, that.users) && + Objects.equals(groups, that.groups) && + Objects.equals(roles, that.roles) && + Objects.equals(accessTypes, that.accessTypes) && + Objects.equals(maskInfo, that.maskInfo) && + isConditional == that.isConditional; + } + } + + @Override + public String toString() { + return toString(new StringBuilder()).toString(); + } + + public StringBuilder toString(StringBuilder sb) { + sb.append("{"); + + if (users != null && !users.isEmpty()) { + sb.append("users:["); + for (String user : users) { + sb.append(user).append(' '); + } + sb.append("] "); + } + + if (groups != null && !groups.isEmpty()) { + sb.append("groups:["); + for (String group : groups) { + sb.append(group).append(' '); + } + sb.append("] "); + } + + if (roles != null && !roles.isEmpty()) { + sb.append("roles:["); + for (String role : roles) { + sb.append(role).append(' '); + } + sb.append("] "); + } + + if (accessTypes != null && !accessTypes.isEmpty()) { + sb.append("accessTypes:["); + for (String accessType : accessTypes) { + sb.append(accessType).append(' '); + } + sb.append("] "); + } + + sb.append("maskInfo="); + maskInfo.toString(sb); + sb.append(" isConditional=").append(isConditional); + + sb.append("}"); + + return sb; + } + } + + @JsonAutoDetect(fieldVisibility=JsonAutoDetect.Visibility.ANY) + @JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL) + @JsonIgnoreProperties(ignoreUnknown=true) + @XmlRootElement + @XmlAccessorType(XmlAccessType.FIELD) + public static class RowFilterResult implements Serializable { + private static final long serialVersionUID = 1L; + + private final Set users; + private final Set groups; + private final Set roles; + private final Set accessTypes; + private final RangerPolicyItemRowFilterInfo filterInfo; + private boolean isConditional = false; + + public RowFilterResult(Set users, Set groups, Set roles, Set accessTypes, RangerPolicyItemRowFilterInfo filterInfo) { + this.users = users; + this.groups = groups; + this.roles = roles; + this.accessTypes = accessTypes; + this.filterInfo = filterInfo; + } + + public RowFilterResult(RowFilterResult that) { + this.users = that.users; + this.groups = that.groups; + this.roles = that.roles; + this.accessTypes = that.accessTypes; + this.filterInfo = that.filterInfo; + this.isConditional = that.isConditional; + } + + public Set getUsers() { return users; } + + public Set getGroups() { return groups; } + + public Set getRoles() { return roles; } + + public Set getAccessTypes() { return accessTypes; } + + public RangerPolicyItemRowFilterInfo getFilterInfo() { return filterInfo; } + + public boolean getIsConditional() { return isConditional; } + + public void setIsConditional(boolean isConditional) { this.isConditional = isConditional; } + + @Override + public int hashCode() { return Objects.hash(users, groups, roles, accessTypes, filterInfo, isConditional); } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } else if (other == null || getClass() != other.getClass()) { + return false; + } else { + RowFilterResult that = (RowFilterResult) other; + + return Objects.equals(users, that.users) && + Objects.equals(groups, that.groups) && + Objects.equals(roles, that.roles) && + Objects.equals(accessTypes, that.accessTypes) && + Objects.equals(filterInfo, that.filterInfo) && + isConditional == that.isConditional; + } + } + + @Override + public String toString() { + return toString(new StringBuilder()).toString(); + } + + public StringBuilder toString(StringBuilder sb) { + sb.append("{"); + + if (users != null && !users.isEmpty()) { + sb.append("users:["); + for (String user : users) { + sb.append(user).append(' '); + } + sb.append("] "); + } + + if (groups != null && !groups.isEmpty()) { + sb.append("groups:["); + for (String group : groups) { + sb.append(group).append(' '); + } + sb.append("] "); + } + + if (roles != null && !roles.isEmpty()) { + sb.append("roles:["); + for (String role : roles) { + sb.append(role).append(' '); + } + sb.append("] "); + } + + if (accessTypes != null && !accessTypes.isEmpty()) { + sb.append("accessTypes:["); + for (String accessType : accessTypes) { + sb.append(accessType).append(' '); + } + sb.append("] "); + } + + sb.append("filterInfo="); + filterInfo.toString(sb); + sb.append(" isConditional=").append(isConditional); + + sb.append("}"); + + return sb; + } + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerResourceAccessInfo.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerResourceAccessInfo.java new file mode 100644 index 00000000000..12f3dfd3f26 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerResourceAccessInfo.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.policyengine; + +import java.util.HashSet; +import java.util.Set; + +public class RangerResourceAccessInfo { + final private RangerAccessRequest request; + final private Set allowedUsers; + final private Set allowedGroups; + final private Set deniedUsers; + final private Set deniedGroups; + + + public RangerResourceAccessInfo(RangerAccessRequest request) { + this.request = request; + this.allowedUsers = new HashSet<>(); + this.allowedGroups = new HashSet<>(); + this.deniedUsers = new HashSet<>(); + this.deniedGroups = new HashSet<>(); + } + + public RangerResourceAccessInfo(RangerResourceAccessInfo other) { + this.request = other.request; + this.allowedUsers = other.allowedUsers == null ? new HashSet() : new HashSet(other.allowedUsers); + this.allowedGroups = other.allowedGroups == null ? new HashSet() : new HashSet(other.allowedGroups); + this.deniedUsers = other.deniedUsers == null ? new HashSet() : new HashSet(other.deniedUsers); + this.deniedGroups = other.deniedGroups == null ? new HashSet() : new HashSet(other.deniedGroups); + } + + public RangerAccessRequest getRequest() { + return request; + } + + public Set getAllowedUsers() { + return allowedUsers; + } + + public Set getAllowedGroups() { + return allowedGroups; + } + + public Set getDeniedUsers() { + return deniedUsers; + } + + public Set getDeniedGroups() { + return deniedGroups; + } + + @Override + public String toString( ) { + StringBuilder sb = new StringBuilder(); + + toString(sb); + + return sb.toString(); + } + + public StringBuilder toString(StringBuilder sb) { + sb.append("RangerResourceAccessInfo={"); + + sb.append("request={"); + if(request != null) { + sb.append(request); + } + sb.append("} "); + + sb.append("allowedUsers={"); + for(String user : allowedUsers) { + sb.append(user).append(" "); + } + sb.append("} "); + + sb.append("allowedGroups={"); + for(String group : allowedGroups) { + sb.append(group).append(" "); + } + sb.append("} "); + + sb.append("deniedUsers={"); + for(String user : deniedUsers) { + sb.append(user).append(" "); + } + sb.append("} "); + + sb.append("deniedGroups={"); + for(String group : deniedGroups) { + sb.append(group).append(" "); + } + sb.append("} "); + + sb.append("}"); + + return sb; + } + +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerResourceTrie.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerResourceTrie.java new file mode 100644 index 00000000000..ffcf768b19e --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerResourceTrie.java @@ -0,0 +1,1217 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.policyengine; + + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.atlas.plugin.model.RangerPolicy.RangerPolicyResource; +import org.apache.atlas.plugin.model.RangerServiceDef.RangerResourceDef; +import org.apache.atlas.plugin.policyresourcematcher.RangerPolicyResourceEvaluator; +import org.apache.atlas.plugin.resourcematcher.RangerAbstractResourceMatcher; +import org.apache.atlas.plugin.resourcematcher.RangerResourceMatcher; +import org.apache.atlas.plugin.util.RangerPerfTracer; +import org.apache.atlas.plugin.util.ServiceDefUtil; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; + +import static org.apache.atlas.plugin.resourcematcher.RangerPathResourceMatcher.DEFAULT_PATH_SEPARATOR_CHAR; +import static org.apache.atlas.plugin.resourcematcher.RangerPathResourceMatcher.OPTION_PATH_SEPARATOR; + +public class RangerResourceTrie { + private static final Log LOG = LogFactory.getLog(RangerResourceTrie.class); + private static final Log TRACE_LOG = RangerPerfTracer.getPerfLogger("resourcetrie.trace"); + private static final Log PERF_TRIE_INIT_LOG = RangerPerfTracer.getPerfLogger("resourcetrie.init"); + private static final Log PERF_TRIE_OP_LOG = RangerPerfTracer.getPerfLogger("resourcetrie.op"); + + private static final String DEFAULT_WILDCARD_CHARS = "*?"; + private static final String TRIE_BUILDER_THREAD_COUNT = "ranger.policyengine.trie.builder.thread.count"; + + private final RangerResourceDef resourceDef; + private final boolean optIgnoreCase; + private final boolean optWildcard; + private final String wildcardChars; + private final boolean isOptimizedForRetrieval; + private final Character separatorChar; + private Set inheritedEvaluators; + private final TrieNode root; + + public RangerResourceTrie(RangerResourceDef resourceDef, List evaluators) { + this(resourceDef, evaluators, true, null); + } + + public RangerResourceTrie(RangerResourceTrie other) { + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_TRIE_INIT_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_TRIE_INIT_LOG, "RangerResourceTrie.copyTrie(name=" + other.resourceDef.getName() + ")"); + } + + this.resourceDef = other.resourceDef; + this.optIgnoreCase = other.optIgnoreCase; + this.optWildcard = other.optWildcard; + this.wildcardChars = other.wildcardChars; + this.isOptimizedForRetrieval = false; + this.separatorChar = other.separatorChar; + this.inheritedEvaluators = other.inheritedEvaluators != null ? new HashSet<>(other.inheritedEvaluators) : null; + this.root = copyTrieSubtree(other.root, null); + + RangerPerfTracer.logAlways(perf); + + if (PERF_TRIE_INIT_LOG.isDebugEnabled()) { + PERF_TRIE_INIT_LOG.debug(toString()); + } + + if (TRACE_LOG.isTraceEnabled()) { + StringBuilder sb = new StringBuilder(); + root.toString("", sb); + TRACE_LOG.trace("Trie Dump from RangerResourceTrie.copyTrie(name=" + other.resourceDef.getName() + "):\n{" + sb.toString() + "}"); + } + } + + public RangerResourceTrie(RangerResourceDef resourceDef, List evaluators, boolean isOptimizedForRetrieval, RangerPluginContext pluginContext) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerResourceTrie(" + resourceDef.getName() + ", evaluatorCount=" + evaluators.size() + ", isOptimizedForRetrieval=" + isOptimizedForRetrieval + ")"); + } + + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_TRIE_INIT_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_TRIE_INIT_LOG, "RangerResourceTrie.init(name=" + resourceDef.getName() + ")"); + } + + Configuration config = pluginContext != null ? pluginContext.getConfig() : null; + int builderThreadCount = config != null ? config.getInt(TRIE_BUILDER_THREAD_COUNT, 1) : 1; + + if (builderThreadCount < 1) { + builderThreadCount = 1; + } + + if (TRACE_LOG.isTraceEnabled()) { + TRACE_LOG.trace("builderThreadCount is set to [" + builderThreadCount + "]"); + } + + Map matcherOptions = resourceDef.getMatcherOptions(); + boolean optReplaceTokens = RangerAbstractResourceMatcher.getOptionReplaceTokens(matcherOptions); + String tokenReplaceSpecialChars = ""; + + if(optReplaceTokens) { + char delimiterStart = RangerAbstractResourceMatcher.getOptionDelimiterStart(matcherOptions); + char delimiterEnd = RangerAbstractResourceMatcher.getOptionDelimiterEnd(matcherOptions); + char delimiterEscape = RangerAbstractResourceMatcher.getOptionDelimiterEscape(matcherOptions); + + tokenReplaceSpecialChars += delimiterStart; + tokenReplaceSpecialChars += delimiterEnd; + tokenReplaceSpecialChars += delimiterEscape; + } + + this.resourceDef = resourceDef; + this.optIgnoreCase = RangerAbstractResourceMatcher.getOptionIgnoreCase(matcherOptions); + this.optWildcard = RangerAbstractResourceMatcher.getOptionWildCard(matcherOptions); + this.wildcardChars = optWildcard ? DEFAULT_WILDCARD_CHARS + tokenReplaceSpecialChars : "" + tokenReplaceSpecialChars; + this.isOptimizedForRetrieval = isOptimizedForRetrieval; + this.separatorChar = ServiceDefUtil.getCharOption(matcherOptions, OPTION_PATH_SEPARATOR, DEFAULT_PATH_SEPARATOR_CHAR); + + TrieNode tmpRoot = buildTrie(resourceDef, evaluators, builderThreadCount); + + if (builderThreadCount > 1 && tmpRoot == null) { // if multi-threaded trie-creation failed, build using a single thread + this.root = buildTrie(resourceDef, evaluators, 1); + } else { + this.root = tmpRoot; + } + + wrapUpUpdate(); + + RangerPerfTracer.logAlways(perf); + + if (PERF_TRIE_INIT_LOG.isDebugEnabled()) { + PERF_TRIE_INIT_LOG.debug(toString()); + } + + if (TRACE_LOG.isTraceEnabled()) { + StringBuilder sb = new StringBuilder(); + root.toString("", sb); + TRACE_LOG.trace("Trie Dump from RangerResourceTrie.init(name=" + resourceDef.getName() + "):\n{" + sb.toString() + "}"); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerResourceTrie(" + resourceDef.getName() + ", evaluatorCount=" + evaluators.size() + ", isOptimizedForRetrieval=" + isOptimizedForRetrieval + "): " + toString()); + } + } + + public Set getInheritedEvaluators() { + return inheritedEvaluators; + } + + public Set getEvaluatorsForResource(Object resource) { + return getEvaluatorsForResource(resource, RangerAccessRequest.ResourceMatchingScope.SELF); + } + + public Set getEvaluatorsForResource(Object resource, RangerAccessRequest.ResourceMatchingScope scope) { + if (resource instanceof String) { + return getEvaluatorsForResource((String) resource, scope); + } else if (resource instanceof Collection) { + if (CollectionUtils.isEmpty((Collection) resource)) { // treat empty collection same as empty-string + return getEvaluatorsForResource("", scope); + } else { + @SuppressWarnings("unchecked") + Collection resources = (Collection) resource; + + return getEvaluatorsForResources(resources, scope); + } + } + + return null; + } + + public void add(RangerPolicyResource resource, T evaluator) { + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_TRIE_INIT_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_TRIE_INIT_LOG, "RangerResourceTrie.add(name=" + resource + ")"); + } + + if (resource == null) { + if (evaluator.isAncestorOf(resourceDef)) { + addInheritedEvaluator(evaluator); + } + } else { + if (resource.getIsExcludes()) { + addInheritedEvaluator(evaluator); + } else { + if (CollectionUtils.isNotEmpty(resource.getValues())) { + for (String value : resource.getValues()) { + insert(root, value, resource.getIsRecursive(), evaluator); + } + } + } + } + + RangerPerfTracer.logAlways(perf); + + if (TRACE_LOG.isTraceEnabled()) { + StringBuilder sb = new StringBuilder(); + root.toString("", sb); + TRACE_LOG.trace("Trie Dump from RangerResourceTrie.add(name=" + resource + "):\n{" + sb.toString() + "}"); + } + } + + public void delete(RangerPolicyResource resource, T evaluator) { + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_TRIE_INIT_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_TRIE_INIT_LOG, "RangerResourceTrie.delete(name=" + resource + ")"); + } + + if (resource == null) { + if (evaluator.isAncestorOf(resourceDef)) { + removeInheritedEvaluator(evaluator); + } + } else if (resource.getIsExcludes()) { + removeInheritedEvaluator(evaluator); + } else { + for (String value : resource.getValues()) { + TrieNode node = getNodeForResource(value); + if (node != null) { + node.removeEvaluatorFromSubtree(evaluator); + } + } + } + + RangerPerfTracer.logAlways(perf); + + if (TRACE_LOG.isTraceEnabled()) { + StringBuilder sb = new StringBuilder(); + root.toString("", sb); + TRACE_LOG.trace("Trie Dump from RangerResourceTrie.delete(name=" + resource + "):\n{" + sb.toString() + "}"); + } + } + + public void wrapUpUpdate() { + if (root != null) { + root.wrapUpUpdate(); + if (TRACE_LOG.isTraceEnabled()) { + StringBuilder sb = new StringBuilder(); + root.toString("", sb); + TRACE_LOG.trace("Trie Dump from RangerResourceTrie.wrapUpUpdate(name=" + resourceDef.getName() + "):\n{" + sb.toString() + "}"); + } + } + } + + TrieNode getRoot() { + return root; + } + + private void addInheritedEvaluator(T evaluator) { + if (inheritedEvaluators == null) { + inheritedEvaluators = new HashSet<>(); + } + + inheritedEvaluators.add(evaluator); + } + + private void removeInheritedEvaluator(T evaluator) { + if (CollectionUtils.isNotEmpty(inheritedEvaluators) && inheritedEvaluators.contains(evaluator)) { + inheritedEvaluators.remove(evaluator); + if (CollectionUtils.isEmpty(inheritedEvaluators)) { + inheritedEvaluators = null; + } + } + } + + private TrieNode copyTrieSubtree(final TrieNode source, final TrieNode parent) { + if (TRACE_LOG.isTraceEnabled()) { + StringBuilder sb = new StringBuilder(); + source.toString(sb); + TRACE_LOG.trace("==> copyTrieSubtree(" + sb + ")"); + } + + TrieNode dest = new TrieNode<>(source.str); + + if (parent != null) { + parent.addChild(dest); + } + + synchronized (source.children) { + dest.isSetup = source.isSetup; + dest.isSharingParentWildcardEvaluators = source.isSharingParentWildcardEvaluators; + + if (source.isSharingParentWildcardEvaluators) { + if (dest.getParent() != null) { + dest.wildcardEvaluators = dest.getParent().getWildcardEvaluators(); + } else { + dest.wildcardEvaluators = null; + } + } else { + if (source.wildcardEvaluators != null) { + dest.wildcardEvaluators = new HashSet<>(source.wildcardEvaluators); + } else { + dest.wildcardEvaluators = null; + } + } + + if (source.evaluators != null) { + if (source.evaluators == source.wildcardEvaluators) { + dest.evaluators = dest.wildcardEvaluators; + } else { + dest.evaluators = new HashSet<>(source.evaluators); + } + } else { + dest.evaluators = null; + } + } + + Map> children = source.getChildren(); + + for (Map.Entry> entry : children.entrySet()) { + copyTrieSubtree(entry.getValue(), dest); + } + + if (TRACE_LOG.isTraceEnabled()) { + StringBuilder sourceAsString = new StringBuilder(), destAsString = new StringBuilder(); + source.toString(sourceAsString); + dest.toString(destAsString); + + TRACE_LOG.trace("<== copyTrieSubtree(" + sourceAsString + ") : " + destAsString); + } + + return dest; + } + + private TrieNode buildTrie(RangerResourceDef resourceDef, List evaluators, int builderThreadCount) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> buildTrie(" + resourceDef.getName() + ", evaluatorCount=" + evaluators.size() + ", isMultiThreaded=" + (builderThreadCount > 1) + ")"); + } + + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_TRIE_INIT_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_TRIE_INIT_LOG, "RangerResourceTrie.init(resourceDef=" + resourceDef.getName() + ")"); + } + + TrieNode ret = new TrieNode<>(null); + final boolean isMultiThreaded = builderThreadCount > 1; + final List builderThreads; + final Map builderThreadMap; + final String resourceName = resourceDef.getName(); + int lastUsedThreadIndex = 0; + + if (isMultiThreaded) { + builderThreads = new ArrayList<>(); + + for (int i = 0; i < builderThreadCount; i++) { + ResourceTrieBuilderThread t = new ResourceTrieBuilderThread(); + + t.setDaemon(true); + builderThreads.add(t); + t.start(); + } + + builderThreadMap = new HashMap<>(); + } else { + builderThreads = null; + builderThreadMap = null; + } + + for (T evaluator : evaluators) { + Map policyResources = evaluator.getPolicyResource(); + RangerPolicyResource policyResource = policyResources != null ? policyResources.get(resourceName) : null; + + if (policyResource == null) { + if (evaluator.isAncestorOf(resourceDef)) { + addInheritedEvaluator(evaluator); + } + + continue; + } + + if (policyResource.getIsExcludes()) { + addInheritedEvaluator(evaluator); + } else { + RangerResourceMatcher resourceMatcher = evaluator.getResourceMatcher(resourceName); + + if (resourceMatcher != null && (resourceMatcher.isMatchAny())) { + ret.addWildcardEvaluator(evaluator); + } else { + if (CollectionUtils.isNotEmpty(policyResource.getValues())) { + for (String resource : policyResource.getValues()) { + if (!isMultiThreaded) { + insert(ret, resource, policyResource.getIsRecursive(), evaluator); + } else { + try { + lastUsedThreadIndex = insert(ret, resource, policyResource.getIsRecursive(), evaluator, builderThreadMap, builderThreads, lastUsedThreadIndex); + } catch (InterruptedException ex) { + LOG.error("Failed to dispatch " + resource + " to " + builderThreads.get(lastUsedThreadIndex)); + LOG.error("Failing and retrying with one thread"); + + ret = null; + + break; + } + } + } + + if (ret == null) { + break; + } + } + } + } + } + + if (ret != null) { + if (isMultiThreaded) { + for (ResourceTrieBuilderThread t : builderThreads) { + try { + // Send termination signal to each thread + t.add("", false, null); + // Wait for threads to finish work + t.join(); + ret.getChildren().putAll(t.getSubtrees()); + } catch (InterruptedException ex) { + LOG.error("BuilderThread " + t + " was interrupted:", ex); + LOG.error("Failing and retrying with one thread"); + + ret = null; + + break; + } + } + + cleanUpThreads(builderThreads); + } + } + + RangerPerfTracer.logAlways(perf); + + if(LOG.isDebugEnabled()) { + LOG.debug("<== buildTrie(" + resourceDef.getName() + ", evaluatorCount=" + evaluators.size() + ", isMultiThreaded=" + isMultiThreaded + ") :" + ret); + } + + return ret; + } + + private void cleanUpThreads(List builderThreads) { + if (CollectionUtils.isNotEmpty(builderThreads)) { + for (ResourceTrieBuilderThread t : builderThreads) { + try { + if (t.isAlive()) { + t.interrupt(); + t.join(); + } + } catch (InterruptedException ex) { + LOG.error("Could not terminate thread " + t); + } + } + } + } + + private TrieData getTrieData() { + TrieData ret = new TrieData(); + + root.populateTrieData(ret); + + ret.maxDepth = getMaxDepth(); + + return ret; + } + + private int getMaxDepth() { + return root.getMaxDepth(); + } + + private Character getLookupChar(char ch) { + return optIgnoreCase ? Character.toLowerCase(ch) : ch; + } + + private Character getLookupChar(String str, int index) { + return getLookupChar(str.charAt(index)); + } + + private int insert(TrieNode currentRoot, String resource, boolean isRecursive, T evaluator, Map builderThreadMap, List builderThreads, int lastUsedThreadIndex) throws InterruptedException { + int ret = lastUsedThreadIndex; + final String prefix = getNonWildcardPrefix(resource); + + if (StringUtils.isNotEmpty(prefix)) { + char c = getLookupChar(prefix.charAt(0)); + Integer index = builderThreadMap.get(c); + + if (index == null) { + ret = index = (lastUsedThreadIndex + 1) % builderThreads.size(); + builderThreadMap.put(c, index); + } + + builderThreads.get(index).add(resource, isRecursive, evaluator); + } else { + currentRoot.addWildcardEvaluator(evaluator); + } + + return ret; + } + + private void insert(TrieNode currentRoot, String resource, boolean isRecursive, T evaluator) { + TrieNode curr = currentRoot; + final String prefix = getNonWildcardPrefix(resource); + final boolean isWildcard = prefix.length() != resource.length(); + + if (StringUtils.isNotEmpty(prefix)) { + curr = curr.getOrCreateChild(prefix); + } + + if(isWildcard || isRecursive) { + curr.addWildcardEvaluator(evaluator); + } else { + curr.addEvaluator(evaluator); + } + + } + + private String getNonWildcardPrefix(String str) { + int minIndex = str.length(); + + for (int i = 0; i < wildcardChars.length(); i++) { + int index = str.indexOf(wildcardChars.charAt(i)); + + if (index != -1 && index < minIndex) { + minIndex = index; + } + } + + return str.substring(0, minIndex); + } + + private Set getEvaluatorsForResource(String resource, RangerAccessRequest.ResourceMatchingScope scope) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerResourceTrie.getEvaluatorsForResource(" + resource + ", " + scope + ")"); + } + + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_TRIE_OP_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_TRIE_OP_LOG, "RangerResourceTrie.getEvaluatorsForResource(resource=" + resource + ")"); + } + + TrieNode curr = root; + TrieNode parent = null; + TrieNode child = null; + final int len = resource.length(); + int i = 0; + + while (i < len) { + if (!isOptimizedForRetrieval) { + curr.setupIfNeeded(parent); + } + + child = curr.getChild(getLookupChar(resource, i)); + + if (child == null) { + break; + } + + final String childStr = child.getStr(); + + if (!resource.regionMatches(optIgnoreCase, i, childStr, 0, childStr.length())) { + break; + } + + parent = curr; + curr = child; + i += childStr.length(); + } + + if (!isOptimizedForRetrieval) { + curr.setupIfNeeded(parent); + } + + boolean isSelfMatch = (i == len); + Set ret = isSelfMatch ? curr.getEvaluators() : curr.getWildcardEvaluators(); + boolean includeEvaluatorsOfChildResources = scope == RangerAccessRequest.ResourceMatchingScope.SELF_OR_CHILD; + + if (includeEvaluatorsOfChildResources) { + final Set childEvalautors = new HashSet<>(); + final boolean resourceEndsWithSep = resource.charAt(resource.length() - 1) == separatorChar; + + if (isSelfMatch) { // resource == path(curr) + if (resourceEndsWithSep) { // ex: resource=/tmp/ + curr.getChildren().values().stream().forEach(c -> c.collectChildEvaluators(separatorChar, 0, childEvalautors)); + } else { // ex: resource=/tmp + curr = curr.getChild(separatorChar); + + if (curr != null) { + curr.collectChildEvaluators(separatorChar, 1, childEvalautors); + } + } + } else if (child != null) { // resource != path(child) ex: (resource=/tmp, path(child)=/tmp/test.txt or path(child)=/tmpdir) + int remainingLen = len - i; + boolean isPrefixMatch = child.getStr().regionMatches(optIgnoreCase, 0, resource, i, remainingLen); + + if (isPrefixMatch) { + if (resourceEndsWithSep) { // ex: resource=/tmp/ + child.collectChildEvaluators(separatorChar, remainingLen, childEvalautors); + } else if (child.getStr().charAt(remainingLen) == separatorChar) { // ex: resource=/tmp + child.collectChildEvaluators(separatorChar, remainingLen + 1, childEvalautors); + } + } + } + + if (CollectionUtils.isNotEmpty(childEvalautors)) { + if (CollectionUtils.isNotEmpty(ret)) { + childEvalautors.addAll(ret); + } + + ret = childEvalautors; + } + } + + RangerPerfTracer.logAlways(perf); + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerResourceTrie.getEvaluatorsForResource(" + resource + ", " + scope + "): evaluatorCount=" + (ret == null ? 0 : ret.size())); + } + + return ret; + } + + private TrieNode getNodeForResource(String resource) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerResourceTrie.getNodeForResource(" + resource + ")"); + } + + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_TRIE_OP_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_TRIE_OP_LOG, "RangerResourceTrie.getNodeForResource(resource=" + resource + ")"); + } + + TrieNode curr = root; + final int len = resource.length(); + int i = 0; + + while (i < len) { + final TrieNode child = curr.getChild(getLookupChar(resource, i)); + + if (child == null) { + break; + } + + final String childStr = child.getStr(); + + if (!resource.regionMatches(optIgnoreCase, i, childStr, 0, childStr.length())) { + break; + } + + curr = child; + i += childStr.length(); + } + + RangerPerfTracer.logAlways(perf); + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerResourceTrie.getNodeForResource(" + resource + ")"); + } + + return curr; + } + + private Set getEvaluatorsForResources(Collection resources, RangerAccessRequest.ResourceMatchingScope scope) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerResourceTrie.getEvaluatorsForResources(" + resources + ")"); + } + + Set ret = null; + Map evaluatorsMap = null; + + for (String resource : resources) { + Set resourceEvaluators = getEvaluatorsForResource(resource, scope); + + if (CollectionUtils.isEmpty(resourceEvaluators)) { + continue; + } + + if (evaluatorsMap == null) { + if (ret == null) { // first resource: don't create map yet + ret = resourceEvaluators; + } else if (ret != resourceEvaluators) { // if evaluator list is same as earlier resources, retain the list, else create a map + evaluatorsMap = new HashMap<>(); + + for (T evaluator : ret) { + evaluatorsMap.put(evaluator.getGuid(), evaluator); + } + + ret = null; + } + } + + if (evaluatorsMap != null) { + for (T evaluator : resourceEvaluators) { + evaluatorsMap.put(evaluator.getGuid(), evaluator); + } + } + } + + if (ret == null && evaluatorsMap != null) { + ret = new HashSet<>(evaluatorsMap.values()); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerResourceTrie.getEvaluatorsForResources(" + resources + "): evaluatorCount=" + (ret == null ? 0 : ret.size())); + } + + return ret; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + + TrieData trieData = getTrieData(); + + sb.append("resourceName=").append(resourceDef.getName()); + sb.append("; optIgnoreCase=").append(optIgnoreCase); + sb.append("; optWildcard=").append(optWildcard); + sb.append("; wildcardChars=").append(wildcardChars); + sb.append("; nodeCount=").append(trieData.nodeCount); + sb.append("; leafNodeCount=").append(trieData.leafNodeCount); + sb.append("; singleChildNodeCount=").append(trieData.singleChildNodeCount); + sb.append("; maxDepth=").append(trieData.maxDepth); + sb.append("; evaluatorListCount=").append(trieData.evaluatorListCount); + sb.append("; wildcardEvaluatorListCount=").append(trieData.wildcardEvaluatorListCount); + sb.append("; evaluatorListRefCount=").append(trieData.evaluatorListRefCount); + sb.append("; wildcardEvaluatorListRefCount=").append(trieData.wildcardEvaluatorListRefCount); + + return sb.toString(); + } + + class ResourceTrieBuilderThread extends Thread { + + class WorkItem { + final String resourceName; + final boolean isRecursive; + final T evaluator; + + WorkItem(String resourceName, boolean isRecursive, T evaluator) { + this.resourceName = resourceName; + this.isRecursive = isRecursive; + this.evaluator = evaluator; + } + + @Override + public String toString() { + return + "resourceName=" + resourceName + + "isRecursive=" + isRecursive + + "evaluator=" + (evaluator != null? evaluator.getId() : null); + } + } + + private final TrieNode thisRoot = new TrieNode<>(null); + private final BlockingQueue workQueue = new LinkedBlockingQueue<>(); + + ResourceTrieBuilderThread() { + } + + void add(String resourceName, boolean isRecursive, T evaluator) throws InterruptedException { + workQueue.put(new WorkItem(resourceName, isRecursive, evaluator)); + } + + Map> getSubtrees() { return thisRoot.getChildren(); } + + @Override + public void run() { + if (LOG.isDebugEnabled()) { + LOG.debug("Running " + this); + } + + while (true) { + final WorkItem workItem; + + try { + workItem = workQueue.take(); + } catch (InterruptedException exception) { + LOG.error("Thread=" + this + " is interrupted", exception); + + break; + } + + if (workItem.evaluator != null) { + insert(thisRoot, workItem.resourceName, workItem.isRecursive, workItem.evaluator); + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Received termination signal. " + workItem); + } + + break; + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("Exiting " + this); + } + } + } + + static class TrieData { + int nodeCount; + int leafNodeCount; + int singleChildNodeCount; + int maxDepth; + int evaluatorListCount; + int wildcardEvaluatorListCount; + int evaluatorListRefCount; + int wildcardEvaluatorListRefCount; + } + + class TrieNode { + private String str; + private TrieNode parent; + private final Map> children = new HashMap<>(); + private Set evaluators; + private Set wildcardEvaluators; + private boolean isSharingParentWildcardEvaluators; + private volatile boolean isSetup = false; + + TrieNode(String str) { + this.str = str; + } + + String getStr() { + return str; + } + + void setStr(String str) { + this.str = str; + } + + TrieNode getParent() { + return parent; + } + + void setParent(TrieNode parent) { + this.parent = parent; + } + + Map> getChildren() { + return children; + } + + Set getEvaluators() { + return evaluators; + } + + Set getWildcardEvaluators() { + return wildcardEvaluators; + } + + TrieNode getChild(Character ch) { + return children.get(ch); + } + + void populateTrieData(TrieData trieData) { + trieData.nodeCount++; + + if (wildcardEvaluators != null) { + if (isSharingParentWildcardEvaluators) { + trieData.wildcardEvaluatorListRefCount++; + } else { + trieData.wildcardEvaluatorListCount++; + } + } + + if (evaluators != null) { + if (evaluators == wildcardEvaluators) { + trieData.evaluatorListRefCount++; + } else { + trieData.evaluatorListCount++; + } + } + + if (!children.isEmpty()) { + if (children.size() == 1) { + trieData.singleChildNodeCount++; + } + + for (Map.Entry> entry : children.entrySet()) { + TrieNode child = entry.getValue(); + + child.populateTrieData(trieData); + } + } else { + trieData.leafNodeCount++; + } + } + + int getMaxDepth() { + int ret = 0; + + for (Map.Entry> entry : children.entrySet()) { + TrieNode child = entry.getValue(); + + int maxChildDepth = child.getMaxDepth(); + + if (maxChildDepth > ret) { + ret = maxChildDepth; + } + } + + return ret + 1; + } + + TrieNode getOrCreateChild(String str) { + int len = str.length(); + TrieNode child = children.get(getLookupChar(str, 0)); + + if (child == null) { + child = new TrieNode<>(str); + + addChild(child); + } else { + final String childStr = child.getStr(); + final int childStrLen = childStr.length(); + final boolean isExactMatch = optIgnoreCase ? StringUtils.equalsIgnoreCase(childStr, str) : StringUtils.equals(childStr, str); + + if (!isExactMatch) { + final int numOfCharactersToMatch = Math.min(childStrLen, len); + int index = 1; + + for (; index < numOfCharactersToMatch; index++) { + if (getLookupChar(childStr, index) != getLookupChar(str, index)) { + break; + } + } + + if (index == numOfCharactersToMatch) { + // Matched all + if (childStrLen > len) { + // Existing node has longer string, need to break up this node + TrieNode newChild = new TrieNode<>(str); + + this.addChild(newChild); + child.setStr(childStr.substring(index)); + newChild.addChild(child); + + child = newChild; + } else { + // This is a longer string, build a child with leftover string + child = child.getOrCreateChild(str.substring(index)); + } + } else { + // Partial match for both; both have leftovers + String matchedPart = str.substring(0, index); + TrieNode newChild = new TrieNode<>(matchedPart); + + this.addChild(newChild); + child.setStr(childStr.substring(index)); + newChild.addChild(child); + + child = newChild.getOrCreateChild(str.substring(index)); + } + } + } + + return child; + } + + private void addChild(TrieNode child) { + children.put(getLookupChar(child.getStr(), 0), child); + child.setParent(this); + } + + void addEvaluator(U evaluator) { + if (evaluators == null) { + evaluators = new HashSet<>(); + } + + evaluators.add(evaluator); + } + + void addWildcardEvaluator(U evaluator) { + undoSetup(); + + if (wildcardEvaluators == null) { + wildcardEvaluators = new HashSet<>(); + } + + if (!wildcardEvaluators.contains(evaluator)) { + wildcardEvaluators.add(evaluator); + } + } + + void removeEvaluator(U evaluator) { + if (CollectionUtils.isNotEmpty(evaluators) && evaluators.contains(evaluator)) { + evaluators.remove(evaluator); + + if (CollectionUtils.isEmpty(evaluators)) { + evaluators = null; + } + } + } + + void removeWildcardEvaluator(U evaluator) { + if (CollectionUtils.isNotEmpty(wildcardEvaluators)) { + wildcardEvaluators.remove(evaluator); + + if (CollectionUtils.isEmpty(wildcardEvaluators)) { + wildcardEvaluators = null; + } + } + } + + void undoSetup() { + if (isSetup) { + for (TrieNode child : children.values()) { + child.undoSetup(); + } + + if (evaluators != null) { + if (evaluators == wildcardEvaluators) { + evaluators = null; + } else { + if (wildcardEvaluators != null) { + evaluators.removeAll(wildcardEvaluators); + + if (CollectionUtils.isEmpty(evaluators)) { + evaluators = null; + } + } + } + } + + if (wildcardEvaluators != null) { + if (isSharingParentWildcardEvaluators) { + wildcardEvaluators = null; + } else { + Set parentWildcardEvaluators = getParent() == null ? null : getParent().getWildcardEvaluators(); + + if (parentWildcardEvaluators != null) { + wildcardEvaluators.removeAll(parentWildcardEvaluators); + + if (CollectionUtils.isEmpty(wildcardEvaluators)) { + wildcardEvaluators = null; + } + } + } + } + + isSharingParentWildcardEvaluators = false; + isSetup = false; + } + } + + void removeSelfFromTrie() { + if (evaluators == null && wildcardEvaluators == null && children.size() == 0) { + TrieNode parent = getParent(); + if (parent != null) { + parent.children.remove(str.charAt(0)); + } + } + } + + void wrapUpUpdate() { + if (isOptimizedForRetrieval) { + RangerPerfTracer postSetupPerf = null; + + if (RangerPerfTracer.isPerfTraceEnabled(PERF_TRIE_INIT_LOG)) { + postSetupPerf = RangerPerfTracer.getPerfTracer(PERF_TRIE_INIT_LOG, "RangerResourceTrie.init(name=" + resourceDef.getName() + "-postSetup)"); + } + + postSetup(null); + + RangerPerfTracer.logAlways(postSetupPerf); + } + } + + void postSetup(Set parentWildcardEvaluators) { + setup(parentWildcardEvaluators); + + for (Map.Entry> entry : children.entrySet()) { + TrieNode child = entry.getValue(); + + child.postSetup(wildcardEvaluators); + } + } + + void setupIfNeeded(TrieNode parent) { + boolean setupNeeded = !isSetup; + + if (setupNeeded) { + synchronized (this.children) { + setupNeeded = !isSetup; + + if (setupNeeded) { + setup(parent == null ? null : parent.getWildcardEvaluators()); + + if (TRACE_LOG.isTraceEnabled()) { + StringBuilder sb = new StringBuilder(); + this.toString(sb); + TRACE_LOG.trace("Set up is completed for this TriNode as a part of access evaluation : [" + sb + "]"); + } + } + } + } + } + + void setup(Set parentWildcardEvaluators) { + if (!isSetup) { + // finalize wildcard-evaluators list by including parent's wildcard evaluators + if (parentWildcardEvaluators != null) { + if (CollectionUtils.isEmpty(this.wildcardEvaluators)) { + this.wildcardEvaluators = parentWildcardEvaluators; + } else { + for (U evaluator : parentWildcardEvaluators) { + addWildcardEvaluator(evaluator); + } + } + } + + this.isSharingParentWildcardEvaluators = wildcardEvaluators == parentWildcardEvaluators; + + // finalize evaluators list by including wildcard evaluators + if (wildcardEvaluators != null) { + if (CollectionUtils.isEmpty(this.evaluators)) { + this.evaluators = wildcardEvaluators; + } else { + for (U evaluator : wildcardEvaluators) { + addEvaluator(evaluator); + } + } + } + + isSetup = true; + } + } + + void collectChildEvaluators(Character sep, int startIdx, Set childEvaluators) { + setupIfNeeded(getParent()); + + final int sepPos = startIdx < str.length() ? str.indexOf(sep, startIdx) : -1; + + if (sepPos == -1) { // ex: startIdx=5, path(str)=/tmp/test, path(a child) could be: /tmp/test.txt, /tmp/test/, /tmp/test/a, /tmp/test/a/b + if (this.evaluators != null) { + childEvaluators.addAll(this.evaluators); + } + + children.values().stream().forEach(c -> c.collectChildEvaluators(sep, 0, childEvaluators)); + } else if (sepPos == (str.length() - 1)) { // ex: str=/tmp/test/, startIdx=5 + if (this.evaluators != null) { + childEvaluators.addAll(this.evaluators); + } + } + } + + private void removeEvaluatorFromSubtree(U evaluator) { + if (CollectionUtils.isNotEmpty(wildcardEvaluators) && wildcardEvaluators.contains(evaluator)) { + undoSetup(); + removeWildcardEvaluator(evaluator); + } else { + removeEvaluator(evaluator); + } + removeSelfFromTrie(); + } + + void toString(StringBuilder sb) { + String nodeValue = this.str; + + sb.append("nodeValue=").append(nodeValue); + sb.append("; isSetup=").append(isSetup); + sb.append("; isSharingParentWildcardEvaluators=").append(isSharingParentWildcardEvaluators); + sb.append("; childCount=").append(children.size()); + sb.append("; evaluators=[ "); + if (evaluators != null) { + for (U evaluator : evaluators) { + sb.append(evaluator.getId()).append(" "); + } + } + sb.append("]"); + + sb.append("; wildcardEvaluators=[ "); + if (wildcardEvaluators != null) { + for (U evaluator : wildcardEvaluators) { + sb.append(evaluator.getId()).append(" "); + } + } + sb.append("]"); + } + + void toString(String prefix, StringBuilder sb) { + String nodeValue = prefix + (str != null ? str : ""); + + sb.append(prefix); + toString(sb); + sb.append("]\n"); + + for (Map.Entry> entry : children.entrySet()) { + TrieNode child = entry.getValue(); + + child.toString(nodeValue, sb); + } + } + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerTagAccessRequest.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerTagAccessRequest.java new file mode 100644 index 00000000000..e3372110e86 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerTagAccessRequest.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.policyengine; + + +import org.apache.commons.lang.StringUtils; +import org.apache.atlas.plugin.contextenricher.RangerTagForEval; +import org.apache.atlas.plugin.model.RangerServiceDef; +import org.apache.atlas.plugin.policyresourcematcher.RangerPolicyResourceMatcher; +import org.apache.atlas.plugin.util.RangerAccessRequestUtil; + +import java.util.Map; + +public class RangerTagAccessRequest extends RangerAccessRequestImpl { + private final RangerPolicyResourceMatcher.MatchType matchType; + public RangerTagAccessRequest(RangerTagForEval resourceTag, RangerServiceDef tagServiceDef, RangerAccessRequest request) { + matchType = resourceTag.getMatchType(); + super.setResource(new RangerTagResource(resourceTag.getType(), tagServiceDef)); + super.setUser(request.getUser()); + super.setUserGroups(request.getUserGroups()); + super.setUserRoles(request.getUserRoles()); + super.setAction(request.getAction()); + super.setAccessType(request.getAccessType()); + super.setAccessTime(request.getAccessTime()); + super.setRequestData(request.getRequestData()); + super.setAccessorsRequested(request.isAccessorsRequested()); + + Map requestContext = request.getContext(); + + RangerAccessRequestUtil.setCurrentTagInContext(request.getContext(), resourceTag); + RangerAccessRequestUtil.setCurrentResourceInContext(request.getContext(), request.getResource()); + RangerAccessRequestUtil.setCurrentUserInContext(request.getContext(), request.getUser()); + + String owner = request.getResource() != null ? request.getResource().getOwnerUser() : null; + + if (StringUtils.isNotEmpty(owner)) { + RangerAccessRequestUtil.setOwnerInContext(request.getContext(), owner); + } + super.setContext(requestContext); + + super.setClientType(request.getClientType()); + super.setClientIPAddress(request.getClientIPAddress()); + super.setRemoteIPAddress(request.getRemoteIPAddress()); + super.setForwardedAddresses(request.getForwardedAddresses()); + super.setSessionId(request.getSessionId()); + super.setResourceMatchingScope(request.getResourceMatchingScope()); + } + public RangerPolicyResourceMatcher.MatchType getMatchType() { + return matchType; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerTagResource.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerTagResource.java new file mode 100644 index 00000000000..7ce8a5d7473 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerTagResource.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.policyengine; + + +import org.apache.atlas.plugin.model.RangerServiceDef; + +public class RangerTagResource extends RangerAccessResourceImpl { + private static final String KEY_TAG = "tag"; + + + public RangerTagResource(String tagType, RangerServiceDef tagServiceDef) { + super.setValue(KEY_TAG, tagType); + super.setServiceDef(tagServiceDef); + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyevaluator/RangerAbstractPolicyEvaluator.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyevaluator/RangerAbstractPolicyEvaluator.java new file mode 100644 index 00000000000..6db63f5f32e --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyevaluator/RangerAbstractPolicyEvaluator.java @@ -0,0 +1,207 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.policyevaluator; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.plugin.model.RangerPolicy; +import org.apache.atlas.plugin.model.RangerServiceDef; +import org.apache.atlas.plugin.model.RangerServiceDef.RangerResourceDef; +import org.apache.atlas.plugin.policyengine.RangerAccessRequest; +import org.apache.atlas.plugin.policyengine.RangerPluginContext; +import org.apache.atlas.plugin.policyengine.RangerPolicyEngineOptions; +import org.apache.atlas.plugin.util.ServiceDefUtil; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +public abstract class RangerAbstractPolicyEvaluator implements RangerPolicyEvaluator { + private static final Log LOG = LogFactory.getLog(RangerAbstractPolicyEvaluator.class); + + private RangerPolicy policy; + private RangerServiceDef serviceDef; + private RangerResourceDef leafResourceDef; + private int evalOrder; + protected RangerPluginContext pluginContext = null; + + + public void setPluginContext(RangerPluginContext pluginContext) { this.pluginContext = pluginContext; } + + public RangerPluginContext getPluginContext() { return pluginContext; } + + @Override + public void init(RangerPolicy policy, RangerServiceDef serviceDef, RangerPolicyEngineOptions options) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerAbstractPolicyEvaluator.init(" + policy + ", " + serviceDef + ")"); + } + + this.policy = getPrunedPolicy(policy); + this.serviceDef = serviceDef; + this.leafResourceDef = ServiceDefUtil.getLeafResourceDef(serviceDef, getPolicyResource()); + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerAbstractPolicyEvaluator.init(" + this.policy + ", " + serviceDef + ")"); + } + } + + @Override + public long getId() { + return policy != null ? policy.getId() :-1; + } + + @Override + public String getGuid() { + return policy != null ? policy.getGuid() : "-1"; + } + + @Override + public Map getPolicyResource() { + return policy !=null ? policy.getResources() : null; + } + + @Override + public RangerPolicy getPolicy() { + return policy; + } + + @Override + public int getPolicyPriority() { + return policy != null && policy.getPolicyPriority() != null ? policy.getPolicyPriority() : RangerPolicy.POLICY_PRIORITY_NORMAL; + } + + @Override + public RangerServiceDef getServiceDef() { + return serviceDef; + } + + @Override + public boolean isAncestorOf(RangerResourceDef resourceDef) { + return ServiceDefUtil.isAncestorOf(serviceDef, leafResourceDef, resourceDef); + } + + public boolean hasAllow() { + return policy != null && CollectionUtils.isNotEmpty(policy.getPolicyItems()); + } + + protected boolean hasMatchablePolicyItem(RangerAccessRequest request) { + return hasAllow() || hasDeny(); + } + + public boolean hasDeny() { + return policy != null && (policy.getIsDenyAllElse() || CollectionUtils.isNotEmpty(policy.getDenyPolicyItems())); + } + + private RangerPolicy getPrunedPolicy(final RangerPolicy policy) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerAbstractPolicyEvaluator.getPrunedPolicy(" + policy + ")"); + } + + final RangerPolicy ret; + + final boolean isPruningNeeded; + final List prunedAllowItems; + final List prunedDenyItems; + final List prunedAllowExceptions; + final List prunedDenyExceptions; + + final RangerPluginContext pluginContext = getPluginContext(); + + if (pluginContext != null && pluginContext.getConfig().getPolicyEngineOptions().evaluateDelegateAdminOnly) { + prunedAllowItems = policy.getPolicyItems().stream().filter(RangerPolicy.RangerPolicyItem::getDelegateAdmin).collect(Collectors.toList()); + prunedDenyItems = policy.getDenyPolicyItems().stream().filter(RangerPolicy.RangerPolicyItem::getDelegateAdmin).collect(Collectors.toList()); + prunedAllowExceptions = policy.getAllowExceptions().stream().filter(RangerPolicy.RangerPolicyItem::getDelegateAdmin).collect(Collectors.toList()); + prunedDenyExceptions = policy.getDenyExceptions().stream().filter(RangerPolicy.RangerPolicyItem::getDelegateAdmin).collect(Collectors.toList()); + + isPruningNeeded = prunedAllowItems.size() != policy.getPolicyItems().size() + || prunedDenyItems.size() != policy.getDenyPolicyItems().size() + || prunedAllowExceptions.size() != policy.getAllowExceptions().size() + || prunedDenyExceptions.size() != policy.getDenyExceptions().size(); + } else { + prunedAllowItems = null; + prunedDenyItems = null; + prunedAllowExceptions = null; + prunedDenyExceptions = null; + isPruningNeeded = false; + } + + if (!isPruningNeeded) { + ret = policy; + } else { + ret = new RangerPolicy(); + ret.updateFrom(policy); + + ret.setId(policy.getId()); + ret.setGuid(policy.getGuid()); + ret.setVersion(policy.getVersion()); + ret.setServiceType(policy.getServiceType()); + + ret.setPolicyItems(prunedAllowItems); + ret.setDenyPolicyItems(prunedDenyItems); + ret.setAllowExceptions(prunedAllowExceptions); + ret.setDenyExceptions(prunedDenyExceptions); + } + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerAbstractPolicyEvaluator.getPrunedPolicy(isPruningNeeded=" + isPruningNeeded + ") : " + ret); + } + + return ret; + } + + @Override + public int getEvalOrder() { + return evalOrder; + } + @Override + public boolean isAuditEnabled() { + return policy != null && policy.getIsAuditEnabled(); + } + + public void setEvalOrder(int evalOrder) { + this.evalOrder = evalOrder; + } + + @Override + public PolicyACLSummary getPolicyACLSummary() { return null; } + + @Override + public String toString( ) { + StringBuilder sb = new StringBuilder(); + + toString(sb); + + return sb.toString(); + } + + public StringBuilder toString(StringBuilder sb) { + sb.append("RangerAbstractPolicyEvaluator={"); + + sb.append("policy={"); + if (policy != null) { + policy.toString(sb); + } + sb.append("} "); + + sb.append("}"); + + return sb; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyevaluator/RangerAbstractPolicyItemEvaluator.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyevaluator/RangerAbstractPolicyItemEvaluator.java new file mode 100644 index 00000000000..86b58f59d1e --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyevaluator/RangerAbstractPolicyItemEvaluator.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.atlas.plugin.policyevaluator; + + +import org.apache.commons.collections.CollectionUtils; +import org.apache.atlas.plugin.conditionevaluator.RangerConditionEvaluator; +import org.apache.atlas.plugin.model.RangerPolicy; +import org.apache.atlas.plugin.model.RangerPolicy.RangerPolicyItem; +import org.apache.atlas.plugin.model.RangerServiceDef; +import org.apache.atlas.plugin.policyengine.RangerPolicyEngine; +import org.apache.atlas.plugin.policyengine.RangerPolicyEngineOptions; + +import java.util.Collections; +import java.util.List; + + +public abstract class RangerAbstractPolicyItemEvaluator implements RangerPolicyItemEvaluator { + + private static final int RANGER_POLICY_ITEM_EVAL_ORDER_DEFAULT = 1000; + + private static final int RANGER_POLICY_ITEM_EVAL_ORDER_MAX_DISCOUNT_USERSGROUPS = 100; + private static final int RANGER_POLICY_ITEM_EVAL_ORDER_MAX_DISCOUNT_ACCESS_TYPES = 25; + private static final int RANGER_POLICY_ITEM_EVAL_ORDER_MAX_DISCOUNT_CUSTOM_CONDITIONS = 25; + private static final int RANGER_POLICY_ITEM_EVAL_ORDER_CUSTOM_CONDITION_PENALTY = 5; + + final RangerPolicy policy; + final RangerPolicyEngineOptions options; + final RangerServiceDef serviceDef; + final RangerPolicyItem policyItem; + final int policyItemType; + final int policyItemIndex; + final long policyId; + final int evalOrder; + + List conditionEvaluators = Collections.emptyList(); + + RangerAbstractPolicyItemEvaluator(RangerServiceDef serviceDef, RangerPolicy policy, RangerPolicyItem policyItem, int policyItemType, int policyItemIndex, RangerPolicyEngineOptions options) { + this.serviceDef = serviceDef; + this.policyItem = policyItem; + this.policyItemType = policyItemType; + this.policyItemIndex = policyItemIndex; + this.options = options; + this.policyId = policy != null && policy.getId() != null ? policy.getId() : -1; + this.evalOrder = computeEvalOrder(); + this.policy = policy; + } + + @Override + public List getConditionEvaluators() { + return conditionEvaluators; + } + + @Override + public int getEvalOrder() { + return evalOrder; + } + + @Override + public RangerPolicyItem getPolicyItem() { + return policyItem; + } + + @Override + public int getPolicyItemType() { + return policyItemType; + } + + @Override + public int getPolicyItemIndex() { + return policyItemIndex; + } + + @Override + public String getComments() { + return null; + } + + protected String getServiceType() { + return serviceDef != null ? serviceDef.getName() : null; + } + + protected boolean getConditionsDisabledOption() { + return options != null && options.disableCustomConditions; + } + + private int computeEvalOrder() { + int evalOrder = RANGER_POLICY_ITEM_EVAL_ORDER_DEFAULT; + + if(policyItem != null) { + if((CollectionUtils.isNotEmpty(policyItem.getGroups()) && policyItem.getGroups().contains(RangerPolicyEngine.GROUP_PUBLIC)) + || (CollectionUtils.isNotEmpty(policyItem.getUsers()) && policyItem.getUsers().contains(RangerPolicyEngine.USER_CURRENT))) { + evalOrder -= RANGER_POLICY_ITEM_EVAL_ORDER_MAX_DISCOUNT_USERSGROUPS; + } else { + int userGroupCount = 0; + + if(! CollectionUtils.isEmpty(policyItem.getUsers())) { + userGroupCount += policyItem.getUsers().size(); + } + + if(! CollectionUtils.isEmpty(policyItem.getGroups())) { + userGroupCount += policyItem.getGroups().size(); + } + + evalOrder -= Math.min(RANGER_POLICY_ITEM_EVAL_ORDER_MAX_DISCOUNT_USERSGROUPS, userGroupCount); + } + + if(CollectionUtils.isNotEmpty(policyItem.getAccesses())) { + evalOrder -= Math.round(((float)RANGER_POLICY_ITEM_EVAL_ORDER_MAX_DISCOUNT_ACCESS_TYPES * policyItem.getAccesses().size()) / serviceDef.getAccessTypes().size()); + } + + int customConditionsPenalty = 0; + if(CollectionUtils.isNotEmpty(policyItem.getConditions())) { + customConditionsPenalty = RANGER_POLICY_ITEM_EVAL_ORDER_CUSTOM_CONDITION_PENALTY * policyItem.getConditions().size(); + } + int customConditionsDiscount = RANGER_POLICY_ITEM_EVAL_ORDER_MAX_DISCOUNT_CUSTOM_CONDITIONS - customConditionsPenalty; + if(customConditionsDiscount > 0) { + evalOrder -= customConditionsDiscount; + } + } + + return evalOrder; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyevaluator/RangerAuditPolicyEvaluator.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyevaluator/RangerAuditPolicyEvaluator.java new file mode 100644 index 00000000000..aee66be1549 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyevaluator/RangerAuditPolicyEvaluator.java @@ -0,0 +1,405 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.policyevaluator; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.plugin.model.AuditFilter; +import org.apache.atlas.plugin.model.RangerPolicy; +import org.apache.atlas.plugin.model.RangerPolicy.RangerPolicyItem; +import org.apache.atlas.plugin.model.RangerPolicy.RangerPolicyItemAccess; +import org.apache.atlas.plugin.model.RangerServiceDef; +import org.apache.atlas.plugin.model.RangerServiceDef.RangerResourceDef; +import org.apache.atlas.plugin.policyengine.*; +import org.apache.atlas.plugin.policyresourcematcher.RangerPolicyResourceMatcher; +import org.apache.atlas.plugin.util.RangerAccessRequestUtil; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + + +public class RangerAuditPolicyEvaluator extends RangerDefaultPolicyEvaluator { + private static final Log LOG = LogFactory.getLog(RangerAuditPolicyEvaluator.class); + + private final RangerAuditPolicy auditPolicy; + private final boolean matchAnyResource; + private final List auditItemEvaluators = new ArrayList<>(); + + public RangerAuditPolicyEvaluator(AuditFilter auditFilter, int priority) { + this.auditPolicy = new RangerAuditPolicy(auditFilter, priority); + this.matchAnyResource = MapUtils.isEmpty(auditFilter.getResources()); + + if (LOG.isDebugEnabled()) { + LOG.debug("RangerAuditPolicyEvaluator(auditFilter=" + auditFilter + ", priority=" + priority + ", matchAnyResource=" + matchAnyResource + ")"); + } + } + + public RangerAuditPolicy getAuditPolicy() { return auditPolicy; } + + @Override + public void init(RangerPolicy policy, RangerServiceDef serviceDef, RangerPolicyEngineOptions options) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerAuditPolicyEvaluator.init(" + auditPolicy.getId() + ")"); + } + + super.init(auditPolicy, serviceDef, options); + + int policyItemIndex = 1; + + for (RangerAuditPolicyItem policyItem : auditPolicy.getAuditPolicyItems()) { + RangerAuditPolicyItemEvaluator itemEvaluator = new RangerAuditPolicyItemEvaluator(serviceDef, auditPolicy, policyItem, policyItemIndex, options); + auditItemEvaluators.add(itemEvaluator); + policyItemIndex = policyItemIndex + 1; + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerAuditPolicyEvaluator.init(" + auditPolicy.getId() + ")"); + } + } + + @Override + public boolean isAncestorOf(RangerResourceDef resourceDef) { + return matchAnyResource || super.isAncestorOf(resourceDef); + } + + @Override + public void evaluate(RangerAccessRequest request, RangerAccessResult result) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerAuditPolicyEvaluator.evaluate(" + auditPolicy.getId() + ", " + request + ", " + result + ")"); + } + + if (request != null && result != null) { + if (!result.getIsAuditedDetermined()) { + if (matchResource(request)) { + evaluatePolicyItems(request, result); + } + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerAuditPolicyEvaluator.evaluate(" + auditPolicy.getId() + ", " + request + ", " + result + ")"); + } + } + + @Override + protected void preprocessPolicy(RangerPolicy policy, RangerServiceDef serviceDef) { + super.preprocessPolicy(policy, serviceDef); + + Map> impliedAccessGrants = getImpliedAccessGrants(serviceDef); + + if (impliedAccessGrants == null || impliedAccessGrants.isEmpty()) { + return; + } + + preprocessPolicyItems(auditPolicy.getAuditPolicyItems(), impliedAccessGrants); + } + + private boolean matchResource(RangerAccessRequest request) { + final boolean ret; + + if (!matchAnyResource) { + RangerPolicyResourceMatcher.MatchType matchType; + + if (RangerTagAccessRequest.class.isInstance(request)) { + matchType = ((RangerTagAccessRequest) request).getMatchType(); + + if (matchType == RangerPolicyResourceMatcher.MatchType.ANCESTOR) { + matchType = RangerPolicyResourceMatcher.MatchType.SELF; + } + } else { + RangerPolicyResourceMatcher resourceMatcher = getPolicyResourceMatcher(); + + if (resourceMatcher != null) { + matchType = resourceMatcher.getMatchType(request.getResource(), request.getContext()); + } else { + matchType = RangerPolicyResourceMatcher.MatchType.NONE; + } + } + + if (request.isAccessTypeAny()) { + ret = matchType != RangerPolicyResourceMatcher.MatchType.NONE; + } else if (request.getResourceMatchingScope() == RangerAccessRequest.ResourceMatchingScope.SELF_OR_DESCENDANTS) { + ret = matchType != RangerPolicyResourceMatcher.MatchType.NONE; + } else { + ret = matchType == RangerPolicyResourceMatcher.MatchType.SELF || matchType == RangerPolicyResourceMatcher.MatchType.SELF_AND_ALL_DESCENDANTS; + } + } else { + ret = true; + } + + return ret; + } + + private void evaluatePolicyItems(RangerAccessRequest request, RangerAccessResult result) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerAuditPolicyEvaluator.evaluatePolicyItems(" + auditPolicy.getId() + ", " + request + ", " + result + ")"); + } + + for (RangerAuditPolicyItemEvaluator itemEvaluator : auditItemEvaluators) { + if (itemEvaluator.isMatch(request, result)) { + Boolean isAudited = itemEvaluator.getIsAudited(); + + if (isAudited != null) { + result.setIsAudited(isAudited); + + break; + } + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerAuditPolicyEvaluator.evaluatePolicyItems(" + auditPolicy.getId() + ", " + request + ", " + result + ")"); + } + } + + @Override + public StringBuilder toString(StringBuilder sb) { + sb.append("RangerAuditPolicyEvaluator={"); + + super.toString(sb); + + sb.append("auditPolicy={"); + auditPolicy.toString(sb); + sb.append("}"); + + sb.append(" matchAnyResource={").append(matchAnyResource).append("}"); + + sb.append("}"); + + return sb; + } + + + public static class RangerAuditPolicyItemEvaluator extends RangerDefaultPolicyItemEvaluator { + private final RangerAuditPolicyItem auditPolicyItem; + private final boolean matchAnyResult; + private final boolean matchAnyUser; + private final boolean matchAnyAction; + private final boolean hasResourceOwner; + + public RangerAuditPolicyItemEvaluator(RangerServiceDef serviceDef, RangerPolicy policy, RangerAuditPolicyItem policyItem, int policyItemIndex, RangerPolicyEngineOptions options) { + super(serviceDef, policy, policyItem, POLICY_ITEM_TYPE_ALLOW, policyItemIndex, options); + + this.auditPolicyItem = policyItem; + this.matchAnyResult = policyItem.getAccessResult() == null; + + List users = policyItem.getUsers(); + List groups = policyItem.getGroups(); + List roles = policyItem.getRoles(); + this.matchAnyUser = (CollectionUtils.isEmpty(users) && CollectionUtils.isEmpty(groups) && CollectionUtils.isEmpty(roles)) || + (CollectionUtils.isNotEmpty(groups) && groups.contains(RangerPolicyEngine.GROUP_PUBLIC)) || + (CollectionUtils.isNotEmpty(users) && users.contains(RangerPolicyEngine.USER_CURRENT)); + this.matchAnyAction = policyItem.getActions().isEmpty() && policyItem.getAccessTypes().isEmpty(); + this.hasResourceOwner = CollectionUtils.isNotEmpty(users) && users.contains(RangerPolicyEngine.RESOURCE_OWNER); + + if (LOG.isDebugEnabled()) { + LOG.debug("RangerAuditPolicyItemEvaluator(" + auditPolicyItem + ", matchAnyUser=" + matchAnyUser + ", matchAnyAction=" + matchAnyAction + ", hasResourceOwner=" + hasResourceOwner + ")"); + } + } + + public Boolean getIsAudited() { return auditPolicyItem.getIsAudited(); } + + public boolean isMatch(RangerAccessRequest request, RangerAccessResult result) { + boolean ret = matchAccessResult(result) && + matchUserGroupRole(request) && + matchAction(request); + + if (LOG.isDebugEnabled()) { + LOG.debug("RangerAuditPolicyItemEvaluator.isMatch(" + request + ", " + result + "): ret=" + ret); + } + + return ret; + } + + private boolean matchAccessResult(RangerAccessResult result) { + boolean ret = matchAnyResult; + + if (!ret) { + switch (auditPolicyItem.getAccessResult()) { + case DENIED: + ret = result.getIsAccessDetermined() && !result.getIsAllowed(); + break; + + case ALLOWED: + ret = result.getIsAccessDetermined() && result.getIsAllowed(); + break; + + case NOT_DETERMINED: + ret = !result.getIsAccessDetermined(); + break; + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("RangerAuditPolicyItemEvaluator.matchAccessResult(" + result + "): ret=" + ret); + } + + return ret; + } + + private boolean matchUserGroupRole(RangerAccessRequest request) { + boolean ret = matchAnyUser; + + if (!ret) { + + if (auditPolicyItem.getUsers() != null && request.getUser() != null) { + ret = auditPolicyItem.getUsers().contains(request.getUser()); + + if (!ret && hasResourceOwner) { + String owner = request.getResource() != null ? request.getResource().getOwnerUser() : null; + ret = request.getUser().equals(owner); + } + } + + if (!ret && auditPolicyItem.getGroups() != null && request.getUserGroups() != null) { + ret = CollectionUtils.containsAny(auditPolicyItem.getGroups(), request.getUserGroups()); + } + + if (!ret && auditPolicyItem.getRoles() != null) { + ret = CollectionUtils.containsAny(auditPolicyItem.getRoles(), RangerAccessRequestUtil.getCurrentUserRolesFromContext(request.getContext())); + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("RangerAuditPolicyItemEvaluator.matchUserGroupRole(" + request + "): ret=" + ret); + } + + return ret; + } + + private boolean matchAction(RangerAccessRequest request) { + boolean ret = matchAnyAction; + + if (!ret) { + if (request.getAction() != null) { + ret = auditPolicyItem.getActions().contains(request.getAction()); + } + + if (!ret && request.getAccessType() != null) { + ret = auditPolicyItem.getAccessTypes().contains(request.getAccessType()); + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("RangerAuditPolicyItemEvaluator.matchAction(" + request + "): ret=" + ret); + } + + return ret; + } + } + + public static class RangerAuditPolicy extends RangerPolicy { + private final List auditPolicyItems; + + public RangerAuditPolicy(AuditFilter auditFilter, int priority) { + setId((long)priority); + setResources(auditFilter.getResources()); + setPolicyType(POLICY_TYPE_AUDIT); + setPolicyPriority(priority); + + this.auditPolicyItems = Collections.singletonList(new RangerAuditPolicyItem(auditFilter)); + } + + public List getAuditPolicyItems() { return auditPolicyItems; } + } + + public static class RangerAuditPolicyItem extends RangerPolicyItem { + private final AuditFilter.AccessResult accessResult; + private final Set actions; + private final Set accessTypes; + private final Boolean isAudited; + + public RangerAuditPolicyItem(AuditFilter auditFilter) { + super(getPolicyItemAccesses(auditFilter.getAccessTypes()), auditFilter.getUsers(), auditFilter.getGroups(), auditFilter.getRoles(), null, null); + + this.accessResult = auditFilter.getAccessResult(); + this.actions = auditFilter.getActions() != null ? new HashSet<>(auditFilter.getActions()) : Collections.emptySet(); + this.accessTypes = auditFilter.getAccessTypes() != null ? new HashSet<>(auditFilter.getAccessTypes()) : Collections.emptySet(); + this.isAudited = auditFilter.getIsAudited(); + } + + public Set getActions() { return actions; } + + public Set getAccessTypes() { return accessTypes; } + + public AuditFilter.AccessResult getAccessResult() { return accessResult; } + + public Boolean getIsAudited() { return isAudited; } + + @Override + public StringBuilder toString(StringBuilder sb) { + if (sb == null) { + sb = new StringBuilder(); + } + + sb.append("RangerAuditPolicyItem={"); + super.toString(sb); + sb.append(" accessResult={").append(accessResult).append("}"); + + sb.append(" actions={"); + if (actions != null) { + for (String action : actions) { + if (action != null) { + sb.append(action).append(" "); + } + } + } + sb.append("}"); + + sb.append(" accessTypes={"); + if (accessTypes != null) { + for (String accessTypes : accessTypes) { + if (accessTypes != null) { + sb.append(accessTypes).append(" "); + } + } + } + sb.append("}"); + + sb.append(" isAudited={").append(isAudited).append("}"); + sb.append("}"); + + return sb; + } + + private static List getPolicyItemAccesses(List accessTypes) { + final List ret; + + if (accessTypes != null) { + ret = new ArrayList<>(accessTypes.size()); + + for (String accessType : accessTypes) { + ret.add(new RangerPolicyItemAccess(accessType)); + } + } else { + ret = Collections.emptyList(); + } + + return ret; + } + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyevaluator/RangerCachedPolicyEvaluator.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyevaluator/RangerCachedPolicyEvaluator.java new file mode 100644 index 00000000000..03f6aef5f32 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyevaluator/RangerCachedPolicyEvaluator.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.policyevaluator; + +/* + * this class is deprecated, as the functionality is moved to base class RangerOptimizedPolicyEvaluator. + * Keeping the class simply for backward compatibility, in case this is used anywhere + */ +public class RangerCachedPolicyEvaluator extends RangerOptimizedPolicyEvaluator { +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyevaluator/RangerCustomConditionEvaluator.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyevaluator/RangerCustomConditionEvaluator.java new file mode 100644 index 00000000000..5be694246f9 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyevaluator/RangerCustomConditionEvaluator.java @@ -0,0 +1,202 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.policyevaluator; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.plugin.conditionevaluator.RangerConditionEvaluator; +import org.apache.atlas.plugin.model.RangerPolicy; +import org.apache.atlas.plugin.model.RangerPolicy.RangerPolicyItem; +import org.apache.atlas.plugin.model.RangerPolicy.RangerPolicyItemCondition; +import org.apache.atlas.plugin.model.RangerServiceDef; +import org.apache.atlas.plugin.policyengine.RangerPolicyEngineOptions; +import org.apache.atlas.plugin.util.RangerPerfTracer; + +import java.util.ArrayList; +import java.util.List; + + +public class RangerCustomConditionEvaluator { + + private static final Log LOG = LogFactory.getLog(RangerCustomConditionEvaluator.class); + private static final Log PERF_POLICY_INIT_LOG = RangerPerfTracer.getPerfLogger("policy.init"); + private static final Log PERF_POLICYITEM_INIT_LOG = RangerPerfTracer.getPerfLogger("policyitem.init"); + private static final Log PERF_POLICYCONDITION_INIT_LOG = RangerPerfTracer.getPerfLogger("policycondition.init"); + + public List getRangerPolicyConditionEvaluator(RangerPolicy policy, + RangerServiceDef serviceDef, + RangerPolicyEngineOptions options) { + List conditionEvaluators = new ArrayList<>(); + + if (!getConditionsDisabledOption(options) && CollectionUtils.isNotEmpty(policy.getConditions())) { + + RangerPerfTracer perf = null; + + long policyId = policy.getId(); + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_POLICY_INIT_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_POLICY_INIT_LOG, "RangerCustomConditionEvaluator.init(policyId=" + policyId + ")"); + } + + for (RangerPolicy.RangerPolicyItemCondition condition : policy.getConditions()) { + RangerServiceDef.RangerPolicyConditionDef conditionDef = getConditionDef(condition.getType(),serviceDef); + + if (conditionDef == null) { + LOG.error("RangerCustomConditionEvaluator.getRangerPolicyConditionEvaluator(policyId=" + policyId + "): conditionDef '" + condition.getType() + "' not found. Ignoring the condition"); + + continue; + } + + RangerConditionEvaluator conditionEvaluator = newConditionEvaluator(conditionDef.getEvaluator()); + + if (conditionEvaluator != null) { + conditionEvaluator.setServiceDef(serviceDef); + conditionEvaluator.setConditionDef(conditionDef); + conditionEvaluator.setPolicyItemCondition(condition); + + RangerPerfTracer perfConditionInit = null; + + if (RangerPerfTracer.isPerfTraceEnabled(PERF_POLICYCONDITION_INIT_LOG)) { + perfConditionInit = RangerPerfTracer.getPerfTracer(PERF_POLICYCONDITION_INIT_LOG, "RangerConditionEvaluator.init(policyId=" + policyId + "policyConditionType=" + condition.getType() + ")"); + } + + conditionEvaluator.init(); + + RangerPerfTracer.log(perfConditionInit); + + conditionEvaluators.add(conditionEvaluator); + } else { + LOG.error("RangerCustomConditionEvaluator.getRangerPolicyConditionEvaluator(policyId=" + policyId + "): failed to init Policy ConditionEvaluator '" + condition.getType() + "'; evaluatorClassName='" + conditionDef.getEvaluator() + "'"); + } + } + + RangerPerfTracer.log(perf); + } + return conditionEvaluators; + } + + + public List getPolicyItemConditionEvaluator(RangerPolicy policy, + RangerPolicyItem policyItem, + RangerServiceDef serviceDef, + RangerPolicyEngineOptions options, + int policyItemIndex) { + + List conditionEvaluators = new ArrayList<>(); + + if (!getConditionsDisabledOption(options) && CollectionUtils.isNotEmpty(policyItem.getConditions())) { + + RangerPerfTracer perf = null; + + Long policyId = policy.getId(); + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_POLICYITEM_INIT_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_POLICYITEM_INIT_LOG, "RangerPolicyItemEvaluator.getRangerPolicyConditionEvaluator(policyId=" + policyId + ",policyItemIndex=" + policyItemIndex + ")"); + } + + for (RangerPolicyItemCondition condition : policyItem.getConditions()) { + RangerServiceDef.RangerPolicyConditionDef conditionDef = getConditionDef(condition.getType(), serviceDef); + + if (conditionDef == null) { + LOG.error("RangerCustomConditionEvaluator.getPolicyItemConditionEvaluator(policyId=" + policyId + "): conditionDef '" + condition.getType() + "' not found. Ignoring the condition"); + + continue; + } + + RangerConditionEvaluator conditionEvaluator = newConditionEvaluator(conditionDef.getEvaluator()); + + if (conditionEvaluator != null) { + conditionEvaluator.setServiceDef(serviceDef); + conditionEvaluator.setConditionDef(conditionDef); + conditionEvaluator.setPolicyItemCondition(condition); + + RangerPerfTracer perfConditionInit = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_POLICYCONDITION_INIT_LOG)) { + perfConditionInit = RangerPerfTracer.getPerfTracer(PERF_POLICYCONDITION_INIT_LOG, "RangerConditionEvaluator.init(policyId=" + policyId + ",policyItemIndex=" + policyItemIndex + ",policyConditionType=" + condition.getType() + ")"); + } + + conditionEvaluator.init(); + + RangerPerfTracer.log(perfConditionInit); + + conditionEvaluators.add(conditionEvaluator); + } else { + LOG.error("RangerCustomConditionEvaluator.getPolicyItemConditionEvaluator(policyId=" + policyId + "): failed to init PolicyItem ConditionEvaluator '" + condition.getType() + "'; evaluatorClassName='" + conditionDef.getEvaluator() + "'"); + } + } + RangerPerfTracer.log(perf); + } + return conditionEvaluators; + } + + private RangerServiceDef.RangerPolicyConditionDef getConditionDef(String conditionName, RangerServiceDef serviceDef) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerCustomConditionEvaluator.getConditionDef(" + conditionName + ")"); + } + + RangerServiceDef.RangerPolicyConditionDef ret = null; + + if (serviceDef != null && CollectionUtils.isNotEmpty(serviceDef.getPolicyConditions())) { + for(RangerServiceDef.RangerPolicyConditionDef conditionDef : serviceDef.getPolicyConditions()) { + if(StringUtils.equals(conditionName, conditionDef.getName())) { + ret = conditionDef; + break; + } + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerCustomConditionEvaluator.getConditionDef(" + conditionName + "): " + ret); + } + + return ret; + } + + + private RangerConditionEvaluator newConditionEvaluator(String className) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerCustomConditionEvaluator.newConditionEvaluator(" + className + ")"); + } + + RangerConditionEvaluator evaluator = null; + + try { + @SuppressWarnings("unchecked") + Class matcherClass = (Class)Class.forName(className); + + evaluator = matcherClass.newInstance(); + } catch(Throwable t) { + LOG.error("RangerCustomConditionEvaluator.newConditionEvaluator(" + className + "): error instantiating evaluator", t); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerCustomConditionEvaluator.newConditionEvaluator(" + className + "): " + evaluator); + } + + return evaluator; + } + + private boolean getConditionsDisabledOption(RangerPolicyEngineOptions options) { + return options != null && options.disableCustomConditions; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyevaluator/RangerDataMaskPolicyItemEvaluator.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyevaluator/RangerDataMaskPolicyItemEvaluator.java new file mode 100644 index 00000000000..9ec0ebaeebd --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyevaluator/RangerDataMaskPolicyItemEvaluator.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.atlas.plugin.policyevaluator; + +import org.apache.atlas.plugin.model.RangerPolicy.RangerPolicyItemDataMaskInfo; + + +public interface RangerDataMaskPolicyItemEvaluator extends RangerPolicyItemEvaluator { + void init(); + + RangerPolicyItemDataMaskInfo getDataMaskInfo(); +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyevaluator/RangerDefaultDataMaskPolicyItemEvaluator.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyevaluator/RangerDefaultDataMaskPolicyItemEvaluator.java new file mode 100644 index 00000000000..fb62a6f6c03 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyevaluator/RangerDefaultDataMaskPolicyItemEvaluator.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.atlas.plugin.policyevaluator; + +import org.apache.atlas.plugin.model.RangerPolicy; +import org.apache.atlas.plugin.model.RangerPolicy.RangerDataMaskPolicyItem; +import org.apache.atlas.plugin.model.RangerPolicy.RangerPolicyItemDataMaskInfo; +import org.apache.atlas.plugin.model.RangerServiceDef; +import org.apache.atlas.plugin.policyengine.RangerAccessResult; +import org.apache.atlas.plugin.policyengine.RangerPolicyEngineOptions; +import org.apache.atlas.plugin.policyresourcematcher.RangerPolicyResourceMatcher; + + +public class RangerDefaultDataMaskPolicyItemEvaluator extends RangerDefaultPolicyItemEvaluator implements RangerDataMaskPolicyItemEvaluator { + final private RangerDataMaskPolicyItem dataMaskPolicyItem; + + public RangerDefaultDataMaskPolicyItemEvaluator(RangerServiceDef serviceDef, RangerPolicy policy, RangerDataMaskPolicyItem policyItem, int policyItemIndex, RangerPolicyEngineOptions options) { + super(serviceDef, policy, policyItem, RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_DATAMASK, policyItemIndex, options); + + dataMaskPolicyItem = policyItem; + } + + @Override + public RangerPolicyItemDataMaskInfo getDataMaskInfo() { + return dataMaskPolicyItem == null ? null : dataMaskPolicyItem.getDataMaskInfo(); + } + + @Override + public void updateAccessResult(RangerPolicyEvaluator policyEvaluator, RangerAccessResult result, RangerPolicyResourceMatcher.MatchType matchType) { + RangerPolicyItemDataMaskInfo dataMaskInfo = getDataMaskInfo(); + + if (result.getMaskType() == null && dataMaskInfo != null) { + result.setMaskType(dataMaskInfo.getDataMaskType()); + result.setMaskCondition(dataMaskInfo.getConditionExpr()); + result.setMaskedValue(dataMaskInfo.getValueExpr()); + policyEvaluator.updateAccessResult(result, matchType, true, getComments()); + } + } + +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java new file mode 100644 index 00000000000..8ad61867dfa --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java @@ -0,0 +1,1432 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.policyevaluator; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.plugin.conditionevaluator.RangerAbstractConditionEvaluator; +import org.apache.atlas.plugin.conditionevaluator.RangerConditionEvaluator; +import org.apache.atlas.plugin.model.RangerPolicy; +import org.apache.atlas.plugin.model.RangerPolicy.RangerDataMaskPolicyItem; +import org.apache.atlas.plugin.model.RangerPolicy.RangerPolicyItem; +import org.apache.atlas.plugin.model.RangerPolicy.RangerPolicyItemAccess; +import org.apache.atlas.plugin.model.RangerPolicy.RangerPolicyResource; +import org.apache.atlas.plugin.model.RangerPolicy.RangerRowFilterPolicyItem; +import org.apache.atlas.plugin.model.RangerServiceDef; +import org.apache.atlas.plugin.model.RangerServiceDef.RangerAccessTypeDef; +import org.apache.atlas.plugin.model.RangerValiditySchedule; +import org.apache.atlas.plugin.policyengine.RangerAccessRequest; +import org.apache.atlas.plugin.policyengine.RangerAccessResource; +import org.apache.atlas.plugin.policyengine.RangerAccessResult; +import org.apache.atlas.plugin.policyengine.RangerPolicyEngine; +import org.apache.atlas.plugin.policyengine.RangerPolicyEngineOptions; +import org.apache.atlas.plugin.policyengine.RangerResourceAccessInfo; +import org.apache.atlas.plugin.policyengine.RangerTagAccessRequest; +import org.apache.atlas.plugin.policyresourcematcher.RangerDefaultPolicyResourceMatcher; +import org.apache.atlas.plugin.policyresourcematcher.RangerPolicyResourceMatcher; +import org.apache.atlas.plugin.resourcematcher.RangerResourceMatcher; +import org.apache.atlas.plugin.util.RangerAccessRequestUtil; +import org.apache.atlas.plugin.util.RangerPerfTracer; +import org.apache.atlas.plugin.util.ServiceDefUtil; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + + +public class RangerDefaultPolicyEvaluator extends RangerAbstractPolicyEvaluator { + private static final Log LOG = LogFactory.getLog(RangerDefaultPolicyEvaluator.class); + + private static final Log PERF_POLICY_INIT_LOG = RangerPerfTracer.getPerfLogger("policy.init"); + private static final Log PERF_POLICY_INIT_ACLSUMMARY_LOG = RangerPerfTracer.getPerfLogger("policy.init.ACLSummary"); + private static final Log PERF_POLICY_REQUEST_LOG = RangerPerfTracer.getPerfLogger("policy.request"); + private static final Log PERF_POLICYCONDITION_REQUEST_LOG = RangerPerfTracer.getPerfLogger("policycondition.request"); + + private RangerPolicyResourceMatcher resourceMatcher; + private List validityScheduleEvaluators; + private List allowEvaluators; + private List denyEvaluators; + private List allowExceptionEvaluators; + private List denyExceptionEvaluators; + private int customConditionsCount; + private List dataMaskEvaluators; + private List rowFilterEvaluators; + private List conditionEvaluators; + private String perfTag; + private PolicyACLSummary aclSummary = null; + private boolean useAclSummaryForEvaluation = false; + private boolean disableRoleResolution = true; + + protected boolean needsDynamicEval() { return resourceMatcher != null && resourceMatcher.getNeedsDynamicEval(); } + + @Override + public int getCustomConditionsCount() { + return customConditionsCount; + } + + @Override + public int getValidityScheduleEvaluatorsCount() { + return validityScheduleEvaluators.size(); + } + + @Override + public RangerPolicyResourceMatcher getPolicyResourceMatcher() { return resourceMatcher; } + + @Override + public RangerResourceMatcher getResourceMatcher(String resourceName) { + return resourceMatcher != null ? resourceMatcher.getResourceMatcher(resourceName) : null; + } + + @Override + public void init(RangerPolicy policy, RangerServiceDef serviceDef, RangerPolicyEngineOptions options) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerDefaultPolicyEvaluator.init()"); + } + + StringBuilder perfTagBuffer = new StringBuilder(); + if (policy != null) { + perfTagBuffer.append("policyId=").append(policy.getId()).append(", policyName=").append(policy.getName()); + } + + perfTag = perfTagBuffer.toString(); + + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_POLICY_INIT_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_POLICY_INIT_LOG, "RangerPolicyEvaluator.init(" + perfTag + ")"); + } + + super.init(policy, serviceDef, options); + + policy = getPolicy(); + + preprocessPolicy(policy, serviceDef); + + resourceMatcher = new RangerDefaultPolicyResourceMatcher(); + + resourceMatcher.setServiceDef(serviceDef); + resourceMatcher.setPolicy(policy); + resourceMatcher.setServiceDefHelper(options.getServiceDefHelper()); + resourceMatcher.init(); + + if(policy != null) { + validityScheduleEvaluators = createValidityScheduleEvaluators(policy); + + this.disableRoleResolution = options.disableRoleResolution; + + if (!options.disableAccessEvaluationWithPolicyACLSummary) { + aclSummary = createPolicyACLSummary(); + } + + useAclSummaryForEvaluation = aclSummary != null; + + if (useAclSummaryForEvaluation) { + allowEvaluators = Collections.emptyList(); + denyEvaluators = Collections.emptyList(); + allowExceptionEvaluators = Collections.emptyList(); + denyExceptionEvaluators = Collections.emptyList(); + } else { + allowEvaluators = createPolicyItemEvaluators(policy, serviceDef, options, RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_ALLOW); + + if (ServiceDefUtil.getOption_enableDenyAndExceptionsInPolicies(serviceDef, pluginContext)) { + denyEvaluators = createPolicyItemEvaluators(policy, serviceDef, options, RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_DENY); + allowExceptionEvaluators = createPolicyItemEvaluators(policy, serviceDef, options, RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_ALLOW_EXCEPTIONS); + denyExceptionEvaluators = createPolicyItemEvaluators(policy, serviceDef, options, RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_DENY_EXCEPTIONS); + } else { + denyEvaluators = Collections.emptyList(); + allowExceptionEvaluators = Collections.emptyList(); + denyExceptionEvaluators = Collections.emptyList(); + } + } + + dataMaskEvaluators = createDataMaskPolicyItemEvaluators(policy, serviceDef, options, policy.getDataMaskPolicyItems()); + rowFilterEvaluators = createRowFilterPolicyItemEvaluators(policy, serviceDef, options, policy.getRowFilterPolicyItems()); + conditionEvaluators = createRangerPolicyConditionEvaluator(policy, serviceDef, options); + } else { + validityScheduleEvaluators = Collections.emptyList(); + allowEvaluators = Collections.emptyList(); + denyEvaluators = Collections.emptyList(); + allowExceptionEvaluators = Collections.emptyList(); + denyExceptionEvaluators = Collections.emptyList(); + dataMaskEvaluators = Collections.emptyList(); + rowFilterEvaluators = Collections.emptyList(); + conditionEvaluators = Collections.emptyList(); + } + + RangerPolicyItemEvaluator.EvalOrderComparator comparator = new RangerPolicyItemEvaluator.EvalOrderComparator(); + Collections.sort(allowEvaluators, comparator); + Collections.sort(denyEvaluators, comparator); + Collections.sort(allowExceptionEvaluators, comparator); + Collections.sort(denyExceptionEvaluators, comparator); + + /* dataMask, rowFilter policyItems must be evaulated in the order given in the policy; hence no sort + Collections.sort(dataMaskEvaluators); + Collections.sort(rowFilterEvaluators); + */ + + RangerPerfTracer.log(perf); + + if (useAclSummaryForEvaluation && (StringUtils.isEmpty(policy.getPolicyType()) || policy.getPolicyType().equals(RangerPolicy.POLICY_TYPE_ACCESS))) { + LOG.info("PolicyEvaluator for policy:[" + policy.getId() + "] is set up to use ACL Summary to evaluate access"); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerDefaultPolicyEvaluator.init()"); + } + } + + @Override + public boolean isApplicable(Date accessTime) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerDefaultPolicyEvaluator.isApplicable(" + accessTime + ")"); + } + + boolean ret = false; + + if (accessTime != null && CollectionUtils.isNotEmpty(validityScheduleEvaluators)) { + for (RangerValidityScheduleEvaluator evaluator : validityScheduleEvaluators) { + if (evaluator.isApplicable(accessTime.getTime())) { + ret = true; + break; + } + } + } else { + ret = true; + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerDefaultPolicyEvaluator.isApplicable(" + accessTime + ") : " + ret); + } + + return ret; + } + + @Override + public void evaluate(RangerAccessRequest request, RangerAccessResult result) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerDefaultPolicyEvaluator.evaluate(policyId=" + getPolicy().getId() + ", " + request + ", " + result + ")"); + } + + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_POLICY_REQUEST_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_POLICY_REQUEST_LOG, "RangerPolicyEvaluator.evaluate(requestHashCode=" + Integer.toHexString(System.identityHashCode(request)) + "," + + perfTag + ")"); + } + + if (request != null && result != null) { + + if (!result.getIsAccessDetermined() || !result.getIsAuditedDetermined() || request.isAccessorsRequested()) { + RangerPolicyResourceMatcher.MatchType matchType; + + if (RangerTagAccessRequest.class.isInstance(request)) { + matchType = ((RangerTagAccessRequest) request).getMatchType(); + if (matchType == RangerPolicyResourceMatcher.MatchType.ANCESTOR) { + matchType = RangerPolicyResourceMatcher.MatchType.SELF; + } + } else { + if (request.getResourceMatchingScope() == RangerAccessRequest.ResourceMatchingScope.SELF_OR_CHILD) { + request.getContext().put(RangerAccessRequest.RANGER_ACCESS_REQUEST_SCOPE_STRING, RangerAccessRequest.ResourceMatchingScope.SELF_OR_CHILD); + } + matchType = resourceMatcher != null ? resourceMatcher.getMatchType(request.getResource(), request.getContext()) : RangerPolicyResourceMatcher.MatchType.NONE; + request.getContext().remove(RangerAccessRequest.RANGER_ACCESS_REQUEST_SCOPE_STRING); + } + + final boolean isMatched; + + if (request.isAccessTypeAny() || Boolean.TRUE.equals(RangerAccessRequestUtil.getIsAnyAccessInContext(request.getContext()))) { + isMatched = matchType != RangerPolicyResourceMatcher.MatchType.NONE; + } else if (request.getResourceMatchingScope() == RangerAccessRequest.ResourceMatchingScope.SELF_OR_DESCENDANTS) { + isMatched = matchType != RangerPolicyResourceMatcher.MatchType.NONE; + } else { + isMatched = matchType == RangerPolicyResourceMatcher.MatchType.SELF || matchType == RangerPolicyResourceMatcher.MatchType.SELF_AND_ALL_DESCENDANTS; + } + + if (isMatched) { + //Evaluate Policy Level Custom Conditions, if any and allowed then go ahead for policyItem level evaluation + if(matchPolicyCustomConditions(request)) { + if (!result.getIsAuditedDetermined()) { + if (isAuditEnabled()) { + result.setIsAudited(true); + result.setAuditPolicyId(getPolicy().getGuid()); + } + } + if (!result.getIsAccessDetermined() || request.isAccessorsRequested()) { + if (hasMatchablePolicyItem(request)) { + evaluatePolicyItems(request, matchType, result); + } + } + } + } + } + } + + RangerPerfTracer.log(perf); + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerDefaultPolicyEvaluator.evaluate(policyId=" + getPolicy().getId() + ", " + request + ", " + result + ")"); + } + } + + @Override + public boolean isMatch(RangerAccessResource resource, Map evalContext) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerDefaultPolicyEvaluator.isMatch(" + resource + ", " + evalContext + ")"); + } + + boolean ret = false; + + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_POLICY_REQUEST_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_POLICY_REQUEST_LOG, "RangerPolicyEvaluator.isMatch(resource=" + resource.getAsString() + "," + evalContext + "," + perfTag + ")"); + } + + if(resourceMatcher != null) { + ret = resourceMatcher.isMatch(resource, evalContext); + } + + RangerPerfTracer.log(perf); + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerDefaultPolicyEvaluator.isMatch(" + resource + ", " + evalContext + "): " + ret); + } + + return ret; + } + + @Override + public boolean isCompleteMatch(RangerAccessResource resource, Map evalContext) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerDefaultPolicyEvaluator.isCompleteMatch(" + resource + ", " + evalContext + ")"); + } + + boolean ret = resourceMatcher != null && resourceMatcher.isCompleteMatch(resource, evalContext); + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerDefaultPolicyEvaluator.isCompleteMatch(" + resource + "): " + ret); + } + + return ret; + } + + @Override + public boolean isCompleteMatch(Map resources, Map evalContext) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerDefaultPolicyEvaluator.isCompleteMatch(" + resources + ", " + evalContext + ")"); + } + + boolean ret = resourceMatcher != null && resourceMatcher.isCompleteMatch(resources, evalContext); + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerDefaultPolicyEvaluator.isCompleteMatch(" + resources + ", " + evalContext + "): " + ret); + } + + return ret; + } + + @Override + public Set getAllowedAccesses(RangerAccessResource resource, String user, Set userGroups, Set roles, Set accessTypes) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerDefaultPolicyEvaluator.getAllowedAccesses(" + resource + ", " + user + ", " + userGroups + ", " + roles + ", " + accessTypes + ")"); + } + + Set ret = null; + + if (isMatch(resource, null)) { + ret = new HashSet<>(); + for (String accessType : accessTypes) { + if (isAccessAllowed(user, userGroups, roles, resource.getOwnerUser(), accessType)) { + ret.add(accessType); + } + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerDefaultPolicyEvaluator.getAllowedAccesses(" + resource + ", " + user + ", " + userGroups + ", " + roles + ", " + accessTypes + "): " + ret); + } + + return ret; + } + + @Override + public Set getAllowedAccesses(Map resources, String user, Set userGroups, Set roles, Set accessTypes, Map evalContext) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerDefaultPolicyEvaluator.getAllowedAccesses(" + getPolicy().getId() + ", " + user + ", " + userGroups + ", " + roles + ", " + accessTypes + ", " + evalContext + ")"); + } + + Set ret = null; + + if (isMatch(resources, evalContext)) { + if (CollectionUtils.isNotEmpty(accessTypes)) { + ret = new HashSet<>(); + for (String accessType : accessTypes) { + if (isAccessAllowed(user, userGroups, roles, null, accessType)) { + ret.add(accessType); + } + } + } else { + if (isAccessAllowed(user, userGroups, roles, null, null)) { + ret = new HashSet<>(); + } + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerDefaultPolicyEvaluator.getAllowedAccesses(" + getPolicy().getId() + ", " + user + ", " + userGroups + ", " + roles + ", " + accessTypes + ", " + evalContext + "): " + ret); + } + + return ret; + } + /* + * This is used only by test code + */ + + @Override + public boolean isAccessAllowed(Map resources, String user, Set userGroups, String accessType) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerDefaultPolicyEvaluator.isAccessAllowed(" + resources + ", " + user + ", " + userGroups + ", " + accessType + ")"); + } + + boolean ret = isAccessAllowed(user, userGroups, null, null, accessType) && isMatch(resources, null); + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerDefaultPolicyEvaluator.isAccessAllowed(" + resources + ", " + user + ", " + userGroups + ", " + accessType + "): " + ret); + } + + return ret; + } + + + @Override + public void getResourceAccessInfo(RangerAccessRequest request, RangerResourceAccessInfo result) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerDefaultPolicyEvaluator.getResourceAccessInfo(" + request + ", " + result + ")"); + } + RangerPolicyResourceMatcher.MatchType matchType; + if (RangerTagAccessRequest.class.isInstance(request)) { + matchType = ((RangerTagAccessRequest) request).getMatchType(); + } else { + matchType = resourceMatcher != null ? resourceMatcher.getMatchType(request.getResource(), request.getContext()) : RangerPolicyResourceMatcher.MatchType.NONE; + } + + final boolean isMatched = matchType != RangerPolicyResourceMatcher.MatchType.NONE; + + if (isMatched) { + + if (CollectionUtils.isNotEmpty(allowEvaluators)) { + Set users = new HashSet<>(); + Set groups = new HashSet<>(); + + getResourceAccessInfo(request, allowEvaluators, users, groups); + + if (CollectionUtils.isNotEmpty(allowExceptionEvaluators)) { + Set exceptionUsers = new HashSet<>(); + Set exceptionGroups = new HashSet<>(); + + getResourceAccessInfo(request, allowExceptionEvaluators, exceptionUsers, exceptionGroups); + + users.removeAll(exceptionUsers); + groups.removeAll(exceptionGroups); + } + + result.getAllowedUsers().addAll(users); + result.getAllowedGroups().addAll(groups); + } + if (matchType != RangerPolicyResourceMatcher.MatchType.DESCENDANT) { + if (CollectionUtils.isNotEmpty(denyEvaluators)) { + Set users = new HashSet(); + Set groups = new HashSet(); + + getResourceAccessInfo(request, denyEvaluators, users, groups); + + if (CollectionUtils.isNotEmpty(denyExceptionEvaluators)) { + Set exceptionUsers = new HashSet(); + Set exceptionGroups = new HashSet(); + + getResourceAccessInfo(request, denyExceptionEvaluators, exceptionUsers, exceptionGroups); + + users.removeAll(exceptionUsers); + groups.removeAll(exceptionGroups); + } + + result.getDeniedUsers().addAll(users); + result.getDeniedGroups().addAll(groups); + } + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerDefaultPolicyEvaluator.getResourceAccessInfo(" + request + ", " + result + ")"); + } + } + + /* + This API is called by policy-engine to support components which need to statically determine Ranger ACLs + for a given resource. It will always return a non-null object. The accesses that cannot be determined + statically will be marked as CONDITIONAL. + */ + + @Override + public PolicyACLSummary getPolicyACLSummary() { + if (aclSummary == null) { + boolean forceCreation = true; + aclSummary = createPolicyACLSummary(forceCreation); + } + + return aclSummary; + } + + @Override + public void updateAccessResult(RangerAccessResult result, RangerPolicyResourceMatcher.MatchType matchType, boolean isAllowed, String reason) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerDefaultPolicyEvaluator.updateAccessResult(" + result + ", " + matchType +", " + isAllowed + ", " + reason + ", " + getId() + ")"); + } + if (!isAllowed) { + RangerAccessResource resource = result.getAccessRequest().getResource(); + if (resource != null && MapUtils.isNotEmpty(resource.getAsMap())) { + result.setIsAllowed(false); + result.setPolicyPriority(getPolicyPriority()); + result.setPolicyId(getGuid()); + result.setReason(reason); + result.setPolicyVersion(getPolicy().getVersion()); + } + } else { + if (!result.getIsAllowed()) { // if access is not yet allowed by another policy + if (matchType != RangerPolicyResourceMatcher.MatchType.ANCESTOR) { + result.setIsAllowed(true); + result.setPolicyPriority(getPolicyPriority()); + result.setPolicyId(getGuid()); + result.setReason(reason); + result.setPolicyVersion(getPolicy().getVersion()); + } + } + } + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerDefaultPolicyEvaluator.updateAccessResult(" + result + ", " + matchType +", " + isAllowed + ", " + reason + ", " + getId() + ")"); + } + } + + /* + This API is only called during initialization of Policy Evaluator if policy-engine is configured to use + PolicyACLSummary for access evaluation (that is, if disableAccessEvaluationWithPolicyACLSummary option + is set to false). It may return null object if all accesses for all user/groups cannot be determined statically. + */ + + private PolicyACLSummary createPolicyACLSummary() { + boolean forceCreation = false; + return createPolicyACLSummary(forceCreation); + } + + private PolicyACLSummary createPolicyACLSummary(boolean isCreationForced) { + PolicyACLSummary ret = null; + RangerPerfTracer perf = null; + + if (RangerPerfTracer.isPerfTraceEnabled(PERF_POLICY_INIT_ACLSUMMARY_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_POLICY_INIT_ACLSUMMARY_LOG, "RangerPolicyEvaluator.init.ACLSummary(" + perfTag + ")"); + } + + RangerPolicy policy; + if (!disableRoleResolution && hasRoles(getPolicy())) { + policy = getPolicyWithRolesResolved(getPolicy()); + } else { + policy = getPolicy(); + } + + final boolean hasNonPublicGroupOrConditionsInAllowExceptions = hasNonPublicGroupOrConditions(policy.getAllowExceptions()); + final boolean hasNonPublicGroupOrConditionsInDenyExceptions = hasNonPublicGroupOrConditions(policy.getDenyExceptions()); + final boolean hasPublicGroupInAllowAndUsersInAllowExceptions = hasPublicGroupAndUserInException(policy.getPolicyItems(), policy.getAllowExceptions()); + final boolean hasPublicGroupInDenyAndUsersInDenyExceptions = hasPublicGroupAndUserInException(policy.getDenyPolicyItems(), policy.getDenyExceptions()); + final boolean hasContextSensitiveSpecification = hasContextSensitiveSpecification(); + final boolean hasRoles = hasRoles(policy); + final boolean isUsableForEvaluation = !hasNonPublicGroupOrConditionsInAllowExceptions + && !hasNonPublicGroupOrConditionsInDenyExceptions + && !hasPublicGroupInAllowAndUsersInAllowExceptions + && !hasPublicGroupInDenyAndUsersInDenyExceptions + && !hasContextSensitiveSpecification + && !hasRoles; + + if (isUsableForEvaluation || isCreationForced) { + ret = new PolicyACLSummary(); + + for (RangerPolicyItem policyItem : policy.getDenyPolicyItems()) { + ret.processPolicyItem(policyItem, RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_DENY, hasNonPublicGroupOrConditionsInDenyExceptions || hasPublicGroupInDenyAndUsersInDenyExceptions); + } + + if (!hasNonPublicGroupOrConditionsInDenyExceptions && !hasPublicGroupInDenyAndUsersInDenyExceptions) { + for (RangerPolicyItem policyItem : policy.getDenyExceptions()) { + ret.processPolicyItem(policyItem, RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_DENY_EXCEPTIONS, false); + } + } + + for (RangerPolicyItem policyItem : policy.getPolicyItems()) { + ret.processPolicyItem(policyItem, RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_ALLOW, hasNonPublicGroupOrConditionsInAllowExceptions || hasPublicGroupInAllowAndUsersInAllowExceptions); + } + + if (!hasNonPublicGroupOrConditionsInAllowExceptions && !hasPublicGroupInAllowAndUsersInAllowExceptions) { + for (RangerPolicyItem policyItem : policy.getAllowExceptions()) { + ret.processPolicyItem(policyItem, RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_ALLOW_EXCEPTIONS, false); + } + } + + for (RangerRowFilterPolicyItem policyItem : policy.getRowFilterPolicyItems()) { + ret.processRowFilterPolicyItem(policyItem); + } + + for (RangerDataMaskPolicyItem policyItem : policy.getDataMaskPolicyItems()) { + ret.processDataMaskPolicyItem(policyItem); + } + + final boolean isDenyAllElse = Boolean.TRUE.equals(policy.getIsDenyAllElse()); + + final Set allAccessTypeNames; + + if (isDenyAllElse) { + allAccessTypeNames = new HashSet<>(); + RangerServiceDef serviceDef = getServiceDef(); + for (RangerAccessTypeDef accessTypeDef :serviceDef.getAccessTypes()) { + if (!StringUtils.equalsIgnoreCase(accessTypeDef.getName(), "all")) { + allAccessTypeNames.add(accessTypeDef.getName()); + } + } + } else { + allAccessTypeNames = Collections.EMPTY_SET; + } + + ret.finalizeAcls(isDenyAllElse, allAccessTypeNames); + } + + RangerPerfTracer.logAlways(perf); + + return ret; + } + + private RangerPolicy getPolicyWithRolesResolved(final RangerPolicy policy) { + // Create new policy with no roles in it + // For each policyItem, expand roles into users and groups; and replace all policyItems with expanded roles - TBD + + RangerPolicy ret = new RangerPolicy(); + ret.updateFrom(policy); + ret.setId(policy.getId()); + ret.setGuid(policy.getGuid()); + ret.setVersion(policy.getVersion()); + + List policyItems = new ArrayList<>(); + List denyPolicyItems = new ArrayList<>(); + List allowExceptions = new ArrayList<>(); + List denyExceptions = new ArrayList<>(); + List dataMaskPolicyItems = new ArrayList<>(); + List rowFilterPolicyItems = new ArrayList<>(); + + for (RangerPolicyItem policyItem : policy.getPolicyItems()) { + RangerPolicyItem newPolicyItem = new RangerPolicyItem(policyItem.getAccesses(), policyItem.getUsers(), policyItem.getGroups(), policyItem.getRoles(), policyItem.getConditions(), policyItem.getDelegateAdmin()); + getPolicyItemWithRolesResolved(newPolicyItem, policyItem); + + policyItems.add(newPolicyItem); + } + ret.setPolicyItems(policyItems); + + for (RangerPolicyItem policyItem : policy.getDenyPolicyItems()) { + RangerPolicyItem newPolicyItem = new RangerPolicyItem(policyItem.getAccesses(), policyItem.getUsers(), policyItem.getGroups(), policyItem.getRoles(), policyItem.getConditions(), policyItem.getDelegateAdmin()); + getPolicyItemWithRolesResolved(newPolicyItem, policyItem); + + denyPolicyItems.add(newPolicyItem); + } + ret.setDenyPolicyItems(denyPolicyItems); + + for (RangerPolicyItem policyItem : policy.getAllowExceptions()) { + RangerPolicyItem newPolicyItem = new RangerPolicyItem(policyItem.getAccesses(), policyItem.getUsers(), policyItem.getGroups(), policyItem.getRoles(), policyItem.getConditions(), policyItem.getDelegateAdmin()); + getPolicyItemWithRolesResolved(newPolicyItem, policyItem); + + allowExceptions.add(newPolicyItem); + } + ret.setAllowExceptions(allowExceptions); + + for (RangerPolicyItem policyItem : policy.getDenyExceptions()) { + RangerPolicyItem newPolicyItem = new RangerPolicyItem(policyItem.getAccesses(), policyItem.getUsers(), policyItem.getGroups(), policyItem.getRoles(), policyItem.getConditions(), policyItem.getDelegateAdmin()); + getPolicyItemWithRolesResolved(newPolicyItem, policyItem); + + denyExceptions.add(newPolicyItem); + } + ret.setDenyExceptions(denyExceptions); + + for (RangerDataMaskPolicyItem policyItem : policy.getDataMaskPolicyItems()) { + RangerDataMaskPolicyItem newPolicyItem = new RangerDataMaskPolicyItem(policyItem.getAccesses(), policyItem.getDataMaskInfo(), policyItem.getUsers(), policyItem.getGroups(), policyItem.getRoles(), policyItem.getConditions(), policyItem.getDelegateAdmin()); + getPolicyItemWithRolesResolved(newPolicyItem, policyItem); + + dataMaskPolicyItems.add(newPolicyItem); + } + ret.setDataMaskPolicyItems(dataMaskPolicyItems); + + for (RangerRowFilterPolicyItem policyItem : policy.getRowFilterPolicyItems()) { + RangerRowFilterPolicyItem newPolicyItem = new RangerRowFilterPolicyItem(policyItem.getRowFilterInfo(), policyItem.getAccesses(), policyItem.getUsers(), policyItem.getGroups(), policyItem.getRoles(), policyItem.getConditions(), policyItem.getDelegateAdmin()); + getPolicyItemWithRolesResolved(newPolicyItem, policyItem); + + rowFilterPolicyItems.add(newPolicyItem); + } + ret.setRowFilterPolicyItems(rowFilterPolicyItems); + + return ret; + } + + private void getPolicyItemWithRolesResolved(RangerPolicyItem newPolicyItem, final RangerPolicyItem policyItem) { + Set usersFromRoles = new HashSet<>(); + Set groupsFromRoles = new HashSet<>(); + + List roles = policyItem.getRoles(); + + for (String role : roles) { + Set users = getPluginContext().getAuthContext().getRangerRolesUtil().getRoleToUserMapping().get(role); + Set groups = getPluginContext().getAuthContext().getRangerRolesUtil().getRoleToGroupMapping().get(role); + if (CollectionUtils.isNotEmpty(users)) { + usersFromRoles.addAll(users); + } + if (CollectionUtils.isNotEmpty(groups)) { + groupsFromRoles.addAll(groups); + } + if (CollectionUtils.isNotEmpty(usersFromRoles) || CollectionUtils.isNotEmpty(groupsFromRoles)) { + usersFromRoles.addAll(policyItem.getUsers()); + groupsFromRoles.addAll(policyItem.getGroups()); + + newPolicyItem.setUsers(new ArrayList<>(usersFromRoles)); + newPolicyItem.setGroups(new ArrayList<>(groupsFromRoles)); + newPolicyItem.setRoles(null); + } + } + } + + private boolean hasPublicGroupAndUserInException(List grants, List exceptionItems) { + boolean ret = false; + + if (CollectionUtils.isNotEmpty(exceptionItems)) { + boolean hasPublicGroupInGrant = false; + + for (RangerPolicyItem policyItem : grants) { + if (policyItem.getGroups().contains(RangerPolicyEngine.GROUP_PUBLIC) || policyItem.getUsers().contains(RangerPolicyEngine.USER_CURRENT)) { + hasPublicGroupInGrant = true; + break; + } + } + + if (hasPublicGroupInGrant) { + boolean hasPublicGroupInException = false; + + for (RangerPolicyItem policyItem : exceptionItems) { + if (policyItem.getGroups().contains(RangerPolicyEngine.GROUP_PUBLIC) || policyItem.getUsers().contains(RangerPolicyEngine.USER_CURRENT)) { + hasPublicGroupInException = true; + break; + } + } + + if (!hasPublicGroupInException) { + ret = true; + } + } + } + + return ret; + } + + protected void evaluatePolicyItems(RangerAccessRequest request, RangerPolicyResourceMatcher.MatchType matchType, RangerAccessResult result) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerDefaultPolicyEvaluator.evaluatePolicyItems(" + request + ", " + result + ", " + matchType + ")"); + } + + if (useAclSummaryForEvaluation && (StringUtils.isEmpty(getPolicy().getPolicyType()) || getPolicy().getPolicyType().equals(RangerPolicy.POLICY_TYPE_ACCESS))) { + if (LOG.isDebugEnabled()) { + LOG.debug("Using ACL Summary for access evaluation. PolicyId=[" + getId() + "]"); + } + Integer accessResult = lookupPolicyACLSummary(request.getUser(), request.getUserGroups(), request.getUserRoles(), request.isAccessTypeAny() || Boolean.TRUE.equals(RangerAccessRequestUtil.getIsAnyAccessInContext(request.getContext())) ? RangerPolicyEngine.ANY_ACCESS : request.getAccessType()); + if (accessResult != null) { + updateAccessResult(result, matchType, accessResult.equals(ACCESS_ALLOWED), null); + } else if (getPolicy().getIsDenyAllElse()) { + updateAccessResult(result, RangerPolicyResourceMatcher.MatchType.NONE, false, "matched deny-all-else policy"); + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Using policyItemEvaluators for access evaluation. PolicyId=[" + getId() + "]"); + } + + RangerPolicyItemEvaluator matchedPolicyItem = getMatchingPolicyItem(request, result); + + if (matchedPolicyItem != null) { + matchedPolicyItem.updateAccessResult(this, result, matchType); + } else if (getPolicy().getIsDenyAllElse() && (StringUtils.isEmpty(getPolicy().getPolicyType()) || getPolicy().getPolicyType().equals(RangerPolicy.POLICY_TYPE_ACCESS))) { + updateAccessResult(result, RangerPolicyResourceMatcher.MatchType.NONE, false, "matched deny-all-else policy"); + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerDefaultPolicyEvaluator.evaluatePolicyItems(" + request + ", " + result + ", " + matchType + ")"); + } + } + + private Integer lookupPolicyACLSummary(String user, Set userGroups, Set userRoles, String accessType) { + Integer accessResult = null; + + Map accesses = aclSummary.getUsersAccessInfo().get(user); + + accessResult = lookupAccess(user, accessType, accesses); + + if (accessResult == null) { + + Set groups = new HashSet<>(); + groups.add(RangerPolicyEngine.GROUP_PUBLIC); + groups.addAll(userGroups); + + for (String userGroup : groups) { + accesses = aclSummary.getGroupsAccessInfo().get(userGroup); + accessResult = lookupAccess(userGroup, accessType, accesses); + if (accessResult != null) { + break; + } + } + + if (userRoles !=null) { + for (String userRole : userRoles) { + accesses = aclSummary.getRolesAccessInfo().get(userRole); + accessResult = lookupAccess(userRole, accessType, accesses); + if (accessResult != null) { + break; + } + } + } + } + + return accessResult; + } + + private Integer lookupAccess(String userOrGroup, String accessType, Map accesses) { + Integer ret = null; + if (accesses != null) { + if (accessType.equals(RangerPolicyEngine.ANY_ACCESS)) { + ret = getAccessResultForAnyAccess(accesses); + } else { + PolicyACLSummary.AccessResult accessResult = accesses.get(accessType); + if (accessResult != null) { + if (accessResult.getResult() == ACCESS_CONDITIONAL) { + LOG.error("Access should not be conditional at this point! user=[" + userOrGroup + "], " + "accessType=[" + accessType + "]"); + } else { + ret = accessResult.getResult(); + } + } + } + } + return ret; + } + + private Integer getAccessResultForAnyAccess(Map accesses) { + final Integer ret; + + int allowedAccessCount = 0; + int deniedAccessCount = 0; + + for (Map.Entry entry : accesses.entrySet()) { + if (StringUtils.equals(entry.getKey(), RangerPolicyEngine.ADMIN_ACCESS)) { + // Don't count admin access if present + continue; + } + PolicyACLSummary.AccessResult accessResult = entry.getValue(); + if (accessResult.getResult() == ACCESS_ALLOWED) { + allowedAccessCount++; + break; + } else if (accessResult.getResult() == ACCESS_DENIED) { + deniedAccessCount++; + } + } + + if (allowedAccessCount > 0) { + // At least one access allowed + ret = ACCESS_ALLOWED; + } else if (deniedAccessCount == getServiceDef().getAccessTypes().size()) { + // All accesses explicitly denied + ret = ACCESS_DENIED; + } else { + ret = null; + } + + return ret; + } + + protected RangerPolicyItemEvaluator getDeterminingPolicyItem(String user, Set userGroups, Set roles, String owner, String accessType) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerDefaultPolicyEvaluator.getDeterminingPolicyItem(" + user + ", " + userGroups + ", " + roles + ", " + owner + ", " + accessType + ")"); + } + + RangerPolicyItemEvaluator ret = null; + + /* + * 1. if a deny matches without hitting any deny-exception, return that + * 2. if an allow matches without hitting any allow-exception, return that + */ + ret = getMatchingPolicyItem(user, userGroups, roles, owner, accessType, denyEvaluators, denyExceptionEvaluators); + + if(ret == null) { + ret = getMatchingPolicyItem(user, userGroups, roles, owner, accessType, allowEvaluators, allowExceptionEvaluators); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerDefaultPolicyEvaluator.getDeterminingPolicyItem(" + user + ", " + userGroups + ", " + roles + ", " + owner + ", " + accessType + "): " + ret); + } + + return ret; + } + + private void getResourceAccessInfo(RangerAccessRequest request, List policyItems, Set users, Set groups) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerDefaultPolicyEvaluator.getResourceAccessInfo(" + request + ", " + policyItems + ", " + users + ", " + groups + ")"); + } + + if (CollectionUtils.isNotEmpty(policyItems)) { + for (RangerPolicyItemEvaluator policyItemEvaluator : policyItems) { + if (policyItemEvaluator.matchAccessType(request.getAccessType()) && policyItemEvaluator.matchCustomConditions(request)) { + if (CollectionUtils.isNotEmpty(policyItemEvaluator.getPolicyItem().getUsers())) { + users.addAll(policyItemEvaluator.getPolicyItem().getUsers()); + } + + if (CollectionUtils.isNotEmpty(policyItemEvaluator.getPolicyItem().getGroups())) { + groups.addAll(policyItemEvaluator.getPolicyItem().getGroups()); + } + } + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerDefaultPolicyEvaluator.getResourceAccessInfo(" + request + ", " + policyItems + ", " + users + ", " + groups + ")"); + } + } + + protected boolean isMatch(RangerPolicy policy, Map evalContext) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerDefaultPolicyEvaluator.isMatch(" + policy.getId() + ", " + evalContext + ")"); + } + + final boolean ret = isMatch(policy.getResources(), evalContext); + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerDefaultPolicyEvaluator.isMatch(" + policy.getId() + ", " + evalContext + "): " + ret); + } + + return ret; + } + + protected boolean isMatch(Map resources, Map evalContext) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerDefaultPolicyEvaluator.isMatch(" + resources + ", " + evalContext + ")"); + } + + boolean ret = resourceMatcher != null && resourceMatcher.isMatch(resources, evalContext); + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerDefaultPolicyEvaluator.isMatch(" + resources + ", " + evalContext + "): " + ret); + } + + return ret; + } + + protected boolean isAccessAllowed(String user, Set userGroups, Set roles, String owner, String accessType) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerDefaultPolicyEvaluator.isAccessAllowed(" + user + ", " + userGroups + ", " + roles + ", " + owner + ", " + accessType + ")"); + } + + boolean ret = false; + + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_POLICY_REQUEST_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_POLICY_REQUEST_LOG, "RangerPolicyEvaluator.isAccessAllowed(hashCode=" + Integer.toHexString(System.identityHashCode(this)) + "," + perfTag + ")"); + } + + if (useAclSummaryForEvaluation && (StringUtils.isEmpty(getPolicy().getPolicyType()) || getPolicy().getPolicyType().equals(RangerPolicy.POLICY_TYPE_ACCESS))) { + if (LOG.isDebugEnabled()) { + LOG.debug("Using ACL Summary for checking if access is allowed. PolicyId=[" + getId() +"]"); + } + + Integer accessResult = StringUtils.isEmpty(accessType) ? null : lookupPolicyACLSummary(user, userGroups, roles, accessType); + if (accessResult != null && accessResult.equals(ACCESS_ALLOWED)) { + ret = true; + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Using policyItemEvaluators for checking if access is allowed. PolicyId=[" + getId() +"]"); + } + + RangerPolicyItemEvaluator item = this.getDeterminingPolicyItem(user, userGroups, roles, owner, accessType); + + if (item != null && item.getPolicyItemType() == RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_ALLOW) { + ret = true; + } + } + + RangerPerfTracer.log(perf); + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerDefaultPolicyEvaluator.isAccessAllowed(" + user + ", " + userGroups + ", " + roles + ", " + owner + ", " + accessType + "): " + ret); + } + + return ret; + } + + public StringBuilder toString(StringBuilder sb) { + sb.append("RangerDefaultPolicyEvaluator={"); + + super.toString(sb); + + sb.append("resourceMatcher={"); + if(resourceMatcher != null) { + resourceMatcher.toString(sb); + } + sb.append("} "); + + sb.append("}"); + + return sb; + } + + protected void preprocessPolicy(RangerPolicy policy, RangerServiceDef serviceDef) { + if(policy == null || (!hasAllow() && !hasDeny()) || serviceDef == null) { + return; + } + + Map> impliedAccessGrants = getImpliedAccessGrants(serviceDef); + + if(impliedAccessGrants == null || impliedAccessGrants.isEmpty()) { + return; + } + + preprocessPolicyItems(policy.getPolicyItems(), impliedAccessGrants); + preprocessPolicyItems(policy.getDenyPolicyItems(), impliedAccessGrants); + preprocessPolicyItems(policy.getAllowExceptions(), impliedAccessGrants); + preprocessPolicyItems(policy.getDenyExceptions(), impliedAccessGrants); + preprocessPolicyItems(policy.getDataMaskPolicyItems(), impliedAccessGrants); + preprocessPolicyItems(policy.getRowFilterPolicyItems(), impliedAccessGrants); + } + + protected void preprocessPolicyItems(List policyItems, Map> impliedAccessGrants) { + for(RangerPolicyItem policyItem : policyItems) { + if(CollectionUtils.isEmpty(policyItem.getAccesses())) { + continue; + } + + // Only one round of 'expansion' is done; multi-level impliedGrants (like shown below) are not handled for now + // multi-level impliedGrants: given admin=>write; write=>read: must imply admin=>read,write + for(Map.Entry> e : impliedAccessGrants.entrySet()) { + String accessType = e.getKey(); + Collection impliedGrants = e.getValue(); + + RangerPolicyItemAccess access = getAccess(policyItem, accessType); + + if(access == null) { + continue; + } + + for(String impliedGrant : impliedGrants) { + RangerPolicyItemAccess impliedAccess = getAccess(policyItem, impliedGrant); + + if(impliedAccess == null) { + impliedAccess = new RangerPolicyItemAccess(impliedGrant, access.getIsAllowed()); + + policyItem.getAccesses().add(impliedAccess); + } else { + if(! impliedAccess.getIsAllowed()) { + impliedAccess.setIsAllowed(access.getIsAllowed()); + } + } + } + } + } + } + + protected Map> getImpliedAccessGrants(RangerServiceDef serviceDef) { + Map> ret = null; + + if(serviceDef != null && !CollectionUtils.isEmpty(serviceDef.getAccessTypes())) { + for(RangerAccessTypeDef accessTypeDef : serviceDef.getAccessTypes()) { + if(!CollectionUtils.isEmpty(accessTypeDef.getImpliedGrants())) { + if(ret == null) { + ret = new HashMap<>(); + } + + Collection impliedAccessGrants = ret.get(accessTypeDef.getName()); + + if(impliedAccessGrants == null) { + impliedAccessGrants = new HashSet<>(); + + ret.put(accessTypeDef.getName(), impliedAccessGrants); + } + + impliedAccessGrants.addAll(accessTypeDef.getImpliedGrants()); + } + } + } + + return ret; + } + + private RangerPolicyItemAccess getAccess(RangerPolicyItem policyItem, String accessType) { + RangerPolicyItemAccess ret = null; + + if(policyItem != null && CollectionUtils.isNotEmpty(policyItem.getAccesses())) { + for(RangerPolicyItemAccess itemAccess : policyItem.getAccesses()) { + if(StringUtils.equalsIgnoreCase(itemAccess.getType(), accessType)) { + ret = itemAccess; + + break; + } + } + } + + return ret; + } + + private List createValidityScheduleEvaluators(RangerPolicy policy) { + List ret = null; + + if (CollectionUtils.isNotEmpty(policy.getValiditySchedules())) { + ret = new ArrayList<>(); + + for (RangerValiditySchedule schedule : policy.getValiditySchedules()) { + ret.add(new RangerValidityScheduleEvaluator(schedule)); + } + } else { + ret = Collections.emptyList(); + } + + return ret; + } + + private List createPolicyItemEvaluators(RangerPolicy policy, RangerServiceDef serviceDef, RangerPolicyEngineOptions options, int policyItemType) { + List ret = null; + List policyItems = null; + + if(isPolicyItemTypeEnabled(serviceDef, policyItemType)) { + if (policyItemType == RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_ALLOW) { + policyItems = policy.getPolicyItems(); + } else if (policyItemType == RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_DENY) { + policyItems = policy.getDenyPolicyItems(); + } else if (policyItemType == RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_ALLOW_EXCEPTIONS) { + policyItems = policy.getAllowExceptions(); + } else if (policyItemType == RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_DENY_EXCEPTIONS) { + policyItems = policy.getDenyExceptions(); + } + } + + if(CollectionUtils.isNotEmpty(policyItems)) { + ret = new ArrayList<>(); + + int policyItemCounter = 1; + + for(RangerPolicyItem policyItem : policyItems) { + RangerPolicyItemEvaluator itemEvaluator = new RangerDefaultPolicyItemEvaluator(serviceDef, policy, policyItem, policyItemType, policyItemCounter++, options); + + itemEvaluator.init(); + + ret.add(itemEvaluator); + + if(CollectionUtils.isNotEmpty(itemEvaluator.getConditionEvaluators())) { + customConditionsCount += itemEvaluator.getConditionEvaluators().size(); + } + } + } else { + ret = Collections.emptyList(); + } + + return ret; + } + + private List createDataMaskPolicyItemEvaluators(RangerPolicy policy, RangerServiceDef serviceDef, RangerPolicyEngineOptions options, List policyItems) { + List ret = null; + + if(CollectionUtils.isNotEmpty(policyItems)) { + ret = new ArrayList<>(); + + int policyItemCounter = 1; + + for(RangerDataMaskPolicyItem policyItem : policyItems) { + RangerDataMaskPolicyItemEvaluator itemEvaluator = new RangerDefaultDataMaskPolicyItemEvaluator(serviceDef, policy, policyItem, policyItemCounter++, options); + + itemEvaluator.init(); + + ret.add(itemEvaluator); + + if(CollectionUtils.isNotEmpty(itemEvaluator.getConditionEvaluators())) { + customConditionsCount += itemEvaluator.getConditionEvaluators().size(); + } + } + } else { + ret = Collections.emptyList(); + } + + return ret; + } + + private List createRowFilterPolicyItemEvaluators(RangerPolicy policy, RangerServiceDef serviceDef, RangerPolicyEngineOptions options, List policyItems) { + List ret = null; + + if(CollectionUtils.isNotEmpty(policyItems)) { + ret = new ArrayList<>(); + + int policyItemCounter = 1; + + for(RangerRowFilterPolicyItem policyItem : policyItems) { + RangerRowFilterPolicyItemEvaluator itemEvaluator = new RangerDefaultRowFilterPolicyItemEvaluator(serviceDef, policy, policyItem, policyItemCounter++, options); + + itemEvaluator.init(); + + ret.add(itemEvaluator); + + if(CollectionUtils.isNotEmpty(itemEvaluator.getConditionEvaluators())) { + customConditionsCount += itemEvaluator.getConditionEvaluators().size(); + } + } + } else { + ret = Collections.emptyList(); + } + + return ret; + } + + private boolean isPolicyItemTypeEnabled(RangerServiceDef serviceDef, int policyItemType) { + boolean ret = true; + + if(policyItemType == RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_DENY || + policyItemType == RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_ALLOW_EXCEPTIONS || + policyItemType == RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_DENY_EXCEPTIONS) { + ret = ServiceDefUtil.getOption_enableDenyAndExceptionsInPolicies(serviceDef, pluginContext); + } + + return ret; + } + + private static boolean hasNonPublicGroupOrConditions(List policyItems) { + boolean ret = false; + for (RangerPolicyItem policyItem : policyItems) { + if (CollectionUtils.isNotEmpty(policyItem.getConditions())) { + ret = true; + break; + } + List allGroups = policyItem.getGroups(); + if (CollectionUtils.isNotEmpty(allGroups) && !allGroups.contains(RangerPolicyEngine.GROUP_PUBLIC)) { + ret = true; + break; + } + } + return ret; + } + + protected RangerPolicyItemEvaluator getMatchingPolicyItem(RangerAccessRequest request, RangerAccessResult result) { + RangerPolicyItemEvaluator ret = null; + + String policyType = getPolicy().getPolicyType(); + if (StringUtils.isEmpty(policyType)) { + policyType = RangerPolicy.POLICY_TYPE_ACCESS; + } + + switch (policyType) { + case RangerPolicy.POLICY_TYPE_ACCESS: { + ret = getMatchingPolicyItemForAccessPolicyForSpecificAccess(request, result); + break; + } + case RangerPolicy.POLICY_TYPE_DATAMASK: { + ret = getMatchingPolicyItem(request, result, dataMaskEvaluators); + break; + } + case RangerPolicy.POLICY_TYPE_ROWFILTER: { + ret = getMatchingPolicyItem(request, result, rowFilterEvaluators); + break; + } + default: + break; + } + + return ret; + } + + protected RangerPolicyItemEvaluator getMatchingPolicyItemForAccessPolicyForSpecificAccess(RangerAccessRequest request, RangerAccessResult result) { + RangerPolicyItemEvaluator ret = getMatchingPolicyItem(request, result, denyEvaluators, denyExceptionEvaluators); + + if(request.isAccessorsRequested() || (ret == null && !result.getIsAccessDetermined())) { // a deny policy could have set isAllowed=true, but in such case it wouldn't set isAccessDetermined=true + ret = getMatchingPolicyItem(request, result, allowEvaluators, allowExceptionEvaluators); + } + + return ret; + } + + protected T getMatchingPolicyItem(RangerAccessRequest request, RangerAccessResult result, + List evaluators) { + T ret = getMatchingPolicyItem(request, result, evaluators, null); + + return ret; + } + + private T getMatchingPolicyItem(RangerAccessRequest request, RangerAccessResult result, + List evaluators, List exceptionEvaluators) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerDefaultPolicyEvaluator.getMatchingPolicyItem(" + request + ")"); + } + + T ret = null; + + if(CollectionUtils.isNotEmpty(evaluators)) { + for (T evaluator : evaluators) { + if(evaluator.isMatch(request)) { + ret = evaluator; + + if (!request.isAccessorsRequested()) { + break; + } + result.addMatchedItemEvaluator(evaluator); + } + } + } + + if(ret != null && CollectionUtils.isNotEmpty(exceptionEvaluators)) { + for (T exceptionEvaluator : exceptionEvaluators) { + if(exceptionEvaluator.isMatch(request)) { + if(LOG.isDebugEnabled()) { + LOG.debug("RangerDefaultPolicyEvaluator.getMatchingPolicyItem(" + request + "): found exception policyItem(" + exceptionEvaluator.getPolicyItem() + "); ignoring the matchedPolicyItem(" + ret.getPolicyItem() + ")"); + } + + ret = null; + + break; + } + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerDefaultPolicyEvaluator.getMatchingPolicyItem(" + request + "): " + ret); + } + + return ret; + } + + private T getMatchingPolicyItem(String user, Set userGroups, Set roles, String owner, String accessType, List evaluators, List exceptionEvaluators) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerDefaultPolicyEvaluator.getMatchingPolicyItem(" + user + ", " + userGroups + ", " + roles + ", " + owner + ", " + accessType + ")"); + } + + T ret = null; + + if(CollectionUtils.isNotEmpty(evaluators)) { + for (T evaluator : evaluators) { + if(evaluator.matchUserGroupAndOwner(user, userGroups, roles, owner) && evaluator.matchAccessType(accessType)) { + ret = evaluator; + + break; + } + } + } + + if(ret != null && CollectionUtils.isNotEmpty(exceptionEvaluators)) { + for (T exceptionEvaluator : exceptionEvaluators) { + if(exceptionEvaluator.matchUserGroupAndOwner(user, userGroups, roles, owner) && exceptionEvaluator.matchAccessType(accessType)) { + if(LOG.isDebugEnabled()) { + LOG.debug("RangerDefaultPolicyEvaluator.getMatchingPolicyItem(" + user + ", " + userGroups + ", " + accessType + "): found exception policyItem(" + exceptionEvaluator.getPolicyItem() + "); ignoring the matchedPolicyItem(" + ret.getPolicyItem() + ")"); + } + + ret = null; + + break; + } + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerDefaultPolicyEvaluator.getMatchingPolicyItem(" + user + ", " + userGroups + ", " + roles + ", " + owner + ", " + accessType + "): " + ret); + } + return ret; + } + + // Policy Level Condition evaluator + private boolean matchPolicyCustomConditions(RangerAccessRequest request) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerDefaultPolicyEvaluator.matchPolicyCustomConditions(" + request + ")"); + } + + boolean ret = true; + + if (CollectionUtils.isNotEmpty(conditionEvaluators)) { + if(LOG.isDebugEnabled()) { + LOG.debug("RangerDefaultPolicyEvaluator.matchPolicyCustomConditions(): conditionCount=" + conditionEvaluators.size()); + } + for(RangerConditionEvaluator conditionEvaluator : conditionEvaluators) { + if(LOG.isDebugEnabled()) { + LOG.debug("evaluating condition: " + conditionEvaluator); + } + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_POLICYCONDITION_REQUEST_LOG)) { + + String conditionType = null; + if (conditionEvaluator instanceof RangerAbstractConditionEvaluator) { + conditionType = ((RangerAbstractConditionEvaluator)conditionEvaluator).getPolicyItemCondition().getType(); + } + + perf = RangerPerfTracer.getPerfTracer(PERF_POLICYCONDITION_REQUEST_LOG, "RangerConditionEvaluator.matchPolicyCustomConditions(policyId=" + getId() + ",policyConditionType=" + conditionType + ")"); + } + + boolean conditionEvalResult = conditionEvaluator.isMatched(request); + + RangerPerfTracer.log(perf); + + if (!conditionEvalResult) { + if(LOG.isDebugEnabled()) { + LOG.debug(conditionEvaluator + " returned false"); + } + ret = false; + break; + } + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerDefaultPolicyEvaluator.matchCustomConditions(" + request + "): " + ret); + } + + return ret; + } + + private List createRangerPolicyConditionEvaluator(RangerPolicy policy, + RangerServiceDef serviceDef, + RangerPolicyEngineOptions options) { + List rangerConditionEvaluators = null; + + RangerCustomConditionEvaluator rangerConditionEvaluator = new RangerCustomConditionEvaluator(); + + rangerConditionEvaluators = rangerConditionEvaluator.getRangerPolicyConditionEvaluator(policy,serviceDef,options); + + if (rangerConditionEvaluators != null) { + customConditionsCount += rangerConditionEvaluators.size(); + } + + return rangerConditionEvaluators; + } + +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyevaluator/RangerDefaultPolicyItemEvaluator.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyevaluator/RangerDefaultPolicyItemEvaluator.java new file mode 100644 index 00000000000..d2e14ad2764 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyevaluator/RangerDefaultPolicyItemEvaluator.java @@ -0,0 +1,335 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.atlas.plugin.policyevaluator; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.plugin.conditionevaluator.RangerAbstractConditionEvaluator; +import org.apache.atlas.plugin.conditionevaluator.RangerConditionEvaluator; +import org.apache.atlas.plugin.model.RangerPolicy; +import org.apache.atlas.plugin.model.RangerPolicy.RangerPolicyItem; +import org.apache.atlas.plugin.model.RangerPolicy.RangerPolicyItemAccess; +import org.apache.atlas.plugin.model.RangerServiceDef; +import org.apache.atlas.plugin.model.RangerServiceDef.RangerPolicyConditionDef; +import org.apache.atlas.plugin.policyengine.RangerAccessRequest; +import org.apache.atlas.plugin.policyengine.RangerAccessResource; +import org.apache.atlas.plugin.policyengine.RangerAccessResult; +import org.apache.atlas.plugin.policyengine.RangerPolicyEngine; +import org.apache.atlas.plugin.policyengine.RangerPolicyEngineOptions; +import org.apache.atlas.plugin.policyresourcematcher.RangerPolicyResourceMatcher; +import org.apache.atlas.plugin.util.RangerAccessRequestUtil; +import org.apache.atlas.plugin.util.RangerPerfTracer; + +import java.util.Collections; +import java.util.List; +import java.util.Set; + + +public class RangerDefaultPolicyItemEvaluator extends RangerAbstractPolicyItemEvaluator { + private static final Log LOG = LogFactory.getLog(RangerDefaultPolicyItemEvaluator.class); + + private static final Log PERF_POLICYITEM_REQUEST_LOG = RangerPerfTracer.getPerfLogger("policyitem.request"); + private static final Log PERF_POLICYCONDITION_REQUEST_LOG = RangerPerfTracer.getPerfLogger("policycondition.request"); + + private boolean hasCurrentUser; + private boolean hasResourceOwner; + + public RangerDefaultPolicyItemEvaluator(RangerServiceDef serviceDef, RangerPolicy policy, RangerPolicyItem policyItem, int policyItemType, int policyItemIndex, RangerPolicyEngineOptions options) { + super(serviceDef, policy, policyItem, policyItemType, policyItemIndex, options); + } + + public void init() { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerDefaultPolicyItemEvaluator(policyId=" + policyId + ", policyItem=" + policyItem + ", serviceType=" + getServiceType() + ", conditionsDisabled=" + getConditionsDisabledOption() + ")"); + } + + RangerCustomConditionEvaluator rangerCustomConditionEvaluator = new RangerCustomConditionEvaluator(); + + conditionEvaluators = rangerCustomConditionEvaluator.getPolicyItemConditionEvaluator(policy,policyItem,serviceDef,options,policyItemIndex); + + List users = policyItem.getUsers(); + this.hasCurrentUser = CollectionUtils.isNotEmpty(users) && users.contains(RangerPolicyEngine.USER_CURRENT); + this.hasResourceOwner = CollectionUtils.isNotEmpty(users) && users.contains(RangerPolicyEngine.RESOURCE_OWNER); + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerDefaultPolicyItemEvaluator(policyId=" + policyId + ", conditionsCount=" + getConditionEvaluators().size() + ")"); + } + } + + @Override + public boolean isMatch(RangerAccessRequest request) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerDefaultPolicyItemEvaluator.isMatch(" + request + ")"); + } + + boolean ret = false; + + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_POLICYITEM_REQUEST_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_POLICYITEM_REQUEST_LOG, "RangerPolicyItemEvaluator.isMatch(resource=" + request.getResource().getAsString() + ")"); + } + + if(policyItem != null) { + if(matchUserGroupAndOwner(request)) { + if (request.isAccessTypeDelegatedAdmin()) { // used only in grant/revoke scenario + if (policyItem.getDelegateAdmin()) { + ret = true; + } + } else if (CollectionUtils.isNotEmpty(policyItem.getAccesses())) { + boolean isAccessTypeMatched = false; + + for (RangerPolicy.RangerPolicyItemAccess access : policyItem.getAccesses()) { + if (access.getIsAllowed() && StringUtils.equalsIgnoreCase(access.getType(), request.getAccessType())) { + isAccessTypeMatched = true; + break; + } + } + + if(isAccessTypeMatched) { + if(matchCustomConditions(request)) { + ret = true; + } + } + } + } + } + + RangerPerfTracer.log(perf); + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerDefaultPolicyItemEvaluator.isMatch(" + request + "): " + ret); + } + + return ret; + } + + @Override + public boolean matchUserGroupAndOwner(String user, Set userGroups, Set roles, String owner) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerDefaultPolicyItemEvaluator.matchUserGroup(" + policyItem + ", " + user + ", " + userGroups + ", " + roles + ", " + owner + ")"); + } + + boolean ret = false; + + if(policyItem != null) { + if(!ret && user != null && policyItem.getUsers() != null) { + ret = hasCurrentUser || policyItem.getUsers().contains(user); + } + if(!ret && userGroups != null && policyItem.getGroups() != null) { + ret = policyItem.getGroups().contains(RangerPolicyEngine.GROUP_PUBLIC) || + !Collections.disjoint(policyItem.getGroups(), userGroups); + } + if (!ret && CollectionUtils.isNotEmpty(roles) && CollectionUtils.isNotEmpty(policyItem.getRoles())) { + ret = !Collections.disjoint(policyItem.getRoles(), roles); + } + if (!ret && hasResourceOwner) { + ret = user != null && user.equals(owner); + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerDefaultPolicyItemEvaluator.matchUserGroup(" + policyItem + ", " + user + ", " + userGroups + ", " + roles + ", " + owner + "): " + ret); + } + + return ret; + } + + private boolean matchUserGroupAndOwner(RangerAccessRequest request) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerDefaultPolicyItemEvaluator.matchUserGroupAndOwner(" + request + ")"); + } + + if (request.isAccessorsRequested()) { + //skip checking user/group/owner if policy item in case get accessors request + return true; + } + + boolean ret = false; + + String user = request.getUser(); + Set userGroups = request.getUserGroups(); + + RangerAccessResource accessedResource = request.getResource(); + String resourceOwner = accessedResource != null ? accessedResource.getOwnerUser() : null; + + if (!ret) { + Set roles = null; + if (CollectionUtils.isNotEmpty(policyItem.getRoles())) { + roles = RangerAccessRequestUtil.getCurrentUserRolesFromContext(request.getContext()); + } + ret = matchUserGroupAndOwner(user, userGroups, roles, resourceOwner); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerDefaultPolicyItemEvaluator.matchUserGroupAndOwner(" + request + "): " + ret); + } + + return ret; + } + @Override + public boolean matchAccessType(String accessType) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerDefaultPolicyItemEvaluator.matchAccessType(" + accessType + ")"); + } + + boolean ret = false; + + if(policyItem != null) { + boolean isAdminAccess = StringUtils.equals(accessType, RangerPolicyEngine.ADMIN_ACCESS); + + if(isAdminAccess) { + ret = policyItem.getDelegateAdmin(); + } else { + if(CollectionUtils.isNotEmpty(policyItem.getAccesses())) { + boolean isAnyAccess = StringUtils.equals(accessType, RangerPolicyEngine.ANY_ACCESS); + + for(RangerPolicyItemAccess itemAccess : policyItem.getAccesses()) { + if(! itemAccess.getIsAllowed()) { + continue; + } + + if(isAnyAccess) { + ret = true; + + break; + } else if(StringUtils.equalsIgnoreCase(itemAccess.getType(), accessType)) { + ret = true; + + break; + } + } + } else if (StringUtils.isEmpty(accessType)) { + ret = true; + } + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerDefaultPolicyItemEvaluator.matchAccessType(" + accessType + "): " + ret); + } + + return ret; + } + + @Override + public boolean matchCustomConditions(RangerAccessRequest request) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerDefaultPolicyItemEvaluator.matchCustomConditions(" + request + ")"); + } + + boolean ret = true; + + if (CollectionUtils.isNotEmpty(conditionEvaluators)) { + if(LOG.isDebugEnabled()) { + LOG.debug("RangerDefaultPolicyItemEvaluator.matchCustomConditions(): conditionCount=" + conditionEvaluators.size()); + } + for(RangerConditionEvaluator conditionEvaluator : conditionEvaluators) { + if(LOG.isDebugEnabled()) { + LOG.debug("evaluating condition: " + conditionEvaluator); + } + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_POLICYCONDITION_REQUEST_LOG)) { + + String conditionType = null; + if (conditionEvaluator instanceof RangerAbstractConditionEvaluator) { + conditionType = ((RangerAbstractConditionEvaluator)conditionEvaluator).getPolicyItemCondition().getType(); + } + + perf = RangerPerfTracer.getPerfTracer(PERF_POLICYCONDITION_REQUEST_LOG, "RangerConditionEvaluator.matchCondition(policyId=" + policyId + ",policyItemIndex=" + getPolicyItemIndex() + ",policyConditionType=" + conditionType + ")"); + } + + boolean conditionEvalResult = conditionEvaluator.isMatched(request); + + RangerPerfTracer.log(perf); + + if (!conditionEvalResult) { + if(LOG.isDebugEnabled()) { + LOG.debug(conditionEvaluator + " returned false"); + } + + ret = false; + + break; + } + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerDefaultPolicyItemEvaluator.matchCustomConditions(" + request + "): " + ret); + } + + return ret; + } + + @Override + public void updateAccessResult(RangerPolicyEvaluator policyEvaluator, RangerAccessResult result, RangerPolicyResourceMatcher.MatchType matchType) { + policyEvaluator.updateAccessResult(result, matchType, getPolicyItemType() != POLICY_ITEM_TYPE_DENY, getComments()); + } + + RangerPolicyConditionDef getConditionDef(String conditionName) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerDefaultPolicyItemEvaluator.getConditionDef(" + conditionName + ")"); + } + + RangerPolicyConditionDef ret = null; + + if (serviceDef != null && CollectionUtils.isNotEmpty(serviceDef.getPolicyConditions())) { + for(RangerPolicyConditionDef conditionDef : serviceDef.getPolicyConditions()) { + if(StringUtils.equals(conditionName, conditionDef.getName())) { + ret = conditionDef; + + break; + } + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerDefaultPolicyItemEvaluator.getConditionDef(" + conditionName + "): " + ret); + } + + return ret; + } + + RangerConditionEvaluator newConditionEvaluator(String className) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerDefaultPolicyItemEvaluator.newConditionEvaluator(" + className + ")"); + } + + RangerConditionEvaluator evaluator = null; + + try { + @SuppressWarnings("unchecked") + Class matcherClass = (Class)Class.forName(className); + + evaluator = matcherClass.newInstance(); + } catch(Throwable t) { + LOG.error("RangerDefaultPolicyItemEvaluator.newConditionEvaluator(" + className + "): error instantiating evaluator", t); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerDefaultPolicyItemEvaluator.newConditionEvaluator(" + className + "): " + evaluator); + } + + return evaluator; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyevaluator/RangerDefaultRowFilterPolicyItemEvaluator.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyevaluator/RangerDefaultRowFilterPolicyItemEvaluator.java new file mode 100644 index 00000000000..72c26df9d4b --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyevaluator/RangerDefaultRowFilterPolicyItemEvaluator.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.atlas.plugin.policyevaluator; + +import org.apache.atlas.plugin.model.RangerPolicy; +import org.apache.atlas.plugin.model.RangerPolicy.RangerPolicyItemRowFilterInfo; +import org.apache.atlas.plugin.model.RangerPolicy.RangerRowFilterPolicyItem; +import org.apache.atlas.plugin.model.RangerServiceDef; +import org.apache.atlas.plugin.policyengine.RangerAccessResult; +import org.apache.atlas.plugin.policyengine.RangerPolicyEngineOptions; +import org.apache.atlas.plugin.policyresourcematcher.RangerPolicyResourceMatcher; + + +public class RangerDefaultRowFilterPolicyItemEvaluator extends RangerDefaultPolicyItemEvaluator implements RangerRowFilterPolicyItemEvaluator { + final private RangerRowFilterPolicyItem rowFilterPolicyItem; + + public RangerDefaultRowFilterPolicyItemEvaluator(RangerServiceDef serviceDef, RangerPolicy policy, RangerRowFilterPolicyItem policyItem, int policyItemIndex, RangerPolicyEngineOptions options) { + super(serviceDef, policy, policyItem, POLICY_ITEM_TYPE_DATAMASK, policyItemIndex, options); + + rowFilterPolicyItem = policyItem; + } + + @Override + public RangerPolicyItemRowFilterInfo getRowFilterInfo() { + return rowFilterPolicyItem == null ? null : rowFilterPolicyItem.getRowFilterInfo(); + } + + @Override + public void updateAccessResult(RangerPolicyEvaluator policyEvaluator, RangerAccessResult result, RangerPolicyResourceMatcher.MatchType matchType) { + RangerPolicyItemRowFilterInfo rowFilterInfo = getRowFilterInfo(); + + if (result.getFilterExpr() == null && rowFilterInfo != null) { + result.setFilterExpr(rowFilterInfo.getFilterExpr()); + policyEvaluator.updateAccessResult(result, matchType, true, getComments()); + } + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyevaluator/RangerOptimizedPolicyEvaluator.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyevaluator/RangerOptimizedPolicyEvaluator.java new file mode 100644 index 00000000000..3b0f4c686de --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyevaluator/RangerOptimizedPolicyEvaluator.java @@ -0,0 +1,368 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.policyevaluator; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.plugin.model.RangerPolicy; +import org.apache.atlas.plugin.model.RangerServiceDef; +import org.apache.atlas.plugin.policyengine.RangerAccessRequest; +import org.apache.atlas.plugin.policyengine.RangerAccessResource; +import org.apache.atlas.plugin.policyengine.RangerPolicyEngine; +import org.apache.atlas.plugin.policyengine.RangerPolicyEngineOptions; +import org.apache.atlas.plugin.util.RangerAccessRequestUtil; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +public class RangerOptimizedPolicyEvaluator extends RangerDefaultPolicyEvaluator { + private static final Log LOG = LogFactory.getLog(RangerOptimizedPolicyEvaluator.class); + + private Set roles = new HashSet<>(); + private Set groups = new HashSet<>(); + private Set users = new HashSet<>(); + private Set accessPerms = new HashSet<>(); + private boolean delegateAdmin; + private boolean hasAllPerms; + private boolean hasPublicGroup; + private boolean hasCurrentUser; + private boolean hasResourceOwner; + + // For computation of priority + private static final String RANGER_POLICY_EVAL_MATCH_ANY_PATTERN_STRING = "*"; + private static final String RANGER_POLICY_EVAL_MATCH_ONE_CHARACTER_STRING = "?"; + + private static final int RANGER_POLICY_EVAL_SCORE_DEFAULT = 10000; + + private static final int RANGER_POLICY_EVAL_SCORE_MAX_DISCOUNT_RESOURCE = 100; + private static final int RANGER_POLICY_EVAL_SCORE_MAX_DISCOUNT_USERSGROUPS = 25; + private static final int RANGER_POLICY_EVAL_SCORE_MAX_DISCOUNT_ACCESS_TYPES = 25; + private static final int RANGER_POLICY_EVAL_SCORE_MAX_DISCOUNT_CUSTOM_CONDITIONS = 25; + + private static final int RANGER_POLICY_EVAL_SCORE_RESOURCE_DISCOUNT_MATCH_ANY_WILDCARD = 25; + private static final int RANGER_POLICY_EVAL_SCORE_RESOURCE_DISCOUNT_HAS_MATCH_ANY_WILDCARD = 10; + private static final int RANGER_POLICY_EVAL_SCORE_RESOURCE_DISCOUNT_HAS_MATCH_ONE_CHARACTER_WILDCARD = 5; + private static final int RANGER_POLICY_EVAL_SCORE_RESOURCE_DISCOUNT_IS_EXCLUDES = 5; + private static final int RANGER_POLICY_EVAL_SCORE_RESORUCE_DISCOUNT_IS_RECURSIVE = 5; + private static final int RANGER_POLICY_EVAL_SCORE_CUSTOM_CONDITION_PENALTY = 5; + private static final int RANGER_POLICY_EVAL_SCORE_DYNAMIC_RESOURCE_EVAL_PENALTY = 20; + + @Override + public void init(RangerPolicy policy, RangerServiceDef serviceDef, RangerPolicyEngineOptions options) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerOptimizedPolicyEvaluator.init()"); + } + + super.init(policy, serviceDef, options); + + preprocessPolicyItems(policy.getPolicyItems()); + preprocessPolicyItems(policy.getDenyPolicyItems()); + preprocessPolicyItems(policy.getAllowExceptions()); + preprocessPolicyItems(policy.getDenyExceptions()); + preprocessPolicyItems(policy.getDataMaskPolicyItems()); + preprocessPolicyItems(policy.getRowFilterPolicyItems()); + + hasAllPerms = checkIfHasAllPerms(); + + for (String user : users) { + if (!hasCurrentUser && RangerPolicyEngine.USER_CURRENT.equalsIgnoreCase(user)) { + hasCurrentUser = true; + } + if (!hasResourceOwner && RangerPolicyEngine.RESOURCE_OWNER.equalsIgnoreCase(user)) { + hasResourceOwner = true; + } + if (hasCurrentUser && hasResourceOwner) { + break; + } + } + + if (policy.getIsDenyAllElse()) { + hasPublicGroup = true; + } else { + for (String group : groups) { + if (RangerPolicyEngine.GROUP_PUBLIC.equalsIgnoreCase(group)) { + hasPublicGroup = true; + break; + } + } + } + + setEvalOrder(computeEvalOrder()); + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerOptimizedPolicyEvaluator.init()"); + } + } + + static class LevelResourceNames implements Comparable { + final int level; + final RangerPolicy.RangerPolicyResource policyResource; + + public LevelResourceNames(int level, RangerPolicy.RangerPolicyResource policyResource) { + this.level = level; + this.policyResource = policyResource; + } + + @Override + public int compareTo(LevelResourceNames other) { + // Sort in ascending order of level numbers + return Integer.compare(this.level, other.level); + } + + @Override + public boolean equals(Object other) { + boolean ret = false; + if (other != null && (other instanceof LevelResourceNames)) { + ret = this == other || compareTo((LevelResourceNames) other) == 0; + } + return ret; + } + + @Override + public int hashCode() { + return Objects.hashCode(this.level); + } + } + + public int computeEvalOrder() { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerOptimizedPolicyEvaluator.computeEvalOrder()"); + } + + int evalOrder = RANGER_POLICY_EVAL_SCORE_DEFAULT; + + RangerServiceDef serviceDef = getServiceDef(); + List resourceDefs = serviceDef.getResources(); + RangerPolicy policy = getPolicy(); + List tmpList = new ArrayList<>(); + + for (Map.Entry kv : policy.getResources().entrySet()) { + String resourceName = kv.getKey(); + RangerPolicy.RangerPolicyResource policyResource = kv.getValue(); + List resourceValues = policyResource.getValues(); + + if(CollectionUtils.isNotEmpty(resourceValues)) { + for (RangerServiceDef.RangerResourceDef resourceDef : resourceDefs) { + if (resourceName.equals(resourceDef.getName())) { + tmpList.add(new LevelResourceNames(resourceDef.getLevel(), policyResource)); + break; + } + } + } + } + Collections.sort(tmpList); // Sort in ascending order of levels + + int resourceDiscount = 0; + for (LevelResourceNames item : tmpList) { + // Expect lowest level first + boolean foundStarWildcard = false; + boolean foundQuestionWildcard = false; + boolean foundMatchAny = false; + + for (String resourceName : item.policyResource.getValues()) { + if (resourceName.isEmpty() || RANGER_POLICY_EVAL_MATCH_ANY_PATTERN_STRING.equals(resourceName)) { + foundMatchAny = true; + break; + } else if (resourceName.contains(RANGER_POLICY_EVAL_MATCH_ANY_PATTERN_STRING)) { + foundStarWildcard = true; + } else if (resourceName.contains(RANGER_POLICY_EVAL_MATCH_ONE_CHARACTER_STRING)) { + foundQuestionWildcard = true; + } + } + if (foundMatchAny) { + resourceDiscount += RANGER_POLICY_EVAL_SCORE_RESOURCE_DISCOUNT_MATCH_ANY_WILDCARD; + } else { + if (foundStarWildcard) { + resourceDiscount += RANGER_POLICY_EVAL_SCORE_RESOURCE_DISCOUNT_HAS_MATCH_ANY_WILDCARD; + } else if (foundQuestionWildcard) { + resourceDiscount += RANGER_POLICY_EVAL_SCORE_RESOURCE_DISCOUNT_HAS_MATCH_ONE_CHARACTER_WILDCARD; + } + + RangerPolicy.RangerPolicyResource resource = item.policyResource; + + if (resource.getIsExcludes()) { + resourceDiscount += RANGER_POLICY_EVAL_SCORE_RESOURCE_DISCOUNT_IS_EXCLUDES; + } + + if (resource.getIsRecursive()) { + resourceDiscount += RANGER_POLICY_EVAL_SCORE_RESORUCE_DISCOUNT_IS_RECURSIVE; + } + } + } + if (needsDynamicEval()) { + evalOrder += RANGER_POLICY_EVAL_SCORE_DYNAMIC_RESOURCE_EVAL_PENALTY; + } + + evalOrder -= Math.min(RANGER_POLICY_EVAL_SCORE_MAX_DISCOUNT_RESOURCE, resourceDiscount); + + if (hasPublicGroup || hasCurrentUser) { + evalOrder -= RANGER_POLICY_EVAL_SCORE_MAX_DISCOUNT_USERSGROUPS; + } else { + evalOrder -= Math.min(groups.size() + users.size(), RANGER_POLICY_EVAL_SCORE_MAX_DISCOUNT_USERSGROUPS); + } + + evalOrder -= Math.round(((float)RANGER_POLICY_EVAL_SCORE_MAX_DISCOUNT_ACCESS_TYPES * accessPerms.size()) / serviceDef.getAccessTypes().size()); + + int customConditionsDiscount = RANGER_POLICY_EVAL_SCORE_MAX_DISCOUNT_CUSTOM_CONDITIONS - (RANGER_POLICY_EVAL_SCORE_CUSTOM_CONDITION_PENALTY * this.getCustomConditionsCount()); + if(customConditionsDiscount > 0) { + evalOrder -= customConditionsDiscount; + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerOptimizedPolicyEvaluator.computeEvalOrder(), policyName:" + policy.getName() + ", priority:" + evalOrder); + } + + return evalOrder; + } + + @Override + protected boolean isAccessAllowed(String user, Set userGroups, Set roles, String owner, String accessType) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerOptimizedPolicyEvaluator.isAccessAllowed(" + user + ", " + userGroups + ", " + roles + ", " + owner + ", " + accessType + ")"); + } + + boolean ret = hasMatchablePolicyItem(user, userGroups, roles, owner, accessType) && super.isAccessAllowed(user, userGroups, roles, owner, accessType); + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerOptimizedPolicyEvaluator.isAccessAllowed(" + user + ", " + userGroups + ", " + roles + ", " + owner + ", " + accessType + "): " + ret); + } + + return ret; + } + + @Override + protected boolean hasMatchablePolicyItem(RangerAccessRequest request) { + boolean ret = false; + + if (request.isAccessorsRequested() || hasPublicGroup || hasCurrentUser || isOwnerMatch(request) || users.contains(request.getUser()) || CollectionUtils.containsAny(groups, request.getUserGroups()) || (CollectionUtils.isNotEmpty(roles) && CollectionUtils.containsAny(roles, RangerAccessRequestUtil.getCurrentUserRolesFromContext(request.getContext())))) { + if (hasAllPerms || request.isAccessTypeAny()) { + ret = true; + } else { + ret = accessPerms.contains(request.getAccessType()); + if (!ret) { + if (request.isAccessTypeDelegatedAdmin()) { + ret = delegateAdmin; + } + } + } + } + + return ret; + } + + private boolean isOwnerMatch(RangerAccessRequest request) { + boolean ret = false; + + if (hasResourceOwner) { + RangerAccessResource accessedResource = request.getResource(); + String resourceOwner = accessedResource != null ? accessedResource.getOwnerUser() : null; + String user = request.getUser(); + + if (user != null && resourceOwner != null && user.equals(resourceOwner)) { + ret = true; + } + } + + return ret; + } + + private boolean hasMatchablePolicyItem(String user, Set userGroups, Set rolesFromContext, String owner, String accessType) { + boolean ret = false; + + boolean hasRole = false; + if (CollectionUtils.isNotEmpty(roles)) { + if (CollectionUtils.isNotEmpty(rolesFromContext)) { + hasRole = CollectionUtils.containsAny(roles, rolesFromContext); + } + } + + if (hasPublicGroup || hasCurrentUser || users.contains(user) || CollectionUtils.containsAny(groups, userGroups) || hasRole || (hasResourceOwner && StringUtils.equals(user, owner))) { + if (hasAllPerms) { + ret = true; + } else { + boolean isAccessTypeAny = StringUtils.isEmpty(accessType) || StringUtils.equals(accessType, RangerPolicyEngine.ANY_ACCESS); + ret = isAccessTypeAny || accessPerms.contains(accessType); + + if (!ret) { + if (StringUtils.equals(accessType, RangerPolicyEngine.ADMIN_ACCESS)) { + ret = delegateAdmin; + } + } + } + } + + return ret; + } + + private void preprocessPolicyItems(List policyItems) { + if(CollectionUtils.isNotEmpty(policyItems)) { + for (RangerPolicy.RangerPolicyItem item : policyItems) { + delegateAdmin = delegateAdmin || item.getDelegateAdmin(); + + List policyItemAccesses = item.getAccesses(); + for(RangerPolicy.RangerPolicyItemAccess policyItemAccess : policyItemAccesses) { + + if (policyItemAccess.getIsAllowed()) { + String accessType = policyItemAccess.getType(); + accessPerms.add(accessType); + } + } + + roles.addAll(item.getRoles()); + groups.addAll(item.getGroups()); + users.addAll(item.getUsers()); + + } + } + } + + private boolean checkIfHasAllPerms() { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerOptimizedPolicyEvaluator.checkIfHasAllPerms()"); + } + boolean result = true; + + if (getPolicy().getIsDenyAllElse()) { + hasAllPerms = true; + } else { + List serviceAccessTypes = getServiceDef().getAccessTypes(); + for (RangerServiceDef.RangerAccessTypeDef serviceAccessType : serviceAccessTypes) { + if (!accessPerms.contains(serviceAccessType.getName())) { + result = false; + break; + } + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerOptimizedPolicyEvaluator.checkIfHasAllPerms(), " + result); + } + + return result; + } + +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyevaluator/RangerPolicyEvaluator.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyevaluator/RangerPolicyEvaluator.java new file mode 100644 index 00000000000..7c738fa40a3 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyevaluator/RangerPolicyEvaluator.java @@ -0,0 +1,618 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.policyevaluator; + + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.atlas.plugin.model.RangerPolicy; +import org.apache.atlas.plugin.model.RangerPolicy.RangerDataMaskPolicyItem; +import org.apache.atlas.plugin.model.RangerPolicy.RangerPolicyItem; +import org.apache.atlas.plugin.model.RangerPolicy.RangerPolicyItemAccess; +import org.apache.atlas.plugin.model.RangerPolicy.RangerPolicyResource; +import org.apache.atlas.plugin.model.RangerPolicy.RangerRowFilterPolicyItem; +import org.apache.atlas.plugin.model.RangerServiceDef; +import org.apache.atlas.plugin.policyengine.RangerAccessRequest; +import org.apache.atlas.plugin.policyengine.RangerAccessResource; +import org.apache.atlas.plugin.policyengine.RangerAccessResult; +import org.apache.atlas.plugin.policyengine.RangerPolicyEngine; +import org.apache.atlas.plugin.policyengine.RangerPolicyEngineOptions; +import org.apache.atlas.plugin.policyengine.RangerResourceACLs.DataMaskResult; +import org.apache.atlas.plugin.policyengine.RangerResourceACLs.RowFilterResult; +import org.apache.atlas.plugin.policyengine.RangerResourceAccessInfo; +import org.apache.atlas.plugin.policyresourcematcher.RangerPolicyResourceEvaluator; +import org.apache.atlas.plugin.policyresourcematcher.RangerPolicyResourceMatcher; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.apache.atlas.plugin.policyevaluator.RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_ALLOW; +import static org.apache.atlas.plugin.policyevaluator.RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_ALLOW_EXCEPTIONS; +import static org.apache.atlas.plugin.policyevaluator.RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_DENY; +import static org.apache.atlas.plugin.policyevaluator.RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_DENY_EXCEPTIONS; + +public interface RangerPolicyEvaluator extends RangerPolicyResourceEvaluator { + Comparator EVAL_ORDER_COMPARATOR = new PolicyEvalOrderComparator(); + Comparator NAME_COMPARATOR = new PolicyNameComparator(); + + // computation of PolicyACLSummary rely on following specific values + Integer ACCESS_DENIED = -1; + Integer ACCESS_UNDETERMINED = 0; + Integer ACCESS_ALLOWED = 1; + Integer ACCESS_CONDITIONAL = 2; + + String EVALUATOR_TYPE_AUTO = "auto"; + String EVALUATOR_TYPE_OPTIMIZED = "optimized"; + String EVALUATOR_TYPE_CACHED = "cached"; + + void init(RangerPolicy policy, RangerServiceDef serviceDef, RangerPolicyEngineOptions options); + + RangerPolicy getPolicy(); + + RangerServiceDef getServiceDef(); + + boolean hasAllow(); + + boolean hasDeny(); + + int getPolicyPriority(); + + boolean isApplicable(Date accessTime); + + int getEvalOrder(); + + int getCustomConditionsCount(); + + int getValidityScheduleEvaluatorsCount(); + + boolean isAuditEnabled(); + + void evaluate(RangerAccessRequest request, RangerAccessResult result); + + boolean isMatch(RangerAccessResource resource, Map evalContext); + + boolean isCompleteMatch(RangerAccessResource resource, Map evalContext); + + boolean isCompleteMatch(Map resources, Map evalContext); + + boolean isAccessAllowed(Map resources, String user, Set userGroups, String accessType); + + void updateAccessResult(RangerAccessResult result, RangerPolicyResourceMatcher.MatchType matchType, boolean isAllowed, String reason); + + void getResourceAccessInfo(RangerAccessRequest request, RangerResourceAccessInfo result); + + Set getAllowedAccesses(RangerAccessResource resource, String user, Set userGroups, Set roles, Set accessTypes); + + Set getAllowedAccesses(Map resources, String user, Set userGroups, Set roles, Set accessTypes, Map evalContext); + + PolicyACLSummary getPolicyACLSummary(); + + default boolean hasContextSensitiveSpecification() { + RangerPolicy policy = getPolicy(); + + for (RangerPolicyItem policyItem : policy.getPolicyItems()) { + if (hasContextSensitiveSpecification(policyItem)) { + return true; + } + } + for (RangerPolicyItem policyItem : policy.getDenyPolicyItems()) { + if (hasContextSensitiveSpecification(policyItem)) { + return true; + } + } + for (RangerPolicyItem policyItem : policy.getAllowExceptions()) { + if (hasContextSensitiveSpecification(policyItem)) { + return true; + } + } + for (RangerPolicyItem policyItem : policy.getDenyExceptions()) { + if (hasContextSensitiveSpecification(policyItem)) { + return true; + } + } + return false; + } + + default boolean hasRoles(final RangerPolicy policy) { + for (RangerPolicyItem policyItem : policy.getPolicyItems()) { + if (hasRoles(policyItem)) { + return true; + } + } + for (RangerPolicyItem policyItem : policy.getDenyPolicyItems()) { + if (hasRoles(policyItem)) { + return true; + } + } + for (RangerPolicyItem policyItem : policy.getAllowExceptions()) { + if (hasRoles(policyItem)) { + return true; + } + } + for (RangerPolicyItem policyItem : policy.getDenyExceptions()) { + if (hasRoles(policyItem)) { + return true; + } + } + for (RangerDataMaskPolicyItem policyItem : policy.getDataMaskPolicyItems()) { + if (hasRoles(policyItem)) { + return true; + } + } + for (RangerRowFilterPolicyItem policyItem : policy.getRowFilterPolicyItems()) { + if (hasRoles(policyItem)) { + return true; + } + } + return false; + } + + static boolean hasContextSensitiveSpecification(RangerPolicyItem policyItem) { + return CollectionUtils.isNotEmpty(policyItem.getConditions()) || policyItem.getUsers().contains(RangerPolicyEngine.RESOURCE_OWNER); /* || policyItem.getGroups().contains(RangerPolicyEngine.RESOURCE_GROUP_OWNER) */ + } + + static boolean hasRoles(RangerPolicyItem policyItem) { + return CollectionUtils.isNotEmpty(policyItem.getRoles()); + } + + class PolicyEvalOrderComparator implements Comparator, Serializable { + @Override + public int compare(RangerPolicyEvaluator me, RangerPolicyEvaluator other) { + int result = Integer.compare(other.getPolicyPriority(), me.getPolicyPriority()); + + return result == 0 ? compareNormal(me, other) : result; + } + + private int compareNormal(RangerPolicyEvaluator me, RangerPolicyEvaluator other) { + final int result; + + if (me.hasDeny() && !other.hasDeny()) { + result = -1; + } else if (!me.hasDeny() && other.hasDeny()) { + result = 1; + } else { + result = Integer.compare(me.getEvalOrder(), other.getEvalOrder()); + } + + return result; + } + } + + class PolicyNameComparator implements Comparator, Serializable { + @Override + public int compare(RangerPolicyEvaluator me, RangerPolicyEvaluator other) { + int result = Integer.compare(other.getPolicyPriority(), me.getPolicyPriority()); + + return result == 0 ? compareNormal(me, other) : result; + } + + private int compareNormal(RangerPolicyEvaluator me, RangerPolicyEvaluator other) { + final int result; + + if (me.hasDeny() && !other.hasDeny()) { + result = -1; + } else if (!me.hasDeny() && other.hasDeny()) { + result = 1; + } else { + result = me.getPolicy().getName().compareTo(other.getPolicy().getName()); + } + + return result; + } + } + + class PolicyACLSummary { + private final Map> usersAccessInfo = new HashMap<>(); + private final Map> groupsAccessInfo = new HashMap<>(); + private final Map> rolesAccessInfo = new HashMap<>(); + private final List rowFilters = new ArrayList<>(); + private final List dataMasks = new ArrayList<>(); + + private enum AccessorType { USER, GROUP, ROLE } + + public static class AccessResult { + private int result; + private final boolean hasSeenDeny; + + public AccessResult(int result) { + this(result, false); + } + + public AccessResult(int result, boolean hasSeenDeny) { + this.result = result; + this.hasSeenDeny = hasSeenDeny; + } + + public int getResult() { + return result; + } + + public void setResult(int result) { + this.result = result; + } + + public boolean getHasSeenDeny() { + return hasSeenDeny; + } + + @Override + public String toString() { + if (result == RangerPolicyEvaluator.ACCESS_ALLOWED) + return "ALLOWED, hasSeenDeny=" + hasSeenDeny; + if (result == RangerPolicyEvaluator.ACCESS_DENIED) + return "NOT_ALLOWED, hasSeenDeny=" + hasSeenDeny; + if (result == RangerPolicyEvaluator.ACCESS_CONDITIONAL) + return "CONDITIONAL_ALLOWED, hasSeenDeny=" + hasSeenDeny; + return "NOT_DETERMINED, hasSeenDeny=" + hasSeenDeny; + } + } + + PolicyACLSummary() { + } + + public Map> getUsersAccessInfo() { + return usersAccessInfo; + } + + public Map> getGroupsAccessInfo() { + return groupsAccessInfo; + } + + public Map> getRolesAccessInfo() { + return rolesAccessInfo; + } + + public List getRowFilters() { return rowFilters; } + + public List getDataMasks() { return dataMasks; } + + void processPolicyItem(RangerPolicyItem policyItem, int policyItemType, boolean isConditional) { + final Integer result; + final boolean hasContextSensitiveSpecification = CollectionUtils.isNotEmpty(policyItem.getConditions()); + + switch (policyItemType) { + case POLICY_ITEM_TYPE_ALLOW: + result = (hasContextSensitiveSpecification || isConditional) ? ACCESS_CONDITIONAL : ACCESS_ALLOWED; + break; + + case POLICY_ITEM_TYPE_ALLOW_EXCEPTIONS: + result = (hasContextSensitiveSpecification || isConditional) ? null : ACCESS_DENIED; + break; + + case POLICY_ITEM_TYPE_DENY: + result = (hasContextSensitiveSpecification || isConditional) ? ACCESS_CONDITIONAL : ACCESS_DENIED; + break; + + case POLICY_ITEM_TYPE_DENY_EXCEPTIONS: + result = (hasContextSensitiveSpecification || isConditional) ? null : ACCESS_ALLOWED; + break; + + default: + result = null; + break; + } + + if (result != null) { + final List accesses; + + if (policyItem.getDelegateAdmin()) { + accesses = new ArrayList<>(); + + accesses.add(new RangerPolicyItemAccess(RangerPolicyEngine.ADMIN_ACCESS, policyItem.getDelegateAdmin())); + accesses.addAll(policyItem.getAccesses()); + } else { + accesses = policyItem.getAccesses(); + } + + final List groups = policyItem.getGroups(); + final List users = policyItem.getUsers(); + final List roles = policyItem.getRoles(); + boolean hasPublicGroup = false; + + for (RangerPolicyItemAccess access : accesses) { + for (String user : users) { + if (StringUtils.equals(user, RangerPolicyEngine.USER_CURRENT)) { + hasPublicGroup = true; + continue; + } else if (StringUtils.isBlank(user)) { + continue; + } + + addAccess(user, AccessorType.USER, access.getType(), result, policyItemType); + } + + for (String group : groups) { + if (StringUtils.equals(group, RangerPolicyEngine.GROUP_PUBLIC)) { + hasPublicGroup = true; + continue; + } + + addAccess(group, AccessorType.GROUP, access.getType(), result, policyItemType); + } + + if (hasPublicGroup) { + addAccess(RangerPolicyEngine.GROUP_PUBLIC, AccessorType.GROUP, access.getType(), result, policyItemType); + } + + for (String role : roles) { + addAccess(role, AccessorType.ROLE, access.getType(), result, policyItemType); + } + } + } + } + + void processRowFilterPolicyItem(RangerRowFilterPolicyItem policyItem) { + Set users = new HashSet<>(policyItem.getUsers()); + Set groups = new HashSet<>(policyItem.getGroups()); + Set roles = new HashSet<>(policyItem.getRoles()); + Set accessTypes = new HashSet<>(); + + policyItem.getAccesses().forEach(accessType -> accessTypes.add(accessType.getType())); + + if (users.contains(RangerPolicyEngine.USER_CURRENT)) { // replace with public group + users.remove(RangerPolicyEngine.USER_CURRENT); + + groups.add(RangerPolicyEngine.GROUP_PUBLIC); + } + + RowFilterResult filterResult = new RowFilterResult(users, groups, roles, accessTypes, policyItem.getRowFilterInfo()); + + if (RangerPolicyEvaluator.hasContextSensitiveSpecification(policyItem)) { + filterResult.setIsConditional(true); + } + + rowFilters.add(filterResult); + } + + void processDataMaskPolicyItem(RangerDataMaskPolicyItem policyItem) { + Set users = new HashSet<>(policyItem.getUsers()); + Set groups = new HashSet<>(policyItem.getGroups()); + Set roles = new HashSet<>(policyItem.getRoles()); + Set accessTypes = new HashSet<>(); + + policyItem.getAccesses().forEach(accessType -> accessTypes.add(accessType.getType())); + + if (users.contains(RangerPolicyEngine.USER_CURRENT)) { // replace with public group + users.remove(RangerPolicyEngine.USER_CURRENT); + + groups.add(RangerPolicyEngine.GROUP_PUBLIC); + } + + DataMaskResult dataMaskResult = new DataMaskResult(users, groups, roles, accessTypes, policyItem.getDataMaskInfo()); + + if (RangerPolicyEvaluator.hasContextSensitiveSpecification(policyItem)) { + dataMaskResult.setIsConditional(true); + } + + dataMasks.add(dataMaskResult); + } + + void finalizeAcls(final boolean isDenyAllElse, final Set allAccessTypeNames) { + Map publicGroupAccessInfo = groupsAccessInfo.get(RangerPolicyEngine.GROUP_PUBLIC); + + if (publicGroupAccessInfo != null) { + + // For each accessType in public, retrieve access + for (Map.Entry entry : publicGroupAccessInfo.entrySet()) { + final String accessType = entry.getKey(); + final AccessResult accessResult = entry.getValue(); + final int access = accessResult.getResult(); + + if (access == ACCESS_DENIED || access == ACCESS_ALLOWED) { + List keysToRemove = null; + + for (Map.Entry> mapEntry : usersAccessInfo.entrySet()) { + Map mapValue = mapEntry.getValue(); + + mapValue.remove(accessType); + + if (mapValue.isEmpty()) { + if (keysToRemove == null) { + keysToRemove = new ArrayList<>(); + } + + keysToRemove.add(mapEntry.getKey()); + } + } + + if (keysToRemove != null) { + for (String keyToRemove : keysToRemove) { + usersAccessInfo.remove(keyToRemove); + } + + keysToRemove.clear(); + } + + for (Map.Entry> mapEntry : groupsAccessInfo.entrySet()) { + if (!StringUtils.equals(mapEntry.getKey(), RangerPolicyEngine.GROUP_PUBLIC)) { + Map mapValue = mapEntry.getValue(); + + mapValue.remove(accessType); + + if (mapValue.isEmpty()) { + if (keysToRemove == null) { + keysToRemove = new ArrayList<>(); + } + + keysToRemove.add(mapEntry.getKey()); + } + } + } + + + if (keysToRemove != null) { + for (String keyToRemove : keysToRemove) { + groupsAccessInfo.remove(keyToRemove); + } + + keysToRemove.clear(); + } + } + } + } + + if (isDenyAllElse) { + // Go through all usersAccessInfo and groupsAccessInfo and mark ACCESS_UNDETERMINED to ACCESS_DENIED + for (Map.Entry> mapEntry : usersAccessInfo.entrySet()) { + for (Map.Entry accessEntry : mapEntry.getValue().entrySet()) { + AccessResult result = accessEntry.getValue(); + if (result.getResult() == ACCESS_UNDETERMINED) { + result.setResult(ACCESS_DENIED); + } + } + } + + for (Map.Entry> mapEntry : groupsAccessInfo.entrySet()) { + for (Map.Entry accessEntry : mapEntry.getValue().entrySet()) { + AccessResult result = accessEntry.getValue(); + if (result.getResult() == ACCESS_UNDETERMINED) { + result.setResult(ACCESS_DENIED); + } + } + } + + // Mark all unseen accessTypeNames are having no permission + for (Map.Entry> mapEntry : usersAccessInfo.entrySet()) { + for (String accessTypeName : allAccessTypeNames) { + if (!mapEntry.getValue().keySet().contains(accessTypeName)) { + mapEntry.getValue().put(accessTypeName, new AccessResult(ACCESS_DENIED, true)); + } + } + } + + for (Map.Entry> mapEntry : groupsAccessInfo.entrySet()) { + for (String accessTypeName : allAccessTypeNames) { + if (!mapEntry.getValue().keySet().contains(accessTypeName)) { + mapEntry.getValue().put(accessTypeName, new AccessResult(ACCESS_DENIED, true)); + } + } + } + + publicGroupAccessInfo = groupsAccessInfo.computeIfAbsent(RangerPolicyEngine.GROUP_PUBLIC, k -> new HashMap<>()); + + Set accessTypeNamesInPublicGroup = publicGroupAccessInfo.keySet(); + + for (String accessTypeName : allAccessTypeNames) { + if (!accessTypeNamesInPublicGroup.contains(accessTypeName)) { + boolean isDenyAccess = true; + for (Map.Entry> mapEntry : usersAccessInfo.entrySet()) { + AccessResult result = mapEntry.getValue().get(accessTypeName); + if (result == null || result.getResult() != ACCESS_DENIED) { + isDenyAccess = false; + break; + } + } + if (isDenyAccess) { + for (Map.Entry> mapEntry : groupsAccessInfo.entrySet()) { + if (!StringUtils.equals(mapEntry.getKey(), RangerPolicyEngine.GROUP_PUBLIC)) { + AccessResult result = mapEntry.getValue().get(accessTypeName); + if (result == null || result.getResult() != ACCESS_DENIED) { + isDenyAccess = false; + break; + } + } + } + } + publicGroupAccessInfo.put(accessTypeName, new AccessResult(isDenyAccess ? ACCESS_DENIED : ACCESS_CONDITIONAL, true)); + } + } + } + } + + private void addAccess(String accessorName, AccessorType accessorType, String accessType, Integer access, int policyItemType) { + final Map> accessorsAccessInfo; + + switch (accessorType) { + case USER: + accessorsAccessInfo = usersAccessInfo; + break; + + case GROUP: + accessorsAccessInfo = groupsAccessInfo; + break; + + case ROLE: + accessorsAccessInfo = rolesAccessInfo; + break; + + default: + return; + } + + final Map accessorAccessInfo = accessorsAccessInfo.computeIfAbsent(accessorName, k -> new HashMap<>()); + final AccessResult currentAccess = accessorAccessInfo.get(accessType); + + if (currentAccess == null) { + if (policyItemType == POLICY_ITEM_TYPE_ALLOW || policyItemType == POLICY_ITEM_TYPE_DENY) { + accessorAccessInfo.put(accessType, new AccessResult(access, policyItemType == POLICY_ITEM_TYPE_DENY)); + } + } else { + if (access.equals(RangerPolicyEvaluator.ACCESS_DENIED)) { + if (currentAccess.getResult() == ACCESS_CONDITIONAL) { + currentAccess.setResult(access); + } else { + int updatedAccessValue = currentAccess.getResult() + access; + + if (policyItemType == POLICY_ITEM_TYPE_DENY) { + updatedAccessValue = (updatedAccessValue < ACCESS_DENIED) ? ACCESS_DENIED : updatedAccessValue; + } else { + updatedAccessValue = (updatedAccessValue < ACCESS_UNDETERMINED) ? ACCESS_UNDETERMINED : updatedAccessValue; + } + + currentAccess.setResult(updatedAccessValue); + } + } else if (access.equals(RangerPolicyEvaluator.ACCESS_ALLOWED)) { + if (currentAccess.getResult() == ACCESS_CONDITIONAL) { + if (policyItemType == POLICY_ITEM_TYPE_ALLOW) { + currentAccess.setResult(access); + } + } else { + int updatedAccessValue = currentAccess.getResult() + access; + boolean replaceValue = false; + + if (policyItemType == POLICY_ITEM_TYPE_ALLOW) { + updatedAccessValue = (updatedAccessValue > ACCESS_ALLOWED) ? ACCESS_ALLOWED : updatedAccessValue; + replaceValue = true; // Forget earlier stashed hasSeenDeny + } else { + updatedAccessValue = (updatedAccessValue > ACCESS_UNDETERMINED) ? ACCESS_UNDETERMINED : updatedAccessValue; + } + + if (replaceValue) { + accessorAccessInfo.put(accessType, new AccessResult(updatedAccessValue)); + } else { + currentAccess.setResult(updatedAccessValue); + } + } + } else { + if (currentAccess.getResult() == ACCESS_UNDETERMINED) { + currentAccess.setResult(access); + } + } + } + } + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyevaluator/RangerPolicyItemEvaluator.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyevaluator/RangerPolicyItemEvaluator.java new file mode 100644 index 00000000000..333b90ec51c --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyevaluator/RangerPolicyItemEvaluator.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.atlas.plugin.policyevaluator; + +import org.apache.atlas.plugin.conditionevaluator.RangerConditionEvaluator; +import org.apache.atlas.plugin.model.RangerPolicy.RangerPolicyItem; +import org.apache.atlas.plugin.policyengine.RangerAccessRequest; +import org.apache.atlas.plugin.policyengine.RangerAccessResult; +import org.apache.atlas.plugin.policyresourcematcher.RangerPolicyResourceMatcher; + +import java.io.Serializable; +import java.util.Comparator; +import java.util.List; +import java.util.Set; + +public interface RangerPolicyItemEvaluator { + int POLICY_ITEM_TYPE_ALLOW = 0; + int POLICY_ITEM_TYPE_DENY = 1; + int POLICY_ITEM_TYPE_ALLOW_EXCEPTIONS = 2; + int POLICY_ITEM_TYPE_DENY_EXCEPTIONS = 3; + int POLICY_ITEM_TYPE_DATAMASK = 4; + int POLICY_ITEM_TYPE_ROWFILTER = 5; + + void init(); + + RangerPolicyItem getPolicyItem(); + + int getPolicyItemType(); + + int getPolicyItemIndex(); + + String getComments(); + + List getConditionEvaluators(); + + int getEvalOrder(); + + boolean isMatch(RangerAccessRequest request); + + boolean matchUserGroupAndOwner(String user, Set userGroups, Set roles, String owner); + + boolean matchAccessType(String accessType); + + boolean matchCustomConditions(RangerAccessRequest request); + + class EvalOrderComparator implements Comparator, Serializable { + @Override + public int compare(RangerPolicyItemEvaluator me, RangerPolicyItemEvaluator other) { + return Integer.compare(me.getEvalOrder(), other.getEvalOrder()); + } + } + void updateAccessResult(RangerPolicyEvaluator policyEvaluator, RangerAccessResult result, RangerPolicyResourceMatcher.MatchType matchType); +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyevaluator/RangerRowFilterPolicyItemEvaluator.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyevaluator/RangerRowFilterPolicyItemEvaluator.java new file mode 100644 index 00000000000..0e19b01c548 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyevaluator/RangerRowFilterPolicyItemEvaluator.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.atlas.plugin.policyevaluator; + +import org.apache.atlas.plugin.model.RangerPolicy.RangerPolicyItemRowFilterInfo; + + +public interface RangerRowFilterPolicyItemEvaluator extends RangerPolicyItemEvaluator { + void init(); + + RangerPolicyItemRowFilterInfo getRowFilterInfo(); +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyevaluator/RangerValidityScheduleEvaluator.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyevaluator/RangerValidityScheduleEvaluator.java new file mode 100644 index 00000000000..7c5f26d5dbc --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyevaluator/RangerValidityScheduleEvaluator.java @@ -0,0 +1,588 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.policyevaluator; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.plugin.model.RangerValidityRecurrence; +import org.apache.atlas.plugin.model.RangerValiditySchedule; +import org.apache.atlas.plugin.resourcematcher.ScheduledTimeAlwaysMatcher; +import org.apache.atlas.plugin.resourcematcher.ScheduledTimeExactMatcher; +import org.apache.atlas.plugin.resourcematcher.ScheduledTimeMatcher; +import org.apache.atlas.plugin.resourcematcher.ScheduledTimeRangeMatcher; +import org.apache.atlas.plugin.util.RangerPerfTracer; + +import javax.annotation.Nonnull; +import java.text.DateFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Collections; +import java.util.Date; +import java.util.GregorianCalendar; +import java.util.List; +import java.util.TimeZone; + +public class RangerValidityScheduleEvaluator { + private static final Log LOG = LogFactory.getLog(RangerValidityScheduleEvaluator.class); + private static final Log PERF_LOG = LogFactory.getLog("test.perf.RangerValidityScheduleEvaluator"); + + private final static TimeZone defaultTZ = TimeZone.getDefault(); + + private static final ThreadLocal DATE_FORMATTER = new ThreadLocal() { + @Override + protected DateFormat initialValue() { + return new SimpleDateFormat(RangerValiditySchedule.VALIDITY_SCHEDULE_DATE_STRING_SPECIFICATION); + } + }; + + private final Date startTime; + private final Date endTime; + private final String timeZone; + private final List recurrenceEvaluators = new ArrayList<>(); + + public RangerValidityScheduleEvaluator(@Nonnull RangerValiditySchedule validitySchedule) { + this(validitySchedule.getStartTime(), validitySchedule.getEndTime(), validitySchedule.getTimeZone(), validitySchedule.getRecurrences()); + } + + public RangerValidityScheduleEvaluator(String startTimeStr, String endTimeStr, String timeZone, List recurrences) { + Date startTime = null; + Date endTime = null; + + if (StringUtils.isNotEmpty(startTimeStr)) { + try { + startTime = DATE_FORMATTER.get().parse(startTimeStr); + } catch (ParseException exception) { + LOG.error("Error parsing startTime:[" + startTimeStr + "]", exception); + } + } + + if (StringUtils.isNotEmpty(endTimeStr)) { + try { + endTime = DATE_FORMATTER.get().parse(endTimeStr); + } catch (ParseException exception) { + LOG.error("Error parsing endTime:[" + endTimeStr + "]", exception); + } + } + + this.startTime = startTime; + this.endTime = endTime; + this.timeZone = timeZone; + + if (CollectionUtils.isNotEmpty(recurrences)) { + for (RangerValidityRecurrence recurrence : recurrences) { + recurrenceEvaluators.add(new RangerRecurrenceEvaluator(recurrence)); + } + } + } + + public boolean isApplicable(long accessTime) { + if (LOG.isDebugEnabled()) { + LOG.debug("===> isApplicable(accessTime=" + accessTime + ")"); + } + + boolean ret = false; + RangerPerfTracer perf = null; + + if (RangerPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_LOG, "RangerValidityScheduleEvaluator.isApplicable(accessTime=" + accessTime + ")"); + } + + long startTimeInMSs = startTime == null ? 0 : startTime.getTime(); + long endTimeInMSs = endTime == null ? 0 : endTime.getTime(); + + if (StringUtils.isNotBlank(timeZone)) { + TimeZone targetTZ = TimeZone.getTimeZone(timeZone); + + if (startTimeInMSs > 0) { + startTimeInMSs = getAdjustedTime(startTimeInMSs, targetTZ); + } + + if (endTimeInMSs > 0) { + endTimeInMSs = getAdjustedTime(endTimeInMSs, targetTZ); + } + } + + if ((startTimeInMSs == 0 || accessTime >= startTimeInMSs) && (endTimeInMSs == 0 || accessTime <= endTimeInMSs)) { + if (CollectionUtils.isEmpty(recurrenceEvaluators)) { + ret = true; + } else { + Calendar now = new GregorianCalendar(); + now.setTime(new Date(accessTime)); + + for (RangerRecurrenceEvaluator recurrenceEvaluator : recurrenceEvaluators) { + ret = recurrenceEvaluator.isApplicable(now); + + if (ret) { + break; + } + } + } + } + + RangerPerfTracer.log(perf); + + if (LOG.isDebugEnabled()) { + LOG.debug("<=== isApplicable(accessTime=" + accessTime + ") :" + ret); + } + return ret; + } + + public static long getAdjustedTime(long localTime, TimeZone timeZone) { + long ret = localTime; + + if (LOG.isDebugEnabled()) { + LOG.debug("Input:[" + new Date(localTime) + ", target-timezone" + timeZone + "], default-timezone:[" + defaultTZ + "]"); + } + + if (!defaultTZ.equals(timeZone)) { + int targetOffset = timeZone.getOffset(localTime); + int defaultOffset = defaultTZ.getOffset(localTime); + int diff = defaultOffset - targetOffset; + + if (LOG.isDebugEnabled()) { + LOG.debug("Offset of target-timezone from UTC :[" + targetOffset + "]"); + LOG.debug("Offset of default-timezone from UTC :[" + defaultOffset + "]"); + LOG.debug("Difference between default-timezone and target-timezone :[" + diff + "]"); + } + + ret += diff; + } + + if (LOG.isDebugEnabled()) { + LOG.debug("Output:[" + new Date(ret) + "]"); + } + + return ret; + } + + static class RangerRecurrenceEvaluator { + private final List minutes = new ArrayList<>(); + private final List hours = new ArrayList<>(); + private final List daysOfMonth = new ArrayList<>(); + private final List daysOfWeek = new ArrayList<>(); + private final List months = new ArrayList<>(); + private final List years = new ArrayList<>(); + private final RangerValidityRecurrence recurrence; + private int intervalInMinutes = 0; + + + public RangerRecurrenceEvaluator(RangerValidityRecurrence recurrence) { + this.recurrence = recurrence; + + if (recurrence != null) { + intervalInMinutes = RangerValidityRecurrence.ValidityInterval.getValidityIntervalInMinutes(recurrence.getInterval()); + + if (intervalInMinutes > 0) { + addScheduledTime(RangerValidityRecurrence.RecurrenceSchedule.ScheduleFieldSpec.minute, minutes); + addScheduledTime(RangerValidityRecurrence.RecurrenceSchedule.ScheduleFieldSpec.hour, hours); + addScheduledTime(RangerValidityRecurrence.RecurrenceSchedule.ScheduleFieldSpec.dayOfMonth, daysOfMonth); + addScheduledTime(RangerValidityRecurrence.RecurrenceSchedule.ScheduleFieldSpec.dayOfWeek, daysOfWeek); + addScheduledTime(RangerValidityRecurrence.RecurrenceSchedule.ScheduleFieldSpec.month, months); + addScheduledTime(RangerValidityRecurrence.RecurrenceSchedule.ScheduleFieldSpec.year, years); + } + } + } + + public boolean isApplicable(Calendar now) { + boolean ret = false; + + RangerPerfTracer perf = null; + + if (RangerPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_LOG, "RangerRecurrenceEvaluator.isApplicable(accessTime=" + now.getTime().getTime() + ")"); + } + + if (recurrence != null && intervalInMinutes > 0) { // recurring schedule + + if (LOG.isDebugEnabled()) { + LOG.debug("Access-Time:[" + now.getTime() + "]"); + } + + Calendar startOfInterval = getClosestPastEpoch(now); + + if (startOfInterval != null) { + if (LOG.isDebugEnabled()) { + LOG.debug("Start-of-Interval:[" + startOfInterval.getTime() + "]"); + } + + Calendar endOfInterval = (Calendar) startOfInterval.clone(); + endOfInterval.add(Calendar.MINUTE, recurrence.getInterval().getMinutes()); + endOfInterval.add(Calendar.HOUR, recurrence.getInterval().getHours()); + endOfInterval.add(Calendar.DAY_OF_MONTH, recurrence.getInterval().getDays()); + + endOfInterval.getTime(); // for recomputation + now.getTime(); + + if (LOG.isDebugEnabled()) { + LOG.debug("End-of-Interval:[" + endOfInterval.getTime() + "]"); + } + + ret = startOfInterval.compareTo(now) <= 0 && endOfInterval.compareTo(now) >= 0; + } + + } else { + ret = true; + } + + RangerPerfTracer.log(perf); + return ret; + } + + private void addScheduledTime(RangerValidityRecurrence.RecurrenceSchedule.ScheduleFieldSpec fieldSpec, List list) { + final String str = recurrence.getSchedule().getFieldValue(fieldSpec); + final boolean isMonth = fieldSpec == RangerValidityRecurrence.RecurrenceSchedule.ScheduleFieldSpec.month; + + if (StringUtils.isNotBlank(str)) { + String[] specs = str.split(","); + + for (String spec : specs) { + String[] range = spec.split("-"); + + if (range.length == 1) { + if (StringUtils.equals(range[0], RangerValidityRecurrence.RecurrenceSchedule.WILDCARD)) { + list.clear(); + list.add(new ScheduledTimeAlwaysMatcher()); + + break; + } else { + list.add(new ScheduledTimeExactMatcher(Integer.valueOf(range[0]) - (isMonth ? 1 : 0))); + } + } else { + if (StringUtils.equals(range[0], RangerValidityRecurrence.RecurrenceSchedule.WILDCARD) || StringUtils.equals(range[1], RangerValidityRecurrence.RecurrenceSchedule.WILDCARD)) { + list.clear(); + list.add(new ScheduledTimeAlwaysMatcher()); + + break; + } else { + list.add(new ScheduledTimeRangeMatcher(Integer.valueOf(range[0]) - (isMonth ? 1 : 0), Integer.valueOf(range[1]) - (isMonth ? 1 : 0))); + } + } + } + + Collections.reverse(list); + } + + } + + /* + Given a Calendar object, get the closest, earlier Calendar object based on configured validity schedules. + Returns - a valid Calendar object. Throws exception if any errors during processing or no suitable Calendar object is found. + Description - Typically, a caller will call this with Calendar constructed with current time, and use returned object + along with specified interval to ensure that next schedule time is after the input Calendar. + Algorithm - This involves doing a Calendar arithmetic (subtraction) with borrow. The tricky parts are ensuring that + Calendar arithmetic yields a valid Calendar object. + - Start with minutes, and then hours. + - Must make sure that the later of the two Calendars - one computed with dayOfMonth, another computed with + dayOfWeek - is picked + - For dayOfMonth calculation, consider that months have different number of days + */ + + private static class ValueWithBorrow { + int value; + boolean borrow; + + ValueWithBorrow() { + } + + ValueWithBorrow(int value) { + this(value, false); + } + + ValueWithBorrow(int value, boolean borrow) { + this.value = value; + this.borrow = borrow; + } + + void setValue(int value) { + this.value = value; + } + + void setBorrow(boolean borrow) { + this.borrow = borrow; + } + + int getValue() { + return value; + } + + boolean getBorrow() { + return borrow; + } + + @Override + public String toString() { + return "value=" + value + ", borrow=" + borrow; + } + } + + private Calendar getClosestPastEpoch(Calendar current) { + Calendar ret = null; + + try { + ValueWithBorrow input = new ValueWithBorrow(); + + input.setValue(current.get(Calendar.MINUTE)); + input.setBorrow(false); + ValueWithBorrow closestMinute = getPastFieldValueWithBorrow(RangerValidityRecurrence.RecurrenceSchedule.ScheduleFieldSpec.minute, minutes, input); + + input.setValue(current.get(Calendar.HOUR_OF_DAY)); + input.setBorrow(closestMinute.borrow); + ValueWithBorrow closestHour = getPastFieldValueWithBorrow(RangerValidityRecurrence.RecurrenceSchedule.ScheduleFieldSpec.hour, hours, input); + + Calendar dayOfMonthCalendar = getClosestDayOfMonth(current, closestMinute, closestHour); + + Calendar dayOfWeekCalendar = getClosestDayOfWeek(current, closestMinute, closestHour); + + ret = getEarlierCalendar(dayOfMonthCalendar, dayOfWeekCalendar); + + if (LOG.isDebugEnabled()) { + LOG.debug("ClosestPastEpoch:[" + (ret != null ? ret.getTime() : null) + "]"); + } + + } catch (Exception e) { + LOG.error("Could not find ClosestPastEpoch, Exception=", e); + } + return ret; + } + + private Calendar getClosestDayOfMonth(Calendar current, ValueWithBorrow closestMinute, ValueWithBorrow closestHour) throws Exception { + Calendar ret = null; + if (StringUtils.isNotBlank(recurrence.getSchedule().getDayOfMonth())) { + int initialDayOfMonth = current.get(Calendar.DAY_OF_MONTH); + + int currentDayOfMonth = initialDayOfMonth, currentMonth = current.get(Calendar.MONTH), currentYear = current.get(Calendar.YEAR); + int maximumDaysInPreviousMonth = getMaximumValForPreviousMonth(current); + + if (closestHour.borrow) { + initialDayOfMonth--; + Calendar dayOfMonthCalc = (GregorianCalendar) current.clone(); + dayOfMonthCalc.add(Calendar.DAY_OF_MONTH, -1); + dayOfMonthCalc.getTime(); + int previousDayOfMonth = dayOfMonthCalc.get(Calendar.DAY_OF_MONTH); + if (initialDayOfMonth < previousDayOfMonth) { + if (LOG.isDebugEnabled()) { + LOG.debug("Need to borrow from previous month, initialDayOfMonth:[" + initialDayOfMonth + "], previousDayOfMonth:[" + previousDayOfMonth + "], dayOfMonthCalc:[" + dayOfMonthCalc.getTime() + "]"); + } + currentDayOfMonth = previousDayOfMonth; + currentMonth = dayOfMonthCalc.get(Calendar.MONTH); + currentYear = dayOfMonthCalc.get(Calendar.YEAR); + maximumDaysInPreviousMonth = getMaximumValForPreviousMonth(dayOfMonthCalc); + } else if (initialDayOfMonth == previousDayOfMonth) { + if (LOG.isDebugEnabled()) { + LOG.debug("No need to borrow from previous month, initialDayOfMonth:[" + initialDayOfMonth + "], previousDayOfMonth:[" + previousDayOfMonth + "]"); + } + } else { + LOG.error("Should not get here, initialDayOfMonth:[" + initialDayOfMonth + "], previousDayOfMonth:[" + previousDayOfMonth + "]"); + throw new Exception("Should not get here, initialDayOfMonth:[" + initialDayOfMonth + "], previousDayOfMonth:[" + previousDayOfMonth + "]"); + } + } + if (LOG.isDebugEnabled()) { + LOG.debug("currentDayOfMonth:[" + currentDayOfMonth + "], maximumDaysInPreviourMonth:[" + maximumDaysInPreviousMonth + "]"); + } + ValueWithBorrow input = new ValueWithBorrow(); + input.setValue(currentDayOfMonth); + input.setBorrow(false); + ValueWithBorrow closestDayOfMonth = null; + do { + int i = 0; + try { + closestDayOfMonth = getPastFieldValueWithBorrow(RangerValidityRecurrence.RecurrenceSchedule.ScheduleFieldSpec.dayOfMonth, daysOfMonth, input, maximumDaysInPreviousMonth); + } catch (Exception e) { + i++; + Calendar c = (GregorianCalendar) current.clone(); + c.set(Calendar.YEAR, currentYear); + c.set(Calendar.MONTH, currentMonth); + c.set(Calendar.DAY_OF_MONTH, currentDayOfMonth); + c.add(Calendar.MONTH, -i); + c.getTime(); + currentMonth = c.get(Calendar.MONTH); + currentYear = c.get(Calendar.YEAR); + currentDayOfMonth = c.get(Calendar.DAY_OF_MONTH); + maximumDaysInPreviousMonth = getMaximumValForPreviousMonth(c); + input.setValue(currentDayOfMonth); + input.setBorrow(false); + } + } while (closestDayOfMonth == null); + + // Build calendar for dayOfMonth + ret = new GregorianCalendar(); + ret.set(Calendar.DAY_OF_MONTH, closestDayOfMonth.value); + ret.set(Calendar.HOUR_OF_DAY, closestHour.value); + ret.set(Calendar.MINUTE, closestMinute.value); + ret.set(Calendar.SECOND, 0); + ret.set(Calendar.MILLISECOND, 0); + + ret.set(Calendar.YEAR, currentYear); + + if (closestDayOfMonth.borrow) { + ret.set(Calendar.MONTH, currentMonth - 1); + } else { + ret.set(Calendar.MONTH, currentMonth); + } + ret.getTime(); // For recomputation + + if (LOG.isDebugEnabled()) { + LOG.debug("Best guess using DAY_OF_MONTH:[" + ret.getTime() + "]"); + } + } + return ret; + } + + private Calendar getClosestDayOfWeek(Calendar current, ValueWithBorrow closestMinute, ValueWithBorrow closestHour) throws Exception { + Calendar ret = null; + if (StringUtils.isNotBlank(recurrence.getSchedule().getDayOfWeek())) { + ValueWithBorrow input = new ValueWithBorrow(); + + input.setValue(current.get(Calendar.DAY_OF_WEEK)); + input.setBorrow(closestHour.borrow); + + + ValueWithBorrow closestDayOfWeek = getPastFieldValueWithBorrow(RangerValidityRecurrence.RecurrenceSchedule.ScheduleFieldSpec.dayOfWeek, daysOfWeek, input); + + int daysToGoback = closestHour.borrow ? 1 : 0; + int range = RangerValidityRecurrence.RecurrenceSchedule.ScheduleFieldSpec.dayOfWeek.maximum - RangerValidityRecurrence.RecurrenceSchedule.ScheduleFieldSpec.dayOfWeek.minimum + 1; + + if (closestDayOfWeek.borrow) { + if (input.value - closestDayOfWeek.value != daysToGoback) { + daysToGoback = range + input.value - closestDayOfWeek.value; + } + } else { + daysToGoback = input.value - closestDayOfWeek.value; + } + + if (LOG.isDebugEnabled()) { + LOG.debug("Need to go back [" + daysToGoback + "] days to match dayOfWeek"); + } + + ret = (GregorianCalendar) current.clone(); + ret.set(Calendar.MINUTE, closestMinute.value); + ret.set(Calendar.HOUR_OF_DAY, closestHour.value); + ret.add(Calendar.DAY_OF_MONTH, (0 - daysToGoback)); + ret.set(Calendar.SECOND, 0); + ret.set(Calendar.MILLISECOND, 0); + + ret.getTime(); + + if (LOG.isDebugEnabled()) { + LOG.debug("Best guess using DAY_OF_WEEK:[" + ret.getTime() + "]"); + } + } + return ret; + + } + + private int getMaximumValForPreviousMonth(Calendar current) { + Calendar cal = (Calendar) current.clone(); + cal.add(Calendar.MONTH, -1); + cal.getTime(); // For recomputation + + return cal.getActualMaximum(Calendar.DAY_OF_MONTH); + } + + private Calendar getEarlierCalendar(Calendar dayOfMonthCalendar, Calendar dayOfWeekCalendar) throws Exception { + + Calendar withDayOfMonth = fillOutCalendar(dayOfMonthCalendar); + if (LOG.isDebugEnabled()) { + LOG.debug("dayOfMonthCalendar:[" + (withDayOfMonth != null ? withDayOfMonth.getTime() : null) + "]"); + } + + Calendar withDayOfWeek = fillOutCalendar(dayOfWeekCalendar); + if (LOG.isDebugEnabled()) { + LOG.debug("dayOfWeekCalendar:[" + (withDayOfWeek != null ? withDayOfWeek.getTime() : null) + "]"); + } + + if (withDayOfMonth != null && withDayOfWeek != null) { + return withDayOfMonth.after(withDayOfWeek) ? withDayOfMonth : withDayOfWeek; + } else if (withDayOfMonth == null) { + return withDayOfWeek; + } else { + return withDayOfMonth; + } + } + + private Calendar fillOutCalendar(Calendar calendar) throws Exception { + Calendar ret = null; + + if (calendar != null) { + ValueWithBorrow input = new ValueWithBorrow(calendar.get(Calendar.MONTH)); + ValueWithBorrow closestMonth = getPastFieldValueWithBorrow(RangerValidityRecurrence.RecurrenceSchedule.ScheduleFieldSpec.month, months, input); + + input.setValue(calendar.get(Calendar.YEAR)); + input.setBorrow(closestMonth.borrow); + ValueWithBorrow closestYear = getPastFieldValueWithBorrow(RangerValidityRecurrence.RecurrenceSchedule.ScheduleFieldSpec.year, years, input); + + // Build calendar + ret = (Calendar) calendar.clone(); + ret.set(Calendar.YEAR, closestYear.value); + ret.set(Calendar.MONTH, closestMonth.value); + ret.set(Calendar.SECOND, 0); + + ret.getTime(); // for recomputation + if (LOG.isDebugEnabled()) { + LOG.debug("Filled-out-Calendar:[" + ret.getTime() + "]"); + } + } + return ret; + } + + private ValueWithBorrow getPastFieldValueWithBorrow(RangerValidityRecurrence.RecurrenceSchedule.ScheduleFieldSpec fieldSpec, List searchList, ValueWithBorrow input) throws Exception { + return getPastFieldValueWithBorrow(fieldSpec, searchList, input, fieldSpec.maximum); + } + + private ValueWithBorrow getPastFieldValueWithBorrow(RangerValidityRecurrence.RecurrenceSchedule.ScheduleFieldSpec fieldSpec, List searchList, ValueWithBorrow input, int maximum) throws Exception { + + ValueWithBorrow ret; + boolean borrow = false; + + int value = input.value - (input.borrow ? 1 : 0); + + if (CollectionUtils.isNotEmpty(searchList)) { + int range = fieldSpec.maximum - fieldSpec.minimum + 1; + + for (int i = 0; i < range; i++, value--) { + if (value < fieldSpec.minimum) { + value = maximum; + borrow = true; + } + for (ScheduledTimeMatcher time : searchList) { + if (time.isMatch(value)) { + if (LOG.isDebugEnabled()) { + LOG.debug("Found match in field:[" + fieldSpec + "], value:[" + value + "], borrow:[" + borrow + "], maximum:[" + maximum + "]"); + } + return new ValueWithBorrow(value, borrow); + } + } + } + // Not found + throw new Exception("No match found in field:[" + fieldSpec + "] for [input=" + input + "]"); + } else { + if (value < fieldSpec.minimum) { + value = maximum; + } + ret = new ValueWithBorrow(value, false); + } + return ret; + } + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyresourcematcher/RangerDefaultPolicyResourceMatcher.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyresourcematcher/RangerDefaultPolicyResourceMatcher.java new file mode 100644 index 00000000000..b6f969ebf5f --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyresourcematcher/RangerDefaultPolicyResourceMatcher.java @@ -0,0 +1,798 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.policyresourcematcher; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.plugin.model.RangerPolicy; +import org.apache.atlas.plugin.model.RangerPolicy.RangerPolicyResource; +import org.apache.atlas.plugin.model.RangerServiceDef; +import org.apache.atlas.plugin.model.RangerServiceDef.RangerResourceDef; +import org.apache.atlas.plugin.model.validation.RangerServiceDefHelper; +import org.apache.atlas.plugin.policyengine.RangerAccessResource; +import org.apache.atlas.plugin.policyengine.RangerAccessResourceImpl; +import org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher; +import org.apache.atlas.plugin.resourcematcher.RangerResourceMatcher; +import org.apache.atlas.plugin.util.RangerPerfTracer; + +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class RangerDefaultPolicyResourceMatcher implements RangerPolicyResourceMatcher { + private static final Log LOG = LogFactory.getLog(RangerDefaultPolicyResourceMatcher.class); + + private static final Log PERF_POLICY_RESOURCE_MATCHER_INIT_LOG = RangerPerfTracer.getPerfLogger("policyresourcematcher.init"); + private static final Log PERF_POLICY_RESOURCE_MATCHER_MATCH_LOG = RangerPerfTracer.getPerfLogger("policyresourcematcher.match"); + + protected RangerServiceDef serviceDef; + protected String policyType; + protected Map policyResources; + + private Map allMatchers; + private boolean needsDynamicEval = false; + private List validResourceHierarchy; + private boolean isInitialized = false; + private RangerServiceDefHelper serviceDefHelper; + + @Override + public void setServiceDef(RangerServiceDef serviceDef) { + if (isInitialized) { + LOG.warn("RangerDefaultPolicyResourceMatcher is already initialized. init() must be done again after updating serviceDef"); + } + + this.serviceDef = serviceDef; + } + + @Override + public void setPolicy(RangerPolicy policy) { + if (isInitialized) { + LOG.warn("RangerDefaultPolicyResourceMatcher is already initialized. init() must be done again after updating policy"); + } + + if (policy == null) { + setPolicyResources(null, RangerPolicy.POLICY_TYPE_ACCESS); + } else { + setPolicyResources(policy.getResources(), StringUtils.isEmpty(policy.getPolicyType()) ? RangerPolicy.POLICY_TYPE_ACCESS : policy.getPolicyType()); + } + } + + @Override + public void setPolicyResources(Map policyResources) { + if (isInitialized) { + LOG.warn("RangerDefaultPolicyResourceMatcher is already initialized. init() must be done again after updating policy-resources"); + } + + setPolicyResources(policyResources, RangerPolicy.POLICY_TYPE_ACCESS); + } + + @Override + public void setPolicyResources(Map policyResources, String policyType) { + this.policyResources = policyResources; + this.policyType = policyType; + } + + @Override + public void setServiceDefHelper(RangerServiceDefHelper serviceDefHelper) { + this.serviceDefHelper = serviceDefHelper; + } + + @Override + public RangerServiceDef getServiceDef() { + return serviceDef; + } + + @Override + public RangerResourceMatcher getResourceMatcher(String resourceName) { + return allMatchers != null ? allMatchers.get(resourceName) : null; + } + + @Override + public boolean getNeedsDynamicEval() { return needsDynamicEval; } + + @Override + public void init() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerDefaultPolicyResourceMatcher.init()"); + } + + allMatchers = null; + needsDynamicEval = false; + validResourceHierarchy = null; + isInitialized = false; + + String errorText = ""; + + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_POLICY_RESOURCE_MATCHER_INIT_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_POLICY_RESOURCE_MATCHER_INIT_LOG, "RangerDefaultPolicyResourceMatcher.init()"); + } + + if (policyResources != null && !policyResources.isEmpty() && serviceDef != null) { + serviceDefHelper = serviceDefHelper == null ? new RangerServiceDefHelper(serviceDef, false) : serviceDefHelper; + + Set> resourceHierarchies = serviceDefHelper.getResourceHierarchies(policyType, policyResources.keySet()); + int validHierarchiesCount = 0; + + for (List resourceHierarchy : resourceHierarchies) { + + if (isHierarchyValidForResources(resourceHierarchy, policyResources)) { + validHierarchiesCount++; + + if (validHierarchiesCount == 1) { + validResourceHierarchy = resourceHierarchy; + } else { + validResourceHierarchy = null; + } + } else { + LOG.warn("RangerDefaultPolicyResourceMatcher.init(): gaps found in policyResources, skipping hierarchy:[" + resourceHierarchies + "]"); + } + } + + if (validHierarchiesCount > 0) { + allMatchers = new HashMap<>(); + + for (List resourceHierarchy : resourceHierarchies) { + for (RangerResourceDef resourceDef : resourceHierarchy) { + String resourceName = resourceDef.getName(); + + if (allMatchers.containsKey(resourceName)) { + continue; + } + + RangerPolicyResource policyResource = policyResources.get(resourceName); + + if (policyResource == null) { + if (LOG.isDebugEnabled()) { + LOG.debug("RangerDefaultPolicyResourceMatcher.init(): no matcher created for " + resourceName + ". Continuing ..."); + } + + continue; + } + + RangerResourceMatcher matcher = createResourceMatcher(resourceDef, policyResource); + + if (matcher != null) { + if (!needsDynamicEval && matcher.getNeedsDynamicEval()) { + needsDynamicEval = true; + } + + allMatchers.put(resourceName, matcher); + } else { + LOG.error("RangerDefaultPolicyResourceMatcher.init(): failed to find matcher for resource " + resourceName); + + allMatchers = null; + errorText = "no matcher found for resource " + resourceName; + + break; + } + } + + if (allMatchers == null) { + break; + } + } + } else { + errorText = "policyResources elements are not part of any valid resourcedef hierarchy."; + } + } else { + errorText = "policyResources is null or empty, or serviceDef is null."; + } + + if (allMatchers == null && !RangerPolicy.POLICY_TYPE_AUDIT.equals(policyType)) { + serviceDefHelper = null; + validResourceHierarchy = null; + + Set policyResourceKeys = policyResources == null ? null : policyResources.keySet(); + String serviceDefName = serviceDef == null ? "" : serviceDef.getName(); + StringBuilder keysString = new StringBuilder(); + + if (CollectionUtils.isNotEmpty(policyResourceKeys)) { + for (String policyResourceKeyName : policyResourceKeys) { + keysString.append(policyResourceKeyName).append(" "); + } + } + + LOG.error("RangerDefaultPolicyResourceMatcher.init() failed: " + errorText + " (serviceDef=" + serviceDefName + ", policyResourceKeys=" + keysString.toString()); + } else { + isInitialized = true; + } + + RangerPerfTracer.log(perf); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerDefaultPolicyResourceMatcher.init(): ret=" + isInitialized); + } + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + + return toString(sb).toString(); + } + + @Override + public StringBuilder toString(StringBuilder sb) { + sb.append("RangerDefaultPolicyResourceMatcher={"); + + sb.append("isInitialized=").append(isInitialized).append(", "); + + sb.append("matchers={"); + if(allMatchers != null) { + for(RangerResourceMatcher matcher : allMatchers.values()) { + sb.append("{").append(matcher).append("} "); + } + } + sb.append("} "); + + sb.append("}"); + + return sb; + } + + @Override + public boolean isCompleteMatch(RangerAccessResource resource, Map evalContext) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerDefaultPolicyResourceMatcher.isCompleteMatch(" + resource + ", " + evalContext + ")"); + } + + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_POLICY_RESOURCE_MATCHER_MATCH_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_POLICY_RESOURCE_MATCHER_MATCH_LOG, "RangerDefaultPolicyResourceMatcher.grantRevokeMatch()"); + } + + boolean ret = false; + Collection resourceKeys = resource == null ? null : resource.getKeys(); + Collection policyKeys = policyResources == null ? null : policyResources.keySet(); + boolean keysMatch = resourceKeys != null && policyKeys != null && CollectionUtils.isEqualCollection(resourceKeys, policyKeys); + + if (keysMatch) { + for (RangerResourceDef resourceDef : serviceDef.getResources()) { + String resourceName = resourceDef.getName(); + Object resourceValue = resource.getValue(resourceName); + RangerResourceMatcher matcher = getResourceMatcher(resourceName); + + if (resourceValue == null) { + ret = matcher == null || matcher.isCompleteMatch(null, evalContext); + } else if (resourceValue instanceof String) { + String strValue = (String) resourceValue; + + if (StringUtils.isEmpty(strValue)) { + ret = matcher == null || matcher.isCompleteMatch(strValue, evalContext); + } else { + ret = matcher != null && matcher.isCompleteMatch(strValue, evalContext); + } + } else { // return false for any other type of resourceValue + ret = false; + } + + if (!ret) { + break; + } + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("isCompleteMatch(): keysMatch=false. resourceKeys=" + resourceKeys + "; policyKeys=" + policyKeys); + } + } + + RangerPerfTracer.log(perf); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerDefaultPolicyResourceMatcher.isCompleteMatch(" + resource + ", " + evalContext + "): " + ret); + } + + return ret; + } + + @Override + public boolean isCompleteMatch(Map resources, Map evalContext) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerDefaultPolicyResourceMatcher.isCompleteMatch(" + resources + ", " + evalContext + ")"); + } + + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_POLICY_RESOURCE_MATCHER_MATCH_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_POLICY_RESOURCE_MATCHER_MATCH_LOG, "RangerDefaultPolicyResourceMatcher.applyPolicyMatch()"); + } + + boolean ret = false; + Collection resourceKeys = resources == null ? null : resources.keySet(); + Collection policyKeys = policyResources == null ? null : policyResources.keySet(); + boolean keysMatch = resourceKeys != null && policyKeys != null && CollectionUtils.isEqualCollection(resourceKeys, policyKeys); + + if (keysMatch) { + for (RangerResourceDef resourceDef : serviceDef.getResources()) { + String resourceName = resourceDef.getName(); + RangerPolicyResource resourceValues = resources.get(resourceName); + RangerPolicyResource policyValues = policyResources == null ? null : policyResources.get(resourceName); + + if (resourceValues == null || CollectionUtils.isEmpty(resourceValues.getValues())) { + ret = (policyValues == null || CollectionUtils.isEmpty(policyValues.getValues())); + } else if (policyValues != null && CollectionUtils.isNotEmpty(policyValues.getValues())) { + ret = CollectionUtils.isEqualCollection(resourceValues.getValues(), policyValues.getValues()); + } + + if (!ret) { + break; + } + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("isCompleteMatch(): keysMatch=false. resourceKeys=" + resourceKeys + "; policyKeys=" + policyKeys); + } + } + + RangerPerfTracer.log(perf); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerDefaultPolicyResourceMatcher.isCompleteMatch(" + resources + ", " + evalContext + "): " + ret); + } + + return ret; + } + + @Override + public boolean isMatch(RangerPolicy policy, MatchScope scope, Map evalContext) { + boolean ret = false; + + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_POLICY_RESOURCE_MATCHER_MATCH_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_POLICY_RESOURCE_MATCHER_MATCH_LOG, "RangerDefaultPolicyResourceMatcher.getPoliciesNonLegacy()"); + } + + Map resources = policy.getResources(); + + if (policy.getPolicyType().equals(policyType) && MapUtils.isNotEmpty(resources)) { + List hierarchy = getMatchingHierarchy(resources.keySet()); + + if (CollectionUtils.isNotEmpty(hierarchy)) { + MatchType matchType = MatchType.NONE; + RangerAccessResourceImpl accessResource = new RangerAccessResourceImpl(); + + accessResource.setServiceDef(serviceDef); + + // Build up accessResource resourceDef by resourceDef. + // For each resourceDef, + // examine policy-values one by one. + // The first value that is acceptable, that is, + // value matches in any way, is used for that resourceDef, and + // next resourceDef is processed. + // If none of the values matches, the policy as a whole definitely will not match, + // therefore, the match is failed + // After all resourceDefs are processed, and some match is achieved at every + // level, the final matchType (which is for the entire policy) is checked against + // requested scope to determine the match-result. + + // Unit tests in TestDefaultPolicyResourceForPolicy.java, TestDefaultPolicyResourceMatcher.java + // test_defaultpolicyresourcematcher_for_hdfs_policy.json, and + // test_defaultpolicyresourcematcher_for_hive_policy.json, and + // test_defaultPolicyResourceMatcher.json + + boolean skipped = false; + + for (RangerResourceDef resourceDef : hierarchy) { + String name = resourceDef.getName(); + RangerPolicyResource policyResource = resources.get(name); + + if (policyResource != null && CollectionUtils.isNotEmpty(policyResource.getValues())) { + ret = false; + matchType = MatchType.NONE; + + if (!skipped) { + for (String value : policyResource.getValues()) { + accessResource.setValue(name, value); + + matchType = getMatchType(accessResource, evalContext); + + if (matchType != MatchType.NONE) { // One value for this resourceDef matched + ret = true; + break; + } + } + } else { + break; + } + } else { + skipped = true; + } + + if (!ret) { // None of the values specified for this resourceDef matched, no point in continuing with next resourceDef + break; + } + } + + ret = ret && isMatch(scope, matchType); + } + } + + RangerPerfTracer.log(perf); + + return ret; + } + + @Override + public boolean isMatch(RangerAccessResource resource, Map evalContext) { + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_POLICY_RESOURCE_MATCHER_MATCH_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_POLICY_RESOURCE_MATCHER_MATCH_LOG, "RangerDefaultPolicyResourceMatcher.grantRevokeMatch()"); + } + + /* + * There is already API to get the delegateAdmin permissions for a map of policyResources. + * That implementation should be reused for figuring out delegateAdmin permissions for a resource as well. + */ + + Map policyResources = null; + + for (RangerResourceDef resourceDef : serviceDef.getResources()) { + String resourceName = resourceDef.getName(); + Object resourceValue = resource.getValue(resourceName); + if (resourceValue instanceof String) { + String strValue = (String) resourceValue; + + if (policyResources == null) { + policyResources = new HashMap<>(); + } + policyResources.put(resourceName, new RangerPolicyResource(strValue)); + } else if (resourceValue != null) { // return false for any other type of resourceValue + policyResources = null; + + break; + } + } + final boolean ret = MapUtils.isNotEmpty(policyResources) && isMatch(policyResources, evalContext); + + RangerPerfTracer.log(perf); + + return ret; + } + + @Override + public boolean isMatch(Map resources, Map evalContext) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerDefaultPolicyResourceMatcher.isMatch(" + resources + ", " + evalContext + ")"); + } + + boolean ret = false; + + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_POLICY_RESOURCE_MATCHER_MATCH_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_POLICY_RESOURCE_MATCHER_MATCH_LOG, "RangerDefaultPolicyResourceMatcher.delegateAdminMatch()"); + } + + if(serviceDef != null && serviceDef.getResources() != null) { + Collection resourceKeys = resources == null ? null : resources.keySet(); + Collection policyKeys = policyResources == null ? null : policyResources.keySet(); + + boolean keysMatch = CollectionUtils.isEmpty(resourceKeys) || (policyKeys != null && policyKeys.containsAll(resourceKeys)); + + if(keysMatch) { + for(RangerResourceDef resourceDef : serviceDef.getResources()) { + String resourceName = resourceDef.getName(); + RangerPolicyResource resourceValues = resources == null ? null : resources.get(resourceName); + List values = resourceValues == null ? null : resourceValues.getValues(); + RangerResourceMatcher matcher = allMatchers == null ? null : allMatchers.get(resourceName); + + if (matcher != null) { + if (CollectionUtils.isNotEmpty(values)) { + for (String value : values) { + ret = matcher.isMatch(value, evalContext); + if (!ret) { + break; + } + } + } else { + ret = matcher.isMatchAny(); + } + } else { + ret = CollectionUtils.isEmpty(values); + } + + if(! ret) { + break; + } + } + } else { + if(LOG.isDebugEnabled()) { + LOG.debug("isMatch(): keysMatch=false. resourceKeys=" + resourceKeys + "; policyKeys=" + policyKeys); + } + } + } + + RangerPerfTracer.log(perf); + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerDefaultPolicyResourceMatcher.isMatch(" + resources + ", " + evalContext + "): " + ret); + } + + return ret; + } + + @Override + public boolean isMatch(RangerAccessResource resource, MatchScope scope, Map evalContext) { + MatchType matchType = getMatchType(resource, evalContext); + return isMatch(scope, matchType); + } + + @Override + public MatchType getMatchType(RangerAccessResource resource, Map evalContext) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerDefaultPolicyResourceMatcher.getMatchType(" + resource + evalContext + ")"); + } + MatchType ret = MatchType.NONE; + + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_POLICY_RESOURCE_MATCHER_MATCH_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_POLICY_RESOURCE_MATCHER_MATCH_LOG, "RangerDefaultPolicyResourceMatcher.getMatchType()"); + } + + if (resource != null && policyResources != null) { + int resourceKeysSize = resource.getKeys() == null ? 0 : resource.getKeys().size(); + + if (policyResources.size() == 0 && resourceKeysSize == 0) { + ret = MatchType.SELF; + } else { + List hierarchy = getMatchingHierarchy(resource); + if (CollectionUtils.isNotEmpty(hierarchy)) { + + int lastNonAnyMatcherIndex = -1; + int matchersSize = 0; + + for (RangerResourceDef resourceDef : hierarchy) { + RangerResourceMatcher matcher = getResourceMatcher(resourceDef.getName()); + if (matcher != null) { + if (!matcher.isMatchAny()) { + lastNonAnyMatcherIndex = matchersSize; + } + matchersSize++; + } else { + break; + } + } + + if (resourceKeysSize == 0) { + ret = MatchType.SELF; + } + + for (RangerResourceDef resourceDef : hierarchy) { + + RangerResourceMatcher matcher = getResourceMatcher(resourceDef.getName()); + Object resourceValue = resource.getValue(resourceDef.getName()); + + if (matcher != null) { + if (resourceValue != null || matcher.isMatchAny()) { + if (matcher.isMatch(resourceValue, evalContext)) { + ret = MatchType.SELF; + } else { + ret = MatchType.NONE; + break; + } + } + } else { + if (resourceValue != null) { + // More resource-values than matchers + ret = MatchType.ANCESTOR; + } + break; + } + } + + if (ret == MatchType.SELF && resourceKeysSize < policyResources.size()) { + // More matchers than resource-values + if (resourceKeysSize > lastNonAnyMatcherIndex) { + // all remaining matchers which matched resource value of null are of type Any + ret = MatchType.SELF_AND_ALL_DESCENDANTS; + } else { + ret = MatchType.DESCENDANT; + } + } + } + } + } + + RangerPerfTracer.log(perf); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerDefaultPolicyResourceMatcher.getMatchType(" + resource + evalContext + "): " + ret); + } + + return ret; + } + + public static boolean isHierarchyValidForResources(List hierarchy, Map resources) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> isHierarchyValidForResources(" + StringUtils.join(hierarchy, ",") + ")"); + } + boolean ret = true; + + if (hierarchy != null) { + boolean skipped = false; + + for (RangerResourceDef resourceDef : hierarchy) { + String resourceName = resourceDef.getName(); + Object resourceValue = resources.get(resourceName); + + if (resourceValue == null) { + if (!skipped) { + skipped = true; + } + } else { + if (skipped) { + ret = false; + break; + } + } + } + } else { + ret = false; + } + if (LOG.isDebugEnabled()) { + LOG.debug("<== isHierarchyValidForResources(" + StringUtils.join(hierarchy, ",") + ") : " + ret); + } + return ret; + } + + private List getMatchingHierarchy(Set resourceKeys) { + List ret = null; + + if (CollectionUtils.isNotEmpty(resourceKeys) && serviceDefHelper != null) { + Set> resourceHierarchies = serviceDefHelper.getResourceHierarchies(policyType, resourceKeys); + + // pick the shortest hierarchy + for (List resourceHierarchy : resourceHierarchies) { + if (ret == null) { + ret = resourceHierarchy; + } else { + if (resourceHierarchy.size() < ret.size()) { + ret = resourceHierarchy; + if (ret.size() == resourceKeys.size()) { + break; + } + } + } + } + } + + return ret; + } + + private List getMatchingHierarchy(RangerAccessResource resource) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerDefaultPolicyResourceMatcher.getMatchingHierarchy(" + resource + ")"); + } + + final List ret; + + Set policyResourcesKeySet = policyResources.keySet(); + Set resourceKeySet = resource.getKeys(); + + if (CollectionUtils.isNotEmpty(resourceKeySet)) { + List aValidHierarchy = null; + + if (validResourceHierarchy != null && serviceDefHelper != null) { + if (serviceDefHelper.hierarchyHasAllResources(validResourceHierarchy, resourceKeySet)) { + aValidHierarchy = validResourceHierarchy; + } + } else { + if (policyResourcesKeySet.containsAll(resourceKeySet)) { + aValidHierarchy = getMatchingHierarchy(policyResourcesKeySet); + } else if (resourceKeySet.containsAll(policyResourcesKeySet)) { + aValidHierarchy = getMatchingHierarchy(resourceKeySet); + } + } + ret = isHierarchyValidForResources(aValidHierarchy, resource.getAsMap()) ? aValidHierarchy : null; + } else { + ret = validResourceHierarchy != null ? validResourceHierarchy : getMatchingHierarchy(policyResourcesKeySet); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerDefaultPolicyResourceMatcher.getMatchingHierarchy(" + resource + "): " + ret); + } + + return ret; + } + + private boolean isMatch(final MatchScope scope, final MatchType matchType) { + final boolean ret; + switch (scope) { + case SELF: { + ret = matchType == MatchType.SELF || matchType == MatchType.SELF_AND_ALL_DESCENDANTS; + break; + } + case ANCESTOR: { + ret = matchType == MatchType.ANCESTOR; + break; + } + case DESCENDANT: { + ret = matchType == MatchType.DESCENDANT; + break; + } + case SELF_OR_ANCESTOR: { + ret = matchType == MatchType.SELF || matchType == MatchType.SELF_AND_ALL_DESCENDANTS || matchType == MatchType.ANCESTOR; + break; + } + case SELF_OR_DESCENDANT: { + ret = matchType == MatchType.SELF || matchType == MatchType.SELF_AND_ALL_DESCENDANTS || matchType == MatchType.DESCENDANT; + break; + } + default: { + ret = matchType != MatchType.NONE; + break; + } + } + return ret; + } + + private static RangerResourceMatcher createResourceMatcher(RangerResourceDef resourceDef, RangerPolicyResource resource) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerDefaultPolicyResourceMatcher.createResourceMatcher(" + resourceDef + ", " + resource + ")"); + } + + RangerResourceMatcher ret = null; + + if (resourceDef != null) { + String resName = resourceDef.getName(); + String clsName = resourceDef.getMatcher(); + + if (!StringUtils.isEmpty(clsName)) { + try { + @SuppressWarnings("unchecked") + Class matcherClass = (Class) Class.forName(clsName); + + ret = matcherClass.newInstance(); + } catch (Exception excp) { + LOG.error("failed to instantiate resource matcher '" + clsName + "' for '" + resName + "'. Default resource matcher will be used", excp); + } + } + + + if (ret == null) { + ret = new RangerDefaultResourceMatcher(); + } + + ret.setResourceDef(resourceDef); + ret.setPolicyResource(resource); + ret.init(); + } else { + LOG.error("RangerDefaultPolicyResourceMatcher: RangerResourceDef is null"); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerDefaultPolicyResourceMatcher.createResourceMatcher(" + resourceDef + ", " + resource + "): " + ret); + } + + return ret; + } + +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyresourcematcher/RangerPolicyResourceEvaluator.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyresourcematcher/RangerPolicyResourceEvaluator.java new file mode 100644 index 00000000000..029bd62a95d --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyresourcematcher/RangerPolicyResourceEvaluator.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.policyresourcematcher; + + +import org.apache.atlas.plugin.model.RangerPolicy; +import org.apache.atlas.plugin.model.RangerServiceDef; +import org.apache.atlas.plugin.resourcematcher.RangerResourceMatcher; + +import java.util.Map; + +public interface RangerPolicyResourceEvaluator { + long getId(); + + String getGuid(); + + RangerPolicyResourceMatcher getPolicyResourceMatcher(); + + Map getPolicyResource(); + + RangerResourceMatcher getResourceMatcher(String resourceName); + + boolean isAncestorOf(RangerServiceDef.RangerResourceDef resourceDef); +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyresourcematcher/RangerPolicyResourceMatcher.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyresourcematcher/RangerPolicyResourceMatcher.java new file mode 100644 index 00000000000..34a64f38041 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyresourcematcher/RangerPolicyResourceMatcher.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.policyresourcematcher; + +import org.apache.atlas.plugin.model.RangerPolicy; +import org.apache.atlas.plugin.model.RangerPolicy.RangerPolicyResource; +import org.apache.atlas.plugin.model.RangerServiceDef; +import org.apache.atlas.plugin.model.validation.RangerServiceDefHelper; +import org.apache.atlas.plugin.policyengine.RangerAccessResource; +import org.apache.atlas.plugin.resourcematcher.RangerResourceMatcher; + +import java.util.Map; + +public interface RangerPolicyResourceMatcher { + enum MatchScope { SELF, SELF_OR_DESCENDANT, SELF_OR_ANCESTOR, DESCENDANT, ANCESTOR, ANY, SELF_AND_ALL_DESCENDANTS} + enum MatchType { NONE, SELF, DESCENDANT, ANCESTOR, SELF_AND_ALL_DESCENDANTS} + + void setServiceDef(RangerServiceDef serviceDef); + + void setPolicy(RangerPolicy policy); + + void setPolicyResources(Map policyResources); + + void setPolicyResources(Map policyResources, String policyType); + + void setServiceDefHelper(RangerServiceDefHelper serviceDefHelper); + + void init(); + + RangerServiceDef getServiceDef(); + + RangerResourceMatcher getResourceMatcher(String resourceName); + + boolean isMatch(RangerAccessResource resource, Map evalContext); + + boolean isMatch(Map resources, Map evalContext); + + boolean isMatch(RangerAccessResource resource, MatchScope scope, Map evalContext); + + boolean isMatch(RangerPolicy policy, MatchScope scope, Map evalContext); + + MatchType getMatchType(RangerAccessResource resource, Map evalContext); + + boolean isCompleteMatch(RangerAccessResource resource, Map evalContext); + + boolean isCompleteMatch(Map resources, Map evalContext); + + boolean getNeedsDynamicEval(); + + StringBuilder toString(StringBuilder sb); +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/resourcematcher/RangerAbstractResourceMatcher.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/resourcematcher/RangerAbstractResourceMatcher.java new file mode 100644 index 00000000000..f0b2ff325fa --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/resourcematcher/RangerAbstractResourceMatcher.java @@ -0,0 +1,582 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.resourcematcher; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.io.IOCase; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.plugin.model.RangerPolicy.RangerPolicyResource; +import org.apache.atlas.plugin.model.RangerServiceDef.RangerResourceDef; +import org.apache.atlas.plugin.util.ServiceDefUtil; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; + + +public abstract class RangerAbstractResourceMatcher implements RangerResourceMatcher { + private static final Log LOG = LogFactory.getLog(RangerAbstractResourceMatcher.class); + + public final static String WILDCARD_ASTERISK = "*"; + + public final static String OPTION_IGNORE_CASE = "ignoreCase"; + public final static String OPTION_QUOTED_CASE_SENSITIVE = "quotedCaseSensitive"; + public final static String OPTION_QUOTE_CHARS = "quoteChars"; + public final static String OPTION_WILD_CARD = "wildCard"; + public final static String OPTION_REPLACE_TOKENS = "replaceTokens"; + public final static String OPTION_TOKEN_DELIMITER_START = "tokenDelimiterStart"; + public final static String OPTION_TOKEN_DELIMITER_END = "tokenDelimiterEnd"; + public final static String OPTION_TOKEN_DELIMITER_ESCAPE = "tokenDelimiterEscape"; + public final static String OPTION_TOKEN_DELIMITER_PREFIX = "tokenDelimiterPrefix"; + + protected RangerResourceDef resourceDef; + protected RangerPolicyResource policyResource; + + protected boolean optIgnoreCase; + protected boolean optQuotedCaseSensitive; + protected String optQuoteChars = "\""; + protected boolean optWildCard; + + protected List policyValues; + protected boolean policyIsExcludes; + protected boolean isMatchAny; + protected ResourceMatcherWrapper resourceMatchers; + + protected boolean optReplaceTokens; + protected char startDelimiterChar = '{'; + protected char endDelimiterChar = '}'; + protected char escapeChar = '\\'; + protected String tokenPrefix = ""; + + @Override + public void setResourceDef(RangerResourceDef resourceDef) { + this.resourceDef = resourceDef; + } + + @Override + public void setPolicyResource(RangerPolicyResource policyResource) { + this.policyResource = policyResource; + } + + @Override + public void init() { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerAbstractResourceMatcher.init()"); + } + + Map options = resourceDef != null ? resourceDef.getMatcherOptions() : null; + + optIgnoreCase = getOptionIgnoreCase(options); + optQuotedCaseSensitive = getOptionQuotedCaseSensitive(options); + optQuoteChars = getOptionQuoteChars(options); + optWildCard = getOptionWildCard(options); + + policyValues = new ArrayList<>(); + policyIsExcludes = policyResource != null && policyResource.getIsExcludes(); + + if (policyResource != null && policyResource.getValues() != null) { + for (String policyValue : policyResource.getValues()) { + if (StringUtils.isEmpty(policyValue)) { + continue; + } + policyValues.add(policyValue); + } + } + + optReplaceTokens = getOptionReplaceTokens(options); + + if(optReplaceTokens) { + startDelimiterChar = getOptionDelimiterStart(options); + endDelimiterChar = getOptionDelimiterEnd(options); + escapeChar = getOptionDelimiterEscape(options); + tokenPrefix = getOptionDelimiterPrefix(options); + + if(escapeChar == startDelimiterChar || escapeChar == endDelimiterChar || + tokenPrefix.indexOf(escapeChar) != -1 || tokenPrefix.indexOf(startDelimiterChar) != -1 || + tokenPrefix.indexOf(endDelimiterChar) != -1) { + String resouceName = resourceDef == null ? "" : resourceDef.getName(); + + String msg = "Invalid token-replacement parameters for resource '" + resouceName + "': { "; + msg += (OPTION_TOKEN_DELIMITER_START + "='" + startDelimiterChar + "'; "); + msg += (OPTION_TOKEN_DELIMITER_END + "='" + endDelimiterChar + "'; "); + msg += (OPTION_TOKEN_DELIMITER_ESCAPE + "='" + escapeChar + "'; "); + msg += (OPTION_TOKEN_DELIMITER_PREFIX + "='" + tokenPrefix + "' }. "); + msg += "Token replacement disabled"; + + LOG.error(msg); + + optReplaceTokens = false; + } + } + + resourceMatchers = buildResourceMatchers(); + isMatchAny = resourceMatchers == null || CollectionUtils.isEmpty(resourceMatchers.getResourceMatchers()); + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerAbstractResourceMatcher.init()"); + } + } + + @Override + public boolean isMatchAny() { return isMatchAny; } + + public boolean getNeedsDynamicEval() { + return resourceMatchers != null && resourceMatchers.getNeedsDynamicEval(); + } + + public static boolean getOptionIgnoreCase(Map options) { + return ServiceDefUtil.getBooleanOption(options, OPTION_IGNORE_CASE, true); + } + + public static boolean getOptionQuotedCaseSensitive(Map options) { + return ServiceDefUtil.getBooleanOption(options, OPTION_QUOTED_CASE_SENSITIVE, false); + } + + public static String getOptionQuoteChars(Map options) { + return ServiceDefUtil.getOption(options, OPTION_QUOTE_CHARS, "\""); + } + + public static boolean getOptionWildCard(Map options) { + return ServiceDefUtil.getBooleanOption(options, OPTION_WILD_CARD, true); + } + + public static boolean getOptionReplaceTokens(Map options) { + return ServiceDefUtil.getBooleanOption(options, OPTION_REPLACE_TOKENS, true); + } + + public static char getOptionDelimiterStart(Map options) { + return ServiceDefUtil.getCharOption(options, OPTION_TOKEN_DELIMITER_START, '{'); + } + + public static char getOptionDelimiterEnd(Map options) { + return ServiceDefUtil.getCharOption(options, OPTION_TOKEN_DELIMITER_END, '}'); + } + + public static char getOptionDelimiterEscape(Map options) { + return ServiceDefUtil.getCharOption(options, OPTION_TOKEN_DELIMITER_ESCAPE, '\\'); + } + + public static String getOptionDelimiterPrefix(Map options) { + return ServiceDefUtil.getOption(options, OPTION_TOKEN_DELIMITER_PREFIX, ""); + } + + protected ResourceMatcherWrapper buildResourceMatchers() { + List resourceMatchers = new ArrayList<>(); + boolean needsDynamicEval = false; + + for (String policyValue : policyValues) { + ResourceMatcher matcher = getMatcher(policyValue); + + if (matcher != null) { + if (matcher.isMatchAny()) { + resourceMatchers.clear(); + break; + } + if (!needsDynamicEval && matcher.getNeedsDynamicEval()) { + needsDynamicEval = true; + } + resourceMatchers.add(matcher); + } + } + + Collections.sort(resourceMatchers, new ResourceMatcher.PriorityComparator()); + + return CollectionUtils.isNotEmpty(resourceMatchers) ? + new ResourceMatcherWrapper(needsDynamicEval, resourceMatchers) : null; + } + + @Override + public boolean isCompleteMatch(String resource, Map evalContext) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerAbstractResourceMatcher.isCompleteMatch(" + resource + ", " + evalContext + ")"); + } + + boolean ret = false; + + if(CollectionUtils.isEmpty(policyValues)) { + ret = StringUtils.isEmpty(resource); + } else if(policyValues.size() == 1) { + String policyValue = policyValues.get(0); + + if(isMatchAny) { + ret = StringUtils.isEmpty(resource) || StringUtils.containsOnly(resource, WILDCARD_ASTERISK); + } else { + ret = optIgnoreCase && !(optQuotedCaseSensitive && ResourceMatcher.startsWithAnyChar(resource, optQuoteChars)) ? StringUtils.equalsIgnoreCase(resource, policyValue) : StringUtils.equals(resource, policyValue); + } + + if(policyIsExcludes) { + ret = !ret; + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerAbstractResourceMatcher.isCompleteMatch(" + resource + ", " + evalContext + "): " + ret); + } + + return ret; + } + + @Override + public String toString( ) { + StringBuilder sb = new StringBuilder(); + + toString(sb); + + return sb.toString(); + } + + public StringBuilder toString(StringBuilder sb) { + sb.append("RangerAbstractResourceMatcher={"); + + sb.append("resourceDef={"); + if(resourceDef != null) { + resourceDef.toString(sb); + } + sb.append("} "); + sb.append("policyResource={"); + if(policyResource != null) { + policyResource.toString(sb); + } + sb.append("} "); + sb.append("optIgnoreCase={").append(optIgnoreCase).append("} "); + sb.append("optQuotedCaseSensitive={").append(optQuotedCaseSensitive).append("} "); + sb.append("optQuoteChars={").append(optQuoteChars).append("} "); + sb.append("optWildCard={").append(optWildCard).append("} "); + + sb.append("policyValues={"); + if(policyValues != null) { + for(String value : policyValues) { + sb.append(value).append(","); + } + } + sb.append("} "); + + sb.append("policyIsExcludes={").append(policyIsExcludes).append("} "); + sb.append("isMatchAny={").append(isMatchAny).append("} "); + + sb.append("options={"); + if(resourceDef != null && resourceDef.getMatcherOptions() != null) { + for(Map.Entry e : resourceDef.getMatcherOptions().entrySet()) { + sb.append(e.getKey()).append("=").append(e.getValue()).append(';'); + } + } + sb.append("} "); + + sb.append("}"); + + return sb; + } + + boolean isAllValuesRequested(Object resource) { + final boolean result; + + if (resource == null) { + result = true; + } else if (resource instanceof String) { + result = StringUtils.isEmpty((String) resource) || WILDCARD_ASTERISK.equals(resource); + } else { // return false for any other type of resourceValue + result = false; + } + + if (LOG.isDebugEnabled()) { + LOG.debug("isAllValuesRequested(" + resource + "): " + result); + } + return result; + } + + /** + * The only case where excludes flag does NOT change the result is the following: + * - Resource denotes all possible values (i.e. resource in (null, "", "*") + * - where as policy does not allow all possible values (i.e. policy.values().contains("*") + * + */ + public boolean applyExcludes(boolean allValuesRequested, boolean resultWithoutExcludes) { + if (!policyIsExcludes) return resultWithoutExcludes; // not an excludes policy! + if (allValuesRequested && !isMatchAny) return resultWithoutExcludes; // one case where excludes has no effect + return !resultWithoutExcludes; // all other cases flip it + } + + ResourceMatcher getMatcher(String policyValue) { + final int len = policyValue != null ? policyValue.length() : 0; + + if (len == 0) { + return null; + } + + final ResourceMatcher ret; + + int wildcardStartIdx = -1; + int wildcardEndIdx = -1; + boolean needWildcardMatch = false; + + // If optWildcard is true + // If ('?' found or non-contiguous '*'s found in policyValue) + // needWildcardMatch = true + // End + // + // wildcardStartIdx is set to index of first '*' in policyValue or -1 if '*' is not found in policyValue, and + // wildcardEndIdx is set to index of last '*' in policyValue or -1 if '*' is not found in policyValue + // Else + // needWildcardMatch is set to false + // End + if (optWildCard) { + for (int i = 0; i < len; i++) { + final char c = policyValue.charAt(i); + + if (c == '?') { + needWildcardMatch = true; + break; + } else if (c == '*') { + if (wildcardEndIdx == -1 || wildcardEndIdx == (i - 1)) { + wildcardEndIdx = i; + if (wildcardStartIdx == -1) { + wildcardStartIdx = i; + } + } else { + needWildcardMatch = true; + break; + } + } + } + } + + if (needWildcardMatch) { // test?, test*a*, test*a*b, *test*a + ret = optIgnoreCase ? (optQuotedCaseSensitive ? new QuotedCaseSensitiveWildcardMatcher(policyValue, optQuoteChars) : new CaseInsensitiveWildcardMatcher(policyValue)) : new CaseSensitiveWildcardMatcher(policyValue); + } else if (wildcardStartIdx == -1) { // test, testa, testab + ret = optIgnoreCase ? (optQuotedCaseSensitive ? new QuotedCaseSensitiveStringMatcher(policyValue, optQuoteChars) : new CaseInsensitiveStringMatcher(policyValue)) : new CaseSensitiveStringMatcher(policyValue); + } else if (wildcardStartIdx == 0) { // *test, **test, *testa, *testab + String matchStr = policyValue.substring(wildcardEndIdx + 1); + ret = optIgnoreCase ? (optQuotedCaseSensitive ? new QuotedCaseSensitiveEndsWithMatcher(matchStr, optQuoteChars) : new CaseInsensitiveEndsWithMatcher(matchStr)) : new CaseSensitiveEndsWithMatcher(matchStr); + } else if (wildcardEndIdx != (len - 1)) { // test*a, test*ab + ret = optIgnoreCase ? (optQuotedCaseSensitive ? new QuotedCaseSensitiveWildcardMatcher(policyValue, optQuoteChars) : new CaseInsensitiveWildcardMatcher(policyValue)) : new CaseSensitiveWildcardMatcher(policyValue); + } else { // test*, test**, testa*, testab* + String matchStr = policyValue.substring(0, wildcardStartIdx); + ret = optIgnoreCase ? (optQuotedCaseSensitive ? new QuotedCaseSensitiveStartsWithMatcher(matchStr, optQuoteChars) : new CaseInsensitiveStartsWithMatcher(matchStr)) : new CaseSensitiveStartsWithMatcher(matchStr); + } + + if(optReplaceTokens) { + ret.setDelimiters(startDelimiterChar, endDelimiterChar, escapeChar, tokenPrefix); + } + + return ret; + } +} + +final class CaseSensitiveStringMatcher extends ResourceMatcher { + CaseSensitiveStringMatcher(String value) { + super(value); + } + + @Override + boolean isMatch(String resourceValue, Map evalContext) { + return StringUtils.equals(resourceValue, getExpandedValue(evalContext)); + } + int getPriority() { return 1 + (getNeedsDynamicEval() ? DYNAMIC_EVALUATION_PENALTY : 0);} +} + +final class CaseInsensitiveStringMatcher extends ResourceMatcher { + CaseInsensitiveStringMatcher(String value) { super(value); } + + @Override + boolean isMatch(String resourceValue, Map evalContext) { + return StringUtils.equalsIgnoreCase(resourceValue, getExpandedValue(evalContext)); + } + int getPriority() {return 2 + (getNeedsDynamicEval() ? DYNAMIC_EVALUATION_PENALTY : 0); } +} + +final class QuotedCaseSensitiveStringMatcher extends ResourceMatcher { + private final String quoteChars; + + QuotedCaseSensitiveStringMatcher(String value, String quoteChars) { + super(value); + + this.quoteChars = quoteChars; + } + + @Override + boolean isMatch(String resourceValue, Map evalContext) { + if (startsWithAnyChar(resourceValue, quoteChars)) { + return StringUtils.equals(resourceValue, getExpandedValue(evalContext)); + } else { + return StringUtils.equalsIgnoreCase(resourceValue, getExpandedValue(evalContext)); + } + } + + int getPriority() {return 2 + (getNeedsDynamicEval() ? DYNAMIC_EVALUATION_PENALTY : 0); } +} + +final class CaseSensitiveStartsWithMatcher extends ResourceMatcher { + CaseSensitiveStartsWithMatcher(String value) { + super(value); + } + + @Override + boolean isMatch(String resourceValue, Map evalContext) { + return StringUtils.startsWith(resourceValue, getExpandedValue(evalContext)); + } + int getPriority() { return 3 + (getNeedsDynamicEval() ? DYNAMIC_EVALUATION_PENALTY : 0);} +} + +final class CaseInsensitiveStartsWithMatcher extends ResourceMatcher { + CaseInsensitiveStartsWithMatcher(String value) { super(value); } + + @Override + boolean isMatch(String resourceValue, Map evalContext) { + return StringUtils.startsWithIgnoreCase(resourceValue, getExpandedValue(evalContext)); + } + int getPriority() { return 4 + (getNeedsDynamicEval() ? DYNAMIC_EVALUATION_PENALTY : 0); } +} + +final class QuotedCaseSensitiveStartsWithMatcher extends ResourceMatcher { + private final String quoteChars; + + QuotedCaseSensitiveStartsWithMatcher(String value, String quoteChars) { + super(value); + + this.quoteChars = quoteChars; + } + + @Override + boolean isMatch(String resourceValue, Map evalContext) { + if (startsWithAnyChar(resourceValue, quoteChars)) { + return StringUtils.startsWith(resourceValue, getExpandedValue(evalContext)); + } else { + return StringUtils.startsWithIgnoreCase(resourceValue, getExpandedValue(evalContext)); + } + } + + int getPriority() { return 4 + (getNeedsDynamicEval() ? DYNAMIC_EVALUATION_PENALTY : 0); } +} + +final class CaseSensitiveEndsWithMatcher extends ResourceMatcher { + CaseSensitiveEndsWithMatcher(String value) { + super(value); + } + + @Override + boolean isMatch(String resourceValue, Map evalContext) { + return StringUtils.endsWith(resourceValue, getExpandedValue(evalContext)); + } + int getPriority() { return 3 + (getNeedsDynamicEval() ? DYNAMIC_EVALUATION_PENALTY : 0); } +} + +final class CaseInsensitiveEndsWithMatcher extends ResourceMatcher { + CaseInsensitiveEndsWithMatcher(String value) { + super(value); + } + + @Override + boolean isMatch(String resourceValue, Map evalContext) { + return StringUtils.endsWithIgnoreCase(resourceValue, getExpandedValue(evalContext)); + } + int getPriority() { return 4 + (getNeedsDynamicEval() ? DYNAMIC_EVALUATION_PENALTY : 0); } +} + +final class QuotedCaseSensitiveEndsWithMatcher extends ResourceMatcher { + private final String quoteChars; + + QuotedCaseSensitiveEndsWithMatcher(String value, String quoteChars) { + super(value); + + this.quoteChars = quoteChars; + } + + @Override + boolean isMatch(String resourceValue, Map evalContext) { + if (startsWithAnyChar(resourceValue, quoteChars)) { + return StringUtils.endsWith(resourceValue, getExpandedValue(evalContext)); + } else { + return StringUtils.endsWithIgnoreCase(resourceValue, getExpandedValue(evalContext)); + } + } + + int getPriority() { return 4 + (getNeedsDynamicEval() ? DYNAMIC_EVALUATION_PENALTY : 0); } +} + +final class CaseSensitiveWildcardMatcher extends ResourceMatcher { + CaseSensitiveWildcardMatcher(String value) { + super(value); + } + + @Override + boolean isMatch(String resourceValue, Map evalContext) { + return FilenameUtils.wildcardMatch(resourceValue, getExpandedValue(evalContext), IOCase.SENSITIVE); + } + int getPriority() { return 5 + (getNeedsDynamicEval() ? DYNAMIC_EVALUATION_PENALTY : 0); } +} + + +final class CaseInsensitiveWildcardMatcher extends ResourceMatcher { + CaseInsensitiveWildcardMatcher(String value) { + super(value); + } + + @Override + boolean isMatch(String resourceValue, Map evalContext) { + return FilenameUtils.wildcardMatch(resourceValue, getExpandedValue(evalContext), IOCase.INSENSITIVE); + } + int getPriority() {return 6 + (getNeedsDynamicEval() ? DYNAMIC_EVALUATION_PENALTY : 0); } +} + +final class QuotedCaseSensitiveWildcardMatcher extends ResourceMatcher { + private final String quoteChars; + + QuotedCaseSensitiveWildcardMatcher(String value, String quoteChars) { + super(value); + + this.quoteChars = quoteChars; + } + + @Override + boolean isMatch(String resourceValue, Map evalContext) { + IOCase caseSensitivity = startsWithAnyChar(resourceValue, quoteChars) ? IOCase.SENSITIVE : IOCase.INSENSITIVE; + + return FilenameUtils.wildcardMatch(resourceValue, getExpandedValue(evalContext), caseSensitivity); + } + + int getPriority() {return 6 + (getNeedsDynamicEval() ? DYNAMIC_EVALUATION_PENALTY : 0); } +} + +final class ResourceMatcherWrapper { + private final boolean needsDynamicEval; + private final List resourceMatchers; + + ResourceMatcherWrapper() { + this(false, null); + } + + ResourceMatcherWrapper(boolean needsDynamicEval, List resourceMatchers) { + this.needsDynamicEval = needsDynamicEval; + this.resourceMatchers = resourceMatchers; + } + + boolean getNeedsDynamicEval() { + return needsDynamicEval; + } + + List getResourceMatchers() { + return resourceMatchers; + } +} + diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/resourcematcher/RangerDefaultResourceMatcher.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/resourcematcher/RangerDefaultResourceMatcher.java new file mode 100644 index 00000000000..a9be0dc6612 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/resourcematcher/RangerDefaultResourceMatcher.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.resourcematcher; + + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import java.util.Collection; +import java.util.Map; + + +public class RangerDefaultResourceMatcher extends RangerAbstractResourceMatcher { + private static final Log LOG = LogFactory.getLog(RangerDefaultResourceMatcher.class); + + @Override + public boolean isMatch(Object resource, Map evalContext) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerDefaultResourceMatcher.isMatch(" + resource + ", " + evalContext + ")"); + } + + boolean ret = false; + boolean allValuesRequested = isAllValuesRequested(resource); + + if(allValuesRequested || isMatchAny) { + ret = isMatchAny; + } else { + if (resource instanceof String) { + String strValue = (String) resource; + + for (ResourceMatcher resourceMatcher : resourceMatchers.getResourceMatchers()) { + ret = resourceMatcher.isMatch(strValue, evalContext); + if (ret) { + break; + } + } + } else if (resource instanceof Collection) { + @SuppressWarnings("unchecked") + Collection collValue = (Collection) resource; + + for (ResourceMatcher resourceMatcher : resourceMatchers.getResourceMatchers()) { + ret = resourceMatcher.isMatchAny(collValue, evalContext); + if (ret) { + break; + } + } + } + + } + + ret = applyExcludes(allValuesRequested, ret); + + if (ret == false) { + if(LOG.isDebugEnabled()) { + StringBuilder sb = new StringBuilder(); + sb.append("["); + for (String policyValue: policyValues) { + sb.append(policyValue); + sb.append(" "); + } + sb.append("]"); + + LOG.debug("RangerDefaultResourceMatcher.isMatch returns FALSE, (resource=" + resource + ", policyValues=" + sb.toString() + ")"); + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerDefaultResourceMatcher.isMatch(" + resource + ", " + evalContext + "): " + ret); + } + + return ret; + } + + public StringBuilder toString(StringBuilder sb) { + sb.append("RangerDefaultResourceMatcher={"); + + super.toString(sb); + + sb.append("}"); + + return sb; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/resourcematcher/RangerPathResourceMatcher.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/resourcematcher/RangerPathResourceMatcher.java new file mode 100644 index 00000000000..1c2fd8b2d31 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/resourcematcher/RangerPathResourceMatcher.java @@ -0,0 +1,450 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.resourcematcher; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.io.IOCase; +import org.apache.commons.lang.ArrayUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.plugin.policyengine.RangerAccessRequest; +import org.apache.atlas.plugin.util.ServiceDefUtil; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.function.BiFunction; + + +public class RangerPathResourceMatcher extends RangerDefaultResourceMatcher { + private static final Log LOG = LogFactory.getLog(RangerPathResourceMatcher.class); + + public static final String OPTION_PATH_SEPARATOR = "pathSeparatorChar"; + public static final char DEFAULT_PATH_SEPARATOR_CHAR = org.apache.hadoop.fs.Path.SEPARATOR_CHAR; + + private boolean policyIsRecursive; + private Character pathSeparatorChar = '/'; + + @Override + public void init() { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerPathResourceMatcher.init()"); + } + + Map options = resourceDef == null ? null : resourceDef.getMatcherOptions(); + + policyIsRecursive = policyResource != null && policyResource.getIsRecursive(); + pathSeparatorChar = ServiceDefUtil.getCharOption(options, OPTION_PATH_SEPARATOR, DEFAULT_PATH_SEPARATOR_CHAR); + + super.init(); + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerPathResourceMatcher.init()"); + } + } + + @Override + + protected ResourceMatcherWrapper buildResourceMatchers() { + List resourceMatchers = new ArrayList<>(); + boolean needsDynamicEval = false; + + for (String policyValue : policyValues) { + if (optWildCard && policyIsRecursive) { + if (policyValue.charAt(policyValue.length() - 1) == pathSeparatorChar) { + policyValue += WILDCARD_ASTERISK; + } + } + + ResourceMatcher matcher = getMatcher(policyValue); + + if (matcher != null) { + if (matcher.isMatchAny()) { + resourceMatchers.clear(); + break; + } + if (!needsDynamicEval && matcher.getNeedsDynamicEval()) { + needsDynamicEval = true; + } + resourceMatchers.add(matcher); + } + } + + Collections.sort(resourceMatchers, new ResourceMatcher.PriorityComparator()); + + return CollectionUtils.isNotEmpty(resourceMatchers) ? + new ResourceMatcherWrapper(needsDynamicEval, resourceMatchers) : null; + } + + @Override + ResourceMatcher getMatcher(String policyValue) { + if (!policyIsRecursive) { + return getPathMatcher(policyValue); + } + + final int len = policyValue != null ? policyValue.length() : 0; + + if (len == 0) { + return null; + } + + // To ensure that when policyValue is single '*', ResourceMatcher created here returns true for isMatchAny() + if (optWildCard && WILDCARD_ASTERISK.equals(policyValue)) { + return new CaseInsensitiveStringMatcher(""); + } + + boolean isWildcardPresent = false; + + if (optWildCard) { + for (int i = 0; i < len; i++) { + final char c = policyValue.charAt(i); + + if (c == '?' || c == '*') { + isWildcardPresent = true; + break; + } + } + } + + final ResourceMatcher ret; + + if (isWildcardPresent) { + ret = new RecursiveWildcardResourceMatcher(policyValue, pathSeparatorChar, optIgnoreCase, RangerPathResourceMatcher::isRecursiveWildCardMatch, optIgnoreCase ? 8 : 7); + } else { + ret = new RecursivePathResourceMatcher(policyValue, pathSeparatorChar, optIgnoreCase ? StringUtils::equalsIgnoreCase : StringUtils::equals, optIgnoreCase ? StringUtils::startsWithIgnoreCase : StringUtils::startsWith, optIgnoreCase ? 8 : 7); + } + + if (optReplaceTokens) { + ret.setDelimiters(startDelimiterChar, endDelimiterChar, escapeChar, tokenPrefix); + } + + return ret; + } + + static boolean isRecursiveWildCardMatch(String pathToCheck, String wildcardPath, Character pathSeparatorChar, IOCase caseSensitivity) { + + boolean ret = false; + + if (! StringUtils.isEmpty(pathToCheck)) { + String[] pathElements = StringUtils.split(pathToCheck, pathSeparatorChar); + + if(! ArrayUtils.isEmpty(pathElements)) { + StringBuilder sb = new StringBuilder(); + + if(pathToCheck.charAt(0) == pathSeparatorChar) { + sb.append(pathSeparatorChar); // preserve the initial pathSeparatorChar + } + + for(String p : pathElements) { + sb.append(p); + + ret = FilenameUtils.wildcardMatch(sb.toString(), wildcardPath, caseSensitivity); + + if (ret) { + break; + } + + sb.append(pathSeparatorChar); + } + + sb = null; + } else { // pathToCheck consists of only pathSeparatorChar + ret = FilenameUtils.wildcardMatch(pathToCheck, wildcardPath, caseSensitivity); + } + } + return ret; + } + + public StringBuilder toString(StringBuilder sb) { + sb.append("RangerPathResourceMatcher={"); + + super.toString(sb); + + sb.append("policyIsRecursive={").append(policyIsRecursive).append("} "); + + sb.append("}"); + + return sb; + } + + private ResourceMatcher getPathMatcher(String policyValue) { + final int len = policyValue != null ? policyValue.length() : 0; + + if (len == 0) { + return null; + } + + final ResourceMatcher ret; + + int wildcardStartIdx = -1; + int wildcardEndIdx = -1; + boolean needWildcardMatch = false; + + // If optWildcard is true + // If ('?' found or non-contiguous '*'s found in policyValue) + // needWildcardMatch = true + // End + // + // wildcardStartIdx is set to index of first '*' in policyValue or -1 if '*' is not found in policyValue, and + // wildcardEndIdx is set to index of last '*' in policyValue or -1 if '*' is not found in policyValue + // Else + // needWildcardMatch is set to false + // End + if (optWildCard) { + for (int i = 0; i < len; i++) { + final char c = policyValue.charAt(i); + + if (c == '?') { + needWildcardMatch = true; + break; + } else if (c == '*') { + if (wildcardEndIdx == -1 || wildcardEndIdx == (i - 1)) { + wildcardEndIdx = i; + if (wildcardStartIdx == -1) { + wildcardStartIdx = i; + } + } else { + needWildcardMatch = true; + break; + } + } + } + } + + if (needWildcardMatch) { // test?, test*a*, test*a*b, *test*a + ret = new WildcardResourceMatcher(policyValue, pathSeparatorChar, optIgnoreCase, FilenameUtils::wildcardMatch, 6); + } else if (wildcardStartIdx == -1) { // test, testa, testab + ret = new StringResourceMatcher(policyValue, pathSeparatorChar, optIgnoreCase ? StringUtils::equalsIgnoreCase : StringUtils::equals, optIgnoreCase ? 2 : 1); + } else if (wildcardStartIdx == 0) { // *test, **test, *testa, *testab + String matchStr = policyValue.substring(wildcardEndIdx + 1); + ret = new StringResourceMatcher(matchStr, pathSeparatorChar, optIgnoreCase ? StringUtils::endsWithIgnoreCase : StringUtils::endsWith, optIgnoreCase ? 4 : 3); + } else if (wildcardEndIdx != (len - 1)) { // test*a, test*ab + ret = new WildcardResourceMatcher(policyValue, pathSeparatorChar, optIgnoreCase, FilenameUtils::wildcardMatch, 6); + } else { // test*, test**, testa*, testab* + String matchStr = policyValue.substring(0, wildcardStartIdx); + ret = new StringResourceMatcher(matchStr, pathSeparatorChar, optIgnoreCase ? StringUtils::startsWithIgnoreCase : StringUtils::startsWith, optIgnoreCase ? 4 : 3); + } + + if (optReplaceTokens) { + ret.setDelimiters(startDelimiterChar, endDelimiterChar, escapeChar, tokenPrefix); + } + + return ret; + } + + interface TriFunction { + R apply(T t, U u, V v); + } + + interface QuadFunction { + R apply(T t, U u, V v, W w); + } + + static abstract class PathResourceMatcher extends ResourceMatcher { + final char pathSeparatorChar; + final int priority; + + PathResourceMatcher(String value, char pathSeparatorChar, int priority) { + super(value); + this.pathSeparatorChar = pathSeparatorChar; + this.priority = priority; + } + int getPriority() { + return priority + (getNeedsDynamicEval() ? DYNAMIC_EVALUATION_PENALTY : 0); + } + } + + static class StringResourceMatcher extends PathResourceMatcher { + final BiFunction function; + StringResourceMatcher(String value, char pathSeparatorChar, BiFunction function, int priority) { + super(value, pathSeparatorChar, priority); + this.function = function; + } + @Override + boolean isMatch(String resourceValue, Map evalContext) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> StringResourceMatcher.isMatch(resourceValue=" + resourceValue + ", evalContext=" + evalContext + ")"); + } + String expandedValue = getExpandedValue(evalContext); + boolean ret = function.apply(resourceValue, expandedValue); + if (!ret) { + RangerAccessRequest.ResourceMatchingScope scope = MapUtils.isNotEmpty(evalContext) ? (RangerAccessRequest.ResourceMatchingScope) evalContext.get(RangerAccessRequest.RANGER_ACCESS_REQUEST_SCOPE_STRING) : null; + if (scope == RangerAccessRequest.ResourceMatchingScope.SELF_OR_CHILD) { + int lastLevelSeparatorIndex = expandedValue.lastIndexOf(pathSeparatorChar); + if (lastLevelSeparatorIndex != -1) { + String shorterExpandedValue = expandedValue.substring(0, lastLevelSeparatorIndex); + if (resourceValue.charAt(resourceValue.length()-1) == pathSeparatorChar) { + resourceValue = resourceValue.substring(0, resourceValue.length()-1); + } + ret = function.apply(resourceValue, shorterExpandedValue); + } + } + } + if (LOG.isDebugEnabled()) { + LOG.debug("<== StringResourceMatcher.isMatch(resourceValue=" + resourceValue + ", expandedValue=" + expandedValue + ") : result:[" + ret + "]"); + } + return ret; + } + + } + + static class WildcardResourceMatcher extends PathResourceMatcher { + final TriFunction function; + final IOCase ioCase; + + WildcardResourceMatcher(String value, char pathSeparatorChar, boolean optIgnoreCase, TriFunction function, int priority) { + super(value, pathSeparatorChar, priority); + this.function = function; + this.ioCase = optIgnoreCase ? IOCase.INSENSITIVE : IOCase.SENSITIVE; + } + @Override + boolean isMatch(String resourceValue, Map evalContext) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> WildcardResourceMatcher.isMatch(resourceValue=" + resourceValue + ", evalContext=" + evalContext + ")"); + } + String expandedValue = getExpandedValue(evalContext); + boolean ret = function.apply(resourceValue, expandedValue, ioCase); + if (!ret) { + RangerAccessRequest.ResourceMatchingScope scope = MapUtils.isNotEmpty(evalContext) ? (RangerAccessRequest.ResourceMatchingScope) evalContext.get(RangerAccessRequest.RANGER_ACCESS_REQUEST_SCOPE_STRING) : null; + if (scope == RangerAccessRequest.ResourceMatchingScope.SELF_OR_CHILD) { + int lastLevelSeparatorIndex = expandedValue.lastIndexOf(pathSeparatorChar); + if (lastLevelSeparatorIndex != -1) { + String shorterExpandedValue = expandedValue.substring(0, lastLevelSeparatorIndex); + if (resourceValue.charAt(resourceValue.length()-1) == pathSeparatorChar) { + resourceValue = resourceValue.substring(0, resourceValue.length()-1); + } + ret = function.apply(resourceValue, shorterExpandedValue, ioCase); + } + } + } + if (LOG.isDebugEnabled()) { + LOG.debug("<== WildcardResourceMatcher.isMatch(resourceValue=" + resourceValue + ", expandedValue=" + expandedValue + ") : result:[" + ret + "]"); + } + return ret; + } + } + + static class RecursiveWildcardResourceMatcher extends PathResourceMatcher { + final QuadFunction function; + final IOCase ioCase; + + RecursiveWildcardResourceMatcher(String value, char pathSeparatorChar, boolean optIgnoreCase, QuadFunction function, int priority) { + super(value, pathSeparatorChar, priority); + this.function = function; + this.ioCase = optIgnoreCase ? IOCase.INSENSITIVE : IOCase.SENSITIVE; + } + @Override + boolean isMatch(String resourceValue, Map evalContext) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RecursiveWildcardResourceMatcher.isMatch(resourceValue=" + resourceValue + ", evalContext=" + evalContext + ")"); + } + String expandedValue = getExpandedValue(evalContext); + boolean ret = function.apply(resourceValue, expandedValue, pathSeparatorChar, ioCase); + if (!ret) { + RangerAccessRequest.ResourceMatchingScope scope = MapUtils.isNotEmpty(evalContext) ? (RangerAccessRequest.ResourceMatchingScope) evalContext.get(RangerAccessRequest.RANGER_ACCESS_REQUEST_SCOPE_STRING) : null; + if (scope == RangerAccessRequest.ResourceMatchingScope.SELF_OR_CHILD) { + int lastLevelSeparatorIndex = expandedValue.lastIndexOf(pathSeparatorChar); + if (lastLevelSeparatorIndex != -1) { + String shorterExpandedValue = expandedValue.substring(0, lastLevelSeparatorIndex); + if (resourceValue.charAt(resourceValue.length()-1) == pathSeparatorChar) { + resourceValue = resourceValue.substring(0, resourceValue.length()-1); + } + ret = function.apply(resourceValue, shorterExpandedValue, pathSeparatorChar, ioCase); + } + } + } + if (LOG.isDebugEnabled()) { + LOG.debug("<== RecursiveWildcardResourceMatcher.isMatch(resourceValue=" + resourceValue + ", expandedValue=" + expandedValue + ") : result:[" + ret + "]"); + } + return ret; + } + } + + static class RecursivePathResourceMatcher extends PathResourceMatcher { + String valueWithoutSeparator; + String valueWithSeparator; + + final BiFunction primaryFunction; + final BiFunction fallbackFunction; + + RecursivePathResourceMatcher(String value, char pathSeparatorChar, BiFunction primaryFunction, BiFunction fallbackFunction, int priority) { + super(value, pathSeparatorChar, priority); + this.primaryFunction = primaryFunction; + this.fallbackFunction = fallbackFunction; + } + + String getStringToCompare(String policyValue) { + if (StringUtils.isEmpty(policyValue)) { + return policyValue; + } + return (policyValue.lastIndexOf(pathSeparatorChar) == policyValue.length() - 1) ? + policyValue.substring(0, policyValue.length() - 1) : policyValue; + } + + @Override + boolean isMatch(String resourceValue, Map evalContext) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RecursivePathResourceMatcher.isMatch(resourceValue=" + resourceValue + ", evalContext=" + evalContext + ")"); + } + final String noSeparator; + if (getNeedsDynamicEval()) { + String expandedPolicyValue = getExpandedValue(evalContext); + noSeparator = expandedPolicyValue != null ? getStringToCompare(expandedPolicyValue) : null; + } else { + if (valueWithoutSeparator == null && value != null) { + valueWithoutSeparator = getStringToCompare(value); + valueWithSeparator = valueWithoutSeparator + pathSeparatorChar; + } + noSeparator = valueWithoutSeparator; + } + + boolean ret = primaryFunction.apply(resourceValue, noSeparator); + + if (!ret && noSeparator != null) { + final String withSeparator = getNeedsDynamicEval() ? noSeparator + pathSeparatorChar : valueWithSeparator; + ret = fallbackFunction.apply(resourceValue, withSeparator); + + if (!ret) { + RangerAccessRequest.ResourceMatchingScope scope = MapUtils.isNotEmpty(evalContext) ? (RangerAccessRequest.ResourceMatchingScope) evalContext.get(RangerAccessRequest.RANGER_ACCESS_REQUEST_SCOPE_STRING) : null; + if (scope == RangerAccessRequest.ResourceMatchingScope.SELF_OR_CHILD) { + final int lastLevelSeparatorIndex = noSeparator.lastIndexOf(pathSeparatorChar); + if (lastLevelSeparatorIndex != -1) { + final String shorterExpandedValue = noSeparator.substring(0, lastLevelSeparatorIndex); + if (resourceValue.charAt(resourceValue.length() - 1) == pathSeparatorChar) { + resourceValue = resourceValue.substring(0, resourceValue.length() - 1); + } + ret = primaryFunction.apply(resourceValue, shorterExpandedValue); + } + } + } + } + if (LOG.isDebugEnabled()) { + LOG.debug("<== RecursivePathResourceMatcher.isMatch(resourceValue=" + resourceValue + ", expandedValueWithoutTrailingSeparatorChar=" + noSeparator + ") : result:[" + ret + "]"); + } + + return ret; + } + } + +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/resourcematcher/RangerResourceMatcher.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/resourcematcher/RangerResourceMatcher.java new file mode 100644 index 00000000000..9562bdf0fb6 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/resourcematcher/RangerResourceMatcher.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.resourcematcher; + +import org.apache.atlas.plugin.model.RangerPolicy.RangerPolicyResource; +import org.apache.atlas.plugin.model.RangerServiceDef.RangerResourceDef; + +import java.util.Map; + +public interface RangerResourceMatcher { + void setResourceDef(RangerResourceDef resourceDef); + + void setPolicyResource(RangerPolicyResource policyResource); + + void init(); + + boolean isMatchAny(); + + boolean isMatch(Object resource, Map evalContext); + + boolean isCompleteMatch(String resource, Map evalContext); + + boolean getNeedsDynamicEval(); + +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/resourcematcher/RangerURLResourceMatcher.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/resourcematcher/RangerURLResourceMatcher.java new file mode 100644 index 00000000000..ddfda29667a --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/resourcematcher/RangerURLResourceMatcher.java @@ -0,0 +1,335 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.resourcematcher; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.io.IOCase; +import org.apache.commons.lang.ArrayUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.plugin.util.ServiceDefUtil; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + + +public class RangerURLResourceMatcher extends RangerDefaultResourceMatcher { + private static final Log LOG = LogFactory.getLog(RangerURLResourceMatcher.class); + + public static final String OPTION_PATH_SEPARATOR = "pathSeparatorChar"; + public static final char DEFAULT_PATH_SEPARATOR_CHAR = org.apache.hadoop.fs.Path.SEPARATOR_CHAR; + + boolean policyIsRecursive; + char pathSeparatorChar = DEFAULT_PATH_SEPARATOR_CHAR; + + @Override + public void init() { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerURLResourceMatcher.init()"); + } + + Map options = resourceDef == null ? null : resourceDef.getMatcherOptions(); + + policyIsRecursive = policyResource != null && policyResource.getIsRecursive(); + pathSeparatorChar = ServiceDefUtil.getCharOption(options, OPTION_PATH_SEPARATOR, DEFAULT_PATH_SEPARATOR_CHAR); + + super.init(); + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerURLResourceMatcher.init()"); + } + } + + @Override + + protected ResourceMatcherWrapper buildResourceMatchers() { + List resourceMatchers = new ArrayList<>(); + boolean needsDynamicEval = false; + + for (String policyValue : policyValues) { + if (optWildCard && policyIsRecursive) { + if (policyValue.charAt(policyValue.length() - 1) == pathSeparatorChar) { + policyValue += WILDCARD_ASTERISK; + } + } + + ResourceMatcher matcher = getMatcher(policyValue); + + if (matcher != null) { + if (matcher.isMatchAny()) { + resourceMatchers.clear(); + break; + } + if (!needsDynamicEval && matcher.getNeedsDynamicEval()) { + needsDynamicEval = true; + } + resourceMatchers.add(matcher); + } + } + + Collections.sort(resourceMatchers, new ResourceMatcher.PriorityComparator()); + + return CollectionUtils.isNotEmpty(resourceMatchers) ? + new ResourceMatcherWrapper(needsDynamicEval, resourceMatchers) : null; + } + + @Override + ResourceMatcher getMatcher(String policyValue) { + if(! policyIsRecursive) { + return super.getMatcher(policyValue); + } + + final int len = policyValue != null ? policyValue.length() : 0; + + if (len == 0) { + return null; + } + + // To ensure that when policyValue is single '*', ResourceMatcher created here returns true for isMatchAny() + if (optWildCard && WILDCARD_ASTERISK.equals(policyValue)) { + return new CaseInsensitiveStringMatcher(""); + } + + boolean isWildcardPresent = false; + + if (optWildCard) { + for (int i = 0; i < len; i++) { + final char c = policyValue.charAt(i); + + if (c == '?' || c == '*') { + isWildcardPresent = true; + break; + } + } + } + + final ResourceMatcher ret; + + if (isWildcardPresent) { + ret = optIgnoreCase ? new CaseInsensitiveURLRecursiveWildcardMatcher(policyValue, pathSeparatorChar) + : new CaseSensitiveURLRecursiveWildcardMatcher(policyValue, pathSeparatorChar); + } else { + ret = optIgnoreCase ? new CaseInsensitiveURLRecursiveMatcher(policyValue, pathSeparatorChar) : new CaseSensitiveURLRecursiveMatcher(policyValue, pathSeparatorChar); + } + + if (optReplaceTokens) { + ret.setDelimiters(startDelimiterChar, endDelimiterChar, escapeChar, tokenPrefix); + } + + return ret; + } + + static boolean isRecursiveWildCardMatch(String pathToCheck, String wildcardPath, char pathSeparatorChar, IOCase caseSensitivity) { + + boolean ret = false; + + String url = StringUtils.trim(pathToCheck); + + if (!StringUtils.isEmpty(url) && isPathURLType(url)) { + String scheme = getScheme(url); + if (StringUtils.isEmpty(scheme)) { + return ret; + } + + String path = getPathWithOutScheme(url); + + String[] pathElements = StringUtils.split(path, pathSeparatorChar); + + if (!ArrayUtils.isEmpty(pathElements)) { + StringBuilder sb = new StringBuilder(); + + sb.append(scheme); + + if (pathToCheck.charAt(0) == pathSeparatorChar) { + sb.append(pathSeparatorChar); // preserve the initial pathSeparatorChar + } + + for (String p : pathElements) { + sb.append(p); + + ret = FilenameUtils.wildcardMatch(sb.toString(), wildcardPath, caseSensitivity); + + if (ret) { + break; + } + + sb.append(pathSeparatorChar); + } + + sb = null; + } else { // pathToCheck consists of only pathSeparatorChar + ret = FilenameUtils.wildcardMatch(pathToCheck, wildcardPath, caseSensitivity); + } + } + + return ret; + } + + public StringBuilder toString(StringBuilder sb) { + sb.append("RangerURLResourceMatcher={"); + + super.toString(sb); + + sb.append("policyIsRecursive={").append(policyIsRecursive).append("} "); + + sb.append("}"); + + return sb; + } + + static boolean isPathURLType(String url) { + + Pattern p1 = Pattern.compile(":/{2}"); + Matcher m1 = p1.matcher(url); + + Pattern p2 = Pattern.compile(":/{3,}"); + Matcher m2 = p2.matcher(url); + + return (m1.find() && !(m2.find())); + } + + + static String getScheme(String url){ + return StringUtils.substring(url,0,(StringUtils.indexOf(url,":") + 3)); + } + + static String getPathWithOutScheme(String url) { + return StringUtils.substring(url,(StringUtils.indexOf(url,":") + 2)); + } +} + +final class CaseSensitiveURLRecursiveWildcardMatcher extends ResourceMatcher { + private final char levelSeparatorChar; + CaseSensitiveURLRecursiveWildcardMatcher(String value, char levelSeparatorChar) { + super(value); + this.levelSeparatorChar = levelSeparatorChar; + } + + @Override + boolean isMatch(String resourceValue, Map evalContext) { + return RangerURLResourceMatcher.isRecursiveWildCardMatch(resourceValue, getExpandedValue(evalContext), levelSeparatorChar, IOCase.SENSITIVE); + } + int getPriority() { return 7 + (getNeedsDynamicEval() ? DYNAMIC_EVALUATION_PENALTY : 0);} +} + +final class CaseInsensitiveURLRecursiveWildcardMatcher extends ResourceMatcher { + private final char levelSeparatorChar; + CaseInsensitiveURLRecursiveWildcardMatcher(String value, char levelSeparatorChar) { + super(value); + this.levelSeparatorChar = levelSeparatorChar; + } + + @Override + boolean isMatch(String resourceValue, Map evalContext) { + return RangerURLResourceMatcher.isRecursiveWildCardMatch(resourceValue, getExpandedValue(evalContext), levelSeparatorChar, IOCase.INSENSITIVE); + } + int getPriority() { return 8 + (getNeedsDynamicEval() ? DYNAMIC_EVALUATION_PENALTY : 0);} + +} + +abstract class RecursiveMatcher extends ResourceMatcher { + final char levelSeparatorChar; + String valueWithoutSeparator; + String valueWithSeparator; + + RecursiveMatcher(String value, char levelSeparatorChar) { + super(value); + this.levelSeparatorChar = levelSeparatorChar; + } + + String getStringToCompare(String policyValue) { + if (StringUtils.isEmpty(policyValue)) { + return policyValue; + } + return (policyValue.lastIndexOf(levelSeparatorChar) == policyValue.length() - 1) ? + policyValue.substring(0, policyValue.length() - 1) : policyValue; + } +} + +final class CaseSensitiveURLRecursiveMatcher extends RecursiveMatcher { + CaseSensitiveURLRecursiveMatcher(String value, char levelSeparatorChar) { + super(value, levelSeparatorChar); + } + + @Override + boolean isMatch(String resourceValue, Map evalContext) { + + final String noSeparator; + if (getNeedsDynamicEval()) { + String expandedPolicyValue = getExpandedValue(evalContext); + noSeparator = expandedPolicyValue != null ? getStringToCompare(expandedPolicyValue) : null; + } else { + if (valueWithoutSeparator == null && value != null) { + valueWithoutSeparator = getStringToCompare(value); + valueWithSeparator = valueWithoutSeparator + Character.toString(levelSeparatorChar); + } + noSeparator = valueWithoutSeparator; + } + + boolean ret = StringUtils.equals(resourceValue, noSeparator); + + if (!ret && noSeparator != null) { + final String withSeparator = getNeedsDynamicEval() ? noSeparator + Character.toString(levelSeparatorChar) : valueWithSeparator; + ret = StringUtils.startsWith(resourceValue, withSeparator); + } + + return ret; + } + int getPriority() { return 7 + (getNeedsDynamicEval() ? DYNAMIC_EVALUATION_PENALTY : 0);} +} + +final class CaseInsensitiveURLRecursiveMatcher extends RecursiveMatcher { + CaseInsensitiveURLRecursiveMatcher(String value, char levelSeparatorChar) { + super(value, levelSeparatorChar); + } + + @Override + boolean isMatch(String resourceValue, Map evalContext) { + + final String noSeparator; + if (getNeedsDynamicEval()) { + String expandedPolicyValue = getExpandedValue(evalContext); + noSeparator = expandedPolicyValue != null ? getStringToCompare(expandedPolicyValue) : null; + } else { + if (valueWithoutSeparator == null && value != null) { + valueWithoutSeparator = getStringToCompare(value); + valueWithSeparator = valueWithoutSeparator + Character.toString(levelSeparatorChar); + } + noSeparator = valueWithoutSeparator; + } + + boolean ret = StringUtils.equalsIgnoreCase(resourceValue, noSeparator); + + if (!ret && noSeparator != null) { + final String withSeparator = getNeedsDynamicEval() ? noSeparator + Character.toString(levelSeparatorChar) : valueWithSeparator; + ret = StringUtils.startsWithIgnoreCase(resourceValue, withSeparator); + } + + return ret; + } + + int getPriority() { return 8 + (getNeedsDynamicEval() ? DYNAMIC_EVALUATION_PENALTY : 0);} +} \ No newline at end of file diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/resourcematcher/ResourceMatcher.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/resourcematcher/ResourceMatcher.java new file mode 100644 index 00000000000..f23667a5f1b --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/resourcematcher/ResourceMatcher.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.resourcematcher; + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.plugin.util.StringTokenReplacer; + +import java.io.Serializable; +import java.util.Collection; +import java.util.Comparator; +import java.util.Map; + +abstract class ResourceMatcher { + private static final Log LOG = LogFactory.getLog(ResourceMatcher.class); + + protected final String value; + protected StringTokenReplacer tokenReplacer; + + static final int DYNAMIC_EVALUATION_PENALTY = 8; + + ResourceMatcher(String value) { this.value = value; } + + abstract boolean isMatch(String resourceValue, Map evalContext); + abstract int getPriority(); + + boolean isMatchAny() { return value != null && value.length() == 0; } + + boolean getNeedsDynamicEval() { + return tokenReplacer != null; + } + + public boolean isMatchAny(Collection resourceValues, Map evalContext) { + if (resourceValues != null) { + for (String resourceValue : resourceValues) { + if (isMatch(resourceValue, evalContext)) { + return true; + } + } + } + + return false; + } + + @Override + public String toString() { + return this.getClass().getName() + "(" + this.value + ")"; + } + + void setDelimiters(char startDelimiterChar, char endDelimiterChar, char escapeChar, String tokenPrefix) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> setDelimiters(value= " + value + ", startDelimiter=" + startDelimiterChar + + ", endDelimiter=" + endDelimiterChar + ", escapeChar=" + escapeChar + ", prefix=" + tokenPrefix); + } + + if(value != null && (value.indexOf(escapeChar) != -1 || (value.indexOf(startDelimiterChar) != -1 && value.indexOf(endDelimiterChar) != -1))) { + tokenReplacer = new StringTokenReplacer(startDelimiterChar, endDelimiterChar, escapeChar, tokenPrefix); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== setDelimiters(value= " + value + ", startDelimiter=" + startDelimiterChar + + ", endDelimiter=" + endDelimiterChar + ", escapeChar=" + escapeChar + ", prefix=" + tokenPrefix); + } + } + + String getExpandedValue(Map evalContext) { + final String ret; + + if(tokenReplacer != null) { + ret = tokenReplacer.replaceTokens(value, evalContext); + } else { + ret = value; + } + + return ret; + } + + public static boolean startsWithAnyChar(String value, String startChars) { + boolean ret = false; + + if (value != null && value.length() > 0 && startChars != null) { + ret = StringUtils.contains(startChars, value.charAt(0)); + } + + return ret; + } + + public static class PriorityComparator implements Comparator, Serializable { + @Override + public int compare(ResourceMatcher me, ResourceMatcher other) { + return Integer.compare(me.getPriority(), other.getPriority()); + } + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/resourcematcher/ScheduledTimeAlwaysMatcher.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/resourcematcher/ScheduledTimeAlwaysMatcher.java new file mode 100644 index 00000000000..01d52bed72b --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/resourcematcher/ScheduledTimeAlwaysMatcher.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.resourcematcher; + +public class ScheduledTimeAlwaysMatcher implements ScheduledTimeMatcher { + @Override + public boolean isMatch(int currentTime) { + return true; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/resourcematcher/ScheduledTimeExactMatcher.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/resourcematcher/ScheduledTimeExactMatcher.java new file mode 100644 index 00000000000..4160c605dca --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/resourcematcher/ScheduledTimeExactMatcher.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.resourcematcher; + +public class ScheduledTimeExactMatcher implements ScheduledTimeMatcher { + private int scheduledTime; + + public ScheduledTimeExactMatcher(int scheduledTime) { + this.scheduledTime = scheduledTime; + } + @Override + public boolean isMatch(int currentTime) { + return currentTime == scheduledTime; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/resourcematcher/ScheduledTimeMatcher.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/resourcematcher/ScheduledTimeMatcher.java new file mode 100644 index 00000000000..1553a1c4932 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/resourcematcher/ScheduledTimeMatcher.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.resourcematcher; + +public interface ScheduledTimeMatcher { + boolean isMatch(int targetTime); +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/resourcematcher/ScheduledTimeRangeMatcher.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/resourcematcher/ScheduledTimeRangeMatcher.java new file mode 100644 index 00000000000..06487e5d2fe --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/resourcematcher/ScheduledTimeRangeMatcher.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.resourcematcher; + +public class ScheduledTimeRangeMatcher implements ScheduledTimeMatcher { + private int lowerBound; + private int upperBound; + + public ScheduledTimeRangeMatcher(int lowerBound, int upperBound) { + this.lowerBound = lowerBound; + this.upperBound = upperBound; + } + @Override + public boolean isMatch(int currentTime) { + return currentTime >= lowerBound && currentTime <= upperBound; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerAuthContext.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerAuthContext.java new file mode 100644 index 00000000000..092c3455339 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerAuthContext.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.service; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.atlas.plugin.contextenricher.RangerContextEnricher; +import org.apache.atlas.plugin.policyengine.RangerPolicyEngine; +import org.apache.atlas.plugin.util.RangerRoles; +import org.apache.atlas.plugin.util.RangerRolesUtil; + +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +public class RangerAuthContext { + private final Map requestContextEnrichers; + private RangerRolesUtil rolesUtil; + + + public RangerAuthContext(Map requestContextEnrichers, RangerRoles roles) { + this.requestContextEnrichers = requestContextEnrichers != null ? requestContextEnrichers : new ConcurrentHashMap<>(); + + setRoles(roles); + } + + public Map getRequestContextEnrichers() { + return requestContextEnrichers; + } + + public void addOrReplaceRequestContextEnricher(RangerContextEnricher enricher, Object database) { + // concurrentHashMap does not allow null to be inserted into it, so insert a dummy which is checked + // when enrich() is called + requestContextEnrichers.put(enricher, database != null ? database : enricher); + } + + public void cleanupRequestContextEnricher(RangerContextEnricher enricher) { + requestContextEnrichers.remove(enricher); + } + + public void setRoles(RangerRoles roles) { + this.rolesUtil = roles != null ? new RangerRolesUtil(roles) : new RangerRolesUtil(null); + } + + public Set getRolesForUserAndGroups(String user, Set groups) { + RangerRolesUtil rolesUtil = this.rolesUtil; + Map> userRoleMapping = rolesUtil.getUserRoleMapping(); + Map> groupRoleMapping = rolesUtil.getGroupRoleMapping(); + Set allRoles = new HashSet<>(); + + if (MapUtils.isNotEmpty(userRoleMapping) && StringUtils.isNotEmpty(user)) { + Set userRoles = userRoleMapping.get(user); + + if (CollectionUtils.isNotEmpty(userRoles)) { + allRoles.addAll(userRoles); + } + } + + if (MapUtils.isNotEmpty(groupRoleMapping)) { + if (CollectionUtils.isNotEmpty(groups)) { + for (String group : groups) { + Set groupRoles = groupRoleMapping.get(group); + + if (CollectionUtils.isNotEmpty(groupRoles)) { + allRoles.addAll(groupRoles); + } + } + } + + Set publicGroupRoles = groupRoleMapping.get(RangerPolicyEngine.GROUP_PUBLIC); + + if (CollectionUtils.isNotEmpty(publicGroupRoles)) { + allRoles.addAll(publicGroupRoles); + } + } + + return allRoles; + } + + public long getRoleVersion() { return this.rolesUtil.getRoleVersion(); } + + public RangerRolesUtil getRangerRolesUtil() { + return this.rolesUtil; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerAuthContextListener.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerAuthContextListener.java new file mode 100644 index 00000000000..20f608418d1 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerAuthContextListener.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.service; + + +public interface RangerAuthContextListener { + void contextChanged(); +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerBasePlugin.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerBasePlugin.java new file mode 100644 index 00000000000..b224cccc7e1 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerBasePlugin.java @@ -0,0 +1,1235 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.service; + +import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.admin.client.RangerAdminClient; +import org.apache.atlas.admin.client.RangerAdminRESTClient; +import org.apache.atlas.audit.provider.AuditHandler; +import org.apache.atlas.audit.provider.AuditProviderFactory; +import org.apache.atlas.audit.provider.StandAloneAuditProviderFactory; +import org.apache.atlas.authorization.hadoop.config.RangerAuditConfig; +import org.apache.atlas.authorization.hadoop.config.RangerPluginConfig; +import org.apache.atlas.authorization.utils.StringUtil; +import org.apache.atlas.plugin.conditionevaluator.RangerScriptExecutionContext; +import org.apache.atlas.plugin.contextenricher.RangerContextEnricher; +import org.apache.atlas.plugin.contextenricher.RangerTagEnricher; +import org.apache.atlas.plugin.model.RangerPolicy; +import org.apache.atlas.plugin.model.RangerRole; +import org.apache.atlas.plugin.model.RangerServiceDef; +import org.apache.atlas.plugin.policyengine.RangerAccessRequest; +import org.apache.atlas.plugin.policyengine.RangerAccessRequestImpl; +import org.apache.atlas.plugin.policyengine.RangerAccessResourceImpl; +import org.apache.atlas.plugin.policyengine.RangerAccessResult; +import org.apache.atlas.plugin.policyengine.RangerAccessResultProcessor; +import org.apache.atlas.plugin.policyengine.RangerPluginContext; +import org.apache.atlas.plugin.policyengine.RangerPolicyEngine; +import org.apache.atlas.plugin.policyengine.RangerPolicyEngineImpl; +import org.apache.atlas.plugin.policyengine.RangerResourceACLs; +import org.apache.atlas.plugin.policyengine.RangerResourceAccessInfo; +import org.apache.atlas.plugin.policyevaluator.RangerPolicyEvaluator; +import org.apache.atlas.plugin.store.EmbeddedServiceDefsUtil; +import org.apache.atlas.plugin.util.*; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.Hashtable; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; + + +public class RangerBasePlugin { + private static final Log LOG = LogFactory.getLog(RangerBasePlugin.class); + + private final RangerPluginConfig pluginConfig; + private final RangerPluginContext pluginContext; + private final Map logHistoryList = new Hashtable<>(); + private final int logInterval = 30000; // 30 seconds + private final DownloadTrigger accessTrigger = new DownloadTrigger(); + private PolicyRefresher refresher; + private RangerPolicyEngine policyEngine; + private RangerAuthContext currentAuthContext; + private RangerAccessResultProcessor resultProcessor; + private RangerRoles roles; + private RangerUserStore userStore; + private final List chainedPlugins; + private AtlasTypeRegistry typeRegistry = null; + + + public RangerBasePlugin(String serviceType, String appId) { + this(new RangerPluginConfig(serviceType, null, appId, null, null, null)); + } + + public RangerBasePlugin(String serviceType, String serviceName, String appId) { + this(new RangerPluginConfig(serviceType, serviceName, appId, null, null, null)); + } + + public RangerBasePlugin(String serviceType, String serviceName, AtlasTypeRegistry typeRegistry) { + this(new RangerPluginConfig(serviceType, serviceName, null, null, null, null)); + this.typeRegistry = typeRegistry; + } + + public RangerBasePlugin(RangerPluginConfig pluginConfig) { + this.pluginConfig = pluginConfig; + this.pluginContext = new RangerPluginContext(pluginConfig); + + Set superUsers = toSet(pluginConfig.get(pluginConfig.getPropertyPrefix() + ".super.users")); + Set superGroups = toSet(pluginConfig.get(pluginConfig.getPropertyPrefix() + ".super.groups")); + Set auditExcludeUsers = toSet(pluginConfig.get(pluginConfig.getPropertyPrefix() + ".audit.exclude.users")); + Set auditExcludeGroups = toSet(pluginConfig.get(pluginConfig.getPropertyPrefix() + ".audit.exclude.groups")); + Set auditExcludeRoles = toSet(pluginConfig.get(pluginConfig.getPropertyPrefix() + ".audit.exclude.roles")); + Set serviceAdmins = toSet(pluginConfig.get(pluginConfig.getPropertyPrefix() + ".service.admins")); + + setSuperUsersAndGroups(superUsers, superGroups); + setAuditExcludedUsersGroupsRoles(auditExcludeUsers, auditExcludeGroups, auditExcludeRoles); + setIsFallbackSupported(pluginConfig.getBoolean(pluginConfig.getPropertyPrefix() + ".is.fallback.supported", false)); + setServiceAdmins(serviceAdmins); + + RangerScriptExecutionContext.init(pluginConfig); + + this.chainedPlugins = initChainedPlugins(); + } + + public RangerBasePlugin(RangerPluginConfig pluginConfig, ServicePolicies policies, ServiceTags tags, RangerRoles roles) { + this(pluginConfig); + + init(); + + setPolicies(policies); + setRoles(roles); + + if (tags != null) { + RangerTagEnricher tagEnricher = getTagEnricher(); + + if (tagEnricher != null) { + tagEnricher.setServiceTags(tags); + } else { + LOG.warn("RangerBasePlugin(tagsVersion=" + tags.getTagVersion() + "): no tag enricher found. Plugin will not enforce tag-based policies"); + } + } + } + + public static AuditHandler getAuditProvider(String serviceName) { + AuditProviderFactory providerFactory = RangerBasePlugin.getAuditProviderFactory(serviceName); + AuditHandler ret = providerFactory.getAuditProvider(); + + return ret; + } + + public String getServiceType() { + return pluginConfig.getServiceType(); + } + + public String getAppId() { + return pluginConfig.getAppId(); + } + + public RangerPluginConfig getConfig() { + return pluginConfig; + } + + public String getClusterName() { + return pluginConfig.getClusterName(); + } + + public RangerPluginContext getPluginContext() { + return pluginContext; + } + + public RangerAuthContext getCurrentRangerAuthContext() { return currentAuthContext; } + + public List getChainedPlugins() { return chainedPlugins; } + + // For backward compatibility + public RangerAuthContext createRangerAuthContext() { return currentAuthContext; } + + public RangerRoles getRoles() { + return this.roles; + } + + public void setRoles(RangerRoles roles) { + this.roles = roles; + + RangerPolicyEngine policyEngine = this.policyEngine; + + if (policyEngine != null) { + policyEngine.setRoles(roles); + } + + pluginContext.notifyAuthContextChanged(); + } + + public RangerUserStore getUserStore() { + return this.userStore; + } + + public void setUserStore(RangerUserStore userStore) { + this.userStore = userStore; + + // RangerPolicyEngine policyEngine = this.policyEngine; + + // if (policyEngine != null) { + // policyEngine.setUserStore(userStore); + // } + + // pluginContext.notifyAuthContextChanged(); + } + + public void setAuditExcludedUsersGroupsRoles(Set users, Set groups, Set roles) { + pluginConfig.setAuditExcludedUsersGroupsRoles(users, groups, roles); + } + + public void setSuperUsersAndGroups(Set users, Set groups) { + pluginConfig.setSuperUsersGroups(users, groups); + } + + public void setIsFallbackSupported(boolean isFallbackSupported) { + pluginConfig.setIsFallbackSupported(isFallbackSupported); + } + + public void setServiceAdmins(Set users) { + pluginConfig.setServiceAdmins(users); + } + + public RangerServiceDef getServiceDef() { + RangerPolicyEngine policyEngine = this.policyEngine; + + return policyEngine != null ? policyEngine.getServiceDef() : null; + } + + public int getServiceDefId() { + RangerServiceDef serviceDef = getServiceDef(); + + return serviceDef != null && serviceDef.getId() != null ? serviceDef.getId().intValue() : -1; + } + + public String getServiceName() { + return pluginConfig.getServiceName(); + } + + public AuditProviderFactory getAuditProviderFactory() { return RangerBasePlugin.getAuditProviderFactory(getServiceName()); } + + public AtlasTypeRegistry getTypeRegistry() { + return typeRegistry; + } + + public void setTypeRegistry(AtlasTypeRegistry typeRegistry) { + this.typeRegistry = typeRegistry; + } + + public void init() { + cleanup(); + + AuditProviderFactory providerFactory = AuditProviderFactory.getInstance(); + + if (!providerFactory.isInitDone()) { + if (pluginConfig.getProperties() != null) { + providerFactory.init(pluginConfig.getProperties(), getAppId()); + } else { + LOG.error("Audit subsystem is not initialized correctly. Please check audit configuration. "); + LOG.error("No authorization audits will be generated. "); + } + } + + if (!pluginConfig.getPolicyEngineOptions().disablePolicyRefresher) { + refresher = new PolicyRefresher(this); + LOG.info("Created PolicyRefresher Thread(" + refresher.getName() + ")"); + refresher.setDaemon(true); + refresher.startRefresher(); + } + + for (RangerChainedPlugin chainedPlugin : chainedPlugins) { + chainedPlugin.init(); + } + } + + public long getPoliciesVersion() { + RangerPolicyEngine policyEngine = this.policyEngine; + Long ret = policyEngine != null ? policyEngine.getPolicyVersion() : null; + + return ret != null ? ret : -1L; + } + + public long getTagsVersion() { + RangerTagEnricher tagEnricher = getTagEnricher(); + Long ret = tagEnricher != null ? tagEnricher.getServiceTagsVersion() : null; + + return ret != null ? ret : -1L; + } + + public long getRolesVersion() { + RangerPolicyEngine policyEngine = this.policyEngine; + Long ret = policyEngine != null ? policyEngine.getRoleVersion() : null; + + return ret != null ? ret : -1L; + } + + public void setPolicies(ServicePolicies policies) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> setPolicies(" + policies + ")"); + } + + // guard against catastrophic failure during policy engine Initialization or + try { + RangerPolicyEngine oldPolicyEngine = this.policyEngine; + ServicePolicies servicePolicies = null; + boolean isNewEngineNeeded = true; + boolean usePolicyDeltas = false; + + if (policies == null) { + policies = getDefaultSvcPolicies(); + + if (policies == null) { + LOG.error("Could not get default Service Policies. Keeping old policy-engine!"); + isNewEngineNeeded = false; + } + } else { + Boolean hasPolicyDeltas = RangerPolicyDeltaUtil.hasPolicyDeltas(policies); + + if (hasPolicyDeltas == null) { + LOG.info("Downloaded policies do not require policy change !! [" + policies + "]"); + + if (this.policyEngine == null) { + + LOG.info("There are no material changes, and current policy-engine is null! Creating a policy-engine with default service policies"); + ServicePolicies defaultSvcPolicies = getDefaultSvcPolicies(); + + if (defaultSvcPolicies == null) { + LOG.error("Could not get default Service Policies. Keeping old policy-engine! This is a FATAL error as the old policy-engine is null!"); + isNewEngineNeeded = false; + } else { + defaultSvcPolicies.setPolicyVersion(policies.getPolicyVersion()); + policies = defaultSvcPolicies; + isNewEngineNeeded = true; + } + } else { + LOG.info("Keeping old policy-engine!"); + isNewEngineNeeded = false; + } + } else { + if (hasPolicyDeltas.equals(Boolean.TRUE)) { + // Rebuild policies from deltas + RangerPolicyEngineImpl policyEngine = (RangerPolicyEngineImpl) oldPolicyEngine; + + servicePolicies = ServicePolicies.applyDelta(policies, policyEngine); + + if (servicePolicies != null) { + usePolicyDeltas = true; + } else { + LOG.error("Could not apply deltas=" + Arrays.toString(policies.getPolicyDeltas().toArray())); + LOG.warn("Keeping old policy-engine!"); + isNewEngineNeeded = false; + } + } else { + if (policies.getPolicies() == null) { + policies.setPolicies(new ArrayList<>()); + } + } + } + } + + if (isNewEngineNeeded) { + RangerPolicyEngine newPolicyEngine = null; + boolean isPolicyEngineShared = false; + + if (!usePolicyDeltas) { + if (LOG.isDebugEnabled()) { + LOG.debug("Creating engine from policies"); + } + + newPolicyEngine = new RangerPolicyEngineImpl(policies, pluginContext, roles); + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("policy-deltas are not null"); + } + + if (CollectionUtils.isNotEmpty(policies.getPolicyDeltas()) || MapUtils.isNotEmpty(policies.getSecurityZones())) { + if (LOG.isDebugEnabled()) { + LOG.debug("Non empty policy-deltas found. Cloning engine using policy-deltas"); + } + + if (oldPolicyEngine != null) { + RangerPolicyEngineImpl oldPolicyEngineImpl = (RangerPolicyEngineImpl) oldPolicyEngine; + + newPolicyEngine = RangerPolicyEngineImpl.getPolicyEngine(oldPolicyEngineImpl, policies); + } + + if (newPolicyEngine != null) { + if (LOG.isDebugEnabled()) { + LOG.debug("Applied policyDeltas=" + Arrays.toString(policies.getPolicyDeltas().toArray()) + ")"); + } + + isPolicyEngineShared = true; + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Failed to apply policyDeltas=" + Arrays.toString(policies.getPolicyDeltas().toArray()) + "), Creating engine from policies"); + LOG.debug("Creating new engine from servicePolicies:[" + servicePolicies + "]"); + } + + newPolicyEngine = new RangerPolicyEngineImpl(servicePolicies, pluginContext, roles); + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Empty policy-deltas. No need to change policy engine"); + } + } + } + + if (newPolicyEngine != null) { + if (!isPolicyEngineShared) { + newPolicyEngine.setUseForwardedIPAddress(pluginConfig.isUseForwardedIPAddress()); + newPolicyEngine.setTrustedProxyAddresses(pluginConfig.getTrustedProxyAddresses()); + } + + this.policyEngine = newPolicyEngine; + this.currentAuthContext = pluginContext.getAuthContext(); + + pluginContext.notifyAuthContextChanged(); + + if (oldPolicyEngine != null && oldPolicyEngine != newPolicyEngine) { + ((RangerPolicyEngineImpl) oldPolicyEngine).releaseResources(!isPolicyEngineShared); + } + + if (this.refresher != null) { + this.refresher.saveToCache(usePolicyDeltas ? servicePolicies : policies); + } + } + + } else { + LOG.warn("Leaving current policy engine as-is"); + LOG.warn("Policies are not saved to cache. policyVersion in the policy-cache may be different than in Ranger-admin, even though the policies are the same!"); + LOG.warn("Ranger-PolicyVersion:[" + (policies != null ? policies.getPolicyVersion() : -1L) + "], Cached-PolicyVersion:[" + (this.policyEngine != null ? this.policyEngine.getPolicyVersion() : -1L) + "]"); + } + + } catch (Exception e) { + LOG.error("setPolicies: policy engine initialization failed! Leaving current policy engine as-is. Exception : ", e); + } + if (LOG.isDebugEnabled()) { + LOG.debug("<== setPolicies(" + policies + ")"); + } + } + + public void cleanup() { + PolicyRefresher refresher = this.refresher; + this.refresher = null; + + RangerPolicyEngine policyEngine = this.policyEngine; + this.policyEngine = null; + + if (refresher != null) { + refresher.stopRefresher(); + } + + if (policyEngine != null) { + ((RangerPolicyEngineImpl) policyEngine).releaseResources(true); + } + } + + public void setResultProcessor(RangerAccessResultProcessor resultProcessor) { + this.resultProcessor = resultProcessor; + } + + public RangerAccessResultProcessor getResultProcessor() { + return this.resultProcessor; + } + + public RangerAccessResult isAccessAllowed(RangerAccessRequest request) { + return isAccessAllowed(request, resultProcessor); + } + + public Collection isAccessAllowed(Collection requests) { + return isAccessAllowed(requests, resultProcessor); + } + + public RangerAccessResult isAccessAllowed(RangerAccessRequest request, RangerAccessResultProcessor resultProcessor) { + RangerAccessResult ret = null; + RangerPolicyEngine policyEngine = this.policyEngine; + + if (policyEngine != null) { + ret = policyEngine.evaluatePolicies(request, RangerPolicy.POLICY_TYPE_ACCESS, null); + } + + if (ret != null) { + for (RangerChainedPlugin chainedPlugin : chainedPlugins) { + RangerAccessResult chainedResult = chainedPlugin.isAccessAllowed(request); + + if (chainedResult != null) { + updateResultFromChainedResult(ret, chainedResult); + } + } + + } + + if (policyEngine != null) { + policyEngine.evaluateAuditPolicies(ret); + } + + if (resultProcessor != null) { + resultProcessor.processResult(ret); + } + + return ret; + } + + public Collection isAccessAllowed(Collection requests, RangerAccessResultProcessor resultProcessor) { + Collection ret = null; + RangerPolicyEngine policyEngine = this.policyEngine; + + if (policyEngine != null) { + ret = policyEngine.evaluatePolicies(requests, RangerPolicy.POLICY_TYPE_ACCESS, null); + } + + if (CollectionUtils.isNotEmpty(ret)) { + for (RangerChainedPlugin chainedPlugin : chainedPlugins) { + Collection chainedResults = chainedPlugin.isAccessAllowed(requests); + + if (CollectionUtils.isNotEmpty(chainedResults)) { + Iterator iterRet = ret.iterator(); + Iterator iterChainedResults = chainedResults.iterator(); + + while (iterRet.hasNext() && iterChainedResults.hasNext()) { + RangerAccessResult result = iterRet.next(); + RangerAccessResult chainedResult = iterChainedResults.next(); + + if (result != null && chainedResult != null) { + updateResultFromChainedResult(result, chainedResult); + } + } + } + } + } + + if (policyEngine != null && CollectionUtils.isNotEmpty(ret)) { + for (RangerAccessResult result : ret) { + policyEngine.evaluateAuditPolicies(result); + } + } + + if (resultProcessor != null) { + resultProcessor.processResults(ret); + } + + return ret; + } + + public RangerAccessResult getAssetAccessors(RangerAccessRequest request) { + RangerAccessResult ret = null; + RangerPolicyEngine policyEngine = this.policyEngine; + + if (policyEngine != null) { + ret = policyEngine.evaluatePolicies(request, RangerPolicy.POLICY_TYPE_ACCESS, null); + } + + return ret; + } + + public RangerAccessResult evalDataMaskPolicies(RangerAccessRequest request, RangerAccessResultProcessor resultProcessor) { + RangerPolicyEngine policyEngine = this.policyEngine; + RangerAccessResult ret = null; + + if(policyEngine != null) { + ret = policyEngine.evaluatePolicies(request, RangerPolicy.POLICY_TYPE_DATAMASK, resultProcessor); + + policyEngine.evaluateAuditPolicies(ret); + } + + return ret; + } + + public RangerAccessResult evalRowFilterPolicies(RangerAccessRequest request, RangerAccessResultProcessor resultProcessor) { + RangerPolicyEngine policyEngine = this.policyEngine; + RangerAccessResult ret = null; + + if(policyEngine != null) { + ret = policyEngine.evaluatePolicies(request, RangerPolicy.POLICY_TYPE_ROWFILTER, resultProcessor); + + policyEngine.evaluateAuditPolicies(ret); + } + + return ret; + } + + public void evalAuditPolicies(RangerAccessResult result) { + RangerPolicyEngine policyEngine = this.policyEngine; + + if (policyEngine != null) { + policyEngine.evaluateAuditPolicies(result); + } + } + + public RangerResourceAccessInfo getResourceAccessInfo(RangerAccessRequest request) { + RangerPolicyEngine policyEngine = this.policyEngine; + + if(policyEngine != null) { + return policyEngine.getResourceAccessInfo(request); + } + + return null; + } + + public RangerResourceACLs getResourceACLs(RangerAccessRequest request) { + return getResourceACLs(request, null); + } + + public RangerResourceACLs getResourceACLs(RangerAccessRequest request, String policyType) { + RangerResourceACLs ret = null; + RangerPolicyEngine policyEngine = this.policyEngine; + + if(policyEngine != null) { + ret = policyEngine.getResourceACLs(request, policyType); + } + + for (RangerChainedPlugin chainedPlugin : chainedPlugins) { + RangerResourceACLs chainedResourceACLs = chainedPlugin.getResourceACLs(request, policyType); + + if (chainedResourceACLs != null) { + if (LOG.isDebugEnabled()) { + LOG.debug("Chained-plugin returned non-null ACLs!!"); + } + if (chainedPlugin.isAuthorizeOnlyWithChainedPlugin()) { + if (LOG.isDebugEnabled()) { + LOG.debug("Chained-plugin is configured to ignore Base-plugin's ACLs"); + } + ret = chainedResourceACLs; + break; + } else { + if (ret != null) { + ret = getMergedResourceACLs(ret, chainedResourceACLs); + } + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Chained-plugin returned null ACLs!!"); + } + } + } + + return ret; + } + + public Set getRolesFromUserAndGroups(String user, Set groups) { + RangerPolicyEngine policyEngine = this.policyEngine; + + if(policyEngine != null) { + return policyEngine.getRolesFromUserAndGroups(user, groups); + } + + return null; + } + + public RangerRoles getRangerRoles() { + RangerPolicyEngine policyEngine = this.policyEngine; + + if(policyEngine != null) { + return policyEngine.getRangerRoles(); + } + + return null; + } + + public Set getRangerRoleForPrincipal(String principal, String type) { + Set ret = new HashSet<>(); + Set rangerRoles = null; + Map> roleMapping = null; + RangerRoles roles = getRangerRoles(); + if (roles != null) { + rangerRoles = roles.getRangerRoles(); + } + + if (rangerRoles != null) { + RangerPluginContext rangerPluginContext = policyEngine.getPluginContext(); + if (rangerPluginContext != null) { + RangerAuthContext rangerAuthContext = rangerPluginContext.getAuthContext(); + if (rangerAuthContext != null) { + RangerRolesUtil rangerRolesUtil = rangerAuthContext.getRangerRolesUtil(); + if (rangerRolesUtil != null) { + switch (type) { + case "USER": + roleMapping = rangerRolesUtil.getUserRoleMapping(); + break; + case "GROUP": + roleMapping = rangerRolesUtil.getGroupRoleMapping(); + break; + case "ROLE": + roleMapping = rangerRolesUtil.getRoleRoleMapping(); + break; + } + } + } + } + if (roleMapping != null) { + Set principalRoles = roleMapping.get(principal); + if (CollectionUtils.isNotEmpty(principalRoles)) { + for (String role : principalRoles) { + for (RangerRole rangerRole : rangerRoles) { + if (rangerRole.getName().equals(role)) { + ret.add(rangerRole); + } + } + } + } + } + } + return ret; + } + + public boolean isServiceAdmin(String userName) { + boolean ret = false; + + RangerPolicyEngine policyEngine = this.policyEngine; + + if(policyEngine != null) { + RangerPolicyEngineImpl rangerPolicyEngine = (RangerPolicyEngineImpl) policyEngine; + ret = rangerPolicyEngine.isServiceAdmin(userName); + } + + return ret; + } + + public RangerRole createRole(RangerRole request, RangerAccessResultProcessor resultProcessor) throws Exception { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerBasePlugin.createRole(" + request + ")"); + } + + RangerRole ret = getAdminClient().createRole(request); + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerBasePlugin.createRole(" + request + ")"); + } + return ret; + } + + public void dropRole(String execUser, String roleName, RangerAccessResultProcessor resultProcessor) throws Exception { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerBasePlugin.dropRole(" + roleName + ")"); + } + + getAdminClient().dropRole(execUser, roleName); + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerBasePlugin.dropRole(" + roleName + ")"); + } + } + + public List getUserRoles(String execUser, RangerAccessResultProcessor resultProcessor) throws Exception { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerBasePlugin.getUserRoleNames(" + execUser + ")"); + } + + final List ret = getAdminClient().getUserRoles(execUser); + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerBasePlugin.getUserRoleNames(" + execUser + ")"); + } + return ret; + } + + public List getAllRoles(String execUser, RangerAccessResultProcessor resultProcessor) throws Exception { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerBasePlugin.getAllRoles()"); + } + + final List ret = getAdminClient().getAllRoles(execUser); + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerBasePlugin.getAllRoles()"); + } + return ret; + } + + public RangerRole getRole(String execUser, String roleName, RangerAccessResultProcessor resultProcessor) throws Exception { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerBasePlugin.getPrincipalsForRole(" + roleName + ")"); + } + + final RangerRole ret = getAdminClient().getRole(execUser, roleName); + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerBasePlugin.getPrincipalsForRole(" + roleName + ")"); + } + return ret; + } + + public void grantRole(GrantRevokeRoleRequest request, RangerAccessResultProcessor resultProcessor) throws Exception { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerBasePlugin.grantRole(" + request + ")"); + } + + getAdminClient().grantRole(request); + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerBasePlugin.grantRole(" + request + ")"); + } + } + + public void revokeRole(GrantRevokeRoleRequest request, RangerAccessResultProcessor resultProcessor) throws Exception { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerBasePlugin.revokeRole(" + request + ")"); + } + + getAdminClient().revokeRole(request); + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerBasePlugin.revokeRole(" + request + ")"); + } + } + + public void grantAccess(GrantRevokeRequest request, RangerAccessResultProcessor resultProcessor) throws Exception { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerBasePlugin.grantAccess(" + request + ")"); + } + + boolean isSuccess = false; + + try { + RangerPolicyEngine policyEngine = this.policyEngine; + + if (policyEngine != null) { + request.setZoneName(policyEngine.getUniquelyMatchedZoneName(request)); + } + + getAdminClient().grantAccess(request); + + isSuccess = true; + } finally { + auditGrantRevoke(request, "grant", isSuccess, resultProcessor); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerBasePlugin.grantAccess(" + request + ")"); + } + } + + public void revokeAccess(GrantRevokeRequest request, RangerAccessResultProcessor resultProcessor) throws Exception { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerBasePlugin.revokeAccess(" + request + ")"); + } + + boolean isSuccess = false; + + try { + RangerPolicyEngine policyEngine = this.policyEngine; + + if (policyEngine != null) { + request.setZoneName(policyEngine.getUniquelyMatchedZoneName(request)); + } + + getAdminClient().revokeAccess(request); + + isSuccess = true; + } finally { + auditGrantRevoke(request, "revoke", isSuccess, resultProcessor); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerBasePlugin.revokeAccess(" + request + ")"); + } + } + + public void registerAuthContextEventListener(RangerAuthContextListener authContextListener) { + this.pluginContext.setAuthContextListener(authContextListener); + } + + public static RangerAdminClient createAdminClient(RangerPluginConfig pluginConfig) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerBasePlugin.createAdminClient(" + pluginConfig.getServiceName() + ", " + pluginConfig.getAppId() + ", " + pluginConfig.getPropertyPrefix() + ")"); + } + + RangerAdminClient ret = null; + String propertyName = pluginConfig.getPropertyPrefix() + ".policy.source.impl"; + String policySourceImpl = pluginConfig.get(propertyName); + + if(StringUtils.isEmpty(policySourceImpl)) { + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("Value for property[%s] was null or empty. Unexpected! Will use policy source of type[%s]", propertyName, RangerAdminRESTClient.class.getName())); + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("Value for property[%s] was [%s].", propertyName, policySourceImpl)); + } + + try { + @SuppressWarnings("unchecked") + Class adminClass = (Class)Class.forName(policySourceImpl); + + ret = adminClass.newInstance(); + } catch (Exception excp) { + LOG.error("failed to instantiate policy source of type '" + policySourceImpl + "'. Will use policy source of type '" + RangerAdminRESTClient.class.getName() + "'", excp); + } + } + + if(ret == null) { + ret = new RangerAdminRESTClient(); + } + + ret.init(pluginConfig.getServiceName(), pluginConfig.getAppId(), pluginConfig.getPropertyPrefix(), pluginConfig); + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerBasePlugin.createAdminClient(" + pluginConfig.getServiceName() + ", " + pluginConfig.getAppId() + ", " + pluginConfig.getPropertyPrefix() + "): policySourceImpl=" + policySourceImpl + ", client=" + ret); + } + return ret; + } + + public void refreshPoliciesAndTags() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> refreshPoliciesAndTags()"); + } + + try { + RangerPolicyEngine policyEngine = this.policyEngine; + + // Synch-up policies + long oldPolicyVersion = policyEngine.getPolicyVersion(); + + if (refresher != null) { + refresher.syncPoliciesWithAdmin(accessTrigger); + } + + policyEngine = this.policyEngine; // might be updated in syncPoliciesWithAdmin() + + long newPolicyVersion = policyEngine.getPolicyVersion(); + + if (oldPolicyVersion == newPolicyVersion) { + // Synch-up tags + RangerTagEnricher tagEnricher = getTagEnricher(); + + if (tagEnricher != null) { + tagEnricher.syncTagsWithAdmin(accessTrigger); + } + } + } catch (InterruptedException exception) { + LOG.error("Failed to update policy-engine, continuing to use old policy-engine and/or tags", exception); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== refreshPoliciesAndTags()"); + } + } + + + private void auditGrantRevoke(GrantRevokeRequest request, String action, boolean isSuccess, RangerAccessResultProcessor resultProcessor) { + if(request != null && resultProcessor != null) { + RangerAccessRequestImpl accessRequest = new RangerAccessRequestImpl(); + + accessRequest.setResource(new RangerAccessResourceImpl(StringUtil.toStringObjectMap(request.getResource()))); + accessRequest.setUser(request.getGrantor()); + accessRequest.setAccessType(RangerPolicyEngine.ANY_ACCESS); + accessRequest.setAction(action); + accessRequest.setClientIPAddress(request.getClientIPAddress()); + accessRequest.setClientType(request.getClientType()); + accessRequest.setRequestData(request.getRequestData()); + accessRequest.setSessionId(request.getSessionId()); + + // call isAccessAllowed() to determine if audit is enabled or not + RangerAccessResult accessResult = isAccessAllowed(accessRequest, null); + + if(accessResult != null && accessResult.getIsAudited()) { + accessRequest.setAccessType(action); + accessResult.setIsAllowed(isSuccess); + + if(! isSuccess) { + accessResult.setPolicyId("-1"); + } + + resultProcessor.processResult(accessResult); + } + } + } + + private RangerServiceDef getDefaultServiceDef() { + RangerServiceDef ret = null; + + if (StringUtils.isNotBlank(getServiceType())) { + try { + ret = EmbeddedServiceDefsUtil.instance().getEmbeddedServiceDef(getServiceType()); + } catch (Exception exp) { + LOG.error("Could not get embedded service-def for " + getServiceType()); + } + } + return ret; + } + + private ServicePolicies getDefaultSvcPolicies() { + ServicePolicies ret = null; + RangerServiceDef serviceDef = getServiceDef(); + + if (serviceDef == null) { + serviceDef = getDefaultServiceDef(); + } + + if (serviceDef != null) { + ret = new ServicePolicies(); + + ret.setServiceDef(serviceDef); + ret.setServiceName(getServiceName()); + ret.setPolicies(new ArrayList()); + } + + return ret; + } + + public boolean logErrorMessage(String message) { + LogHistory log = logHistoryList.get(message); + if (log == null) { + log = new LogHistory(); + logHistoryList.put(message, log); + } + if ((System.currentTimeMillis() - log.lastLogTime) > logInterval) { + log.lastLogTime = System.currentTimeMillis(); + int counter = log.counter; + log.counter = 0; + if( counter > 0) { + message += ". Messages suppressed before: " + counter; + } + LOG.error(message); + return true; + } else { + log.counter++; + } + return false; + } + + private Set toSet(String value) { + return StringUtils.isNotBlank(value) ? StringUtil.toSet(value) : Collections.emptySet(); + } + + static private final class LogHistory { + long lastLogTime; + int counter; + } + + public RangerTagEnricher getTagEnricher() { + RangerTagEnricher ret = null; + RangerAuthContext authContext = getCurrentRangerAuthContext(); + + if (authContext != null) { + Map contextEnricherMap = authContext.getRequestContextEnrichers(); + + if (MapUtils.isNotEmpty(contextEnricherMap)) { + Set contextEnrichers = contextEnricherMap.keySet(); + + for (RangerContextEnricher enricher : contextEnrichers) { + if (enricher instanceof RangerTagEnricher) { + ret = (RangerTagEnricher) enricher; + + break; + } + } + } + } + return ret; + } + + public static RangerResourceACLs getMergedResourceACLs(RangerResourceACLs baseACLs, RangerResourceACLs chainedACLs) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerBasePlugin.getMergedResourceACLs()"); + LOG.debug("baseACLs:[" + baseACLs + "]"); + LOG.debug("chainedACLS:[" + chainedACLs + "]"); + } + + overrideACLs(chainedACLs, baseACLs, RangerRolesUtil.ROLES_FOR.USER); + overrideACLs(chainedACLs, baseACLs, RangerRolesUtil.ROLES_FOR.GROUP); + overrideACLs(chainedACLs, baseACLs, RangerRolesUtil.ROLES_FOR.ROLE); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerBasePlugin.getMergedResourceACLs() : ret:[" + baseACLs + "]"); + } + return baseACLs; + } + + private RangerAdminClient getAdminClient() throws Exception { + PolicyRefresher refresher = this.refresher; + RangerAdminClient admin = refresher == null ? null : refresher.getRangerAdminClient(); + + if(admin == null) { + throw new Exception("ranger-admin client is null"); + } + return admin; + } + + private List initChainedPlugins() { + List ret = new ArrayList<>(); + String chainedServicePropPrefix = pluginConfig.getPropertyPrefix() + ".chained.services"; + + for (String chainedService : StringUtil.toList(pluginConfig.get(chainedServicePropPrefix))) { + if (StringUtils.isBlank(chainedService)) { + continue; + } + + String className = pluginConfig.get(chainedServicePropPrefix + "." + chainedService + ".impl"); + + if (StringUtils.isBlank(className)) { + LOG.error("Ignoring chained service " + chainedService + ": no impl class specified"); + + continue; + } + + try { + @SuppressWarnings("unchecked") + Class pluginClass = (Class) Class.forName(className); + RangerChainedPlugin chainedPlugin = pluginClass.getConstructor(RangerBasePlugin.class, String.class).newInstance(this, chainedService); + + ret.add(chainedPlugin); + } catch (Throwable t) { + LOG.error("initChainedPlugins(): error instantiating plugin impl " + className, t); + } + } + + return ret; + } + + private void updateResultFromChainedResult(RangerAccessResult result, RangerAccessResult chainedResult) { + boolean overrideResult = false; + + if (chainedResult.getIsAccessDetermined()) { // only if chained-result is definitive + // override if result is not definitive or chained-result is by a higher priority policy + overrideResult = !result.getIsAccessDetermined() || chainedResult.getPolicyPriority() > result.getPolicyPriority(); + + if (!overrideResult) { + // override if chained-result is from the same policy priority, and if denies access + if (chainedResult.getPolicyPriority() == result.getPolicyPriority() && !chainedResult.getIsAllowed()) { + // let's not override if result is already denied + if (result.getIsAllowed()) { + overrideResult = true; + } + } + } + } + + if (overrideResult) { + result.setIsAllowed(chainedResult.getIsAllowed()); + result.setIsAccessDetermined(chainedResult.getIsAccessDetermined()); + result.setPolicyId(chainedResult.getPolicyId()); + result.setPolicyVersion(chainedResult.getPolicyVersion()); + result.setPolicyPriority(chainedResult.getPolicyPriority()); + result.setZoneName(chainedResult.getZoneName()); + } + + if (!result.getIsAuditedDetermined() && chainedResult.getIsAuditedDetermined()) { + result.setIsAudited(chainedResult.getIsAudited()); + result.setAuditPolicyId(chainedResult.getAuditPolicyId()); + } + } + + private static void overrideACLs(final RangerResourceACLs chainedResourceACLs, RangerResourceACLs baseResourceACLs, final RangerRolesUtil.ROLES_FOR userType) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerBasePlugin.overrideACLs(isUser=" + userType.name() + ")"); + } + Map> chainedACLs = null; + Map> baseACLs = null; + + switch (userType) { + case USER: + chainedACLs = chainedResourceACLs.getUserACLs(); + baseACLs = baseResourceACLs.getUserACLs(); + break; + case GROUP: + chainedACLs = chainedResourceACLs.getGroupACLs(); + baseACLs = baseResourceACLs.getGroupACLs(); + break; + case ROLE: + chainedACLs = chainedResourceACLs.getRoleACLs(); + baseACLs = baseResourceACLs.getRoleACLs(); + break; + default: + break; + } + + for (Map.Entry> chainedPermissionsMap : chainedACLs.entrySet()) { + String name = chainedPermissionsMap.getKey(); + Map chainedPermissions = chainedPermissionsMap.getValue(); + Map basePermissions = baseACLs.get(name); + + for (Map.Entry chainedPermission : chainedPermissions.entrySet()) { + String chainedAccessType = chainedPermission.getKey(); + RangerResourceACLs.AccessResult chainedAccessResult = chainedPermission.getValue(); + RangerResourceACLs.AccessResult baseAccessResult = basePermissions == null ? null : basePermissions.get(chainedAccessType); + + final boolean useChainedAccessResult; + + if (baseAccessResult == null) { + useChainedAccessResult = true; + } else { + if (chainedAccessResult.getPolicy().getPolicyPriority() > baseAccessResult.getPolicy().getPolicyPriority()) { + useChainedAccessResult = true; + } else if (chainedAccessResult.getPolicy().getPolicyPriority().equals(baseAccessResult.getPolicy().getPolicyPriority())) { + if (chainedAccessResult.getResult() == baseAccessResult.getResult()) { + useChainedAccessResult = true; + } else { + useChainedAccessResult = chainedAccessResult.getResult() == RangerPolicyEvaluator.ACCESS_DENIED; + } + } else { // chainedAccessResult.getPolicy().getPolicyPriority() < baseAccessResult.getPolicy().getPolicyPriority() + useChainedAccessResult = false; + } + } + + final RangerResourceACLs.AccessResult finalAccessResult = useChainedAccessResult ? chainedAccessResult : baseAccessResult; + + switch (userType) { + case USER: + baseResourceACLs.setUserAccessInfo(name, chainedAccessType, finalAccessResult.getResult(), finalAccessResult.getPolicy()); + break; + case GROUP: + baseResourceACLs.setGroupAccessInfo(name, chainedAccessType, finalAccessResult.getResult(), finalAccessResult.getPolicy()); + break; + case ROLE: + baseResourceACLs.setRoleAccessInfo(name, chainedAccessType, finalAccessResult.getResult(), finalAccessResult.getPolicy()); + break; + default: + break; + } + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerBasePlugin.mergeACLsOneWay(isUser=" + userType.name() + ")"); + } + } + + private static AuditProviderFactory getAuditProviderFactory(String serviceName) { + AuditProviderFactory ret = AuditProviderFactory.getInstance(); + + if (!ret.isInitDone()) { + LOG.warn("RangerBasePlugin.getAuditProviderFactory(serviceName=" + serviceName + "): audit not initialized yet. Will use stand-alone audit factory"); + + ret = StandAloneAuditProviderFactory.getInstance(); + + if (!ret.isInitDone()) { + RangerAuditConfig conf = new RangerAuditConfig(); + + if (conf.isInitSuccess()) { + ret.init(conf.getProperties(), "StandAlone"); + } + } + } + + return ret; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerBaseService.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerBaseService.java new file mode 100644 index 00000000000..35107e3dadb --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerBaseService.java @@ -0,0 +1,466 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.service; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.security.SecureClientLogin; +import org.apache.hadoop.security.authentication.util.KerberosName; +import org.apache.atlas.authorization.hadoop.config.RangerAdminConfig; +import org.apache.atlas.plugin.model.RangerPolicy; +import org.apache.atlas.plugin.model.RangerPolicy.RangerPolicyItem; +import org.apache.atlas.plugin.model.RangerPolicy.RangerPolicyItemAccess; +import org.apache.atlas.plugin.model.RangerPolicy.RangerPolicyResource; +import org.apache.atlas.plugin.model.RangerService; +import org.apache.atlas.plugin.model.RangerServiceDef; +import org.apache.atlas.plugin.model.validation.RangerServiceDefHelper; +import org.apache.atlas.plugin.resourcematcher.RangerAbstractResourceMatcher; +import org.apache.atlas.plugin.util.ServiceDefUtil; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; + + +public abstract class RangerBaseService { + private static final Log LOG = LogFactory.getLog(RangerBaseService.class); + + protected static final String ADMIN_USER_PRINCIPAL = "ranger.admin.kerberos.principal"; + protected static final String ADMIN_USER_KEYTAB = "ranger.admin.kerberos.keytab"; + protected static final String LOOKUP_PRINCIPAL = "ranger.lookup.kerberos.principal"; + protected static final String LOOKUP_KEYTAB = "ranger.lookup.kerberos.keytab"; + protected static final String RANGER_AUTH_TYPE = "hadoop.security.authentication"; + + protected static final String KERBEROS_TYPE = "kerberos"; + + private static final String PROP_DEFAULT_POLICY_PREFIX = "default-policy."; + private static final String PROP_DEFAULT_POLICY_NAME_SUFFIX = "name"; + + + protected RangerServiceDef serviceDef; + protected RangerService service; + + protected Map configs; + protected String serviceName; + protected String serviceType; + protected String lookUpUser; + + protected final RangerAdminConfig config; + + public RangerBaseService() { + this.config = RangerAdminConfig.getInstance(); + String authType = config.get(RANGER_AUTH_TYPE,"simple"); + String lookupPrincipal = config.get(LOOKUP_PRINCIPAL); + String lookupKeytab = config.get(LOOKUP_KEYTAB); + lookUpUser = getLookupUser(authType, lookupPrincipal, lookupKeytab); + } + + public void init(RangerServiceDef serviceDef, RangerService service) { + this.serviceDef = serviceDef; + this.service = service; + this.configs = service.getConfigs(); + this.serviceName = service.getName(); + this.serviceType = service.getType(); + } + + /** + * @return the serviceDef + */ + public RangerServiceDef getServiceDef() { + return serviceDef; + } + + /** + * @return the service + */ + public RangerService getService() { + return service; + } + + public Map getConfigs() { + return configs; + } + + public void setConfigs(Map configs) { + this.configs = configs; + } + + public String getServiceName() { + return serviceName; + } + + public void setServiceName(String serviceName) { + this.serviceName = serviceName; + } + + public String getServiceType() { + return serviceType; + } + + public void setServiceType(String serviceType) { + this.serviceType = serviceType; + } + + public RangerAdminConfig getConfig() { return config; } + + public abstract Map validateConfig() throws Exception; + + public abstract List lookupResource(ResourceLookupContext context) throws Exception; + + public List getDefaultRangerPolicies() throws Exception { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerBaseService.getDefaultRangerPolicies() "); + } + + List ret = new ArrayList(); + + try { + // we need to create one policy for each resource hierarchy + RangerServiceDefHelper serviceDefHelper = new RangerServiceDefHelper(serviceDef); + for (List aHierarchy : serviceDefHelper.filterHierarchies_containsOnlyMandatoryResources(RangerPolicy.POLICY_TYPE_ACCESS)) { + RangerPolicy policy = getDefaultPolicy(aHierarchy); + if (policy != null) { + ret.add(policy); + } + } + } catch (Exception e) { + LOG.error("Error getting default polcies for Service: " + service.getName(), e); + } + + final Boolean additionalDefaultPolicySetup = Boolean.valueOf(configs.get("setup.additional.default.policies")); + + if (additionalDefaultPolicySetup) { + LOG.info(getServiceName() + ": looking for additional default policies in service-config"); + + Set policyIndexes = new TreeSet<>(); + + for (String configName : configs.keySet()) { + if (configName.startsWith(PROP_DEFAULT_POLICY_PREFIX) && configName.endsWith(PROP_DEFAULT_POLICY_NAME_SUFFIX)) { + policyIndexes.add(configName.substring(PROP_DEFAULT_POLICY_PREFIX.length(), configName.length() - PROP_DEFAULT_POLICY_NAME_SUFFIX.length() - 1)); + } + } + + LOG.info(getServiceName() + ": found " + policyIndexes.size() + " additional default policies in service-config"); + + for (String policyIndex : policyIndexes) { + String policyPropertyPrefix = PROP_DEFAULT_POLICY_PREFIX + policyIndex + "."; + String resourcePropertyPrefix = policyPropertyPrefix + "resource."; + Map policyResources = getResourcesForPrefix(resourcePropertyPrefix); + + if (MapUtils.isNotEmpty(policyResources)) { + addCustomRangerDefaultPolicies(ret, policyResources, policyPropertyPrefix); + } else { + LOG.warn(getServiceName() + ": no resources specified for default policy with prefix '" + policyPropertyPrefix + "'. Ignored"); + } + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerBaseService.getDefaultRangerPolicies(): " + ret); + } + + return ret; + } + + private Map getResourcesForPrefix(String resourcePropertyPrefix) { + Map policyResourceMap = new HashMap(); + + if (configs != null) { + for (Map.Entry entry : configs.entrySet()) { + String configName = entry.getKey(); + String configValue = entry.getValue(); + + if(configName.startsWith(resourcePropertyPrefix) && StringUtils.isNotBlank(configValue)){ + RangerPolicyResource rPolRes = new RangerPolicyResource(); + String resourceKey = configName.substring(resourcePropertyPrefix.length()); + List resourceList = new ArrayList(Arrays.asList(configValue.split(","))); + + rPolRes.setIsExcludes(false); + rPolRes.setIsRecursive(false); + rPolRes.setValues(resourceList); + policyResourceMap.put(resourceKey, rPolRes); + } + } + } + + return policyResourceMap; + } + + private void addCustomRangerDefaultPolicies(List ret, Map policyResourceMap, String policyPropertyPrefix) throws Exception { + String policyName = configs.get(policyPropertyPrefix + PROP_DEFAULT_POLICY_NAME_SUFFIX); + String description = configs.get(policyPropertyPrefix + "description"); + + if (StringUtils.isEmpty(description)) { + description = "Policy for " + policyName; + } + + RangerPolicy policy = new RangerPolicy(); + + policy.setName(policyName); + policy.setIsEnabled(true); + policy.setVersion(1L); + policy.setIsAuditEnabled(true); + policy.setService(serviceName); + policy.setDescription(description); + policy.setName(policyName); + policy.setResources(policyResourceMap); + + for (int i = 1; ; i++) { + String policyItemPropertyPrefix = policyPropertyPrefix + "policyItem." + i + "."; + String policyItemUsers = configs.get(policyItemPropertyPrefix + "users"); + String policyItemGroups = configs.get(policyItemPropertyPrefix + "groups"); + String policyItemRoles = configs.get(policyItemPropertyPrefix + "roles"); + String policyItemAccessTypes = configs.get(policyItemPropertyPrefix + "accessTypes"); + String isDelegateAdmin = configs.get(policyItemPropertyPrefix + "isDelegateAdmin"); + + if (StringUtils.isEmpty(policyItemAccessTypes) || + (StringUtils.isEmpty(policyItemUsers) && StringUtils.isEmpty(policyItemGroups) && StringUtils.isEmpty(policyItemRoles))) { + + break; + } + + RangerPolicyItem policyItem = new RangerPolicyItem(); + + policyItem.setDelegateAdmin(Boolean.parseBoolean(isDelegateAdmin)); + + if (StringUtils.isNotBlank(policyItemUsers)) { + policyItem.setUsers(Arrays.asList(policyItemUsers.split(","))); + } + + if (StringUtils.isNotBlank(policyItemGroups)) { + policyItem.setGroups(Arrays.asList(policyItemGroups.split(","))); + } + + if (StringUtils.isNotBlank(policyItemRoles)) { + policyItem.setRoles(Arrays.asList(policyItemRoles.split(","))); + } + + if (StringUtils.isNotBlank(policyItemAccessTypes)) { + for (String accessType : Arrays.asList(policyItemAccessTypes.split(","))) { + RangerPolicyItemAccess polAccess = new RangerPolicyItemAccess(accessType, true); + + policyItem.getAccesses().add(polAccess); + } + } + + policy.getPolicyItems().add(policyItem); + } + + LOG.info(getServiceName() + ": adding default policy: name=" + policy.getName()); + + ret.add(policy); + } + + private RangerPolicy getDefaultPolicy(List resourceHierarchy) throws Exception { + + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerBaseService.getDefaultPolicy()"); + } + + RangerPolicy policy = new RangerPolicy(); + + String policyName=buildPolicyName(resourceHierarchy); + + policy.setIsEnabled(true); + policy.setVersion(1L); + policy.setName(policyName); + policy.setService(service.getName()); + policy.setDescription("Policy for " + policyName); + policy.setIsAuditEnabled(true); + policy.setResources(createDefaultPolicyResource(resourceHierarchy)); + + List policyItems = new ArrayList(); + //Create Default policy item for the service user + RangerPolicy.RangerPolicyItem policyItem = createDefaultPolicyItem(policy.getResources()); + policyItems.add(policyItem); + policy.setPolicyItems(policyItems); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerBaseService.getDefaultPolicy()" + policy); + } + + return policy; + } + + private RangerPolicy.RangerPolicyItem createDefaultPolicyItem(Map policyResources) throws Exception { + + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerBaseService.createDefaultPolicyItem()"); + } + + RangerPolicy.RangerPolicyItem policyItem = new RangerPolicy.RangerPolicyItem(); + + policyItem.setUsers(getUserList()); + policyItem.setGroups(getGroupList()); + List accesses = getAllowedAccesses(policyResources); + policyItem.setAccesses(accesses); + + policyItem.setDelegateAdmin(true); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerBaseService.createDefaultPolicyItem(): " + policyItem ); + } + return policyItem; + } + + protected List getAllowedAccesses(Map policyResources) { + List ret = new ArrayList(); + + RangerServiceDef.RangerResourceDef leafResourceDef = ServiceDefUtil.getLeafResourceDef(serviceDef, policyResources); + + if (leafResourceDef != null) { + Set accessTypeRestrictions = leafResourceDef.getAccessTypeRestrictions(); + + for (RangerServiceDef.RangerAccessTypeDef accessTypeDef : serviceDef.getAccessTypes()) { + boolean isAccessTypeAllowed = CollectionUtils.isEmpty(accessTypeRestrictions) || accessTypeRestrictions.contains(accessTypeDef.getName()); + + if (isAccessTypeAllowed) { + RangerPolicy.RangerPolicyItemAccess access = new RangerPolicy.RangerPolicyItemAccess(); + access.setType(accessTypeDef.getName()); + access.setIsAllowed(true); + ret.add(access); + } + } + } + return ret; + } + + protected Map createDefaultPolicyResource(List resourceHierarchy) throws Exception { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerBaseService.createDefaultPolicyResource()"); + } + Map resourceMap = new HashMap<>(); + + for (RangerServiceDef.RangerResourceDef resourceDef : resourceHierarchy) { + RangerPolicy.RangerPolicyResource polRes = new RangerPolicy.RangerPolicyResource(); + + polRes.setIsExcludes(false); + polRes.setIsRecursive(resourceDef.getRecursiveSupported()); + polRes.setValue(RangerAbstractResourceMatcher.WILDCARD_ASTERISK); + + resourceMap.put(resourceDef.getName(), polRes); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerBaseService.createDefaultPolicyResource():" + resourceMap); + } + return resourceMap; + } + + private String buildPolicyName(List resourceHierarchy) { + StringBuilder sb = new StringBuilder("all"); + if (CollectionUtils.isNotEmpty(resourceHierarchy)) { + int resourceDefCount = 0; + for (RangerServiceDef.RangerResourceDef resourceDef : resourceHierarchy) { + if (resourceDefCount > 0) { + sb.append(", "); + } else { + sb.append(" - "); + } + sb.append(resourceDef.getName()); + resourceDefCount++; + } + } + return sb.toString().trim(); + } + + private List getUserList() { + List ret = new ArrayList<>(); + + HashSet uniqueUsers = new HashSet(); + String[] users = config.getStrings("ranger.default.policy.users"); + + if (users != null) { + for (String user : users) { + uniqueUsers.add(user); + } + } + + Map serviceConfig = service.getConfigs(); + if (serviceConfig != null ) { + String serviceConfigUser = serviceConfig.get("username"); + if (StringUtils.isNotBlank(serviceConfigUser)){ + uniqueUsers.add(serviceConfig.get("username")); + } + String defaultUsers = serviceConfig.get("default.policy.users"); + if (!StringUtils.isEmpty(defaultUsers)) { + List defaultUserList = new ArrayList<>(Arrays.asList(StringUtils.split(defaultUsers,","))); + if (!defaultUserList.isEmpty()) { + uniqueUsers.addAll(defaultUserList); + } + } + } + + ret.addAll(uniqueUsers); + return ret; + } + private List getGroupList() { + List ret = new ArrayList<>(); + + HashSet uniqueGroups = new HashSet(); + String[] groups = config.getStrings("ranger.default.policy.groups"); + + if (groups != null) { + for (String group : groups) { + uniqueGroups.add(group); + } + } + + Map serviceConfig = service.getConfigs(); + if (serviceConfig != null) { + String defaultGroups = serviceConfig.get("default.policy.groups"); + if (!StringUtils.isEmpty(defaultGroups)) { + List defaultGroupList = new ArrayList<>(Arrays.asList(StringUtils.split(defaultGroups, ","))); + if (!defaultGroupList.isEmpty()) { + uniqueGroups.addAll(defaultGroupList); + } + } + } + ret.addAll(uniqueGroups); + + return ret; + } + + protected String getLookupUser(String authType, String lookupPrincipal, String lookupKeytab) { + String lookupUser = null; + if(!StringUtils.isEmpty(authType) && authType.equalsIgnoreCase(KERBEROS_TYPE)){ + if(SecureClientLogin.isKerberosCredentialExists(lookupPrincipal, lookupKeytab)){ + KerberosName krbName = new KerberosName(lookupPrincipal); + try { + lookupUser = krbName.getShortName(); + } catch (IOException e) { + LOG.error("Unknown lookup user", e); + } + } + } + return lookupUser; + } + + +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerChainedPlugin.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerChainedPlugin.java new file mode 100644 index 00000000000..58daf8efd1e --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerChainedPlugin.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.service; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.plugin.policyengine.RangerAccessRequest; +import org.apache.atlas.plugin.policyengine.RangerAccessResult; +import org.apache.atlas.plugin.policyengine.RangerResourceACLs; + +import java.util.Collection; + +public abstract class RangerChainedPlugin { + private static final Log LOG = LogFactory.getLog(RangerChainedPlugin.class); + + protected final RangerBasePlugin rootPlugin; + protected final String serviceType; + protected final String serviceName; + protected final RangerBasePlugin plugin; + + protected RangerChainedPlugin(RangerBasePlugin rootPlugin, String serviceType, String serviceName) { + LOG.info("RangerChainedPlugin(" + serviceType + ", " + serviceName + ")"); + + this.rootPlugin = rootPlugin; + this.serviceType = serviceType; + this.serviceName = serviceName; + this.plugin = buildChainedPlugin(serviceType, serviceName, rootPlugin.getAppId()); + } + + public void init() { + LOG.info("==> RangerChainedPlugin.init(" + serviceType + ", " + serviceName + ")"); + + this.plugin.init(); + + LOG.info("<== RangerChainedPlugin.init(" + serviceType + ", " + serviceName + ")"); + } + + protected RangerBasePlugin buildChainedPlugin(String serviceType, String serviceName, String appId) { + return new RangerBasePlugin(serviceType, serviceName, appId); + } + + public abstract RangerAccessResult isAccessAllowed(RangerAccessRequest request); + + public abstract Collection isAccessAllowed(Collection requests); + + public abstract RangerResourceACLs getResourceACLs(RangerAccessRequest request); + + public abstract RangerResourceACLs getResourceACLs(RangerAccessRequest request, String policyType); + + public boolean isAuthorizeOnlyWithChainedPlugin() { return false; } + +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerDefaultRequestProcessor.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerDefaultRequestProcessor.java new file mode 100644 index 00000000000..cdef3fca548 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerDefaultRequestProcessor.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.service; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.atlas.plugin.contextenricher.RangerContextEnricher; +import org.apache.atlas.plugin.policyengine.PolicyEngine; +import org.apache.atlas.plugin.policyengine.RangerAccessRequest; +import org.apache.atlas.plugin.policyengine.RangerAccessRequestImpl; +import org.apache.atlas.plugin.policyengine.RangerAccessRequestProcessor; +import org.apache.atlas.plugin.policyengine.RangerAccessResource; +import org.apache.atlas.plugin.policyengine.RangerMutableResource; +import org.apache.atlas.plugin.util.RangerAccessRequestUtil; + +import java.util.List; +import java.util.Set; + +public class RangerDefaultRequestProcessor implements RangerAccessRequestProcessor { + + protected final PolicyEngine policyEngine; + + public RangerDefaultRequestProcessor(PolicyEngine policyEngine) { + this.policyEngine = policyEngine; + } + + @Override + public void preProcess(RangerAccessRequest request) { + + setResourceServiceDef(request); + if (request instanceof RangerAccessRequestImpl) { + RangerAccessRequestImpl reqImpl = (RangerAccessRequestImpl) request; + + if (reqImpl.getClientIPAddress() == null) { + reqImpl.extractAndSetClientIPAddress(policyEngine.getUseForwardedIPAddress(), policyEngine.getTrustedProxyAddresses()); + } + + if(policyEngine.getPluginContext() != null) { + if (reqImpl.getClusterName() == null) { + reqImpl.setClusterName(policyEngine.getPluginContext().getClusterName()); + } + + if (reqImpl.getClusterType() == null) { + reqImpl.setClusterType(policyEngine.getPluginContext().getClusterType()); + } + } + } + + RangerAccessRequestUtil.setCurrentUserInContext(request.getContext(), request.getUser()); + + String owner = request.getResource() != null ? request.getResource().getOwnerUser() : null; + + if (StringUtils.isNotEmpty(owner)) { + RangerAccessRequestUtil.setOwnerInContext(request.getContext(), owner); + } + + Set roles = request.getUserRoles(); + if (CollectionUtils.isEmpty(roles)) { + roles = policyEngine.getPluginContext().getAuthContext().getRolesForUserAndGroups(request.getUser(), request.getUserGroups()); + } + + if (CollectionUtils.isNotEmpty(roles)) { + RangerAccessRequestUtil.setCurrentUserRolesInContext(request.getContext(), roles); + } + + enrich(request); + } + + @Override + public void enrich(RangerAccessRequest request) { + List enrichers = policyEngine.getAllContextEnrichers(); + + if (!CollectionUtils.isEmpty(enrichers)) { + for(RangerContextEnricher enricher : enrichers) { + enricher.enrich(request); + } + } + } + + private void setResourceServiceDef(RangerAccessRequest request) { + RangerAccessResource resource = request.getResource(); + + if (resource.getServiceDef() == null) { + if (resource instanceof RangerMutableResource) { + RangerMutableResource mutable = (RangerMutableResource) resource; + mutable.setServiceDef(policyEngine.getServiceDef()); + } + } + } + +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerDefaultService.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerDefaultService.java new file mode 100644 index 00000000000..d2541c46666 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerDefaultService.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.service; + +import org.apache.commons.collections.ListUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import java.util.List; +import java.util.Map; + +public class RangerDefaultService extends RangerBaseService { + private static final Log LOG = LogFactory.getLog(RangerDefaultService.class); + + @Override + public Map validateConfig() throws Exception { + if(LOG.isDebugEnabled()) { + LOG.debug("RangerDefaultService.validateConfig Service: (" + serviceName + " ), returning empty map"); + } + return MapUtils.EMPTY_MAP; + } + + @Override + public List lookupResource(ResourceLookupContext context) throws Exception { + if(LOG.isDebugEnabled()) { + LOG.debug("RangerDefaultService.lookupResource Context: (" + context + "), returning empty list"); + } + return ListUtils.EMPTY_LIST; + } + +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/ResourceLookupContext.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/ResourceLookupContext.java new file mode 100644 index 00000000000..444488d1c92 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/ResourceLookupContext.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.service; + +import org.codehaus.jackson.annotate.JsonAutoDetect; +import org.codehaus.jackson.annotate.JsonAutoDetect.Visibility; +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.List; +import java.util.Map; + +@JsonAutoDetect(getterVisibility=Visibility.NONE, setterVisibility=Visibility.NONE, fieldVisibility=Visibility.ANY) +@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL ) +@JsonIgnoreProperties(ignoreUnknown=true) +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class ResourceLookupContext { + private String userInput; + private String resourceName; + private Map> resources; + + + public ResourceLookupContext() { + + } + + /** + * @return the userInput + */ + public String getUserInput() { + return userInput; + } + /** + * @param userInput the userInput to set + */ + public void setUserInput(String userInput) { + this.userInput = userInput; + } + /** + * @return the resourceName + */ + public String getResourceName() { + return resourceName; + } + /** + * @param resourceName the resourceName to set + */ + public void setResourceName(String resourceName) { + this.resourceName = resourceName; + } + /** + * @return the resources + */ + public Map> getResources() { + return resources; + } + /** + * @param resources the resources to set + */ + public void setResources(Map> resources) { + this.resources = resources; + } + + @Override + public String toString() { + return String.format("ResourceLookupContext={resourceName=%s,userInput=%s,resources=%s}", resourceName, userInput, resources); + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/AbstractPredicateUtil.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/AbstractPredicateUtil.java new file mode 100644 index 00000000000..1ddc65b6165 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/AbstractPredicateUtil.java @@ -0,0 +1,1053 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.store; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.collections.Predicate; +import org.apache.commons.collections.PredicateUtils; +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.lang.ObjectUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.atlas.plugin.model.RangerBaseModelObject; +import org.apache.atlas.plugin.model.RangerPolicy; +import org.apache.atlas.plugin.model.RangerPolicy.RangerPolicyItem; +import org.apache.atlas.plugin.model.RangerPolicy.RangerPolicyResource; +import org.apache.atlas.plugin.model.RangerSecurityZone; +import org.apache.atlas.plugin.model.RangerService; +import org.apache.atlas.plugin.model.RangerServiceDef; +import org.apache.atlas.plugin.model.RangerServiceDef.RangerResourceDef; +import org.apache.atlas.plugin.util.SearchFilter; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class AbstractPredicateUtil { + private static Map> sorterMap = new HashMap<>(); + + public void applyFilter(List objList, SearchFilter filter) { + if(CollectionUtils.isEmpty(objList)) { + return; + } + + Predicate pred = getPredicate(filter); + + if(pred != null) { + CollectionUtils.filter(objList, pred); + } + + Comparator sorter = getSorter(filter); + + if(sorter != null) { + Collections.sort(objList, sorter); + } + } + + public Predicate getPredicate(SearchFilter filter) { + if(filter == null || filter.isEmpty()) { + return null; + } + + List predicates = new ArrayList<>(); + + addPredicates(filter, predicates); + + Predicate ret = CollectionUtils.isEmpty(predicates) ? null : PredicateUtils.allPredicate(predicates); + + return ret; + } + + public void addPredicates(SearchFilter filter, List predicates) { + addPredicateForServiceType(filter.getParam(SearchFilter.SERVICE_TYPE), predicates); + addPredicateForServiceTypeId(filter.getParam(SearchFilter.SERVICE_TYPE_ID), predicates); + addPredicateForServiceName(filter.getParam(SearchFilter.SERVICE_NAME), predicates); + // addPredicateForServiceId(filter.getParam(SearchFilter.SERVICE_ID), predicates); // not supported + addPredicateForPolicyName(filter.getParam(SearchFilter.POLICY_NAME), predicates); + addPredicateForPolicyId(filter.getParam(SearchFilter.POLICY_ID), predicates); + addPredicateForIsEnabled(filter.getParam(SearchFilter.IS_ENABLED), predicates); + addPredicateForIsRecursive(filter.getParam(SearchFilter.IS_RECURSIVE), predicates); + addPredicateForTagServiceName(filter.getParam(SearchFilter.TAG_SERVICE_NAME), predicates); + // addPredicateForTagServiceId(filter.getParam(SearchFilter.TAG_SERVICE_ID), predicates); // not supported + addPredicateForUserName(filter.getParam(SearchFilter.USER), predicates); + addPredicateForGroupName(filter.getParam(SearchFilter.GROUP), predicates); + addPredicateForRoleName(filter.getParam(SearchFilter.ROLE), predicates); + addPredicateForResources(filter.getParamsWithPrefix(SearchFilter.RESOURCE_PREFIX, true), predicates); + addPredicateForPolicyResource(filter.getParam(SearchFilter.POL_RESOURCE), predicates); + addPredicateForPartialPolicyName(filter.getParam(SearchFilter.POLICY_NAME_PARTIAL), predicates); + addPredicateForResourceSignature(filter.getParam(SearchFilter.RESOURCE_SIGNATURE), predicates); + addPredicateForPolicyType(filter.getParam(SearchFilter.POLICY_TYPE), predicates); + addPredicateForPolicyPriority(filter.getParam(SearchFilter.POLICY_PRIORITY), predicates); + addPredicateForPartialPolicyLabels(filter.getParam(SearchFilter.POLICY_LABELS_PARTIAL), predicates); + addPredicateForZoneName(filter.getParam(SearchFilter.ZONE_NAME), predicates); + // addPredicateForZoneId(filter.getParam(SearchFilter.ZONE_ID), predicates); // not supported + } + + public Comparator getSorter(SearchFilter filter) { + String sortBy = filter == null ? null : filter.getSortBy(); + + if(StringUtils.isEmpty(sortBy)) { + return null; + } + + Comparator ret = sorterMap.get(sortBy); + + return ret; + } + + public final static Comparator idComparator = new Comparator() { + @Override + public int compare(RangerBaseModelObject o1, RangerBaseModelObject o2) { + Long val1 = (o1 != null) ? o1.getId() : null; + Long val2 = (o2 != null) ? o2.getId() : null; + + return ObjectUtils.compare(val1, val2); + } + }; + + protected final static Comparator createTimeComparator = new Comparator() { + @Override + public int compare(RangerBaseModelObject o1, RangerBaseModelObject o2) { + Date val1 = (o1 != null) ? o1.getCreateTime() : null; + Date val2 = (o2 != null) ? o2.getCreateTime() : null; + + return ObjectUtils.compare(val1, val2); + } + }; + + protected final static Comparator updateTimeComparator = new Comparator() { + @Override + public int compare(RangerBaseModelObject o1, RangerBaseModelObject o2) { + Date val1 = (o1 != null) ? o1.getUpdateTime() : null; + Date val2 = (o2 != null) ? o2.getUpdateTime() : null; + + return ObjectUtils.compare(val1, val2); + } + }; + + protected final static Comparator serviceDefNameComparator = new Comparator() { + @Override + public int compare(RangerBaseModelObject o1, RangerBaseModelObject o2) { + String val1 = null; + String val2 = null; + + if(o1 != null) { + if(o1 instanceof RangerServiceDef) { + val1 = ((RangerServiceDef)o1).getName(); + } else if(o1 instanceof RangerService) { + val1 = ((RangerService)o1).getType(); + } + } + + if(o2 != null) { + if(o2 instanceof RangerServiceDef) { + val2 = ((RangerServiceDef)o2).getName(); + } else if(o2 instanceof RangerService) { + val2 = ((RangerService)o2).getType(); + } + } + + return ObjectUtils.compare(val1, val2); + } + }; + + protected final static Comparator serviceNameComparator = new Comparator() { + @Override + public int compare(RangerBaseModelObject o1, RangerBaseModelObject o2) { + String val1 = null; + String val2 = null; + + if(o1 != null) { + if(o1 instanceof RangerPolicy) { + val1 = ((RangerPolicy)o1).getService(); + } else if(o1 instanceof RangerService) { + val1 = ((RangerService)o1).getType(); + } + } + + if(o2 != null) { + if(o2 instanceof RangerPolicy) { + val2 = ((RangerPolicy)o2).getService(); + } else if(o2 instanceof RangerService) { + val2 = ((RangerService)o2).getType(); + } + } + + return ObjectUtils.compare(val1, val2); + } + }; + + protected final static Comparator policyNameComparator = new Comparator() { + @Override + public int compare(RangerBaseModelObject o1, RangerBaseModelObject o2) { + String val1 = (o1 instanceof RangerPolicy) ? ((RangerPolicy)o1).getName() : null; + String val2 = (o2 instanceof RangerPolicy) ? ((RangerPolicy)o2).getName() : null; + + return ObjectUtils.compare(val1, val2); + } + }; + + public final static Comparator resourceLevelComparator = new Comparator() { + @Override + public int compare(RangerResourceDef o1, RangerResourceDef o2) { + Integer val1 = (o1 != null) ? o1.getLevel() : null; + Integer val2 = (o2 != null) ? o2.getLevel() : null; + + return ObjectUtils.compare(val1, val2); + } + }; + + protected final static Comparator zoneNameComparator = new Comparator() { + @Override + public int compare(RangerBaseModelObject o1, RangerBaseModelObject o2) { + String val1 = (o1 instanceof RangerSecurityZone) ? ((RangerSecurityZone)o1).getName() : null; + String val2 = (o2 instanceof RangerSecurityZone) ? ((RangerSecurityZone)o2).getName() : null; + + return ObjectUtils.compare(val1, val2); + } + }; + + static { + sorterMap.put(SearchFilter.SERVICE_TYPE, serviceDefNameComparator); + sorterMap.put(SearchFilter.SERVICE_TYPE_ID, idComparator); + sorterMap.put(SearchFilter.SERVICE_NAME, serviceNameComparator); + sorterMap.put(SearchFilter.SERVICE_TYPE_ID, idComparator); + sorterMap.put(SearchFilter.POLICY_NAME, policyNameComparator); + sorterMap.put(SearchFilter.POLICY_ID, idComparator); + sorterMap.put(SearchFilter.CREATE_TIME, createTimeComparator); + sorterMap.put(SearchFilter.UPDATE_TIME, updateTimeComparator); + sorterMap.put(SearchFilter.ZONE_ID, idComparator); + sorterMap.put(SearchFilter.ZONE_NAME, zoneNameComparator); + } + + private Predicate addPredicateForServiceType(final String serviceType, List predicates) { + if(StringUtils.isEmpty(serviceType)) { + return null; + } + + Predicate ret = new Predicate() { + @Override + public boolean evaluate(Object object) { + if(object == null) { + return false; + } + + boolean ret = false; + + if(object instanceof RangerServiceDef) { + RangerServiceDef serviceDef = (RangerServiceDef)object; + String svcType = serviceDef.getName(); + + ret = StringUtils.equals(svcType, serviceType); + } else { + ret = true; + } + + return ret; + } + }; + + if(predicates != null) { + predicates.add(ret); + } + + return ret; + } + + private Predicate addPredicateForServiceTypeId(final String serviceTypeId, List predicates) { + if(StringUtils.isEmpty(serviceTypeId)) { + return null; + } + + Predicate ret = new Predicate() { + @Override + public boolean evaluate(Object object) { + if(object == null) { + return false; + } + + boolean ret = false; + + if(object instanceof RangerServiceDef) { + RangerServiceDef serviceDef = (RangerServiceDef)object; + Long svcDefId = serviceDef.getId(); + + if(svcDefId != null) { + ret = StringUtils.equals(serviceTypeId, svcDefId.toString()); + } + } else { + ret = true; + } + + return ret; + } + }; + + if(predicates != null) { + predicates.add(ret); + } + + return ret; + } + + private Predicate addPredicateForServiceName(final String serviceName, List predicates) { + if(StringUtils.isEmpty(serviceName)) { + return null; + } + + Predicate ret = new Predicate() { + @Override + public boolean evaluate(Object object) { + if(object == null) { + return false; + } + + boolean ret = false; + + if(object instanceof RangerPolicy) { + RangerPolicy policy = (RangerPolicy)object; + + ret = StringUtils.equals(serviceName, policy.getService()); + } else if(object instanceof RangerService) { + RangerService service = (RangerService)object; + + ret = StringUtils.equals(serviceName, service.getName()); + } else { + ret = true; + } + + return ret; + } + }; + + if(ret != null) { + predicates.add(ret); + } + + return ret; + } + + private Predicate addPredicateForPolicyName(final String policyName, List predicates) { + if(StringUtils.isEmpty(policyName)) { + return null; + } + + Predicate ret = new Predicate() { + @Override + public boolean evaluate(Object object) { + if(object == null) { + return false; + } + + boolean ret = false; + + if(object instanceof RangerPolicy) { + RangerPolicy policy = (RangerPolicy)object; + + ret = StringUtils.equals(policyName, policy.getName()); + } else { + ret = true; + } + + return ret; + } + }; + + if(predicates != null) { + predicates.add(ret); + } + + return ret; + } + + private Predicate addPredicateForPartialPolicyName(final String policyName, List predicates) { + if(StringUtils.isEmpty(policyName)) { + return null; + } + + Predicate ret = new Predicate() { + @Override + public boolean evaluate(Object object) { + if(object == null) { + return false; + } + + boolean ret = false; + + if(object instanceof RangerPolicy) { + RangerPolicy policy = (RangerPolicy)object; + + ret = StringUtils.containsIgnoreCase(policy.getName(), policyName); + } else { + ret = true; + } + + return ret; + } + }; + + if(predicates != null) { + predicates.add(ret); + } + + return ret; + } + + private Predicate addPredicateForPolicyId(final String policyId, List predicates) { + if(StringUtils.isEmpty(policyId)) { + return null; + } + + Predicate ret = new Predicate() { + @Override + public boolean evaluate(Object object) { + if(object == null) { + return false; + } + + boolean ret = false; + + if(object instanceof RangerPolicy) { + RangerPolicy policy = (RangerPolicy)object; + + if(policy.getId() != null) { + ret = StringUtils.equals(policyId, policy.getId().toString()); + } + } else { + ret = true; + } + + return ret; + } + }; + + if(predicates != null) { + predicates.add(ret); + } + + return ret; + } + + private Predicate addPredicateForUserName(final String userName, List predicates) { + if(StringUtils.isEmpty(userName)) { + return null; + } + + Predicate ret = new Predicate() { + @Override + public boolean evaluate(Object object) { + if(object == null) { + return false; + } + + boolean ret = false; + + if(object instanceof RangerPolicy) { + RangerPolicy policy = (RangerPolicy)object; + + List[] policyItemsList = new List[] { policy.getPolicyItems(), + policy.getDenyPolicyItems(), + policy.getAllowExceptions(), + policy.getDenyExceptions(), + policy.getDataMaskPolicyItems(), + policy.getRowFilterPolicyItems() + }; + + for(List policyItemsObj : policyItemsList) { + @SuppressWarnings("unchecked") + List policyItems = (List)policyItemsObj; + + for(RangerPolicyItem policyItem : policyItems) { + if(! policyItem.getUsers().isEmpty()) { + for(String user : policyItem.getUsers()) { + if(StringUtils.containsIgnoreCase(user, userName)) { + ret = true; + break; + } + } + } + } + if (ret) { + break; + } + } + } else { + ret = true; + } + + return ret; + } + }; + + if(predicates != null) { + predicates.add(ret); + } + + return ret; + } + + private Predicate addPredicateForGroupName(final String groupName, List predicates) { + if(StringUtils.isEmpty(groupName)) { + return null; + } + + Predicate ret = new Predicate() { + @Override + public boolean evaluate(Object object) { + if(object == null) { + return false; + } + + boolean ret = false; + + if(object instanceof RangerPolicy) { + RangerPolicy policy = (RangerPolicy)object; + + List[] policyItemsList = new List[] { policy.getPolicyItems(), + policy.getDenyPolicyItems(), + policy.getAllowExceptions(), + policy.getDenyExceptions(), + policy.getDataMaskPolicyItems(), + policy.getRowFilterPolicyItems() + }; + + for(List policyItemsObj : policyItemsList) { + @SuppressWarnings("unchecked") + List policyItems = (List)policyItemsObj; + + for(RangerPolicyItem policyItem : policyItems) { + if(! policyItem.getGroups().isEmpty()) { + for(String group : policyItem.getGroups()) { + if(StringUtils.containsIgnoreCase(group, groupName)) { + ret = true; + break; + } + } + } + } + if (ret) { + break; + } + } + } else { + ret = true; + } + + return ret; + } + }; + + if(predicates != null) { + predicates.add(ret); + } + + return ret; + } + + private Predicate addPredicateForRoleName(final String roleName, List predicates) { + if(StringUtils.isEmpty(roleName)) { + return null; + } + + Predicate ret = new Predicate() { + @Override + public boolean evaluate(Object object) { + if(object == null) { + return false; + } + + boolean ret = false; + + if(object instanceof RangerPolicy) { + RangerPolicy policy = (RangerPolicy)object; + + List[] policyItemsList = new List[] { policy.getPolicyItems(), + policy.getDenyPolicyItems(), + policy.getAllowExceptions(), + policy.getDenyExceptions(), + policy.getDataMaskPolicyItems(), + policy.getRowFilterPolicyItems() + }; + for(List policyItemsObj : policyItemsList) { + @SuppressWarnings("unchecked") + List policyItems = (List)policyItemsObj; + + for(RangerPolicyItem policyItem : policyItems) { + if(! policyItem.getRoles().isEmpty()) { + for(String role : policyItem.getRoles()) { + if(StringUtils.containsIgnoreCase(role, roleName)) { + ret = true; + break; + } + } + } + } + if (ret) { + break; + } + } + }else { + ret = true; + } + + return ret; + } + }; + + if(predicates != null) { + predicates.add(ret); + } + + return ret; + + } + + private Predicate addPredicateForIsEnabled(final String status, List predicates) { + if(StringUtils.isEmpty(status)) { + return null; + } + + Predicate ret = new Predicate() { + @Override + public boolean evaluate(Object object) { + if(object == null) { + return false; + } + + boolean ret = false; + + if(object instanceof RangerBaseModelObject) { + RangerBaseModelObject obj = (RangerBaseModelObject)object; + + if(Boolean.parseBoolean(status)) { + ret = obj.getIsEnabled(); + } else { + ret = !obj.getIsEnabled(); + } + } else { + ret = true; + } + + return ret; + } + }; + + if(predicates != null) { + predicates.add(ret); + } + + return ret; + } + + private Predicate addPredicateForResources(final Map resources, List predicates) { + if(MapUtils.isEmpty(resources)) { + return null; + } + + Predicate ret = new Predicate() { + @Override + public boolean evaluate(Object object) { + if(object == null) { + return false; + } + + boolean ret = false; + + if(object instanceof RangerPolicy) { + RangerPolicy policy = (RangerPolicy)object; + + if(! MapUtils.isEmpty(policy.getResources())) { + int numFound = 0; + for(String name : resources.keySet()) { + boolean isMatch = false; + + RangerPolicyResource policyResource = policy.getResources().get(name); + + if(policyResource != null && !CollectionUtils.isEmpty(policyResource.getValues())) { + String val = resources.get(name); + + if(policyResource.getValues().contains(val)) { + isMatch = true; + } else { + for(String policyResourceValue : policyResource.getValues()) { + if(FilenameUtils.wildcardMatch(val, policyResourceValue)) { + isMatch = true; + break; + } + } + } + } + + if(isMatch) { + numFound++; + } else { + break; + } + } + + ret = numFound == resources.size(); + } + } else { + ret = true; + } + + return ret; + } + }; + + if(predicates != null) { + predicates.add(ret); + } + + return ret; + } + + private Predicate addPredicateForPolicyResource(final String resourceValue, List predicates) { + if(StringUtils.isEmpty(resourceValue)) { + return null; + } + + Predicate ret = new Predicate() { + @Override + public boolean evaluate(Object object) { + if(object == null) { + return false; + } + + boolean ret = false; + + if(object instanceof RangerPolicy) { + RangerPolicy policy = (RangerPolicy)object; + Map policyResources = policy.getResources(); + + if(MapUtils.isNotEmpty(policyResources)) { + + for (Map.Entry entry : policyResources.entrySet()) { + + RangerPolicyResource policyResource = entry.getValue(); + + if (policyResource != null && CollectionUtils.isNotEmpty(policyResource.getValues())) { + + for (String policyResoureValue : policyResource.getValues()) { + if (StringUtils.containsIgnoreCase(policyResoureValue, resourceValue)) { + ret = true; + + break; + } + } + } + + } + + } + } else { + ret = true; + } + + return ret; + } + }; + + if(predicates != null) { + predicates.add(ret); + } + + return ret; + } + + private Predicate addPredicateForIsRecursive(final String isRecursiveStr, List predicates) { + if(StringUtils.isEmpty(isRecursiveStr)) { + return null; + } + + final boolean isRecursive = Boolean.parseBoolean(isRecursiveStr); + + Predicate ret = new Predicate() { + @Override + public boolean evaluate(Object object) { + if(object == null) { + return false; + } + + boolean ret = true; + + if(object instanceof RangerPolicy) { + RangerPolicy policy = (RangerPolicy)object; + + if(! MapUtils.isEmpty(policy.getResources())) { + for(Map.Entry e : policy.getResources().entrySet()) { + RangerPolicyResource resValue = e.getValue(); + + if(resValue.getIsRecursive() == null) { + ret = !isRecursive; + } else { + ret = resValue.getIsRecursive().booleanValue() == isRecursive; + } + + if(ret) { + break; + } + } + } + } + + return ret; + } + }; + + if(predicates != null) { + predicates.add(ret); + } + + return ret; + } + + private Predicate addPredicateForTagServiceName(final String tagServiceName, List predicates) { + if(StringUtils.isEmpty(tagServiceName)) { + return null; + } + + Predicate ret = new Predicate() { + @Override + public boolean evaluate(Object object) { + if(object == null) { + return false; + } + + boolean ret = false; + + if(object instanceof RangerService) { + RangerService service = (RangerService)object; + + ret = StringUtils.equals(tagServiceName, service.getTagService()); + } else { + ret = true; + } + + return ret; + } + }; + + if(ret != null) { + predicates.add(ret); + } + + return ret; + } + + private Predicate addPredicateForResourceSignature(String signature, List predicates) { + + Predicate ret = createPredicateForResourceSignature(signature); + + if(predicates != null && ret != null) { + predicates.add(ret); + } + + return ret; + } + + private Predicate addPredicateForPolicyType(final String policyType, List predicates) { + if(StringUtils.isEmpty(policyType)) { + return null; + } + + Predicate ret = new Predicate() { + @Override + public boolean evaluate(Object object) { + if(object == null) { + return false; + } + + boolean ret = true; + + if(object instanceof RangerPolicy) { + RangerPolicy policy = (RangerPolicy)object; + + if(policy.getPolicyType() != null) { + ret = StringUtils.equalsIgnoreCase(policyType, policy.getPolicyType().toString()); + } + } + + return ret; + } + }; + + if(predicates != null) { + predicates.add(ret); + } + + return ret; + } + + private Predicate addPredicateForPartialPolicyLabels(final String policyLabels, List predicates) { + if (StringUtils.isEmpty(policyLabels)) { + return null; + } + + Predicate ret = new Predicate() { + @Override + public boolean evaluate(Object object) { + if (object == null) { + return false; + } + boolean ret = false; + + if (object instanceof RangerPolicy) { + RangerPolicy policy = (RangerPolicy) object; + // exact match + /*if (policy.getPolicyLabels().contains(policyLabels)) { + ret = true; + }*/ + /*partial match*/ + for (String label :policy.getPolicyLabels()){ + ret = StringUtils.containsIgnoreCase(label, policyLabels); + if(ret){ + return ret; + } + } + + } else { + ret = true; + } + return ret; + } + }; + if (predicates != null) { + predicates.add(ret); + } + + return ret; + } + + private Predicate addPredicateForPolicyPriority(final String policyPriority, List predicates) { + if(StringUtils.isEmpty(policyPriority)) { + return null; + } + + Predicate ret = new Predicate() { + @Override + public boolean evaluate(Object object) { + if (object == null) { + return false; + } + + boolean ret = true; + + if (object instanceof RangerPolicy) { + RangerPolicy policy = (RangerPolicy) object; + + Integer priority = policy.getPolicyPriority() != null ? policy.getPolicyPriority() : RangerPolicy.POLICY_PRIORITY_NORMAL; + + if (priority == RangerPolicy.POLICY_PRIORITY_NORMAL) { + ret = StringUtils.equalsIgnoreCase(policyPriority, policy.POLICY_PRIORITY_NAME_NORMAL) + || StringUtils.equalsIgnoreCase(policyPriority, priority.toString()); + } else if (priority == RangerPolicy.POLICY_PRIORITY_OVERRIDE) { + ret = StringUtils.equalsIgnoreCase(policyPriority, policy.POLICY_PRIORITY_NAME_OVERRIDE) + || StringUtils.equalsIgnoreCase(policyPriority, priority.toString()); + } else { + ret = false; + } + } + return ret; + } + + }; + + if(predicates != null) { + predicates.add(ret); + } + + return ret; + } + + + public Predicate createPredicateForResourceSignature(final String policySignature) { + + if (StringUtils.isEmpty(policySignature)) { + return null; + } + + return new Predicate() { + @Override + public boolean evaluate(Object object) { + if(object == null) { + return false; + } + + boolean ret = false; + + if (object instanceof RangerPolicy) { + RangerPolicy policy = (RangerPolicy)object; + + ret = StringUtils.equals(policy.getResourceSignature(), policySignature); + } else { + ret = true; + } + + return ret; + } + }; + } + private Predicate addPredicateForZoneName(final String zoneName, List predicates) { + + if(StringUtils.isEmpty(zoneName)) { + return null; + } + + Predicate ret = new Predicate() { + @Override + public boolean evaluate(Object object) { + if(object == null) { + return false; + } + + final boolean ret; + + if(object instanceof RangerPolicy) { + RangerPolicy policy = (RangerPolicy)object; + + if (policy.getZoneName() != null) { + ret = StringUtils.equals(zoneName, policy.getZoneName()); + } else { + ret = StringUtils.isEmpty(zoneName); + } + } else if (object instanceof RangerSecurityZone) { + RangerSecurityZone securityZone = (RangerSecurityZone)object; + + return StringUtils.equals(securityZone.getName(), zoneName); + } else { + ret = true; + } + + return ret; + } + }; + + if(predicates != null) { + predicates.add(ret); + } + + return ret; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/AbstractServiceStore.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/AbstractServiceStore.java new file mode 100644 index 00000000000..a60df515de4 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/AbstractServiceStore.java @@ -0,0 +1,644 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.store; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.authorization.hadoop.config.RangerAdminConfig; +import org.apache.atlas.plugin.model.RangerBaseModelObject; +import org.apache.atlas.plugin.model.RangerPolicy; +import org.apache.atlas.plugin.model.RangerService; +import org.apache.atlas.plugin.model.RangerServiceDef; +import org.apache.atlas.plugin.util.SearchFilter; +import org.apache.atlas.services.tag.RangerServiceTag; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Objects; + +public abstract class AbstractServiceStore implements ServiceStore { + private static final Log LOG = LogFactory.getLog(AbstractServiceStore.class); + + public static final String COMPONENT_ACCESSTYPE_SEPARATOR = ":"; + + public static final String AUTOPROPAGATE_ROWFILTERDEF_TO_TAG_PROP = "ranger.servicedef.autopropagate.rowfilterdef.to.tag"; + + public static final boolean AUTOPROPAGATE_ROWFILTERDEF_TO_TAG_PROP_DEFAULT = false; + + private static final int MAX_ACCESS_TYPES_IN_SERVICE_DEF = 1000; + + private final RangerAdminConfig config; + + // when a service-def is updated, the updated service-def should be made available to plugins + // this is achieved by incrementing policyVersion of all its services + protected abstract void updateServicesForServiceDefUpdate(RangerServiceDef serviceDef) throws Exception; + + protected AbstractServiceStore() { + this.config = RangerAdminConfig.getInstance(); + } + + @Override + public void updateTagServiceDefForAccessTypes() throws Exception { + if (LOG.isDebugEnabled()) { + LOG.debug("==> ServiceDefDBStore.updateTagServiceDefForAccessTypes()"); + } + List allServiceDefs = getServiceDefs(new SearchFilter()); + for (RangerServiceDef serviceDef : allServiceDefs) { + updateTagServiceDefForUpdatingAccessTypes(serviceDef); + } + if (LOG.isDebugEnabled()) { + LOG.debug("<== ServiceDefDBStore.updateTagServiceDefForAccessTypes()"); + } + } + + @Override + public PList getPaginatedServiceDefs(SearchFilter filter) throws Exception { + List resultList = getServiceDefs(filter); + + return CollectionUtils.isEmpty(resultList) ? new PList() : new PList(resultList, 0, resultList.size(), + (long) resultList.size(), resultList.size(), filter.getSortType(), filter.getSortBy()); + } + + @Override + public PList getPaginatedServices(SearchFilter filter) throws Exception { + List resultList = getServices(filter); + + return CollectionUtils.isEmpty(resultList) ? new PList() : new PList(resultList, 0, resultList.size(), (long) resultList.size(), + resultList.size(), filter.getSortType(), filter.getSortBy()); + } + + @Override + public PList getPaginatedPolicies(SearchFilter filter) throws Exception { + List resultList = getPolicies(filter); + + return CollectionUtils.isEmpty(resultList) ? new PList() : new PList(resultList, 0, resultList.size(), (long) resultList.size(), + resultList.size(), filter.getSortType(), filter.getSortBy()); + } + + @Override + public PList getPaginatedServicePolicies(Long serviceId, SearchFilter filter) throws Exception { + List resultList = getServicePolicies(serviceId, filter); + + return CollectionUtils.isEmpty(resultList) ? new PList() : new PList(resultList, 0, resultList.size(), (long) resultList.size(), + resultList.size(), filter.getSortType(), filter.getSortBy()); + } + + @Override + public PList getPaginatedServicePolicies(String serviceName, SearchFilter filter) throws Exception { + List resultList = getServicePolicies(serviceName, filter); + + return CollectionUtils.isEmpty(resultList) ? new PList() : new PList(resultList, 0, resultList.size(), (long) resultList.size(), + resultList.size(), filter.getSortType(), filter.getSortBy()); + } + + @Override + public Long getServicePolicyVersion(String serviceName) { + RangerService service = null; + try { + service = getServiceByName(serviceName); + } catch (Exception exception) { + LOG.error("Failed to get service object for service:" + serviceName); + } + return service != null ? service.getPolicyVersion() : null; + } + + protected void postCreate(RangerBaseModelObject obj) throws Exception { + if (obj instanceof RangerServiceDef) { + updateTagServiceDefForUpdatingAccessTypes((RangerServiceDef) obj); + } + } + + protected void postUpdate(RangerBaseModelObject obj) throws Exception { + if (obj instanceof RangerServiceDef) { + RangerServiceDef serviceDef = (RangerServiceDef) obj; + + updateTagServiceDefForUpdatingAccessTypes(serviceDef); + updateServicesForServiceDefUpdate(serviceDef); + } + } + + protected void postDelete(RangerBaseModelObject obj) throws Exception { + if (obj instanceof RangerServiceDef) { + updateTagServiceDefForDeletingAccessTypes(((RangerServiceDef) obj).getName()); + } + } + + public static long getNextVersion(Long currentVersion) { + return currentVersion == null ? 1L : currentVersion + 1; + } + + private RangerServiceDef.RangerAccessTypeDef findAccessTypeDef(long itemId, List accessTypeDefs) { + RangerServiceDef.RangerAccessTypeDef ret = null; + + for (RangerServiceDef.RangerAccessTypeDef accessTypeDef : accessTypeDefs) { + if (itemId == accessTypeDef.getItemId()) { + ret = accessTypeDef; + break; + } + } + return ret; + } + + private boolean updateTagAccessTypeDef(RangerServiceDef.RangerAccessTypeDef tagAccessType, RangerServiceDef.RangerAccessTypeDef svcAccessType, String prefix) { + + boolean isUpdated = false; + + if (!Objects.equals(tagAccessType.getName().substring(prefix.length()), svcAccessType.getName())) { + isUpdated = true; + } else if (!Objects.equals(tagAccessType.getLabel(), svcAccessType.getLabel())) { + isUpdated = true; + } else if (!Objects.equals(tagAccessType.getRbKeyLabel(), svcAccessType.getRbKeyLabel())) { + isUpdated = true; + } else { + Collection tagImpliedGrants = tagAccessType.getImpliedGrants(); + Collection svcImpliedGrants = svcAccessType.getImpliedGrants(); + + int tagImpliedGrantsLen = tagImpliedGrants == null ? 0 : tagImpliedGrants.size(); + int svcImpliedGrantsLen = svcImpliedGrants == null ? 0 : svcImpliedGrants.size(); + + if (tagImpliedGrantsLen != svcImpliedGrantsLen) { + isUpdated = true; + } else if (tagImpliedGrantsLen > 0) { + for (String svcImpliedGrant : svcImpliedGrants) { + if (!tagImpliedGrants.contains(prefix + svcImpliedGrant)) { + isUpdated = true; + break; + } + } + } + } + + if (isUpdated) { + tagAccessType.setName(prefix + svcAccessType.getName()); + tagAccessType.setLabel(svcAccessType.getLabel()); + tagAccessType.setRbKeyLabel(svcAccessType.getRbKeyLabel()); + + tagAccessType.setImpliedGrants(new HashSet()); + if (CollectionUtils.isNotEmpty(svcAccessType.getImpliedGrants())) { + for (String svcImpliedGrant : svcAccessType.getImpliedGrants()) { + tagAccessType.getImpliedGrants().add(prefix + svcImpliedGrant); + } + } + } + return isUpdated; + } + + private boolean updateTagAccessTypeDefs(List svcDefAccessTypes, List tagDefAccessTypes, + long itemIdOffset, String prefix) { + + List toAdd = new ArrayList<>(); + List toUpdate = new ArrayList<>(); + List toDelete = new ArrayList<>(); + + for (RangerServiceDef.RangerAccessTypeDef svcAccessType : svcDefAccessTypes) { + long tagAccessTypeItemId = svcAccessType.getItemId() + itemIdOffset; + + RangerServiceDef.RangerAccessTypeDef tagAccessType = findAccessTypeDef(tagAccessTypeItemId, tagDefAccessTypes); + + if (tagAccessType == null) { + tagAccessType = new RangerServiceDef.RangerAccessTypeDef(); + + tagAccessType.setItemId(tagAccessTypeItemId); + tagAccessType.setName(prefix + svcAccessType.getName()); + tagAccessType.setLabel(svcAccessType.getLabel()); + tagAccessType.setRbKeyLabel(svcAccessType.getRbKeyLabel()); + + tagAccessType.setImpliedGrants(new HashSet()); + if (CollectionUtils.isNotEmpty(svcAccessType.getImpliedGrants())) { + for (String svcImpliedGrant : svcAccessType.getImpliedGrants()) { + tagAccessType.getImpliedGrants().add(prefix + svcImpliedGrant); + } + } + + toAdd.add(tagAccessType); + } + } + + + for (RangerServiceDef.RangerAccessTypeDef tagAccessType : tagDefAccessTypes) { + if (tagAccessType.getName().startsWith(prefix)) { + long svcAccessTypeItemId = tagAccessType.getItemId() - itemIdOffset; + + RangerServiceDef.RangerAccessTypeDef svcAccessType = findAccessTypeDef(svcAccessTypeItemId, svcDefAccessTypes); + + if (svcAccessType == null) { // accessType has been deleted in service + toDelete.add(tagAccessType); + } else if (updateTagAccessTypeDef(tagAccessType, svcAccessType, prefix)) { + toUpdate.add(tagAccessType); + } + } + } + + boolean updateNeeded = false; + + if (CollectionUtils.isNotEmpty(toAdd) || CollectionUtils.isNotEmpty(toUpdate) || CollectionUtils.isNotEmpty(toDelete)) { + if (LOG.isDebugEnabled()) { + for (RangerServiceDef.RangerAccessTypeDef accessTypeDef : toDelete) { + LOG.debug("accessTypeDef-to-delete:[" + accessTypeDef + "]"); + } + + for (RangerServiceDef.RangerAccessTypeDef accessTypeDef : toUpdate) { + LOG.debug("accessTypeDef-to-update:[" + accessTypeDef + "]"); + } + for (RangerServiceDef.RangerAccessTypeDef accessTypeDef : toAdd) { + LOG.debug("accessTypeDef-to-add:[" + accessTypeDef + "]"); + } + } + + tagDefAccessTypes.addAll(toAdd); + tagDefAccessTypes.removeAll(toDelete); + + updateNeeded = true; + } + return updateNeeded; + } + + private void updateTagServiceDefForUpdatingAccessTypes(RangerServiceDef serviceDef) throws Exception { + if (StringUtils.equals(serviceDef.getName(), EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_TAG_NAME)) { + return; + } + + if (EmbeddedServiceDefsUtil.instance().getTagServiceDefId() == -1) { + LOG.info("AbstractServiceStore.updateTagServiceDefForUpdatingAccessTypes(" + serviceDef.getName() + "): tag service-def does not exist"); + } + + RangerServiceDef tagServiceDef; + try { + tagServiceDef = this.getServiceDef(EmbeddedServiceDefsUtil.instance().getTagServiceDefId()); + } catch (Exception e) { + LOG.error("AbstractServiceStore.updateTagServiceDefForUpdatingAccessTypes" + serviceDef.getName() + "): could not find TAG ServiceDef.. ", e); + throw e; + } + + if (tagServiceDef == null) { + LOG.error("AbstractServiceStore.updateTagServiceDefForUpdatingAccessTypes(" + serviceDef.getName() + "): could not find TAG ServiceDef.. "); + + return; + } + + String serviceDefName = serviceDef.getName(); + String prefix = serviceDefName + COMPONENT_ACCESSTYPE_SEPARATOR; + + List svcDefAccessTypes = serviceDef.getAccessTypes(); + List tagDefAccessTypes = tagServiceDef.getAccessTypes(); + + long itemIdOffset = serviceDef.getId() * (MAX_ACCESS_TYPES_IN_SERVICE_DEF + 1); + + boolean updateNeeded = updateTagAccessTypeDefs(svcDefAccessTypes, tagDefAccessTypes, itemIdOffset, prefix); + + if (updateTagServiceDefForUpdatingDataMaskDef(tagServiceDef, serviceDef, itemIdOffset, prefix)) { + updateNeeded = true; + } + + if (updateTagServiceDefForUpdatingRowFilterDef(tagServiceDef, serviceDef, itemIdOffset, prefix)) { + updateNeeded = true; + } + + boolean resourceUpdated = updateResourceInTagServiceDef(tagServiceDef); + + updateNeeded = updateNeeded || resourceUpdated; + + if (updateNeeded) { + try { + updateServiceDef(tagServiceDef); + LOG.info("AbstractServiceStore.updateTagServiceDefForUpdatingAccessTypes -- updated TAG service def with " + serviceDefName + " access types"); + } catch (Exception e) { + LOG.error("AbstractServiceStore.updateTagServiceDefForUpdatingAccessTypes -- Failed to update TAG ServiceDef.. ", e); + throw e; + } + } + } + + private void updateTagServiceDefForDeletingAccessTypes(String serviceDefName) throws Exception { + if (EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_TAG_NAME.equals(serviceDefName)) { + return; + } + + RangerServiceDef tagServiceDef; + try { + tagServiceDef = this.getServiceDef(EmbeddedServiceDefsUtil.instance().getTagServiceDefId()); + } catch (Exception e) { + LOG.error("AbstractServiceStore.updateTagServiceDefForDeletingAccessTypes(" + serviceDefName + "): could not find TAG ServiceDef.. ", e); + throw e; + } + + if (tagServiceDef == null) { + LOG.error("AbstractServiceStore.updateTagServiceDefForDeletingAccessTypes(" + serviceDefName + "): could not find TAG ServiceDef.. "); + + return; + } + + List accessTypes = new ArrayList<>(); + + for (RangerServiceDef.RangerAccessTypeDef accessType : tagServiceDef.getAccessTypes()) { + if (accessType.getName().startsWith(serviceDefName + COMPONENT_ACCESSTYPE_SEPARATOR)) { + accessTypes.add(accessType); + } + } + + tagServiceDef.getAccessTypes().removeAll(accessTypes); + + updateTagServiceDefForDeletingDataMaskDef(tagServiceDef, serviceDefName); + + updateTagServiceDefForDeletingRowFilterDef(tagServiceDef, serviceDefName); + + updateResourceInTagServiceDef(tagServiceDef); + + try { + updateServiceDef(tagServiceDef); + LOG.info("AbstractServiceStore.updateTagServiceDefForDeletingAccessTypes -- updated TAG service def with " + serviceDefName + " access types"); + } catch (Exception e) { + LOG.error("AbstractServiceStore.updateTagServiceDefForDeletingAccessTypes -- Failed to update TAG ServiceDef.. ", e); + throw e; + } + } + + private boolean updateTagServiceDefForUpdatingDataMaskDef(RangerServiceDef tagServiceDef, RangerServiceDef serviceDef, long itemIdOffset, String prefix) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> AbstractServiceStore.updateTagServiceDefForUpdatingDataMaskDef(" + serviceDef.getName() + ")"); + } + boolean ret = false; + + RangerServiceDef.RangerDataMaskDef svcDataMaskDef = serviceDef.getDataMaskDef(); + RangerServiceDef.RangerDataMaskDef tagDataMaskDef = tagServiceDef.getDataMaskDef(); + + List svcDefMaskTypes = svcDataMaskDef.getMaskTypes(); + List tagDefMaskTypes = tagDataMaskDef.getMaskTypes(); + + List svcDefAccessTypes = svcDataMaskDef.getAccessTypes(); + List tagDefAccessTypes = tagDataMaskDef.getAccessTypes(); + + List maskTypesToAdd = new ArrayList<>(); + List maskTypesToUpdate = new ArrayList<>(); + List maskTypesToDelete = new ArrayList<>(); + + for (RangerServiceDef.RangerDataMaskTypeDef svcMaskType : svcDefMaskTypes) { + long tagMaskTypeItemId = itemIdOffset + svcMaskType.getItemId(); + RangerServiceDef.RangerDataMaskTypeDef foundTagMaskType = null; + for (RangerServiceDef.RangerDataMaskTypeDef tagMaskType : tagDefMaskTypes) { + if (tagMaskType.getItemId().equals(tagMaskTypeItemId)) { + foundTagMaskType = tagMaskType; + break; + } + } + if (foundTagMaskType == null) { + RangerServiceDef.RangerDataMaskTypeDef tagMaskType = new RangerServiceDef.RangerDataMaskTypeDef(svcMaskType); + tagMaskType.setName(prefix + svcMaskType.getName()); + tagMaskType.setItemId(itemIdOffset + svcMaskType.getItemId()); + tagMaskType.setLabel(svcMaskType.getLabel()); + tagMaskType.setRbKeyLabel(svcMaskType.getRbKeyLabel()); + maskTypesToAdd.add(tagMaskType); + } + } + + for (RangerServiceDef.RangerDataMaskTypeDef tagMaskType : tagDefMaskTypes) { + if (StringUtils.startsWith(tagMaskType.getName(), prefix)) { + + RangerServiceDef.RangerDataMaskTypeDef foundSvcMaskType = null; + for (RangerServiceDef.RangerDataMaskTypeDef svcMaskType : svcDefMaskTypes) { + long tagMaskTypeItemId = itemIdOffset + svcMaskType.getItemId(); + if (tagMaskType.getItemId().equals(tagMaskTypeItemId)) { + foundSvcMaskType = svcMaskType; + break; + } + } + if (foundSvcMaskType == null) { + maskTypesToDelete.add(tagMaskType); + continue; + } + + RangerServiceDef.RangerDataMaskTypeDef checkTagMaskType = new RangerServiceDef.RangerDataMaskTypeDef(foundSvcMaskType); + + checkTagMaskType.setName(prefix + foundSvcMaskType.getName()); + checkTagMaskType.setItemId(itemIdOffset + foundSvcMaskType.getItemId()); + + if (!checkTagMaskType.equals(tagMaskType)) { + tagMaskType.setLabel(checkTagMaskType.getLabel()); + tagMaskType.setDescription(checkTagMaskType.getDescription()); + tagMaskType.setTransformer(checkTagMaskType.getTransformer()); + tagMaskType.setDataMaskOptions(checkTagMaskType.getDataMaskOptions()); + tagMaskType.setRbKeyLabel(checkTagMaskType.getRbKeyLabel()); + tagMaskType.setRbKeyDescription(checkTagMaskType.getRbKeyDescription()); + maskTypesToUpdate.add(tagMaskType); + } + } + } + + if (CollectionUtils.isNotEmpty(maskTypesToAdd) || CollectionUtils.isNotEmpty(maskTypesToUpdate) || CollectionUtils.isNotEmpty(maskTypesToDelete)) { + ret = true; + + if (LOG.isDebugEnabled()) { + for (RangerServiceDef.RangerDataMaskTypeDef maskTypeDef : maskTypesToDelete) { + LOG.debug("maskTypeDef-to-delete:[" + maskTypeDef + "]"); + } + + for (RangerServiceDef.RangerDataMaskTypeDef maskTypeDef : maskTypesToUpdate) { + LOG.debug("maskTypeDef-to-update:[" + maskTypeDef + "]"); + } + + for (RangerServiceDef.RangerDataMaskTypeDef maskTypeDef : maskTypesToAdd) { + LOG.debug("maskTypeDef-to-add:[" + maskTypeDef + "]"); + } + } + + tagDefMaskTypes.removeAll(maskTypesToDelete); + tagDefMaskTypes.addAll(maskTypesToAdd); + + tagDataMaskDef.setMaskTypes(tagDefMaskTypes); + } + + boolean tagMaskDefAccessTypesUpdated = updateTagAccessTypeDefs(svcDefAccessTypes, tagDefAccessTypes, itemIdOffset, prefix); + + if (tagMaskDefAccessTypesUpdated) { + tagDataMaskDef.setAccessTypes(tagDefAccessTypes); + ret = true; + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== AbstractServiceStore.updateTagServiceDefForUpdatingDataMaskDef(" + serviceDef.getName() + ") : " + ret); + } + + return ret; + } + + private void updateTagServiceDefForDeletingDataMaskDef(RangerServiceDef tagServiceDef, String serviceDefName) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> AbstractServiceStore.updateTagServiceDefForDeletingDataMaskDef(" + serviceDefName + ")"); + } + RangerServiceDef.RangerDataMaskDef tagDataMaskDef = tagServiceDef.getDataMaskDef(); + + if (tagDataMaskDef == null) { + return; + } + + String prefix = serviceDefName + COMPONENT_ACCESSTYPE_SEPARATOR; + + List accessTypes = new ArrayList<>(); + + for (RangerServiceDef.RangerAccessTypeDef accessType : tagDataMaskDef.getAccessTypes()) { + if (accessType.getName().startsWith(prefix)) { + accessTypes.add(accessType); + } + } + List maskTypes = new ArrayList<>(); + for (RangerServiceDef.RangerDataMaskTypeDef maskType : tagDataMaskDef.getMaskTypes()) { + if (maskType.getName().startsWith(prefix)) { + maskTypes.add(maskType); + } + } + tagDataMaskDef.getAccessTypes().removeAll(accessTypes); + tagDataMaskDef.getMaskTypes().removeAll(maskTypes); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== AbstractServiceStore.updateTagServiceDefForDeletingDataMaskDef(" + serviceDefName + ")"); + } + } + + private boolean updateTagServiceDefForUpdatingRowFilterDef(RangerServiceDef tagServiceDef, RangerServiceDef serviceDef, long itemIdOffset, String prefix) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> AbstractServiceStore.updateTagServiceDefForUpdatingRowFilterDef(" + serviceDef.getName() + ")"); + } + boolean ret = false; + + boolean autopropagateRowfilterdefToTag = config.getBoolean(AUTOPROPAGATE_ROWFILTERDEF_TO_TAG_PROP, AUTOPROPAGATE_ROWFILTERDEF_TO_TAG_PROP_DEFAULT); + + if (autopropagateRowfilterdefToTag) { + RangerServiceDef.RangerRowFilterDef svcRowFilterDef = serviceDef.getRowFilterDef(); + RangerServiceDef.RangerRowFilterDef tagRowFilterDef = tagServiceDef.getRowFilterDef(); + + List svcDefAccessTypes = svcRowFilterDef.getAccessTypes(); + List tagDefAccessTypes = tagRowFilterDef.getAccessTypes(); + + boolean tagRowFilterAccessTypesUpdated = updateTagAccessTypeDefs(svcDefAccessTypes, tagDefAccessTypes, itemIdOffset, prefix); + + if (tagRowFilterAccessTypesUpdated) { + tagRowFilterDef.setAccessTypes(tagDefAccessTypes); + ret = true; + } + } + if (LOG.isDebugEnabled()) { + LOG.debug("<== AbstractServiceStore.updateTagServiceDefForUpdatingRowFilterDef(" + serviceDef.getName() + ") : " + ret); + } + + return ret; + } + + private void updateTagServiceDefForDeletingRowFilterDef(RangerServiceDef tagServiceDef, String serviceDefName) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> AbstractServiceStore.updateTagServiceDefForDeletingRowFilterDef(" + serviceDefName + ")"); + } + RangerServiceDef.RangerRowFilterDef tagRowFilterDef = tagServiceDef.getRowFilterDef(); + + if (tagRowFilterDef == null) { + return; + } + + String prefix = serviceDefName + COMPONENT_ACCESSTYPE_SEPARATOR; + + List accessTypes = new ArrayList<>(); + + for (RangerServiceDef.RangerAccessTypeDef accessType : tagRowFilterDef.getAccessTypes()) { + if (accessType.getName().startsWith(prefix)) { + accessTypes.add(accessType); + } + } + + tagRowFilterDef.getAccessTypes().removeAll(accessTypes); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== AbstractServiceStore.updateTagServiceDefForDeletingRowFilterDef(" + serviceDefName + ")"); + } + } + + private boolean updateResourceInTagServiceDef(RangerServiceDef tagServiceDef) throws Exception { + if (LOG.isDebugEnabled()) { + LOG.debug("==> AbstractServiceStore.updateResourceInTagServiceDef(" + tagServiceDef + ")"); + } + boolean ret = false; + + final RangerServiceDef.RangerResourceDef accessPolicyTagResource = getResourceDefForTagResource(tagServiceDef.getResources()); + + final List resources = new ArrayList<>(); + + if (accessPolicyTagResource == null) { + LOG.warn("Resource with name :[" + RangerServiceTag.TAG_RESOURCE_NAME + "] not found in tag-service-definition!!"); + } else { + resources.add(accessPolicyTagResource); + } + + RangerServiceDef.RangerDataMaskDef dataMaskDef = tagServiceDef.getDataMaskDef(); + + if (dataMaskDef != null) { + if (CollectionUtils.isNotEmpty(dataMaskDef.getAccessTypes())) { + if (CollectionUtils.isEmpty(dataMaskDef.getResources())) { + dataMaskDef.setResources(resources); + ret = true; + } + } else { + if (CollectionUtils.isNotEmpty(dataMaskDef.getResources())) { + dataMaskDef.setResources(null); + ret = true; + } + } + } + + RangerServiceDef.RangerRowFilterDef rowFilterDef = tagServiceDef.getRowFilterDef(); + + if (rowFilterDef != null) { + boolean autopropagateRowfilterdefToTag = config.getBoolean(AUTOPROPAGATE_ROWFILTERDEF_TO_TAG_PROP, AUTOPROPAGATE_ROWFILTERDEF_TO_TAG_PROP_DEFAULT); + if (autopropagateRowfilterdefToTag) { + if (CollectionUtils.isNotEmpty(rowFilterDef.getAccessTypes())) { + if (CollectionUtils.isEmpty(rowFilterDef.getResources())) { + rowFilterDef.setResources(resources); + ret = true; + } + } else { + if (CollectionUtils.isNotEmpty(rowFilterDef.getResources())) { + rowFilterDef.setResources(null); + ret = true; + } + } + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== AbstractServiceStore.updateResourceInTagServiceDef(" + tagServiceDef + ") : " + ret); + } + return ret; + } + + private RangerServiceDef.RangerResourceDef getResourceDefForTagResource(List resourceDefs) { + RangerServiceDef.RangerResourceDef ret = null; + + if (CollectionUtils.isNotEmpty(resourceDefs)) { + for (RangerServiceDef.RangerResourceDef resourceDef : resourceDefs) { + if (resourceDef.getName().equals(RangerServiceTag.TAG_RESOURCE_NAME)) { + ret = resourceDef; + break; + } + } + } + return ret; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/AbstractTagStore.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/AbstractTagStore.java new file mode 100644 index 00000000000..e0f30e743d1 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/AbstractTagStore.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.store; + +public abstract class AbstractTagStore implements TagStore { + + protected ServiceStore svcStore; + + @Override + public void init() throws Exception { + // Empty + } + + @Override + final public void setServiceStore(ServiceStore svcStore) { + this.svcStore = svcStore; + } + + @Override + final public ServiceStore getServiceStore() { + return svcStore; + } + +} + + diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/EmbeddedServiceDefsUtil.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/EmbeddedServiceDefsUtil.java new file mode 100755 index 00000000000..270141f22fb --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/EmbeddedServiceDefsUtil.java @@ -0,0 +1,363 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.store; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.authorization.hadoop.config.RangerAdminConfig; +import org.apache.atlas.plugin.model.RangerServiceDef; +import org.apache.atlas.plugin.util.ServiceDefUtil; + +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/* + * This utility class deals with service-defs embedded in ranger-plugins-common + * library (hdfs/hbase/hive/knox/storm/..). If any of these service-defs + * don't exist in the given service store, they will be created in the store + * using the embedded definitions. + * + * init() method should be called from ServiceStore implementations to + * initialize embedded service-defs. + */ +public class EmbeddedServiceDefsUtil { + private static final Log LOG = LogFactory.getLog(EmbeddedServiceDefsUtil.class); + + + // following servicedef list should be reviewed/updated whenever a new embedded service-def is added + public static final String DEFAULT_BOOTSTRAP_SERVICEDEF_LIST = "tag,hdfs,hbase,hive,kms,knox,storm,yarn,kafka,solr,atlas,nifi,nifi-registry,sqoop,kylin,elasticsearch,presto,ozone,kudu,schema-registry"; + private static final String PROPERTY_SUPPORTED_SERVICE_DEFS = "ranger.supportedcomponents"; + private Set supportedServiceDefs; + public static final String EMBEDDED_SERVICEDEF_TAG_NAME = "tag"; + public static final String EMBEDDED_SERVICEDEF_HDFS_NAME = "hdfs"; + public static final String EMBEDDED_SERVICEDEF_HBASE_NAME = "hbase"; + public static final String EMBEDDED_SERVICEDEF_HIVE_NAME = "hive"; + public static final String EMBEDDED_SERVICEDEF_KMS_NAME = "kms"; + public static final String EMBEDDED_SERVICEDEF_KNOX_NAME = "knox"; + public static final String EMBEDDED_SERVICEDEF_STORM_NAME = "storm"; + public static final String EMBEDDED_SERVICEDEF_YARN_NAME = "yarn"; + public static final String EMBEDDED_SERVICEDEF_KAFKA_NAME = "kafka"; + public static final String EMBEDDED_SERVICEDEF_SOLR_NAME = "solr"; + public static final String EMBEDDED_SERVICEDEF_SCHEMA_REGISTRY_NAME = "schema-registry"; + public static final String EMBEDDED_SERVICEDEF_NIFI_NAME = "nifi"; + public static final String EMBEDDED_SERVICEDEF_NIFI_REGISTRY_NAME = "nifi-registry"; + public static final String EMBEDDED_SERVICEDEF_ATLAS_NAME = "atlas"; + public static final String EMBEDDED_SERVICEDEF_WASB_NAME = "wasb"; + public static final String EMBEDDED_SERVICEDEF_SQOOP_NAME = "sqoop"; + public static final String EMBEDDED_SERVICEDEF_KYLIN_NAME = "kylin"; + public static final String EMBEDDED_SERVICEDEF_ABFS_NAME = "abfs"; + public static final String EMBEDDED_SERVICEDEF_ELASTICSEARCH_NAME = "elasticsearch"; + public static final String EMBEDDED_SERVICEDEF_PRESTO_NAME = "presto"; + public static final String EMBEDDED_SERVICEDEF_OZONE_NAME = "ozone"; + public static final String EMBEDDED_SERVICEDEF_KUDU_NAME = "kudu"; + + public static final String PROPERTY_CREATE_EMBEDDED_SERVICE_DEFS = "ranger.service.store.create.embedded.service-defs"; + + public static final String HDFS_IMPL_CLASS_NAME = "org.apache.atlas.services.hdfs.RangerServiceHdfs"; + public static final String HBASE_IMPL_CLASS_NAME = "org.apache.atlas.services.hbase.RangerServiceHBase"; + public static final String HIVE_IMPL_CLASS_NAME = "org.apache.atlas.services.hive.RangerServiceHive"; + public static final String KMS_IMPL_CLASS_NAME = "org.apache.atlas.services.kms.RangerServiceKMS"; + public static final String KNOX_IMPL_CLASS_NAME = "org.apache.atlas.services.knox.RangerServiceKnox"; + public static final String STORM_IMPL_CLASS_NAME = "org.apache.atlas.services.storm.RangerServiceStorm"; + public static final String YARN_IMPL_CLASS_NAME = "org.apache.atlas.services.yarn.RangerServiceYarn"; + public static final String KAFKA_IMPL_CLASS_NAME = "org.apache.atlas.services.kafka.RangerServiceKafka"; + public static final String SOLR_IMPL_CLASS_NAME = "org.apache.atlas.services.solr.RangerServiceSolr"; + public static final String SCHEMA_REGISTRY_IMPL_CLASS_NAME = "org.apache.atlas.services.schemaregistry.RangerServiceSchemaRegistry"; + public static final String NIFI_IMPL_CLASS_NAME = "org.apache.atlas.services.nifi.RangerServiceNiFi"; + public static final String ATLAS_IMPL_CLASS_NAME = "org.apache.atlas.services.atlas.RangerServiceAtlas"; + public static final String PRESTO_IMPL_CLASS_NAME = "org.apache.atlas.services.presto.RangerServicePresto"; + public static final String OZONE_IMPL_CLASS_NAME = "org.apache.atlas.services.ozone.RangerServiceOzone"; + public static final String KUDU_IMPL_CLASS_NAME = "org.apache.atlas.services.kudu.RangerServiceKudu"; + + private static EmbeddedServiceDefsUtil instance = new EmbeddedServiceDefsUtil(); + + private boolean createEmbeddedServiceDefs = true; + private RangerServiceDef hdfsServiceDef; + private RangerServiceDef hBaseServiceDef; + private RangerServiceDef hiveServiceDef; + private RangerServiceDef kmsServiceDef; + private RangerServiceDef knoxServiceDef; + private RangerServiceDef stormServiceDef; + private RangerServiceDef yarnServiceDef; + private RangerServiceDef kafkaServiceDef; + private RangerServiceDef solrServiceDef; + private RangerServiceDef schemaRegistryServiceDef; + private RangerServiceDef nifiServiceDef; + private RangerServiceDef nifiRegistryServiceDef; + private RangerServiceDef atlasServiceDef; + private RangerServiceDef wasbServiceDef; + private RangerServiceDef sqoopServiceDef; + private RangerServiceDef kylinServiceDef; + private RangerServiceDef abfsServiceDef; + private RangerServiceDef elasticsearchServiceDef; + private RangerServiceDef prestoServiceDef; + private RangerServiceDef ozoneServiceDef; + private RangerServiceDef kuduServiceDef; + + private RangerServiceDef tagServiceDef; + + private final Gson gsonBuilder; + private final RangerAdminConfig config; + + /** Private constructor to restrict instantiation of this singleton utility class. */ + private EmbeddedServiceDefsUtil() { + gsonBuilder = new GsonBuilder().setDateFormat("yyyyMMdd-HH:mm:ss.SSS-Z").setPrettyPrinting().create(); + config = RangerAdminConfig.getInstance(); + } + + public static EmbeddedServiceDefsUtil instance() { + return instance; + } + + public void init(ServiceStore store) { + LOG.info("==> EmbeddedServiceDefsUtil.init()"); + + try { + createEmbeddedServiceDefs = config.getBoolean(PROPERTY_CREATE_EMBEDDED_SERVICE_DEFS, true); + + supportedServiceDefs =getSupportedServiceDef(); + /* + * Maintaining the following service-def create-order is critical for the + * the legacy service-defs (HDFS/HBase/Hive/Knox/Storm) to be assigned IDs + * that were used in earlier version (0.4) */ + hdfsServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_HDFS_NAME); + hBaseServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_HBASE_NAME); + hiveServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_HIVE_NAME); + kmsServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_KMS_NAME); + knoxServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_KNOX_NAME); + stormServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_STORM_NAME); + yarnServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_YARN_NAME); + kafkaServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_KAFKA_NAME); + solrServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_SOLR_NAME); + schemaRegistryServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_SCHEMA_REGISTRY_NAME); + nifiServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_NIFI_NAME); + nifiRegistryServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_NIFI_REGISTRY_NAME); + atlasServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_ATLAS_NAME); + + tagServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_TAG_NAME); + wasbServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_WASB_NAME); + sqoopServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_SQOOP_NAME); + kylinServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_KYLIN_NAME); + abfsServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_ABFS_NAME); + elasticsearchServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_ELASTICSEARCH_NAME); + prestoServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_PRESTO_NAME); + ozoneServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_OZONE_NAME); + kuduServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_KUDU_NAME); + + // Ensure that tag service def is updated with access types of all service defs + store.updateTagServiceDefForAccessTypes(); + } catch(Throwable excp) { + LOG.fatal("EmbeddedServiceDefsUtil.init(): failed", excp); + } + + LOG.info("<== EmbeddedServiceDefsUtil.init()"); + } + + public long getHdfsServiceDefId() { + return getId(hdfsServiceDef); + } + + public long getHBaseServiceDefId() { + return getId(hBaseServiceDef); + } + + public long getHiveServiceDefId() { + return getId(hiveServiceDef); + } + + public long getKmsServiceDefId() { + return getId(kmsServiceDef); + } + + public long getKnoxServiceDefId() { + return getId(knoxServiceDef); + } + + public long getStormServiceDefId() { + return getId(stormServiceDef); + } + + public long getYarnServiceDefId() { + return getId(yarnServiceDef); + } + + public long getKafkaServiceDefId() { + return getId(kafkaServiceDef); + } + + public long getSolrServiceDefId() { + return getId(solrServiceDef); + } + + public long getSchemaRegistryServiceDefId() { + return getId(schemaRegistryServiceDef); + } + + public long getNiFiServiceDefId() { + return getId(nifiServiceDef); + } + + public long getNiFiRegistryServiceDefId() { + return getId(nifiRegistryServiceDef); + } + + public long getAtlasServiceDefId() { + return getId(atlasServiceDef); + } + + public long getSqoopServiceDefId() { + return getId(sqoopServiceDef); + } + + public long getKylinServiceDefId() { + return getId(kylinServiceDef); + } + + public long getElasticsearchServiceDefId() { + return getId(elasticsearchServiceDef); + } + public long getTagServiceDefId() { return getId(tagServiceDef); } + + public long getWasbServiceDefId() { return getId(wasbServiceDef); } + + public long getAbfsServiceDefId() { return getId(abfsServiceDef); } + + public long getPrestoServiceDefId() { return getId(prestoServiceDef); } + + public long getOzoneServiceDefId() { return getId(ozoneServiceDef); } + + public long getKuduServiceDefId() { return getId(kuduServiceDef); } + + public RangerServiceDef getEmbeddedServiceDef(String defType) throws Exception { + RangerServiceDef serviceDef=null; + if(StringUtils.isNotEmpty(defType)){ + serviceDef=loadEmbeddedServiceDef(defType); + } + return serviceDef; + } + + public static boolean isRecursiveEnabled(final RangerServiceDef rangerServiceDef, final String resourceDefName) { + boolean ret = false; + List resourceDefs = rangerServiceDef.getResources(); + for(RangerServiceDef.RangerResourceDef resourceDef:resourceDefs) { + if (resourceDefName.equals(resourceDef.getName())) { + ret = resourceDef.getRecursiveSupported(); + break; + } + } + return ret; + } + + private long getId(RangerServiceDef serviceDef) { + return serviceDef == null || serviceDef.getId() == null ? -1 : serviceDef.getId().longValue(); + } + + private RangerServiceDef getOrCreateServiceDef(ServiceStore store, String serviceDefName) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> EmbeddedServiceDefsUtil.getOrCreateServiceDef(" + serviceDefName + ")"); + } + + RangerServiceDef ret = null; + boolean createServiceDef = (CollectionUtils.isEmpty(supportedServiceDefs) || supportedServiceDefs.contains(serviceDefName)); + try { + ret = store.getServiceDefByName(serviceDefName); + if(ret == null && createEmbeddedServiceDefs && createServiceDef) { + ret = ServiceDefUtil.normalize(loadEmbeddedServiceDef(serviceDefName)); + + LOG.info("creating embedded service-def " + serviceDefName); + if (ret.getId() != null) { + store.setPopulateExistingBaseFields(true); + try { + ret = store.createServiceDef(ret); + } finally { + store.setPopulateExistingBaseFields(false); + } + } else { + ret = store.createServiceDef(ret); + } + LOG.info("created embedded service-def " + serviceDefName); + } + } catch(Exception excp) { + LOG.fatal("EmbeddedServiceDefsUtil.getOrCreateServiceDef(): failed to load/create serviceType " + serviceDefName, excp); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== EmbeddedServiceDefsUtil.getOrCreateServiceDef(" + serviceDefName + "): " + ret); + } + + return ret; + } + + private RangerServiceDef loadEmbeddedServiceDef(String serviceType) throws Exception { + if(LOG.isDebugEnabled()) { + LOG.debug("==> EmbeddedServiceDefsUtil.loadEmbeddedServiceDef(" + serviceType + ")"); + } + + RangerServiceDef ret = null; + + String resource = "/service-defs/ranger-servicedef-" + serviceType + ".json"; + + InputStream inStream = getClass().getResourceAsStream(resource); + + InputStreamReader reader = new InputStreamReader(inStream); + + ret = gsonBuilder.fromJson(reader, RangerServiceDef.class); + + //Set DEFAULT displayName if missing + if (ret != null && StringUtils.isBlank(ret.getDisplayName())) { + ret.setDisplayName(ret.getName()); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("==> EmbeddedServiceDefsUtil.loadEmbeddedServiceDef(" + serviceType + ")"); + } + + return ret; + } + + private Set getSupportedServiceDef(){ + Set supportedServiceDef =new HashSet<>(); + try{ + String ranger_supportedcomponents = config.get(PROPERTY_SUPPORTED_SERVICE_DEFS, DEFAULT_BOOTSTRAP_SERVICEDEF_LIST); + if(StringUtils.isBlank(ranger_supportedcomponents) || "all".equalsIgnoreCase(ranger_supportedcomponents)){ + ranger_supportedcomponents=DEFAULT_BOOTSTRAP_SERVICEDEF_LIST; + } + String[] supportedComponents=ranger_supportedcomponents.split(","); + if(supportedComponents!=null && supportedComponents.length>0){ + for(String element:supportedComponents){ + if(!StringUtils.isBlank(element)){ + element=element.toLowerCase(); + supportedServiceDef.add(element); + } + } + } + }catch(Exception ex){ + LOG.error("EmbeddedServiceDefsUtil.getSupportedServiceDef(): failed", ex); + } + return supportedServiceDef; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/GeolocationStore.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/GeolocationStore.java new file mode 100644 index 00000000000..c995ba668ec --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/GeolocationStore.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.store; + +import org.apache.atlas.plugin.geo.RangerGeolocationData; +import org.apache.atlas.plugin.geo.RangerGeolocationDatabase; + +import java.util.Map; + +public interface GeolocationStore { + void init(Map context); + + RangerGeolocationData getGeoLocation(String ipAddress); + RangerGeolocationDatabase getGeoDatabase(); +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/PList.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/PList.java new file mode 100644 index 00000000000..e00e8627a4a --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/PList.java @@ -0,0 +1,194 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.store; + +import java.util.List; + +public class PList implements java.io.Serializable { + + private static final long serialVersionUID = 1L; + + /** + * Start index for the result + */ + protected int startIndex; + /** + * Page size used for the result + */ + protected int pageSize; + /** + * Total records in the database for the given search conditions + */ + protected long totalCount; + /** + * Number of rows returned for the search condition + */ + protected int resultSize; + /** + * Sort type. Either desc or asc + */ + protected String sortType; + /** + * Comma seperated list of the fields for sorting + */ + protected String sortBy; + + protected long queryTimeMS = System.currentTimeMillis(); + + protected List list; + /** + * Default constructor. This will set all the attributes to default value. + */ + public PList() { + startIndex = 0; + pageSize = 0; + totalCount = 0; + resultSize = 0; + sortType = null; + sortBy = null; + } + + public PList(List list, int startIndex, int pageSize, long totalCount, int resultSize, String sortType, String sortBy) { + this.list = list; + this.startIndex = startIndex; + this.pageSize = pageSize; + this.totalCount = totalCount; + this.resultSize = resultSize; + this.sortType = sortType; + this.sortBy = sortBy; + + } + + public int getListSize() { + return list == null ? 0 : list.size(); + } + + public void setList(List list) {this.list = list;} + + public List getList() { + return list; + } + + /** + * This method sets the value to the member attribute startIndex. You + * cannot set null to the attribute. + * + * @param startIndex + * Value to set member attribute startIndex + */ + public void setStartIndex(int startIndex) { + this.startIndex = startIndex; + } + public int getStartIndex() { return startIndex; } + + + /** + * This method sets the value to the member attribute pageSize. You + * cannot set null to the attribute. + * + * @param pageSize + * Value to set member attribute pageSize + */ + public void setPageSize(int pageSize) { + this.pageSize = pageSize; + } + public int getPageSize() { return pageSize; } + + + /** + * This method sets the value to the member attribute totalCount. You + * cannot set null to the attribute. + * + * @param totalCount + * Value to set member attribute totalCount + */ + public void setTotalCount(long totalCount) { + this.totalCount = totalCount; + } + public long getTotalCount() { return totalCount; } + + + + /** + * This method sets the value to the member attribute resultSize. You + * cannot set null to the attribute. + * + * @param resultSize + * Value to set member attribute resultSize + */ + public void setResultSize(int resultSize) { + this.resultSize = resultSize; + } + + /** + * Returns the value for the member attribute resultSize + * + * @return int - value of member attribute resultSize. + */ + public int getResultSize() { + return getListSize(); + } + + /** + * This method sets the value to the member attribute sortType. You + * cannot set null to the attribute. + * + * @param sortType + * Value to set member attribute sortType + */ + public void setSortType(String sortType) { + this.sortType = sortType; + } + public String getSortType() { return sortType; } + + + + /** + * This method sets the value to the member attribute sortBy. You + * cannot set null to the attribute. + * + * @param sortBy + * Value to set member attribute sortBy + */ + public void setSortBy(String sortBy) { + this.sortBy = sortBy; + } + public String getSortBy() { return sortBy; } + + + + + + + + /* + * (non-Javadoc) + * + * @see java.lang.Object#toString() + */ + @Override + public String toString() { + return "PList [startIndex=" + startIndex + ", pageSize=" + + pageSize + ", totalCount=" + totalCount + + ", resultSize=" + resultSize + ", sortType=" + + sortType + ", sortBy=" + sortBy + ", queryTimeMS=" + + queryTimeMS + "]"; + } +} \ No newline at end of file diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/RangerServiceResourceSignature.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/RangerServiceResourceSignature.java new file mode 100644 index 00000000000..1a399abbc6a --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/RangerServiceResourceSignature.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.store; + +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.atlas.authorization.hadoop.config.RangerAdminConfig; +import org.apache.atlas.plugin.model.RangerPolicy; +import org.apache.atlas.plugin.model.RangerServiceResource; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +public class RangerServiceResourceSignature { + private final String _string; + private final String _hash; + + public RangerServiceResourceSignature(RangerServiceResource serviceResource) { + _string = ServiceResourceSerializer.toString(serviceResource); + if (RangerAdminConfig.getInstance().isFipsEnabled()) { + _hash = DigestUtils.sha512Hex(_string); + } else { + _hash = DigestUtils.sha256Hex(_string); + } + } + + String asString() { + return _string; + } + + public String getSignature() { + return _hash; + } + + static class ServiceResourceSerializer { + + static final int _SignatureVersion = 1; + + static public String toString(final RangerServiceResource serviceResource) { + // invalid/empty serviceResource gets a deterministic signature as if it had an + // empty resource string + Map resource = serviceResource.getResourceElements(); + Map resources = new TreeMap<>(); + for (Map.Entry entry : resource.entrySet()) { + String resourceName = entry.getKey(); + ResourceSerializer resourceView = new ResourceSerializer(entry.getValue()); + resources.put(resourceName, resourceView); + } + String resourcesAsString = resources.toString(); + return String.format("{version=%d,resource=%s}", _SignatureVersion, resourcesAsString); + } + + static class ResourceSerializer { + final RangerPolicy.RangerPolicyResource _policyResource; + + ResourceSerializer(RangerPolicy.RangerPolicyResource policyResource) { + _policyResource = policyResource; + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("{"); + if (_policyResource != null) { + builder.append("values="); + if (_policyResource.getValues() != null) { + List values = new ArrayList<>(_policyResource.getValues()); + Collections.sort(values); + builder.append(values); + } + + builder.append(",excludes="); + if (_policyResource.getIsExcludes() == null) { // null is same as false + builder.append(Boolean.FALSE); + } else { + builder.append(_policyResource.getIsExcludes()); + } + + builder.append(",recursive="); + if (_policyResource.getIsRecursive() == null) { // null is the same as false + builder.append(Boolean.FALSE); + } else { + builder.append(_policyResource.getIsRecursive()); + } + } + builder.append("}"); + return builder.toString(); + } + } + } +} + diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/RolePredicateUtil.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/RolePredicateUtil.java new file mode 100644 index 00000000000..0f8b6ffe905 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/RolePredicateUtil.java @@ -0,0 +1,315 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.store; + +import org.apache.commons.collections.Predicate; +import org.apache.commons.lang.StringUtils; +import org.apache.atlas.plugin.model.RangerRole; +import org.apache.atlas.plugin.util.SearchFilter; + +import java.util.List; + +public class RolePredicateUtil extends AbstractPredicateUtil { + + public RolePredicateUtil() { + super(); + } + + @Override + public void addPredicates(SearchFilter filter, List predicates) { + addPredicateForRoleName(filter.getParam(SearchFilter.ROLE_NAME), predicates); + addPredicateForRoleId(filter.getParam(SearchFilter.ROLE_ID), predicates); + addPredicateForGroupName(filter.getParam(SearchFilter.GROUP_NAME), predicates); + addPredicateForUserName(filter.getParam(SearchFilter.USER_NAME), predicates); + + addPredicateForPartialRoleName(filter.getParam(SearchFilter.ROLE_NAME_PARTIAL), predicates); + addPredicateForPartialGroupName(filter.getParam(SearchFilter.GROUP_NAME_PARTIAL), predicates); + addPredicateForPartialUserName(filter.getParam(SearchFilter.USER_NAME_PARTIAL), predicates); + } + + private Predicate addPredicateForRoleName(final String roleName, List predicates) { + if(StringUtils.isEmpty(roleName)) { + return null; + } + + Predicate ret = new Predicate() { + @Override + public boolean evaluate(Object object) { + if(object == null) { + return false; + } + + boolean ret = false; + + if(object instanceof RangerRole) { + RangerRole role = (RangerRole) object; + + ret = StringUtils.equals(role.getName(), roleName); + + if (!ret) { + List roles = role.getRoles(); + + for (RangerRole.RoleMember member : roles) { + ret = StringUtils.equals(role.getName(), roleName); + + if (ret) { + break; + } + } + } + } + return ret; + } + }; + + if(predicates != null) { + predicates.add(ret); + } + + return ret; + } + + private Predicate addPredicateForPartialRoleName(final String roleNamePartial, List predicates) { + if(StringUtils.isEmpty(roleNamePartial)) { + return null; + } + + Predicate ret = new Predicate() { + @Override + public boolean evaluate(Object object) { + if(object == null) { + return false; + } + + boolean ret = false; + + if(object instanceof RangerRole) { + RangerRole role = (RangerRole) object; + + ret = StringUtils.containsIgnoreCase(role.getName(), roleNamePartial); + + if (!ret) { + List roles = role.getRoles(); + + for (RangerRole.RoleMember member : roles) { + ret = StringUtils.containsIgnoreCase(role.getName(), roleNamePartial); + + if (ret) { + break; + } + } + } + } + return ret; + } + }; + + if(predicates != null) { + predicates.add(ret); + } + + return ret; + } + + private Predicate addPredicateForRoleId(final String roleId, List predicates) { + if(StringUtils.isEmpty(roleId)) { + return null; + } + + Predicate ret = new Predicate() { + @Override + public boolean evaluate(Object object) { + if(object == null) { + return false; + } + + boolean ret = false; + + if(object instanceof RangerRole) { + RangerRole role = (RangerRole) object; + + ret = StringUtils.equals(roleId, role.getId().toString()); + } + + return ret; + } + }; + + if(predicates != null) { + predicates.add(ret); + } + + return ret; + } + + private Predicate addPredicateForGroupName(final String groupName, List predicates) { + if(StringUtils.isEmpty(groupName)) { + return null; + } + + Predicate ret = new Predicate() { + @Override + public boolean evaluate(Object object) { + if(object == null) { + return false; + } + + boolean ret = false; + + if(object instanceof RangerRole) { + RangerRole role = (RangerRole) object; + + List groups = role.getGroups(); + + for (RangerRole.RoleMember member : groups) { + ret = StringUtils.equals(member.getName(), groupName); + + if (ret) { + break; + } + } + } + return ret; + } + }; + + if(predicates != null) { + predicates.add(ret); + } + + return ret; + } + + private Predicate addPredicateForPartialGroupName(final String groupNamePartial, List predicates) { + if(StringUtils.isEmpty(groupNamePartial)) { + return null; + } + + Predicate ret = new Predicate() { + @Override + public boolean evaluate(Object object) { + if(object == null) { + return false; + } + + boolean ret = false; + + if(object instanceof RangerRole) { + RangerRole role = (RangerRole) object; + + List groups = role.getGroups(); + + for (RangerRole.RoleMember member : groups) { + ret = StringUtils.containsIgnoreCase(member.getName(), groupNamePartial); + + if (ret) { + break; + } + } + } + return ret; + } + }; + + if(predicates != null) { + predicates.add(ret); + } + + return ret; + } + + private Predicate addPredicateForUserName(final String userName, List predicates) { + if(StringUtils.isEmpty(userName)) { + return null; + } + + Predicate ret = new Predicate() { + @Override + public boolean evaluate(Object object) { + if(object == null) { + return false; + } + + boolean ret = false; + + if(object instanceof RangerRole) { + RangerRole role = (RangerRole) object; + + List users = role.getUsers(); + + for (RangerRole.RoleMember member : users) { + ret = StringUtils.equals(member.getName(), userName); + + if (ret) { + break; + } + } + } + return ret; + } + }; + + if(predicates != null) { + predicates.add(ret); + } + + return ret; + } + + private Predicate addPredicateForPartialUserName(final String userNamePartial, List predicates) { + if(StringUtils.isEmpty(userNamePartial)) { + return null; + } + + Predicate ret = new Predicate() { + @Override + public boolean evaluate(Object object) { + if(object == null) { + return false; + } + + boolean ret = false; + + if(object instanceof RangerRole) { + RangerRole role = (RangerRole) object; + + List users = role.getUsers(); + + for (RangerRole.RoleMember member : users) { + ret = StringUtils.containsIgnoreCase(member.getName(), userNamePartial); + + if (ret) { + break; + } + } + } + return ret; + } + }; + + if(predicates != null) { + predicates.add(ret); + } + + return ret; + } + +} + diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/RoleStore.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/RoleStore.java new file mode 100644 index 00000000000..c38c512c52f --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/RoleStore.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.store; + +import org.apache.atlas.plugin.model.RangerRole; +import org.apache.atlas.plugin.util.RangerRoles; +import org.apache.atlas.plugin.util.SearchFilter; + +import java.util.List; + +public interface RoleStore { + + void init() throws Exception; + + RangerRole createRole(RangerRole role, Boolean createNonExistUserGroup) throws Exception; + + RangerRole updateRole(RangerRole role, Boolean createNonExistUserGroup) throws Exception; + + void deleteRole(String roleName) throws Exception; + + void deleteRole(Long roleId) throws Exception; + + RangerRole getRole(Long id) throws Exception; + + RangerRole getRole(String name) throws Exception; + + List getRoles(SearchFilter filter) throws Exception; + + List getRoleNames(SearchFilter filter) throws Exception; + + RangerRoles getRoles(String serviceName, Long lastKnownRoleVersion) throws Exception; + + Long getRoleVersion(String serviceName); + + boolean roleExists(Long id) throws Exception; + + boolean roleExists(String name) throws Exception; +} + diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/SecurityZonePredicateUtil.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/SecurityZonePredicateUtil.java new file mode 100644 index 00000000000..bd2aa023fb9 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/SecurityZonePredicateUtil.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.store; + +import org.apache.commons.collections.Predicate; +import org.apache.commons.lang.StringUtils; +import org.apache.atlas.plugin.model.RangerSecurityZone; +import org.apache.atlas.plugin.util.SearchFilter; + +import java.util.List; + +public class SecurityZonePredicateUtil extends AbstractPredicateUtil { + + public SecurityZonePredicateUtil() { + super(); + } + + @Override + public void addPredicates(SearchFilter filter, List predicates) { + //super.addPredicates(filter, predicates); + + addPredicateForServiceName(filter.getParam(SearchFilter.SERVICE_NAME), predicates); + addPredicateForMatchingZoneId(filter.getParam(SearchFilter.ZONE_ID), predicates); + addPredicateForNonMatchingZoneName(filter.getParam(SearchFilter.ZONE_NAME), predicates); + } + + private Predicate addPredicateForServiceName(final String serviceName, List predicates) { + if(StringUtils.isEmpty(serviceName)) { + return null; + } + + Predicate ret = new Predicate() { + @Override + public boolean evaluate(Object object) { + if(object == null) { + return false; + } + + boolean ret = false; + + if(object instanceof RangerSecurityZone) { + RangerSecurityZone securityZone = (RangerSecurityZone) object; + + ret = securityZone.getServices().get(serviceName) != null; + } + + return ret; + } + }; + + if(predicates != null) { + predicates.add(ret); + } + + return ret; + } + + private Predicate addPredicateForMatchingZoneId(final String zoneId, List predicates) { + if (StringUtils.isEmpty(zoneId)) { + return null; + } + + Predicate ret = new Predicate() { + @Override + public boolean evaluate(Object object) { + if(object == null) { + return false; + } + + boolean ret = false; + + if(object instanceof RangerSecurityZone) { + RangerSecurityZone securityZone = (RangerSecurityZone) object; + + if (StringUtils.equals(zoneId, securityZone.getId().toString())) { + ret = true; + } + } + + return ret; + } + }; + + if(predicates != null) { + predicates.add(ret); + } + + return ret; + } + + private Predicate addPredicateForNonMatchingZoneName(final String zoneName, List predicates) { + + Predicate ret = new Predicate() { + @Override + public boolean evaluate(Object object) { + if(object == null) { + return false; + } + + boolean ret = false; + + if(object instanceof RangerSecurityZone) { + RangerSecurityZone securityZone = (RangerSecurityZone) object; + + if (StringUtils.isEmpty(zoneName) || !StringUtils.equals(zoneName, securityZone.getName())) { + ret = true; + } + } + + return ret; + } + }; + + if(predicates != null) { + predicates.add(ret); + } + + return ret; + } +} + diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/SecurityZoneStore.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/SecurityZoneStore.java new file mode 100644 index 00000000000..d23036be48f --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/SecurityZoneStore.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.store; + +import org.apache.atlas.plugin.model.RangerSecurityZone; +import org.apache.atlas.plugin.util.SearchFilter; + +import java.util.List; +import java.util.Map; + +public interface SecurityZoneStore { + + void init() throws Exception; + + RangerSecurityZone createSecurityZone(RangerSecurityZone securityZone) throws Exception; + + RangerSecurityZone updateSecurityZoneById(RangerSecurityZone securityZone) throws Exception; + + void deleteSecurityZoneByName(String zoneName) throws Exception; + + void deleteSecurityZoneById(Long zoneId) throws Exception; + + RangerSecurityZone getSecurityZone(Long id) throws Exception; + + RangerSecurityZone getSecurityZoneByName(String name) throws Exception; + + List getSecurityZones(SearchFilter filter) throws Exception; + + Map getSecurityZonesForService(String serviceName); + +} + diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/ServicePredicateUtil.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/ServicePredicateUtil.java new file mode 100644 index 00000000000..642ca2a7e70 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/ServicePredicateUtil.java @@ -0,0 +1,233 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.store; + +import org.apache.commons.collections.Predicate; +import org.apache.commons.lang.StringUtils; +import org.apache.atlas.plugin.model.RangerPolicy; +import org.apache.atlas.plugin.model.RangerService; +import org.apache.atlas.plugin.model.RangerServiceDef; +import org.apache.atlas.plugin.util.SearchFilter; + +import java.util.List; + +public class ServicePredicateUtil extends AbstractPredicateUtil { + private ServiceStore serviceStore; + + public ServicePredicateUtil(ServiceStore serviceStore) { + super(); + this.serviceStore = serviceStore; + } + + @Override + public void addPredicates(SearchFilter filter, List predicates) { + super.addPredicates(filter, predicates); + + addPredicateForServiceType(filter.getParam(SearchFilter.SERVICE_TYPE), predicates); + addPredicateForServiceId(filter.getParam(SearchFilter.SERVICE_ID), predicates); + addPredicateForTagSeviceName(filter.getParam(SearchFilter.TAG_SERVICE_NAME), predicates); + addPredicateForTagSeviceId(filter.getParam(SearchFilter.TAG_SERVICE_ID), predicates); + } + + private String getServiceType(String serviceName) { + RangerService service = null; + + try { + if (serviceStore != null) { + service = serviceStore.getServiceByName(serviceName); + } + } catch(Exception excp) { + // ignore + } + + return service != null ? service.getType() : null; + } + + private Long getServiceId(String serviceName) { + RangerService service = null; + + try { + if (serviceStore != null) { + service = serviceStore.getServiceByName(serviceName); + } + } catch(Exception excp) { + // ignore + } + + return service != null ? service.getId() : null; + } + + + private Predicate addPredicateForServiceType(final String serviceType, List predicates) { + if(StringUtils.isEmpty(serviceType)) { + return null; + } + + Predicate ret = new Predicate() { + @Override + public boolean evaluate(Object object) { + if(object == null) { + return false; + } + + boolean ret = false; + + if(object instanceof RangerPolicy) { + RangerPolicy policy = (RangerPolicy)object; + + ret = StringUtils.equals(serviceType, getServiceType(policy.getService())); + } else if(object instanceof RangerService) { + RangerService service = (RangerService)object; + + ret = StringUtils.equals(serviceType, service.getType()); + } else if(object instanceof RangerServiceDef) { + RangerServiceDef serviceDef = (RangerServiceDef)object; + + ret = StringUtils.equals(serviceType, serviceDef.getName()); + } + + return ret; + } + }; + + if(predicates != null) { + predicates.add(ret); + } + + return ret; + } + + private Predicate addPredicateForServiceId(final String serviceId, List predicates) { + if(StringUtils.isEmpty(serviceId)) { + return null; + } + + Predicate ret = new Predicate() { + @Override + public boolean evaluate(Object object) { + if(object == null) { + return false; + } + + boolean ret = false; + + if(object instanceof RangerPolicy) { + RangerPolicy policy = (RangerPolicy)object; + Long svcId = getServiceId(policy.getService()); + + if(svcId != null) { + ret = StringUtils.equals(serviceId, svcId.toString()); + } + } else if(object instanceof RangerService) { + RangerService service = (RangerService)object; + + if(service.getId() != null) { + ret = StringUtils.equals(serviceId, service.getId().toString()); + } + } else { + ret = true; + } + + return ret; + } + }; + + if(predicates != null) { + predicates.add(ret); + } + + return ret; + } + + private Predicate addPredicateForTagSeviceName(final String tagServiceName, List predicates) { + if(StringUtils.isEmpty(tagServiceName)) { + return null; + } + + Predicate ret = new Predicate() { + @Override + public boolean evaluate(Object object) { + if(object == null) { + return false; + } + + boolean ret = false; + + if(object instanceof RangerService) { + RangerService service = (RangerService)object; + + ret = StringUtils.equals(tagServiceName, service.getTagService()); + } else { + ret = true; + } + + return ret; + } + }; + + if(predicates != null) { + predicates.add(ret); + } + + return ret; + } + + private Predicate addPredicateForTagSeviceId(final String tagServiceId, List predicates) { + if(StringUtils.isEmpty(tagServiceId)) { + return null; + } + + Predicate ret = new Predicate() { + @Override + public boolean evaluate(Object object) { + if(object == null) { + return false; + } + + boolean ret = false; + + if(object instanceof RangerService) { + RangerService service = (RangerService)object; + + if(! StringUtils.isEmpty(service.getTagService())) { + RangerService tagService = null; + + try { + tagService = serviceStore.getServiceByName(service.getTagService()); + } catch(Exception excp) { + } + + ret = tagService != null && tagService.getId() != null && StringUtils.equals(tagServiceId, tagService.getId().toString()); + } + } else { + ret = true; + } + + return ret; + } + }; + + if(predicates != null) { + predicates.add(ret); + } + + return ret; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/ServiceStore.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/ServiceStore.java new file mode 100644 index 00000000000..91e31f74d2c --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/ServiceStore.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.store; + +import org.apache.atlas.plugin.model.RangerPolicy; +import org.apache.atlas.plugin.model.RangerSecurityZone; +import org.apache.atlas.plugin.model.RangerService; +import org.apache.atlas.plugin.model.RangerServiceDef; +import org.apache.atlas.plugin.util.SearchFilter; +import org.apache.atlas.plugin.util.ServicePolicies; + +import java.util.List; +import java.util.Map; + +public interface ServiceStore { + + String OPTION_FORCE_RENAME = "forceRename"; + + void init() throws Exception; + + RangerServiceDef createServiceDef(RangerServiceDef serviceDef) throws Exception; + + RangerServiceDef updateServiceDef(RangerServiceDef serviceDef) throws Exception; + + void deleteServiceDef(Long id, Boolean forceDelete) throws Exception; + + void updateTagServiceDefForAccessTypes() throws Exception; + + RangerServiceDef getServiceDef(Long id) throws Exception; + + RangerServiceDef getServiceDefByName(String name) throws Exception; + + RangerServiceDef getServiceDefByDisplayName(String name) throws Exception; + + List getServiceDefs(SearchFilter filter) throws Exception; + + PList getPaginatedServiceDefs(SearchFilter filter) throws Exception; + + RangerService createService(RangerService service) throws Exception; + + RangerService updateService(RangerService service, Map options) throws Exception; + + void deleteService(Long id) throws Exception; + + RangerService getService(Long id) throws Exception; + + RangerService getServiceByName(String name) throws Exception; + + RangerService getServiceByDisplayName(String displayName) throws Exception; + + List getServices(SearchFilter filter) throws Exception; + + PList getPaginatedServices(SearchFilter filter) throws Exception; + + RangerPolicy createPolicy(RangerPolicy policy) throws Exception; + + RangerPolicy updatePolicy(RangerPolicy policy) throws Exception; + + void deletePolicy(RangerPolicy policy, RangerService service) throws Exception; + + void deletePolicy(RangerPolicy policy) throws Exception; + + boolean policyExists(Long id) throws Exception; + + RangerPolicy getPolicy(Long id) throws Exception; + + List getPolicies(SearchFilter filter) throws Exception; + + Long getPolicyId(final Long serviceId, final String policyName, final Long zoneId); + + PList getPaginatedPolicies(SearchFilter filter) throws Exception; + + List getPoliciesByResourceSignature(String serviceName, String policySignature, Boolean isPolicyEnabled) throws Exception; + + List getServicePolicies(Long serviceId, SearchFilter filter) throws Exception; + + PList getPaginatedServicePolicies(Long serviceId, SearchFilter filter) throws Exception; + + List getServicePolicies(String serviceName, SearchFilter filter) throws Exception; + + PList getPaginatedServicePolicies(String serviceName, SearchFilter filter) throws Exception; + + ServicePolicies getServicePoliciesIfUpdated(String serviceName, Long lastKnownVersion, boolean needsBackwardCompatibility) throws Exception; + + Long getServicePolicyVersion(String serviceName); + + ServicePolicies getServicePolicyDeltasOrPolicies(String serviceName, Long lastKnownVersion) throws Exception; + + ServicePolicies getServicePolicyDeltas(String serviceName, Long lastKnownVersion) throws Exception; + + ServicePolicies getServicePolicies(String serviceName, Long lastKnownVersion) throws Exception; + + RangerPolicy getPolicyFromEventTime(String eventTimeStr, Long policyId); + + void setPopulateExistingBaseFields(Boolean populateExistingBaseFields); + + Boolean getPopulateExistingBaseFields(); + + RangerSecurityZone getSecurityZone(Long id) throws Exception; + + RangerSecurityZone getSecurityZone(String name) throws Exception; + + long getPoliciesCount(final String serviceName); + + Map getServiceConfigForPlugin(Long serviceId); +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/StoredServiceResource.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/StoredServiceResource.java new file mode 100644 index 00000000000..e8badc86cbe --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/StoredServiceResource.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.store; + +import org.apache.atlas.plugin.model.RangerPolicy; +import org.codehaus.jackson.annotate.JsonAutoDetect; +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.Map; + +@JsonAutoDetect(fieldVisibility=JsonAutoDetect.Visibility.ANY) +@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown=true) +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) + +public class StoredServiceResource implements java.io.Serializable { + private final Map resourceElements; + private final String ownerName; + private final Map additionalInfo; + + public StoredServiceResource() { + this(null, null, null); + } + + public StoredServiceResource(Map resourceElements, String ownerName, Map additionalInfo) { + this.resourceElements = resourceElements; + this.ownerName = ownerName; + this.additionalInfo = additionalInfo; + } + + public Map getResourceElements() { + return resourceElements; + } + public String getOwnerName() { + return ownerName; + } + public Map getAdditionalInfo() { + return additionalInfo; + } +} \ No newline at end of file diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/TagPredicateUtil.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/TagPredicateUtil.java new file mode 100644 index 00000000000..f011b9ed323 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/TagPredicateUtil.java @@ -0,0 +1,383 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.store; + +import org.apache.commons.collections.Predicate; +import org.apache.commons.lang.StringUtils; +import org.apache.atlas.plugin.model.RangerServiceResource; +import org.apache.atlas.plugin.model.RangerTag; +import org.apache.atlas.plugin.model.RangerTagDef; +import org.apache.atlas.plugin.model.RangerTagResourceMap; +import org.apache.atlas.plugin.util.SearchFilter; + +import java.util.List; + +public class TagPredicateUtil extends AbstractPredicateUtil { + + public TagPredicateUtil() { super(); } + + @Override + public void addPredicates(SearchFilter filter, List predicates) { + super.addPredicates(filter, predicates); + + addPredicateForTagDefId(filter.getParam(SearchFilter.TAG_DEF_ID), predicates); + addPredicateForTagDefGuid(filter.getParam(SearchFilter.TAG_DEF_GUID), predicates); + + addPredicateForTagId(filter.getParam(SearchFilter.TAG_ID), predicates); + addPredicateForTagGuid(filter.getParam(SearchFilter.TAG_GUID), predicates); + addPredicateForTagType(filter.getParam(SearchFilter.TAG_TYPE), predicates); + + addPredicateForResourceId(filter.getParam(SearchFilter.TAG_RESOURCE_ID), predicates); + addPredicateForResourceGuid(filter.getParam(SearchFilter.TAG_RESOURCE_GUID), predicates); + addPredicateForServiceResourceServiceName(filter.getParam(SearchFilter.TAG_RESOURCE_SERVICE_NAME), predicates); + addPredicateForResourceSignature(filter.getParam(SearchFilter.TAG_RESOURCE_SIGNATURE), predicates); + + addPredicateForTagResourceMapId(filter.getParam(SearchFilter.TAG_MAP_ID), predicates); + } + + private Predicate addPredicateForTagDefId(final String id, List predicates) { + if (StringUtils.isEmpty(id)) { + return null; + } + + Predicate ret = new Predicate() { + @Override + public boolean evaluate(Object object) { + + boolean ret = false; + + if (object == null) { + return ret; + } + + if (object instanceof RangerTagDef) { + RangerTagDef tagDef = (RangerTagDef) object; + + ret = StringUtils.equals(id, tagDef.getId().toString()); + } + + return ret; + } + }; + + if (predicates != null) { + predicates.add(ret); + } + + return ret; + } + + private Predicate addPredicateForTagDefGuid(final String guid, List predicates) { + if (StringUtils.isEmpty(guid)) { + return null; + } + + Predicate ret = new Predicate() { + @Override + public boolean evaluate(Object object) { + + boolean ret = false; + + if (object == null) { + return ret; + } + + if (object instanceof RangerTagDef) { + RangerTagDef tagDef = (RangerTagDef) object; + + ret = StringUtils.equals(guid, tagDef.getGuid()); + } + + return ret; + } + }; + + if (predicates != null) { + predicates.add(ret); + } + + return ret; + } + + private Predicate addPredicateForTagId(final String id, List predicates) { + if (StringUtils.isEmpty(id)) { + return null; + } + + Predicate ret = new Predicate() { + @Override + public boolean evaluate(Object object) { + + boolean ret = false; + + if (object == null) { + return ret; + } + + if (object instanceof RangerTag) { + RangerTag tag = (RangerTag) object; + + ret = StringUtils.equals(id, tag.getId().toString()); + } else if (object instanceof RangerTagResourceMap) { + RangerTagResourceMap tagResourceMap = (RangerTagResourceMap) object; + ret = StringUtils.equals(id, tagResourceMap.getTagId().toString()); + } + + return ret; + } + }; + + if (predicates != null) { + predicates.add(ret); + } + + return ret; + } + + private Predicate addPredicateForTagGuid(final String guid, List predicates) { + if (StringUtils.isEmpty(guid)) { + return null; + } + + Predicate ret = new Predicate() { + @Override + public boolean evaluate(Object object) { + + boolean ret = false; + + if (object == null) { + return ret; + } + + if (object instanceof RangerTag) { + RangerTag tag = (RangerTag) object; + + ret = StringUtils.equals(guid, tag.getGuid()); + } + + return ret; + } + }; + + if (predicates != null) { + predicates.add(ret); + } + + return ret; + } + + private Predicate addPredicateForTagType(final String type, List predicates) { + if (StringUtils.isEmpty(type)) { + return null; + } + + Predicate ret = new Predicate() { + @Override + public boolean evaluate(Object object) { + + boolean ret = false; + + if (object == null) { + return ret; + } + + if (object instanceof RangerTagDef) { + RangerTagDef tagDef = (RangerTagDef) object; + + ret = StringUtils.equals(type, tagDef.getName()); + } else if (object instanceof RangerTag) { + RangerTag tag = (RangerTag) object; + + ret = StringUtils.equals(type, tag.getType()); + } + + return ret; + } + }; + + if (predicates != null) { + predicates.add(ret); + } + + return ret; + } + + private Predicate addPredicateForResourceId(final String id, List predicates) { + if (StringUtils.isEmpty(id)) { + return null; + } + + Predicate ret = new Predicate() { + @Override + public boolean evaluate(Object object) { + + boolean ret = false; + + if (object == null) { + return ret; + } + + if (object instanceof RangerServiceResource) { + RangerServiceResource resource = (RangerServiceResource) object; + + ret = StringUtils.equals(id, resource.getId().toString()); + } else if(object instanceof RangerTagResourceMap) { + RangerTagResourceMap tagResourceMap = (RangerTagResourceMap)object; + + ret = StringUtils.equals(id, tagResourceMap.getId().toString()); + } + + return ret; + } + }; + + if (predicates != null) { + predicates.add(ret); + } + + return ret; + } + + private Predicate addPredicateForResourceGuid(final String id, List predicates) { + if (StringUtils.isEmpty(id)) { + return null; + } + + Predicate ret = new Predicate() { + @Override + public boolean evaluate(Object object) { + + boolean ret = false; + + if (object == null) { + return ret; + } + + if (object instanceof RangerServiceResource) { + RangerServiceResource resource = (RangerServiceResource) object; + + ret = StringUtils.equals(id, resource.getGuid()); + } + + return ret; + } + }; + + if (predicates != null) { + predicates.add(ret); + } + + return ret; + } + + private Predicate addPredicateForServiceResourceServiceName(final String serviceName, List predicates) { + if (serviceName == null || StringUtils.isEmpty(serviceName)) { + return null; + } + + Predicate ret = new Predicate() { + @Override + public boolean evaluate(Object object) { + + boolean ret = false; + + if (object == null) { + return ret; + } + + if (object instanceof RangerServiceResource) { + RangerServiceResource resource = (RangerServiceResource) object; + ret = StringUtils.equals(resource.getServiceName(), serviceName); + } + + return ret; + } + }; + + if (predicates != null) { + predicates.add(ret); + } + + return ret; + } + + private Predicate addPredicateForResourceSignature(final String signature, List predicates) { + if (StringUtils.isEmpty(signature)) { + return null; + } + + Predicate ret = new Predicate() { + @Override + public boolean evaluate(Object object) { + + boolean ret = false; + + if (object == null) { + return ret; + } + + if (object instanceof RangerServiceResource) { + RangerServiceResource resource = (RangerServiceResource) object; + + ret = StringUtils.equals(signature, resource.getResourceSignature()); + } + + return ret; + } + }; + + if (predicates != null) { + predicates.add(ret); + } + + return ret; + } + + private Predicate addPredicateForTagResourceMapId(final String id, List predicates) { + if (StringUtils.isEmpty(id)) { + return null; + } + + Predicate ret = new Predicate() { + @Override + public boolean evaluate(Object object) { + + boolean ret = false; + + if (object == null) { + return ret; + } + + if (object instanceof RangerTagResourceMap) { + RangerTagResourceMap tagResourceMap = (RangerTagResourceMap) object; + ret = StringUtils.equals(id, tagResourceMap.getId().toString()); + } + + return ret; + } + }; + + if (predicates != null) { + predicates.add(ret); + } + + return ret; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/TagStore.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/TagStore.java new file mode 100644 index 00000000000..baa9b05060c --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/TagStore.java @@ -0,0 +1,144 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.store; + +import org.apache.atlas.plugin.model.*; +import org.apache.atlas.plugin.util.SearchFilter; +import org.apache.atlas.plugin.util.ServiceTags; + +import java.util.List; + +/** + * Interface to backing store for the top-level TAG model objects + */ + +public interface TagStore { + void init() throws Exception; + + void setServiceStore(ServiceStore svcStore); + + ServiceStore getServiceStore(); + + RangerTagDef createTagDef(RangerTagDef tagDef) throws Exception; + + RangerTagDef updateTagDef(RangerTagDef TagDef) throws Exception; + + void deleteTagDefByName(String name) throws Exception; + + void deleteTagDef(Long id) throws Exception; + + RangerTagDef getTagDef(Long id) throws Exception; + + RangerTagDef getTagDefByGuid(String guid) throws Exception; + + RangerTagDef getTagDefByName(String name) throws Exception; + + List getTagDefs(SearchFilter filter) throws Exception; + + PList getPaginatedTagDefs(SearchFilter filter) throws Exception; + + List getTagTypes() throws Exception; + + + RangerTag createTag(RangerTag tag) throws Exception; + + RangerTag updateTag(RangerTag tag) throws Exception; + + void deleteTag(Long id) throws Exception; + + RangerTag getTag(Long id) throws Exception; + + RangerTag getTagByGuid(String guid) throws Exception; + + List getTagIdsForResourceId(Long resourceId) throws Exception; + + List getTagsByType(String name) throws Exception; + + List getTagsForResourceId(Long resourceId) throws Exception; + + List getTagsForResourceGuid(String resourceGuid) throws Exception; + + List getTags(SearchFilter filter) throws Exception; + + PList getPaginatedTags(SearchFilter filter) throws Exception; + + + RangerServiceResource createServiceResource(RangerServiceResource resource) throws Exception; + + RangerServiceResource updateServiceResource(RangerServiceResource resource) throws Exception; + + void refreshServiceResource(Long resourceId) throws Exception; + + void deleteServiceResource(Long id) throws Exception; + + void deleteServiceResourceByGuid(String guid) throws Exception; + + RangerServiceResource getServiceResource(Long id) throws Exception; + + RangerServiceResource getServiceResourceByGuid(String guid) throws Exception; + + List getServiceResourcesByService(String serviceName) throws Exception; + + List getServiceResourceGuidsByService(String serviceName) throws Exception; + + RangerServiceResource getServiceResourceByServiceAndResourceSignature(String serviceName, String resourceSignature) throws Exception; + + List getServiceResources(SearchFilter filter) throws Exception; + + PList getPaginatedServiceResources(SearchFilter filter) throws Exception; + + + RangerTagResourceMap createTagResourceMap(RangerTagResourceMap tagResourceMap) throws Exception; + + void deleteTagResourceMap(Long id) throws Exception; + + RangerTagResourceMap getTagResourceMap(Long id) throws Exception; + + RangerTagResourceMap getTagResourceMapByGuid(String guid) throws Exception; + + List getTagResourceMapsForTagId(Long tagId) throws Exception; + + List getTagResourceMapsForTagGuid(String tagGuid) throws Exception; + + List getTagResourceMapsForResourceId(Long resourceId) throws Exception; + + List getTagResourceMapsForResourceGuid(String resourceGuid) throws Exception; + + RangerTagResourceMap getTagResourceMapForTagAndResourceId(Long tagId, Long resourceId) throws Exception; + + RangerTagResourceMap getTagResourceMapForTagAndResourceGuid(String tagGuid, String resourceGuid) throws Exception; + + List getTagResourceMaps(SearchFilter filter) throws Exception; + + PList getPaginatedTagResourceMaps(SearchFilter filter) throws Exception; + + + ServiceTags getServiceTagsIfUpdated(String serviceName, Long lastKnownVersion, boolean needsBackwardCompatibility) throws Exception; + ServiceTags getServiceTags(String serviceName, Long lastKnownVersion) throws Exception; + ServiceTags getServiceTagsDelta(String serviceName, Long lastKnownVersion) throws Exception; + + + Long getTagVersion(String serviceName); + + void deleteAllTagObjectsForService(String serviceName) throws Exception; + + boolean isInPlaceTagUpdateSupported(); + +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/TagValidator.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/TagValidator.java new file mode 100644 index 00000000000..0b966fc80d3 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/TagValidator.java @@ -0,0 +1,329 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.store; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.atlas.plugin.model.*; + +import java.util.List; + +public class TagValidator { + private TagStore tagStore; + + public TagValidator() {} + + public void setTagStore(TagStore tagStore) { + this.tagStore = tagStore; + } + + public RangerTagDef preCreateTagDef(final RangerTagDef tagDef, boolean updateIfExists) throws Exception { + String name = tagDef.getName(); + + if (StringUtils.isBlank(name)) { + throw new Exception("TagDef has no name"); + } + + RangerTagDef existing = tagStore.getTagDefByName(name); + + return existing; + } + + public RangerTag preCreateTag(final RangerTag tag) throws Exception { + if(StringUtils.isBlank(tag.getType()) ) { + throw new Exception("Tag has no type"); + } + + RangerTag ret = null; + + String guid = tag.getGuid(); + if (! StringUtils.isBlank(guid)) { + ret = tagStore.getTagByGuid(guid); + } + + return ret; + } + + public void preUpdateTag(final Long id, final RangerTag tag) throws Exception { + if (StringUtils.isBlank(tag.getType())) { + throw new Exception("Tag has no type"); + } + + if (id == null) { + throw new Exception("Invalid/null id"); + } + + RangerTag existing = tagStore.getTag(id); + + if (existing == null) { + throw new Exception("Attempt to update nonexistant tag, id=" + id); + } + + if (!StringUtils.equals(tag.getType(), existing.getType())) { + throw new Exception("Attempt to change the tag-type"); + } + + tag.setId(existing.getId()); + tag.setGuid(existing.getGuid()); + } + + public void preUpdateTagByGuid(String guid, final RangerTag tag) throws Exception { + if (StringUtils.isBlank(tag.getType())) { + throw new Exception("Tag has no type"); + } + + RangerTag existing = tagStore.getTagByGuid(guid); + if (existing == null) { + throw new Exception("Attempt to update nonexistent tag, guid=" + guid); + } + + if (!StringUtils.equals(tag.getType(), existing.getType())) { + throw new Exception("Attempt to change the tag-type"); + } + + tag.setId(existing.getId()); + tag.setGuid(existing.getGuid()); + } + + public RangerTag preDeleteTag(Long id) throws Exception { + if (id == null) { + throw new Exception("Invalid/null id"); + } + + RangerTag existing = tagStore.getTag(id); + + if (existing == null) { + throw new Exception("Attempt to delete nonexistent tag, id=" + id); + } + + List associations = tagStore.getTagResourceMapsForTagId(existing.getId()); + if (CollectionUtils.isNotEmpty(associations)) { + throw new Exception("Attempt to delete tag which is associated with a service-resource, id=" + id); + } + return existing; + } + + public RangerTag preDeleteTagByGuid(String guid) throws Exception { + RangerTag exiting = tagStore.getTagByGuid(guid); + + if (exiting == null) { + throw new Exception("Attempt to delete nonexistent tag, guid=" + guid); + } + + List associations = tagStore.getTagResourceMapsForTagId(exiting.getId()); + if (CollectionUtils.isNotEmpty(associations)) { + throw new Exception("Attempt to delete tag which is associated with a service-resource, guid=" + guid); + } + return exiting; + } + + public RangerServiceResource preCreateServiceResource(RangerServiceResource resource) throws Exception { + RangerServiceResource ret = null; + + if (StringUtils.isBlank(resource.getServiceName()) || MapUtils.isEmpty(resource.getResourceElements())) { + throw new Exception("No serviceName or resource in RangerServiceResource"); + } + + String guid = resource.getGuid(); + if (! StringUtils.isBlank(guid)) { + ret = tagStore.getServiceResourceByGuid(guid); + } + + if (ret == null) { + RangerServiceResourceSignature serializer = new RangerServiceResourceSignature(resource); + resource.setResourceSignature(serializer.getSignature()); + } + + return ret; + } + + public void preUpdateServiceResource(Long id, RangerServiceResource resource) throws Exception { + if (StringUtils.isBlank(resource.getServiceName()) || MapUtils.isEmpty(resource.getResourceElements())) { + throw new Exception("No serviceName or resource in RangerServiceResource"); + } + + if (id == null) { + throw new Exception("Invalid/null id"); + } + + RangerServiceResource existing = tagStore.getServiceResource(id); + if (existing == null) { + throw new Exception("Attempt to update nonexistent resource, id=" + id); + } + + if (!StringUtils.equals(existing.getServiceName(), resource.getServiceName())) { + throw new Exception("Attempt to change service-name for existing service-resource"); + } + + RangerServiceResourceSignature serializer = new RangerServiceResourceSignature(resource); + + resource.setId(existing.getId()); + resource.setGuid(existing.getGuid()); + resource.setResourceSignature(serializer.getSignature()); + } + + public void preUpdateServiceResourceByGuid(String guid, RangerServiceResource resource) throws Exception { + if (StringUtils.isBlank(resource.getServiceName()) || MapUtils.isEmpty(resource.getResourceElements())) { + throw new Exception("No serviceName or resource in RangerServiceResource"); + } + + RangerServiceResource existing = tagStore.getServiceResourceByGuid(guid); + if (existing == null) { + throw new Exception("Attempt to update nonexistent resource, guid=" + guid); + } + + if (!StringUtils.equals(existing.getServiceName(), resource.getServiceName())) { + throw new Exception("Attempt to change service-name for existing service-resource"); + } + + RangerServiceResourceSignature serializer = new RangerServiceResourceSignature(resource); + + resource.setId(existing.getId()); + resource.setGuid(guid); + resource.setResourceSignature(serializer.getSignature()); + } + + public RangerServiceResource preDeleteServiceResource(Long id) throws Exception { + RangerServiceResource existing = tagStore.getServiceResource(id); + + if (existing == null) { + throw new Exception("Attempt to delete nonexistent resource, id=" + id); + } + + List associations = tagStore.getTagResourceMapsForResourceId(existing.getId()); + if (CollectionUtils.isNotEmpty(associations)) { + throw new Exception("Attempt to delete serviceResource which is associated with a tag, id=" + id); + } + + return existing; + } + + public RangerServiceResource preDeleteServiceResourceByGuid(String guid, boolean deleteReferences) throws Exception { + RangerServiceResource existing = tagStore.getServiceResourceByGuid(guid); + + if (existing == null) { + throw new Exception("Attempt to delete nonexistent resource, guid=" + guid); + } + + List associations = tagStore.getTagResourceMapsForResourceId(existing.getId()); + if (CollectionUtils.isNotEmpty(associations) && !deleteReferences) { + throw new Exception("Attempt to delete serviceResource which is associated with a tag, guid=" + guid); + } + + return existing; + } + + public RangerTagResourceMap preCreateTagResourceMap(String tagGuid, String resourceGuid) throws Exception { + if (StringUtils.isBlank(resourceGuid) || StringUtils.isBlank(tagGuid)) { + throw new Exception("Both resourceGuid and resourceId need to be non-empty"); + } + + RangerTagResourceMap existing = tagStore.getTagResourceMapForTagAndResourceGuid(tagGuid, resourceGuid); + + if (existing != null) { + throw new Exception("Attempt to create existing association between resourceId=" + resourceGuid + " and tagId=" + tagGuid); + } + + RangerServiceResource existingServiceResource = tagStore.getServiceResourceByGuid(resourceGuid); + + if(existingServiceResource == null) { + throw new Exception("No resource found for guid=" + resourceGuid); + } + + RangerTag existingTag = tagStore.getTagByGuid(tagGuid); + + if(existingTag == null) { + throw new Exception("No tag found for guid=" + tagGuid); + } + + RangerTagResourceMap newTagResourceMap = new RangerTagResourceMap(); + newTagResourceMap.setResourceId(existingServiceResource.getId()); + newTagResourceMap.setTagId(existingTag.getId()); + + return newTagResourceMap; + } + + public RangerTagResourceMap preCreateTagResourceMapByIds(Long tagId, Long resourceId) throws Exception { + RangerTagResourceMap existing = tagStore.getTagResourceMapForTagAndResourceId(tagId, resourceId); + + if (existing != null) { + throw new Exception("Attempt to create existing association between resourceId=" + resourceId + " and tagId=" + tagId); + } + + RangerServiceResource existingServiceResource = tagStore.getServiceResource(resourceId); + + if(existingServiceResource == null) { + throw new Exception("No resource found for id=" + resourceId); + } + + RangerTag existingTag = tagStore.getTag(tagId); + + if(existingTag == null) { + throw new Exception("No tag found for id=" + tagId); + } + + RangerTagResourceMap newTagResourceMap = new RangerTagResourceMap(); + newTagResourceMap.setResourceId(resourceId); + newTagResourceMap.setTagId(tagId); + + return newTagResourceMap; + } + + public RangerTagResourceMap preDeleteTagResourceMap(Long id) throws Exception { + RangerTagResourceMap existing = tagStore.getTagResourceMap(id); + + if (existing == null) { + throw new Exception("Attempt to delete nonexistent tagResourceMap(id=" + id + ")"); + } + + return existing; + } + + public RangerTagResourceMap preDeleteTagResourceMapByGuid(String guid) throws Exception { + RangerTagResourceMap existing = tagStore.getTagResourceMapByGuid(guid); + + if (existing == null) { + throw new Exception("Attempt to delete nonexistent tagResourceMap(guid=" + guid + ")"); + } + + return existing; + } + + public RangerTagResourceMap preDeleteTagResourceMap(String tagGuid, String resourceGuid) throws Exception { + RangerTagResourceMap existing = tagStore.getTagResourceMapForTagAndResourceGuid(tagGuid, resourceGuid); + + if (existing == null) { + throw new Exception("Attempt to delete nonexistent association between resourceId=" + resourceGuid + " and tagId=" + tagGuid); + } + + return existing; + } + + public RangerTagResourceMap preDeleteTagResourceMapByIds(Long tagId, Long resourceId) throws Exception { + RangerTagResourceMap existing = tagStore.getTagResourceMapForTagAndResourceId(tagId, resourceId); + + if (existing == null) { + throw new Exception("Attempt to delete nonexistent association between resourceId=" + resourceId + " and tagId=" + tagId); + } + + return existing; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/file/GeolocationFileStore.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/file/GeolocationFileStore.java new file mode 100644 index 00000000000..df82e4f323b --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/file/GeolocationFileStore.java @@ -0,0 +1,256 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.store.file; + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.plugin.geo.GeolocationMetadata; +import org.apache.atlas.plugin.geo.RangerGeolocationData; +import org.apache.atlas.plugin.geo.RangerGeolocationDatabase; +import org.apache.atlas.plugin.store.GeolocationStore; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.Reader; +import java.util.HashMap; +import java.util.Map; + +public class GeolocationFileStore implements GeolocationStore { + private static final Log LOG = LogFactory.getLog(GeolocationFileStore.class); + + public static final String GeoLineCommentIdentifier = "#"; + public static final Character GeoFieldsSeparator = ','; + + public static final String PROP_GEOLOCATION_FILE_LOCATION = "FilePath"; + public static final String PROP_GEOLOCATION_FILE_REINIT = "ForceRead"; + public static final String PROP_GEOLOCATION_IP_IN_DOT_FORMAT = "IPInDotFormat"; + + private static Map geolocationDBMap = new HashMap<>(); + + private RangerGeolocationDatabase geolocationDatabase; + + private boolean isMetalineProcessed; + private boolean useDotFormat; + + @Override + public void init(final Map context) { + + String filePathToGeolocationFile = context.get(PROP_GEOLOCATION_FILE_LOCATION); + + if (StringUtils.isBlank(filePathToGeolocationFile)) { + filePathToGeolocationFile = "/etc/ranger/data/geo.txt"; + } + + String reinit = context.get(PROP_GEOLOCATION_FILE_REINIT); + boolean reinitialize = reinit == null || Boolean.parseBoolean(reinit); + + String ipInDotFormat = context.get(PROP_GEOLOCATION_IP_IN_DOT_FORMAT); + useDotFormat = ipInDotFormat == null || Boolean.parseBoolean(ipInDotFormat); + + + if (LOG.isDebugEnabled()) { + LOG.debug("GeolocationFileStore.init() - Geolocation file location=" + filePathToGeolocationFile); + LOG.debug("GeolocationFileStore.init() - Reinitialize flag =" + reinitialize); + LOG.debug("GeolocationFileStore.init() - UseDotFormat flag =" + useDotFormat); + } + + RangerGeolocationDatabase database = geolocationDBMap.get(filePathToGeolocationFile); + + if (database == null || reinitialize) { + RangerGeolocationDatabase newDatabase = build(filePathToGeolocationFile); + if (newDatabase != null) { + geolocationDBMap.put(filePathToGeolocationFile, newDatabase); + database = newDatabase; + } else { + LOG.error("GeolocationFileStore.init() - Could not build database. Using old database if present."); + } + } + geolocationDatabase = database; + + if (geolocationDatabase == null) { + LOG.error("GeolocationFileStore.init() - Cannot build Geolocation database from file " + filePathToGeolocationFile); + } + + } + + @Override + public RangerGeolocationDatabase getGeoDatabase() { + return geolocationDatabase; + } + + @Override + public final RangerGeolocationData getGeoLocation(final String ipAddress) { + RangerGeolocationData ret = null; + + RangerGeolocationDatabase database = geolocationDatabase; // init() may get called when getGeolocation is half-executed + + if (database != null) { + + long start = 0L, end = 0L; + + start = System.currentTimeMillis(); + ret = database.find(ipAddress); + end = System.currentTimeMillis(); + + if (LOG.isDebugEnabled()) { + if (ret == null) { + LOG.debug("GeolocationFileStore.getGeolocation() - " + ipAddress + " not found. Search time = " + (end - start) + " milliseconds"); + } else { + LOG.debug("GeolocationFileStore.getGeolocation() - " + ipAddress + " found. Search time = " + (end - start) + " milliseconds"); + + for (String attrName : database.getMetadata().getLocationDataItemNames()) { + LOG.debug("GeolocationFileStore.getGeolocation() - IPAddress[" + attrName + "]=" + database.getValue(ret, attrName) + ", "); + } + + } + } + } else { + LOG.error("GeolocationFileStore.getGeolocation() - GeoLocationDatabase is not initialized correctly."); + } + + return ret; + } + + private Reader getReader(String dataFileName) throws IOException { + Reader ret = null; + + File f = new File(dataFileName); + + if(f.exists() && f.canRead()) { + LOG.info("GeolocationFileStore: reading location data from file '" + dataFileName + "'"); + + ret = new FileReader(dataFileName); + } else { + InputStream inStr = this.getClass().getResourceAsStream(dataFileName); + + if(inStr != null) { + LOG.info("GeolocationFileStore: reading location data from resource '" + dataFileName + "'"); + + ret = new InputStreamReader(inStr); + } + } + + if(ret == null) { + throw new FileNotFoundException(dataFileName); + } + + return ret; + } + + RangerGeolocationDatabase build(String dataFileName) { + + RangerGeolocationDatabase database = null; + + BufferedReader bufferedReader = null; + long start = 0L, end = 0L; + + start = System.currentTimeMillis(); + + try { + bufferedReader = new BufferedReader(getReader(dataFileName)); + + database = new RangerGeolocationDatabase(); + + String line; + int lineNumber = 0; + isMetalineProcessed = false; + + while(( line = bufferedReader.readLine()) != null) { + lineNumber++; + if (!processLine(lineNumber, line, database)) { + LOG.error("RangerGeolocationDatabaseBuilder.build() - Invalid geo-specification - " + lineNumber + ":" + line); + database = null; + break; + } + } + + bufferedReader.close(); + bufferedReader = null; + } + catch(FileNotFoundException ex) { + LOG.error("RangerGeolocationDatabaseBuilder.build() - Unable to open file '" + dataFileName + "'"); + } + catch(IOException ex) { + LOG.error("RangerGeolocationDatabaseBuilder.build() - Error reading file '" + dataFileName + "', " + ex); + } + finally { + if (bufferedReader != null) { + try { + bufferedReader.close(); + } + catch (Exception exception) { + // Ignore + } + } + } + + end = System.currentTimeMillis(); + + if (LOG.isDebugEnabled()) { + LOG.debug("RangerGeolocationDatabaseBuilder.build() - Time taken for reading file = " + (end - start) + " milliseconds"); + } + + if (database != null) { + database.optimize(); + } + + return database; + } + + private boolean processLine(int lineNumber, String line, RangerGeolocationDatabase database) { + + boolean ret = true; + + line = line.trim(); + + if (!line.startsWith(GeoLineCommentIdentifier)) { + String fields[] = StringUtils.split(line, GeoFieldsSeparator); + if (fields != null) { + if (!isMetalineProcessed) { + GeolocationMetadata metadata = GeolocationMetadata.create(fields, lineNumber); + if (metadata != null) { + database.setMetadata(metadata); + isMetalineProcessed = true; + } else { + LOG.error("GeolocationFileStore.processLine() - Invalid metadata specification " + lineNumber + ":" + line); + ret = false; + } + } else { + RangerGeolocationData data = RangerGeolocationData.create(fields, lineNumber, useDotFormat); + if (data != null) { + database.getData().insert(data); + } else { + LOG.error("GeolocationFileStore.processLine() - Invalid data specification " + lineNumber + ":" + line); + } + } + } else { + LOG.error("GeolocationFileStore.processLine() - Invalid line, skipping.." + lineNumber + ":" + line); + } + } + return ret; + } + +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/DownloadTrigger.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/DownloadTrigger.java new file mode 100644 index 00000000000..0cd0bdaf747 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/DownloadTrigger.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.util; + +public final class DownloadTrigger { + private boolean isNotified = false; + + public synchronized void waitForCompletion() throws InterruptedException { + while (!isNotified) { + wait(); + } + isNotified = false; + } + + public synchronized void signalCompletion() { + isNotified = true; + notifyAll(); + } +} \ No newline at end of file diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/DownloaderTask.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/DownloaderTask.java new file mode 100644 index 00000000000..6d88dec4433 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/DownloaderTask.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.util; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import java.util.TimerTask; +import java.util.concurrent.BlockingQueue; + +public final class DownloaderTask extends TimerTask { + private static final Log LOG = LogFactory.getLog(DownloaderTask.class); + + private final DownloadTrigger timerTrigger = new DownloadTrigger(); + private final BlockingQueue queue; + + public DownloaderTask(BlockingQueue queue) { + this.queue = queue; + } + + @Override + public void run() { + try { + queue.put(timerTrigger); + timerTrigger.waitForCompletion(); + } catch (InterruptedException excp) { + LOG.error("Caught exception. Exiting thread"); + } + } +} \ No newline at end of file diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/GrantRevokeRequest.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/GrantRevokeRequest.java new file mode 100644 index 00000000000..09cf8519275 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/GrantRevokeRequest.java @@ -0,0 +1,443 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.util; + + +import org.apache.htrace.shaded.fasterxml.jackson.annotation.JsonInclude; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class GrantRevokeRequest implements Serializable { + private static final long serialVersionUID = 1L; + + private String grantor; + private Set grantorGroups; + private Map resource; + private Set users; + private Set groups; + private Set roles; + private Set accessTypes; + private List forwardedAddresses; + private String remoteIPAddress; + private Boolean delegateAdmin = Boolean.FALSE; + private Boolean enableAudit = Boolean.TRUE; + private Boolean replaceExistingPermissions = Boolean.FALSE; + private Boolean isRecursive = Boolean.FALSE; + private String clientIPAddress; + private String clientType; + private String requestData; + private String sessionId; + private String clusterName; + private String zoneName; + private String ownerUser; + + public GrantRevokeRequest() { + this(null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null); + } + + public GrantRevokeRequest(String grantor, Set grantorGroups, Map resource, Set users, + Set groups, Set accessTypes, Boolean delegateAdmin, Boolean enableAudit, + Boolean replaceExistingPermissions, Boolean isRecursive, String clientIPAddress, + String clientType, String requestData, String sessionId, String clusterName, String zoneName) { + this(grantor, grantorGroups, resource, users, groups, null, accessTypes, delegateAdmin, enableAudit, replaceExistingPermissions, isRecursive, clientIPAddress, clientType, requestData, sessionId, clusterName, zoneName); + } + + public GrantRevokeRequest(String grantor, Set grantorGroups, Map resource, Set users, + Set groups, Set roles, Set accessTypes, Boolean delegateAdmin, Boolean enableAudit, + Boolean replaceExistingPermissions, Boolean isRecursive, String clientIPAddress, + String clientType, String requestData, String sessionId, String clusterName, String zoneName) { + this(grantor, grantorGroups, resource, users, groups, roles, accessTypes, delegateAdmin, enableAudit, replaceExistingPermissions, isRecursive, clientIPAddress, clientType, requestData, sessionId, clusterName, zoneName, null); + } + + public GrantRevokeRequest(String grantor, Set grantorGroups, Map resource, Set users, + Set groups, Set roles, Set accessTypes, Boolean delegateAdmin, Boolean enableAudit, + Boolean replaceExistingPermissions, Boolean isRecursive, String clientIPAddress, + String clientType, String requestData, String sessionId, String clusterName, String zoneName, String ownerUser) { + setGrantor(grantor); + setGrantorGroups(grantorGroups); + setResource(resource); + setUsers(users); + setGroups(groups); + setRoles(roles); + setAccessTypes(accessTypes); + setDelegateAdmin(delegateAdmin); + setEnableAudit(enableAudit); + setReplaceExistingPermissions(replaceExistingPermissions); + setIsRecursive(isRecursive); + setClientIPAddress(clientIPAddress); + setClientType(clientType); + setRequestData(requestData); + setSessionId(sessionId); + setClusterName(clusterName); + setZoneName(zoneName); + setOwnerUser(ownerUser); + } + + /** + * @return the grantor + */ + public String getGrantor() { + return grantor; + } + + /** + * @param grantor the grantor to set + */ + public void setGrantor(String grantor) { + this.grantor = grantor; + } + + /** + * @return the grantorGroups + */ + public Set getGrantorGroups() { + return grantorGroups; + } + + /** + * @param grantorGroups the grantorGroups to set + */ + public void setGrantorGroups(Set grantorGroups) { + this.grantorGroups = grantorGroups == null ? new HashSet() : grantorGroups; + } + /** + * @return the resource + */ + public Map getResource() { + return resource; + } + + public void setForwardedAddresses(List forwardedAddresses) { + this.forwardedAddresses = (forwardedAddresses == null) ? new ArrayList() : forwardedAddresses; + } + + public void setRemoteIPAddress(String remoteIPAddress) { + this.remoteIPAddress = remoteIPAddress; + } + + + /** + * @param resource the resource to set + */ + public void setResource(Map resource) { + this.resource = resource == null ? new HashMap() : resource; + } + + /** + * @return the users + */ + public Set getUsers() { + return users; + } + + /** + * @param users the users to set + */ + public void setUsers(Set users) { + this.users = users == null ? new HashSet() : users; + } + + /** + * @return the groups + */ + public Set getGroups() { + return groups; + } + + /** + * @param groups the groups to set + */ + public void setGroups(Set groups) { + this.groups = groups == null ? new HashSet() : groups; + } + + /** + * @return the roles + */ + public Set getRoles() { + return roles; + } + + /** + * @param roles the roles to set + */ + public void setRoles(Set roles) { + this.roles = roles == null ? new HashSet() : roles; + } + + + /** + * @return the accessTypes + */ + public Set getAccessTypes() { + return accessTypes; + } + + /** + * @param accessTypes the accessTypes to set + */ + public void setAccessTypes(Set accessTypes) { + this.accessTypes = accessTypes == null ? new HashSet() : accessTypes; + } + + /** + * @return the delegateAdmin + */ + public Boolean getDelegateAdmin() { + return delegateAdmin; + } + + /** + * @param delegateAdmin the delegateAdmin to set + */ + public void setDelegateAdmin(Boolean delegateAdmin) { + this.delegateAdmin = delegateAdmin == null ? Boolean.FALSE : delegateAdmin; + } + + /** + * @return the enableAudit + */ + public Boolean getEnableAudit() { + return enableAudit; + } + + /** + * @param enableAudit the enableAudit to set + */ + public void setEnableAudit(Boolean enableAudit) { + this.enableAudit = enableAudit == null ? Boolean.TRUE : enableAudit; + } + + /** + * @return the ownerUser + */ + public String getOwnerUser() { + return ownerUser; + } + + /** + * @param ownerUser the ownerUser to set + */ + public void setOwnerUser(String ownerUser) { + this.ownerUser = ownerUser; + } + + /** + * @return the replaceExistingPermissions + */ + public Boolean getReplaceExistingPermissions() { + return replaceExistingPermissions; + } + + /** + * @param replaceExistingPermissions the replaceExistingPermissions to set + */ + public void setReplaceExistingPermissions(Boolean replaceExistingPermissions) { + this.replaceExistingPermissions = replaceExistingPermissions == null ? Boolean.FALSE : replaceExistingPermissions; + } + + /** + * @return the isRecursive + */ + public Boolean getIsRecursive() { + return isRecursive; + } + + /** + * @param isRecursive the isRecursive to set + */ + public void setIsRecursive(Boolean isRecursive) { + this.isRecursive = isRecursive == null ? Boolean.FALSE : isRecursive; + } + + /** + * @return the clientIPAddress + */ + public String getClientIPAddress() { + return clientIPAddress; + } + + /** + * @param clientIPAddress the clientIPAddress to set + */ + public void setClientIPAddress(String clientIPAddress) { + this.clientIPAddress = clientIPAddress; + } + + /** + * @return the clientType + */ + public String getClientType() { + return clientType; + } + + /** + * @param clientType the clientType to set + */ + public void setClientType(String clientType) { + this.clientType = clientType; + } + + /** + * @return the requestData + */ + public String getRequestData() { + return requestData; + } + + /** + * @param requestData the requestData to set + */ + public void setRequestData(String requestData) { + this.requestData = requestData; + } + + /** + * @return the sessionId + */ + public String getSessionId() { + return sessionId; + } + + /** + * @param sessionId the sessionId to set + */ + public void setSessionId(String sessionId) { + this.sessionId = sessionId; + } + + /** + * @return the clusterName + */ + public String getClusterName() { + return clusterName; + } + + public String getRemoteIPAddress() { + return remoteIPAddress; + } + + public List getForwardedAddresses() { + return forwardedAddresses; + } + + /** + * @param clusterName the clusterName to set + */ + public void setClusterName(String clusterName) { + this.clusterName = clusterName; + } + + /** + * @return the clusterName + */ + public String getZoneName() { + return zoneName; + } + + /** + * @param zoneName the clusterName to set + */ + public void setZoneName(String zoneName) { + this.zoneName = zoneName; + } + + @Override + public String toString( ) { + StringBuilder sb = new StringBuilder(); + + toString(sb); + + return sb.toString(); + } + + public StringBuilder toString(StringBuilder sb) { + sb.append("GrantRevokeRequest={"); + + sb.append("grantor={").append(grantor).append("} "); + + sb.append("grantorGroups={"); + if(grantorGroups != null) { + for(String grantorGroup : grantorGroups) { + sb.append(grantorGroup).append(" "); + } + } + sb.append("} "); + + sb.append("resource={"); + if(resource != null) { + for(Map.Entry e : resource.entrySet()) { + sb.append(e.getKey()).append("=").append(e.getValue()).append("; "); + } + } + sb.append("} "); + + sb.append("users={"); + if(users != null) { + for(String user : users) { + sb.append(user).append(" "); + } + } + sb.append("} "); + + sb.append("groups={"); + if(groups != null) { + for(String group : groups) { + sb.append(group).append(" "); + } + } + sb.append("} "); + + sb.append("accessTypes={"); + if(accessTypes != null) { + for(String accessType : accessTypes) { + sb.append(accessType).append(" "); + } + } + sb.append("} "); + + sb.append("delegateAdmin={").append(delegateAdmin).append("} "); + sb.append("enableAudit={").append(enableAudit).append("} "); + sb.append("replaceExistingPermissions={").append(replaceExistingPermissions).append("} "); + sb.append("isRecursive={").append(isRecursive).append("} "); + sb.append("clientIPAddress={").append(clientIPAddress).append("} "); + sb.append("clientType={").append(clientType).append("} "); + sb.append("requestData={").append(requestData).append("} "); + sb.append("sessionId={").append(sessionId).append("} "); + sb.append("clusterName={").append(clusterName).append("} "); + sb.append("zoneName={").append(zoneName).append("} "); + + sb.append("}"); + + return sb; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/GrantRevokeRoleRequest.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/GrantRevokeRoleRequest.java new file mode 100644 index 00000000000..0f3811fde63 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/GrantRevokeRoleRequest.java @@ -0,0 +1,299 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.util; + +import org.apache.htrace.shaded.fasterxml.jackson.annotation.JsonInclude; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import java.io.Serializable; +import java.util.HashSet; +import java.util.Set; + + +@JsonInclude(JsonInclude.Include.NON_NULL) +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class GrantRevokeRoleRequest implements Serializable { + private static final long serialVersionUID = 1L; + + private String grantor; + private Set grantorGroups; + private Set targetRoles; + private Set users; + private Set groups; + private Set roles; + private Boolean grantOption = Boolean.FALSE; + private String clientIPAddress; + private String clientType; + private String requestData; + private String sessionId; + private String clusterName; + + public GrantRevokeRoleRequest() { + this(null, null, null, null, null, null, null, null, null, null, null); + } + + public GrantRevokeRoleRequest(String grantor, Set grantorGroups, Set targetRoles, Set users, + Set groups, Set roles, Boolean grantOption, + String clientIPAddress, String clientType, + String requestData, String sessionId) { + setGrantor(grantor); + setGrantorGroups(grantorGroups); + setTargetRoles(targetRoles); + setUsers(users); + setGroups(groups); + setRoles(roles); + setGrantOption(grantOption); + setClientIPAddress(clientIPAddress); + setClientType(clientType); + setRequestData(requestData); + setSessionId(sessionId); + setClusterName(clusterName); + } + + /** + * @return the grantor + */ + public String getGrantor() { + return grantor; + } + + /** + * @param grantor the grantor to set + */ + public void setGrantor(String grantor) { + this.grantor = grantor; + } + + /** + * @return the grantorRoles + */ + public Set getGrantorGroups() { + return grantorGroups; + } + + /** + * @param grantorGroups the grantorRoles to set + */ + + public void setGrantorGroups(Set grantorGroups) { + this.grantorGroups = grantorGroups; + } + + /** + * @return the targetRoles + */ + public Set getTargetRoles() { + return targetRoles; + } + + /** + * @param targetRoles the targetRoles to set + */ + public void setTargetRoles(Set targetRoles) { + this.targetRoles = targetRoles == null ? new HashSet() : targetRoles; + } + + /** + * @return the users + */ + public Set getUsers() { + return users; + } + + /** + * @param users the users to set + */ + public void setUsers(Set users) { + this.users = users == null ? new HashSet() : users; + } + + /** + * @return the groups + */ + public Set getGroups() { + return groups; + } + + /** + * @param groups the groups to set + */ + public void setGroups(Set groups) { + this.groups = groups; + } + + /** + * @return the roles + */ + public Set getRoles() { + return roles; + } + + /** + * @param roles the roles to set + */ + public void setRoles(Set roles) { + this.roles = roles == null ? new HashSet() : roles; + } + + /** + * @return the grantOption + */ + public Boolean getGrantOption() { + return grantOption; + } + + /** + * @param grantOption the grantOption to set + */ + public void setGrantOption(Boolean grantOption) { + this.grantOption = grantOption == null ? Boolean.FALSE : grantOption; + } + + /** + * @return the clientIPAddress + */ + public String getClientIPAddress() { + return clientIPAddress; + } + + /** + * @param clientIPAddress the clientIPAddress to set + */ + public void setClientIPAddress(String clientIPAddress) { + this.clientIPAddress = clientIPAddress; + } + + /** + * @return the clientType + */ + public String getClientType() { + return clientType; + } + + /** + * @param clientType the clientType to set + */ + public void setClientType(String clientType) { + this.clientType = clientType; + } + + /** + * @return the requestData + */ + public String getRequestData() { + return requestData; + } + + /** + * @param requestData the requestData to set + */ + public void setRequestData(String requestData) { + this.requestData = requestData; + } + + /** + * @return the sessionId + */ + public String getSessionId() { + return sessionId; + } + + /** + * @param sessionId the sessionId to set + */ + public void setSessionId(String sessionId) { + this.sessionId = sessionId; + } + + /** + * @return the clusterName + */ + public String getClusterName() { + return clusterName; + } + + /** + * @param clusterName the clusterName to set + */ + public void setClusterName(String clusterName) { + this.clusterName = clusterName; + } + + @Override + public String toString( ) { + StringBuilder sb = new StringBuilder(); + + toString(sb); + + return sb.toString(); + } + + public StringBuilder toString(StringBuilder sb) { + sb.append("GrantRevokeRoleRequest={"); + + sb.append("grantor={").append(grantor).append("} "); + + sb.append("targetRoles={"); + if(targetRoles != null) { + for(String targetRole : targetRoles) { + sb.append(targetRole).append(" "); + } + } + sb.append("} "); + + sb.append("users={"); + if(users != null) { + for(String user : users) { + sb.append(user).append(" "); + } + } + sb.append("} "); + + sb.append("groups={"); + if(roles != null) { + for(String group : groups) { + sb.append(group).append(" "); + } + } + sb.append("} "); + + sb.append("roles={"); + if(roles != null) { + for(String role : roles) { + sb.append(role).append(" "); + } + } + sb.append("} "); + + sb.append("grantOption={").append(grantOption).append("} "); + sb.append("clientIPAddress={").append(clientIPAddress).append("} "); + sb.append("clientType={").append(clientType).append("} "); + sb.append("requestData={").append(requestData).append("} "); + sb.append("sessionId={").append(sessionId).append("} "); + sb.append("clusterName={").append(clusterName).append("} "); + + sb.append("}"); + + return sb; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/JsonUtilsV2.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/JsonUtilsV2.java new file mode 100644 index 00000000000..9632166079a --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/JsonUtilsV2.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.util; + + +import org.apache.htrace.shaded.fasterxml.jackson.core.type.TypeReference; +import org.apache.htrace.shaded.fasterxml.jackson.databind.ObjectMapper; + +import java.io.Serializable; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class JsonUtilsV2 { + + static private final ThreadLocal mapper = new ThreadLocal() { + @Override + protected ObjectMapper initialValue() { + return new ObjectMapper(); + } + }; + + static public ObjectMapper getMapper() { + return mapper.get(); + } + + static public Map jsonToMap(String jsonStr) throws Exception { + final Map ret; + + if (jsonStr == null || jsonStr.isEmpty()) { + ret = new HashMap<>(); + } else { + ret = getMapper().readValue(jsonStr, new TypeReference>() {}); + } + + return ret; + } + + static public String mapToJson(Map map) throws Exception { + return getMapper().writeValueAsString(map); + } + + static public String listToJson(List list) throws Exception { + return getMapper().writeValueAsString(list); + } + + static public String objToJson(Serializable obj) throws Exception { + return getMapper().writeValueAsString(obj); + } + + static public T jsonToObj(String json, Class tClass) throws Exception { + return getMapper().readValue(json, tClass); + } + +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/KeySearchFilter.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/KeySearchFilter.java new file mode 100644 index 00000000000..88c8029c32d --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/KeySearchFilter.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.util; + +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang.StringUtils; + +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +public class KeySearchFilter { + public static final String KEY_NAME = "name";// search, sort + + public static final String START_INDEX = "startIndex"; + public static final String PAGE_SIZE = "pageSize"; + public static final String SORT_BY = "sortBy"; + + private Map params; + private int startIndex; + private int maxRows = Integer.MAX_VALUE; + private boolean getCount = true; + private String sortBy; + private String sortType; + + public KeySearchFilter() { + this(null); + } + + public KeySearchFilter(String name, String value) { + setParam(name, value); + } + + public KeySearchFilter(Map values) { + setParams(values); + } + + public Map getParams() { + return params; + } + + public void setParams(Map params) { + this.params = params; + } + + public String getParam(String name) { + return params == null ? null : params.get(name); + } + + public void setParam(String name, String value) { + if(StringUtils.isEmpty(name) || StringUtils.isEmpty(value)) { + return; + } + + if(params == null) { + params = new HashMap<>(); + } + + params.put(name, value); + } + public boolean isEmpty() { + return MapUtils.isEmpty(params); + } + + public int getStartIndex() { + return startIndex; + } + + public void setStartIndex(int startIndex) { + this.startIndex = startIndex; + } + + public int getMaxRows() { + return maxRows; + } + + public void setMaxRows(int maxRows) { + this.maxRows = maxRows; + } + + public boolean isGetCount() { + return getCount; + } + + public void setGetCount(boolean getCount) { + this.getCount = getCount; + } + + public String getSortBy() { + return sortBy; + } + + public void setSortBy(String sortBy) { + this.sortBy = sortBy; + } + + public String getSortType() { + return sortType; + } + + public void setSortType(String sortType) { + this.sortType = sortType; + } + + @Override + public boolean equals(Object object) { + if (object == null || !(object instanceof KeySearchFilter)) { + return false; + } + KeySearchFilter that = (KeySearchFilter)object; + return Objects.equals(params, that.params); + } + + @Override + public int hashCode() { + return Objects.hash(params); + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/KeycloakUserStore.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/KeycloakUserStore.java new file mode 100644 index 00000000000..b80ba85d8ed --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/KeycloakUserStore.java @@ -0,0 +1,506 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.util; + +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.atlas.RequestContext; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.plugin.model.RangerRole; +import org.apache.atlas.utils.AtlasPerfMetrics; +import org.apache.atlas.utils.AtlasPerfTracer; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; +import org.keycloak.representations.idm.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import static org.apache.atlas.keycloak.client.AtlasKeycloakClient.getKeycloakClient; +import static org.apache.atlas.repository.Constants.*; +import static org.apache.atlas.repository.util.AccessControlUtils.ARGO_SERVICE_USER_NAME; +import static org.apache.atlas.repository.util.AccessControlUtils.BACKEND_SERVICE_USER_NAME; + + +public class KeycloakUserStore { + private static final Logger PERF_LOG = AtlasPerfTracer.getPerfLogger("KeycloakUserStore"); + private static final Logger LOG = LoggerFactory.getLogger(KeycloakUserStore.class); + + private static int NUM_THREADS = 5; + + private static String LOGIN_EVENT_DETAIL_KEY = "custom_required_action"; + private static String LOGIN_EVENT_DETAIL_VALUE = "UPDATE_PROFILE"; + + private static List EVENT_TYPES = Arrays.asList("LOGIN"); + private static List OPERATION_TYPES = Arrays.asList("CREATE", "UPDATE", "DELETE"); + private static List RESOURCE_TYPES = Arrays.asList("USER", "GROUP", "REALM_ROLE", "CLIENT", "REALM_ROLE_MAPPING", "GROUP_MEMBERSHIP", "CLIENT_ROLE_MAPPING"); + + private final String serviceName; + + public KeycloakUserStore(String serviceName) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerRolesProvider(serviceName=" + serviceName + ").RangerRolesProvider()"); + } + + this.serviceName = serviceName; + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerRolesProvider(serviceName=" + serviceName + ").RangerRolesProvider()"); + } + } + + public static ExecutorService getExecutorService(String namePattern) { + ExecutorService service = Executors.newFixedThreadPool(NUM_THREADS, + new ThreadFactoryBuilder().setNameFormat(namePattern + Thread.currentThread().getName()) + .build()); + return service; + } + + public boolean isKeycloakSubjectsStoreUpdated(long cacheLastUpdatedTime) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("getKeycloakSubjectsStoreUpdatedTime"); + if (cacheLastUpdatedTime == -1) { + return true; + } + + long latestKeycloakEventTime = -1L; + + try { + int size = 100; + + for (int from = 0; ; from += size) { + + List adminEvents = getKeycloakClient().getAdminEvents(OPERATION_TYPES, + null, null, null, null, null, null, null, + from, size); + + if (CollectionUtils.isEmpty(adminEvents) || cacheLastUpdatedTime > adminEvents.get(0).getTime()) { + break; + } + + Optional event = adminEvents.stream().filter(x -> RESOURCE_TYPES.contains(x.getResourceType())).findFirst(); + + if (event.isPresent()) { + latestKeycloakEventTime = event.get().getTime(); + break; + } + } + + if (latestKeycloakEventTime > cacheLastUpdatedTime) { + return true; + } + + //check Events for user registration event via OKTA + for (int from = 0; ; from += size) { + + List events = getKeycloakClient().getEvents(EVENT_TYPES, + null, null, null, null, null, from, size); + + if (CollectionUtils.isEmpty(events) || cacheLastUpdatedTime > events.get(0).getTime()) { + break; + } + + Optional event = events.stream().filter(this::isUpdateProfileEvent).findFirst(); + + if (event.isPresent()) { + latestKeycloakEventTime = event.get().getTime(); + break; + } + } + + if (latestKeycloakEventTime > cacheLastUpdatedTime) { + return true; + } + + } catch (Exception e) { + LOG.error("Error while fetching latest event time", e); + } finally { + RequestContext.get().endMetricRecord(metricRecorder); + } + + return false; + } + + private boolean isUpdateProfileEvent(EventRepresentation event) { + return MapUtils.isNotEmpty(event.getDetails()) && + event.getDetails().containsKey(LOGIN_EVENT_DETAIL_KEY) && + event.getDetails().get(LOGIN_EVENT_DETAIL_KEY).equals(LOGIN_EVENT_DETAIL_VALUE); + } + + public RangerRoles loadRolesIfUpdated(long lastUpdatedTime) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("loadRolesIfUpdated"); + + boolean isKeycloakUpdated = isKeycloakSubjectsStoreUpdated(lastUpdatedTime); + if (!isKeycloakUpdated) { + return null; + } + + List kRoles = getKeycloakClient().getAllRoles(); + LOG.info("Found {} keycloak roles", kRoles.size()); + + Set roleSet = new HashSet<>(); + RangerRoles rangerRoles = new RangerRoles(); + List userNamesList = new ArrayList<>(); + + submitCallablesAndWaitToFinish("RoleSubjectsFetcher", + kRoles.stream() + .map(x -> new RoleSubjectsFetcher(x, roleSet, userNamesList)) + .collect(Collectors.toList())); + + processDefaultRole(roleSet); + + LOG.info("Inverting roles"); + invertRoles(roleSet); + + rangerRoles.setRangerRoles(roleSet); + rangerRoles.setServiceName(serviceName); + + Date current = new Date(); + rangerRoles.setRoleUpdateTime(current); + rangerRoles.setServiceName(serviceName); + rangerRoles.setRoleVersion(-1L); + + RequestContext.get().endMetricRecord(recorder); + + return rangerRoles; + } + + public void invertRoles(Set roleSet) { + Map roleMap = new HashMap<>(); + for (RangerRole role : roleSet) { + RangerRole existingRole = roleMap.get(role.getName()); + if (existingRole != null) { + existingRole.setGroups(role.getGroups()); + existingRole.setUsers(role.getUsers()); + } else { + RangerRole newRole = new RangerRole(); + newRole.setName(role.getName()); + newRole.setUsers(role.getUsers()); + newRole.setGroups(role.getGroups()); + roleMap.put(role.getName(), newRole); + } + + List roles = role.getRoles(); + for (RangerRole.RoleMember roleMember : roles) { + if (role.getName().equals("default-roles-default") && roleMember.getName().equals("$guest")) { + continue; + } + RangerRole existingRoleMember = roleMap.get(roleMember.getName()); + if (existingRoleMember != null) { + List existingRoleMemberRoles = existingRoleMember.getRoles(); + // If the role already present in existing role, then skip + if (existingRoleMemberRoles.stream().anyMatch(x -> x.getName().equals(role.getName()))) { + continue; + } + existingRoleMemberRoles.add(new RangerRole.RoleMember(role.getName(), false)); + } else { + RangerRole newRoleMember = new RangerRole(); + newRoleMember.setName(roleMember.getName()); + newRoleMember.setRoles(new ArrayList<>(Arrays.asList(new RangerRole.RoleMember(role.getName(), false)))); + roleMap.put(roleMember.getName(), newRoleMember); + } + } + } + roleSet.clear(); + roleSet.addAll(roleMap.values()); + } + + private void processDefaultRole(Set roleSet) { + Optional defaultRole = roleSet.stream().filter(x -> KEYCLOAK_ROLE_DEFAULT.equals(x.getName())).findFirst(); + + if (defaultRole.isPresent()) { + List realmDefaultRoles = defaultRole.get().getRoles().stream().map(x -> x.getName()).collect(Collectors.toList()); + String tenantDefaultRealmUserRole = ""; + + if (realmDefaultRoles.contains(KEYCLOAK_ROLE_ADMIN)) { + tenantDefaultRealmUserRole = KEYCLOAK_ROLE_ADMIN; + } else if (realmDefaultRoles.contains(KEYCLOAK_ROLE_MEMBER)) { + tenantDefaultRealmUserRole = KEYCLOAK_ROLE_MEMBER; + } else if (realmDefaultRoles.contains(KEYCLOAK_ROLE_GUEST)) { + tenantDefaultRealmUserRole = KEYCLOAK_ROLE_GUEST; + } + + String finalTenantDefaultRealmUserRole = tenantDefaultRealmUserRole; + Optional targetRole = roleSet.stream().filter(x -> finalTenantDefaultRealmUserRole.equals(x.getName())).findFirst(); + + if (targetRole.isPresent()) { + List defaultUsers = new ArrayList<>(defaultRole.get().getUsers()); + List nonGuestUsers = new ArrayList<>(0); + + Optional adminRole = roleSet.stream().filter(x -> KEYCLOAK_ROLE_ADMIN.equals(x.getName())).findFirst(); + adminRole.ifPresent(rangerRole -> nonGuestUsers.addAll(rangerRole.getUsers())); + + Optional memberRole = roleSet.stream().filter(x -> KEYCLOAK_ROLE_MEMBER.equals(x.getName())).findFirst(); + memberRole.ifPresent(rangerRole -> nonGuestUsers.addAll(rangerRole.getUsers())); + + Optional apiTokenDefaultAccessRole = roleSet.stream().filter(x -> KEYCLOAK_ROLE_API_TOKEN.equals(x.getName())).findFirst(); + apiTokenDefaultAccessRole.ifPresent(rangerRole -> nonGuestUsers.addAll(rangerRole.getUsers())); + + defaultUsers.removeAll(nonGuestUsers); + defaultUsers.remove(new RangerRole.RoleMember(ARGO_SERVICE_USER_NAME, false)); + defaultUsers.remove(new RangerRole.RoleMember(BACKEND_SERVICE_USER_NAME, false)); + + targetRole.get().getUsers().addAll(defaultUsers); + } + } + } + + public RangerUserStore loadUserStoreIfUpdated(long lastUpdatedTime) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("loadUserStoreIfUpdated"); + + boolean isKeycloakUpdated = isKeycloakSubjectsStoreUpdated(lastUpdatedTime); + if (!isKeycloakUpdated) { + return null; + } + + Map> userGroupMapping = new HashMap<>(); + + List kUsers = getKeycloakClient().getAllUsers(); + LOG.info("Found {} keycloak users", kUsers.size()); + + List> callables = new ArrayList<>(); + kUsers.forEach(x -> callables.add(new UserGroupsFetcher(x, userGroupMapping))); + + submitCallablesAndWaitToFinish("UserGroupsFetcher", callables); + + RangerUserStore userStore = new RangerUserStore(); + userStore.setUserGroupMapping(userGroupMapping); + Date current = new Date(); + userStore.setUserStoreUpdateTime(current); + userStore.setServiceName(serviceName); + userStore.setUserStoreVersion(-1L); + + RequestContext.get().endMetricRecord(recorder); + + return userStore; + } + + private static RangerRole keycloakRoleToRangerRole(RoleRepresentation kRole) { + AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("keycloakRolesToRangerRoles"); + + RangerRole rangerRole = new RangerRole(); + rangerRole.setName(kRole.getName()); + rangerRole.setDescription(kRole.getDescription() + " " + kRole.getId()); + + RequestContext.get().endMetricRecord(recorder); + return rangerRole; + } + + private static List keycloakGroupsToRangerRoleMember(Set kGroups) { + AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("keycloakGroupsToRangerRoleMember"); + List rangerGroups = new ArrayList<>(); + + for (GroupRepresentation kGroup : kGroups) { + //TODO: Revisit isAdmin flag + rangerGroups.add(new RangerRole.RoleMember(kGroup.getName(), false)); + } + + RequestContext.get().endMetricRecord(recorder); + return rangerGroups; + } + + private static List keycloakUsersToRangerRoleMember(Set kUsers) { + AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("keycloakUsersToRangerRoleMember"); + List rangerUsers = new ArrayList<>(); + + for (UserRepresentation kUser : kUsers) { + //TODO: Revisit isAdmin flag + rangerUsers.add(new RangerRole.RoleMember(kUser.getUsername(), false)); + } + + RequestContext.get().endMetricRecord(recorder); + return rangerUsers; + } + + private static List keycloakRolesToRangerRoleMember(Set kRoles) { + AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("keycloakRolesToRangerRoleMember"); + List rangerRoles = new ArrayList<>(); + + for (RoleRepresentation kRole : kRoles) { + //TODO: Revisit isAdmin flag + rangerRoles.add(new RangerRole.RoleMember(kRole.getName(), false)); + } + + RequestContext.get().endMetricRecord(recorder); + return rangerRoles; + } + + protected static void submitCallablesAndWaitToFinish(String threadName, List> callables) throws AtlasBaseException { + ExecutorService service = getExecutorService(threadName + "-%d-"); + try { + + LOG.info("Submitting callables: {}", threadName); + callables.forEach(service::submit); + + service.shutdown(); + + boolean terminated = service.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS); + LOG.info("awaitTermination done: {}", threadName); + + if (!terminated) { + LOG.warn("Time out occurred while waiting to complete {}", threadName); + } + } catch (InterruptedException e) { + throw new AtlasBaseException(); + } + } + + static class RoleSubjectsFetcher implements Callable { + private Set roleSet; + private RoleRepresentation kRole; + List userNamesList; + + public RoleSubjectsFetcher(RoleRepresentation kRole, + Set roleSet, + List userNamesList) { + this.kRole = kRole; + this.roleSet = roleSet; + this.userNamesList = userNamesList; + } + + @Override + public RangerRole call() throws Exception { + AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("roleSubjectsFetcher"); + final RangerRole rangerRole = keycloakRoleToRangerRole(kRole); + + try { + //get all groups for Roles + Thread groupsFetcher = new Thread(() -> { + int start = 0; + int size = 500; + boolean found = true; + Set ret = new HashSet<>(); + + do { + try { + Set kGroups = getKeycloakClient().getRoleGroupMembers(kRole.getName(), start, size); + if (CollectionUtils.isNotEmpty(kGroups)) { + ret.addAll(kGroups); + start += size; + } else { + found = false; + } + } catch (Exception e) { + LOG.error("Failed to get group members with role", e); + throw new RuntimeException(e); + } + + } while (found && ret.size() % size == 0); + + rangerRole.setGroups(keycloakGroupsToRangerRoleMember(ret)); + }); + groupsFetcher.start(); + + //get all users for Roles + Thread usersFetcher = new Thread(() -> { + int start = 0; + int size = 500; + boolean found = true; + Set ret = new HashSet<>(); + + do { + try { + Set userRepresentations = getKeycloakClient().getRoleUserMembers(kRole.getName(), start, size); + if (CollectionUtils.isNotEmpty(userRepresentations)) { + ret.addAll(userRepresentations); + start += size; + } else { + found = false; + } + } catch (Exception e) { + LOG.error("Failed to get users for role {}", kRole.getName(), e); + throw new RuntimeException(e); + } + + } while (found && ret.size() % size == 0); + + rangerRole.setUsers(keycloakUsersToRangerRoleMember(ret)); + userNamesList.addAll(ret); + }); + usersFetcher.start(); + + //get all roles for Roles + Thread subRolesFetcher = new Thread(() -> { + Set kSubRoles = null; + try { + kSubRoles = getKeycloakClient().getRoleComposites(kRole.getName()); + rangerRole.setRoles(keycloakRolesToRangerRoleMember(kSubRoles)); + } catch (AtlasBaseException e) { + LOG.error("Failed to get composite for role {}", kRole.getName(), e); + throw new RuntimeException(e); + } + }); + subRolesFetcher.start(); + + try { + groupsFetcher.join(); + usersFetcher.join(); + subRolesFetcher.join(); + } catch (InterruptedException e) { + LOG.error("Failed to wait for threads to complete: {}", kRole.getName()); + e.printStackTrace(); + } + + RequestContext.get().endMetricRecord(recorder); + roleSet.add(rangerRole); + } catch (Exception e) { + LOG.error("RoleSubjectsFetcher: Failed to process role {}: {}", kRole.getName(), e.getMessage()); + } finally { + RequestContext.get().endMetricRecord(recorder); + } + + return rangerRole; + } + } + + static class UserGroupsFetcher implements Callable { + private Map> userGroupMapping; + private UserRepresentation kUser; + + public UserGroupsFetcher(UserRepresentation kUser, Map> userGroupMapping) { + this.kUser = kUser; + this.userGroupMapping = userGroupMapping; + } + + @Override + public Object call() throws Exception { + AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("userGroupsFetcher"); + + try { + List kGroups = getKeycloakClient().getGroupsForUserById(kUser.getId()); + userGroupMapping.put(kUser.getUsername(), + kGroups.stream() + .map(GroupRepresentation::getName) + .collect(Collectors.toSet())); + + } catch (Exception e) { + LOG.error("UserGroupsFetcher: Failed to process user {}: {}", kUser.getUsername(), e.getMessage()); + } finally { + RequestContext.get().endMetricRecord(recorder); + } + + return null; + } + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PasswordUtils.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PasswordUtils.java new file mode 100644 index 00000000000..bc18f331d9e --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PasswordUtils.java @@ -0,0 +1,231 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.plugin.util; + +import com.google.common.base.Splitter; +import com.google.common.base.Strings; +import com.google.common.collect.Lists; +import com.sun.jersey.core.util.Base64; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.crypto.Cipher; +import javax.crypto.SecretKey; +import javax.crypto.SecretKeyFactory; +import javax.crypto.spec.IvParameterSpec; +import javax.crypto.spec.PBEKeySpec; +import javax.crypto.spec.PBEParameterSpec; +import java.io.IOException; +import java.security.NoSuchAlgorithmException; +import java.security.SecureRandom; +import java.util.Map; + +public class PasswordUtils { + + private static final Logger LOG = LoggerFactory.getLogger(PasswordUtils.class); + + private final String cryptAlgo; + private String password; + private final int iterationCount; + private final char[] encryptKey; + private final byte[] salt; + private final byte[] iv; + private static final String LEN_SEPARATOR_STR = ":"; + + public static final String PBE_SHA512_AES_128 = "PBEWITHHMACSHA512ANDAES_128"; + + public static final String DEFAULT_CRYPT_ALGO = "PBEWithMD5AndDES"; + public static final String DEFAULT_ENCRYPT_KEY = "tzL1AKl5uc4NKYaoQ4P3WLGIBFPXWPWdu1fRm9004jtQiV"; + public static final String DEFAULT_SALT = "f77aLYLo"; + public static final int DEFAULT_ITERATION_COUNT = 17; + public static final byte[] DEFAULT_INITIAL_VECTOR = new byte[16]; + + public static String encryptPassword(String aPassword) throws IOException { + return build(aPassword).encrypt(); + } + + public static PasswordUtils build(String aPassword) { + return new PasswordUtils(aPassword); + } + + private String encrypt() throws IOException { + String ret = null; + String strToEncrypt = null; + if (password == null) { + strToEncrypt = ""; + } else { + strToEncrypt = password.length() + LEN_SEPARATOR_STR + password; + } + try { + Cipher engine = Cipher.getInstance(cryptAlgo); + PBEKeySpec keySpec = new PBEKeySpec(encryptKey); + SecretKeyFactory skf = SecretKeyFactory.getInstance(cryptAlgo); + SecretKey key = skf.generateSecret(keySpec); + engine.init(Cipher.ENCRYPT_MODE, key, new PBEParameterSpec(salt, iterationCount, new IvParameterSpec(iv))); + byte[] encryptedStr = engine.doFinal(strToEncrypt.getBytes()); + ret = new String(Base64.encode(encryptedStr)); + } + catch(Throwable t) { + LOG.error("Unable to encrypt password due to error", t); + throw new IOException("Unable to encrypt password due to error", t); + } + return ret; + } + + PasswordUtils(String aPassword) { + String[] crypt_algo_array = null; + byte[] SALT; + char[] ENCRYPT_KEY; + if (aPassword != null && aPassword.contains(",")) { + crypt_algo_array = Lists.newArrayList(Splitter.on(",").split(aPassword)).toArray(new String[0]); + } + if (crypt_algo_array != null && crypt_algo_array.length > 4) { + int index = 0; + cryptAlgo = crypt_algo_array[index++]; // 0 + ENCRYPT_KEY = crypt_algo_array[index++].toCharArray(); // 1 + SALT = crypt_algo_array[index++].getBytes(); // 2 + iterationCount = Integer.parseInt(crypt_algo_array[index++]);// 3 + if (needsIv(cryptAlgo)) { + iv = Base64.decode(crypt_algo_array[index++]); + } else { + iv = DEFAULT_INITIAL_VECTOR; + } + password = crypt_algo_array[index++]; + if (crypt_algo_array.length > index) { + for (int i = index; i < crypt_algo_array.length; i++) { + password = password + "," + crypt_algo_array[i]; + } + } + } else { + cryptAlgo = DEFAULT_CRYPT_ALGO; + ENCRYPT_KEY = DEFAULT_ENCRYPT_KEY.toCharArray(); + SALT = DEFAULT_SALT.getBytes(); + iterationCount = DEFAULT_ITERATION_COUNT; + iv = DEFAULT_INITIAL_VECTOR; + password = aPassword; + } + Map env = System.getenv(); + String encryptKeyStr = env.get("ENCRYPT_KEY"); + if (encryptKeyStr == null) { + encryptKey=ENCRYPT_KEY; + }else{ + encryptKey=encryptKeyStr.toCharArray(); + } + String saltStr = env.get("ENCRYPT_SALT"); + if (saltStr == null) { + salt = SALT; + }else{ + salt=saltStr.getBytes(); + } + } + + public static String decryptPassword(String aPassword) throws IOException { + return build(aPassword).decrypt(); + } + + private String decrypt() throws IOException { + String ret = null; + try { + byte[] decodedPassword = Base64.decode(password); + Cipher engine = Cipher.getInstance(cryptAlgo); + PBEKeySpec keySpec = new PBEKeySpec(encryptKey); + SecretKeyFactory skf = SecretKeyFactory.getInstance(cryptAlgo); + SecretKey key = skf.generateSecret(keySpec); + engine.init(Cipher.DECRYPT_MODE, key,new PBEParameterSpec(salt, iterationCount, new IvParameterSpec(iv))); + String decrypted = new String(engine.doFinal(decodedPassword)); + int foundAt = decrypted.indexOf(LEN_SEPARATOR_STR); + if (foundAt > -1) { + if (decrypted.length() > foundAt) { + ret = decrypted.substring(foundAt+1); + } + else { + ret = ""; + } + } + else { + ret = null; + } + } + catch(Throwable t) { + LOG.error("Unable to decrypt password due to error", t); + throw new IOException("Unable to decrypt password due to error", t); + } + return ret; + } + + public static boolean needsIv(String cryptoAlgo) { + if (Strings.isNullOrEmpty(cryptoAlgo)) + return false; + + return PBE_SHA512_AES_128.toLowerCase().equals(cryptoAlgo.toLowerCase()) + || cryptoAlgo.toLowerCase().contains("aes_128") || cryptoAlgo.toLowerCase().contains("aes_256"); + } + + public static String generateIvIfNeeded(String cryptAlgo) throws NoSuchAlgorithmException { + if (!needsIv(cryptAlgo)) + return null; + return generateBase64EncodedIV(); + } + + private static String generateBase64EncodedIV() throws NoSuchAlgorithmException { + byte[] iv = new byte[16]; + SecureRandom.getInstance("NativePRNGNonBlocking").nextBytes(iv); + return new String(Base64.encode(iv)); + } + + public String getCryptAlgo() { + return cryptAlgo; + } + + public String getPassword() { + return password; + } + + public int getIterationCount() { + return iterationCount; + } + + public char[] getEncryptKey() { + return encryptKey; + } + + public byte[] getSalt() { + return salt; + } + + public byte[] getIv() { + return iv; + } + + public String getIvAsString() { + return new String(Base64.encode(getIv())); + } + public static String getDecryptPassword(String password) { + String decryptedPwd = null; + try { + decryptedPwd = decryptPassword(password); + } catch (Exception ex) { + LOG.warn("Password decryption failed, trying original password string."); + decryptedPwd = null; + } finally { + if (decryptedPwd == null) { + decryptedPwd = password; + } + } + return decryptedPwd; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PerfDataRecorder.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PerfDataRecorder.java new file mode 100644 index 00000000000..5c10c9d424d --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PerfDataRecorder.java @@ -0,0 +1,173 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.util; + +import com.google.common.collect.ImmutableMap; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; + +public class PerfDataRecorder { + private static final Log LOG = LogFactory.getLog(PerfDataRecorder.class); + private static final Log PERF = RangerPerfTracer.getPerfLogger(PerfDataRecorder.class); + + private static volatile PerfDataRecorder instance; + private Map perfStatistics = new HashMap<>(); + + public static void initialize(List names) { + if (instance == null) { + synchronized (PerfDataRecorder.class) { + if (instance == null) { + instance = new PerfDataRecorder(names); + } + } + } + } + + public static boolean collectStatistics() { + return instance != null; + } + + public static void printStatistics() { + if (instance != null) { + instance.dumpStatistics(); + } + } + + public static void clearStatistics() { + if (instance != null) { + instance.clear(); + } + } + + public static void recordStatistic(String tag, long elapsedTime) { + if (instance != null) { + instance.record(tag, elapsedTime); + } + } + + private void dumpStatistics() { + List tags = new ArrayList<>(perfStatistics.keySet()); + + Collections.sort(tags); + + for (String tag : tags) { + PerfStatistic perfStatistic = perfStatistics.get(tag); + + long averageTimeSpent = 0L; + + if (perfStatistic.numberOfInvocations.get() != 0L) { + averageTimeSpent = perfStatistic.microSecondsSpent.get()/perfStatistic.numberOfInvocations.get(); + } + + String logMsg = "[" + tag + "]" + + " execCount: " + perfStatistic.numberOfInvocations.get() + + ", totalTimeTaken: " + perfStatistic.microSecondsSpent.get() + " Îŧs" + + ", maxTimeTaken: " + perfStatistic.maxTimeSpent.get() + " Îŧs" + + ", minTimeTaken: " + perfStatistic.minTimeSpent.get() + " Îŧs" + + ", avgTimeTaken: " + averageTimeSpent + " Îŧs"; + + LOG.info(logMsg); + PERF.debug(logMsg); + } + } + + private void clear() { + perfStatistics.clear(); + } + + private void record(String tag, long elapsedTime) { + PerfStatistic perfStatistic = perfStatistics.get(tag); + + if (perfStatistic == null) { + synchronized (PerfDataRecorder.class) { + perfStatistic = perfStatistics.get(tag); + + if(perfStatistic == null) { + perfStatistic = new PerfStatistic(); + perfStatistics.put(tag, perfStatistic); + } + } + } + + perfStatistic.addPerfDataItem(elapsedTime); + } + + private PerfDataRecorder(List names) { + if (CollectionUtils.isNotEmpty(names)) { + for (String name : names) { + // Create structure + perfStatistics.put(name, new PerfStatistic()); + } + } + } + + public static Map exposeStatistics() { + if (instance != null) { + return ImmutableMap.copyOf(instance.perfStatistics); + } + return ImmutableMap.of(); + } + + public static class PerfStatistic { + private AtomicLong numberOfInvocations = new AtomicLong(0L); + private AtomicLong microSecondsSpent = new AtomicLong(0L); + private AtomicLong minTimeSpent = new AtomicLong(Long.MAX_VALUE); + private AtomicLong maxTimeSpent = new AtomicLong(Long.MIN_VALUE); + + void addPerfDataItem(final long timeTaken) { + numberOfInvocations.getAndIncrement(); + microSecondsSpent.getAndAdd(timeTaken); + + long min = minTimeSpent.get(); + if(timeTaken < min) { + minTimeSpent.compareAndSet(min, timeTaken); + } + + long max = maxTimeSpent.get(); + if(timeTaken > max) { + maxTimeSpent.compareAndSet(max, timeTaken); + } + } + + public long getNumberOfInvocations() { + return numberOfInvocations.get(); + } + + public long getMicroSecondsSpent() { + return microSecondsSpent.get(); + } + + public long getMinTimeSpent() { + return minTimeSpent.get(); + } + + public long getMaxTimeSpent() { + return maxTimeSpent.get(); + } + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java new file mode 100644 index 00000000000..aae09a7d262 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java @@ -0,0 +1,530 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.util; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import org.apache.atlas.authz.admin.client.AtlasAuthAdminClient; +import org.apache.atlas.policytransformer.CachePolicyTransformerImpl; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.admin.client.RangerAdminClient; +import org.apache.atlas.authorization.hadoop.config.RangerPluginConfig; +import org.apache.atlas.plugin.policyengine.RangerPluginContext; +import org.apache.atlas.plugin.service.RangerBasePlugin; + +import java.io.File; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.Reader; +import java.io.Writer; +import java.util.Timer; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; + + +public class PolicyRefresher extends Thread { + private static final Log LOG = LogFactory.getLog(PolicyRefresher.class); + + private static final Log PERF_POLICYENGINE_INIT_LOG = RangerPerfTracer.getPerfLogger("policyengine.init"); + + private final RangerBasePlugin plugIn; + private final String serviceType; + private final String serviceName; + private final RangerAdminClient rangerAdmin; + private final AtlasAuthAdminClient atlasAuthAdminClient; + private final RangerRolesProvider rolesProvider; + private final RangerUserStoreProvider userStoreProvider; + private final long pollingIntervalMs; + private final String cacheFileName; + private final String cacheDir; + private final Gson gson; + private final BlockingQueue policyDownloadQueue = new LinkedBlockingQueue<>(); + private Timer policyDownloadTimer; + private long lastKnownVersion = -1L; + private long lastUpdatedTiemInMillis = -1L; + private long lastActivationTimeInMillis; + private boolean policiesSetInPlugin; + private boolean serviceDefSetInPlugin; + + + public PolicyRefresher(RangerBasePlugin plugIn) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> PolicyRefresher(serviceName=" + plugIn.getServiceName() + ").PolicyRefresher()"); + } + LOG.info("==> PolicyRefresher(serviceName=" + plugIn.getServiceName() + ").PolicyRefresher()"); + + RangerPluginConfig pluginConfig = plugIn.getConfig(); + String propertyPrefix = pluginConfig.getPropertyPrefix(); + + this.plugIn = plugIn; + this.serviceType = plugIn.getServiceType(); + this.serviceName = plugIn.getServiceName(); + this.cacheDir = pluginConfig.get(propertyPrefix + ".policy.cache.dir"); + + String appId = StringUtils.isEmpty(plugIn.getAppId()) ? serviceType : plugIn.getAppId(); + String cacheFilename = String.format("%s_%s.json", appId, serviceName); + + cacheFilename = cacheFilename.replace(File.separatorChar, '_'); + cacheFilename = cacheFilename.replace(File.pathSeparatorChar, '_'); + + this.cacheFileName = cacheFilename; + + rangerAdmin = getRangerAdminClient(); + + Gson gson = null; + try { + gson = new GsonBuilder().setDateFormat("yyyyMMdd-HH:mm:ss.SSS-Z").create(); + } catch(Throwable excp) { + LOG.fatal("PolicyRefresher(): failed to create GsonBuilder object", excp); + } + + RangerPluginContext pluginContext = plugIn.getPluginContext(); + this.atlasAuthAdminClient = pluginContext.getAtlasAuthAdminClient(); + this.gson = gson; + this.rolesProvider = new RangerRolesProvider(getServiceType(), appId, getServiceName(), atlasAuthAdminClient, cacheDir, pluginConfig); + this.userStoreProvider = new RangerUserStoreProvider(getServiceType(), appId, getServiceName(), atlasAuthAdminClient, cacheDir, pluginConfig); + this.pollingIntervalMs = pluginConfig.getLong(propertyPrefix + ".policy.pollIntervalMs", 30 * 1000); + + setName("PolicyRefresher(serviceName=" + serviceName + ")-" + getId()); + + if(LOG.isDebugEnabled()) { + LOG.debug("<== PolicyRefresher(serviceName=" + serviceName + ").PolicyRefresher()"); + } + } + + /** + * @return the plugIn + */ + public RangerBasePlugin getPlugin() { + return plugIn; + } + + /** + * @return the serviceType + */ + public String getServiceType() { + return serviceType; + } + + /** + * @return the serviceName + */ + public String getServiceName() { + return serviceName; + } + + /** + * @return the rangerAdmin + */ + public RangerAdminClient getRangerAdminClient() { + return rangerAdmin; + } + + public long getLastActivationTimeInMillis() { + return lastActivationTimeInMillis; + } + + public void setLastActivationTimeInMillis(long lastActivationTimeInMillis) { + this.lastActivationTimeInMillis = lastActivationTimeInMillis; + } + + public void startRefresher() { + loadRoles(); + loadPolicy(); + loadUserStore(); + super.start(); + + policyDownloadTimer = new Timer("policyDownloadTimer", true); + + try { + policyDownloadTimer.schedule(new DownloaderTask(policyDownloadQueue), pollingIntervalMs, pollingIntervalMs); + + LOG.info("Scheduled policyDownloadRefresher to download policies every " + pollingIntervalMs + " milliseconds"); + + } catch (IllegalStateException exception) { + LOG.error("Error scheduling policyDownloadTimer:", exception); + LOG.error("*** Policies will NOT be downloaded every " + pollingIntervalMs + " milliseconds ***"); + + policyDownloadTimer = null; + } + + } + + public void stopRefresher() { + + Timer policyDownloadTimer = this.policyDownloadTimer; + + this.policyDownloadTimer = null; + + if (policyDownloadTimer != null) { + policyDownloadTimer.cancel(); + } + + if (super.isAlive()) { + super.interrupt(); + + boolean setInterrupted = false; + boolean isJoined = false; + + while (!isJoined) { + try { + super.join(); + isJoined = true; + } catch (InterruptedException excp) { + LOG.warn("PolicyRefresher(serviceName=" + serviceName + "): error while waiting for thread to exit", excp); + LOG.warn("Retrying Thread.join(). Current thread will be marked as 'interrupted' after Thread.join() returns"); + setInterrupted = true; + } + } + if (setInterrupted) { + Thread.currentThread().interrupt(); + } + } + } + + public void run() { + + if(LOG.isDebugEnabled()) { + LOG.debug("==> PolicyRefresher(serviceName=" + serviceName + ").run()"); + } + + while(true) { + DownloadTrigger trigger = null; + try { + trigger = policyDownloadQueue.take(); + + loadRoles(); + loadPolicy(); + loadUserStore(); + } catch(InterruptedException excp) { + LOG.info("PolicyRefresher(serviceName=" + serviceName + ").run(): interrupted! Exiting thread", excp); + break; + } finally { + if (trigger != null) { + trigger.signalCompletion(); + } + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== PolicyRefresher(serviceName=" + serviceName + ").run()"); + } + } + + public void syncPoliciesWithAdmin(DownloadTrigger token) throws InterruptedException { + policyDownloadQueue.put(token); + token.waitForCompletion(); + } + + private void loadPolicy() { + + if(LOG.isDebugEnabled()) { + LOG.debug("==> PolicyRefresher(serviceName=" + serviceName + ").loadPolicy()"); + } + + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_POLICYENGINE_INIT_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_POLICYENGINE_INIT_LOG, "PolicyRefresher.loadPolicy(serviceName=" + serviceName + ")"); + long freeMemory = Runtime.getRuntime().freeMemory(); + long totalMemory = Runtime.getRuntime().totalMemory(); + PERF_POLICYENGINE_INIT_LOG.debug("In-Use memory: " + (totalMemory-freeMemory) + ", Free memory:" + freeMemory); + } + + try { + //load policy from PolicyAdmin + ServicePolicies svcPolicies = loadPolicyfromPolicyAdmin(); + + if (svcPolicies == null) { + //if Policy fetch from Policy Admin Fails, load from cache + if (!policiesSetInPlugin) { + svcPolicies = loadFromCache(); + } + } + + if (PERF_POLICYENGINE_INIT_LOG.isDebugEnabled()) { + long freeMemory = Runtime.getRuntime().freeMemory(); + long totalMemory = Runtime.getRuntime().totalMemory(); + PERF_POLICYENGINE_INIT_LOG.debug("In-Use memory: " + (totalMemory - freeMemory) + ", Free memory:" + freeMemory); + } + + if (svcPolicies != null) { + plugIn.setPolicies(svcPolicies); + policiesSetInPlugin = true; + serviceDefSetInPlugin = false; + setLastActivationTimeInMillis(System.currentTimeMillis()); + lastKnownVersion = svcPolicies.getPolicyVersion() != null ? svcPolicies.getPolicyVersion() : -1L; + lastUpdatedTiemInMillis = svcPolicies.getPolicyUpdateTime() != null ? svcPolicies.getPolicyUpdateTime().getTime() : -1L; + } else { + if (!policiesSetInPlugin && !serviceDefSetInPlugin) { + plugIn.setPolicies(null); + serviceDefSetInPlugin = true; + } + } + } catch (RangerServiceNotFoundException snfe) { + if (!serviceDefSetInPlugin) { + disableCache(); + plugIn.setPolicies(null); + serviceDefSetInPlugin = true; + setLastActivationTimeInMillis(System.currentTimeMillis()); + lastKnownVersion = -1; + lastUpdatedTiemInMillis = -1; + } + } catch (Exception excp) { + LOG.error("Encountered unexpected exception, ignoring..", excp); + } + + RangerPerfTracer.log(perf); + + if(LOG.isDebugEnabled()) { + LOG.debug("<== PolicyRefresher(serviceName=" + serviceName + ").loadPolicy()"); + } + } + + private ServicePolicies loadPolicyfromPolicyAdmin() throws RangerServiceNotFoundException { + + if(LOG.isDebugEnabled()) { + LOG.debug("==> PolicyRefresher(serviceName=" + serviceName + ").loadPolicyfromPolicyAdmin()"); + } + + ServicePolicies svcPolicies = null; + + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_POLICYENGINE_INIT_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_POLICYENGINE_INIT_LOG, "PolicyRefresher.loadPolicyFromPolicyAdmin(serviceName=" + serviceName + ")"); + } + + try { + + if (serviceName.equals("atlas") && plugIn.getTypeRegistry() != null && lastUpdatedTiemInMillis == -1) { + RangerRESTUtils restUtils = new RangerRESTUtils(); + CachePolicyTransformerImpl transformer = new CachePolicyTransformerImpl(plugIn.getTypeRegistry()); + + svcPolicies = transformer.getPolicies(serviceName, + restUtils.getPluginId(serviceName, plugIn.getAppId()), + lastUpdatedTiemInMillis); + } else { + svcPolicies = atlasAuthAdminClient.getServicePoliciesIfUpdated(lastUpdatedTiemInMillis); + } + + boolean isUpdated = svcPolicies != null; + + if(isUpdated) { + long newVersion = svcPolicies.getPolicyVersion() == null ? -1 : svcPolicies.getPolicyVersion().longValue(); + + if(!StringUtils.equals(serviceName, svcPolicies.getServiceName())) { + LOG.warn("PolicyRefresher(serviceName=" + serviceName + "): ignoring unexpected serviceName '" + svcPolicies.getServiceName() + "' in service-store"); + + svcPolicies.setServiceName(serviceName); + } + + LOG.info("PolicyRefresher(serviceName=" + serviceName + "): found updated version. lastKnownVersion=" + lastKnownVersion + "; newVersion=" + newVersion); + + } else { + if(LOG.isDebugEnabled()) { + LOG.debug("PolicyRefresher(serviceName=" + serviceName + ").run(): no update found. lastKnownVersion=" + lastKnownVersion); + } + } + } catch (Exception excp) { + LOG.error("PolicyRefresher(serviceName=" + serviceName + "): failed to refresh policies. Will continue to use last known version of policies (" + lastKnownVersion + ")", excp); + svcPolicies = null; + } + + RangerPerfTracer.log(perf); + + if(LOG.isDebugEnabled()) { + LOG.debug("<== PolicyRefresher(serviceName=" + serviceName + ").loadPolicyfromPolicyAdmin()"); + } + + return svcPolicies; + } + + private ServicePolicies loadFromCache() { + + ServicePolicies policies = null; + + if(LOG.isDebugEnabled()) { + LOG.debug("==> PolicyRefresher(serviceName=" + serviceName + ").loadFromCache()"); + } + + File cacheFile = cacheDir == null ? null : new File(cacheDir + File.separator + cacheFileName); + + if(cacheFile != null && cacheFile.isFile() && cacheFile.canRead()) { + Reader reader = null; + + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_POLICYENGINE_INIT_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_POLICYENGINE_INIT_LOG, "PolicyRefresher.loadFromCache(serviceName=" + serviceName + ")"); + } + + try { + reader = new FileReader(cacheFile); + + policies = gson.fromJson(reader, ServicePolicies.class); + + if(policies != null) { + if(!StringUtils.equals(serviceName, policies.getServiceName())) { + LOG.warn("ignoring unexpected serviceName '" + policies.getServiceName() + "' in cache file '" + cacheFile.getAbsolutePath() + "'"); + + policies.setServiceName(serviceName); + } + + lastKnownVersion = policies.getPolicyVersion() == null ? -1 : policies.getPolicyVersion().longValue(); + lastUpdatedTiemInMillis = policies.getPolicyUpdateTime() == null ? -1 : policies.getPolicyUpdateTime().getTime(); + } + } catch (Exception excp) { + LOG.error("failed to load policies from cache file " + cacheFile.getAbsolutePath(), excp); + } finally { + RangerPerfTracer.log(perf); + + if(reader != null) { + try { + reader.close(); + } catch(Exception excp) { + LOG.error("error while closing opened cache file " + cacheFile.getAbsolutePath(), excp); + } + } + } + } else { + LOG.warn("cache file does not exist or not readable '" + (cacheFile == null ? null : cacheFile.getAbsolutePath()) + "'"); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== PolicyRefresher(serviceName=" + serviceName + ").loadFromCache()"); + } + + return policies; + } + public void saveToCache(ServicePolicies policies) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> PolicyRefresher(serviceName=" + serviceName + ").saveToCache()"); + } + + if(policies != null) { + File cacheFile = null; + if (cacheDir != null) { + // Create the cacheDir if it doesn't already exist + File cacheDirTmp = new File(cacheDir); + if (cacheDirTmp.exists()) { + cacheFile = new File(cacheDir + File.separator + cacheFileName); + } else { + try { + cacheDirTmp.mkdirs(); + cacheFile = new File(cacheDir + File.separator + cacheFileName); + } catch (SecurityException ex) { + LOG.error("Cannot create cache directory", ex); + } + } + } + + if(cacheFile != null) { + + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_POLICYENGINE_INIT_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_POLICYENGINE_INIT_LOG, "PolicyRefresher.saveToCache(serviceName=" + serviceName + ")"); + } + + Writer writer = null; + + try { + writer = new FileWriter(cacheFile); + + gson.toJson(policies, writer); + } catch (Exception excp) { + LOG.error("failed to save policies to cache file '" + cacheFile.getAbsolutePath() + "'", excp); + } finally { + if(writer != null) { + try { + writer.close(); + } catch(Exception excp) { + LOG.error("error while closing opened cache file '" + cacheFile.getAbsolutePath() + "'", excp); + } + } + } + + RangerPerfTracer.log(perf); + + } + } else { + LOG.info("policies is null. Nothing to save in cache"); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== PolicyRefresher(serviceName=" + serviceName + ").saveToCache()"); + } + } + + private void disableCache() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> PolicyRefresher.disableCache(serviceName=" + serviceName + ")"); + } + + File cacheFile = cacheDir == null ? null : new File(cacheDir + File.separator + cacheFileName); + + if(cacheFile != null && cacheFile.isFile() && cacheFile.canRead()) { + LOG.warn("Cleaning up local cache"); + String renamedCacheFile = cacheFile.getAbsolutePath() + "_" + System.currentTimeMillis(); + if (!cacheFile.renameTo(new File(renamedCacheFile))) { + LOG.error("Failed to move " + cacheFile.getAbsolutePath() + " to " + renamedCacheFile); + } else { + LOG.warn("Moved " + cacheFile.getAbsolutePath() + " to " + renamedCacheFile); + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("No local policy cache found. No need to disable it!"); + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== PolicyRefresher.disableCache(serviceName=" + serviceName + ")"); + } + } + + private void loadRoles() { + if(LOG.isDebugEnabled()) { + LOG.debug("==> PolicyRefresher(serviceName=" + serviceName + ").loadRoles()"); + } + + //Load the Ranger UserGroup Roles + rolesProvider.loadUserGroupRoles(plugIn); + + if(LOG.isDebugEnabled()) { + LOG.debug("<== PolicyRefresher(serviceName=" + serviceName + ").loadRoles()"); + } + } + + private void loadUserStore() { + if(LOG.isDebugEnabled()) { + LOG.debug("==> PolicyRefresher(serviceName=" + serviceName + ").loadGroups()"); + } + + //Load the Ranger UserGroup Roles + userStoreProvider.loadUserStore(plugIn); + + if(LOG.isDebugEnabled()) { + LOG.debug("<== PolicyRefresher(serviceName=" + serviceName + ").loadRoles()"); + } + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerAccessRequestUtil.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerAccessRequestUtil.java new file mode 100644 index 00000000000..3a37ecdc1ed --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerAccessRequestUtil.java @@ -0,0 +1,187 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.util; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.plugin.contextenricher.RangerTagForEval; +import org.apache.atlas.plugin.policyengine.RangerAccessResource; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +public class RangerAccessRequestUtil { + private static final Log LOG = LogFactory.getLog(RangerAccessRequestUtil.class); + + public static final String KEY_CONTEXT_TAGS = "TAGS"; + public static final String KEY_CONTEXT_TAG_OBJECT = "TAG_OBJECT"; + public static final String KEY_CONTEXT_RESOURCE = "RESOURCE"; + public static final String KEY_CONTEXT_REQUESTED_RESOURCES = "REQUESTED_RESOURCES"; + public static final String KEY_CONTEXT_USERSTORE = "USERSTORE"; + public static final String KEY_TOKEN_NAMESPACE = "token:"; + public static final String KEY_USER = "USER"; + public static final String KEY_OWNER = "OWNER"; + public static final String KEY_ROLES = "ROLES"; + public static final String KEY_CONTEXT_ACCESSTYPES = "ACCESSTYPES"; + public static final String KEY_CONTEXT_IS_ANY_ACCESS = "ISANYACCESS"; + + public static void setRequestTagsInContext(Map context, Set tags) { + if(CollectionUtils.isEmpty(tags)) { + context.remove(KEY_CONTEXT_TAGS); + } else { + context.put(KEY_CONTEXT_TAGS, tags); + } + } + + public static Set getRequestTagsFromContext(Map context) { + Set ret = null; + Object val = context.get(RangerAccessRequestUtil.KEY_CONTEXT_TAGS); + + if (val instanceof Set) { + try { + @SuppressWarnings("unchecked") + Set tags = (Set) val; + + ret = tags; + } catch (Throwable t) { + LOG.error("getRequestTags(): failed to get tags from context", t); + } + } + + return ret; + } + + public static void setCurrentTagInContext(Map context, RangerTagForEval tag) { + context.put(KEY_CONTEXT_TAG_OBJECT, tag); + } + + public static RangerTagForEval getCurrentTagFromContext(Map context) { + RangerTagForEval ret = null; + Object val = context.get(KEY_CONTEXT_TAG_OBJECT); + + if(val instanceof RangerTagForEval) { + ret = (RangerTagForEval)val; + } + + return ret; + } + + public static void setRequestedResourcesInContext(Map context, RangerRequestedResources resources) { + context.put(KEY_CONTEXT_REQUESTED_RESOURCES, resources); + } + + public static RangerRequestedResources getRequestedResourcesFromContext(Map context) { + RangerRequestedResources ret = null; + Object val = context.get(KEY_CONTEXT_REQUESTED_RESOURCES); + + if(val instanceof RangerRequestedResources) { + ret = (RangerRequestedResources)val; + } + + return ret; + } + + public static void setCurrentResourceInContext(Map context, RangerAccessResource resource) { + context.put(KEY_CONTEXT_RESOURCE, resource); + } + + public static RangerAccessResource getCurrentResourceFromContext(Map context) { + RangerAccessResource ret = null; + Object val = MapUtils.isNotEmpty(context) ? context.get(KEY_CONTEXT_RESOURCE) : null; + + if(val instanceof RangerAccessResource) { + ret = (RangerAccessResource)val; + } + + return ret; + } + + public static Map copyContext(Map context) { + final Map ret; + + if(MapUtils.isEmpty(context)) { + ret = new HashMap<>(); + } else { + ret = new HashMap<>(context); + + ret.remove(KEY_CONTEXT_TAGS); + ret.remove(KEY_CONTEXT_TAG_OBJECT); + ret.remove(KEY_CONTEXT_RESOURCE); + // don't remove REQUESTED_RESOURCES + } + + return ret; + } + + public static void setCurrentUserInContext(Map context, String user) { + setTokenInContext(context, KEY_USER, user); + } + public static void setOwnerInContext(Map context, String owner) { + setTokenInContext(context, KEY_OWNER, owner); + } + public static String getCurrentUserFromContext(Map context) { + Object ret = getTokenFromContext(context, KEY_USER); + return ret != null ? ret.toString() : ""; + } + + public static void setTokenInContext(Map context, String tokenName, Object tokenValue) { + String tokenNameWithNamespace = KEY_TOKEN_NAMESPACE + tokenName; + context.put(tokenNameWithNamespace, tokenValue); + } + public static Object getTokenFromContext(Map context, String tokenName) { + String tokenNameWithNamespace = KEY_TOKEN_NAMESPACE + tokenName; + return MapUtils.isNotEmpty(context) ? context.get(tokenNameWithNamespace) : null; + } + + public static void setCurrentUserRolesInContext(Map context, Set roles) { + setTokenInContext(context, KEY_ROLES, roles); + } + public static Set getCurrentUserRolesFromContext(Map context) { + Object ret = getTokenFromContext(context, KEY_ROLES); + return ret != null ? (Set) ret : Collections.EMPTY_SET; + } + + public static void setRequestUserStoreInContext(Map context, RangerUserStore rangerUserStore) { + context.put(KEY_CONTEXT_USERSTORE, rangerUserStore); + } + + public static RangerUserStore getRequestUserStoreFromContext(Map context) { + RangerUserStore ret = null; + Object val = context.get(KEY_CONTEXT_USERSTORE); + + if(val instanceof RangerUserStore) { + ret = (RangerUserStore) val; + } + + return ret; + } + + public static void setIsAnyAccessInContext(Map context, Boolean value) { + context.put(KEY_CONTEXT_IS_ANY_ACCESS, value); + } + + public static Boolean getIsAnyAccessInContext(Map context) { + return (Boolean)context.get(KEY_CONTEXT_IS_ANY_ACCESS); + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerCommonConstants.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerCommonConstants.java new file mode 100644 index 00000000000..da939a7ad0f --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerCommonConstants.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.util; + +public class RangerCommonConstants { + + public static final String PROP_COOKIE_NAME = "ranger.admin.cookie.name"; + public static final String DEFAULT_COOKIE_NAME = "RANGERADMINSESSIONID"; + + public static final String RANGER_ADMIN_SUFFIX_POLICY_DELTA = ".supports.policy.deltas"; + public static final String PLUGIN_CONFIG_SUFFIX_POLICY_DELTA = ".supports.policy.deltas"; + + public static final String RANGER_ADMIN_SUFFIX_TAG_DELTA = ".supports.tag.deltas"; + public static final String PLUGIN_CONFIG_SUFFIX_TAG_DELTA = ".supports.tag.deltas"; + + public static final String RANGER_ADMIN_SUFFIX_IN_PLACE_POLICY_UPDATES = ".supports.in.place.policy.updates"; + public static final String PLUGIN_CONFIG_SUFFIX_IN_PLACE_POLICY_UPDATES = ".supports.in.place.policy.updates"; + + public static final String RANGER_ADMIN_SUFFIX_IN_PLACE_TAG_UPDATES = ".supports.in.place.tag.updates"; + public static final String PLUGIN_CONFIG_SUFFIX_IN_PLACE_TAG_UPDATES = ".supports.in.place.tag.updates"; + + public static final boolean RANGER_ADMIN_SUFFIX_POLICY_DELTA_DEFAULT = false; + public static final boolean PLUGIN_CONFIG_SUFFIX_POLICY_DELTA_DEFAULT = false; + + public static final boolean RANGER_ADMIN_SUFFIX_TAG_DELTA_DEFAULT = false; + public static final boolean PLUGIN_CONFIG_SUFFIX_TAG_DELTA_DEFAULT = false; + + public static final boolean RANGER_ADMIN_SUFFIX_IN_PLACE_POLICY_UPDATES_DEFAULT = false; + public static final boolean PLUGIN_CONFIG_SUFFIX_IN_PLACE_POLICY_UPDATES_DEFAULT = false; + + public static final boolean RANGER_ADMIN_SUFFIX_IN_PLACE_TAG_UPDATES_DEFAULT = false; + public static final boolean PLUGIN_CONFIG_SUFFIX_IN_PLACE_TAG_UPDATES_DEFAULT = false; + + public static final boolean POLICY_REST_CLIENT_SESSION_COOKIE_ENABLED = true; + + public static final String SCRIPT_OPTION_ENABLE_JSON_CTX = "enableJsonCtx"; + public static final String SCRIPT_VAR_CONTEXT = "_ctx"; + public static final String SCRIPT_VAR_CONTEXT_JSON = "_ctx_json"; + public static final String SCRIPT_FIELD_ACCESS_TIME = "accessTime"; + public static final String SCRIPT_FIELD_ACCESS_TYPE = "accessType"; + public static final String SCRIPT_FIELD_ACTION = "action"; + public static final String SCRIPT_FIELD_CLIENT_IP_ADDRESS = "clientIPAddress"; + public static final String SCRIPT_FIELD_CLIENT_TYPE = "clientType"; + public static final String SCRIPT_FIELD_CLUSTER_NAME = "clusterName"; + public static final String SCRIPT_FIELD_CLUSTER_TYPE = "clusterType"; + public static final String SCRIPT_FIELD_FORWARDED_ADDRESSES = "forwardedAddresses"; + public static final String SCRIPT_FIELD_REMOTE_IP_ADDRESS = "remoteIPAddress"; + public static final String SCRIPT_FIELD_REQUEST_DATA = "requestData"; + public static final String SCRIPT_FIELD_RESOURCE = "resource"; + public static final String SCRIPT_FIELD_RESOURCE_OWNER_USER = "resourceOwnerUser"; + public static final String SCRIPT_FIELD_RESOURCE_MATCHING_SCOPE = "resourceMatchingScope"; + public static final String SCRIPT_FIELD_TAG = "tag"; + public static final String SCRIPT_FIELD_TAGS = "tags"; + public static final String SCRIPT_FIELD_USER = "user"; + public static final String SCRIPT_FIELD_USER_ATTRIBUTES = "userAttributes"; + public static final String SCRIPT_FIELD_USER_GROUPS = "userGroups"; + public static final String SCRIPT_FIELD_USER_GROUP_ATTRIBUTES = "userGroupAttributes"; + public static final String SCRIPT_FIELD_USER_ROLES = "userRoles"; + public static final String SCRIPT_FIELD_REQUEST = "request"; +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerMetricsUtil.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerMetricsUtil.java new file mode 100644 index 00000000000..ee7516ed211 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerMetricsUtil.java @@ -0,0 +1,189 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.util; + +import com.google.gson.Gson; +import org.apache.commons.lang.StringUtils; +import org.apache.atlas.plugin.model.RangerMetrics; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.FileWriter; +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryMXBean; +import java.lang.management.MemoryPoolMXBean; +import java.lang.management.MemoryType; +import java.lang.management.MemoryUsage; +import java.lang.management.OperatingSystemMXBean; +import java.lang.management.RuntimeMXBean; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Connect Worker system and runtime information. + */ +public class RangerMetricsUtil { + + private static final Logger LOG = LoggerFactory.getLogger(RangerMetricsUtil.class); + private static final OperatingSystemMXBean OS; + private static final MemoryMXBean MEM_BEAN; + public static final String NL = System.getProperty("line.separator"); + + private static final RuntimeMXBean RUNTIME = ManagementFactory.getRuntimeMXBean(); + private static final String JVM_MACHINE_ACTUAL_NAME = RUNTIME.getVmName(); + private static final String VERSION = RUNTIME.getVmVersion(); + private static final String JVM_MACHINE_REPRESENTATION_NAME = RUNTIME.getName(); + private static final long UP_TIME_OF_JVM = RUNTIME.getUptime(); + private static final String JVM_VENDOR_NAME = RUNTIME.getVmVendor(); + + + static { + OS = ManagementFactory.getOperatingSystemMXBean(); + MEM_BEAN = ManagementFactory.getMemoryMXBean(); + } + + public Map getValues() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerMetricsUtil.getValues()"); + } + + Map values = new LinkedHashMap<>(); + values.put("os.spec", StringUtils.join(Arrays.asList(addSystemInfo()), ", ")); + values.put("os.vcpus", String.valueOf(OS.getAvailableProcessors())); + values.put("memory", addMemoryDetails()); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerMetricsUtil.getValues()" + values); + } + + return values; + } + + /** + * collect the pool division of java + */ + protected Map getPoolDivision() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerMetricsUtil.getPoolDivision()"); + } + + Map poolDivisionValues = new LinkedHashMap<>(); + for (MemoryPoolMXBean mpBean : ManagementFactory.getMemoryPoolMXBeans()) { + if (mpBean.getType() == MemoryType.HEAP) { + poolDivisionValues.put(mpBean.getName(), mpBean.getUsage()); + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerMetricsUtil.getPoolDivision()" + poolDivisionValues); + } + + return poolDivisionValues; + } + + /** + * Add memory details + */ + protected Map addMemoryDetails() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerMetricsUtil.addMemoryDetails()"); + } + + Map memory = new LinkedHashMap<>(); + MemoryUsage memHeapUsage = MEM_BEAN.getHeapMemoryUsage(); + MemoryUsage nonHeapUsage = MEM_BEAN.getNonHeapMemoryUsage(); + memory.put("heapInit", String.valueOf(memHeapUsage.getInit())); + memory.put("heapMax", String.valueOf(memHeapUsage.getMax())); + memory.put("heapCommitted", String.valueOf(memHeapUsage.getCommitted())); + memory.put("heapUsed", String.valueOf(memHeapUsage.getUsed())); + memory.put("nonHeapInit", String.valueOf(nonHeapUsage.getInit())); + memory.put("nonHeapMax", String.valueOf(nonHeapUsage.getMax())); + memory.put("nonHeapCommitted", String.valueOf(nonHeapUsage.getCommitted())); + memory.put("nonHeapUsed", String.valueOf(nonHeapUsage.getUsed())); + memory.put("memory_pool_usages", getPoolDivision()); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerMetricsUtil.addMemoryDetails()" + memory); + } + + return memory; + } + + /** + * Collect system information. + */ + protected String[] addSystemInfo() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerMetricsUtil.addSystemInfo()"); + } + + String[] osInfo = { OS.getName(), OS.getArch(), OS.getVersion() }; + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerMetricsUtil.addSystemInfo()" + osInfo); + } + + return osInfo; + } + + public RangerMetrics getVMStatus() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerMetricsUtil.getVMStatus()"); + } + + Map jvm = new LinkedHashMap<>(); + Map vmDetails = new LinkedHashMap<>(); + vmDetails.put("JVM Machine Actual Name", JVM_MACHINE_ACTUAL_NAME); + vmDetails.put("version", VERSION); + vmDetails.put("JVM Machine Representation Name", JVM_MACHINE_REPRESENTATION_NAME); + vmDetails.put("Up time of JVM", UP_TIME_OF_JVM); + vmDetails.put("JVM Vendor Name", JVM_VENDOR_NAME); + vmDetails.putAll(getValues()); + jvm.put("jvm", vmDetails); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerMetricsUtil.getVMStatus() " + jvm); + } + + return new RangerMetrics(jvm); + } + + public void writeMetricsToFile(File filePath) throws Throwable { + + RangerMetrics rangerMetrics = null; + rangerMetrics = getVMStatus(); + if (null == rangerMetrics || null == filePath) { + LOG.debug("RangerMetrics or filePath can not be null)"); + return; + } + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerMetricsUtil.writeMetricsToFIle() for path: "+ filePath); + } + Gson gson = new Gson(); + try (FileWriter file = new FileWriter(filePath)) { + gson.toJson(rangerMetrics, file); + file.flush(); + } catch (Exception e ) { + LOG.error("RangerMetricsUtil.writeMetricsToFile() got an error",e); + throw e; + } + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerObjectFactory.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerObjectFactory.java new file mode 100644 index 00000000000..72c2f68287f --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerObjectFactory.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.util; + +import org.apache.atlas.plugin.model.RangerPolicy; +import org.apache.atlas.plugin.model.RangerPolicyResourceSignature; +import org.apache.atlas.plugin.model.RangerServiceDef; +import org.apache.atlas.plugin.model.validation.RangerServiceDefHelper; + +public class RangerObjectFactory { + public RangerPolicyResourceSignature createPolicyResourceSignature(RangerPolicy policy) { + return new RangerPolicyResourceSignature(policy); + } + + public RangerServiceDefHelper createServiceDefHelper(RangerServiceDef serviceDef) { + return new RangerServiceDefHelper(serviceDef); + } + + public RangerServiceDefHelper createServiceDefHelper(RangerServiceDef serviceDef, boolean useCache) { + return new RangerServiceDefHelper(serviceDef, useCache, true); + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPerfCollectorTracer.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPerfCollectorTracer.java new file mode 100644 index 00000000000..98990a64f87 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPerfCollectorTracer.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.util; + +import org.apache.commons.logging.Log; + +public class RangerPerfCollectorTracer extends RangerPerfTracer { + private final long startTimeNanos; + + public RangerPerfCollectorTracer(Log logger, String tag, String data) { + super(logger, tag, data); + startTimeNanos = System.nanoTime(); + } + + @Override + public void log() { + // Collect elapsed time in microseconds + PerfDataRecorder.recordStatistic(tag, ((System.nanoTime() - startTimeNanos) + 500) / 1000); + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPerfTracer.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPerfTracer.java new file mode 100644 index 00000000000..e34d7ae690b --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPerfTracer.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.util; + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +public class RangerPerfTracer { + protected final Log logger; + protected final String tag; + protected final String data; + private final long startTimeMs; + + private static long reportingThresholdMs; + + private final static String tagEndMarker = "("; + + public static Log getPerfLogger(String name) { + return LogFactory.getLog("org.apache.atlas.auth.perf." + name); + } + + public static Log getPerfLogger(Class cls) { + return RangerPerfTracer.getPerfLogger(cls.getName()); + } + + public static boolean isPerfTraceEnabled(Log logger) { + return logger.isDebugEnabled(); + } + + public static RangerPerfTracer getPerfTracer(Log logger, String tag) { + String data = ""; + String realTag = ""; + + if (tag != null) { + int indexOfTagEndMarker = StringUtils.indexOf(tag, tagEndMarker); + if (indexOfTagEndMarker != -1) { + realTag = StringUtils.substring(tag, 0, indexOfTagEndMarker); + data = StringUtils.substring(tag, indexOfTagEndMarker); + } else { + realTag = tag; + } + } + return RangerPerfTracerFactory.getPerfTracer(logger, realTag, data); + } + + public static RangerPerfTracer getPerfTracer(Log logger, String tag, String data) { + return RangerPerfTracerFactory.getPerfTracer(logger, tag, data); + } + + public static void log(RangerPerfTracer tracer) { + if(tracer != null) { + tracer.log(); + } + } + + public static void logAlways(RangerPerfTracer tracer) { + if(tracer != null) { + tracer.logAlways(); + } + } + public RangerPerfTracer(Log logger, String tag, String data) { + this.logger = logger; + this.tag = tag; + this.data = data; + startTimeMs = System.currentTimeMillis(); + } + + public final String getTag() { + return tag; + } + + public final long getStartTime() { + return startTimeMs; + } + + public final long getElapsedTime() { + return System.currentTimeMillis() - startTimeMs; + } + + public void log() { + long elapsedTime = getElapsedTime(); + if (elapsedTime > reportingThresholdMs) { + logger.debug("[PERF] " + tag + data + ": " + elapsedTime); + } + } + public void logAlways() { + long elapsedTime = getElapsedTime(); + logger.debug("[PERF] " + tag + data + ": " + elapsedTime); + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPerfTracerFactory.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPerfTracerFactory.java new file mode 100644 index 00000000000..24a7ca9d931 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPerfTracerFactory.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.util; + +import org.apache.commons.logging.Log; + +public class RangerPerfTracerFactory { + + static RangerPerfTracer getPerfTracer(Log logger, String tag, String data) { + + RangerPerfTracer ret = null; + + if (PerfDataRecorder.collectStatistics()) { + ret = new RangerPerfCollectorTracer(logger, tag, data); + } else if (logger.isDebugEnabled()) { + ret = new RangerPerfTracer(logger, tag, data); + } + return ret; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPluginCapability.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPluginCapability.java new file mode 100644 index 00000000000..a80b495f1bc --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPluginCapability.java @@ -0,0 +1,184 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.util; + +import org.apache.atlas.authorization.utils.JsonUtils; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +public class RangerPluginCapability { + + /* + - tag-policies + - allowExceptions/deny/denyExceptions + - masking/row-filtering + - Macros - like ${USER} + - tag-based masking/row-filtering + - audit mode support + - service-def changes - isValidLeaf + - validity periods + - policy priority + - security zones + - policy-level conditions + - deny AllElse policies + - roles + - role download timer + - Audit-excluded-users + - Chained plugins + - Super-user permission + - UserStore download + - Audit-policies + */ + private final long pluginCapabilities; + private static final String baseRangerCapabilities = computeBaseCapabilities(); + + // Any new RANGER_PLUGIN_CAPABILITY needs to be added to the end of this enum. Existing enumerator order should *NOT* be changed // + public enum RangerPluginFeature { + RANGER_PLUGIN_CAPABILITY_TAG_POLICIES("Tag Policies"), + RANGER_PLUGIN_CAPABILITY_MASKING_AND_ROW_FILTERING("Masking and Row-filtering"), + RANGER_PLUGIN_CAPABILITY_MACROS("Macros"), + RANGER_PLUGIN_CAPABILITY_AUDIT_MODE("Audit Mode"), + RANGER_PLUGIN_CAPABILITY_RESOURCE_IS_VALID_LEAF("Support for leaf node"), + RANGER_PLUGIN_CAPABILITY_VALIDITY_PERIOD("Validity Period"), + RANGER_PLUGIN_CAPABILITY_POLICY_PRIORITY("Policy Priority"), + RANGER_PLUGIN_CAPABILITY_SECURITY_ZONE("Security Zone"), + RANGER_PLUGIN_CAPABILITY_POLICY_LEVEL_CONDITION("Policy-level Condition"), + RANGER_PLUGIN_CAPABILITY_DENY_ALL_ELSE_POLICY("Deny-all-else Policy"), + RANGER_PLUGIN_CAPABILITY_ROLE("Role"), + RANGER_PLUGIN_CAPABILITY_ROLE_DOWNLOAD_TIMER("Role Timer"), + RANGER_PLUGIN_CAPABILITY_AUDIT_EXCLUDED_USERS("Audit-Excluded Users"), + RANGER_PLUGIN_CAPABILITY_CHAINED_PLUGINS("Chained Plugins"), + RANGER_PLUGIN_CAPABILITY_SUPERUSER_PERMISSIONS("Super-user Permissions"), + RANGER_PLUGIN_CAPABILITY_USERSTORE_DOWNLOAD("UserStore Download"), + RANGER_PLUGIN_CAPABILITY_AUDIT_POLICY("Audit Policy"); + + private final String name; + RangerPluginFeature(String name) { + this.name = name; + } + String getName() { return name; } + } + + + public RangerPluginCapability() { + + long vector = 0L; + for (RangerPluginFeature feature : RangerPluginFeature.values()) { + vector += 1L << feature.ordinal(); + } + pluginCapabilities = vector; + } + + public RangerPluginCapability(long pluginCapabilities) { + this.pluginCapabilities = pluginCapabilities; + } + + public RangerPluginCapability(List capabilities) { + + long vector = 0L; + + for (String capability : capabilities) { + RangerPluginFeature feature; + try { + feature = RangerPluginFeature.valueOf(capability); + vector += 1L << feature.ordinal(); + } catch (Exception e) { + // Ignore + } + } + this.pluginCapabilities = vector; + } + + public long getPluginCapabilities() { + return pluginCapabilities; + } + + public List compare(RangerPluginCapability other) { + final List ret; + + if (pluginCapabilities != other.pluginCapabilities) { + + long mismatchedCapabilitiesVector = this.pluginCapabilities ^ other.pluginCapabilities; + + List missingFeatures = toStrings(mismatchedCapabilitiesVector); + + if (mismatchedCapabilitiesVector > (1L << (RangerPluginFeature.values().length))) { + missingFeatures.add("unknown"); + } + + ret = missingFeatures; + } else { + ret = Collections.EMPTY_LIST; + } + + return ret; + } + + @Override + public String toString() { + List capabilities = toStrings(pluginCapabilities); + return JsonUtils.objectToJson(capabilities); + } + + public static String getBaseRangerCapabilities() { + return baseRangerCapabilities; + } + + private static List toStrings(long vector) { + + List ret = new ArrayList<>(); + + for (RangerPluginFeature feature : RangerPluginFeature.values()) { + long test = 1L << feature.ordinal(); + if ((test & vector) > 0) { + ret.add(feature.name()); + } + } + + return ret; + } + + + private static String computeBaseCapabilities() { + List baseCapabilities = Arrays.asList(RangerPluginFeature.RANGER_PLUGIN_CAPABILITY_TAG_POLICIES.getName() + , RangerPluginFeature.RANGER_PLUGIN_CAPABILITY_MASKING_AND_ROW_FILTERING.getName() + , RangerPluginFeature.RANGER_PLUGIN_CAPABILITY_MACROS.getName() + , RangerPluginFeature.RANGER_PLUGIN_CAPABILITY_AUDIT_MODE.getName() + , RangerPluginFeature.RANGER_PLUGIN_CAPABILITY_RESOURCE_IS_VALID_LEAF.getName() + , RangerPluginFeature.RANGER_PLUGIN_CAPABILITY_VALIDITY_PERIOD.getName() + , RangerPluginFeature.RANGER_PLUGIN_CAPABILITY_POLICY_PRIORITY.getName() + , RangerPluginFeature.RANGER_PLUGIN_CAPABILITY_SECURITY_ZONE.getName() + , RangerPluginFeature.RANGER_PLUGIN_CAPABILITY_POLICY_LEVEL_CONDITION.getName() + , RangerPluginFeature.RANGER_PLUGIN_CAPABILITY_DENY_ALL_ELSE_POLICY.getName() + , RangerPluginFeature.RANGER_PLUGIN_CAPABILITY_ROLE.getName() + , RangerPluginFeature.RANGER_PLUGIN_CAPABILITY_ROLE_DOWNLOAD_TIMER.getName() + , RangerPluginFeature.RANGER_PLUGIN_CAPABILITY_AUDIT_EXCLUDED_USERS.getName() + , RangerPluginFeature.RANGER_PLUGIN_CAPABILITY_CHAINED_PLUGINS.getName() + , RangerPluginFeature.RANGER_PLUGIN_CAPABILITY_SUPERUSER_PERMISSIONS.getName() + , RangerPluginFeature.RANGER_PLUGIN_CAPABILITY_USERSTORE_DOWNLOAD.getName() + , RangerPluginFeature.RANGER_PLUGIN_CAPABILITY_AUDIT_POLICY.getName()); + + return Long.toHexString(new RangerPluginCapability(baseCapabilities).getPluginCapabilities()); + } + +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java new file mode 100644 index 00000000000..e3cb0f36979 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java @@ -0,0 +1,244 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.util; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.plugin.model.RangerPolicy; +import org.apache.atlas.plugin.model.RangerPolicyDelta; +import org.apache.atlas.plugin.store.EmbeddedServiceDefsUtil; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; + +public class RangerPolicyDeltaUtil { + + private static final Log LOG = LogFactory.getLog(RangerPolicyDeltaUtil.class); + + private static final Log PERF_POLICY_DELTA_LOG = RangerPerfTracer.getPerfLogger("policy.delta"); + + public static List applyDeltas(List policies, List deltas, String serviceType) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> applyDeltas(serviceType=" + serviceType + ")"); + } + + List ret; + + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_POLICY_DELTA_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_POLICY_DELTA_LOG, "RangerPolicyDelta.applyDeltas()"); + } + + boolean hasExpectedServiceType = false; + + if (CollectionUtils.isNotEmpty(deltas)) { + if (LOG.isDebugEnabled()) { + LOG.debug("applyDeltas(deltas=" + Arrays.toString(deltas.toArray()) + ", serviceType=" + serviceType + ")"); + } + + for (RangerPolicyDelta delta : deltas) { + if (serviceType.equals(delta.getServiceType())) { + hasExpectedServiceType = true; + break; + } else if (!serviceType.equals(EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_TAG_NAME) && !delta.getServiceType().equals(EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_TAG_NAME)) { + LOG.warn("Found unexpected serviceType in policyDelta:[" + delta + "]. Was expecting serviceType:[" + serviceType + "]. Should NOT have come here!! Ignoring delta and continuing"); + } + } + + if (hasExpectedServiceType) { + ret = new ArrayList<>(policies); + + for (RangerPolicyDelta delta : deltas) { + if (!serviceType.equals(delta.getServiceType())) { + continue; + } + + int changeType = delta.getChangeType(); + + if (changeType == RangerPolicyDelta.CHANGE_TYPE_POLICY_CREATE || changeType == RangerPolicyDelta.CHANGE_TYPE_POLICY_UPDATE || changeType == RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE) { + Long policyId = delta.getPolicyId(); + + if (policyId == null) { + continue; + } + + List deletedPolicies = new ArrayList<>(); + + Iterator iter = ret.iterator(); + + while (iter.hasNext()) { + RangerPolicy policy = iter.next(); + if (policyId.equals(policy.getId()) && (changeType == RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE || changeType == RangerPolicyDelta.CHANGE_TYPE_POLICY_UPDATE)) { + deletedPolicies.add(policy); + iter.remove(); + } + } + + switch(changeType) { + case RangerPolicyDelta.CHANGE_TYPE_POLICY_CREATE: { + if (CollectionUtils.isNotEmpty(deletedPolicies)) { + LOG.warn("Unexpected: found existing policy for CHANGE_TYPE_POLICY_CREATE: " + Arrays.toString(deletedPolicies.toArray())); + } + break; + } + case RangerPolicyDelta.CHANGE_TYPE_POLICY_UPDATE: { + if (CollectionUtils.isEmpty(deletedPolicies) || deletedPolicies.size() > 1) { + LOG.warn("Unexpected: found no policy or multiple policies for CHANGE_TYPE_POLICY_UPDATE: " + Arrays.toString(deletedPolicies.toArray())); + } + break; + } + case RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE: { + if (CollectionUtils.isEmpty(deletedPolicies) || deletedPolicies.size() > 1) { + LOG.warn("Unexpected: found no policy or multiple policies for CHANGE_TYPE_POLICY_DELETE: " + Arrays.toString(deletedPolicies.toArray())); + } + break; + } + default: + break; + } + + if (changeType != RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE) { + ret.add(delta.getPolicy()); + } + } else { + LOG.warn("Found unexpected changeType in policyDelta:[" + delta + "]. Ignoring delta"); + } + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("applyDeltas - none of the deltas is for " + serviceType + ")"); + } + ret = policies; + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("applyDeltas called with empty deltas. Will return policies without change"); + } + ret = policies; + } + + if (CollectionUtils.isNotEmpty(deltas) && hasExpectedServiceType && CollectionUtils.isNotEmpty(ret)) { + ret.sort(RangerPolicy.POLICY_ID_COMPARATOR); + } + + RangerPerfTracer.log(perf); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== applyDeltas(serviceType=" + serviceType + "): " + ret); + } + return ret; + } + + public static boolean isValidDeltas(List deltas, String componentServiceType) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> isValidDeltas(deltas=" + Arrays.toString(deltas.toArray()) + ", componentServiceType=" + componentServiceType +")"); + } + boolean isValid = true; + + for (RangerPolicyDelta delta : deltas) { + final Integer changeType = delta.getChangeType(); + final Long policyId = delta.getPolicyId(); + + if (changeType == null) { + isValid = false; + break; + } + + if (changeType != RangerPolicyDelta.CHANGE_TYPE_POLICY_CREATE + && changeType != RangerPolicyDelta.CHANGE_TYPE_POLICY_UPDATE + && changeType != RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE) { + isValid = false; + } else if (policyId == null) { + isValid = false; + } else { + final String serviceType = delta.getServiceType(); + final String policyType = delta.getPolicyType(); + + if (serviceType == null || (!serviceType.equals(EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_TAG_NAME) && + !serviceType.equals(componentServiceType))) { + isValid = false; + } else if (StringUtils.isEmpty(policyType) || (!RangerPolicy.POLICY_TYPE_ACCESS.equals(policyType) + && !RangerPolicy.POLICY_TYPE_DATAMASK.equals(policyType) + && !RangerPolicy.POLICY_TYPE_ROWFILTER.equals(policyType))) { + isValid = false; + } + } + + if (!isValid) { + break; + } + } + if (LOG.isDebugEnabled()) { + LOG.debug("<== isValidDeltas(deltas=" + Arrays.toString(deltas.toArray()) + ", componentServiceType=" + componentServiceType +"): " + isValid); + } + return isValid; + } + + public static Boolean hasPolicyDeltas(final ServicePolicies servicePolicies) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> hasPolicyDeltas(servicePolicies:[" + servicePolicies + "]"); + } + final Boolean ret; + + if (servicePolicies == null) { + LOG.error("ServicePolicies are null!"); + ret = null; + } else { + boolean isPoliciesExistInSecurityZones = false; + boolean isPolicyDeltasExistInSecurityZones = false; + + if (MapUtils.isNotEmpty(servicePolicies.getSecurityZones())) { + for (ServicePolicies.SecurityZoneInfo element : servicePolicies.getSecurityZones().values()) { + if (CollectionUtils.isNotEmpty(element.getPolicies()) && CollectionUtils.isEmpty(element.getPolicyDeltas())) { + isPoliciesExistInSecurityZones = true; + } + if (CollectionUtils.isEmpty(element.getPolicies()) && CollectionUtils.isNotEmpty(element.getPolicyDeltas())) { + isPolicyDeltasExistInSecurityZones = true; + } + } + } + + boolean isPoliciesExist = CollectionUtils.isNotEmpty(servicePolicies.getPolicies()) || (servicePolicies.getTagPolicies() != null && CollectionUtils.isNotEmpty(servicePolicies.getTagPolicies().getPolicies())) || isPoliciesExistInSecurityZones; + boolean isPolicyDeltasExist = CollectionUtils.isNotEmpty(servicePolicies.getPolicyDeltas()) || isPolicyDeltasExistInSecurityZones; + + if (isPoliciesExist && isPolicyDeltasExist) { + LOG.warn("ServicePolicies contain both policies and policy-deltas!! Cannot build policy-engine from these servicePolicies. Please check server-side code!"); + LOG.warn("Downloaded ServicePolicies are [" + servicePolicies + "]"); + ret = null; + } else if (!isPoliciesExist && !isPolicyDeltasExist) { + LOG.warn("ServicePolicies do not contain any policies or policy-deltas!! There are no material changes in the policies."); + LOG.warn("Downloaded ServicePolicies are [" + servicePolicies + "]"); + ret = null; + } else { + ret = isPolicyDeltasExist; + } + } + if (LOG.isDebugEnabled()) { + LOG.debug("<== hasPolicyDeltas(servicePolicies:[" + servicePolicies + "], ret:[" + ret + "]"); + } + return ret; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerRESTClient.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerRESTClient.java new file mode 100644 index 00000000000..4a265a84307 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerRESTClient.java @@ -0,0 +1,652 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.util; + +import com.fasterxml.jackson.jaxrs.json.JacksonJsonProvider; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.sun.jersey.api.client.Client; +import com.sun.jersey.api.client.ClientHandlerException; +import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.WebResource; +import com.sun.jersey.api.client.config.ClientConfig; +import com.sun.jersey.api.client.config.DefaultClientConfig; +import com.sun.jersey.api.client.filter.HTTPBasicAuthFilter; +import com.sun.jersey.client.urlconnection.HTTPSProperties; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang.Validate; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.atlas.authorization.hadoop.utils.RangerCredentialProvider; +import org.apache.atlas.authorization.utils.StringUtil; + +import javax.net.ssl.HostnameVerifier; +import javax.net.ssl.KeyManager; +import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLSession; +import javax.net.ssl.TrustManager; +import javax.net.ssl.TrustManagerFactory; +import javax.ws.rs.core.Cookie; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.security.KeyManagementException; +import java.security.KeyStore; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.security.SecureRandom; +import java.security.UnrecoverableKeyException; +import java.security.cert.CertificateException; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.Set; + + +public class RangerRESTClient { + private static final Log LOG = LogFactory.getLog(RangerRESTClient.class); + + public static final String RANGER_PROP_POLICYMGR_URL = "ranger.service.store.rest.url"; + public static final String RANGER_PROP_POLICYMGR_SSLCONFIG_FILENAME = "ranger.service.store.rest.ssl.config.file"; + + public static final String RANGER_POLICYMGR_CLIENT_KEY_FILE = "xasecure.policymgr.clientssl.keystore"; + public static final String RANGER_POLICYMGR_CLIENT_KEY_FILE_TYPE = "xasecure.policymgr.clientssl.keystore.type"; + public static final String RANGER_POLICYMGR_CLIENT_KEY_FILE_CREDENTIAL = "xasecure.policymgr.clientssl.keystore.credential.file"; + public static final String RANGER_POLICYMGR_CLIENT_KEY_FILE_CREDENTIAL_ALIAS = "sslKeyStore"; + public static final String RANGER_POLICYMGR_CLIENT_KEY_FILE_TYPE_DEFAULT = "jks"; + + public static final String RANGER_POLICYMGR_TRUSTSTORE_FILE = "xasecure.policymgr.clientssl.truststore"; + public static final String RANGER_POLICYMGR_TRUSTSTORE_FILE_TYPE = "xasecure.policymgr.clientssl.truststore.type"; + public static final String RANGER_POLICYMGR_TRUSTSTORE_FILE_CREDENTIAL = "xasecure.policymgr.clientssl.truststore.credential.file"; + public static final String RANGER_POLICYMGR_TRUSTSTORE_FILE_CREDENTIAL_ALIAS = "sslTrustStore"; + public static final String RANGER_POLICYMGR_TRUSTSTORE_FILE_TYPE_DEFAULT = "jks"; + + public static final String RANGER_SSL_KEYMANAGER_ALGO_TYPE = KeyManagerFactory.getDefaultAlgorithm(); + public static final String RANGER_SSL_TRUSTMANAGER_ALGO_TYPE = TrustManagerFactory.getDefaultAlgorithm(); + public static final String RANGER_SSL_CONTEXT_ALGO_TYPE = "TLS"; + + private String mUrl; + private String mSslConfigFileName; + private String mUsername; + private String mPassword; + private boolean mIsSSL; + + private String mKeyStoreURL; + private String mKeyStoreAlias; + private String mKeyStoreFile; + private String mKeyStoreType; + private String mTrustStoreURL; + private String mTrustStoreAlias; + private String mTrustStoreFile; + private String mTrustStoreType; + private Gson gsonBuilder; + private int mRestClientConnTimeOutMs; + private int mRestClientReadTimeOutMs; + private int lastKnownActiveUrlIndex; + + private final List configuredURLs; + + private volatile Client client; + + + public RangerRESTClient(String url, String sslConfigFileName, Configuration config) { + mUrl = url; + mSslConfigFileName = sslConfigFileName; + configuredURLs = StringUtil.getURLs(mUrl); + + setLastKnownActiveUrlIndex((new Random()).nextInt(getConfiguredURLs().size())); + + init(config); + } + + public String getUrl() { + return mUrl; + } + + public void setUrl(String url) { + this.mUrl = url; + } + + public String getUsername() { + return mUsername; + } + + public String getPassword() { + return mPassword; + } + + public int getRestClientConnTimeOutMs() { + return mRestClientConnTimeOutMs; + } + + public void setRestClientConnTimeOutMs(int mRestClientConnTimeOutMs) { + this.mRestClientConnTimeOutMs = mRestClientConnTimeOutMs; + } + + public int getRestClientReadTimeOutMs() { + return mRestClientReadTimeOutMs; + } + + public void setRestClientReadTimeOutMs(int mRestClientReadTimeOutMs) { + this.mRestClientReadTimeOutMs = mRestClientReadTimeOutMs; + } + + public void setBasicAuthInfo(String username, String password) { + mUsername = username; + mPassword = password; + } + + public WebResource getResource(String relativeUrl) { + WebResource ret = getClient().resource(getUrl() + relativeUrl); + + return ret; + } + + public String toJson(Object obj) { + return gsonBuilder.toJson(obj); + } + + public T fromJson(String json, Class cls) { + return gsonBuilder.fromJson(json, cls); + } + + public Client getClient() { + // result saves on access time when client is built at the time of the call + Client result = client; + if(result == null) { + synchronized(this) { + result = client; + if(result == null) { + client = result = buildClient(); + } + } + } + + return result; + } + + private Client buildClient() { + Client client = null; + + if (mIsSSL) { + KeyManager[] kmList = getKeyManagers(); + TrustManager[] tmList = getTrustManagers(); + SSLContext sslContext = getSSLContext(kmList, tmList); + ClientConfig config = new DefaultClientConfig(); + + config.getClasses().add(JacksonJsonProvider.class); // to handle List<> unmarshalling + + HostnameVerifier hv = new HostnameVerifier() { + public boolean verify(String urlHostName, SSLSession session) { + return session.getPeerHost().equals(urlHostName); + } + }; + + config.getProperties().put(HTTPSProperties.PROPERTY_HTTPS_PROPERTIES, new HTTPSProperties(hv, sslContext)); + + client = Client.create(config); + } + + if(client == null) { + ClientConfig config = new DefaultClientConfig(); + + config.getClasses().add(JacksonJsonProvider.class); // to handle List<> unmarshalling + + client = Client.create(config); + } + + if(StringUtils.isNotEmpty(mUsername) && StringUtils.isNotEmpty(mPassword)) { + client.addFilter(new HTTPBasicAuthFilter(mUsername, mPassword)); + } + + // Set Connection Timeout and ReadTime for the PolicyRefresh + client.setConnectTimeout(mRestClientConnTimeOutMs); + client.setReadTimeout(mRestClientReadTimeOutMs); + + return client; + } + + public void resetClient(){ + client = null; + } + + private void init(Configuration config) { + try { + gsonBuilder = new GsonBuilder().setDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZ").create(); + } catch(Throwable excp) { + LOG.fatal("RangerRESTClient.init(): failed to create GsonBuilder object", excp); + } + + mIsSSL = StringUtils.containsIgnoreCase(mUrl, "https"); + + if (mIsSSL) { + + InputStream in = null; + + try { + in = getFileInputStream(mSslConfigFileName); + + if (in != null) { + config.addResource(in); + } + + mKeyStoreURL = config.get(RANGER_POLICYMGR_CLIENT_KEY_FILE_CREDENTIAL); + mKeyStoreAlias = RANGER_POLICYMGR_CLIENT_KEY_FILE_CREDENTIAL_ALIAS; + mKeyStoreType = config.get(RANGER_POLICYMGR_CLIENT_KEY_FILE_TYPE, RANGER_POLICYMGR_CLIENT_KEY_FILE_TYPE_DEFAULT); + mKeyStoreFile = config.get(RANGER_POLICYMGR_CLIENT_KEY_FILE); + + mTrustStoreURL = config.get(RANGER_POLICYMGR_TRUSTSTORE_FILE_CREDENTIAL); + mTrustStoreAlias = RANGER_POLICYMGR_TRUSTSTORE_FILE_CREDENTIAL_ALIAS; + mTrustStoreType = config.get(RANGER_POLICYMGR_TRUSTSTORE_FILE_TYPE, RANGER_POLICYMGR_TRUSTSTORE_FILE_TYPE_DEFAULT); + mTrustStoreFile = config.get(RANGER_POLICYMGR_TRUSTSTORE_FILE); + } catch (IOException ioe) { + LOG.error("Unable to load SSL Config FileName: [" + mSslConfigFileName + "]", ioe); + } finally { + close(in, mSslConfigFileName); + } + + } + } + + private KeyManager[] getKeyManagers() { + KeyManager[] kmList = null; + + String keyStoreFilepwd = getCredential(mKeyStoreURL, mKeyStoreAlias); + + kmList = getKeyManagers(mKeyStoreFile,keyStoreFilepwd); + return kmList; + } + + public KeyManager[] getKeyManagers(String keyStoreFile, String keyStoreFilePwd) { + KeyManager[] kmList = null; + + if (StringUtils.isNotEmpty(keyStoreFile) && StringUtils.isNotEmpty(keyStoreFilePwd)) { + InputStream in = null; + + try { + in = getFileInputStream(keyStoreFile); + + if (in != null) { + KeyStore keyStore = KeyStore.getInstance(mKeyStoreType); + + keyStore.load(in, keyStoreFilePwd.toCharArray()); + + KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance(RANGER_SSL_KEYMANAGER_ALGO_TYPE); + + keyManagerFactory.init(keyStore, keyStoreFilePwd.toCharArray()); + + kmList = keyManagerFactory.getKeyManagers(); + } else { + LOG.error("Unable to obtain keystore from file [" + keyStoreFile + "]"); + throw new IllegalStateException("Unable to find keystore file :" + keyStoreFile); + } + } catch (KeyStoreException e) { + LOG.error("Unable to obtain from KeyStore :" + e.getMessage(), e); + throw new IllegalStateException("Unable to init keystore:" + e.getMessage(), e); + } catch (NoSuchAlgorithmException e) { + LOG.error("SSL algorithm is NOT available in the environment", e); + throw new IllegalStateException("SSL algorithm is NOT available in the environment :" + e.getMessage(), e); + } catch (CertificateException e) { + LOG.error("Unable to obtain the requested certification ", e); + throw new IllegalStateException("Unable to obtain the requested certification :" + e.getMessage(), e); + } catch (FileNotFoundException e) { + LOG.error("Unable to find the necessary SSL Keystore Files", e); + throw new IllegalStateException("Unable to find keystore file :" + keyStoreFile + ", error :" + e.getMessage(), e); + } catch (IOException e) { + LOG.error("Unable to read the necessary SSL Keystore Files", e); + throw new IllegalStateException("Unable to read keystore file :" + keyStoreFile + ", error :" + e.getMessage(), e); + } catch (UnrecoverableKeyException e) { + LOG.error("Unable to recover the key from keystore", e); + throw new IllegalStateException("Unable to recover the key from keystore :" + keyStoreFile+", error :" + e.getMessage(), e); + } finally { + close(in, keyStoreFile); + } + } + + return kmList; + } + + private TrustManager[] getTrustManagers() { + TrustManager[] tmList = null; + if (StringUtils.isNotEmpty(mTrustStoreURL) && StringUtils.isNotEmpty(mTrustStoreAlias)) { + String trustStoreFilepwd = getCredential(mTrustStoreURL, mTrustStoreAlias); + if (StringUtils.isNotEmpty(trustStoreFilepwd)) { + tmList = getTrustManagers(mTrustStoreFile, trustStoreFilepwd); + } + } + return tmList; + } + + public TrustManager[] getTrustManagers(String trustStoreFile, String trustStoreFilepwd) { + TrustManager[] tmList = null; + + if (StringUtils.isNotEmpty(trustStoreFile) && StringUtils.isNotEmpty(trustStoreFilepwd)) { + InputStream in = null; + + try { + in = getFileInputStream(trustStoreFile); + + if (in != null) { + KeyStore trustStore = KeyStore.getInstance(mTrustStoreType); + + trustStore.load(in, trustStoreFilepwd.toCharArray()); + + TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(RANGER_SSL_TRUSTMANAGER_ALGO_TYPE); + + trustManagerFactory.init(trustStore); + + tmList = trustManagerFactory.getTrustManagers(); + } else { + LOG.error("Unable to obtain truststore from file [" + trustStoreFile + "]"); + throw new IllegalStateException("Unable to find truststore file :" + trustStoreFile); + } + } catch (KeyStoreException e) { + LOG.error("Unable to obtain from KeyStore", e); + throw new IllegalStateException("Unable to init keystore:" + e.getMessage(), e); + } catch (NoSuchAlgorithmException e) { + LOG.error("SSL algorithm is NOT available in the environment :" + e.getMessage(), e); + throw new IllegalStateException("SSL algorithm is NOT available in the environment :" + e.getMessage(), e); + } catch (CertificateException e) { + LOG.error("Unable to obtain the requested certification :" + e.getMessage(), e); + throw new IllegalStateException("Unable to obtain the requested certification :" + e.getMessage(), e); + } catch (FileNotFoundException e) { + LOG.error("Unable to find the necessary SSL TrustStore File:" + trustStoreFile, e); + throw new IllegalStateException("Unable to find trust store file :" + trustStoreFile + ", error :" + e.getMessage(), e); + } catch (IOException e) { + LOG.error("Unable to read the necessary SSL TrustStore Files :" + trustStoreFile, e); + throw new IllegalStateException("Unable to read the trust store file :" + trustStoreFile + ", error :" + e.getMessage(), e); + } finally { + close(in, trustStoreFile); + } + } + + return tmList; + } + + protected SSLContext getSSLContext(KeyManager[] kmList, TrustManager[] tmList) { + if (tmList == null) { + try { + String algo = TrustManagerFactory.getDefaultAlgorithm() ; + TrustManagerFactory tmf = TrustManagerFactory.getInstance(algo) ; + tmf.init((KeyStore)null) ; + tmList = tmf.getTrustManagers() ; + } + catch(NoSuchAlgorithmException | KeyStoreException | IllegalStateException e) { + LOG.error("Unable to get the default SSL TrustStore for the JVM",e); + tmList = null; + } + } + Validate.notNull(tmList, "TrustManager is not specified"); + try { + SSLContext sslContext = SSLContext.getInstance(RANGER_SSL_CONTEXT_ALGO_TYPE); + + sslContext.init(kmList, tmList, new SecureRandom()); + + return sslContext; + } catch (NoSuchAlgorithmException e) { + LOG.error("SSL algorithm is not available in the environment", e); + throw new IllegalStateException("SSL algorithm is not available in the environment: " + e.getMessage(), e); + } catch (KeyManagementException e) { + LOG.error("Unable to initials the SSLContext", e); + throw new IllegalStateException("Unable to initials the SSLContex: " + e.getMessage(), e); + } + } + + private String getCredential(String url, String alias) { + return RangerCredentialProvider.getInstance().getCredentialString(url, alias); + } + + private InputStream getFileInputStream(String fileName) throws IOException { + InputStream in = null; + + if(StringUtils.isNotEmpty(fileName)) { + File f = new File(fileName); + + if (f.exists()) { + in = new FileInputStream(f); + } + else { + in = ClassLoader.getSystemResourceAsStream(fileName); + } + } + + return in; + } + + private void close(InputStream str, String filename) { + if (str != null) { + try { + str.close(); + } catch (IOException excp) { + LOG.error("Error while closing file: [" + filename + "]", excp); + } + } + } + + public ClientResponse get(String relativeUrl, Map params) throws Exception { + ClientResponse finalResponse = null; + int startIndex = this.lastKnownActiveUrlIndex; + int currentIndex = 0; + + for (int index = 0; index < configuredURLs.size(); index++) { + try { + currentIndex = (startIndex + index) % configuredURLs.size(); + + WebResource webResource = getClient().resource(configuredURLs.get(currentIndex) + relativeUrl); + webResource = setQueryParams(webResource, params); + + finalResponse = webResource.accept(RangerRESTUtils.REST_EXPECTED_MIME_TYPE).type(RangerRESTUtils.REST_MIME_TYPE_JSON).get(ClientResponse.class); + + if (finalResponse != null) { + setLastKnownActiveUrlIndex(currentIndex); + break; + } + } catch (ClientHandlerException ex) { + LOG.warn("Failed to communicate with Ranger Admin, URL : " + configuredURLs.get(currentIndex)); + processException(index, ex); + } + } + return finalResponse; + } + + public ClientResponse get(String relativeUrl, Map params, Cookie sessionId) throws Exception{ + ClientResponse finalResponse = null; + int startIndex = this.lastKnownActiveUrlIndex; + int currentIndex = 0; + + for (int index = 0; index < configuredURLs.size(); index++) { + try { + currentIndex = (startIndex + index) % configuredURLs.size(); + + WebResource webResource = createWebResourceForCookieAuth(currentIndex, relativeUrl); + webResource = setQueryParams(webResource, params); + WebResource.Builder br = webResource.getRequestBuilder().cookie(sessionId); + finalResponse = br.accept(RangerRESTUtils.REST_EXPECTED_MIME_TYPE).type(RangerRESTUtils.REST_MIME_TYPE_JSON).get(ClientResponse.class); + + if (finalResponse != null) { + setLastKnownActiveUrlIndex(currentIndex); + break; + } + } catch (ClientHandlerException ex) { + LOG.warn("Failed to communicate with Ranger Admin, URL : "+configuredURLs.get(currentIndex)); + processException(index, ex); + } + } + return finalResponse; + } + + public ClientResponse post(String relativeUrl, Map params, Object obj) throws Exception { + ClientResponse finalResponse = null; + int startIndex = this.lastKnownActiveUrlIndex; + int currentIndex = 0; + + for (int index = 0; index < configuredURLs.size(); index++) { + try { + currentIndex = (startIndex + index) % configuredURLs.size(); + + WebResource webResource = getClient().resource(configuredURLs.get(currentIndex) + relativeUrl); + webResource = setQueryParams(webResource, params); + finalResponse = webResource.accept(RangerRESTUtils.REST_EXPECTED_MIME_TYPE).type(RangerRESTUtils.REST_MIME_TYPE_JSON).post(ClientResponse.class, toJson(obj)); + if (finalResponse != null) { + setLastKnownActiveUrlIndex(currentIndex); + break; + } + } catch (ClientHandlerException ex) { + LOG.warn("Failed to communicate with Ranger Admin, URL : " + configuredURLs.get(currentIndex)); + processException(index, ex); + } + } + return finalResponse; + } + + public ClientResponse delete(String relativeUrl, Map params) throws Exception { + ClientResponse finalResponse = null; + int startIndex = this.lastKnownActiveUrlIndex; + int currentIndex = 0; + + for (int index = 0; index < configuredURLs.size(); index++) { + try { + currentIndex = (startIndex + index) % configuredURLs.size(); + + WebResource webResource = getClient().resource(configuredURLs.get(currentIndex) + relativeUrl); + webResource = setQueryParams(webResource, params); + + finalResponse = webResource.accept(RangerRESTUtils.REST_EXPECTED_MIME_TYPE).type(RangerRESTUtils.REST_MIME_TYPE_JSON).delete(ClientResponse.class); + if (finalResponse != null) { + setLastKnownActiveUrlIndex(currentIndex); + break; + } + } catch (ClientHandlerException ex) { + LOG.warn("Failed to communicate with Ranger Admin, URL : " + configuredURLs.get(currentIndex)); + processException(index, ex); + } + } + return finalResponse; + } + + public ClientResponse put(String relativeUrl, Map params, Object obj) throws Exception { + ClientResponse finalResponse = null; + int startIndex = this.lastKnownActiveUrlIndex; + int currentIndex = 0; + for (int index = 0; index < configuredURLs.size(); index++) { + try { + currentIndex = (startIndex + index) % configuredURLs.size(); + + WebResource webResource = getClient().resource(configuredURLs.get(currentIndex) + relativeUrl); + webResource = setQueryParams(webResource, params); + finalResponse = webResource.accept(RangerRESTUtils.REST_EXPECTED_MIME_TYPE).type(RangerRESTUtils.REST_MIME_TYPE_JSON).put(ClientResponse.class, toJson(obj)); + if (finalResponse != null) { + setLastKnownActiveUrlIndex(currentIndex); + break; + } + } catch (ClientHandlerException ex) { + LOG.warn("Failed to communicate with Ranger Admin, URL : " + configuredURLs.get(currentIndex)); + processException(index, ex); + } + } + return finalResponse; + } + + public ClientResponse put(String relativeURL, Object request, Cookie sessionId) throws Exception { + ClientResponse response = null; + int startIndex = this.lastKnownActiveUrlIndex; + int currentIndex = 0; + + for (int index = 0; index < configuredURLs.size(); index++) { + try { + currentIndex = (startIndex + index) % configuredURLs.size(); + + WebResource webResource = createWebResourceForCookieAuth(currentIndex, relativeURL); + WebResource.Builder br = webResource.getRequestBuilder().cookie(sessionId); + response = br.accept(RangerRESTUtils.REST_EXPECTED_MIME_TYPE).type(RangerRESTUtils.REST_MIME_TYPE_JSON) + .put(ClientResponse.class, toJson(request)); + if (response != null) { + setLastKnownActiveUrlIndex(currentIndex); + break; + } + } catch (ClientHandlerException e) { + LOG.warn("Failed to communicate with Ranger Admin, URL : " + configuredURLs.get(currentIndex)); + processException(index, e); + } + } + return response; + } + + protected static WebResource setQueryParams(WebResource webResource, Map params) { + WebResource ret = webResource; + if (webResource != null && params != null) { + Set> entrySet= params.entrySet(); + for (Map.Entry entry : entrySet) { + ret = ret.queryParam(entry.getKey(), entry.getValue()); + } + } + return ret; + } + + protected void setLastKnownActiveUrlIndex(int lastKnownActiveUrlIndex) { + this.lastKnownActiveUrlIndex = lastKnownActiveUrlIndex; + } + + protected WebResource createWebResourceForCookieAuth(int currentIndex, String relativeURL) { + Client cookieClient = getClient(); + cookieClient.removeAllFilters(); + WebResource ret = cookieClient.resource(configuredURLs.get(currentIndex) + relativeURL); + return ret; + } + + protected void processException(int index, ClientHandlerException e) throws Exception { + if (index == configuredURLs.size() - 1) { + LOG.error("Failed to communicate with all Ranger Admin's URL's : [ " + configuredURLs + " ]"); + throw e; + } + } + + public int getLastKnownActiveUrlIndex() { + return lastKnownActiveUrlIndex; + } + + public List getConfiguredURLs() { + return configuredURLs; + } + + public boolean isSSL() { + return mIsSSL; + } + + public void setSSL(boolean mIsSSL) { + this.mIsSSL = mIsSSL; + } + + protected void setClient(Client client) { + this.client = client; + } + + protected void setKeyStoreType(String mKeyStoreType) { + this.mKeyStoreType = mKeyStoreType; + } + + protected void setTrustStoreType(String mTrustStoreType) { + this.mTrustStoreType = mTrustStoreType; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerRESTUtils.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerRESTUtils.java new file mode 100644 index 00000000000..48ccc19051e --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerRESTUtils.java @@ -0,0 +1,164 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.util; + + +//import com.kstruct.gethostname4j.Hostname; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import java.net.InetAddress; +import java.net.UnknownHostException; + +/** + * Since this class does not retain any state. It isn't a singleton for testability. + * + */ +public class RangerRESTUtils { + + private static final Log LOG = LogFactory.getLog(RangerRESTUtils.class); + + public static final String REST_URL_POLICY_GET_FOR_SERVICE_IF_UPDATED = "/service/plugins/policies/download/"; + public static final String REST_URL_SERVICE_GRANT_ACCESS = "/service/plugins/services/grant/"; + public static final String REST_URL_SERVICE_REVOKE_ACCESS = "/service/plugins/services/revoke/"; + + public static final String REST_URL_POLICY_GET_FOR_SECURE_SERVICE_IF_UPDATED = "/service/plugins/secure/policies/download/"; + public static final String REST_URL_SECURE_SERVICE_GRANT_ACCESS = "/service/plugins/secure/services/grant/"; + public static final String REST_URL_SECURE_SERVICE_REVOKE_ACCESS = "/service/plugins/secure/services/revoke/"; + + public static final String REST_URL_SERVICE_CREATE_ROLE = "/service/public/v2/api/roles/"; + public static final String REST_URL_SERVICE_DROP_ROLE = "/service/public/v2/api/roles/name/"; + public static final String REST_URL_SERVICE_GET_ALL_ROLES = "/service/public/v2/api/roles/names/"; + public static final String REST_URL_SERVICE_GET_USER_ROLES = "/service/public/v2/api/roles/user/"; + public static final String REST_URL_SERVICE_GET_ROLE_INFO = "/service/public/v2/api/roles/name/"; + public static final String REST_URL_SERVICE_GRANT_ROLE = "/service/public/v2/api/roles/grant/"; + public static final String REST_URL_SERVICE_REVOKE_ROLE = "/service/public/v2/api/roles/revoke/"; + + public static final String REST_URL_SERVICE_SERCURE_GET_USER_GROUP_ROLES = "/service/roles/secure/download/"; + public static final String REST_URL_SERVICE_GET_USER_GROUP_ROLES = "/service/roles/download/"; + + public static final String REST_URL_GET_SERVICE_TAGS_IF_UPDATED = "/service/tags/download/"; + public static final String REST_URL_GET_SECURE_SERVICE_TAGS_IF_UPDATED = "/service/tags/secure/download/"; + public static final String SERVICE_NAME_PARAM = "serviceName"; + public static final String LAST_KNOWN_TAG_VERSION_PARAM = "lastKnownVersion"; + public static final String PATTERN_PARAM = "pattern"; + + public static final String REST_URL_LOOKUP_TAG_NAMES = "/service/tags/lookup"; + + public static final String REST_EXPECTED_MIME_TYPE = "application/json"; + public static final String REST_MIME_TYPE_JSON = "application/json"; + + public static final String REST_PARAM_LAST_KNOWN_POLICY_VERSION = "lastKnownVersion"; + public static final String REST_PARAM_LAST_ACTIVATION_TIME = "lastActivationTime"; + public static final String REST_PARAM_PLUGIN_ID = "pluginId"; + + public static final String REST_PARAM_LAST_KNOWN_ROLE_VERSION = "lastKnownRoleVersion"; + + public static final String REST_PARAM_LAST_KNOWN_USERSTORE_VERSION = "lastKnownUserStoreVersion"; + public static final String REST_URL_SERVICE_SERCURE_GET_USERSTORE = "/service/xusers/secure/download/"; + public static final String REST_URL_SERVICE_GET_USERSTORE = "/service/xusers/download/"; + + private static final int MAX_PLUGIN_ID_LEN = 255; + + public static final String REST_PARAM_CLUSTER_NAME = "clusterName"; + public static final String REST_PARAM_SUPPORTS_POLICY_DELTAS = "supportsPolicyDeltas"; + public static final String REST_PARAM_SUPPORTS_TAG_DELTAS = "supportsTagDeltas"; + + public static final String REST_PARAM_ZONE_NAME = "zoneName"; + public static final String REST_PARAM_EXEC_USER = "execUser"; + + public static final String REST_PARAM_CAPABILITIES = "pluginCapabilities"; + + public static String hostname; + + static { + try { + //hostname = Hostname.getHostname(); + hostname = InetAddress.getLocalHost().getHostName(); + } + catch(Exception e) { + LOG.error("ERROR: Unable to find hostname for the agent ", e); + hostname = "unknownHost"; + } + } + + public String getPluginId(String serviceName, String appId) { + String hostName = null; + + try { + hostName = InetAddress.getLocalHost().getHostName(); + } catch (UnknownHostException e) { + LOG.error("ERROR: Unable to find hostname for the agent ", e); + hostName = "unknownHost"; + } + + String ret = hostName + "-" + serviceName; + + if(! StringUtils.isEmpty(appId)) { + ret = appId + "@" + ret; + } + + if (ret.length() > MAX_PLUGIN_ID_LEN ) { + ret = ret.substring(0,MAX_PLUGIN_ID_LEN); + } + + return ret ; + } + /* + * This method returns the hostname of agents. + */ + public String getAgentHostname() { + return hostname; + } + + public String getHostnameFromPluginId(String pluginId, String serviceName) { + String ret = ""; + + if (StringUtils.isNotBlank(pluginId)) { + int lastIndex; + String[] parts = pluginId.split("@"); + int index = parts.length > 1 ? 1 : 0; + if (StringUtils.isNotBlank(serviceName)) { + lastIndex = StringUtils.lastIndexOf(parts[index], serviceName); + if (lastIndex > 1) { + ret = parts[index].substring(0, lastIndex-1); + } + } else { + lastIndex = StringUtils.lastIndexOf(parts[index], "-"); + if (lastIndex > 0) { + ret = parts[index].substring(0, lastIndex); + } + } + } + + return ret; + } + public String getAppIdFromPluginId(String pluginId) { + String ret = "**Unknown**"; + + if (StringUtils.isNotBlank(pluginId)) { + String[] parts = pluginId.split("@"); + ret = parts[0]; + } + + return ret; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerReadWriteLock.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerReadWriteLock.java new file mode 100644 index 00000000000..06af2dc6863 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerReadWriteLock.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.util; + +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +public class RangerReadWriteLock { + + private static final RangerLock NO_OP_LOCK = new RangerLock(null); + + private final ReentrantReadWriteLock lock; + + public RangerReadWriteLock(boolean isUseLock) { + lock = isUseLock ? new ReentrantReadWriteLock(true) : null; + } + + public RangerLock getReadLock() { + final RangerLock ret; + if (lock != null) { + ReentrantReadWriteLock.ReadLock readLock = lock.readLock(); + readLock.lock(); + ret = new RangerLock(readLock); + } else { + ret = NO_OP_LOCK; + } + return ret; + } + + public RangerLock getWriteLock() { + final RangerLock ret; + if (lock != null) { + boolean isLocked = false; + ReentrantReadWriteLock.WriteLock writeLock = lock.writeLock(); + while (!isLocked) { + isLocked = writeLock.tryLock(); + // Another thread has acquired write lock. Hint to scheduler to schedule some other thread if needed + if (!isLocked && lock.getReadLockCount() == 0) { + Thread.yield(); + } + } + ret = new RangerLock(writeLock); + } else { + ret = NO_OP_LOCK; + } + return ret; + } + + @Override + public String toString() { + if (lock != null) { + return "ReadWriteLock:[" + lock.toString() + "], ReadLock:[" + lock.readLock().toString() + "], WriteLock:[" + lock.writeLock().toString() + "]"; + } else { + return "ReadWriteLock:[null]"; + } + } + + static final public class RangerLock implements AutoCloseable { + private final Lock lock; + + private RangerLock(Lock lock) { + this.lock = lock; + } + + @Override + public void close() { + if (lock != null) { + lock.unlock(); + } + } + + public boolean isLockingEnabled() { + return lock != null; + } + + @Override + public String toString() { + return lock == null ? "null" : lock.toString(); + } + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerRequestedResources.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerRequestedResources.java new file mode 100644 index 00000000000..fa4272aa67c --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerRequestedResources.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +package org.apache.atlas.plugin.util; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.htrace.shaded.fasterxml.jackson.annotation.JsonInclude; +import org.apache.atlas.plugin.policyengine.RangerAccessResource; +import org.apache.atlas.plugin.policyresourcematcher.RangerPolicyResourceMatcher; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +@JsonInclude(JsonInclude.Include.NON_NULL) +/*@JsonAutoDetect(getterVisibility= JsonAutoDetect.Visibility.NONE, setterVisibility= JsonAutoDetect.Visibility.NONE, fieldVisibility= JsonAutoDetect.Visibility.ANY) +@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL ) +@JsonIgnoreProperties(ignoreUnknown=true)*/ +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) + +public class RangerRequestedResources { + private List requestedResources = new ArrayList<>(); + + public RangerRequestedResources() { + } + + public void addRequestedResource(RangerAccessResource requestedResource) { + if (requestedResource != null) { + + if (CollectionUtils.isEmpty(requestedResources)) { + requestedResources = new ArrayList<>(); + } + + boolean exists = requestedResources.contains(requestedResource); + + if (!exists) { + requestedResources.add(requestedResource); + } + } + } + + public boolean isMutuallyExcluded(final List matchers, final Map evalContext) { + boolean ret = true; + + int matchedCount = 0; + + if (!CollectionUtils.isEmpty(matchers) && !CollectionUtils.isEmpty(requestedResources) && requestedResources.size() > 1) { + + for (RangerAccessResource resource : requestedResources) { + + for (RangerPolicyResourceMatcher matcher : matchers) { + + if (matcher.isMatch(resource, evalContext) && matchedCount++ > 0) { + ret = false; + break; + } + } + } + } + + return ret; + } + + @Override + public String toString( ) { + StringBuilder sb = new StringBuilder(); + + toString(sb); + + return sb.toString(); + } + + StringBuilder toString(StringBuilder sb) { + sb.append("AllRequestedHiveResources={"); + for (RangerAccessResource resource : requestedResources) { + if (resource != null) { + sb.append(resource.getAsString()); + sb.append("; "); + } + } + sb.append("} "); + + return sb; + } +} + + diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerRoles.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerRoles.java new file mode 100644 index 00000000000..083871f1e22 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerRoles.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.util; + +import org.apache.htrace.shaded.fasterxml.jackson.annotation.JsonInclude; +import org.apache.atlas.plugin.model.RangerRole; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import java.io.Serializable; +import java.util.Date; +import java.util.Set; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class RangerRoles implements Serializable { + private static final long serialVersionUID = 1L; + + private String serviceName; + private Long roleVersion; + private Date roleUpdateTime; + private Set rangerRoles; + + public String getServiceName() { + return serviceName; + } + + public void setServiceName(String serviceName) { + this.serviceName = serviceName; + } + + public Long getRoleVersion() { + return roleVersion; + } + + public void setRoleVersion(Long roleVersion) { + this.roleVersion = roleVersion; + } + + public Date getRoleUpdateTime() { + return roleUpdateTime; + } + + public void setRoleUpdateTime(Date roleUpdateTime) { + this.roleUpdateTime = roleUpdateTime; + } + + public Set getRangerRoles(){ + return this.rangerRoles; + } + + public void setRangerRoles(Set rangerRoles){ + this.rangerRoles = rangerRoles; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerRolesProvider.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerRolesProvider.java new file mode 100644 index 00000000000..4607d6192a4 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerRolesProvider.java @@ -0,0 +1,375 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.util; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import org.apache.atlas.authz.admin.client.AtlasAuthAdminClient; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.atlas.admin.client.RangerAdminClient; +import org.apache.atlas.plugin.service.RangerBasePlugin; + +import java.io.File; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.Reader; +import java.io.Writer; +import java.util.Date; +import java.util.HashSet; + + +public class RangerRolesProvider { + private static final Log LOG = LogFactory.getLog(RangerRolesProvider.class); + + private static final Log PERF_POLICYENGINE_INIT_LOG = RangerPerfTracer.getPerfLogger("policyengine.init"); + + private final String serviceType; + private final String serviceName; + private final AtlasAuthAdminClient atlasAuthAdminClient; + private final KeycloakUserStore keycloakUserStore; + + private final String cacheFileName; + private final String cacheFileNamePrefix; + private final String cacheDir; + private final Gson gson; + private final boolean disableCacheIfServiceNotFound; + + private long lastUpdatedTimeInMillis = -1; + private long lastActivationTimeInMillis; + private long lastKnownRoleVersion = -1L; + private boolean rangerUserGroupRolesSetInPlugin; + private boolean serviceDefSetInPlugin; + + public RangerRolesProvider(String serviceType, String appId, String serviceName, AtlasAuthAdminClient atlasAuthAdminClient, String cacheDir, Configuration config) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerRolesProvider(serviceName=" + serviceName + ").RangerRolesProvider()"); + } + + this.serviceType = serviceType; + this.serviceName = serviceName; + this.atlasAuthAdminClient = atlasAuthAdminClient; + this.keycloakUserStore = new KeycloakUserStore(serviceType); + + + if (StringUtils.isEmpty(appId)) { + appId = serviceType; + } + + cacheFileNamePrefix = "roles"; + String cacheFilename = String.format("%s_%s_%s.json", appId, serviceName, cacheFileNamePrefix); + cacheFilename = cacheFilename.replace(File.separatorChar, '_'); + cacheFilename = cacheFilename.replace(File.pathSeparatorChar, '_'); + + this.cacheFileName = cacheFilename; + this.cacheDir = cacheDir; + + Gson gson = null; + try { + gson = new GsonBuilder().setDateFormat("yyyyMMdd-HH:mm:ss.SSS-Z").create(); + } catch (Throwable excp) { + LOG.fatal("RangerRolesProvider(): failed to create GsonBuilder object", excp); + } + this.gson = gson; + + String propertyPrefix = "ranger.plugin." + serviceType; + disableCacheIfServiceNotFound = config.getBoolean(propertyPrefix + ".disable.cache.if.servicenotfound", true); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerRolesProvider(serviceName=" + serviceName + ").RangerRolesProvider()"); + } + } + + public long getLastUpdatedTimeInMillis() { + return lastUpdatedTimeInMillis; + } + + public void setLastUpdatedTimeInMillis(long lastUpdatedTimeInMillis) { + this.lastUpdatedTimeInMillis = lastUpdatedTimeInMillis; + } + + public long getLastActivationTimeInMillis() { + return lastActivationTimeInMillis; + } + + public void setLastActivationTimeInMillis(long lastActivationTimeInMillis) { + this.lastActivationTimeInMillis = lastActivationTimeInMillis; + } + + public void loadUserGroupRoles(RangerBasePlugin plugIn) { + + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerRolesProvider(serviceName= " + serviceName + " serviceType= " + serviceType +").loadUserGroupRoles()"); + } + + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_POLICYENGINE_INIT_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_POLICYENGINE_INIT_LOG, "RangerRolesProvider.loadUserGroupRoles(serviceName=" + serviceName + ")"); + long freeMemory = Runtime.getRuntime().freeMemory(); + long totalMemory = Runtime.getRuntime().totalMemory(); + PERF_POLICYENGINE_INIT_LOG.debug("In-Use memory: " + (totalMemory-freeMemory) + ", Free memory:" + freeMemory); + } + + try { + //load userGroupRoles from ranger admin + RangerRoles roles = loadUserGroupRolesFromAdmin(); + + if (roles == null) { + //if userGroupRoles fetch from ranger Admin Fails, load from cache + if (!rangerUserGroupRolesSetInPlugin) { + roles = loadUserGroupRolesFromCache(); + } + } + + if (PERF_POLICYENGINE_INIT_LOG.isDebugEnabled()) { + long freeMemory = Runtime.getRuntime().freeMemory(); + long totalMemory = Runtime.getRuntime().totalMemory(); + PERF_POLICYENGINE_INIT_LOG.debug("In-Use memory: " + (totalMemory - freeMemory) + ", Free memory:" + freeMemory); + } + + if (roles != null) { + plugIn.setRoles(roles); + rangerUserGroupRolesSetInPlugin = true; + setLastActivationTimeInMillis(System.currentTimeMillis()); + lastKnownRoleVersion = roles.getRoleVersion() == null ? -1 : roles.getRoleVersion(); + lastUpdatedTimeInMillis = roles.getRoleUpdateTime() == null ? -1 : roles.getRoleUpdateTime().getTime(); + } else { + if (!rangerUserGroupRolesSetInPlugin && !serviceDefSetInPlugin) { + plugIn.setRoles(null); + serviceDefSetInPlugin = true; + } + } + } catch (RangerServiceNotFoundException snfe) { + if (disableCacheIfServiceNotFound) { + disableCache(); + plugIn.setRoles(null); + setLastActivationTimeInMillis(System.currentTimeMillis()); + lastKnownRoleVersion = -1L; + lastUpdatedTimeInMillis = -1L; + serviceDefSetInPlugin = true; + } + } catch (Exception excp) { + LOG.error("Encountered unexpected exception, ignoring..", excp); + } + + RangerPerfTracer.log(perf); + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerRolesProvider(serviceName=" + serviceName + ").loadUserGroupRoles()"); + } + } + + private RangerRoles loadUserGroupRolesFromAdmin() throws RangerServiceNotFoundException { + + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerRolesProvider(serviceName=" + serviceName + ").loadUserGroupRolesFromAdmin()"); + } + + RangerRoles roles = null; + + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_POLICYENGINE_INIT_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_POLICYENGINE_INIT_LOG, "RangerRolesProvider.loadUserGroupRolesFromAdmin(serviceName=" + serviceName + ")"); + } + + try { + if ("atlas".equals(serviceName)) { + roles = keycloakUserStore.loadRolesIfUpdated(lastUpdatedTimeInMillis); + } else { + roles = atlasAuthAdminClient.getRolesIfUpdated(lastUpdatedTimeInMillis); + } + + boolean isUpdated = roles != null; + + if(isUpdated) { + long newVersion = roles.getRoleVersion() == null ? -1 : roles.getRoleVersion().longValue(); + saveToCache(roles); + LOG.info("RangerRolesProvider(serviceName=" + serviceName + "): found updated version. lastKnownRoleVersion=" + lastKnownRoleVersion + "; newVersion=" + newVersion ); + } else { + if(LOG.isDebugEnabled()) { + LOG.debug("RangerRolesProvider(serviceName=" + serviceName + ").run(): no update found. lastKnownRoleVersion=" + lastKnownRoleVersion ); + } + } + } catch (Exception excp) { + LOG.error("RangerRolesProvider(serviceName=" + serviceName + "): failed to refresh roles. Will continue to use last known version of roles (" + "lastKnowRoleVersion= " + lastKnownRoleVersion, excp); + roles = null; + } + + RangerPerfTracer.log(perf); + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerRolesProvider(serviceName=" + serviceName + " serviceType= " + serviceType + " ).loadUserGroupRolesFromAdmin()"); + } + + return roles; + } + + private RangerRoles loadUserGroupRolesFromCache() { + + RangerRoles roles = null; + + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerRolesProvider(serviceName=" + serviceName + ").loadUserGroupRolesFromCache()"); + } + + File cacheFile = cacheDir == null ? null : new File(cacheDir + File.separator + cacheFileName); + + if (cacheFile != null && cacheFile.isFile() && cacheFile.canRead()) { + Reader reader = null; + + RangerPerfTracer perf = null; + + if (RangerPerfTracer.isPerfTraceEnabled(PERF_POLICYENGINE_INIT_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_POLICYENGINE_INIT_LOG, "RangerRolesProvider.loadUserGroupRolesFromCache(serviceName=" + serviceName + ")"); + } + + try { + reader = new FileReader(cacheFile); + + roles = gson.fromJson(reader, RangerRoles.class); + + if (roles != null) { + if (!StringUtils.equals(serviceName, roles.getServiceName())) { + LOG.warn("ignoring unexpected serviceName '" + roles.getServiceName() + "' in cache file '" + cacheFile.getAbsolutePath() + "'"); + + roles.setServiceName(serviceName); + } + + lastKnownRoleVersion = roles.getRoleVersion() == null ? -1 : roles.getRoleVersion().longValue(); + lastUpdatedTimeInMillis = roles.getRoleUpdateTime() == null ? -1 : roles.getRoleUpdateTime().getTime(); + } + } catch (Exception excp) { + LOG.error("failed to load userGroupRoles from cache file " + cacheFile.getAbsolutePath(), excp); + } finally { + RangerPerfTracer.log(perf); + + if (reader != null) { + try { + reader.close(); + } catch (Exception excp) { + LOG.error("error while closing opened cache file " + cacheFile.getAbsolutePath(), excp); + } + } + } + } else { + roles = new RangerRoles(); + roles.setServiceName(serviceName); + roles.setRoleVersion(-1L); + roles.setRoleUpdateTime(null); + roles.setRangerRoles(new HashSet<>()); + saveToCache(roles); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerRolesProvider(serviceName=" + serviceName + ").RangerRolesProvider()"); + } + + return roles; + } + + public void saveToCache(RangerRoles roles) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerRolesProvider(serviceName=" + serviceName + ").saveToCache()"); + } + + if(roles != null) { + File cacheFile = null; + if (cacheDir != null) { + // Create the cacheDir if it doesn't already exist + File cacheDirTmp = new File(cacheDir); + if (cacheDirTmp.exists()) { + cacheFile = new File(cacheDir + File.separator + cacheFileName); + } else { + try { + cacheDirTmp.mkdirs(); + cacheFile = new File(cacheDir + File.separator + cacheFileName); + } catch (SecurityException ex) { + LOG.error("Cannot create cache directory", ex); + } + } + } + + if(cacheFile != null) { + + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_POLICYENGINE_INIT_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_POLICYENGINE_INIT_LOG, "RangerRolesProvider.saveToCache(serviceName=" + serviceName + ")"); + } + + Writer writer = null; + + try { + writer = new FileWriter(cacheFile); + + gson.toJson(roles, writer); + } catch (Exception excp) { + LOG.error("failed to save roles to cache file '" + cacheFile.getAbsolutePath() + "'", excp); + } finally { + if(writer != null) { + try { + writer.close(); + } catch(Exception excp) { + LOG.error("error while closing opened cache file '" + cacheFile.getAbsolutePath() + "'", excp); + } + } + } + + RangerPerfTracer.log(perf); + } + } else { + LOG.info("roles is null. Nothing to save in cache"); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerRolesProvider.saveToCache(serviceName=" + serviceName + ")"); + } + } + + private void disableCache() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerRolesProvider.disableCache(serviceName=" + serviceName + ")"); + } + + File cacheFile = cacheDir == null ? null : new File(cacheDir + File.separator + cacheFileName); + + if(cacheFile != null && cacheFile.isFile() && cacheFile.canRead()) { + LOG.warn("Cleaning up local RangerRoles cache"); + String renamedCacheFile = cacheFile.getAbsolutePath() + "_" + System.currentTimeMillis(); + if (!cacheFile.renameTo(new File(renamedCacheFile))) { + LOG.error("Failed to move " + cacheFile.getAbsolutePath() + " to " + renamedCacheFile); + } else { + LOG.warn("Moved " + cacheFile.getAbsolutePath() + " to " + renamedCacheFile); + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("No local RangerRoles cache found. No need to disable it!"); + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerRolesProvider.disableCache(serviceName=" + serviceName + ")"); + } + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerRolesUtil.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerRolesUtil.java new file mode 100644 index 00000000000..78e9e46b144 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerRolesUtil.java @@ -0,0 +1,172 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.util; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.atlas.plugin.model.RangerRole; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class RangerRolesUtil { + private final long roleVersion; + private final Map> userRoleMapping = new HashMap<>(); + private final Map> groupRoleMapping = new HashMap<>(); + private final Map> roleRoleMapping = new HashMap<>(); + + private final Map> roleToUserMapping = new HashMap<>(); + private final Map> roleToGroupMapping = new HashMap<>(); + + private RangerRoles roles = null; + public enum ROLES_FOR {USER, GROUP, ROLE} + + public RangerRolesUtil(RangerRoles roles) { + if (roles != null) { + this.roles = roles; + roleVersion = roles.getRoleVersion(); + + if (CollectionUtils.isNotEmpty(roles.getRangerRoles())) { + for (RangerRole role : roles.getRangerRoles()) { + Set containedRoles = getAllContainedRoles(roles.getRangerRoles(), role); + + buildMap(userRoleMapping, role, containedRoles, ROLES_FOR.USER); + buildMap(groupRoleMapping, role, containedRoles, ROLES_FOR.GROUP); + buildMap(roleRoleMapping, role, containedRoles, ROLES_FOR.ROLE); + + Set roleUsers = new HashSet<>(); + Set roleGroups = new HashSet<>(); + + addMemberNames(role.getUsers(), roleUsers); + addMemberNames(role.getGroups(), roleGroups); + + for (RangerRole containedRole : containedRoles) { + addMemberNames(containedRole.getUsers(), roleUsers); + addMemberNames(containedRole.getGroups(), roleGroups); + } + + roleToUserMapping.put(role.getName(), roleUsers); + roleToGroupMapping.put(role.getName(), roleGroups); + } + + } + } else { + roleVersion = -1L; + } + } + + public long getRoleVersion() { return roleVersion; } + + public RangerRoles getRoles() { + return this.roles; + } + + public Map> getUserRoleMapping() { + return this.userRoleMapping; + } + + public Map> getGroupRoleMapping() { + return this.groupRoleMapping; + } + + public Map> getRoleRoleMapping() { + return this.roleRoleMapping; + } + + public Map> getRoleToUserMapping() { + return this.roleToUserMapping; + } + + public Map> getRoleToGroupMapping() { + return this.roleToGroupMapping; + } + + private Set getAllContainedRoles(Set roles, RangerRole role) { + Set allRoles = new HashSet<>(); + + allRoles.add(role); + addContainedRoles(allRoles, roles, role); + + return allRoles; + } + + private void addContainedRoles(Set allRoles, Set roles, RangerRole role) { + List roleMembers = role.getRoles(); + + for (RangerRole.RoleMember roleMember : roleMembers) { + RangerRole containedRole = getContainedRole(roles, roleMember.getName()); + + if (containedRole!= null && !allRoles.contains(containedRole)) { + allRoles.add(containedRole); + addContainedRoles(allRoles, roles, containedRole); + } + } + } + + private void buildMap(Map> map, RangerRole role, Set containedRoles, ROLES_FOR roleFor) { + buildMap(map, role, role.getName(), roleFor); + + for (RangerRole containedRole : containedRoles) { + buildMap(map, containedRole, role.getName(), roleFor); + } + } + + private void buildMap(Map> map, RangerRole role, String roleName, ROLES_FOR roles_for) { + List userOrGroupOrRole = null; + switch(roles_for) { + case USER: + userOrGroupOrRole = role.getUsers(); + break; + case GROUP: + userOrGroupOrRole = role.getGroups(); + break; + case ROLE: + userOrGroupOrRole = role.getRoles(); + break; + } + if (CollectionUtils.isNotEmpty(userOrGroupOrRole)) { + getRoleMap(map, roleName, userOrGroupOrRole); + } + } + + private void getRoleMap(Map> map, String roleName, List userOrGroupOrRole) { + for (RangerRole.RoleMember roleMember : userOrGroupOrRole) { + if (StringUtils.isNotEmpty(roleMember.getName())) { + Set roleNames = map.computeIfAbsent(roleMember.getName(), k -> new HashSet<>()); + roleNames.add(roleName); + } + } + } + + private RangerRole getContainedRole(Set roles, String role) { + return (roles.stream().filter(containedRole -> role.equals(containedRole.getName())).findAny().orElse(null)); + } + + private void addMemberNames(List members, Set names) { + for (RangerRole.RoleMember member : members) { + names.add(member.getName()); + } + } +} + + diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerServiceNotFoundException.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerServiceNotFoundException.java new file mode 100644 index 00000000000..e1c836bc8b3 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerServiceNotFoundException.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.util; + +import org.apache.commons.lang.StringUtils; + +public class RangerServiceNotFoundException extends Exception { + static private final String formatString = "\"RANGER_ERROR_SERVICE_NOT_FOUND: ServiceName=%s\""; + public RangerServiceNotFoundException(String serviceName) { + super(serviceName); + } + public static final String buildExceptionMsg(String serviceName) { + return String.format(formatString, serviceName); + } + public static final void throwExceptionIfServiceNotFound(String serviceName, String exceptionMsg) throws RangerServiceNotFoundException { + String expectedExceptionMsg = buildExceptionMsg(serviceName); + if (StringUtils.startsWith(exceptionMsg, expectedExceptionMsg)) { + throw new RangerServiceNotFoundException(serviceName); + } + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerServiceTagsDeltaUtil.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerServiceTagsDeltaUtil.java new file mode 100644 index 00000000000..87be65297e5 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerServiceTagsDeltaUtil.java @@ -0,0 +1,177 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.util; + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.plugin.model.RangerServiceResource; +import org.apache.atlas.plugin.model.RangerTag; +import org.apache.atlas.plugin.model.RangerTagDef; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +public class RangerServiceTagsDeltaUtil { + + private static final Log LOG = LogFactory.getLog(RangerServiceTagsDeltaUtil.class); + + private static final Log PERF_TAGS_DELTA_LOG = RangerPerfTracer.getPerfLogger("tags.delta"); + + /* + It should be possible to call applyDelta() multiple times with serviceTags and delta resulting from previous call to applyDelta() + The end result should be same if called once or multiple times. + */ + static public ServiceTags applyDelta(ServiceTags serviceTags, ServiceTags delta) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerServiceTagsDeltaUtil.applyDelta()"); + } + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_TAGS_DELTA_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_TAGS_DELTA_LOG, "RangerServiceTagsDeltaUtil.applyDelta()"); + } + if (serviceTags != null && !serviceTags.getIsDelta() && delta != null && delta.getIsDelta()) { + serviceTags.setServiceName(delta.getServiceName()); + serviceTags.setTagVersion(delta.getTagVersion()); + + // merge + Map tags = serviceTags.getTags(); + List serviceResources = serviceTags.getServiceResources(); + Map> resourceToTagIds = serviceTags.getResourceToTagIds(); + boolean isAnyMaterialChange = false; + + for (Map.Entry tagEntry : delta.getTags().entrySet()) { + if (StringUtils.isEmpty(tagEntry.getValue().getType())) { + if (null != tags.remove(tagEntry.getKey())) { + isAnyMaterialChange = true; + } + } else { + tags.put(tagEntry.getKey(), tagEntry.getValue()); + } + } + + // This could be expensive - compute time is M x N ( M - number of old tagged service resources, N - number of changed service resources) + + Map deletedServiceResources = new HashMap<>(); + List addedServiceResources = new ArrayList<>(); + + for (RangerServiceResource serviceResource : delta.getServiceResources()) { + + boolean found = false; + Iterator iter = serviceResources.iterator(); + + while (iter.hasNext()) { + RangerServiceResource existingResource = iter.next(); + + if (serviceResource.getId().equals(existingResource.getId())) { + if (!StringUtils.isEmpty(serviceResource.getResourceSignature())) { + if (!serviceResource.getResourceSignature().equals(existingResource.getResourceSignature())) { // ServiceResource changed + iter.remove(); + existingResource.setResourceSignature(null); + addedServiceResources.add(existingResource); + break; + } + } else { + iter.remove(); + deletedServiceResources.put(serviceResource.getId(), serviceResource.getId()); + } + found = true; + break; + } + } + if (!found) { + if (StringUtils.isNotEmpty(serviceResource.getResourceSignature())) { + serviceResources.add(serviceResource); + isAnyMaterialChange = true; + } + } + } + + for (Long deletedServiceResourceId : deletedServiceResources.keySet()) { + resourceToTagIds.remove(deletedServiceResourceId); + } + + resourceToTagIds.putAll(delta.getResourceToTagIds()); + + // Ensure that any modified service-resources are at head of list of service-resources in delta + // So that in in setServiceTags(), they get cleaned out first, and service-resource with new spec gets added + + addedServiceResources.addAll(delta.getServiceResources()); + delta.setServiceResources(addedServiceResources); + + if (!isAnyMaterialChange) { + if (LOG.isDebugEnabled()) { + LOG.debug("No material change may have occurred because of applying this delta"); + } + } + + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Cannot apply deltas to service-tags as one of preconditions is violated. Returning received serviceTags without applying delta!!"); + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerServiceTagsDeltaUtil.applyDelta()"); + } + + RangerPerfTracer.log(perf); + + return serviceTags; + } + + public static void pruneUnusedAttributes(ServiceTags serviceTags) { + if (serviceTags != null) { + serviceTags.setTagUpdateTime(null); + + for (Map.Entry entry : serviceTags.getTagDefinitions().entrySet()) { + RangerTagDef tagDef = entry.getValue(); + tagDef.setCreatedBy(null); + tagDef.setCreateTime(null); + tagDef.setUpdatedBy(null); + tagDef.setUpdateTime(null); + tagDef.setGuid(null); + } + + for (Map.Entry entry : serviceTags.getTags().entrySet()) { + RangerTag tag = entry.getValue(); + tag.setCreatedBy(null); + tag.setCreateTime(null); + tag.setUpdatedBy(null); + tag.setUpdateTime(null); + tag.setGuid(null); + } + + for (RangerServiceResource serviceResource : serviceTags.getServiceResources()) { + serviceResource.setCreatedBy(null); + serviceResource.setCreateTime(null); + serviceResource.setUpdatedBy(null); + serviceResource.setUpdateTime(null); + serviceResource.setGuid(null); + + serviceResource.setServiceName(null); + } + } + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerSslHelper.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerSslHelper.java new file mode 100644 index 00000000000..187185b2685 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerSslHelper.java @@ -0,0 +1,287 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.util; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.atlas.authorization.hadoop.utils.RangerCredentialProvider; +import org.apache.atlas.authorization.utils.StringUtil; + +import javax.net.ssl.HostnameVerifier; +import javax.net.ssl.KeyManager; +import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLSession; +import javax.net.ssl.TrustManager; +import javax.net.ssl.TrustManagerFactory; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.security.KeyStore; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.security.SecureRandom; +import java.security.UnrecoverableKeyException; +import java.security.cert.CertificateException; + +public class RangerSslHelper { + private static final Log LOG = LogFactory.getLog(RangerSslHelper.class); + + static final String RANGER_POLICYMGR_CLIENT_KEY_FILE = "xasecure.policymgr.clientssl.keystore"; + static final String RANGER_POLICYMGR_CLIENT_KEY_FILE_TYPE = "xasecure.policymgr.clientssl.keystore.type"; + static final String RANGER_POLICYMGR_CLIENT_KEY_FILE_CREDENTIAL = "xasecure.policymgr.clientssl.keystore.credential.file"; + static final String RANGER_POLICYMGR_CLIENT_KEY_FILE_CREDENTIAL_ALIAS = "sslKeyStore"; + static final String RANGER_POLICYMGR_CLIENT_KEY_FILE_TYPE_DEFAULT = "jks"; + + static final String RANGER_POLICYMGR_TRUSTSTORE_FILE = "xasecure.policymgr.clientssl.truststore"; + static final String RANGER_POLICYMGR_TRUSTSTORE_FILE_TYPE = "xasecure.policymgr.clientssl.truststore.type"; + static final String RANGER_POLICYMGR_TRUSTSTORE_FILE_CREDENTIAL = "xasecure.policymgr.clientssl.truststore.credential.file"; + static final String RANGER_POLICYMGR_TRUSTSTORE_FILE_CREDENTIAL_ALIAS = "sslTrustStore"; + static final String RANGER_POLICYMGR_TRUSTSTORE_FILE_TYPE_DEFAULT = "jks"; + + static final String RANGER_SSL_KEYMANAGER_ALGO_TYPE = KeyManagerFactory.getDefaultAlgorithm(); + static final String RANGER_SSL_TRUSTMANAGER_ALGO_TYPE = TrustManagerFactory.getDefaultAlgorithm(); + static final String RANGER_SSL_CONTEXT_ALGO_TYPE = "TLS"; + + private String mKeyStoreURL; + private String mKeyStoreAlias; + private String mKeyStoreFile; + private String mKeyStoreType; + private String mTrustStoreURL; + private String mTrustStoreAlias; + private String mTrustStoreFile; + private String mTrustStoreType; + + final static HostnameVerifier _Hv = new HostnameVerifier() { + + @Override + public boolean verify(String urlHostName, SSLSession session) { + return session.getPeerHost().equals(urlHostName); + } + }; + + final String mSslConfigFileName; + + public RangerSslHelper(String sslConfigFileName) { + mSslConfigFileName = sslConfigFileName; + + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerSslHelper(" + mSslConfigFileName + ")"); + } + + } + + public SSLContext createContext() { + readConfig(); + KeyManager[] kmList = getKeyManagers(); + TrustManager[] tmList = getTrustManagers(); + SSLContext sslContext = getSSLContext(kmList, tmList); + return sslContext; + } + + public HostnameVerifier getHostnameVerifier() { + return _Hv; + } + + void readConfig() { + InputStream in = null; + + try { + Configuration conf = new Configuration(); + + in = getFileInputStream(mSslConfigFileName); + + if (in != null) { + conf.addResource(in); + } + + mKeyStoreURL = conf.get(RANGER_POLICYMGR_CLIENT_KEY_FILE_CREDENTIAL); + mKeyStoreAlias = RANGER_POLICYMGR_CLIENT_KEY_FILE_CREDENTIAL_ALIAS; + mKeyStoreType = conf.get(RANGER_POLICYMGR_CLIENT_KEY_FILE_TYPE, RANGER_POLICYMGR_CLIENT_KEY_FILE_TYPE_DEFAULT); + mKeyStoreFile = conf.get(RANGER_POLICYMGR_CLIENT_KEY_FILE); + + mTrustStoreURL = conf.get(RANGER_POLICYMGR_TRUSTSTORE_FILE_CREDENTIAL); + mTrustStoreAlias = RANGER_POLICYMGR_TRUSTSTORE_FILE_CREDENTIAL_ALIAS; + mTrustStoreType = conf.get(RANGER_POLICYMGR_TRUSTSTORE_FILE_TYPE, RANGER_POLICYMGR_TRUSTSTORE_FILE_TYPE_DEFAULT); + mTrustStoreFile = conf.get(RANGER_POLICYMGR_TRUSTSTORE_FILE); + + if (LOG.isDebugEnabled()) { + LOG.debug(toString()); + } + } + catch(IOException ioe) { + LOG.error("Unable to load SSL Config FileName: [" + mSslConfigFileName + "]", ioe); + } + finally { + close(in, mSslConfigFileName); + } + } + + private KeyManager[] getKeyManagers() { + KeyManager[] kmList = null; + + String keyStoreFilepwd = getCredential(mKeyStoreURL, mKeyStoreAlias); + + if (!StringUtil.isEmpty(mKeyStoreFile) && !StringUtil.isEmpty(keyStoreFilepwd)) { + InputStream in = null; + + try { + in = getFileInputStream(mKeyStoreFile); + + if (in != null) { + KeyStore keyStore = KeyStore.getInstance(mKeyStoreType); + + keyStore.load(in, keyStoreFilepwd.toCharArray()); + + KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance(RANGER_SSL_KEYMANAGER_ALGO_TYPE); + + keyManagerFactory.init(keyStore, keyStoreFilepwd.toCharArray()); + + kmList = keyManagerFactory.getKeyManagers(); + } else { + LOG.error("Unable to obtain keystore from file [" + mKeyStoreFile + "]"); + } + } catch (KeyStoreException e) { + LOG.error("Unable to obtain from KeyStore", e); + } catch (NoSuchAlgorithmException e) { + LOG.error("SSL algorithm is available in the environment", e); + } catch (CertificateException e) { + LOG.error("Unable to obtain the requested certification ", e); + } catch (FileNotFoundException e) { + LOG.error("Unable to find the necessary SSL Keystore and TrustStore Files", e); + } catch (IOException e) { + LOG.error("Unable to read the necessary SSL Keystore and TrustStore Files", e); + } catch (UnrecoverableKeyException e) { + LOG.error("Unable to recover the key from keystore", e); + } finally { + close(in, mKeyStoreFile); + } + } + + return kmList; + } + + private TrustManager[] getTrustManagers() { + TrustManager[] tmList = null; + + String trustStoreFilepwd = getCredential(mTrustStoreURL, mTrustStoreAlias); + + if (!StringUtil.isEmpty(mTrustStoreFile) && !StringUtil.isEmpty(trustStoreFilepwd)) { + InputStream in = null; + + try { + in = getFileInputStream(mTrustStoreFile); + + if (in != null) { + KeyStore trustStore = KeyStore.getInstance(mTrustStoreType); + + trustStore.load(in, trustStoreFilepwd.toCharArray()); + + TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(RANGER_SSL_TRUSTMANAGER_ALGO_TYPE); + + trustManagerFactory.init(trustStore); + + tmList = trustManagerFactory.getTrustManagers(); + } else { + LOG.error("Unable to obtain keystore from file [" + mTrustStoreFile + "]"); + } + } catch (KeyStoreException e) { + LOG.error("Unable to obtain from KeyStore", e); + } catch (NoSuchAlgorithmException e) { + LOG.error("SSL algorithm is available in the environment", e); + } catch (CertificateException e) { + LOG.error("Unable to obtain the requested certification ", e); + } catch (FileNotFoundException e) { + LOG.error("Unable to find the necessary SSL Keystore and TrustStore Files", e); + } catch (IOException e) { + LOG.error("Unable to read the necessary SSL Keystore and TrustStore Files", e); + } finally { + close(in, mTrustStoreFile); + } + } + + return tmList; + } + + private SSLContext getSSLContext(KeyManager[] kmList, TrustManager[] tmList) { + try { + if(tmList != null) { + SSLContext sslContext = SSLContext.getInstance(RANGER_SSL_CONTEXT_ALGO_TYPE); + + sslContext.init(kmList, tmList, new SecureRandom()); + + return sslContext; + } + } catch (NoSuchAlgorithmException e) { + LOG.error("SSL algorithm is available in the environment", e); + } catch (Exception e) { + LOG.error("Unable to initialize the SSLContext", e); + } + + return null; + } + + private String getCredential(String url, String alias) { + return RangerCredentialProvider.getInstance().getCredentialString(url, alias); + } + + private InputStream getFileInputStream(String fileName) throws IOException { + InputStream in = null; + + if(! StringUtil.isEmpty(fileName)) { + File f = new File(fileName); + + if (f.exists()) { + in = new FileInputStream(f); + } + else { + in = ClassLoader.getSystemResourceAsStream(fileName); + } + } + + return in; + } + + private void close(InputStream str, String filename) { + if (str != null) { + try { + str.close(); + } catch (IOException excp) { + LOG.error("Error while closing file: [" + filename + "]", excp); + } + } + } + + @Override + public String toString() { + return "keyStoreAlias=" + mKeyStoreAlias + ", " + + "keyStoreFile=" + mKeyStoreFile + ", " + + "keyStoreType="+ mKeyStoreType + ", " + + "keyStoreURL=" + mKeyStoreURL + ", " + + "trustStoreAlias=" + mTrustStoreAlias + ", " + + "trustStoreFile=" + mTrustStoreFile + ", " + + "trustStoreType=" + mTrustStoreType + ", " + + "trustStoreURL=" + mTrustStoreURL + ; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerUserStore.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerUserStore.java new file mode 100644 index 00000000000..7318648cf85 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerUserStore.java @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.util; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.htrace.shaded.fasterxml.jackson.annotation.JsonInclude; +import org.apache.atlas.plugin.model.GroupInfo; +import org.apache.atlas.plugin.model.UserInfo; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import java.io.Serializable; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class RangerUserStore implements Serializable { + private static final long serialVersionUID = 1L; + public static final String CLOUD_IDENTITY_NAME = "cloud_id"; + + private Long userStoreVersion; + private Date userStoreUpdateTime; + private Map> userAttrMapping; + private Map> groupAttrMapping ; + private Map> userGroupMapping; + private Map userCloudIdMapping; + private Map groupCloudIdMapping; + private String serviceName; + + public RangerUserStore() {this(-1L, null, null, null);} + + public RangerUserStore(Long userStoreVersion, Set users, Set groups, Map> userGroups) { + setUserStoreVersion(userStoreVersion); + setUserStoreUpdateTime(new Date()); + setUserGroupMapping(userGroups); + buildMap(users, groups); + } + + public String getServiceName() { + return serviceName; + } + + public void setServiceName(String serviceName) { + this.serviceName = serviceName; + } + + public Long getUserStoreVersion() { + return userStoreVersion; + } + + public void setUserStoreVersion(Long userStoreVersion) { + this.userStoreVersion = userStoreVersion; + } + + public Date getUserStoreUpdateTime() { + return userStoreUpdateTime; + } + + public void setUserStoreUpdateTime(Date userStoreUpdateTime) { + this.userStoreUpdateTime = userStoreUpdateTime; + } + + public Map> getUserAttrMapping() { + return userAttrMapping; + } + + public void setUserAttrMapping(Map> userAttrMapping) { + this.userAttrMapping = userAttrMapping; + } + + public Map> getGroupAttrMapping() { + return groupAttrMapping; + } + + public void setGroupAttrMapping(Map> groupAttrMapping) { + this.groupAttrMapping = groupAttrMapping; + } + + public Map> getUserGroupMapping() { + return userGroupMapping; + } + + public void setUserGroupMapping(Map> userGroupMapping) { + this.userGroupMapping = userGroupMapping; + } + + public Map getUserCloudIdMapping() { + return userCloudIdMapping; + } + + public void setUserCloudIdMapping(Map userCloudIdMapping) { + this.userCloudIdMapping = userCloudIdMapping; + } + + public Map getGroupCloudIdMapping() { + return groupCloudIdMapping; + } + + public void setGroupCloudIdMapping(Map groupCloudIdMapping) { + this.groupCloudIdMapping = groupCloudIdMapping; + } + + @Override + public String toString( ) { + StringBuilder sb = new StringBuilder(); + + toString(sb); + + return sb.toString(); + } + + public StringBuilder toString(StringBuilder sb) { + sb.append("RangerUserStore={") + .append("userStoreVersion=").append(userStoreVersion).append(", ") + .append("userStoreUpdateTime=").append(userStoreUpdateTime).append(", "); + sb.append("users={"); + if(MapUtils.isNotEmpty(userAttrMapping)) { + for(String user : userAttrMapping.keySet()) { + sb.append(user); + } + } + sb.append("}, "); + sb.append("groups={"); + if(MapUtils.isNotEmpty(groupAttrMapping)) { + for(String group : groupAttrMapping.keySet()) { + sb.append(group); + } + } + sb.append("}"); + sb.append("}"); + + return sb; + } + + private void buildMap(Set users, Set groups) { + if (CollectionUtils.isNotEmpty(users)) { + userAttrMapping = new HashMap<>(); + userCloudIdMapping = new HashMap<>(); + for (UserInfo user : users) { + String username = user.getName(); + Map userAttrs = user.getOtherAttributes(); + if (MapUtils.isNotEmpty(userAttrs)) { + userAttrMapping.put(username, userAttrs); + String cloudId = userAttrs.get(CLOUD_IDENTITY_NAME); + if (StringUtils.isNotEmpty(cloudId)) { + userCloudIdMapping.put(cloudId, username); + } + } + } + } + if (CollectionUtils.isNotEmpty(groups)) { + groupAttrMapping = new HashMap<>(); + groupCloudIdMapping = new HashMap<>(); + for (GroupInfo group : groups) { + String groupname = group.getName(); + Map groupAttrs = group.getOtherAttributes(); + if (MapUtils.isNotEmpty(groupAttrs)) { + groupAttrMapping.put(groupname, groupAttrs); + String cloudId = groupAttrs.get(CLOUD_IDENTITY_NAME); + if (StringUtils.isNotEmpty(cloudId)) { + groupCloudIdMapping.put(cloudId, groupname); + } + } + } + } + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerUserStoreProvider.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerUserStoreProvider.java new file mode 100644 index 00000000000..89980504b06 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerUserStoreProvider.java @@ -0,0 +1,377 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.util; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import org.apache.atlas.authz.admin.client.AtlasAuthAdminClient; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.atlas.admin.client.RangerAdminClient; +import org.apache.atlas.plugin.service.RangerBasePlugin; + +import java.io.File; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.Reader; +import java.io.Writer; +import java.util.Date; +import java.util.HashMap; +import java.util.Set; + + +public class RangerUserStoreProvider { + private static final Log LOG = LogFactory.getLog(RangerUserStoreProvider.class); + + private static final Log PERF_POLICYENGINE_INIT_LOG = RangerPerfTracer.getPerfLogger("policyengine.init"); + + private final String serviceType; + private final String serviceName; + private final AtlasAuthAdminClient atlasAuthAdminClient; + private final KeycloakUserStore keycloakUserStore; + + private final String cacheFileName; + private final String cacheFileNamePrefix; + private final String cacheDir; + private final Gson gson; + private final boolean disableCacheIfServiceNotFound; + + private long lastActivationTimeInMillis; + private long lastUpdateTimeInMillis = -1L; + private long lastKnownUserStoreVersion = -1L; + private boolean rangerUserStoreSetInPlugin; + private boolean serviceDefSetInPlugin; + + public RangerUserStoreProvider(String serviceType, String appId, String serviceName, AtlasAuthAdminClient atlasAuthAdminClient, String cacheDir, Configuration config) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerUserStoreProvider(serviceName=" + serviceName + ").RangerUserStoreProvider()"); + } + + this.serviceType = serviceType; + this.serviceName = serviceName; + this.atlasAuthAdminClient = atlasAuthAdminClient; + + this.keycloakUserStore = new KeycloakUserStore(serviceType); + + if (StringUtils.isEmpty(appId)) { + appId = serviceType; + } + + cacheFileNamePrefix = "userstore"; + String cacheFilename = String.format("%s_%s_%s.json", appId, serviceName, cacheFileNamePrefix); + cacheFilename = cacheFilename.replace(File.separatorChar, '_'); + cacheFilename = cacheFilename.replace(File.pathSeparatorChar, '_'); + + this.cacheFileName = cacheFilename; + this.cacheDir = cacheDir; + + Gson gson = null; + try { + gson = new GsonBuilder().setDateFormat("yyyyMMdd-HH:mm:ss.SSS-Z").create(); + } catch (Throwable excp) { + LOG.fatal("RangerUserStoreProvider(): failed to create GsonBuilder object", excp); + } + this.gson = gson; + + String propertyPrefix = "ranger.plugin." + serviceType; + disableCacheIfServiceNotFound = config.getBoolean(propertyPrefix + ".disable.cache.if.servicenotfound", true); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerUserStoreProvider(serviceName=" + serviceName + ").RangerUserStoreProvider()"); + } + } + + public long getLastActivationTimeInMillis() { + return lastActivationTimeInMillis; + } + + public void setLastActivationTimeInMillis(long lastActivationTimeInMillis) { + this.lastActivationTimeInMillis = lastActivationTimeInMillis; + } + + public boolean isRangerUserStoreSetInPlugin() { + return rangerUserStoreSetInPlugin; + } + + public void setRangerUserStoreSetInPlugin(boolean rangerUserStoreSetInPlugin) { + this.rangerUserStoreSetInPlugin = rangerUserStoreSetInPlugin; + } + + public void loadUserStore(RangerBasePlugin plugIn) { + + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerUserStoreProvider(serviceName= " + serviceName + " serviceType= " + serviceType +").loadUserGroups()"); + } + + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_POLICYENGINE_INIT_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_POLICYENGINE_INIT_LOG, "RangerUserStoreProvider.loadUserGroups(serviceName=" + serviceName + ")"); + long freeMemory = Runtime.getRuntime().freeMemory(); + long totalMemory = Runtime.getRuntime().totalMemory(); + PERF_POLICYENGINE_INIT_LOG.debug("In-Use memory: " + (totalMemory-freeMemory) + ", Free memory:" + freeMemory); + } + + try { + //load userGroupRoles from ranger admin + RangerUserStore userStore = loadUserStoreFromAdmin(); + + if (userStore == null) { + //if userGroupRoles fetch from ranger Admin Fails, load from cache + if (!rangerUserStoreSetInPlugin) { + userStore = loadUserStoreFromCache(); + } + } + + if (PERF_POLICYENGINE_INIT_LOG.isDebugEnabled()) { + long freeMemory = Runtime.getRuntime().freeMemory(); + long totalMemory = Runtime.getRuntime().totalMemory(); + PERF_POLICYENGINE_INIT_LOG.debug("In-Use memory: " + (totalMemory - freeMemory) + ", Free memory:" + freeMemory); + } + + if (userStore != null) { + plugIn.setUserStore(userStore); + rangerUserStoreSetInPlugin = true; + setLastActivationTimeInMillis(System.currentTimeMillis()); + lastKnownUserStoreVersion = userStore.getUserStoreVersion() == null ? -1 : userStore.getUserStoreVersion(); + lastUpdateTimeInMillis = userStore.getUserStoreUpdateTime() == null ? -1 : userStore.getUserStoreUpdateTime().getTime(); + } else { + if (!rangerUserStoreSetInPlugin && !serviceDefSetInPlugin) { + plugIn.setUserStore(userStore); + serviceDefSetInPlugin = true; + } + } + } catch (RangerServiceNotFoundException snfe) { + if (disableCacheIfServiceNotFound) { + disableCache(); + plugIn.setUserStore(null); + setLastActivationTimeInMillis(System.currentTimeMillis()); + lastKnownUserStoreVersion = -1L; + lastUpdateTimeInMillis = -1L; + serviceDefSetInPlugin = true; + } + } catch (Exception excp) { + LOG.error("Encountered unexpected exception, ignoring..", excp); + } + + RangerPerfTracer.log(perf); + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerUserStoreProvider(serviceName=" + serviceName + ").loadUserGroups()"); + } + } + + private RangerUserStore loadUserStoreFromAdmin() throws RangerServiceNotFoundException { + + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerUserStoreProvider(serviceName=" + serviceName + ").loadUserStoreFromAdmin()"); + } + + RangerUserStore userStore = null; + + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_POLICYENGINE_INIT_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_POLICYENGINE_INIT_LOG, "RangerUserStoreProvider.loadUserStoreFromAdmin(serviceName=" + serviceName + ")"); + } + + try { + if ("atlas".equals(serviceName)) { + userStore = keycloakUserStore.loadUserStoreIfUpdated(lastUpdateTimeInMillis); + } else { + userStore = atlasAuthAdminClient.getUserStoreIfUpdated(lastUpdateTimeInMillis); + } + + boolean isUpdated = userStore != null; + + if(isUpdated) { + + long newVersion = userStore.getUserStoreVersion() == null ? -1 : userStore.getUserStoreVersion().longValue(); + saveToCache(userStore); + LOG.info("RangerUserStoreProvider(serviceName=" + serviceName + "): found updated version. lastKnownUserStoreVersion=" + lastKnownUserStoreVersion + "; newVersion=" + newVersion ); + } else { + if(LOG.isDebugEnabled()) { + LOG.debug("RangerUserStoreProvider(serviceName=" + serviceName + ").run(): no update found. lastKnownUserStoreVersion=" + lastKnownUserStoreVersion ); + } + } + } catch (Exception excp) { + LOG.error("RangerUserStoreProvider(serviceName=" + serviceName + "): failed to refresh userStore. Will continue to use last known version of userStore (" + "lastKnowRoleVersion= " + lastKnownUserStoreVersion, excp); + userStore = null; + } + + RangerPerfTracer.log(perf); + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerUserStoreProvider(serviceName=" + serviceName + " serviceType= " + serviceType + " ).loadUserStoreFromAdmin()"); + } + + return userStore; + } + + private RangerUserStore loadUserStoreFromCache() { + + RangerUserStore userStore = null; + + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerUserStoreProvider(serviceName=" + serviceName + ").loadUserStoreFromCache()"); + } + + File cacheFile = cacheDir == null ? null : new File(cacheDir + File.separator + cacheFileName); + + if (cacheFile != null && cacheFile.isFile() && cacheFile.canRead()) { + Reader reader = null; + + RangerPerfTracer perf = null; + + if (RangerPerfTracer.isPerfTraceEnabled(PERF_POLICYENGINE_INIT_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_POLICYENGINE_INIT_LOG, "RangerUserStoreProvider.loadUserStoreFromCache(serviceName=" + serviceName + ")"); + } + + try { + reader = new FileReader(cacheFile); + + userStore = gson.fromJson(reader, RangerUserStore.class); + + if (userStore != null) { + if (!StringUtils.equals(serviceName, userStore.getServiceName())) { + LOG.warn("ignoring unexpected serviceName '" + userStore.getServiceName() + "' in cache file '" + cacheFile.getAbsolutePath() + "'"); + + userStore.setServiceName(serviceName); + } + + lastKnownUserStoreVersion = userStore.getUserStoreVersion() == null ? -1 : userStore.getUserStoreVersion().longValue(); + lastUpdateTimeInMillis = userStore.getUserStoreUpdateTime() == null ? -1 : userStore.getUserStoreUpdateTime().getTime(); + } + } catch (Exception excp) { + LOG.error("failed to load userStore from cache file " + cacheFile.getAbsolutePath(), excp); + } finally { + RangerPerfTracer.log(perf); + + if (reader != null) { + try { + reader.close(); + } catch (Exception excp) { + LOG.error("error while closing opened cache file " + cacheFile.getAbsolutePath(), excp); + } + } + } + } else { + userStore = new RangerUserStore(); + userStore.setServiceName(serviceName); + userStore.setUserStoreVersion(-1L); + userStore.setUserStoreUpdateTime(null); + userStore.setUserGroupMapping(new HashMap>()); + saveToCache(userStore); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerUserStoreProvider(serviceName=" + serviceName + ").RangerUserStoreProvider()"); + } + + return userStore; + } + + public void saveToCache(RangerUserStore userStore) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerUserStoreProvider(serviceName=" + serviceName + ").saveToCache()"); + } + + if(userStore != null) { + File cacheFile = null; + if (cacheDir != null) { + // Create the cacheDir if it doesn't already exist + File cacheDirTmp = new File(cacheDir); + if (cacheDirTmp.exists()) { + cacheFile = new File(cacheDir + File.separator + cacheFileName); + } else { + try { + cacheDirTmp.mkdirs(); + cacheFile = new File(cacheDir + File.separator + cacheFileName); + } catch (SecurityException ex) { + LOG.error("Cannot create cache directory", ex); + } + } + } + + if(cacheFile != null) { + + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_POLICYENGINE_INIT_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_POLICYENGINE_INIT_LOG, "RangerUserStoreProvider.saveToCache(serviceName=" + serviceName + ")"); + } + + Writer writer = null; + + try { + writer = new FileWriter(cacheFile); + + gson.toJson(userStore, writer); + } catch (Exception excp) { + LOG.error("failed to save userStore to cache file '" + cacheFile.getAbsolutePath() + "'", excp); + } finally { + if(writer != null) { + try { + writer.close(); + } catch(Exception excp) { + LOG.error("error while closing opened cache file '" + cacheFile.getAbsolutePath() + "'", excp); + } + } + } + + RangerPerfTracer.log(perf); + } + } else { + LOG.info("userStore is null. Nothing to save in cache"); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerUserStoreProvider.saveToCache(serviceName=" + serviceName + ")"); + } + } + + private void disableCache() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerUserStoreProvider.disableCache(serviceName=" + serviceName + ")"); + } + + File cacheFile = cacheDir == null ? null : new File(cacheDir + File.separator + cacheFileName); + + if(cacheFile != null && cacheFile.isFile() && cacheFile.canRead()) { + LOG.warn("Cleaning up local RangerUserStore cache"); + String renamedCacheFile = cacheFile.getAbsolutePath() + "_" + System.currentTimeMillis(); + if (!cacheFile.renameTo(new File(renamedCacheFile))) { + LOG.error("Failed to move " + cacheFile.getAbsolutePath() + " to " + renamedCacheFile); + } else { + LOG.warn("Moved " + cacheFile.getAbsolutePath() + " to " + renamedCacheFile); + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("No local RangerRoles cache found. No need to disable it!"); + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerUserStoreProvider.disableCache(serviceName=" + serviceName + ")"); + } + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerUserStoreUtil.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerUserStoreUtil.java new file mode 100644 index 00000000000..4e3bf599fb4 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerUserStoreUtil.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.util; + +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang.StringUtils; + +import java.util.Map; + +public class RangerUserStoreUtil { + public static final String CLOUD_IDENTITY_NAME = "cloud_id"; + + public static String getPrintableOptions(Map otherAttributes) { + if (MapUtils.isEmpty(otherAttributes)) return "{}"; + StringBuilder ret = new StringBuilder(); + ret.append("{"); + for (Map.Entry entry : otherAttributes.entrySet()) { + ret.append(entry.getKey()).append(", ").append("[").append(entry.getValue()).append("]").append(","); + } + ret.append("}"); + return ret.toString(); + } + + public static String getAttrVal(Map> attrMap, String name, String attrName) { + String ret = null; + + if (StringUtils.isNotEmpty(name) && StringUtils.isNotEmpty(attrName)) { + Map attrs = attrMap.get(name); + if (MapUtils.isNotEmpty(attrs)) { + ret = attrs.get(attrName); + } + } + return ret; + } + + public String getCloudId(Map> attrMap, String name) { + String cloudId = null; + cloudId = getAttrVal(attrMap, name, CLOUD_IDENTITY_NAME); + return cloudId; + } +} + + diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/SearchFilter.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/SearchFilter.java new file mode 100644 index 00000000000..87e7ed58356 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/SearchFilter.java @@ -0,0 +1,253 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.util; + +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang.StringUtils; + +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + + +public class SearchFilter { + public static final String SERVICE_TYPE = "serviceType"; // search, sort + public static final String SERVICE_TYPE_ID = "serviceTypeId"; // search, sort + public static final String SERVICE_NAME = "serviceName"; // search, sort + public static final String SERVICE_ID = "serviceId"; // search, sort + public static final String POLICY_NAME = "policyName"; // search, sort + public static final String POLICY_ID = "policyId"; // search, sort + public static final String IS_ENABLED = "isEnabled"; // search + public static final String IS_RECURSIVE = "isRecursive"; // search + public static final String TAG_SERVICE_NAME = "tagServiceName"; // search + public static final String TAG_SERVICE_ID = "tagServiceId"; // search + public static final String USER = "user"; // search + public static final String GROUP = "group"; // search + public static final String ROLE = "role"; // search + public static final String RESOURCE_PREFIX = "resource:"; // search + public static final String RESOURCE_MATCH_SCOPE = "resourceMatchScope"; // search - valid values: "self", "ancestor", "self_or_ancestor" + public static final String POL_RESOURCE = "polResource"; // search + public static final String POLICY_NAME_PARTIAL = "policyNamePartial"; // search, sort + public static final String CREATE_TIME = "createTime"; // sort + public static final String UPDATE_TIME = "updateTime"; // sort + public static final String START_INDEX = "startIndex"; + public static final String PAGE_SIZE = "pageSize"; + public static final String SORT_BY = "sortBy"; + public static final String RESOURCE_SIGNATURE = "resourceSignature:"; // search + public static final String POLICY_TYPE = "policyType"; // search + public static final String POLICY_PRIORITY = "policyPriority"; // search + public static final String GUID = "guid"; //search + public static final String POLICY_LABEL = "policyLabel"; // search + public static final String POLICY_LABELS_PARTIAL = "policyLabelsPartial"; // search + public static final String POLICY_LABEL_ID = "policyLabelId"; // search, sort + public static final String ZONE_ID = "zoneId"; // search, sort + public static final String ZONE_NAME = "zoneName"; // search, sort + public static final String ROLE_ID = "roleId"; // search, sort + public static final String ROLE_NAME = "roleName"; // search, sort + public static final String GROUP_NAME = "groupName"; // search, sort + public static final String USER_NAME = "userName"; // search, sort + public static final String ROLE_NAME_PARTIAL = "roleNamePartial"; // search + public static final String GROUP_NAME_PARTIAL = "groupNamePartial"; // search + public static final String USER_NAME_PARTIAL = "userNamePartial"; // search + + public static final String TAG_DEF_ID = "tagDefId"; // search + public static final String TAG_DEF_GUID = "tagDefGuid"; // search + public static final String TAG_TYPE = "tagType"; // search + public static final String TAG_ID = "tagId"; // search + public static final String TAG_GUID = "tagGuid"; // search + public static final String TAG_RESOURCE_ID = "resourceId"; // search + public static final String TAG_RESOURCE_GUID = "resourceGuid"; // search + public static final String TAG_RESOURCE_SERVICE_NAME = "resourceServiceName"; // search + public static final String TAG_RESOURCE_SIGNATURE = "resourceSignature"; // search + public static final String TAG_MAP_ID = "tagResourceMapId"; // search + public static final String TAG_MAP_GUID = "tagResourceMapGuid"; // search + + public static final String SERVICE_NAME_PARTIAL = "serviceNamePartial"; + + public static final String PLUGIN_HOST_NAME = "pluginHostName"; + public static final String PLUGIN_APP_TYPE = "pluginAppType"; + public static final String PLUGIN_ENTITY_TYPE = "pluginEntityType"; + public static final String PLUGIN_IP_ADDRESS = "pluginIpAddress"; + public static final String CLUSTER_NAME = "clusterName"; + public static final String FETCH_ZONE_UNZONE_POLICIES= "fetchZoneAndUnzonePolicies"; + public static final String FETCH_TAG_POLICIES = "fetchTagPolicies"; + public static final String FETCH_ZONE_NAME = "zoneName"; + public static final String FETCH_DENY_CONDITION = "denyCondition"; + + public static final String SERVICE_DISPLAY_NAME = "serviceDisplayName"; // search, sort + public static final String SERVICE_DISPLAY_NAME_PARTIAL = "serviceDisplayNamePartial"; // search + public static final String SERVICE_TYPE_DISPLAY_NAME = "serviceTypeDisplayName"; // search, sort + + private Map params; + private int startIndex; + private int maxRows = Integer.MAX_VALUE; + private boolean getCount = true; + private String sortBy; + private String sortType; + + public SearchFilter() { + this(null); + } + + public SearchFilter(String name, String value) { + setParam(name, value); + } + + public SearchFilter(Map values) { + setParams(values); + } + + public Map getParams() { + return params; + } + + public void setParams(Map params) { + this.params = params; + } + + public String getParam(String name) { + return params == null ? null : params.get(name); + } + + public void setParam(String name, String value) { + if(StringUtils.isEmpty(name) || StringUtils.isEmpty(value)) { + return; + } + + if(params == null) { + params = new HashMap(); + } + + params.put(name, value); + } + public void removeParam(String name) { + + params.remove(name); + } + + public Map getParamsWithPrefix(String prefix, boolean stripPrefix) { + Map ret = null; + + if(prefix == null) { + prefix = StringUtils.EMPTY; + } + + if(params != null) { + for(Map.Entry e : params.entrySet()) { + String name = e.getKey(); + + if(name.startsWith(prefix)) { + if(ret == null) { + ret = new HashMap<>(); + } + + if(stripPrefix) { + name = name.substring(prefix.length()); + } + + ret.put(name, e.getValue()); + } + } + } + + return ret; + } + + public boolean isEmpty() { + return MapUtils.isEmpty(params); + } + + public int getStartIndex() { + return startIndex; + } + + public void setStartIndex(int startIndex) { + this.startIndex = startIndex; + } + + public int getMaxRows() { + return maxRows; + } + + public void setMaxRows(int maxRows) { + this.maxRows = maxRows; + } + + public boolean isGetCount() { + return getCount; + } + + public void setGetCount(boolean getCount) { + this.getCount = getCount; + } + + public String getSortBy() { + return sortBy; + } + + public void setSortBy(String sortBy) { + this.sortBy = sortBy; + } + + public String getSortType() { + return sortType; + } + + public void setSortType(String sortType) { + this.sortType = sortType; + } + + @Override + public boolean equals(Object object) { + if (object == null || !(object instanceof SearchFilter)) { + return false; + } + SearchFilter that = (SearchFilter)object; + return Objects.equals(params, that.params); + } + + @Override + public int hashCode() { + return Objects.hash(params); + } + + @Override + public String toString( ) { + StringBuilder sb = new StringBuilder(); + + toString(sb); + + return sb.toString(); + } + + public StringBuilder toString(StringBuilder sb) { + sb.append("SearchFilter={"); + + sb.append("getCount={").append(getCount).append("} "); + sb.append("maxRows={").append(maxRows).append("} "); + sb.append("params={").append(params).append("} "); + sb.append("sortBy={").append(sortBy).append("} "); + sb.append("sortType={").append(sortType).append("} "); + sb.append("startIndex={").append(startIndex).append("} "); + sb.append("}"); + + return sb; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/ServiceDefUtil.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/ServiceDefUtil.java new file mode 100644 index 00000000000..39e0e236e82 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/ServiceDefUtil.java @@ -0,0 +1,469 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.util; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.atlas.plugin.model.RangerPolicy; +import org.apache.atlas.plugin.model.RangerServiceDef; +import org.apache.atlas.plugin.model.RangerServiceDef.RangerAccessTypeDef; +import org.apache.atlas.plugin.model.RangerServiceDef.RangerDataMaskTypeDef; +import org.apache.atlas.plugin.model.RangerServiceDef.RangerResourceDef; +import org.apache.atlas.plugin.policyengine.RangerPluginContext; +import org.apache.atlas.plugin.store.AbstractServiceStore; +import org.apache.atlas.plugin.store.EmbeddedServiceDefsUtil; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; + +public class ServiceDefUtil { + + public static boolean getOption_enableDenyAndExceptionsInPolicies(RangerServiceDef serviceDef, RangerPluginContext pluginContext) { + boolean ret = false; + + if(serviceDef != null) { + Configuration config = pluginContext != null ? pluginContext.getConfig() : null; + boolean enableDenyAndExceptionsInPoliciesHiddenOption = config == null || config.getBoolean("ranger.servicedef.enableDenyAndExceptionsInPolicies", true); + boolean defaultValue = enableDenyAndExceptionsInPoliciesHiddenOption || StringUtils.equalsIgnoreCase(serviceDef.getName(), EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_TAG_NAME); + + ret = ServiceDefUtil.getBooleanValue(serviceDef.getOptions(), RangerServiceDef.OPTION_ENABLE_DENY_AND_EXCEPTIONS_IN_POLICIES, defaultValue); + } + + return ret; + } + + public static RangerDataMaskTypeDef getDataMaskType(RangerServiceDef serviceDef, String typeName) { + RangerDataMaskTypeDef ret = null; + + if(serviceDef != null && serviceDef.getDataMaskDef() != null) { + List maskTypes = serviceDef.getDataMaskDef().getMaskTypes(); + + if(CollectionUtils.isNotEmpty(maskTypes)) { + for(RangerDataMaskTypeDef maskType : maskTypes) { + if(StringUtils.equals(maskType.getName(), typeName)) { + ret = maskType; + break; + } + } + } + } + + return ret; + } + + public static RangerServiceDef normalize(RangerServiceDef serviceDef) { + normalizeDataMaskDef(serviceDef); + normalizeRowFilterDef(serviceDef); + + return serviceDef; + } + + public static RangerResourceDef getResourceDef(RangerServiceDef serviceDef, String resource) { + RangerResourceDef ret = null; + + if(serviceDef != null && resource != null && CollectionUtils.isNotEmpty(serviceDef.getResources())) { + for(RangerResourceDef resourceDef : serviceDef.getResources()) { + if(StringUtils.equalsIgnoreCase(resourceDef.getName(), resource)) { + ret = resourceDef; + break; + } + } + } + + return ret; + } + + public static Integer getLeafResourceLevel(RangerServiceDef serviceDef, Map policyResource) { + Integer ret = null; + + RangerResourceDef resourceDef = getLeafResourceDef(serviceDef, policyResource); + + if (resourceDef != null) { + ret = resourceDef.getLevel(); + } + + return ret; + } + + public static RangerResourceDef getLeafResourceDef(RangerServiceDef serviceDef, Map policyResource) { + RangerResourceDef ret = null; + + if(serviceDef != null && policyResource != null) { + for(Map.Entry entry : policyResource.entrySet()) { + if (!isEmpty(entry.getValue())) { + String resource = entry.getKey(); + RangerResourceDef resourceDef = ServiceDefUtil.getResourceDef(serviceDef, resource); + + if (resourceDef != null && resourceDef.getLevel() != null) { + if (ret == null) { + ret = resourceDef; + } else if(ret.getLevel() < resourceDef.getLevel()) { + ret = resourceDef; + } + } + } + } + } + + return ret; + } + + public static boolean isAncestorOf(RangerServiceDef serviceDef, RangerResourceDef ancestor, RangerResourceDef descendant) { + + boolean ret = false; + + if (ancestor != null && descendant != null) { + final String ancestorName = ancestor.getName(); + + for (RangerResourceDef node = descendant; node != null; node = ServiceDefUtil.getResourceDef(serviceDef, node.getParent())) { + if (StringUtils.equalsIgnoreCase(ancestorName, node.getParent())) { + ret = true; + break; + } + } + } + + return ret; + } + + public static boolean isEmpty(RangerPolicy.RangerPolicyResource policyResource) { + boolean ret = true; + if (policyResource != null) { + List resourceValues = policyResource.getValues(); + if (CollectionUtils.isNotEmpty(resourceValues)) { + for (String resourceValue : resourceValues) { + if (StringUtils.isNotBlank(resourceValue)) { + ret = false; + break; + } + } + } + } + return ret; + } + + public static String getOption(Map options, String name, String defaultValue) { + String ret = options != null && name != null ? options.get(name) : null; + + if(ret == null) { + ret = defaultValue; + } + + return ret; + } + + public static boolean getBooleanOption(Map options, String name, boolean defaultValue) { + String val = getOption(options, name, null); + + return val == null ? defaultValue : Boolean.parseBoolean(val); + } + + public static char getCharOption(Map options, String name, char defaultValue) { + String val = getOption(options, name, null); + + return StringUtils.isEmpty(val) ? defaultValue : val.charAt(0); + } + + public static RangerServiceDef normalizeAccessTypeDefs(RangerServiceDef serviceDef, final String componentType) { + + if (serviceDef != null && StringUtils.isNotBlank(componentType)) { + + List accessTypeDefs = serviceDef.getAccessTypes(); + + if (CollectionUtils.isNotEmpty(accessTypeDefs)) { + + String prefix = componentType + AbstractServiceStore.COMPONENT_ACCESSTYPE_SEPARATOR; + + List unneededAccessTypeDefs = null; + + for (RangerServiceDef.RangerAccessTypeDef accessTypeDef : accessTypeDefs) { + + String accessType = accessTypeDef.getName(); + + if (StringUtils.startsWith(accessType, prefix)) { + + String newAccessType = StringUtils.removeStart(accessType, prefix); + + accessTypeDef.setName(newAccessType); + + Collection impliedGrants = accessTypeDef.getImpliedGrants(); + + if (CollectionUtils.isNotEmpty(impliedGrants)) { + + Collection newImpliedGrants = null; + + for (String impliedGrant : impliedGrants) { + + if (StringUtils.startsWith(impliedGrant, prefix)) { + + String newImpliedGrant = StringUtils.removeStart(impliedGrant, prefix); + + if (newImpliedGrants == null) { + newImpliedGrants = new ArrayList<>(); + } + + newImpliedGrants.add(newImpliedGrant); + } + } + accessTypeDef.setImpliedGrants(newImpliedGrants); + + } + } else if (StringUtils.contains(accessType, AbstractServiceStore.COMPONENT_ACCESSTYPE_SEPARATOR)) { + if(unneededAccessTypeDefs == null) { + unneededAccessTypeDefs = new ArrayList<>(); + } + + unneededAccessTypeDefs.add(accessTypeDef); + } + } + + if(unneededAccessTypeDefs != null) { + accessTypeDefs.removeAll(unneededAccessTypeDefs); + } + } + } + + return serviceDef; + } + + private static void normalizeDataMaskDef(RangerServiceDef serviceDef) { + if(serviceDef != null && serviceDef.getDataMaskDef() != null) { + List dataMaskResources = serviceDef.getDataMaskDef().getResources(); + List dataMaskAccessTypes = serviceDef.getDataMaskDef().getAccessTypes(); + + if(CollectionUtils.isNotEmpty(dataMaskResources)) { + List resources = serviceDef.getResources(); + List processedDefs = new ArrayList(dataMaskResources.size()); + + for(RangerResourceDef dataMaskResource : dataMaskResources) { + RangerResourceDef processedDef = dataMaskResource; + + for(RangerResourceDef resourceDef : resources) { + if(StringUtils.equals(resourceDef.getName(), dataMaskResource.getName())) { + processedDef = ServiceDefUtil.mergeResourceDef(resourceDef, dataMaskResource); + break; + } + } + + processedDefs.add(processedDef); + } + + serviceDef.getDataMaskDef().setResources(processedDefs); + } + + if(CollectionUtils.isNotEmpty(dataMaskAccessTypes)) { + List accessTypes = serviceDef.getAccessTypes(); + List processedDefs = new ArrayList(accessTypes.size()); + + for(RangerAccessTypeDef dataMaskAccessType : dataMaskAccessTypes) { + RangerAccessTypeDef processedDef = dataMaskAccessType; + + for(RangerAccessTypeDef accessType : accessTypes) { + if(StringUtils.equals(accessType.getName(), dataMaskAccessType.getName())) { + processedDef = ServiceDefUtil.mergeAccessTypeDef(accessType, dataMaskAccessType); + break; + } + } + + processedDefs.add(processedDef); + } + + serviceDef.getDataMaskDef().setAccessTypes(processedDefs); + } + } + } + + private static void normalizeRowFilterDef(RangerServiceDef serviceDef) { + if(serviceDef != null && serviceDef.getRowFilterDef() != null) { + List rowFilterResources = serviceDef.getRowFilterDef().getResources(); + List rowFilterAccessTypes = serviceDef.getRowFilterDef().getAccessTypes(); + + if(CollectionUtils.isNotEmpty(rowFilterResources)) { + List resources = serviceDef.getResources(); + List processedDefs = new ArrayList(rowFilterResources.size()); + + for(RangerResourceDef rowFilterResource : rowFilterResources) { + RangerResourceDef processedDef = rowFilterResource; + + for(RangerResourceDef resourceDef : resources) { + if(StringUtils.equals(resourceDef.getName(), rowFilterResource.getName())) { + processedDef = ServiceDefUtil.mergeResourceDef(resourceDef, rowFilterResource); + break; + } + } + + processedDefs.add(processedDef); + } + + serviceDef.getRowFilterDef().setResources(processedDefs); + } + + if(CollectionUtils.isNotEmpty(rowFilterAccessTypes)) { + List accessTypes = serviceDef.getAccessTypes(); + List processedDefs = new ArrayList(accessTypes.size()); + + for(RangerAccessTypeDef rowFilterAccessType : rowFilterAccessTypes) { + RangerAccessTypeDef processedDef = rowFilterAccessType; + + for(RangerAccessTypeDef accessType : accessTypes) { + if(StringUtils.equals(accessType.getName(), rowFilterAccessType.getName())) { + processedDef = ServiceDefUtil.mergeAccessTypeDef(accessType, rowFilterAccessType); + break; + } + } + + processedDefs.add(processedDef); + } + + serviceDef.getRowFilterDef().setAccessTypes(processedDefs); + } + } + } + + private static RangerResourceDef mergeResourceDef(RangerResourceDef base, RangerResourceDef delta) { + RangerResourceDef ret = new RangerResourceDef(base); + + // retain base values for: itemId, name, type, level, parent, lookupSupported + + if(Boolean.TRUE.equals(delta.getMandatory())) + ret.setMandatory(delta.getMandatory()); + + if(delta.getRecursiveSupported() != null) + ret.setRecursiveSupported(delta.getRecursiveSupported()); + + if(delta.getExcludesSupported() != null) + ret.setExcludesSupported(delta.getExcludesSupported()); + + if(StringUtils.isNotEmpty(delta.getMatcher())) + ret.setMatcher(delta.getMatcher()); + + if(MapUtils.isNotEmpty(delta.getMatcherOptions())) { + if(ret.getMatcherOptions() == null) { + ret.setMatcherOptions(new HashMap()); + } + + for(Map.Entry e : delta.getMatcherOptions().entrySet()) { + ret.getMatcherOptions().put(e.getKey(), e.getValue()); + } + } + + if(StringUtils.isNotEmpty(delta.getValidationRegEx())) + ret.setValidationRegEx(delta.getValidationRegEx()); + + if(StringUtils.isNotEmpty(delta.getValidationMessage())) + ret.setValidationMessage(delta.getValidationMessage()); + + ret.setUiHint(delta.getUiHint()); + + if(StringUtils.isNotEmpty(delta.getLabel())) + ret.setLabel(delta.getLabel()); + + if(StringUtils.isNotEmpty(delta.getDescription())) + ret.setDescription(delta.getDescription()); + + if(StringUtils.isNotEmpty(delta.getRbKeyLabel())) + ret.setRbKeyLabel(delta.getRbKeyLabel()); + + if(StringUtils.isNotEmpty(delta.getRbKeyDescription())) + ret.setRbKeyDescription(delta.getRbKeyDescription()); + + if(StringUtils.isNotEmpty(delta.getRbKeyValidationMessage())) + ret.setRbKeyValidationMessage(delta.getRbKeyValidationMessage()); + + if(CollectionUtils.isNotEmpty(delta.getAccessTypeRestrictions())) + ret.setAccessTypeRestrictions(delta.getAccessTypeRestrictions()); + + boolean copyLeafValue = false; + if (ret.getIsValidLeaf() != null) { + if (!ret.getIsValidLeaf().equals(delta.getIsValidLeaf())) { + copyLeafValue = true; + } + } else { + if (delta.getIsValidLeaf() != null) { + copyLeafValue = true; + } + } + if (copyLeafValue) + ret.setIsValidLeaf(delta.getIsValidLeaf()); + + return ret; + } + + private static RangerAccessTypeDef mergeAccessTypeDef(RangerAccessTypeDef base, RangerAccessTypeDef delta) { + RangerAccessTypeDef ret = new RangerAccessTypeDef(base); + + // retain base values for: itemId, name, impliedGrants + + if(StringUtils.isNotEmpty(delta.getLabel())) + ret.setLabel(delta.getLabel()); + + if(StringUtils.isNotEmpty(delta.getRbKeyLabel())) + ret.setRbKeyLabel(delta.getRbKeyLabel()); + + return ret; + } + + private static boolean getBooleanValue(Map map, String elementName, boolean defaultValue) { + boolean ret = defaultValue; + + if(MapUtils.isNotEmpty(map) && map.containsKey(elementName)) { + String elementValue = map.get(elementName); + + if(StringUtils.isNotEmpty(elementValue)) { + ret = Boolean.valueOf(elementValue.toString()); + } + } + + return ret; + } + + public static Map> getExpandedImpliedGrants(RangerServiceDef serviceDef) { + Map> ret = new HashMap<>(); + + if(serviceDef != null && !CollectionUtils.isEmpty(serviceDef.getAccessTypes())) { + for(RangerAccessTypeDef accessTypeDef : serviceDef.getAccessTypes()) { + if(!CollectionUtils.isEmpty(accessTypeDef.getImpliedGrants())) { + + Collection impliedAccessGrants = ret.get(accessTypeDef.getName()); + + if(impliedAccessGrants == null) { + impliedAccessGrants = new HashSet<>(); + + ret.put(accessTypeDef.getName(), impliedAccessGrants); + } + + impliedAccessGrants.addAll(accessTypeDef.getImpliedGrants()); + impliedAccessGrants.add(accessTypeDef.getName()); + } else { + ret.put(accessTypeDef.getName(), new HashSet<>(Collections.singleton(accessTypeDef.getName()))); + } + } + } + return ret; + } + +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/ServicePolicies.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/ServicePolicies.java new file mode 100644 index 00000000000..547349c8f51 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/ServicePolicies.java @@ -0,0 +1,458 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.util; + + +import org.apache.commons.collections.MapUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.htrace.shaded.fasterxml.jackson.annotation.JsonInclude; +import org.apache.atlas.plugin.model.RangerPolicy; +import org.apache.atlas.plugin.model.RangerPolicyDelta; +import org.apache.atlas.plugin.model.RangerServiceDef; +import org.apache.atlas.plugin.policyengine.RangerPolicyEngine; +import org.apache.atlas.plugin.policyengine.RangerPolicyEngineImpl; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.Arrays; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class ServicePolicies implements java.io.Serializable { + private static final long serialVersionUID = 1L; + private static final Log LOG = LogFactory.getLog(ServicePolicies.class); + + private String serviceName; + private String serviceId; + private Long policyVersion; + private Date policyUpdateTime; + private List policies; + private RangerServiceDef serviceDef; + private String auditMode = RangerPolicyEngine.AUDIT_DEFAULT; + private TagPolicies tagPolicies; + private Map securityZones; + private List policyDeltas; + private Map serviceConfig; + + /** + * @return the serviceName + */ + public String getServiceName() { + return serviceName; + } + /** + * @param serviceName the serviceName to set + */ + public void setServiceName(String serviceName) { + this.serviceName = serviceName; + } + /** + * @return the serviceId + */ + public String getServiceId() { + return serviceId; + } + /** + * @param serviceId the serviceId to set + */ + public void setServiceId(String serviceId) { + this.serviceId = serviceId; + } + /** + * @return the policyVersion + */ + public Long getPolicyVersion() { + return policyVersion; + } + /** + * @param policyVersion the policyVersion to set + */ + public void setPolicyVersion(Long policyVersion) { + this.policyVersion = policyVersion; + } + /** + * @return the policyUpdateTime + */ + public Date getPolicyUpdateTime() { + return policyUpdateTime; + } + /** + * @param policyUpdateTime the policyUpdateTime to set + */ + public void setPolicyUpdateTime(Date policyUpdateTime) { + this.policyUpdateTime = policyUpdateTime; + } + + public Map getServiceConfig() { + return serviceConfig; + } + public void setServiceConfig(Map serviceConfig) { + this.serviceConfig = serviceConfig; + } + + /** + * @return the policies + */ + public List getPolicies() { + return policies; + } + /** + * @param policies the policies to set + */ + public void setPolicies(List policies) { + this.policies = policies; + } + /** + * @return the serviceDef + */ + public RangerServiceDef getServiceDef() { + return serviceDef; + } + /** + * @param serviceDef the serviceDef to set + */ + public void setServiceDef(RangerServiceDef serviceDef) { + this.serviceDef = serviceDef; + } + + public String getAuditMode() { + return auditMode; + } + + public void setAuditMode(String auditMode) { + this.auditMode = auditMode; + } + /** + * @return the tagPolicies + */ + public TagPolicies getTagPolicies() { + return tagPolicies; + } + /** + * @param tagPolicies the tagPolicies to set + */ + public void setTagPolicies(TagPolicies tagPolicies) { + this.tagPolicies = tagPolicies; + } + + public Map getSecurityZones() { return securityZones; } + + public void setSecurityZones(Map securityZones) { + this.securityZones = securityZones; + } + + @Override + public String toString() { + return "serviceName=" + serviceName + ", " + + "serviceId=" + serviceId + ", " + + "policyVersion=" + policyVersion + ", " + + "policyUpdateTime=" + policyUpdateTime + ", " + + "policies=" + policies + ", " + + "tagPolicies=" + tagPolicies + ", " + + "policyDeltas=" + policyDeltas + ", " + + "serviceDef=" + serviceDef + ", " + + "auditMode=" + auditMode + ", " + + "securityZones=" + securityZones + ; + } + public List getPolicyDeltas() { return this.policyDeltas; } + + public void setPolicyDeltas(List policyDeltas) { this.policyDeltas = policyDeltas; } + + @JsonInclude(JsonInclude.Include.NON_NULL) + @XmlRootElement + @XmlAccessorType(XmlAccessType.FIELD) + public static class TagPolicies implements java.io.Serializable { + private static final long serialVersionUID = 1L; + + private String serviceName; + private String serviceId; + private Long policyVersion; + private Date policyUpdateTime; + private List policies; + private RangerServiceDef serviceDef; + private String auditMode = RangerPolicyEngine.AUDIT_DEFAULT; + private Map serviceConfig; + + /** + * @return the serviceName + */ + public String getServiceName() { + return serviceName; + } + /** + * @param serviceName the serviceName to set + */ + public void setServiceName(String serviceName) { + this.serviceName = serviceName; + } + /** + * @return the serviceId + */ + public String getServiceId() { + return serviceId; + } + /** + * @param serviceId the serviceId to set + */ + public void setServiceId(String serviceId) { + this.serviceId = serviceId; + } + /** + * @return the policyVersion + */ + public Long getPolicyVersion() { + return policyVersion; + } + /** + * @param policyVersion the policyVersion to set + */ + public void setPolicyVersion(Long policyVersion) { + this.policyVersion = policyVersion; + } + /** + * @return the policyUpdateTime + */ + public Date getPolicyUpdateTime() { + return policyUpdateTime; + } + /** + * @param policyUpdateTime the policyUpdateTime to set + */ + public void setPolicyUpdateTime(Date policyUpdateTime) { + this.policyUpdateTime = policyUpdateTime; + } + /** + * @return the policies + */ + public List getPolicies() { + return policies; + } + /** + * @param policies the policies to set + */ + public void setPolicies(List policies) { + this.policies = policies; + } + /** + * @return the serviceDef + */ + public RangerServiceDef getServiceDef() { + return serviceDef; + } + /** + * @param serviceDef the serviceDef to set + */ + public void setServiceDef(RangerServiceDef serviceDef) { + this.serviceDef = serviceDef; + } + + public String getAuditMode() { + return auditMode; + } + + public void setAuditMode(String auditMode) { + this.auditMode = auditMode; + } + + public Map getServiceConfig() { + return serviceConfig; + } + + public void setServiceConfig(Map serviceConfig) { + this.serviceConfig = serviceConfig; + } + + @Override + public String toString() { + return "serviceName=" + serviceName + ", " + + "serviceId=" + serviceId + ", " + + "policyVersion=" + policyVersion + ", " + + "policyUpdateTime=" + policyUpdateTime + ", " + + "policies=" + policies + ", " + + "serviceDef=" + serviceDef + ", " + + "auditMode=" + auditMode + + "serviceConfig=" + serviceConfig + ; + } + } + + @JsonInclude(JsonInclude.Include.NON_NULL) + @XmlRootElement + @XmlAccessorType(XmlAccessType.FIELD) + public static class SecurityZoneInfo implements java.io.Serializable { + private static final long serialVersionUID = 1L; + private String zoneName; + private List>> resources; + private List policies; + private List policyDeltas; + private Boolean containsAssociatedTagService; + + public String getZoneName() { + return zoneName; + } + + public List>> getResources() { + return resources; + } + + public List getPolicies() { + return policies; + } + + public List getPolicyDeltas() { return policyDeltas; } + + public Boolean getContainsAssociatedTagService() { return containsAssociatedTagService; } + + public void setZoneName(String zoneName) { + this.zoneName = zoneName; + } + + public void setResources(List>> resources) { + this.resources = resources; + } + + public void setPolicies(List policies) { + this.policies = policies; + } + + public void setPolicyDeltas(List policyDeltas) { this.policyDeltas = policyDeltas; } + + public void setContainsAssociatedTagService(Boolean containsAssociatedTagService) { this.containsAssociatedTagService = containsAssociatedTagService; } + + @Override + public String toString() { + return "zoneName=" + zoneName + ", " + + "resources=" + resources + ", " + + "policies=" + policies + ", " + + "policyDeltas=" + policyDeltas + ", " + + "containsAssociatedTagService=" + containsAssociatedTagService + ; + } + } + + static public ServicePolicies copyHeader(ServicePolicies source) { + ServicePolicies ret = new ServicePolicies(); + + ret.setServiceName(source.getServiceName()); + ret.setServiceId(source.getServiceId()); + ret.setPolicyVersion(source.getPolicyVersion()); + ret.setAuditMode(source.getAuditMode()); + ret.setServiceDef(source.getServiceDef()); + ret.setPolicyUpdateTime(source.getPolicyUpdateTime()); + ret.setSecurityZones(source.getSecurityZones()); + ret.setPolicies(Collections.emptyList()); + ret.setPolicyDeltas(null); + if (source.getTagPolicies() != null) { + TagPolicies tagPolicies = copyHeader(source.getTagPolicies(), source.getServiceDef().getName()); + ret.setTagPolicies(tagPolicies); + } + + return ret; + } + + static public TagPolicies copyHeader(TagPolicies source, String componentServiceName) { + TagPolicies ret = new TagPolicies(); + + ret.setServiceName(source.getServiceName()); + ret.setServiceId(source.getServiceId()); + ret.setPolicyVersion(source.getPolicyVersion()); + ret.setAuditMode(source.getAuditMode()); + ret.setServiceDef(ServiceDefUtil.normalizeAccessTypeDefs(source.getServiceDef(), componentServiceName)); + ret.setPolicyUpdateTime(source.getPolicyUpdateTime()); + ret.setPolicies(Collections.emptyList()); + + return ret; + } + + public static ServicePolicies applyDelta(final ServicePolicies servicePolicies, RangerPolicyEngineImpl policyEngine) { + ServicePolicies ret = copyHeader(servicePolicies); + + List oldResourcePolicies = policyEngine.getResourcePolicies(); + List oldTagPolicies = policyEngine.getTagPolicies(); + + List newResourcePolicies = RangerPolicyDeltaUtil.applyDeltas(oldResourcePolicies, servicePolicies.getPolicyDeltas(), servicePolicies.getServiceDef().getName()); + + ret.setPolicies(newResourcePolicies); + + final List newTagPolicies; + if (servicePolicies.getTagPolicies() != null) { + if (LOG.isDebugEnabled()) { + LOG.debug("applyingDeltas for tag policies"); + } + newTagPolicies = RangerPolicyDeltaUtil.applyDeltas(oldTagPolicies, servicePolicies.getPolicyDeltas(), servicePolicies.getTagPolicies().getServiceDef().getName()); + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("No need to apply deltas for tag policies"); + } + newTagPolicies = oldTagPolicies; + } + + if (LOG.isDebugEnabled()) { + LOG.debug("New tag policies:[" + Arrays.toString(newTagPolicies.toArray()) + "]"); + } + + if (ret.getTagPolicies() != null) { + ret.getTagPolicies().setPolicies(newTagPolicies); + } + + if (MapUtils.isNotEmpty(servicePolicies.getSecurityZones())) { + Map newSecurityZones = new HashMap<>(); + + for (Map.Entry entry : servicePolicies.getSecurityZones().entrySet()) { + String zoneName = entry.getKey(); + SecurityZoneInfo zoneInfo = entry.getValue(); + + List zoneResourcePolicies = policyEngine.getResourcePolicies(zoneName); + // There are no separate tag-policy-repositories for each zone + + if (LOG.isDebugEnabled()) { + LOG.debug("Applying deltas for security-zone:[" + zoneName + "]"); + } + + final List newZonePolicies = RangerPolicyDeltaUtil.applyDeltas(zoneResourcePolicies, zoneInfo.getPolicyDeltas(), servicePolicies.getServiceDef().getName()); + + if (LOG.isDebugEnabled()) { + LOG.debug("New resource policies for security-zone:[" + zoneName + "], zoneResourcePolicies:[" + Arrays.toString(newZonePolicies.toArray())+ "]"); + } + + SecurityZoneInfo newZoneInfo = new SecurityZoneInfo(); + + newZoneInfo.setZoneName(zoneName); + newZoneInfo.setResources(zoneInfo.getResources()); + newZoneInfo.setPolicies(newZonePolicies); + + newSecurityZones.put(zoneName, newZoneInfo); + } + + ret.setSecurityZones(newSecurityZones); + } + + return ret; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/ServiceTags.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/ServiceTags.java new file mode 100644 index 00000000000..a57d4d26004 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/ServiceTags.java @@ -0,0 +1,206 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.util; + + +import org.apache.htrace.shaded.fasterxml.jackson.annotation.JsonInclude; +import org.apache.atlas.plugin.model.RangerServiceResource; +import org.apache.atlas.plugin.model.RangerTag; +import org.apache.atlas.plugin.model.RangerTagDef; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class ServiceTags implements java.io.Serializable { + private static final long serialVersionUID = 1L; + + public static final String OP_ADD_OR_UPDATE = "add_or_update"; + public static final String OP_DELETE = "delete"; + public static final String OP_REPLACE = "replace"; + + public enum TagsChangeExtent { NONE, TAGS, SERVICE_RESOURCE, ALL } + public enum TagsChangeType { NONE, SERVICE_RESOURCE_UPDATE, TAG_UPDATE, TAG_RESOURCE_MAP_UPDATE, RANGER_ADMIN_START, INVALIDATE_TAG_DELTAS, ALL } + + private String op = OP_ADD_OR_UPDATE; + private String serviceName; + private Long tagVersion; + private Date tagUpdateTime; + private Map tagDefinitions; + private Map tags; + private List serviceResources; + private Map> resourceToTagIds; + private Boolean isDelta; + private TagsChangeExtent tagsChangeExtent; + + public ServiceTags() { + this(OP_ADD_OR_UPDATE, null, 0L, null, null, null, null, null); + } + + public ServiceTags(String op, String serviceName, Long tagVersion, Date tagUpdateTime, Map tagDefinitions, + Map tags, List serviceResources, Map> resourceToTagIds) { + this(op, serviceName, tagVersion, tagUpdateTime, tagDefinitions, tags, serviceResources, resourceToTagIds, false, TagsChangeExtent.ALL); + } + public ServiceTags(String op, String serviceName, Long tagVersion, Date tagUpdateTime, Map tagDefinitions, + Map tags, List serviceResources, Map> resourceToTagIds, Boolean isDelta, TagsChangeExtent tagsChangeExtent) { + setOp(op); + setServiceName(serviceName); + setTagVersion(tagVersion); + setTagUpdateTime(tagUpdateTime); + setTagDefinitions(tagDefinitions); + setTags(tags); + setServiceResources(serviceResources); + setResourceToTagIds(resourceToTagIds); + setIsDelta(isDelta); + setTagsChangeExtent(tagsChangeExtent); + } + /** + * @return the op + */ + public String getOp() { + return op; + } + + /** + * @return the serviceName + */ + public String getServiceName() { + return serviceName; + } + + /** + * @param op the op to set + */ + public void setOp(String op) { + this.op = op; + } + + /** + * @param serviceName the serviceName to set + */ + public void setServiceName(String serviceName) { + this.serviceName = serviceName; + } + + /** + * @return the tagVersion + */ + public Long getTagVersion() { + return tagVersion; + } + + /** + * @param tagVersion the version to set + */ + public void setTagVersion(Long tagVersion) { + this.tagVersion = tagVersion; + } + + /** + * @return the tagUpdateTime + */ + public Date getTagUpdateTime() { + return tagUpdateTime; + } + + /** + * @param tagUpdateTime the tagUpdateTime to set + */ + public void setTagUpdateTime(Date tagUpdateTime) { + this.tagUpdateTime = tagUpdateTime; + } + + public Map getTagDefinitions() { + return tagDefinitions; + } + + public void setTagDefinitions(Map tagDefinitions) { + this.tagDefinitions = tagDefinitions == null ? new HashMap() : tagDefinitions; + } + + public Map getTags() { + return tags; + } + + public void setTags(Map tags) { + this.tags = tags == null ? new HashMap() : tags; + } + + public List getServiceResources() { + return serviceResources; + } + + public void setServiceResources(List serviceResources) { + this.serviceResources = serviceResources == null ? new ArrayList() : serviceResources; + } + + public Map> getResourceToTagIds() { + return resourceToTagIds; + } + + public void setResourceToTagIds(Map> resourceToTagIds) { + this.resourceToTagIds = resourceToTagIds == null ? new HashMap>() : resourceToTagIds; + } + + public Boolean getIsDelta() { + return isDelta == null ? Boolean.FALSE : isDelta; + } + + public void setIsDelta(Boolean isDelta) { + this.isDelta = isDelta; + } + + public TagsChangeExtent getTagsChangeExtent() { + return tagsChangeExtent; + } + + public void setTagsChangeExtent(TagsChangeExtent tagsChangeExtent) { + this.tagsChangeExtent = tagsChangeExtent; + } + @Override + public String toString( ) { + StringBuilder sb = new StringBuilder(); + + toString(sb); + + return sb.toString(); + } + + public StringBuilder toString(StringBuilder sb) { + sb.append("ServiceTags={") + .append("op=").append(op).append(", ") + .append("serviceName=").append(serviceName).append(", ") + .append("tagVersion=").append(tagVersion).append(", ") + .append("tagUpdateTime={").append(tagUpdateTime).append("}") + .append("isDelta={").append(isDelta).append("}") + .append("tagsChangeExtent={").append(tagsChangeExtent).append("}") + .append("}"); + + return sb; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/StringTokenReplacer.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/StringTokenReplacer.java new file mode 100644 index 00000000000..54fdc989032 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/StringTokenReplacer.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.util; + +import java.util.Map; + +public class StringTokenReplacer { + private final char startChar; + private final char endChar; + private final char escapeChar; + private final String tokenPrefix; + + public StringTokenReplacer(char startChar, char endChar, char escapeChar, String tokenPrefix) { + this.startChar = startChar; + this.endChar = endChar; + this.escapeChar = escapeChar; + this.tokenPrefix = tokenPrefix; + } + + public String replaceTokens(String value, Map tokens) { + if(tokens == null || tokens.size() < 1 || value == null || value.length() < 1 || + (value.indexOf(startChar) == -1 && value.indexOf(endChar) == -1 && value.indexOf(escapeChar) == -1)) { + return value; + } + + StringBuilder ret = new StringBuilder(); + StringBuilder token = null; + + for(int i = 0; i < value.length(); i++) { + char c = value.charAt(i); + + if(c == escapeChar) { + i++; + if(i < value.length()) { + c = value.charAt(i); + if (token != null) { + // if next char is not the escape char or endChar, retain the escapeChar + if (c != escapeChar && c != endChar) { + token.append(escapeChar); + } + token.append(c); + } else { + // if next char is not the escape char or startChar, retain the escapeChar + if (c != escapeChar && c != startChar) { + ret.append(escapeChar); + } + ret.append(c); + } + } else { + if (token != null) { + token.append(escapeChar); + } else { + ret.append(escapeChar); + } + } + continue; + } + + if(token == null) { // not in token + if(c == startChar) { + token = new StringBuilder(); + } else { + ret.append(c); + } + } else { // in token + if(c == endChar) { + String rawToken = token.toString(); + if (tokenPrefix.length() == 0 || rawToken.startsWith(tokenPrefix)) { + Object replaced = RangerAccessRequestUtil.getTokenFromContext(tokens, rawToken.substring(tokenPrefix.length())); + if (replaced != null) { + ret.append(replaced.toString()); + } else { + ret.append(startChar).append(token).append(endChar); + } + } else { + ret.append(startChar).append(token).append(endChar); + } + token = null; + } else { + token.append(c); + } + } + } + + if(token != null) { // if no endChar is found + ret.append(startChar).append(token); + } + + return ret.toString(); + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/TimedEventUtil.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/TimedEventUtil.java new file mode 100644 index 00000000000..65ad8b66a52 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/TimedEventUtil.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + package org.apache.atlas.plugin.util; + +import java.util.concurrent.Callable; +import java.util.concurrent.TimeUnit; + +public class TimedEventUtil{ + + public static void runWithTimeout(final Runnable runnable, long timeout, TimeUnit timeUnit) throws Exception { + timedTask(new Callable() { + @Override + public Object call() throws Exception { + runnable.run(); + return null; + } + }, timeout, timeUnit); + } + + public static T timedTask(Callable callableObj, long timeout, + TimeUnit timeUnit) throws Exception{ + + return callableObj.call(); + + /* + final ExecutorService executor = Executors.newSingleThreadExecutor(); + final Future future = executor.submit(callableObj); + executor.shutdownNow(); + + try { + return future.get(timeout, timeUnit); + } catch (TimeoutException | InterruptedException | ExecutionException e) { + if(logger.isDebugEnabled()){ + logger.debug("Error executing task", e); + } + Throwable t = e.getCause(); + if (t instanceof Error) { + throw (Error) t; + } else if (t instanceof Exception) { + throw (Exception) e; + } else { + throw new IllegalStateException(t); + } + } + */ + + } + + +} \ No newline at end of file diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/URLEncoderUtil.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/URLEncoderUtil.java new file mode 100644 index 00000000000..aaa18434261 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/URLEncoderUtil.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.util; + +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; + +public class URLEncoderUtil { + + public static String encodeURIParam(String s) throws UnsupportedEncodingException { + + String ret = null; + + try { + ret = URLEncoder.encode(s, "UTF-8").replaceAll("\\+", "%20"); + } catch (UnsupportedEncodingException e) { + throw e; + } + + return ret; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/XMLUtils.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/XMLUtils.java new file mode 100644 index 00000000000..85e6e0eb94a --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/XMLUtils.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.util; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +import javax.xml.XMLConstants; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.InputStream; +import java.util.Map; + +public class XMLUtils { + + private static final Logger LOG = LoggerFactory.getLogger(XMLUtils.class); + + private static final String XMLCONFIG_PROPERTY_TAGNAME = "property"; + private static final String XMLCONFIG_NAME_TAGNAME = "name"; + private static final String XMLCONFIG_VALUE_TAGNAME = "value"; + + public static void loadConfig(String configFileName, Map properties) { + try (InputStream input = getFileInputStream(configFileName)) { + loadConfig(input, properties); + } catch (Exception e) { + LOG.error("Error loading : ", e); + } + } + + public static void loadConfig(InputStream input, Map properties) { + try { + DocumentBuilderFactory xmlDocumentBuilderFactory = DocumentBuilderFactory.newInstance(); + xmlDocumentBuilderFactory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); + xmlDocumentBuilderFactory.setFeature("http://xml.org/sax/features/external-general-entities", false); + xmlDocumentBuilderFactory.setFeature("http://xml.org/sax/features/external-parameter-entities", false); + xmlDocumentBuilderFactory.setIgnoringComments(true); + xmlDocumentBuilderFactory.setNamespaceAware(true); + + DocumentBuilder xmlDocumentBuilder = xmlDocumentBuilderFactory.newDocumentBuilder(); + Document xmlDocument = xmlDocumentBuilder.parse(input); + xmlDocument.getDocumentElement().normalize(); + + NodeList nList = xmlDocument.getElementsByTagName(XMLCONFIG_PROPERTY_TAGNAME); + + for (int temp = 0; temp < nList.getLength(); temp++) { + + Node nNode = nList.item(temp); + + if (nNode.getNodeType() == Node.ELEMENT_NODE) { + + Element eElement = (Element) nNode; + + String propertyName = ""; + String propertyValue = ""; + if (eElement.getElementsByTagName(XMLCONFIG_NAME_TAGNAME).item(0) != null) { + propertyName = eElement.getElementsByTagName(XMLCONFIG_NAME_TAGNAME) + .item(0).getTextContent().trim(); + } + if (eElement.getElementsByTagName(XMLCONFIG_VALUE_TAGNAME).item(0) != null) { + propertyValue = eElement.getElementsByTagName(XMLCONFIG_VALUE_TAGNAME) + .item(0).getTextContent().trim(); + } + + if (properties.get(propertyName) != null) { + properties.remove(propertyName); + } + + properties.put(propertyName, propertyValue); + + } + } + + } catch (Exception e) { + LOG.error("Error loading : ", e); + } + } + + private static InputStream getFileInputStream(String path) throws FileNotFoundException { + + InputStream ret = null; + + // Guard against path traversal attacks + String sanitizedPath = new File(path).getName(); + if ("".equals(sanitizedPath)) { + return null; + } + File f = new File(sanitizedPath); + + if (f.exists()) { + ret = new FileInputStream(f); + } else { + ret = XMLUtils.class.getResourceAsStream(path); + + if (ret == null) { + if (! path.startsWith("/")) { + ret = XMLUtils.class.getResourceAsStream("/" + path); + } + } + + if (ret == null) { + ret = ClassLoader.getSystemClassLoader().getResourceAsStream(path); + if (ret == null) { + if (! path.startsWith("/")) { + ret = ClassLoader.getSystemResourceAsStream("/" + path); + } + } + } + } + + return ret; + } + +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/AbstractCachePolicyTransformer.java b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/AbstractCachePolicyTransformer.java new file mode 100644 index 00000000000..5c0a16b0b6c --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/AbstractCachePolicyTransformer.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.policytransformer; + +import org.apache.atlas.exception.AtlasBaseException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashMap; +import java.util.Map; + +public abstract class AbstractCachePolicyTransformer implements CachePolicyTransformer { + + private static final Logger LOG = LoggerFactory.getLogger(AbstractCachePolicyTransformer.class); + + public static final String PLACEHOLDER_ENTITY = "{entity}"; + public static final String PLACEHOLDER_ENTITY_TYPE = "{entity-type}"; + public static final String PLACEHOLDER_TAG = "{tag}"; + + private static Map TEMPLATES = new HashMap<>(); + + public PolicyTransformerTemplate getTemplate(String fileSuffix) throws AtlasBaseException { + if (!TEMPLATES.containsKey(fileSuffix)) { + try { + TEMPLATES.put(fileSuffix, CacheTransformerTemplateHelper.getTemplate(fileSuffix)); + } catch (AtlasBaseException e) { + LOG.error("Failed to load template for policies: {}", fileSuffix); + throw e; + } + } + + return TEMPLATES.get(fileSuffix); + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformer.java b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformer.java new file mode 100644 index 00000000000..b18311f034e --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformer.java @@ -0,0 +1,27 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.policytransformer; + +import org.apache.atlas.model.instance.AtlasEntityHeader; + +import java.util.List; + +public interface CachePolicyTransformer { + + List transform(AtlasEntityHeader atlasPolicy); +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java new file mode 100644 index 00000000000..4a81129a3c6 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java @@ -0,0 +1,544 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.policytransformer; + +import org.apache.atlas.AtlasException; +import org.apache.atlas.RequestContext; +import org.apache.atlas.discovery.EntityDiscoveryService; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.discovery.AtlasSearchResult; +import org.apache.atlas.model.discovery.IndexSearchParams; +import org.apache.atlas.model.instance.AtlasEntityHeader; +import org.apache.atlas.plugin.util.ServicePolicies; +import org.apache.atlas.plugin.model.RangerPolicy; +import org.apache.atlas.plugin.model.RangerPolicy.RangerDataMaskPolicyItem; +import org.apache.atlas.plugin.model.RangerPolicy.RangerPolicyItem; +import org.apache.atlas.plugin.model.RangerPolicy.RangerPolicyItemAccess; +import org.apache.atlas.plugin.model.RangerPolicy.RangerPolicyItemCondition; +import org.apache.atlas.plugin.model.RangerPolicy.RangerPolicyItemDataMaskInfo; +import org.apache.atlas.plugin.model.RangerPolicy.RangerPolicyResource; +import org.apache.atlas.plugin.model.RangerServiceDef; +import org.apache.atlas.plugin.model.RangerValiditySchedule; +import org.apache.atlas.plugin.util.ServicePolicies.TagPolicies; +import org.apache.atlas.repository.graphdb.AtlasGraph; +import org.apache.atlas.repository.graphdb.janus.AtlasJanusGraph; +import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; +import org.apache.atlas.type.AtlasType; +import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.atlas.utils.AtlasPerfMetrics; +import org.apache.atlas.v1.model.instance.Id; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import javax.inject.Inject; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.apache.atlas.repository.Constants.NAME; +import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; +import static org.apache.atlas.repository.Constants.SERVICE_ENTITY_TYPE; +import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_CATEGORY; +import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_CONNECTION_QN; +import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_IS_ENABLED; +import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_PRIORITY; +import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_SERVICE_NAME; +import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_SUB_CATEGORY; +import static org.apache.atlas.repository.util.AccessControlUtils.POLICY_CATEGORY_PERSONA; +import static org.apache.atlas.repository.util.AccessControlUtils.POLICY_CATEGORY_PURPOSE; +import static org.apache.atlas.repository.util.AccessControlUtils.getIsPolicyEnabled; +import static org.apache.atlas.repository.util.AccessControlUtils.getPolicyCategory; + +@Component +public class CachePolicyTransformerImpl { + private static final Logger LOG = LoggerFactory.getLogger(CachePolicyTransformerImpl.class); + + private static final String RESOURCES_SPLITTER = ":"; + + static final String ATTR_QUALIFIED_NAME = "qualifiedName"; + static final String ATTR_NAME = "name"; + + public static final String ATTR_POLICY_ACTIONS = "policyActions"; + public static final String ATTR_POLICY_TYPE = "policyType"; + public static final String ATTR_POLICY_RESOURCES = "policyResources"; + + public static final String ATTR_SERVICE_SERVICE_TYPE = "authServiceType"; + public static final String ATTR_SERVICE_TAG_SERVICE = "tagService"; + public static final String ATTR_SERVICE_IS_ENABLED = "authServiceIsEnabled"; + public static final String ATTR_SERVICE_LAST_SYNC = "authServicePolicyLastSync"; + + private static final String ATTR_POLICY_RESOURCES_CATEGORY = "policyResourceCategory"; + private static final String ATTR_POLICY_GROUPS = "policyGroups"; + private static final String ATTR_POLICY_USERS = "policyUsers"; + private static final String ATTR_POLICY_ROLES = "policyRoles"; + private static final String ATTR_POLICY_VALIDITY = "policyValiditySchedule"; + private static final String ATTR_POLICY_CONDITIONS = "policyConditions"; + private static final String ATTR_POLICY_MASK_TYPE = "policyMaskType"; + + private static final String RESOURCE_SERVICE_DEF_PATH = "/service-defs/"; + private static final String RESOURCE_SERVICE_DEF_PATTERN = RESOURCE_SERVICE_DEF_PATH + "atlas-servicedef-%s.json"; + + private EntityDiscoveryService discoveryService; + private AtlasGraph graph; + private EntityGraphRetriever entityRetriever; + + private PersonaCachePolicyTransformer personaTransformer; + private PurposeCachePolicyTransformer purposeTransformer; + + private AtlasEntityHeader service; + + @Inject + public CachePolicyTransformerImpl(AtlasTypeRegistry typeRegistry) throws AtlasBaseException { + this.graph = new AtlasJanusGraph(); + this.entityRetriever = new EntityGraphRetriever(graph, typeRegistry); + + personaTransformer = new PersonaCachePolicyTransformer(entityRetriever); + purposeTransformer = new PurposeCachePolicyTransformer(entityRetriever); + + try { + this.discoveryService = new EntityDiscoveryService(typeRegistry, graph, null, null, null, null); + } catch (AtlasException e) { + LOG.error("Failed to initialize discoveryService"); + throw new AtlasBaseException(e.getCause()); + } + } + + public AtlasEntityHeader getService() { + return service; + } + + public ServicePolicies getPolicies(String serviceName, String pluginId, Long lastUpdatedTime) { + //TODO: return only if updated + AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("CachePolicyTransformerImpl.getPolicies." + serviceName); + + ServicePolicies servicePolicies = new ServicePolicies(); + + try { + servicePolicies.setServiceName(serviceName); + + service = getServiceEntity(serviceName); + servicePolicies.setPolicyVersion(-1L); + servicePolicies.setPolicyUpdateTime(new Date()); + + if (service != null) { + List allPolicies = getServicePolicies(service); + servicePolicies.setServiceName(serviceName); + servicePolicies.setServiceId(service.getGuid()); + + String serviceDefName = String.format(RESOURCE_SERVICE_DEF_PATTERN, serviceName); + servicePolicies.setServiceDef(getResourceAsObject(serviceDefName, RangerServiceDef.class)); + + + //Process tag based policies + String tagServiceName = (String) service.getAttribute(ATTR_SERVICE_TAG_SERVICE); + if (StringUtils.isNotEmpty(tagServiceName)) { + AtlasEntityHeader tagService = getServiceEntity(tagServiceName); + + if (tagService != null) { + allPolicies.addAll(getServicePolicies(tagService)); + + TagPolicies tagPolicies = new TagPolicies(); + + tagPolicies.setServiceName(tagServiceName); + tagPolicies.setPolicyUpdateTime(new Date()); + tagPolicies.setServiceId(tagService.getGuid()); + tagPolicies.setPolicyVersion(-1L); + + String tagServiceDefName = String.format(RESOURCE_SERVICE_DEF_PATTERN, tagService.getAttribute(NAME)); + tagPolicies.setServiceDef(getResourceAsObject(tagServiceDefName, RangerServiceDef.class)); + + servicePolicies.setTagPolicies(tagPolicies); + } + } + + AtlasPerfMetrics.MetricRecorder recorderFilterPolicies = RequestContext.get().startMetricRecord("filterPolicies"); + //filter out policies based on serviceName + List policiesA = allPolicies.stream().filter(x -> serviceName.equals(x.getService())).collect(Collectors.toList()); + List policiesB = allPolicies.stream().filter(x -> tagServiceName.equals(x.getService())).collect(Collectors.toList()); + + servicePolicies.setPolicies(policiesA); + servicePolicies.getTagPolicies().setPolicies(policiesB); + + RequestContext.get().endMetricRecord(recorderFilterPolicies); + + LOG.info("Found {} policies", servicePolicies.getPolicies().size()); + } + + } catch (Exception e) { + LOG.error("ERROR in getPolicies {}: ", e); + return null; + } + + RequestContext.get().endMetricRecord(recorder); + return servicePolicies; + } + + private List getServicePolicies(AtlasEntityHeader service) throws AtlasBaseException, IOException { + + List servicePolicies = new ArrayList<>(); + + String serviceName = (String) service.getAttribute("name"); + String serviceType = (String) service.getAttribute("authServiceType"); + List atlasPolicies = getAtlasPolicies(serviceName); + + if (CollectionUtils.isNotEmpty(atlasPolicies)) { + //transform policies + servicePolicies = transformAtlasPoliciesToRangerPolicies(atlasPolicies, serviceType, serviceName); + } + return servicePolicies; + } + + private List transformAtlasPoliciesToRangerPolicies(List atlasPolicies, + String serviceType, + String serviceName) throws IOException, AtlasBaseException { + AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("CachePolicyTransformerImpl."+serviceName+".transformAtlasPoliciesToRangerPolicies"); + + List rangerPolicies = new ArrayList<>(); + try { + for (AtlasEntityHeader atlasPolicy : atlasPolicies) { + + String policyCategory = getPolicyCategory(atlasPolicy); + if (POLICY_CATEGORY_PERSONA.equals(policyCategory)) { + + List transformedAtlasPolicies = personaTransformer.transform(atlasPolicy); + for (AtlasEntityHeader transformedPolicy : transformedAtlasPolicies) { + rangerPolicies.add(toRangerPolicy(transformedPolicy, serviceType)); + } + + } else if (POLICY_CATEGORY_PURPOSE.equals(policyCategory)) { + List transformedAtlasPolicies = purposeTransformer.transform(atlasPolicy); + + for (AtlasEntityHeader transformedPolicy : transformedAtlasPolicies) { + rangerPolicies.add(toRangerPolicy(transformedPolicy, serviceType)); + } + + } else { + rangerPolicies.add(toRangerPolicy(atlasPolicy, serviceType)); + } + } + + } finally { + RequestContext.get().endMetricRecord(recorder); + } + + return rangerPolicies; + } + + private RangerPolicy toRangerPolicy(AtlasEntityHeader atlasPolicy, String serviceType) throws AtlasBaseException, IOException { + RangerPolicy rangerPolicy = getRangerPolicy(atlasPolicy, serviceType); + + //GET policy Item + setPolicyItems(rangerPolicy, atlasPolicy); + + //GET policy Resources + setPolicyResources(rangerPolicy, atlasPolicy); + + return rangerPolicy; + } + + private void setPolicyResources(RangerPolicy rangerPolicy, AtlasEntityHeader atlasPolicy) throws IOException { + List atlasResources = (List) atlasPolicy.getAttribute("policyResources"); + + Map> resourceValuesMap = new HashMap<>(); + + for (String atlasResource : atlasResources) { + String resourceName = atlasResource.split(RESOURCES_SPLITTER)[0]; + + if (!resourceValuesMap.containsKey(resourceName)) { + String resourceNameFinal = resourceName + ":"; + List applicables = atlasResources.stream().filter(x -> x.startsWith(resourceNameFinal)).collect(Collectors.toList()); + List values = applicables.stream().map(x -> x.substring(resourceNameFinal.length())).collect(Collectors.toList()); + resourceValuesMap.put(resourceName, values); + } + } + + Map resources = new HashMap<>(); + for (String key : resourceValuesMap.keySet()) { + RangerPolicyResource resource = new RangerPolicyResource(resourceValuesMap.get(key), false, false); + resources.put(key, resource); + } + + rangerPolicy.setResources(resources); + } + + private T getResourceAsObject(String resourceName, Class clazz) throws IOException { + InputStream stream = getClass().getResourceAsStream(resourceName); + return AtlasType.fromJson(stream, clazz); + } + + private void setPolicyItems(RangerPolicy rangerPolicy, AtlasEntityHeader atlasPolicy) throws AtlasBaseException { + + String policyType = (String) atlasPolicy.getAttribute("policyType"); + + List users = (List) atlasPolicy.getAttribute("policyUsers"); + List groups = (List) atlasPolicy.getAttribute("policyGroups"); + List roles = (List) atlasPolicy.getAttribute("policyRoles"); + + List accesses = new ArrayList<>(); + List actions = (List) atlasPolicy.getAttribute("policyActions"); + + actions.forEach(action -> accesses.add(new RangerPolicyItemAccess(action))); + + + if ("allow".equals(policyType)) { + RangerPolicyItem item = new RangerPolicyItem(); + item.setUsers(users); + item.setGroups(groups); + item.setRoles(roles); + item.setAccesses(accesses); + + rangerPolicy.setPolicyItems(Collections.singletonList(item)); + rangerPolicy.setPolicyType(RangerPolicy.POLICY_TYPE_ACCESS); + + } else if ("deny".equals(policyType)) { + RangerPolicyItem item = new RangerPolicyItem(); + item.setUsers(users); + item.setGroups(groups); + item.setRoles(roles); + item.setAccesses(accesses); + + rangerPolicy.setDenyPolicyItems(Collections.singletonList(item)); + rangerPolicy.setPolicyType(RangerPolicy.POLICY_TYPE_ACCESS); + + } else if ("allowExceptions".equals(policyType)) { + RangerPolicyItem item = new RangerPolicyItem(); + item.setUsers(users); + item.setGroups(groups); + item.setRoles(roles); + item.setAccesses(accesses); + + rangerPolicy.setAllowExceptions(Collections.singletonList(item)); + rangerPolicy.setPolicyType(RangerPolicy.POLICY_TYPE_ACCESS); + + } else if ("denyExceptions".equals(policyType)) { + RangerPolicyItem item = new RangerPolicyItem(); + item.setUsers(users); + item.setGroups(groups); + item.setRoles(roles); + item.setAccesses(accesses); + + rangerPolicy.setDenyExceptions(Collections.singletonList(item)); + rangerPolicy.setPolicyType(RangerPolicy.POLICY_TYPE_ACCESS); + + } else if ("dataMask".equals(policyType)) { + + rangerPolicy.setPolicyType(RangerPolicy.POLICY_TYPE_DATAMASK); + + RangerDataMaskPolicyItem item = new RangerDataMaskPolicyItem(); + item.setUsers(users); + item.setGroups(groups); + item.setRoles(roles); + item.setAccesses(accesses); + + String maskType = (String) atlasPolicy.getAttribute(ATTR_POLICY_MASK_TYPE); + + if (StringUtils.isEmpty(maskType)) { + LOG.error("MASK type not found"); + throw new AtlasBaseException("MASK type not found"); + } + + RangerPolicyItemDataMaskInfo dataMaskInfo = new RangerPolicyItemDataMaskInfo(maskType, null, null); + item.setDataMaskInfo(dataMaskInfo); + + rangerPolicy.setDataMaskPolicyItems(Collections.singletonList(item)); + + } else if ("rowFilter".equals(policyType)) { + rangerPolicy.setPolicyType(RangerPolicy.POLICY_TYPE_ROWFILTER); + //TODO + } + } + + private List getPolicyConditions(AtlasEntityHeader atlasPolicy) { + List ret = new ArrayList<>(); + + if (!atlasPolicy.hasAttribute("policyConditions")) { + return null; + } + + List> conditions = (List>) atlasPolicy.getAttribute("policyConditions"); + + for (HashMap condition : conditions) { + RangerPolicyItemCondition rangerCondition = new RangerPolicyItemCondition(); + + rangerCondition.setType((String) condition.get("policyConditionType")); + rangerCondition.setValues((List) condition.get("policyConditionValues")); + + ret.add(rangerCondition); + } + return ret; + } + + private List getPolicyValiditySchedule(AtlasEntityHeader atlasPolicy) { + List ret = new ArrayList<>(); + + if (!atlasPolicy.hasAttribute("policyValiditySchedule")) { + return null; + } + + List> validitySchedules = (List>) atlasPolicy.getAttribute("policyValiditySchedule"); + + + for (HashMap validitySchedule : validitySchedules) { + RangerValiditySchedule rangerValiditySchedule = new RangerValiditySchedule(); + + rangerValiditySchedule.setStartTime(validitySchedule.get("policyValidityScheduleStartTime")); + rangerValiditySchedule.setEndTime(validitySchedule.get("policyValidityScheduleEndTime")); + rangerValiditySchedule.setTimeZone(validitySchedule.get("policyValidityScheduleTimezone")); + + ret.add(rangerValiditySchedule); + } + return ret; + } + + private List getAtlasPolicies(String serviceName) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("CachePolicyTransformerImpl."+service+".getAtlasPolicies"); + + List ret = new ArrayList<>(); + try { + IndexSearchParams indexSearchParams = new IndexSearchParams(); + Set attributes = new HashSet<>(); + attributes.add(NAME); + attributes.add(ATTR_POLICY_CATEGORY); + attributes.add(ATTR_POLICY_SUB_CATEGORY); + attributes.add(ATTR_POLICY_TYPE); + attributes.add(ATTR_POLICY_SERVICE_NAME); + attributes.add(ATTR_POLICY_USERS); + attributes.add(ATTR_POLICY_GROUPS); + attributes.add(ATTR_POLICY_ROLES); + attributes.add(ATTR_POLICY_ACTIONS); + attributes.add(ATTR_POLICY_RESOURCES); + attributes.add(ATTR_POLICY_RESOURCES_CATEGORY); + attributes.add(ATTR_POLICY_MASK_TYPE); + attributes.add(ATTR_POLICY_PRIORITY); + attributes.add(ATTR_POLICY_VALIDITY); + attributes.add(ATTR_POLICY_CONDITIONS); + attributes.add(ATTR_POLICY_IS_ENABLED); + attributes.add(ATTR_POLICY_CONNECTION_QN); + + Map dsl = getMap("size", 0); + + List> mustClauseList = new ArrayList<>(); + mustClauseList.add(getMap("term", getMap(ATTR_POLICY_SERVICE_NAME, serviceName))); + mustClauseList.add(getMap("match", getMap("__state", Id.EntityState.ACTIVE))); + + dsl.put("query", getMap("bool", getMap("must", mustClauseList))); + + List sortList = new ArrayList<>(0); + sortList.add(getMap("__timestamp", getMap("order", "asc"))); + sortList.add(getMap("__guid", getMap("order", "asc"))); + dsl.put("sort", sortList); + + indexSearchParams.setDsl(dsl); + indexSearchParams.setAttributes(attributes); + + int from = 0; + int size = 100; + boolean found = true; + + do { + dsl.put("from", from); + dsl.put("size", size); + indexSearchParams.setDsl(dsl); + + List headers = discoveryService.directIndexSearch(indexSearchParams).getEntities(); + if (headers != null) { + ret.addAll(headers); + } else { + found = false; + } + + from += size; + + } while (found && ret.size() % size == 0); + + } finally { + RequestContext.get().endMetricRecord(recorder); + } + + return ret; + } + + private AtlasEntityHeader getServiceEntity(String serviceName) throws AtlasBaseException { + IndexSearchParams indexSearchParams = new IndexSearchParams(); + Set attributes = new HashSet<>(); + attributes.add(NAME); + attributes.add(ATTR_SERVICE_SERVICE_TYPE); + attributes.add(ATTR_SERVICE_TAG_SERVICE); + attributes.add(ATTR_SERVICE_IS_ENABLED); + + Map dsl = getMap("size", 1); + + List> mustClauseList = new ArrayList<>(); + mustClauseList.add(getMap("term", getMap("__typeName.keyword", SERVICE_ENTITY_TYPE))); + mustClauseList.add(getMap("term", getMap("name.keyword", serviceName))); + mustClauseList.add(getMap("match", getMap("__state", Id.EntityState.ACTIVE))); + + dsl.put("query", getMap("bool", getMap("must", mustClauseList))); + + indexSearchParams.setDsl(dsl); + indexSearchParams.setAttributes(attributes); + + AtlasSearchResult searchResult = discoveryService.directIndexSearch(indexSearchParams); + + if (searchResult.getEntities() != null) { + return searchResult.getEntities().get(0); + } + + return null; + } + + private Map getMap(String key, Object value) { + Map map = new HashMap<>(); + map.put(key, value); + return map; + } + + private RangerPolicy getRangerPolicy(AtlasEntityHeader atlasPolicy, String serviceType) { + RangerPolicy policy = new RangerPolicy(); + + //policy.setId(atlasPolicy.getGuid()); + policy.setName((String) atlasPolicy.getAttribute(QUALIFIED_NAME)); + policy.setService((String) atlasPolicy.getAttribute(ATTR_POLICY_SERVICE_NAME)); + policy.setServiceType(serviceType); + policy.setGuid(atlasPolicy.getGuid()); + policy.setCreatedBy(atlasPolicy.getCreatedBy()); + policy.setCreateTime(atlasPolicy.getCreateTime()); + policy.setIsEnabled(getIsPolicyEnabled(atlasPolicy)); + + policy.setConditions(getPolicyConditions(atlasPolicy)); + policy.setValiditySchedules(getPolicyValiditySchedule(atlasPolicy)); + + if (atlasPolicy.hasAttribute(ATTR_POLICY_PRIORITY)) { + policy.setPolicyPriority((Integer) atlasPolicy.getAttribute(ATTR_POLICY_PRIORITY)); + } + + return policy; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CacheTransformerTemplateHelper.java b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CacheTransformerTemplateHelper.java new file mode 100644 index 00000000000..075f0c7b575 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CacheTransformerTemplateHelper.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.policytransformer; + +import org.apache.atlas.exception.AtlasBaseException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; + +import static org.apache.atlas.repository.Constants.getStaticFileAsString; + +public class CacheTransformerTemplateHelper { + private static final Logger LOG = LoggerFactory.getLogger(CacheTransformerTemplateHelper.class); + + static final String RESOURCE_POLICY_TRANSFORMER = "templates/policy_cache_transformer_%s.json"; + + public static PolicyTransformerTemplate getTemplate(String fileSuffix) throws AtlasBaseException { + PolicyTransformerTemplate templates; + String jsonTemplate = null; + String fileName = String.format(RESOURCE_POLICY_TRANSFORMER, fileSuffix); + + try { + jsonTemplate = getStaticFileAsString(fileName); + } catch (IOException e) { + LOG.error("Failed to load template for policies: {}", RESOURCE_POLICY_TRANSFORMER); + throw new AtlasBaseException(e); + } + templates = new PolicyTransformerTemplate(); + templates.fromJsonString(jsonTemplate); + + return templates; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/PersonaCachePolicyTransformer.java b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/PersonaCachePolicyTransformer.java new file mode 100644 index 00000000000..6a6d2d3cd99 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/PersonaCachePolicyTransformer.java @@ -0,0 +1,155 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.policytransformer; + +import org.apache.atlas.RequestContext; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.model.instance.AtlasEntityHeader; +import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; +import org.apache.atlas.utils.AtlasPerfMetrics; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; + +import static org.apache.atlas.policytransformer.CachePolicyTransformerImpl.ATTR_NAME; +import static org.apache.atlas.policytransformer.CachePolicyTransformerImpl.ATTR_POLICY_RESOURCES; +import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_ACTIONS; +import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_IS_ENABLED; +import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_RESOURCES_CATEGORY; +import static org.apache.atlas.repository.util.AccessControlUtils.POLICY_SUB_CATEGORY_DATA; +import static org.apache.atlas.repository.util.AccessControlUtils.POLICY_SUB_CATEGORY_METADATA; +import static org.apache.atlas.repository.util.AccessControlUtils.RESOURCES_ENTITY; +import static org.apache.atlas.repository.util.AccessControlUtils.RESOURCES_ENTITY_TYPE; +import static org.apache.atlas.repository.util.AccessControlUtils.getEntityByQualifiedName; +import static org.apache.atlas.repository.util.AccessControlUtils.getFilteredPolicyResources; +import static org.apache.atlas.repository.util.AccessControlUtils.getIsPolicyEnabled; +import static org.apache.atlas.repository.util.AccessControlUtils.getPolicyActions; +import static org.apache.atlas.repository.util.AccessControlUtils.getPolicyConnectionQN; +import static org.apache.atlas.repository.util.AccessControlUtils.getPolicyResources; +import static org.apache.atlas.repository.util.AccessControlUtils.getPolicySubCategory; + +public class PersonaCachePolicyTransformer extends AbstractCachePolicyTransformer { + private static final Logger LOG = LoggerFactory.getLogger(PersonaCachePolicyTransformer.class); + + private final static String TEMPLATE_SUFFIX = "persona"; + + private EntityGraphRetriever entityRetriever = null; + private PolicyTransformerTemplate personaTemplate; + + public PersonaCachePolicyTransformer(EntityGraphRetriever entityRetriever) throws AtlasBaseException { + personaTemplate = getTemplate(TEMPLATE_SUFFIX); + this.entityRetriever = entityRetriever; + } + + public List transform(AtlasEntityHeader atlasPolicy) { + AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("PersonaCachePolicyTransformer.transform"); + List ret = new ArrayList<>(); + + List atlasActions = getPolicyActions(atlasPolicy); + List atlasResources = getPolicyResources(atlasPolicy); + List entityResources = getFilteredPolicyResources(atlasResources, RESOURCES_ENTITY); + List typeResources = getFilteredPolicyResources(atlasResources, RESOURCES_ENTITY_TYPE); + + int index = 0; + for (String atlasAction : atlasActions) { + List currentTemplates = personaTemplate.getTemplate(atlasAction); + + if (CollectionUtils.isEmpty(currentTemplates)) { + LOG.warn("PolicyTransformerImpl: Skipping unknown action {} while transforming policy {}", atlasAction, atlasPolicy.getGuid()); + continue; + } + + for (PolicyTransformerTemplate.TemplatePolicy templatePolicy : currentTemplates) { + AtlasEntityHeader header = new AtlasEntityHeader(atlasPolicy); + + header.setGuid(atlasPolicy.getGuid() + "-" + index++); + + header.setAttribute(ATTR_POLICY_ACTIONS, templatePolicy.getActions()); + header.setAttribute(ATTR_POLICY_RESOURCES_CATEGORY, templatePolicy.getPolicyResourceCategory()); + header.setAttribute(ATTR_POLICY_IS_ENABLED, getIsPolicyEnabled(atlasPolicy)); + + String subCategory = getPolicySubCategory(atlasPolicy); + + List finalResources = new ArrayList<>(); + + for (String templateResource : templatePolicy.getResources()) { + if (templateResource.contains(PLACEHOLDER_ENTITY)) { + for (String entityResource : entityResources) { + finalResources.add(templateResource.replace(PLACEHOLDER_ENTITY, entityResource)); + } + + } else if (templateResource.contains(PLACEHOLDER_ENTITY_TYPE)) { + + if (CollectionUtils.isNotEmpty(typeResources)) { + typeResources.forEach(x -> finalResources.add(templateResource.replace(PLACEHOLDER_ENTITY_TYPE, x))); + } else { + boolean isConnection = false; + + if (POLICY_SUB_CATEGORY_METADATA.equals(subCategory) || POLICY_SUB_CATEGORY_DATA.equals(subCategory)) { + isConnection = isConnectionPolicy(entityResources, atlasPolicy); + } + + if (isConnection) { + finalResources.add(templateResource.replace(PLACEHOLDER_ENTITY_TYPE, "*")); + } else { + finalResources.add(templateResource.replace(PLACEHOLDER_ENTITY_TYPE, "Process")); + finalResources.add(templateResource.replace(PLACEHOLDER_ENTITY_TYPE, "Catalog")); + } + } + } else { + finalResources.add(templateResource); + } + } + header.setAttribute(ATTR_POLICY_RESOURCES, finalResources); + + header.setAttribute(ATTR_NAME, "transformed_policy_persona"); + + ret.add(header); + } + } + + RequestContext.get().endMetricRecord(recorder); + return ret; + } + + private boolean isConnectionPolicy(List assets, AtlasEntityHeader atlasPolicy) { + + if (assets.size() == 1) { + String connQNAttr = getPolicyConnectionQN(atlasPolicy); + + if (StringUtils.isNotEmpty(connQNAttr)) { + return connQNAttr.equals(assets.get(0)); + } else { + AtlasEntity connection; + try { + connection = getEntityByQualifiedName(entityRetriever, assets.get(0)); + } catch (AtlasBaseException abe) { + return false; + } + return connection != null; + } + } + + return false; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/PolicyTransformerTemplate.java b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/PolicyTransformerTemplate.java new file mode 100644 index 00000000000..3bde95eef12 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/PolicyTransformerTemplate.java @@ -0,0 +1,120 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.policytransformer; + +import org.apache.atlas.type.AtlasType; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class PolicyTransformerTemplate { + private static final Logger LOG = LoggerFactory.getLogger(PolicyTransformerTemplate.class); + + private Map> actionToPoliciesMap = new HashMap<>(); + + public PolicyTransformerTemplate() { + } + + public List getTemplate(String action) { + return actionToPoliciesMap.get(action); + } + + public Set getTemplateActions() { + return new HashSet<>(actionToPoliciesMap.keySet()); + } + + public void fromJsonString(String json) { + + Map> templates = AtlasType.fromJson(json, Map.class); + + for (String customAction : templates.keySet()) { + List templatePolicies = templates.get(customAction); + List policies = new ArrayList<>(); + + for (Map policy: templatePolicies) { + TemplatePolicy templatePolicy = new TemplatePolicy(); + + templatePolicy.setActions((List) policy.get("actions")); + templatePolicy.setResources((List) policy.get("resources")); + templatePolicy.setPolicyType((String) policy.get("policyType")); + templatePolicy.setPolicyResourceCategory((String) policy.get("policyResourceCategory")); + templatePolicy.setPolicyServiceName((String) policy.get("policyServiceName")); + + policies.add(templatePolicy); + } + + this.actionToPoliciesMap.put(customAction, policies); + } + } + + class TemplatePolicy { + private String policyServiceName; + private String policyType; + private List resources; + private List actions; + private String policyResourceCategory; + + public String getPolicyServiceName() { + return policyServiceName; + } + + public void setPolicyServiceName(String policyServiceName) { + this.policyServiceName = policyServiceName; + } + + public String getPolicyType() { + return policyType; + } + + public void setPolicyType(String policyType) { + this.policyType = policyType; + } + + public String getPolicyResourceCategory() { + return policyResourceCategory; + } + + public void setPolicyResourceCategory(String category) { + this.policyResourceCategory = category; + } + + public List getResources() { + return resources; + } + + public void setResources(List resources) { + this.resources = resources; + } + + public List getActions() { + return actions; + } + + public void setActions(List actions) { + this.actions = actions; + } + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/PurposeCachePolicyTransformer.java b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/PurposeCachePolicyTransformer.java new file mode 100644 index 00000000000..943a0d3220c --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/PurposeCachePolicyTransformer.java @@ -0,0 +1,121 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.policytransformer; + +import org.apache.atlas.RequestContext; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.instance.AtlasEntityHeader; +import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; +import org.apache.atlas.utils.AtlasPerfMetrics; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import static org.apache.atlas.policytransformer.CachePolicyTransformerImpl.ATTR_POLICY_RESOURCES; +import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_ACTIONS; +import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_IS_ENABLED; +import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_RESOURCES_CATEGORY; +import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_SERVICE_NAME; +import static org.apache.atlas.repository.util.AccessControlUtils.RESOURCES_TAG; +import static org.apache.atlas.repository.util.AccessControlUtils.getFilteredPolicyResources; +import static org.apache.atlas.repository.util.AccessControlUtils.getIsPolicyEnabled; +import static org.apache.atlas.repository.util.AccessControlUtils.getPolicyActions; +import static org.apache.atlas.repository.util.AccessControlUtils.getPolicyResources; + +public class PurposeCachePolicyTransformer extends AbstractCachePolicyTransformer { + private static final Logger LOG = LoggerFactory.getLogger(PurposeCachePolicyTransformer.class); + + private final static String TEMPLATE_SUFFIX = "purpose"; + + private EntityGraphRetriever entityRetriever = null; + private PolicyTransformerTemplate purposeTemplate; + + public PurposeCachePolicyTransformer(EntityGraphRetriever entityRetriever) throws AtlasBaseException { + purposeTemplate = getTemplate(TEMPLATE_SUFFIX); + this.entityRetriever = entityRetriever; + } + + public List transform(AtlasEntityHeader atlasPolicy) { + AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("PurposeCachePolicyTransformer.transform"); + List ret = new ArrayList<>(); + + List atlasActions = getPolicyActions(atlasPolicy); + List atlasResources = getPolicyResources(atlasPolicy); + List tags = getFilteredPolicyResources(atlasResources, RESOURCES_TAG); + + int index = 0; + Set templateActions = purposeTemplate.getTemplateActions(); + List transformableActions = (List) CollectionUtils.intersection(atlasActions, templateActions); + + + for (String atlasAction : transformableActions) { + List currentTemplates = purposeTemplate.getTemplate(atlasAction); + + if (CollectionUtils.isEmpty(currentTemplates)) { + LOG.warn("PurposeCachePolicyTransformer: Skipping unknown action {} while transforming policy {}", atlasAction, atlasPolicy.getGuid()); + continue; + } + + for (PolicyTransformerTemplate.TemplatePolicy templatePolicy : currentTemplates) { + AtlasEntityHeader header = new AtlasEntityHeader(atlasPolicy); + + header.setGuid(atlasPolicy.getGuid() + "-" + index++); + + header.setAttribute(ATTR_POLICY_ACTIONS, templatePolicy.getActions()); + header.setAttribute(ATTR_POLICY_RESOURCES_CATEGORY, templatePolicy.getPolicyResourceCategory()); + header.setAttribute(ATTR_POLICY_IS_ENABLED, getIsPolicyEnabled(atlasPolicy)); + + if (StringUtils.isNotEmpty(templatePolicy.getPolicyServiceName())) { + header.setAttribute(ATTR_POLICY_SERVICE_NAME, templatePolicy.getPolicyServiceName()); + } + + List finalResources = new ArrayList<>(); + + for (String templateResource : templatePolicy.getResources()) { + if (templateResource.contains(PLACEHOLDER_TAG)) { + tags.forEach(tag -> finalResources.add(templateResource.replace(PLACEHOLDER_TAG, tag))); + } else { + finalResources.add(templateResource); + } + } + header.setAttribute(ATTR_POLICY_RESOURCES, finalResources); + + ret.add(header); + } + } + + //prepare a policy for all non-transformable actions + //this will help to reduce number of overall transformed policies + List nonTransformableActions = (List) CollectionUtils.subtract(atlasActions, templateActions); + + if (CollectionUtils.isNotEmpty(nonTransformableActions)) { + AtlasEntityHeader header = new AtlasEntityHeader(atlasPolicy); + header.setGuid(atlasPolicy.getGuid() + "-" + index); + header.setAttribute(ATTR_POLICY_ACTIONS, nonTransformableActions); + ret.add(header); + } + + RequestContext.get().endMetricRecord(recorder); + return ret; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/services/tag/RangerServiceTag.java b/auth-agents-common/src/main/java/org/apache/atlas/services/tag/RangerServiceTag.java new file mode 100644 index 00000000000..55ee56ef7fb --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/services/tag/RangerServiceTag.java @@ -0,0 +1,199 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.services.tag; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.plugin.model.RangerPolicy; +import org.apache.atlas.plugin.model.RangerService; +import org.apache.atlas.plugin.model.RangerServiceDef; +import org.apache.atlas.plugin.service.RangerBaseService; +import org.apache.atlas.plugin.service.ResourceLookupContext; +import org.apache.atlas.plugin.store.TagStore; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.apache.atlas.plugin.policyengine.RangerPolicyEngine.GROUP_PUBLIC; + +public class RangerServiceTag extends RangerBaseService { + + private static final Log LOG = LogFactory.getLog(RangerServiceTag.class); + + public static final String TAG_RESOURCE_NAME = "tag"; + public static final String RANGER_TAG_NAME_EXPIRES_ON = "EXPIRES_ON"; + public static final String RANGER_TAG_EXPIRY_CONDITION_NAME = "accessed-after-expiry"; + + private TagStore tagStore; + + public RangerServiceTag() { + super(); + } + + @Override + public void init(RangerServiceDef serviceDef, RangerService service) { + super.init(serviceDef, service); + } + + public void setTagStore(TagStore tagStore) { + this.tagStore = tagStore; + } + + @Override + public Map validateConfig() throws Exception { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerServiceTag.validateConfig(" + serviceName + " )"); + } + + Map ret = new HashMap<>(); + + ret.put("connectivityStatus", true); + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerServiceTag.validateConfig(" + serviceName + " ): " + ret); + } + + return ret; + } + + @Override + public List lookupResource(ResourceLookupContext context) throws Exception { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerServiceTag.lookupResource(" + context + ")"); + } + + List ret = new ArrayList<>(); + + if (context != null && StringUtils.equals(context.getResourceName(), TAG_RESOURCE_NAME)) { + try { + List tags = tagStore != null ? tagStore.getTagTypes() : null; + + if(CollectionUtils.isNotEmpty(tags)) { + List valuesToExclude = MapUtils.isNotEmpty(context.getResources()) ? context.getResources().get(TAG_RESOURCE_NAME) : null; + + if(CollectionUtils.isNotEmpty(valuesToExclude)) { + tags.removeAll(valuesToExclude); + } + + String valueToMatch = context.getUserInput(); + + if(StringUtils.isNotEmpty(valueToMatch)) { + if(! valueToMatch.endsWith("*")) { + valueToMatch += "*"; + } + + for (String tag : tags) { + if(FilenameUtils.wildcardMatch(tag, valueToMatch)) { + ret.add(tag); + } + } + } + } + } catch (Exception excp) { + LOG.error("RangerServiceTag.lookupResource()", excp); + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerServiceTag.lookupResource(): tag count=" + ret.size()); + } + + return ret; + } + + @Override + public List getDefaultRangerPolicies() throws Exception { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerServiceTag.getDefaultRangerPolicies() "); + } + + List ret = new ArrayList(); + + boolean isConditionDefFound = false; + + List policyConditionDefs = serviceDef.getPolicyConditions(); + + if (CollectionUtils.isNotEmpty(policyConditionDefs)) { + for (RangerServiceDef.RangerPolicyConditionDef conditionDef : policyConditionDefs) { + if (conditionDef.getName().equals(RANGER_TAG_EXPIRY_CONDITION_NAME)) { + isConditionDefFound = true; + break; + } + } + } + + if (isConditionDefFound) { + + ret = super.getDefaultRangerPolicies(); + String tagResourceName = null; + if (!serviceDef.getResources().isEmpty()) { + tagResourceName = serviceDef.getResources().get(0).getName(); + + for (RangerPolicy defaultPolicy : ret) { + + RangerPolicy.RangerPolicyResource tagPolicyResource = defaultPolicy.getResources().get(tagResourceName); + + if (tagPolicyResource != null) { + + String value = RANGER_TAG_NAME_EXPIRES_ON; + + tagPolicyResource.setValue(value); + defaultPolicy.setName(value); + defaultPolicy.setDescription("Policy for data with " + value + " tag"); + + List defaultPolicyItems = defaultPolicy.getPolicyItems(); + + for (RangerPolicy.RangerPolicyItem defaultPolicyItem : defaultPolicyItems) { + + List groups = new ArrayList(); + groups.add(GROUP_PUBLIC); + defaultPolicyItem.setGroups(groups); + + List policyItemConditions = new ArrayList(); + List values = new ArrayList(); + values.add("yes"); + RangerPolicy.RangerPolicyItemCondition policyItemCondition = new RangerPolicy.RangerPolicyItemCondition(RANGER_TAG_EXPIRY_CONDITION_NAME, values); + policyItemConditions.add(policyItemCondition); + + defaultPolicyItem.setConditions(policyItemConditions); + defaultPolicyItem.setDelegateAdmin(Boolean.FALSE); + } + + defaultPolicy.setDenyPolicyItems(defaultPolicyItems); + defaultPolicy.setPolicyItems(null); + } + } + } + } else { + LOG.error("RangerServiceTag.getDefaultRangerPolicies() - Cannot create default TAG policy: Cannot get tagPolicyConditionDef with name=" + RANGER_TAG_EXPIRY_CONDITION_NAME); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerServiceTag.getDefaultRangerPolicies() : " + ret); + } + return ret; + } +} diff --git a/auth-agents-common/src/main/java/org/apache/hadoop/security/KrbPasswordSaverLoginModule.java b/auth-agents-common/src/main/java/org/apache/hadoop/security/KrbPasswordSaverLoginModule.java new file mode 100644 index 00000000000..414ac34e787 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/hadoop/security/KrbPasswordSaverLoginModule.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + package org.apache.hadoop.security; + +import javax.security.auth.Subject; +import javax.security.auth.callback.CallbackHandler; +import javax.security.auth.login.LoginException; +import javax.security.auth.spi.LoginModule; +import java.util.Map; + +public class KrbPasswordSaverLoginModule implements LoginModule { + public static final String USERNAME_PARAM = "javax.security.auth.login.name"; + public static final String PASSWORD_PARAM = "javax.security.auth.login.password"; + + @SuppressWarnings("rawtypes") + private Map sharedState; + + public KrbPasswordSaverLoginModule() { + } + + @Override + public boolean abort() throws LoginException { + return true; + } + + @Override + public boolean commit() throws LoginException { + return true; + } + + @SuppressWarnings("unchecked") + @Override + public void initialize(Subject subject, CallbackHandler callbackhandler, Map sharedMap, Map options) { + + this.sharedState = sharedMap; + + String userName = (options != null) ? (String)options.get(USERNAME_PARAM) : null; + if (userName != null) { + this.sharedState.put(USERNAME_PARAM,userName); + } + String password = (options != null) ? (String)options.get(PASSWORD_PARAM) : null; + + if (password != null) { + this.sharedState.put(PASSWORD_PARAM,password.toCharArray()); + } + } + + @Override + public boolean login() throws LoginException { + return true; + } + + @Override + public boolean logout() throws LoginException { + return true; + } + +} diff --git a/auth-agents-common/src/main/java/org/apache/hadoop/security/SecureClientLogin.java b/auth-agents-common/src/main/java/org/apache/hadoop/security/SecureClientLogin.java new file mode 100644 index 00000000000..39ac3476a63 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/hadoop/security/SecureClientLogin.java @@ -0,0 +1,196 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hadoop.security; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; +import org.apache.hadoop.security.authentication.util.KerberosName; +import org.apache.hadoop.security.authentication.util.KerberosUtil; +import org.apache.hadoop.util.StringUtils; + +import javax.security.auth.Subject; +import javax.security.auth.login.AppConfigurationEntry; +import javax.security.auth.login.AppConfigurationEntry.LoginModuleControlFlag; +import javax.security.auth.login.LoginContext; +import javax.security.auth.login.LoginException; +import java.io.File; +import java.io.IOException; +import java.security.Principal; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +public class SecureClientLogin { + private static final Log LOG = LogFactory.getLog(SecureClientLogin.class); + public static final String HOSTNAME_PATTERN = "_HOST"; + + public synchronized static Subject loginUserFromKeytab(String user, String path) throws IOException { + try { + Subject subject = new Subject(); + SecureClientLoginConfiguration loginConf = new SecureClientLoginConfiguration(true, user, path); + LoginContext login = new LoginContext("hadoop-keytab-kerberos", subject, null, loginConf); + subject.getPrincipals().add(new User(user, AuthenticationMethod.KERBEROS, login)); + login.login(); + return login.getSubject(); + } catch (LoginException le) { + throw new IOException("Login failure for " + user + " from keytab " + path, le); + } + } + + public synchronized static Subject loginUserFromKeytab(String user, String path, String nameRules) throws IOException { + try { + Subject subject = new Subject(); + SecureClientLoginConfiguration loginConf = new SecureClientLoginConfiguration(true, user, path); + LoginContext login = new LoginContext("hadoop-keytab-kerberos", subject, null, loginConf); + KerberosName.setRules(nameRules); + subject.getPrincipals().add(new User(user, AuthenticationMethod.KERBEROS, login)); + login.login(); + return login.getSubject(); + } catch (LoginException le) { + throw new IOException("Login failure for " + user + " from keytab " + path, le); + } + } + + public synchronized static Subject loginUserWithPassword(String user, String password) throws IOException { + try { + Subject subject = new Subject(); + SecureClientLoginConfiguration loginConf = new SecureClientLoginConfiguration(false, user, password); + LoginContext login = new LoginContext("hadoop-keytab-kerberos", subject, null, loginConf); + subject.getPrincipals().add(new User(user, AuthenticationMethod.KERBEROS, login)); + login.login(); + return login.getSubject(); + } catch (LoginException le) { + throw new IOException("Login failure for " + user + " using password ****", le); + } + } + + public synchronized static Subject login(String user) throws IOException { + Subject subject = new Subject(); + subject.getPrincipals().add(new User(user)); + return subject; + } + + public static Set getUserPrincipals(Subject aSubject) { + if (aSubject != null) { + Set list = aSubject.getPrincipals(User.class); + if (list != null) { + Set ret = new HashSet<>(); + ret.addAll(list); + return ret; + } else { + return null; + } + } else { + return null; + } + } + + public static Principal createUserPrincipal(String aLoginName) { + return new User(aLoginName); + } + + public static boolean isKerberosCredentialExists(String principal, String keytabPath){ + boolean isValid = false; + if (keytabPath != null && !keytabPath.isEmpty()) { + File keytabFile = new File(keytabPath); + if (!keytabFile.exists()) { + LOG.warn(keytabPath + " doesn't exist."); + } else if (!keytabFile.canRead()) { + LOG.warn("Unable to read " + keytabPath + ". Please check the file access permissions for user"); + }else{ + isValid = true; + } + } else { + LOG.warn("Can't find keyTab Path : "+keytabPath); + } + if (!(principal != null && !principal.isEmpty() && isValid)) { + isValid = false; + LOG.warn("Can't find principal : "+principal); + } + return isValid; + } + + public static String getPrincipal(String principalConfig, String hostName) throws IOException { + String[] components = getComponents(principalConfig); + if (components == null || components.length != 3 || !HOSTNAME_PATTERN.equals(components[1])) { + return principalConfig; + } else { + if (hostName == null) { + throw new IOException("Can't replace " + HOSTNAME_PATTERN + " pattern since client ranger.service.host is null"); + } + return replacePattern(components, hostName); + } + } + + private static String[] getComponents(String principalConfig) { + if (principalConfig == null) + return null; + return principalConfig.split("[/@]"); + } + + private static String replacePattern(String[] components, String hostname) + throws IOException { + String fqdn = hostname; + if (fqdn == null || fqdn.isEmpty() || "0.0.0.0".equals(fqdn)) { + fqdn = java.net.InetAddress.getLocalHost().getCanonicalHostName(); + } + return components[0] + "/" + StringUtils.toLowerCase(fqdn) + "@" + components[2]; + } +} + +class SecureClientLoginConfiguration extends javax.security.auth.login.Configuration { + private Map kerberosOptions = new HashMap<>(); + private boolean usePassword; + + public SecureClientLoginConfiguration(boolean useKeyTab, String principal, String credential) { + kerberosOptions.put("principal", principal); + kerberosOptions.put("debug", "false"); + if (useKeyTab) { + kerberosOptions.put("useKeyTab", "true"); + kerberosOptions.put("keyTab", credential); + kerberosOptions.put("doNotPrompt", "true"); + } else { + usePassword = true; + kerberosOptions.put("useKeyTab", "false"); + kerberosOptions.put(KrbPasswordSaverLoginModule.USERNAME_PARAM, principal); + kerberosOptions.put(KrbPasswordSaverLoginModule.PASSWORD_PARAM, credential); + kerberosOptions.put("doNotPrompt", "false"); + kerberosOptions.put("useFirstPass", "true"); + kerberosOptions.put("tryFirstPass","false"); + } + kerberosOptions.put("storeKey", "true"); + kerberosOptions.put("refreshKrb5Config", "true"); + } + + @Override + public AppConfigurationEntry[] getAppConfigurationEntry(String appName) { + AppConfigurationEntry KEYTAB_KERBEROS_LOGIN = new AppConfigurationEntry(KerberosUtil.getKrb5LoginModuleName(), LoginModuleControlFlag.REQUIRED, kerberosOptions); + if (usePassword) { + AppConfigurationEntry KERBEROS_PWD_SAVER = new AppConfigurationEntry(KrbPasswordSaverLoginModule.class.getName(), LoginModuleControlFlag.REQUIRED, kerberosOptions); + return new AppConfigurationEntry[] { KERBEROS_PWD_SAVER, KEYTAB_KERBEROS_LOGIN }; + } + else { + return new AppConfigurationEntry[] { KEYTAB_KERBEROS_LOGIN }; + } + } + + +} diff --git a/auth-agents-common/src/main/resources/service-defs/atlas-servicedef-atlas.json b/auth-agents-common/src/main/resources/service-defs/atlas-servicedef-atlas.json new file mode 100644 index 00000000000..0539a562b9b --- /dev/null +++ b/auth-agents-common/src/main/resources/service-defs/atlas-servicedef-atlas.json @@ -0,0 +1,502 @@ +{ + "id": 15, + "name": "atlas", + "displayName": "atlas", + "implClass": "org.apache.atlas.services.atlas.RangerServiceAtlas", + "label": "Atlas Metadata Server", + "description": "Atlas Metadata Server", + "guid": "311a79b7-16f5-46f4-9829-a0224b9999c5", + "resources": [ + { + "itemId": 1, + "name": "type-category", + "type": "string", + "level": 10, + "mandatory": true, + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": true, + "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": "true", + "ignoreCase": "true" + }, + "label": "Type Catagory", + "description": "Type Catagory" + }, + { + "itemId": 2, + "name": "type", + "type": "string", + "level": 20, + "mandatory": true, + "parent": "type-category", + "isValidLeaf": true, + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": true, + "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": "true", + "ignoreCase": "false" + }, + "label": "Type Name", + "description": "Type Name", + "accessTypeRestrictions": ["type-read" ,"type-create", "type-update", "type-delete" ] + }, + { + "itemId": 3, + "name": "entity-type", + "type": "string", + "level": 10, + "mandatory": true, + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": true, + "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": "true", + "ignoreCase": "false" + }, + "label": "Entity Type", + "description": "Entity Type" + }, + { + "itemId": 4, + "name": "entity-classification", + "type": "string", + "level": 20, + "mandatory": true, + "parent": "entity-type", + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": true, + "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": "true", + "ignoreCase": "false" + }, + "label": "Entity Classification", + "description": "Entity Classification" + }, + { + "itemId": 5, + "name": "entity", + "type": "string", + "level": 30, + "mandatory": true, + "parent": "entity-classification", + "isValidLeaf": true, + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": true, + "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": "true", + "ignoreCase": "true" + }, + "label": "Entity ID", + "description": "Entity ID", + "accessTypeRestrictions": ["entity-read", "entity-create", "entity-update", "entity-delete"] + }, + { + "itemId": 6, + "name": "atlas-service", + "type": "string", + "level": 10, + "mandatory": true, + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": true, + "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": "true", + "ignoreCase": "true" + }, + "label": "Atlas Service", + "description": "Atlas Service", + "accessTypeRestrictions": ["admin-import", "admin-export", "admin-purge", "admin-audits", "admin-entity-audits", "admin-repair-index", "admin-task-cud"] + }, + { + "itemId": 7, + "name": "relationship-type", + "type": "string", + "level": 10, + "mandatory": true, + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": true, + "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": "true", + "ignoreCase": "false" + }, + "label": "Relationship Type", + "description": "Relationship Type" + }, + { + "itemId": 8, + "name": "end-one-entity-type", + "type": "string", + "level": 20, + "mandatory": true, + "parent": "relationship-type", + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": true, + "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": "true", + "ignoreCase": "false" + }, + "label": "End1 Entity Type", + "description": "End1 Entity Type" + }, + { + "itemId": 9, + "name": "end-one-entity-classification", + "type": "string", + "level": 30, + "mandatory": true, + "parent": "end-one-entity-type", + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": true, + "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": "true", + "ignoreCase": "false" + }, + "label": "End1 Entity Classification", + "description": "End1 Entity Classification" + }, + { + "itemId": 10, + "name": "end-one-entity", + "type": "string", + "level": 40, + "mandatory": true, + "parent": "end-one-entity-classification", + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": true, + "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": "true", + "ignoreCase": "true" + }, + "label": "End1 Entity ID", + "description": "End1 Entity ID" + }, + { + "itemId": 11, + "name": "end-two-entity-type", + "type": "string", + "level": 50, + "mandatory": true, + "parent": "end-one-entity", + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": true, + "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": "true", + "ignoreCase": "false" + }, + "label": "End2 Entity Type", + "description": "End2 Entity Type" + }, + { + "itemId": 12, + "name": "end-two-entity-classification", + "type": "string", + "level": 60, + "mandatory": true, + "parent": "end-two-entity-type", + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": true, + "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": "true", + "ignoreCase": "false" + }, + "label": "End2 Entity Classification", + "description": "End2 Entity Classification" + }, + { + "itemId": 13, + "name": "end-two-entity", + "type": "string", + "level": 70, + "mandatory": true, + "parent": "end-two-entity-classification", + "isValidLeaf": true, + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": true, + "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": "true", + "ignoreCase": "true" + }, + "label": "End2 Entity ID", + "description": "End2 Entity ID", + "accessTypeRestrictions": [ + "add-relationship", + "update-relationship", + "remove-relationship" + ] + }, + { + "itemId": 14, + "name": "entity-label", + "type": "string", + "level": 40, + "mandatory": true, + "parent": "entity", + "isValidLeaf": true, + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": true, + "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": "true", + "ignoreCase": "true" + }, + "label": "Label", + "description": "Label", + "accessTypeRestrictions": [ + "entity-add-label", + "entity-remove-label" + ] + }, + { + "itemId": 15, + "name": "entity-business-metadata", + "type": "string", + "level": 40, + "mandatory": true, + "parent": "entity", + "isValidLeaf": true, + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": true, + "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": "true", + "ignoreCase": "true" + }, + "label": "Business Metadata", + "description": "Business Metadata", + "accessTypeRestrictions": [ + "entity-update-business-metadata" + ] + }, + { + "itemId": 16, + "name": "classification", + "type": "string", + "level": 40, + "mandatory": true, + "parent": "entity", + "isValidLeaf": true, + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": true, + "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": "true", + "ignoreCase": "false" + }, + "label": "Targetted classifications", + "description": "Targetted classifications", + "accessTypeRestrictions": [ + "entity-add-classification", + "entity-update-classification", + "entity-remove-classification" + ] + } + ], + "accessTypes": [ + { + "itemId": 1, + "name": "type-create", + "label": "Create Type", + "impliedGrants": + [ + "type-read" + ] + }, + { + "itemId": 2, + "name": "type-update", + "label": "Update Type", + "impliedGrants": + [ + "type-read" + ] + }, + { + "itemId": 3, + "name": "type-delete", + "label": "Delete Type", + "impliedGrants": + [ + "type-read" + ] + }, + { + "itemId": 4, + "name": "entity-read", + "label": "Read Entity" + }, + { + "itemId": 5, + "name": "entity-create", + "label": "Create Entity" + }, + { + "itemId": 6, + "name": "entity-update", + "label": "Update Entity" + }, + { + "itemId": 7, + "name": "entity-delete", + "label": "Delete Entity" + }, + { + "itemId": 8, + "name": "entity-add-classification", + "label": "Add Classification" + }, + { + "itemId": 9, + "name": "entity-update-classification", + "label": "Update Classification" + }, + { + "itemId": 10, + "name": "entity-remove-classification", + "label": "Remove Classification" + }, + { + "itemId": 11, + "name": "admin-export", + "label": "Admin Export" + }, + { + "itemId": 12, + "name": "admin-import", + "label": "Admin Import" + }, + { + "itemId": 13, + "name": "add-relationship", + "label": "Add Relationship" + }, + { + "itemId": 14, + "name": "update-relationship", + "label": "Update Relationship" + }, + { + "itemId": 15, + "name": "remove-relationship", + "label": "Remove Relationship" + }, + { + "itemId": 16, + "name": "admin-purge", + "label": "Admin Purge" + }, + { + "itemId": 17, + "name": "entity-add-label", + "label": "Add Label" + }, + { + "itemId": 18, + "name": "entity-remove-label", + "label": "Remove Label" + }, + { + "itemId": 19, + "name": "entity-update-business-metadata", + "label": "Update Business Metadata" + }, + { + "itemId": 20, + "name": "type-read", + "label": "Read Type" + }, + { + "itemId": 21, + "name": "admin-audits", + "label": "Admin Audits" + }, + { + "itemId": 22, + "name": "admin-entity-audits", + "label": "Admin Entity Audits" + }, + { + "itemId": 23, + "name": "admin-repair-index", + "label": "Admin Repair Index" + }, + { + "itemId": 24, + "name": "admin-task-cud", + "label": "Admin task CUD API" + } + + ], + "configs": [ + { + "itemId": 1, + "name": "username", + "type": "string", + "mandatory": true, + "label": "Username" + }, + { + "itemId": 2, + "name": "password", + "type": "password", + "mandatory": true, + "label": "Password" + }, + { + "itemId": 3, + "name": "atlas.rest.address", + "type": "string", + "mandatory": true, + "defaultValue": "http://localhost:21000" + }, + { + "itemId": 4, + "name": "commonNameForCertificate", + "type": "string", + "mandatory": false, + "label": "Common Name for Certificate" + }, + + { + "itemId": 5, + "name": "ranger.plugin.audit.filters", + "type": "string", + "subType": "", + "mandatory": false, + "validationRegEx":"", + "validationMessage": "", + "uiHint":"", + "label": "Ranger Default Audit Filters", + "defaultValue": "[ {'accessResult': 'DENIED', 'isAudited': true}, {'users':['atlas'] ,'isAudited':false} ]" + } + ], + "options": { + "enableDenyAndExceptionsInPolicies": "true" + } +} diff --git a/auth-agents-common/src/main/resources/service-defs/atlas-servicedef-atlas_tag.json b/auth-agents-common/src/main/resources/service-defs/atlas-servicedef-atlas_tag.json new file mode 100644 index 00000000000..98ff10877f5 --- /dev/null +++ b/auth-agents-common/src/main/resources/service-defs/atlas-servicedef-atlas_tag.json @@ -0,0 +1,94 @@ +{ + "id":100, + "name": "tag", + "displayName": "tag", + "implClass": "org.apache.atlas.services.tag.RangerServiceTag", + "label": "TAG", + "description": "TAG Service Definition", + "guid": "0d047248-baff-4cf9-8e9e-d5d377284b2e", + "options": + { + "ui.pages":"tag-based-policies" + }, + "resources": + [ + { + "itemId":1, + "name": "tag", + "type": "string", + "level": 1, + "parent": "", + "mandatory": true, + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": false, + "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { "wildCard":false, "ignoreCase":false }, + "validationRegEx":"", + "validationMessage": "", + "uiHint":"{ \"singleValue\":true }", + "label": "TAG", + "description": "TAG" + } + ], + + "accessTypes": + [ + + ], + + "configs": + [ + { + "itemId":1, + "name": "ranger.plugin.audit.filters", + "type": "string", + "subType": "", + "mandatory": false, + "validationRegEx":"", + "validationMessage": "", + "uiHint":"", + "label": "Ranger Default Audit Filters", + "defaultValue": "[ {'accessResult': 'DENIED', 'isAudited': true} ]" + } + ], + + "enums": + [ + + ], + + "contextEnrichers": + [ + { + "itemId": 1, + "name" : "TagEnricher", + "enricher" : "org.apache.atlas.plugin.contextenricher.RangerTagEnricher", + "enricherOptions" : { + "tagRetrieverClassName": "org.apache.atlas.plugin.contextenricher.RangerAdminTagRetriever", + "tagRefresherPollingInterval": 60000 + } + } + ], + + "policyConditions": + [ + { + "itemId":1, + "name":"accessed-after-expiry", + "evaluator": "org.apache.atlas.plugin.conditionevaluator.RangerScriptTemplateConditionEvaluator", + "evaluatorOptions" : { "scriptTemplate":"ctx.isAccessedAfter('expiry_date');" }, + "uiHint": "{ \"singleValue\":true }", + "label":"Accessed after expiry_date (yes/no)?", + "description": "Accessed after expiry_date? (yes/no)" + }, + { + "itemId":2, + "name":"expression", + "evaluator": "org.apache.atlas.plugin.conditionevaluator.RangerScriptConditionEvaluator", + "evaluatorOptions" : {"engineName":"JavaScript", "ui.isMultiline":"true"}, + "label":"Enter boolean expression", + "description": "Boolean expression" + } + ] +} diff --git a/auth-agents-common/src/main/resources/service-defs/atlas-servicedef-heka.json b/auth-agents-common/src/main/resources/service-defs/atlas-servicedef-heka.json new file mode 100644 index 00000000000..2edbc1f8bbb --- /dev/null +++ b/auth-agents-common/src/main/resources/service-defs/atlas-servicedef-heka.json @@ -0,0 +1,244 @@ +{ + "name": "heka", + "displayName": "heka", + "implClass": "com.heka.ranger.service.RangerServiceHeka", + "label": "Heka", + "description": "Heka", + "id": 204, + "guid": "379a9fe5-1b6e-4091-a584-4890e245e6c2", + "isEnabled": true, + "createdBy": "Admin", + "updatedBy": "Admin", + "version": 1, + "options": { "enableDenyAndExceptionsInPolicies": "true" }, + "configs": [ + { + "itemId": 1, + "name": "service_url", + "type": "string", + "mandatory": true, + "validationRegEx": "", + "validationMessage": "", + "uiHint": "", + "label": "Service Url" + } + ], + "resources": [ + { + "itemId": 1, + "name": "entity-type", + "type": "string", + "level": 10, + "mandatory": true, + "lookupSupported": false, + "recursiveSupported": false, + "excludesSupported": true, + "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { "wildCard": "true", "ignoreCase": "false" }, + "validationRegEx": "", + "validationMessage": "", + "uiHint": "", + "label": "Entity Type", + "description": "Entity Type", + "accessTypeRestrictions": [], + "isValidLeaf": true + }, + { + "itemId": 2, + "name": "entity", + "type": "string", + "level": 20, + "parent": "entity-type", + "mandatory": true, + "lookupSupported": false, + "recursiveSupported": false, + "excludesSupported": true, + "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { "wildCard": "true", "ignoreCase": "true" }, + "validationRegEx": "", + "validationMessage": "", + "uiHint": "", + "label": "Entity ID", + "description": "Entity ID", + "accessTypeRestrictions": [], + "isValidLeaf": true + } + ], + "accessTypes": [ + { "itemId": 1, "name": "select", "label": "Select", "impliedGrants": [] }, + { + "itemId": 12, + "name": "all", + "label": "All", + "impliedGrants": ["select"] + } + ], + "policyConditions": [], + "contextEnrichers": [], + "enums": [], + "dataMaskDef": { + "maskTypes": [ + { + "itemId": 1, + "name": "MASK_REDACT", + "label": "Redact", + "description": "Replace lowercase with \u0027x\u0027, uppercase with \u0027X\u0027, digits with \u00270\u0027", + "transformer": "cast(regexp_replace(regexp_replace(regexp_replace({col},\u0027([A-Z])\u0027, \u0027X\u0027),\u0027([a-z])\u0027,\u0027x\u0027),\u0027([0-9])\u0027,\u00270\u0027) as {type})", + "dataMaskOptions": {} + }, + { + "itemId": 2, + "name": "MASK_SHOW_LAST_4", + "label": "Partial mask: show last 4", + "description": "Show last 4 characters; replace rest with \u0027X\u0027", + "transformer": "cast(concat(regexp_replace(RIGHT({col}, -4), \u0027.\u0027, \u0027X\u0027)), regexp_replace({col}, \u0027(.*)(.{4}$)\u0027, \u0027\\2\u0027)) as {type})", + "dataMaskOptions": {} + }, + { + "itemId": 3, + "name": "MASK_SHOW_FIRST_4", + "label": "Partial mask: show first 4", + "description": "Show first 4 characters; replace rest with \u0027x\u0027", + "transformer": "cast(regexp_replace({col}, \u0027(^.{4})(.*)\u0027, \u0027\\1XXXX\u0027) as {type})", + "dataMaskOptions": {} + }, + { + "itemId": 4, + "name": "MASK_HASH", + "label": "Hash", + "description": "Hash the value of a varchar with sha256", + "transformer": "cast(md5({col}) as {type})", + "dataMaskOptions": {} + }, + { + "itemId": 5, + "name": "MASK_NULL", + "label": "Nullify", + "description": "Replace with NULL", + "dataMaskOptions": {} + }, + { + "itemId": 6, + "name": "MASK_NONE", + "label": "Unmasked (retain original value)", + "description": "No masking", + "dataMaskOptions": {} + }, + { + "itemId": 12, + "name": "MASK_DATE_SHOW_YEAR", + "label": "Date: show only year", + "description": "Date: show only year", + "transformer": "YEAR({col})", + "dataMaskOptions": {} + }, + { + "itemId": 13, + "name": "CUSTOM", + "label": "Custom", + "description": "Custom", + "dataMaskOptions": {} + } + ], + "accessTypes": [ + { + "itemId": 1, + "name": "select", + "label": "Select", + "impliedGrants": [] + } + ], + "resources": [ + { + "itemId": 1, + "name": "entity-type", + "type": "string", + "level": 10, + "mandatory": true, + "lookupSupported": false, + "recursiveSupported": false, + "excludesSupported": true, + "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { "wildCard": "true", "ignoreCase": "false" }, + "validationRegEx": "", + "validationMessage": "", + "uiHint": "", + "label": "Entity Type", + "description": "Entity Type", + "accessTypeRestrictions": [], + "isValidLeaf": true + }, + { + "itemId": 2, + "name": "entity", + "type": "string", + "level": 20, + "parent": "entity-type", + "mandatory": true, + "lookupSupported": false, + "recursiveSupported": false, + "excludesSupported": true, + "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { "wildCard": "true", "ignoreCase": "true" }, + "validationRegEx": "", + "validationMessage": "", + "uiHint": "", + "label": "Entity ID", + "description": "Entity ID", + "accessTypeRestrictions": [], + "isValidLeaf": true + } + ] + }, + "rowFilterDef": { + "accessTypes": [ + { + "itemId": 1, + "name": "select", + "label": "Select", + "impliedGrants": [] + } + ], + "resources": [ + { + "itemId": 1, + "name": "entity-type", + "type": "string", + "level": 10, + "mandatory": true, + "lookupSupported": false, + "recursiveSupported": false, + "excludesSupported": false, + "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { "wildCard": "true", "ignoreCase": "false" }, + "validationRegEx": "", + "validationMessage": "", + "uiHint": "{ \"singleValue\":true }", + "label": "Entity Type", + "description": "Entity Type", + "accessTypeRestrictions": [], + "isValidLeaf": false + }, + { + "itemId": 2, + "name": "entity", + "type": "string", + "level": 20, + "parent": "entity-type", + "mandatory": true, + "lookupSupported": false, + "recursiveSupported": false, + "excludesSupported": false, + "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { "wildCard": "true", "ignoreCase": "true" }, + "validationRegEx": "", + "validationMessage": "", + "uiHint": "{ \"singleValue\":true }", + "label": "Entity ID", + "description": "Entity ID", + "accessTypeRestrictions": [], + "isValidLeaf": true + } + ] + } +} \ No newline at end of file diff --git a/auth-agents-cred/pom.xml b/auth-agents-cred/pom.xml new file mode 100644 index 00000000000..d2f57732302 --- /dev/null +++ b/auth-agents-cred/pom.xml @@ -0,0 +1,52 @@ + + + + + apache-atlas + org.apache.atlas + 3.0.0-SNAPSHOT + + 4.0.0 + + auth-agents-cred + + + 8 + 8 + + + + + + org.apache.hadoop + hadoop-common + ${hadoop.version} + + + net.minidev + json-smart + + + + + + + \ No newline at end of file diff --git a/auth-agents-cred/src/main/java/org/apache/atlas/authorization/credutils/CredentialsProviderUtil.java b/auth-agents-cred/src/main/java/org/apache/atlas/authorization/credutils/CredentialsProviderUtil.java new file mode 100644 index 00000000000..277c15e6039 --- /dev/null +++ b/auth-agents-cred/src/main/java/org/apache/atlas/authorization/credutils/CredentialsProviderUtil.java @@ -0,0 +1,165 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.authorization.credutils; + +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.KerberosCredentials; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.client.config.AuthSchemes; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.atlas.authorization.credutils.kerberos.KerberosCredentialsProvider; +import org.apache.atlas.authorization.credutils.kerberos.KeytabJaasConf; +import org.ietf.jgss.GSSCredential; +import org.ietf.jgss.GSSException; +import org.ietf.jgss.GSSManager; +import org.ietf.jgss.GSSName; +import org.ietf.jgss.Oid; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.security.auth.Subject; +import javax.security.auth.kerberos.KerberosPrincipal; +import javax.security.auth.kerberos.KerberosTicket; +import javax.security.auth.login.Configuration; +import javax.security.auth.login.LoginContext; +import java.math.BigDecimal; +import java.security.AccessControlContext; +import java.security.AccessController; +import java.security.PrivilegedActionException; +import java.security.PrivilegedExceptionAction; +import java.util.Collections; +import java.util.Date; +import java.util.Set; + +public class CredentialsProviderUtil { + private static final Logger logger = LoggerFactory.getLogger(CredentialsProviderUtil.class); + private static final Oid SPNEGO_OID = getSpnegoOid(); + private static final String CRED_CONF_NAME = "ESClientLoginConf"; + public static long ticketExpireTime80 = 0; + + private static Oid getSpnegoOid() { + Oid oid = null; + try { + oid = new Oid("1.3.6.1.5.5.2"); + } catch (GSSException gsse) { + throw new RuntimeException(gsse); + } + return oid; + } + + public static KerberosCredentialsProvider getKerberosCredentials(String user, String password){ + KerberosCredentialsProvider credentialsProvider = new KerberosCredentialsProvider(); + final GSSManager gssManager = GSSManager.getInstance(); + try { + final GSSName gssUserPrincipalName = gssManager.createName(user, GSSName.NT_USER_NAME); + Subject subject = login(user, password); + final AccessControlContext acc = AccessController.getContext(); + final GSSCredential credential = doAsPrivilegedWrapper(subject, + (PrivilegedExceptionAction) () -> gssManager.createCredential(gssUserPrincipalName, + GSSCredential.DEFAULT_LIFETIME, SPNEGO_OID, GSSCredential.INITIATE_ONLY), + acc); + credentialsProvider.setCredentials( + new AuthScope(AuthScope.ANY_HOST, AuthScope.ANY_PORT, AuthScope.ANY_REALM, AuthSchemes.SPNEGO), + new KerberosCredentials(credential)); + } catch (GSSException e) { + logger.error("GSSException:", e); + throw new RuntimeException(e); + } catch (PrivilegedActionException e) { + logger.error("PrivilegedActionException:", e); + throw new RuntimeException(e); + } + return credentialsProvider; + } + + public static synchronized KerberosTicket getTGT(Subject subject) { + Set tickets = subject.getPrivateCredentials(KerberosTicket.class); + for(KerberosTicket ticket: tickets) { + KerberosPrincipal server = ticket.getServer(); + if (server.getName().equals("krbtgt/" + server.getRealm() + "@" + server.getRealm())) { + if (logger.isDebugEnabled()) { + logger.debug("Client principal is \"" + ticket.getClient().getName() + "\"."); + logger.debug("Server principal is \"" + ticket.getServer().getName() + "\"."); + } + return ticket; + } + } + return null; + } + + public static Boolean ticketWillExpire(KerberosTicket ticket){ + long ticketExpireTime = ticket.getEndTime().getTime(); + long currrentTime = new Date().getTime(); + if (logger.isDebugEnabled()) { + logger.debug("TicketExpireTime is:" + ticketExpireTime); + logger.debug("currrentTime is:" + currrentTime); + } + if (ticketExpireTime80 == 0) { + long timeDiff = ticketExpireTime - currrentTime; + long timeDiff20 = Math.round(Float.parseFloat(BigDecimal.valueOf(timeDiff * 0.2).toPlainString())); + ticketExpireTime80 = ticketExpireTime - timeDiff20; + } + if (logger.isDebugEnabled()) { + logger.debug("ticketExpireTime80 is:" + ticketExpireTime80); + } + if (currrentTime > ticketExpireTime80) { + if (logger.isDebugEnabled()) { + logger.debug("Current time is more than 80% of Ticket Expire Time!!"); + } + ticketExpireTime80 = 0; + return true; + } + return false; + } + + public static synchronized Subject login(String userPrincipalName, String keytabPath) throws PrivilegedActionException { + Subject sub = AccessController.doPrivileged((PrivilegedExceptionAction) () -> { + final Subject subject = new Subject(false, Collections.singleton(new KerberosPrincipal(userPrincipalName)), + Collections.emptySet(), Collections.emptySet()); + Configuration conf = new KeytabJaasConf(userPrincipalName, keytabPath, false); + + LoginContext loginContext = new LoginContext(CRED_CONF_NAME, subject, null, conf); + loginContext.login(); + return loginContext.getSubject(); + }); + return sub; + } + + + static T doAsPrivilegedWrapper(final Subject subject, final PrivilegedExceptionAction action, final AccessControlContext acc) + throws PrivilegedActionException { + try { + return AccessController.doPrivileged((PrivilegedExceptionAction) () -> Subject.doAsPrivileged(subject, action, acc)); + } catch (PrivilegedActionException pae) { + if (pae.getCause() instanceof PrivilegedActionException) { + throw (PrivilegedActionException) pae.getCause(); + } + throw pae; + } + } + + public static CredentialsProvider getBasicCredentials(String user, String password) { + CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials(AuthScope.ANY, + new UsernamePasswordCredentials(user, password)); + return credentialsProvider; + } + +} diff --git a/auth-agents-cred/src/main/java/org/apache/atlas/authorization/credutils/kerberos/AbstractJaasConf.java b/auth-agents-cred/src/main/java/org/apache/atlas/authorization/credutils/kerberos/AbstractJaasConf.java new file mode 100644 index 00000000000..fddb8a5e6a5 --- /dev/null +++ b/auth-agents-cred/src/main/java/org/apache/atlas/authorization/credutils/kerberos/AbstractJaasConf.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.authorization.credutils.kerberos; + +import javax.security.auth.login.AppConfigurationEntry; +import javax.security.auth.login.Configuration; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +public abstract class AbstractJaasConf extends Configuration { + private final String userPrincipalName; + private final boolean enableDebugLogs; + + public AbstractJaasConf(final String userPrincipalName, final boolean enableDebugLogs) { + this.userPrincipalName = userPrincipalName; + this.enableDebugLogs = enableDebugLogs; + } + + @Override + public AppConfigurationEntry[] getAppConfigurationEntry(final String name) { + final Map options = new HashMap<>(); + options.put("principal", userPrincipalName); + options.put("isInitiator", Boolean.TRUE.toString()); + options.put("storeKey", Boolean.TRUE.toString()); + options.put("debug", Boolean.toString(enableDebugLogs)); + addOptions(options); + return new AppConfigurationEntry[] { new AppConfigurationEntry("com.sun.security.auth.module.Krb5LoginModule", + AppConfigurationEntry.LoginModuleControlFlag.REQUIRED, Collections.unmodifiableMap(options)) }; + } + + abstract void addOptions(Map options); +} diff --git a/auth-agents-cred/src/main/java/org/apache/atlas/authorization/credutils/kerberos/KerberosCredentialsProvider.java b/auth-agents-cred/src/main/java/org/apache/atlas/authorization/credutils/kerberos/KerberosCredentialsProvider.java new file mode 100644 index 00000000000..1a87548ce6f --- /dev/null +++ b/auth-agents-cred/src/main/java/org/apache/atlas/authorization/credutils/kerberos/KerberosCredentialsProvider.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.authorization.credutils.kerberos; + +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.Credentials; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.client.config.AuthSchemes; + +public class KerberosCredentialsProvider implements CredentialsProvider { + private AuthScope authScope; + private Credentials credentials; + + @Override + public void setCredentials(AuthScope authscope, Credentials credentials) { + if (authscope.getScheme().regionMatches(true, 0, AuthSchemes.SPNEGO, 0, AuthSchemes.SPNEGO.length()) == false) { + throw new IllegalArgumentException("Only " + AuthSchemes.SPNEGO + " auth scheme is supported in AuthScope"); + } + this.authScope = authscope; + this.credentials = credentials; + } + + @Override + public Credentials getCredentials(AuthScope authscope) { + assert this.authScope != null && authscope != null; + return authscope.match(this.authScope) > -1 ? this.credentials : null; + } + + @Override + public void clear() { + this.authScope = null; + this.credentials = null; + } + +} diff --git a/auth-agents-cred/src/main/java/org/apache/atlas/authorization/credutils/kerberos/KeytabJaasConf.java b/auth-agents-cred/src/main/java/org/apache/atlas/authorization/credutils/kerberos/KeytabJaasConf.java new file mode 100644 index 00000000000..2b0efb70a2d --- /dev/null +++ b/auth-agents-cred/src/main/java/org/apache/atlas/authorization/credutils/kerberos/KeytabJaasConf.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.authorization.credutils.kerberos; + +import java.util.Map; + +public class KeytabJaasConf extends AbstractJaasConf { + private final String keytabFilePath; + + public KeytabJaasConf(final String userPrincipalName, final String keytabFilePath, final boolean enableDebugLogs) { + super(userPrincipalName, enableDebugLogs); + this.keytabFilePath = keytabFilePath; + } + + public void addOptions(final Map options) { + options.put("useKeyTab", Boolean.TRUE.toString()); + options.put("keyTab", keytabFilePath); + options.put("doNotPrompt", Boolean.TRUE.toString()); + } + +} diff --git a/auth-agents-cred/src/main/java/org/apache/atlas/authorization/hadoop/utils/RangerCredentialProvider.java b/auth-agents-cred/src/main/java/org/apache/atlas/authorization/hadoop/utils/RangerCredentialProvider.java new file mode 100644 index 00000000000..58d84ba78e3 --- /dev/null +++ b/auth-agents-cred/src/main/java/org/apache/atlas/authorization/hadoop/utils/RangerCredentialProvider.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.authorization.hadoop.utils; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.alias.CredentialProvider; +import org.apache.hadoop.security.alias.CredentialProviderFactory; + +import java.util.List; + + +public final class RangerCredentialProvider { + + private static final Log LOG = LogFactory.getLog(RangerCredentialProvider.class); + + private static final RangerCredentialProvider CRED_PROVIDER = new RangerCredentialProvider(); + + protected RangerCredentialProvider() { + // + } + + public static RangerCredentialProvider getInstance() { + return CRED_PROVIDER; + } + + public String getCredentialString(String url, String alias) { + if (url != null && alias != null) { + List providers = getCredentialProviders(url); + if (providers != null) { + for (CredentialProvider provider : providers) { + try { + CredentialProvider.CredentialEntry credEntry = provider.getCredentialEntry(alias); + if (credEntry != null && credEntry.getCredential() != null) { + return new String(credEntry.getCredential()); + } + } catch (Exception ie) { + LOG.error("Unable to get the Credential Provider from the Configuration", ie); + } + } + } + } + return null; + } + + List getCredentialProviders(String url) { + if (url != null) { + try { + Configuration conf = new Configuration(); + conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, url); + return CredentialProviderFactory.getProviders(conf); + } catch (Exception ie) { + LOG.error("Unable to get the Credential Provider from the Configuration", ie); + } + } + return null; + } + +} diff --git a/auth-audits/pom.xml b/auth-audits/pom.xml new file mode 100644 index 00000000000..c19e686740e --- /dev/null +++ b/auth-audits/pom.xml @@ -0,0 +1,99 @@ + + + + + apache-atlas + org.apache.atlas + 3.0.0-SNAPSHOT + + 4.0.0 + + auth-audits + + + 8 + 8 + 2.9.0 + 1.5.8 + + + + + org.apache.atlas + auth-agents-cred + ${project.version} + + + + commons-lang + commons-lang + ${commons-lang.version} + + + + org.apache.solr + solr-solrj + ${solr.version} + + + io.netty + * + + + org.eclipse.jetty.http2 + * + + + org.apache.commons + commons-math3 + + + commons-io + commons-io + + + org.apache.httpcomponents + * + + + org.apache.zookeeper + * + + + org.codehaus.woodstox + * + + + org.eclipse.jetty + * + + + + + + org.elasticsearch.client + elasticsearch-rest-high-level-client + ${elasticsearch.version} + + + + + \ No newline at end of file diff --git a/auth-audits/src/main/java/org/apache/atlas/audit/destination/AuditDestination.java b/auth-audits/src/main/java/org/apache/atlas/audit/destination/AuditDestination.java new file mode 100644 index 00000000000..d0b2b7d2a94 --- /dev/null +++ b/auth-audits/src/main/java/org/apache/atlas/audit/destination/AuditDestination.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.audit.destination; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.audit.provider.BaseAuditHandler; + +import java.util.Properties; + +/** + * This class needs to be extended by anyone who wants to build custom + * destination + */ +public abstract class AuditDestination extends BaseAuditHandler { + private static final Log logger = LogFactory.getLog(AuditDestination.class); + + public AuditDestination() { + logger.info("AuditDestination() enter"); + } + + /* + * (non-Javadoc) + * + * @see + * org.apache.ranger.audit.provider.AuditProvider#init(java.util.Properties, + * java.lang.String) + */ + @Override + public void init(Properties prop, String basePropertyName) { + super.init(prop, basePropertyName); + } + + /* + * (non-Javadoc) + * + * @see org.apache.ranger.audit.provider.AuditProvider#flush() + */ + @Override + public void flush() { + + } + + @Override + public void start() { + + } + + @Override + public void stop() { + + } + + @Override + public void waitToComplete() { + + } + + @Override + public void waitToComplete(long timeout) { + + } + +} diff --git a/auth-audits/src/main/java/org/apache/atlas/audit/destination/ElasticSearchAuditDestination.java b/auth-audits/src/main/java/org/apache/atlas/audit/destination/ElasticSearchAuditDestination.java new file mode 100644 index 00000000000..114d3fe4eb9 --- /dev/null +++ b/auth-audits/src/main/java/org/apache/atlas/audit/destination/ElasticSearchAuditDestination.java @@ -0,0 +1,331 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.audit.destination; + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.http.HttpHost; +import org.apache.http.auth.AuthSchemeProvider; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.client.config.AuthSchemes; +import org.apache.http.config.Lookup; +import org.apache.http.config.RegistryBuilder; +import org.apache.http.impl.auth.SPNegoSchemeFactory; +import org.apache.atlas.audit.model.AuditEventBase; +import org.apache.atlas.audit.model.AuthzAuditEvent; +import org.apache.atlas.audit.provider.MiscUtil; +import org.apache.atlas.authorization.credutils.CredentialsProviderUtil; +import org.apache.atlas.authorization.credutils.kerberos.KerberosCredentialsProvider; +import org.elasticsearch.action.admin.indices.open.OpenIndexRequest; +import org.elasticsearch.action.bulk.BulkItemResponse; +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestClientBuilder; +import org.elasticsearch.client.RestHighLevelClient; + +import javax.security.auth.Subject; +import javax.security.auth.kerberos.KerberosTicket; +import java.io.File; +import java.security.PrivilegedActionException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Date; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +public class ElasticSearchAuditDestination extends AuditDestination { + private static final Log LOG = LogFactory.getLog(ElasticSearchAuditDestination.class); + + public static final String CONFIG_URLS = "urls"; + public static final String CONFIG_PORT = "port"; + public static final String CONFIG_USER = "user"; + public static final String CONFIG_PWRD = "password"; + public static final String CONFIG_PROTOCOL = "protocol"; + public static final String CONFIG_INDEX = "index"; + public static final String CONFIG_PREFIX = "ranger.audit.elasticsearch"; + public static final String DEFAULT_INDEX = "ranger-audit"; + + private String index = "index"; + private volatile RestHighLevelClient client = null; + private String protocol; + private String user; + private int port; + private String password; + private String hosts; + private Subject subject; + + public ElasticSearchAuditDestination() { + propPrefix = CONFIG_PREFIX; + } + + + @Override + public void init(Properties props, String propPrefix) { + super.init(props, propPrefix); + this.protocol = getStringProperty(props, propPrefix + "." + CONFIG_PROTOCOL, "http"); + this.user = getStringProperty(props, propPrefix + "." + CONFIG_USER, ""); + this.password = getStringProperty(props, propPrefix + "." + CONFIG_PWRD, ""); + this.port = MiscUtil.getIntProperty(props, propPrefix + "." + CONFIG_PORT, 9200); + this.index = getStringProperty(props, propPrefix + "." + CONFIG_INDEX, DEFAULT_INDEX); + this.hosts = getHosts(); + LOG.info("Connecting to ElasticSearch: " + connectionString()); + getClient(); // Initialize client + } + + private String connectionString() { + return String.format(Locale.ROOT, "User:%s, %s://%s:%s/%s", user, protocol, hosts, port, index); + } + + @Override + public void stop() { + super.stop(); + logStatus(); + } + + @Override + public boolean log(Collection events) { + boolean ret = false; + try { + logStatusIfRequired(); + addTotalCount(events.size()); + + RestHighLevelClient client = getClient(); + if (null == client) { + // ElasticSearch is still not initialized. So need return error + addDeferredCount(events.size()); + return ret; + } + + ArrayList eventList = new ArrayList<>(events); + BulkRequest bulkRequest = new BulkRequest(); + try { + for (AuditEventBase event : eventList) { + AuthzAuditEvent authzEvent = (AuthzAuditEvent) event; + String id = authzEvent.getEventId(); + Map doc = toDoc(authzEvent); + bulkRequest.add(new IndexRequest(index).id(id).source(doc)); + } + } catch (Exception ex) { + addFailedCount(eventList.size()); + logFailedEvent(eventList, ex); + } + BulkResponse response = client.bulk(bulkRequest, RequestOptions.DEFAULT); + if (response.status().getStatus() >= 400) { + addFailedCount(eventList.size()); + logFailedEvent(eventList, "HTTP " + response.status().getStatus()); + } else { + BulkItemResponse[] items = response.getItems(); + for (int i = 0; i < items.length; i++) { + AuditEventBase itemRequest = eventList.get(i); + BulkItemResponse itemResponse = items[i]; + if (itemResponse.isFailed()) { + addFailedCount(1); + logFailedEvent(Arrays.asList(itemRequest), itemResponse.getFailureMessage()); + } else { + if(LOG.isDebugEnabled()) { + LOG.debug(String.format("Indexed %s", itemRequest.getEventKey())); + } + addSuccessCount(1); + ret = true; + } + } + } + } catch (Throwable t) { + addDeferredCount(events.size()); + logError("Error sending message to ElasticSearch", t); + } + return ret; + } + + /* + * (non-Javadoc) + * + * @see org.apache.ranger.audit.provider.AuditProvider#flush() + */ + @Override + public void flush() { + + } + + public boolean isAsync() { + return true; + } + + synchronized RestHighLevelClient getClient() { + if (client == null) { + synchronized (ElasticSearchAuditDestination.class) { + if (client == null) { + client = newClient(); + } + } + } + if (subject != null) { + KerberosTicket ticket = CredentialsProviderUtil.getTGT(subject); + try { + if (new Date().getTime() > ticket.getEndTime().getTime()){ + client = null; + CredentialsProviderUtil.ticketExpireTime80 = 0; + newClient(); + } else if (CredentialsProviderUtil.ticketWillExpire(ticket)) { + subject = CredentialsProviderUtil.login(user, password); + } + } catch (PrivilegedActionException e) { + LOG.error("PrivilegedActionException:", e); + throw new RuntimeException(e); + } + } + return client; + } + + private final AtomicLong lastLoggedAt = new AtomicLong(0); + + public static RestClientBuilder getRestClientBuilder(String urls, String protocol, String user, String password, int port) { + RestClientBuilder restClientBuilder = RestClient.builder( + MiscUtil.toArray(urls, ",").stream() + .map(x -> new HttpHost(x, port, protocol)) + .toArray(i -> new HttpHost[i]) + ); + if (StringUtils.isNotBlank(user) && StringUtils.isNotBlank(password) && !user.equalsIgnoreCase("NONE") && !password.equalsIgnoreCase("NONE")) { + if (password.contains("keytab") && new File(password).exists()) { + final KerberosCredentialsProvider credentialsProvider = + CredentialsProviderUtil.getKerberosCredentials(user, password); + Lookup authSchemeRegistry = RegistryBuilder.create() + .register(AuthSchemes.SPNEGO, new SPNegoSchemeFactory()).build(); + restClientBuilder.setHttpClientConfigCallback(clientBuilder -> { + clientBuilder.setDefaultCredentialsProvider(credentialsProvider); + clientBuilder.setDefaultAuthSchemeRegistry(authSchemeRegistry); + return clientBuilder; + }); + } else { + final CredentialsProvider credentialsProvider = + CredentialsProviderUtil.getBasicCredentials(user, password); + restClientBuilder.setHttpClientConfigCallback(clientBuilder -> + clientBuilder.setDefaultCredentialsProvider(credentialsProvider)); + } + } else { + LOG.error("ElasticSearch Credentials not provided!!"); + final CredentialsProvider credentialsProvider = null; + restClientBuilder.setHttpClientConfigCallback(clientBuilder -> + clientBuilder.setDefaultCredentialsProvider(credentialsProvider)); + } + return restClientBuilder; + } + + private RestHighLevelClient newClient() { + try { + if (StringUtils.isNotBlank(user) && StringUtils.isNotBlank(password) && password.contains("keytab") && new File(password).exists()) { + subject = CredentialsProviderUtil.login(user, password); + } + RestClientBuilder restClientBuilder = + getRestClientBuilder(hosts, protocol, user, password, port); + RestHighLevelClient restHighLevelClient = new RestHighLevelClient(restClientBuilder); + if (LOG.isDebugEnabled()) { + LOG.debug("Initialized client"); + } + boolean exits = false; + try { + exits = restHighLevelClient.indices().open(new OpenIndexRequest(this.index), RequestOptions.DEFAULT).isShardsAcknowledged(); + } catch (Exception e) { + LOG.warn("Error validating index " + this.index); + } + if(exits) { + if (LOG.isDebugEnabled()) { + LOG.debug("Index exists"); + } + } else { + LOG.info("Index does not exist"); + } + return restHighLevelClient; + } catch (Throwable t) { + lastLoggedAt.updateAndGet(lastLoggedAt -> { + long now = System.currentTimeMillis(); + long elapsed = now - lastLoggedAt; + if (elapsed > TimeUnit.MINUTES.toMillis(1)) { + LOG.fatal("Can't connect to ElasticSearch server: " + connectionString(), t); + return now; + } else { + return lastLoggedAt; + } + }); + return null; + } + } + + private String getHosts() { + String urls = MiscUtil.getStringProperty(props, propPrefix + "." + CONFIG_URLS); + if (urls != null) { + urls = urls.trim(); + } + if ("NONE".equalsIgnoreCase(urls)) { + urls = null; + } + return urls; + } + + private String getStringProperty(Properties props, String propName, String defaultValue) { + String value = MiscUtil.getStringProperty(props, propName); + if (null == value) { + return defaultValue; + } + return value; + } + + Map toDoc(AuthzAuditEvent auditEvent) { + Map doc = new HashMap(); + doc.put("id", auditEvent.getEventId()); + doc.put("access", auditEvent.getAccessType()); + doc.put("enforcer", auditEvent.getAclEnforcer()); + doc.put("agent", auditEvent.getAgentId()); + doc.put("repo", auditEvent.getRepositoryName()); + doc.put("sess", auditEvent.getSessionId()); + doc.put("reqUser", auditEvent.getUser()); + doc.put("reqEntityGuid", auditEvent.getEntityGuid()); + doc.put("reqData", auditEvent.getRequestData()); + doc.put("resource", auditEvent.getResourcePath()); + doc.put("cliIP", auditEvent.getClientIP()); + doc.put("logType", auditEvent.getLogType()); + doc.put("result", auditEvent.getAccessResult()); + doc.put("policyId", auditEvent.getPolicyId()); + doc.put("repoType", auditEvent.getRepositoryType()); + doc.put("resType", auditEvent.getResourceType()); + doc.put("reason", auditEvent.getResultReason()); + doc.put("action", auditEvent.getAction()); + doc.put("evtTime", auditEvent.getEventTime()); + doc.put("seq_num", auditEvent.getSeqNum()); + doc.put("event_count", auditEvent.getEventCount()); + doc.put("event_dur_ms", auditEvent.getEventDurationMS()); + doc.put("tags", auditEvent.getTags()); + doc.put("cluster", auditEvent.getClusterName()); + doc.put("zoneName", auditEvent.getZoneName()); + doc.put("agentHost", auditEvent.getAgentHostname()); + doc.put("policyVersion", auditEvent.getPolicyVersion()); + return doc; + } + +} diff --git a/auth-audits/src/main/java/org/apache/atlas/audit/destination/FileAuditDestination.java b/auth-audits/src/main/java/org/apache/atlas/audit/destination/FileAuditDestination.java new file mode 100644 index 00000000000..1bbc04edde1 --- /dev/null +++ b/auth-audits/src/main/java/org/apache/atlas/audit/destination/FileAuditDestination.java @@ -0,0 +1,256 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.audit.destination; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.audit.model.AuditEventBase; +import org.apache.atlas.audit.provider.MiscUtil; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.PrintWriter; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Date; +import java.util.List; +import java.util.Properties; + +/** + * This class write the logs to local file + */ +public class FileAuditDestination extends AuditDestination { + private static final Log logger = LogFactory + .getLog(FileAuditDestination.class); + + public static final String PROP_FILE_LOCAL_DIR = "dir"; + public static final String PROP_FILE_LOCAL_FILE_NAME_FORMAT = "filename.format"; + public static final String PROP_FILE_FILE_ROLLOVER = "file.rollover.sec"; + + String baseFolder = null; + String fileFormat = null; + int fileRolloverSec = 24 * 60 * 60; // In seconds + private String logFileNameFormat; + + boolean initDone = false; + + private File logFolder; + PrintWriter logWriter = null; + + private Date fileCreateTime = null; + + private String currentFileName; + + private boolean isStopped = false; + + @Override + public void init(Properties prop, String propPrefix) { + super.init(prop, propPrefix); + + // Initialize properties for this class + // Initial folder and file properties + String logFolderProp = MiscUtil.getStringProperty(props, propPrefix + + "." + PROP_FILE_LOCAL_DIR); + logFileNameFormat = MiscUtil.getStringProperty(props, propPrefix + "." + + PROP_FILE_LOCAL_FILE_NAME_FORMAT); + fileRolloverSec = MiscUtil.getIntProperty(props, propPrefix + "." + + PROP_FILE_FILE_ROLLOVER, fileRolloverSec); + + if (logFolderProp == null || logFolderProp.isEmpty()) { + logger.error("File destination folder is not configured. Please set " + + propPrefix + + "." + + PROP_FILE_LOCAL_DIR + + ". name=" + + getName()); + return; + } + logFolder = new File(logFolderProp); + if (!logFolder.isDirectory()) { + logFolder.mkdirs(); + if (!logFolder.isDirectory()) { + logger.error("FileDestination folder not found and can't be created. folder=" + + logFolder.getAbsolutePath() + ", name=" + getName()); + return; + } + } + logger.info("logFolder=" + logFolder + ", name=" + getName()); + + if (logFileNameFormat == null || logFileNameFormat.isEmpty()) { + logFileNameFormat = "%app-type%_ranger_audit.log"; + } + + logger.info("logFileNameFormat=" + logFileNameFormat + ", destName=" + + getName()); + + initDone = true; + } + + @Override + synchronized public boolean logJSON(Collection events) { + logStatusIfRequired(); + addTotalCount(events.size()); + + if (isStopped) { + logError("log() called after stop was requested. name=" + getName()); + addDeferredCount(events.size()); + return false; + } + + try { + PrintWriter out = getLogFileStream(); + for (String event : events) { + out.println(event); + } + out.flush(); + } catch (Throwable t) { + addDeferredCount(events.size()); + logError("Error writing to log file.", t); + return false; + } + addSuccessCount(events.size()); + return true; + } + + /* + * (non-Javadoc) + * + * @see + * org.apache.ranger.audit.provider.AuditProvider#log(java.util.Collection) + */ + @Override + public boolean log(Collection events) { + if (isStopped) { + addTotalCount(events.size()); + addDeferredCount(events.size()); + logError("log() called after stop was requested. name=" + getName()); + return false; + } + List jsonList = new ArrayList(); + for (AuditEventBase event : events) { + try { + jsonList.add(MiscUtil.stringify(event)); + } catch (Throwable t) { + addTotalCount(1); + addFailedCount(1); + logFailedEvent(event); + logger.error("Error converting to JSON. event=" + event); + } + } + return logJSON(jsonList); + + } + + /* + * (non-Javadoc) + * + * @see org.apache.ranger.audit.provider.AuditProvider#start() + */ + @Override + public void start() { + // Nothing to do here. We will open the file when the first log request + // comes + } + + @Override + synchronized public void stop() { + isStopped = true; + if (logWriter != null) { + try { + logWriter.flush(); + logWriter.close(); + } catch (Throwable t) { + logger.error("Error on closing log writter. Exception will be ignored. name=" + + getName() + ", fileName=" + currentFileName); + } + logWriter = null; + } + logStatus(); + } + + // Helper methods in this class + synchronized private PrintWriter getLogFileStream() throws Exception { + closeFileIfNeeded(); + + // Either there are no open log file or the previous one has been rolled + // over + if (logWriter == null) { + Date currentTime = new Date(); + // Create a new file + String fileName = MiscUtil.replaceTokens(logFileNameFormat, + currentTime.getTime()); + File outLogFile = new File(logFolder, fileName); + if (outLogFile.exists()) { + // Let's try to get the next available file + int i = 0; + while (true) { + i++; + int lastDot = fileName.lastIndexOf('.'); + String baseName = fileName.substring(0, lastDot); + String extension = fileName.substring(lastDot); + String newFileName = baseName + "." + i + extension; + File newLogFile = new File(logFolder, newFileName); + if (!newLogFile.exists()) { + // Move the file + if (!outLogFile.renameTo(newLogFile)) { + logger.error("Error renameing file. " + outLogFile + + " to " + newLogFile); + } + break; + } + } + } + if (!outLogFile.exists()) { + logger.info("Creating new file. destName=" + getName() + + ", fileName=" + fileName); + // Open the file + logWriter = new PrintWriter(new BufferedWriter(new FileWriter( + outLogFile))); + } else { + logWriter = new PrintWriter(new BufferedWriter(new FileWriter( + outLogFile, true))); + } + fileCreateTime = new Date(); + currentFileName = outLogFile.getPath(); + } + return logWriter; + } + + private void closeFileIfNeeded() { + if (logWriter == null) { + return; + } + if (System.currentTimeMillis() - fileCreateTime.getTime() > fileRolloverSec * 1000) { + logger.info("Closing file. Rolling over. name=" + getName() + + ", fileName=" + currentFileName); + try { + logWriter.flush(); + logWriter.close(); + } catch (Throwable t) { + logger.error("Error on closing log writter. Exception will be ignored. name=" + + getName() + ", fileName=" + currentFileName); + } + logWriter = null; + currentFileName = null; + } + } + +} diff --git a/auth-audits/src/main/java/org/apache/atlas/audit/destination/HDFSAuditDestination.java b/auth-audits/src/main/java/org/apache/atlas/audit/destination/HDFSAuditDestination.java new file mode 100644 index 00000000000..1e048ea1073 --- /dev/null +++ b/auth-audits/src/main/java/org/apache/atlas/audit/destination/HDFSAuditDestination.java @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.audit.destination; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.audit.model.AuditEventBase; +import org.apache.atlas.audit.provider.AuditWriterFactory; +import org.apache.atlas.audit.provider.MiscUtil; +import org.apache.atlas.audit.utils.RangerAuditWriter; + +import java.io.File; +import java.security.PrivilegedAction; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +/** + * This class write the logs to local file + */ +public class HDFSAuditDestination extends AuditDestination { + private static final Log logger = LogFactory + .getLog(HDFSAuditDestination.class); + + private Map auditConfigs = null; + private String auditProviderName = null; + private RangerAuditWriter auditWriter = null; + private boolean initDone = false; + private boolean isStopped = false; + + @Override + public void init(Properties prop, String propPrefix) { + super.init(prop, propPrefix); + this.auditProviderName = getName(); + this.auditConfigs = configProps; + + try { + this.auditWriter = getWriter(); + this.initDone = true; + } catch (Exception e) { + logger.error("Error while getting Audit writer", e); + } + } + + @Override + synchronized public boolean logJSON(final Collection events) { + logStatusIfRequired(); + addTotalCount(events.size()); + + if (!initDone) { + addDeferredCount(events.size()); + return false; + } + if (isStopped) { + addDeferredCount(events.size()); + logError("log() called after stop was requested. name=" + getName()); + return false; + } + try { + boolean ret = auditWriter.log(events); + if (!ret) { + addDeferredCount(events.size()); + return false; + } + } catch (Throwable t) { + addDeferredCount(events.size()); + logError("Error writing to log file.", t); + return false; + } finally { + logger.info("Flushing HDFS audit. Event Size:" + events.size()); + if (auditWriter != null) { + flush(); + } + } + addSuccessCount(events.size()); + return true; + } + + @Override + synchronized public boolean logFile(final File file) { + logStatusIfRequired(); + if (!initDone) { + return false; + } + if (isStopped) { + logError("log() called after stop was requested. name=" + getName()); + return false; + } + + try { + boolean ret = auditWriter.logFile(file); + if (!ret) { + return false; + } + } catch (Throwable t) { + logError("Error writing to log file.", t); + return false; + } finally { + logger.info("Flushing HDFS audit. File:" + file.getAbsolutePath() + file.getName()); + if (auditWriter != null) { + flush(); + } + } + return true; + } + + @Override + public void flush() { + logger.info("Flush called. name=" + getName()); + MiscUtil.executePrivilegedAction(new PrivilegedAction() { + @Override + public Void run() { + auditWriter.flush(); + return null; + } + }); + } + + /* + * (non-Javadoc) + * + * @see + * org.apache.ranger.audit.provider.AuditProvider#log(java.util.Collection) + */ + @Override + public boolean log(Collection events) { + if (isStopped) { + logStatusIfRequired(); + addTotalCount(events.size()); + addDeferredCount(events.size()); + logError("log() called after stop was requested. name=" + getName()); + return false; + } + List jsonList = new ArrayList(); + for (AuditEventBase event : events) { + try { + jsonList.add(MiscUtil.stringify(event)); + } catch (Throwable t) { + logger.error("Error converting to JSON. event=" + event); + addTotalCount(1); + addFailedCount(1); + logFailedEvent(event); + } + } + return logJSON(jsonList); + + } + + /* + * (non-Javadoc) + * + * @see org.apache.ranger.audit.provider.AuditProvider#start() + */ + @Override + public void start() { + // Nothing to do here. We will open the file when the first log request + // comes + } + + @Override + synchronized public void stop() { + auditWriter.stop(); + logStatus(); + isStopped = true; + } + + public RangerAuditWriter getWriter() throws Exception { + AuditWriterFactory auditWriterFactory = AuditWriterFactory.getInstance(); + auditWriterFactory.init(props, propPrefix, auditProviderName, auditConfigs); + return auditWriterFactory.getAuditWriter(); + } +} diff --git a/auth-audits/src/main/java/org/apache/atlas/audit/destination/Log4JAuditDestination.java b/auth-audits/src/main/java/org/apache/atlas/audit/destination/Log4JAuditDestination.java new file mode 100644 index 00000000000..cb3c80db28b --- /dev/null +++ b/auth-audits/src/main/java/org/apache/atlas/audit/destination/Log4JAuditDestination.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.audit.destination; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.audit.model.AuditEventBase; +import org.apache.atlas.audit.provider.MiscUtil; + +import java.util.Collection; +import java.util.Properties; + +public class Log4JAuditDestination extends AuditDestination { + private static final Log logger = LogFactory + .getLog(Log4JAuditDestination.class); + + private static Log auditLogger = null; + + public static final String PROP_LOG4J_LOGGER = "logger"; + public static final String DEFAULT_LOGGER_PREFIX = "ranger.audit"; + private String loggerName = null; + + public Log4JAuditDestination() { + logger.info("Log4JAuditDestination() called."); + + } + + @Override + public void init(Properties prop, String propPrefix) { + super.init(prop, propPrefix); + loggerName = MiscUtil.getStringProperty(props, propPrefix + "." + + PROP_LOG4J_LOGGER); + if (loggerName == null || loggerName.isEmpty()) { + loggerName = DEFAULT_LOGGER_PREFIX + "." + getName(); + logger.info("Logger property " + propPrefix + "." + + PROP_LOG4J_LOGGER + " was not set. Constructing default=" + + loggerName); + } + logger.info("Logger name for " + getName() + " is " + loggerName); + auditLogger = LogFactory.getLog(loggerName); + logger.info("Done initializing logger for audit. name=" + getName() + + ", loggerName=" + loggerName); + } + + + @Override + public void stop() { + super.stop(); + logStatus(); + } + + @Override + public boolean log(AuditEventBase event) { + if (!auditLogger.isInfoEnabled()) { + logStatusIfRequired(); + addTotalCount(1); + return true; + } + + if (event != null) { + String eventStr = MiscUtil.stringify(event); + logJSON(eventStr); + } + return true; + } + + @Override + public boolean log(Collection events) { + if (!auditLogger.isInfoEnabled()) { + logStatusIfRequired(); + addTotalCount(events.size()); + return true; + } + + for (AuditEventBase event : events) { + log(event); + } + return true; + } + + @Override + public boolean logJSON(String event) { + logStatusIfRequired(); + addTotalCount(1); + if (!auditLogger.isInfoEnabled()) { + return true; + } + + if (event != null) { + auditLogger.info(event); + addSuccessCount(1); + } + return true; + } + + @Override + public boolean logJSON(Collection events) { + if (!auditLogger.isInfoEnabled()) { + logStatusIfRequired(); + addTotalCount(events.size()); + return true; + } + + for (String event : events) { + logJSON(event); + } + return false; + } + +} diff --git a/auth-audits/src/main/java/org/apache/atlas/audit/destination/SolrAuditDestination.java b/auth-audits/src/main/java/org/apache/atlas/audit/destination/SolrAuditDestination.java new file mode 100644 index 00000000000..7b42926ca96 --- /dev/null +++ b/auth-audits/src/main/java/org/apache/atlas/audit/destination/SolrAuditDestination.java @@ -0,0 +1,489 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.audit.destination; + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.audit.model.AuditEventBase; +import org.apache.atlas.audit.model.AuthzAuditEvent; +import org.apache.atlas.audit.provider.MiscUtil; +import org.apache.atlas.audit.utils.InMemoryJAASConfiguration; +import org.apache.atlas.audit.utils.KerberosAction; +import org.apache.atlas.audit.utils.KerberosJAASConfigUser; +import org.apache.atlas.audit.utils.KerberosUser; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.solr.client.solrj.impl.HttpClientUtil; +import org.apache.solr.client.solrj.impl.Krb5HttpClientBuilder; +import org.apache.solr.client.solrj.impl.LBHttpSolrClient; +import org.apache.solr.client.solrj.impl.SolrHttpClientBuilder; +import org.apache.solr.client.solrj.response.UpdateResponse; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrInputDocument; + +import javax.net.ssl.KeyManager; +import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManager; +import javax.net.ssl.TrustManagerFactory; +import javax.security.auth.login.LoginException; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.security.KeyManagementException; +import java.security.KeyStore; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.security.PrivilegedExceptionAction; +import java.security.SecureRandom; +import java.security.UnrecoverableKeyException; +import java.security.cert.CertificateException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Optional; +import java.util.Properties; + + +public class SolrAuditDestination extends AuditDestination { + private static final Log LOG = LogFactory + .getLog(SolrAuditDestination.class); + + public static final String PROP_SOLR_URLS = "urls"; + public static final String PROP_SOLR_ZK = "zookeepers"; + public static final String PROP_SOLR_COLLECTION = "collection"; + public static final String PROP_SOLR_FORCE_USE_INMEMORY_JAAS_CONFIG = "force.use.inmemory.jaas.config"; + + public static final String DEFAULT_COLLECTION_NAME = "ranger_audits"; + public static final String PROP_JAVA_SECURITY_AUTH_LOGIN_CONFIG = "java.security.auth.login.config"; + + private volatile SolrClient solrClient = null; + private volatile KerberosUser kerberosUser = null; + + public SolrAuditDestination() { + } + + @Override + public void init(Properties props, String propPrefix) { + LOG.info("init() called"); + super.init(props, propPrefix); + init(); + connect(); + } + + @Override + public void stop() { + LOG.info("SolrAuditDestination.stop() called.."); + logStatus(); + + if (solrClient != null) { + try { + solrClient.close(); + } catch (IOException ioe) { + LOG.error("Error while stopping slor!", ioe); + } finally { + solrClient = null; + } + } + + if (kerberosUser != null) { + try { + kerberosUser.logout(); + } catch (LoginException excp) { + LOG.error("Error logging out keytab user", excp); + } finally { + kerberosUser = null; + } + } + } + + synchronized void connect() { + SolrClient me = solrClient; + if (me == null) { + synchronized(SolrAuditDestination.class) { + me = solrClient; + if (solrClient == null) { + KeyManager[] kmList = getKeyManagers(); + TrustManager[] tmList = getTrustManagers(); + SSLContext sslContext = getSSLContext(kmList, tmList); + if(sslContext != null) { + SSLContext.setDefault(sslContext); + } + String urls = MiscUtil.getStringProperty(props, propPrefix + + "." + PROP_SOLR_URLS); + if (urls != null) { + urls = urls.trim(); + } + if (urls != null && urls.equalsIgnoreCase("NONE")) { + urls = null; + } + List solrURLs = new ArrayList(); + String zkHosts = null; + solrURLs = MiscUtil.toArray(urls, ","); + zkHosts = MiscUtil.getStringProperty(props, propPrefix + "." + + PROP_SOLR_ZK); + if (zkHosts != null && zkHosts.equalsIgnoreCase("NONE")) { + zkHosts = null; + } + String collectionName = MiscUtil.getStringProperty(props, + propPrefix + "." + PROP_SOLR_COLLECTION); + if (collectionName == null + || collectionName.equalsIgnoreCase("none")) { + collectionName = DEFAULT_COLLECTION_NAME; + } + + LOG.info("Solr zkHosts=" + zkHosts + ", solrURLs=" + urls + + ", collectionName=" + collectionName); + + if (zkHosts != null && !zkHosts.isEmpty()) { + LOG.info("Connecting to solr cloud using zkHosts=" + + zkHosts); + try { + // Instantiate + Krb5HttpClientBuilder krbBuild = new Krb5HttpClientBuilder(); + SolrHttpClientBuilder kb = krbBuild.getBuilder(); + HttpClientUtil.setHttpClientBuilder(kb); + + final List zkhosts = new ArrayList(Arrays.asList(zkHosts.split(","))); + final CloudSolrClient solrCloudClient = MiscUtil.executePrivilegedAction(new PrivilegedExceptionAction() { + @Override + public CloudSolrClient run() throws Exception { + CloudSolrClient solrCloudClient = new CloudSolrClient.Builder(zkhosts, Optional.empty()).build(); + return solrCloudClient; + }; + }); + + solrCloudClient.setDefaultCollection(collectionName); + me = solrClient = solrCloudClient; + } catch (Throwable t) { + LOG.fatal("Can't connect to Solr server. ZooKeepers=" + + zkHosts, t); + } + } else if (solrURLs != null && !solrURLs.isEmpty()) { + try { + LOG.info("Connecting to Solr using URLs=" + solrURLs); + Krb5HttpClientBuilder krbBuild = new Krb5HttpClientBuilder(); + SolrHttpClientBuilder kb = krbBuild.getBuilder(); + HttpClientUtil.setHttpClientBuilder(kb); + final List solrUrls = solrURLs; + final LBHttpSolrClient lbSolrClient = MiscUtil.executePrivilegedAction(new PrivilegedExceptionAction() { + @Override + public LBHttpSolrClient run() throws Exception { + LBHttpSolrClient.Builder builder = new LBHttpSolrClient.Builder(); + builder.withBaseSolrUrl(solrUrls.get(0)); + builder.withConnectionTimeout(1000); + LBHttpSolrClient lbSolrClient = builder.build(); + return lbSolrClient; + }; + }); + + for (int i = 1; i < solrURLs.size(); i++) { + lbSolrClient.addSolrServer(solrURLs.get(i)); + } + me = solrClient = lbSolrClient; + } catch (Throwable t) { + LOG.fatal("Can't connect to Solr server. URL=" + + solrURLs, t); + } + } + } + } + } + } + + @Override + public boolean log(Collection events) { + boolean ret = false; + try { + logStatusIfRequired(); + addTotalCount(events.size()); + + if (solrClient == null) { + connect(); + if (solrClient == null) { + // Solr is still not initialized. So need return error + addDeferredCount(events.size()); + return ret; + } + } + + final Collection docs = new ArrayList(); + for (AuditEventBase event : events) { + AuthzAuditEvent authzEvent = (AuthzAuditEvent) event; + // Convert AuditEventBase to Solr document + SolrInputDocument document = toSolrDoc(authzEvent); + docs.add(document); + } + try { + final UpdateResponse response = addDocsToSolr(solrClient, docs); + + if (response.getStatus() != 0) { + addFailedCount(events.size()); + logFailedEvent(events, response.toString()); + } else { + addSuccessCount(events.size()); + ret = true; + } + } catch (SolrException ex) { + addFailedCount(events.size()); + logFailedEvent(events, ex); + } + } catch (Throwable t) { + addDeferredCount(events.size()); + logError("Error sending message to Solr", t); + } + return ret; + } + + /* + * (non-Javadoc) + * + * @see org.apache.ranger.audit.provider.AuditProvider#flush() + */ + @Override + public void flush() { + + } + + SolrInputDocument toSolrDoc(AuthzAuditEvent auditEvent) { + SolrInputDocument doc = new SolrInputDocument(); + doc.addField("id", auditEvent.getEventId()); + doc.addField("access", auditEvent.getAccessType()); + doc.addField("enforcer", auditEvent.getAclEnforcer()); + doc.addField("agent", auditEvent.getAgentId()); + doc.addField("repo", auditEvent.getRepositoryName()); + doc.addField("sess", auditEvent.getSessionId()); + doc.addField("reqUser", auditEvent.getUser()); + doc.addField("reqData", auditEvent.getRequestData()); + doc.addField("resource", auditEvent.getResourcePath()); + doc.addField("cliIP", auditEvent.getClientIP()); + doc.addField("logType", auditEvent.getLogType()); + doc.addField("result", auditEvent.getAccessResult()); + doc.addField("policy", auditEvent.getPolicyId()); + doc.addField("repoType", auditEvent.getRepositoryType()); + doc.addField("resType", auditEvent.getResourceType()); + doc.addField("reason", auditEvent.getResultReason()); + doc.addField("action", auditEvent.getAction()); + doc.addField("evtTime", auditEvent.getEventTime()); + doc.addField("seq_num", auditEvent.getSeqNum()); + doc.setField("event_count", auditEvent.getEventCount()); + doc.setField("event_dur_ms", auditEvent.getEventDurationMS()); + doc.setField("tags", auditEvent.getTags()); + doc.setField("cluster", auditEvent.getClusterName()); + doc.setField("zoneName", auditEvent.getZoneName()); + doc.setField("agentHost", auditEvent.getAgentHostname()); + doc.setField("policyVersion", auditEvent.getPolicyVersion()); + + return doc; + } + + public boolean isAsync() { + return true; + } + + private void init() { + LOG.info("==>SolrAuditDestination.init()" ); + try { + // SolrJ requires "java.security.auth.login.config" property to be set to identify itself that it is kerberized. So using a dummy property for it + // Acutal solrclient JAAS configs are read from the ranger--audit.xml present in components conf folder and set by InMemoryJAASConfiguration + // Refer InMemoryJAASConfiguration doc for JAAS Configuration + String confFileName = System.getProperty(PROP_JAVA_SECURITY_AUTH_LOGIN_CONFIG); + LOG.info("In solrAuditDestination.init() : JAAS Configuration set as [" + confFileName + "]"); + if ( System.getProperty(PROP_JAVA_SECURITY_AUTH_LOGIN_CONFIG) == null ) { + if ( MiscUtil.getBooleanProperty(props, propPrefix + "." + PROP_SOLR_FORCE_USE_INMEMORY_JAAS_CONFIG,false) ) { + System.setProperty(PROP_JAVA_SECURITY_AUTH_LOGIN_CONFIG, "/dev/null"); + } else { + LOG.warn("No Client JAAS config present in solr audit config. Ranger Audit to Kerberized Solr will fail..."); + } + } + + LOG.info("Loading SolrClient JAAS config from Ranger audit config if present..."); + + InMemoryJAASConfiguration conf = InMemoryJAASConfiguration.init(props); + + KerberosUser kerberosUser = new KerberosJAASConfigUser("Client", conf); + + if (kerberosUser.getPrincipal() != null) { + this.kerberosUser = kerberosUser; + } + } catch (Exception e) { + LOG.error("ERROR: Unable to load SolrClient JAAS config from Audit config file. Audit to Kerberized Solr will fail...", e); + } finally { + String confFileName = System.getProperty(PROP_JAVA_SECURITY_AUTH_LOGIN_CONFIG); + LOG.info("In solrAuditDestination.init() (finally) : JAAS Configuration set as [" + confFileName + "]"); + } + LOG.info("<==SolrAuditDestination.init()" ); + } + + private KeyManager[] getKeyManagers() { + KeyManager[] kmList = null; + String credentialProviderPath = MiscUtil.getStringProperty(props, RANGER_POLICYMGR_CLIENT_KEY_FILE_CREDENTIAL); + String keyStoreAlias = RANGER_POLICYMGR_CLIENT_KEY_FILE_CREDENTIAL_ALIAS; + String keyStoreFile = MiscUtil.getStringProperty(props, RANGER_POLICYMGR_CLIENT_KEY_FILE); + String keyStoreFilepwd = MiscUtil.getCredentialString(credentialProviderPath, keyStoreAlias); + if (StringUtils.isNotEmpty(keyStoreFile) && StringUtils.isNotEmpty(keyStoreFilepwd)) { + InputStream in = null; + + try { + in = getFileInputStream(keyStoreFile); + + if (in != null) { + String keyStoreType = MiscUtil.getStringProperty(props, RANGER_POLICYMGR_CLIENT_KEY_FILE_TYPE); + keyStoreType = StringUtils.isNotEmpty(keyStoreType) ? keyStoreType : RANGER_POLICYMGR_CLIENT_KEY_FILE_TYPE_DEFAULT; + KeyStore keyStore = KeyStore.getInstance(keyStoreType); + + keyStore.load(in, keyStoreFilepwd.toCharArray()); + + KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance(RANGER_SSL_KEYMANAGER_ALGO_TYPE); + + keyManagerFactory.init(keyStore, keyStoreFilepwd.toCharArray()); + + kmList = keyManagerFactory.getKeyManagers(); + } else { + LOG.error("Unable to obtain keystore from file [" + keyStoreFile + "]"); + } + } catch (KeyStoreException e) { + LOG.error("Unable to obtain from KeyStore :" + e.getMessage(), e); + } catch (NoSuchAlgorithmException e) { + LOG.error("SSL algorithm is NOT available in the environment", e); + } catch (CertificateException e) { + LOG.error("Unable to obtain the requested certification ", e); + } catch (FileNotFoundException e) { + LOG.error("Unable to find the necessary SSL Keystore Files", e); + } catch (IOException e) { + LOG.error("Unable to read the necessary SSL Keystore Files", e); + } catch (UnrecoverableKeyException e) { + LOG.error("Unable to recover the key from keystore", e); + } finally { + close(in, keyStoreFile); + } + } + + return kmList; + } + + private TrustManager[] getTrustManagers() { + TrustManager[] tmList = null; + String credentialProviderPath = MiscUtil.getStringProperty(props, RANGER_POLICYMGR_TRUSTSTORE_FILE_CREDENTIAL); + String trustStoreAlias = RANGER_POLICYMGR_TRUSTSTORE_FILE_CREDENTIAL_ALIAS; + String trustStoreFile = MiscUtil.getStringProperty(props, RANGER_POLICYMGR_TRUSTSTORE_FILE); + String trustStoreFilepwd = MiscUtil.getCredentialString(credentialProviderPath, trustStoreAlias); + if (StringUtils.isNotEmpty(trustStoreFile) && StringUtils.isNotEmpty(trustStoreFilepwd)) { + InputStream in = null; + + try { + in = getFileInputStream(trustStoreFile); + + if (in != null) { + String trustStoreType = MiscUtil.getStringProperty(props, RANGER_POLICYMGR_TRUSTSTORE_FILE_TYPE); + trustStoreType = StringUtils.isNotEmpty(trustStoreType) ? trustStoreType : RANGER_POLICYMGR_TRUSTSTORE_FILE_TYPE_DEFAULT; + KeyStore trustStore = KeyStore.getInstance(trustStoreType); + + trustStore.load(in, trustStoreFilepwd.toCharArray()); + + TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(RANGER_SSL_TRUSTMANAGER_ALGO_TYPE); + + trustManagerFactory.init(trustStore); + + tmList = trustManagerFactory.getTrustManagers(); + } else { + LOG.error("Unable to obtain truststore from file [" + trustStoreFile + "]"); + } + } catch (KeyStoreException e) { + LOG.error("Unable to obtain from KeyStore", e); + } catch (NoSuchAlgorithmException e) { + LOG.error("SSL algorithm is NOT available in the environment :" + e.getMessage(), e); + } catch (CertificateException e) { + LOG.error("Unable to obtain the requested certification :" + e.getMessage(), e); + } catch (FileNotFoundException e) { + LOG.error("Unable to find the necessary SSL TrustStore File:" + trustStoreFile, e); + } catch (IOException e) { + LOG.error("Unable to read the necessary SSL TrustStore Files :" + trustStoreFile, e); + } finally { + close(in, trustStoreFile); + } + } + + return tmList; + } + + private SSLContext getSSLContext(KeyManager[] kmList, TrustManager[] tmList) { + SSLContext sslContext = null; + try { + sslContext = SSLContext.getInstance(RANGER_SSL_CONTEXT_ALGO_TYPE); + if (sslContext != null) { + sslContext.init(kmList, tmList, new SecureRandom()); + } + } catch (NoSuchAlgorithmException e) { + LOG.error("SSL algorithm is not available in the environment", e); + } catch (KeyManagementException e) { + LOG.error("Unable to initialise the SSLContext", e); + } + return sslContext; + } + + private UpdateResponse addDocsToSolr(final SolrClient solrClient, final Collection docs) throws Exception { + final UpdateResponse ret; + + try { + final PrivilegedExceptionAction action = () -> solrClient.add(docs); + + if (kerberosUser != null) { + // execute the privileged action as the given keytab user + final KerberosAction kerberosAction = new KerberosAction<>(kerberosUser, action, LOG); + + ret = (UpdateResponse) kerberosAction.execute(); + } else { + ret = action.run(); + } + } catch (Exception e) { + throw e; + } + + return ret; + } + + private InputStream getFileInputStream(String fileName) throws IOException { + InputStream in = null; + if (StringUtils.isNotEmpty(fileName)) { + File file = new File(fileName); + if (file != null && file.exists()) { + in = new FileInputStream(file); + } else { + in = ClassLoader.getSystemResourceAsStream(fileName); + } + } + return in; + } + + private void close(InputStream str, String filename) { + if (str != null) { + try { + str.close(); + } catch (IOException excp) { + LOG.error("Error while closing file: [" + filename + "]", excp); + } + } + } +} diff --git a/auth-audits/src/main/java/org/apache/atlas/audit/model/AuditEventBase.java b/auth-audits/src/main/java/org/apache/atlas/audit/model/AuditEventBase.java new file mode 100644 index 00000000000..2d9fa2f9460 --- /dev/null +++ b/auth-audits/src/main/java/org/apache/atlas/audit/model/AuditEventBase.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.audit.model; + +import java.util.Date; + +public abstract class AuditEventBase { + + protected AuditEventBase() { + } + + public abstract String getEventKey(); + public abstract Date getEventTime (); + public abstract void setEventCount(long eventCount); + public abstract void setEventDurationMS(long eventDurationMS); +} diff --git a/auth-audits/src/main/java/org/apache/atlas/audit/model/AuthzAuditEvent.java b/auth-audits/src/main/java/org/apache/atlas/audit/model/AuthzAuditEvent.java new file mode 100644 index 00000000000..f0bae5480c7 --- /dev/null +++ b/auth-audits/src/main/java/org/apache/atlas/audit/model/AuthzAuditEvent.java @@ -0,0 +1,585 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.audit.model; + +import com.google.gson.annotations.SerializedName; +import org.apache.commons.lang.StringUtils; + +import java.util.Date; +import java.util.HashSet; +import java.util.Set; + +public class AuthzAuditEvent extends AuditEventBase { + protected static String FIELD_SEPARATOR = ";"; + + protected static final int MAX_ACTION_FIELD_SIZE = 1800; + protected static final int MAX_REQUEST_DATA_FIELD_SIZE = 1800; + + @SerializedName("repoType") + protected int repositoryType = 0; + + @SerializedName("repo") + protected String repositoryName = null; + + @SerializedName("reqUser") + protected String user = null; + + @SerializedName("reqEntityGuid") + protected String entityGuid = null; + + @SerializedName("evtTime") + protected Date eventTime = new Date(); + + @SerializedName("access") + protected String accessType = null; + + @SerializedName("resource") + protected String resourcePath = null; + + @SerializedName("resType") + protected String resourceType = null; + + @SerializedName("action") + protected String action = null; + + @SerializedName("result") + protected short accessResult = 0; // 0 - DENIED; 1 - ALLOWED; HTTP return + // code + + @SerializedName("agent") + protected String agentId = null; + + @SerializedName("policyId") + protected String policyId = "-1"; + + @SerializedName("reason") + protected String resultReason = null; + + @SerializedName("enforcer") + protected String aclEnforcer = null; + + @SerializedName("sess") + protected String sessionId = null; + + @SerializedName("cliType") + protected String clientType = null; + + @SerializedName("cliIP") + protected String clientIP = null; + + @SerializedName("reqData") + protected String requestData = null; + + @SerializedName("agentHost") + protected String agentHostname = null; + + @SerializedName("logType") + protected String logType = null; + + @SerializedName("id") + protected String eventId = null; + + /** + * This to ensure order within a session. Order not guaranteed across + * processes and hosts + */ + @SerializedName("seq_num") + protected long seqNum = 0; + + @SerializedName("event_count") + protected long eventCount = 1; + + @SerializedName("event_dur_ms") + protected long eventDurationMS = 0; + + @SerializedName("tags") + protected Set tags = new HashSet<>(); + + @SerializedName("additional_info") + protected String additionalInfo; + + @SerializedName("cluster_name") + protected String clusterName; + + @SerializedName("zone_name") + protected String zoneName; + + @SerializedName("policy_version") + protected Long policyVersion; + + public AuthzAuditEvent() { + super(); + + this.repositoryType = 0; + } + + public AuthzAuditEvent(int repositoryType, String repositoryName, + String user, Date eventTime, String accessType, + String resourcePath, String resourceType, String action, + short accessResult, String agentId, String policyId, + String resultReason, String aclEnforcer, String sessionId, + String clientType, String clientIP, String requestData, String clusterName) { + this(repositoryType, repositoryName, user, eventTime, accessType, resourcePath, resourceType, action, accessResult, agentId, + policyId, resultReason, aclEnforcer, sessionId, clientType, clientIP, requestData, clusterName, null); + } + + public AuthzAuditEvent(int repositoryType, String repositoryName, + String user, Date eventTime, String accessType, + String resourcePath, String resourceType, String action, + short accessResult, String agentId, String policyId, + String resultReason, String aclEnforcer, String sessionId, + String clientType, String clientIP, String requestData, String clusterName, String zoneName) { + this(repositoryType, repositoryName, user, eventTime, accessType, resourcePath, resourceType, action, accessResult, agentId, + policyId, resultReason, aclEnforcer, sessionId, clientType, clientIP, requestData, clusterName, zoneName, null); + + } + + public AuthzAuditEvent(int repositoryType, String repositoryName, + String user, Date eventTime, String accessType, + String resourcePath, String resourceType, String action, + short accessResult, String agentId, String policyId, + String resultReason, String aclEnforcer, String sessionId, + String clientType, String clientIP, String requestData, String clusterName, String zoneName, Long policyVersion) { + this.repositoryType = repositoryType; + this.repositoryName = repositoryName; + this.user = user; + this.eventTime = eventTime; + this.accessType = accessType; + this.resourcePath = resourcePath; + this.resourceType = resourceType; + this.action = action; + this.accessResult = accessResult; + this.agentId = agentId; + this.policyId = policyId; + this.resultReason = resultReason; + this.aclEnforcer = aclEnforcer; + this.sessionId = sessionId; + this.clientType = clientType; + this.clientIP = clientIP; + this.requestData = requestData; + this.clusterName = clusterName; + this.zoneName = zoneName; + this.policyVersion = policyVersion; + } + + /** + * @return the repositoryType + */ + public int getRepositoryType() { + return repositoryType; + } + + /** + * @param repositoryType + * the repositoryType to set + */ + public void setRepositoryType(int repositoryType) { + this.repositoryType = repositoryType; + } + + /** + * @return the repositoryName + */ + public String getRepositoryName() { + return repositoryName; + } + + /** + * @param repositoryName + * the repositoryName to set + */ + public void setRepositoryName(String repositoryName) { + this.repositoryName = repositoryName; + } + + /** + * @return the user + */ + public String getUser() { + return user; + } + + /** + * @param user + * the user to set + */ + public void setUser(String user) { + this.user = user; + } + + public String getEntityGuid() { + return entityGuid; + } + + public void setEntityGuid(String entityGuid) { + this.entityGuid = entityGuid; + } + + /** + * @return the timeStamp + */ + public Date getEventTime() { + return eventTime; + } + + /** + * @param eventTime + * the eventTime to set + */ + public void setEventTime(Date eventTime) { + this.eventTime = eventTime; + } + + /** + * @return the accessType + */ + public String getAccessType() { + return accessType; + } + + /** + * @param accessType + * the accessType to set + */ + public void setAccessType(String accessType) { + this.accessType = accessType; + } + + /** + * @return the resourcePath + */ + public String getResourcePath() { + return resourcePath; + } + + /** + * @param resourcePath + * the resourcePath to set + */ + public void setResourcePath(String resourcePath) { + this.resourcePath = resourcePath; + } + + /** + * @return the resourceType + */ + public String getResourceType() { + return resourceType; + } + + /** + * @param resourceType + * the resourceType to set + */ + public void setResourceType(String resourceType) { + this.resourceType = resourceType; + } + + /** + * @return the action + */ + public String getAction() { return action; } + + /** + * @param action + * the action to set + */ + public void setAction(String action) { + this.action = action; + } + + /** + * @return the accessResult + */ + public short getAccessResult() { + return accessResult; + } + + /** + * @param accessResult + * the accessResult to set + */ + public void setAccessResult(short accessResult) { + this.accessResult = accessResult; + } + + /** + * @return the agentId + */ + public String getAgentId() { + return agentId; + } + + /** + * @param agentId + * the agentId to set + */ + public void setAgentId(String agentId) { + this.agentId = agentId; + } + + /** + * @return the policyId + */ + public String getPolicyId() { + return policyId; + } + + /** + * @param policyId + * the policyId to set + */ + public void setPolicyId(String policyId) { + this.policyId = policyId; + } + + /** + * @return the resultReason + */ + public String getResultReason() { + return resultReason; + } + + /** + * @param resultReason + * the resultReason to set + */ + public void setResultReason(String resultReason) { + this.resultReason = resultReason; + } + + /** + * @return the aclEnforcer + */ + public String getAclEnforcer() { + return aclEnforcer; + } + + /** + * @param aclEnforcer + * the aclEnforcer to set + */ + public void setAclEnforcer(String aclEnforcer) { + this.aclEnforcer = aclEnforcer; + } + + /** + * @return the sessionId + */ + public String getSessionId() { + return sessionId; + } + + /** + * @param sessionId + * the sessionId to set + */ + public void setSessionId(String sessionId) { + this.sessionId = sessionId; + } + + /** + * @return the clientType + */ + public String getClientType() { + return clientType; + } + + /** + * @param clientType + * the clientType to set + */ + public void setClientType(String clientType) { + this.clientType = clientType; + } + + /** + * @return the clientIP + */ + public String getClientIP() { + return clientIP; + } + + /** + * @param clientIP + * the clientIP to set + */ + public void setClientIP(String clientIP) { + this.clientIP = clientIP; + } + + /** + * @return the requestData + */ + public String getRequestData() { return requestData; } + + /** + * @param requestData + * the requestData to set + */ + public void setRequestData(String requestData) { + this.requestData = requestData; + } + + public String getAgentHostname() { + return agentHostname; + } + + public void setAgentHostname(String agentHostname) { + this.agentHostname = agentHostname; + } + + public String getLogType() { + return logType; + } + + public void setLogType(String logType) { + this.logType = logType; + } + + public String getEventId() { + return eventId; + } + + public void setEventId(String eventId) { + this.eventId = eventId; + } + + public long getSeqNum() { + return seqNum; + } + + public void setSeqNum(long seqNum) { + this.seqNum = seqNum; + } + + public long getEventCount() { + return eventCount; + } + + public void setEventCount(long frequencyCount) { + this.eventCount = frequencyCount; + } + + public long getEventDurationMS() { + return eventDurationMS; + } + + public Set getTags() { + return tags; + } + + public void setEventDurationMS(long frequencyDurationMS) { + this.eventDurationMS = frequencyDurationMS; + } + + public void setTags(Set tags) { + this.tags = tags; + } + + public String getClusterName() { + return clusterName; + } + + public void setZoneName(String zoneName) { + this.zoneName = zoneName; + } + + public String getZoneName() { + return zoneName; + } + + public void setPolicyVersion(Long policyVersion) { + this.policyVersion = policyVersion; + } + + public Long getPolicyVersion() { + return policyVersion; + } + + public void setClusterName(String clusterName) { + this.clusterName = clusterName; + } + + public String getAdditionalInfo() { return this.additionalInfo; } + + public void setAdditionalInfo(String additionalInfo) { this.additionalInfo = additionalInfo; } + + @Override + public String getEventKey() { + String key = user + "^" + accessType + "^" + resourcePath + "^" + + resourceType + "^" + action + "^" + accessResult + "^" + + sessionId + "^" + clientIP; + return key; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + + sb.append("AuthzAuditEvent{"); + toString(sb); + sb.append("}"); + + return sb.toString(); + } + + protected StringBuilder toString(StringBuilder sb) { + sb.append("repositoryType=").append(repositoryType) + .append(FIELD_SEPARATOR).append("repositoryName=") + .append(repositoryName).append(FIELD_SEPARATOR).append("user=") + .append(user).append(FIELD_SEPARATOR).append("eventTime=") + .append(eventTime).append(FIELD_SEPARATOR) + .append("accessType=").append(accessType) + .append(FIELD_SEPARATOR).append("resourcePath=") + .append(resourcePath).append(FIELD_SEPARATOR) + .append("resourceType=").append(resourceType) + .append(FIELD_SEPARATOR).append("action=").append(action) + .append(FIELD_SEPARATOR).append("accessResult=") + .append(accessResult).append(FIELD_SEPARATOR) + .append("agentId=").append(agentId).append(FIELD_SEPARATOR) + .append("policyId=").append(policyId).append(FIELD_SEPARATOR) + .append("resultReason=").append(resultReason) + .append(FIELD_SEPARATOR).append("aclEnforcer=") + .append(aclEnforcer).append(FIELD_SEPARATOR) + .append("sessionId=").append(sessionId).append(FIELD_SEPARATOR) + .append("clientType=").append(clientType) + .append(FIELD_SEPARATOR).append("clientIP=").append(clientIP) + .append(FIELD_SEPARATOR).append("requestData=") + .append(requestData).append(FIELD_SEPARATOR) + .append("agentHostname=").append(agentHostname) + .append(FIELD_SEPARATOR).append("logType=").append(logType) + .append(FIELD_SEPARATOR).append("eventId=").append(eventId) + .append(FIELD_SEPARATOR).append("seq_num=").append(seqNum) + .append(FIELD_SEPARATOR).append("event_count=") + .append(eventCount).append(FIELD_SEPARATOR) + .append("event_dur_ms=").append(eventDurationMS) + .append(FIELD_SEPARATOR) + .append("tags=").append("[") + .append(StringUtils.join(tags, ", ")) + .append("]") + .append(FIELD_SEPARATOR).append("clusterName=").append(clusterName) + .append(FIELD_SEPARATOR).append("zoneName=").append(zoneName) + .append(FIELD_SEPARATOR).append("policyVersion=").append(policyVersion) + .append(FIELD_SEPARATOR).append("additionalInfo=").append(additionalInfo); + + return sb; + } +} diff --git a/auth-audits/src/main/java/org/apache/atlas/audit/model/EnumRepositoryType.java b/auth-audits/src/main/java/org/apache/atlas/audit/model/EnumRepositoryType.java new file mode 100644 index 00000000000..4deeaccc8fe --- /dev/null +++ b/auth-audits/src/main/java/org/apache/atlas/audit/model/EnumRepositoryType.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + package org.apache.atlas.audit.model; + +public final class EnumRepositoryType { + + public static final int HDFS = 1; + + public static final int HBASE = 2; + + public static final int HIVE = 3; + + public static final int XAAGENT = 4; + + public static final int KNOX = 5; + + public static final int STORM = 6; + + +} diff --git a/auth-audits/src/main/java/org/apache/atlas/audit/provider/AsyncAuditProvider.java b/auth-audits/src/main/java/org/apache/atlas/audit/provider/AsyncAuditProvider.java new file mode 100644 index 00000000000..33fee12141d --- /dev/null +++ b/auth-audits/src/main/java/org/apache/atlas/audit/provider/AsyncAuditProvider.java @@ -0,0 +1,289 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + package org.apache.atlas.audit.provider; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.audit.model.AuditEventBase; + +import java.util.Properties; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +public class AsyncAuditProvider extends MultiDestAuditProvider implements + Runnable { + + private static final Log LOG = LogFactory.getLog(AsyncAuditProvider.class); + + private static int sThreadCount = 0; + + private BlockingQueue mQueue = null; + private Thread mThread = null; + private String mName = null; + private int mMaxQueueSize = 10 * 1024; + private int mMaxFlushInterval = 5000; // 5 seconds + + private static final int mStopLoopIntervalSecs = 1; // 1 second + private static final int mWaitToCompleteLoopIntervalSecs = 1; // 1 second + + // Summary of logs handled + private AtomicLong lifeTimeInLogCount = new AtomicLong(0); // Total count, including drop count + private AtomicLong lifeTimeOutLogCount = new AtomicLong(0); + private AtomicLong lifeTimeDropCount = new AtomicLong(0); + private AtomicLong intervalInLogCount = new AtomicLong(0); + private AtomicLong intervalOutLogCount = new AtomicLong(0); + private AtomicLong intervalDropCount = new AtomicLong(0); + private long lastIntervalLogTime = System.currentTimeMillis(); + private int intervalLogDurationMS = 60000; + private long lastFlushTime = System.currentTimeMillis(); + + public AsyncAuditProvider(String name, int maxQueueSize, int maxFlushInterval) { + LOG.info("AsyncAuditProvider(" + name + "): creating.."); + + if(maxQueueSize < 1) { + LOG.warn("AsyncAuditProvider(" + name + "): invalid maxQueueSize=" + maxQueueSize + ". will use default " + mMaxQueueSize); + + maxQueueSize = mMaxQueueSize; + } + + mName = name; + mMaxQueueSize = maxQueueSize; + mMaxFlushInterval = maxFlushInterval; + + mQueue = new ArrayBlockingQueue(mMaxQueueSize); + } + + public AsyncAuditProvider(String name, int maxQueueSize, int maxFlushInterval, AuditHandler provider) { + this(name, maxQueueSize, maxFlushInterval); + + addAuditProvider(provider); + } + + @Override + public void init(Properties props) { + LOG.info("AsyncAuditProvider(" + mName + ").init()"); + + super.init(props); + } + + public int getIntervalLogDurationMS() { + return intervalLogDurationMS; + } + + public void setIntervalLogDurationMS(int intervalLogDurationMS) { + this.intervalLogDurationMS = intervalLogDurationMS; + } + + @Override + public boolean log(AuditEventBase event) { + LOG.debug("AsyncAuditProvider.logEvent(AuditEventBase)"); + + queueEvent(event); + return true; + } + + @Override + public void start() { + mThread = new Thread(this, "AsyncAuditProvider" + (++sThreadCount)); + + mThread.setDaemon(true); + mThread.start(); + + super.start(); + } + + @Override + public void stop() { + LOG.info("==> AsyncAuditProvider.stop()"); + try { + LOG.info("Interrupting child thread of " + mName + "..." ); + mThread.interrupt(); + while (mThread.isAlive()) { + try { + LOG.info(String.format("Waiting for child thread of %s to exit. Sleeping for %d secs", mName, mStopLoopIntervalSecs)); + mThread.join(mStopLoopIntervalSecs * 1000); + } catch (InterruptedException e) { + LOG.warn("Interrupted while waiting for child thread to join! Proceeding with stop", e); + break; + } + } + + super.stop(); + } finally { + LOG.info("<== AsyncAuditProvider.stop()"); + } + } + + @Override + public void waitToComplete() { + waitToComplete(0); + + super.waitToComplete(); + } + + @Override + public void run() { + LOG.info("==> AsyncAuditProvider.run()"); + + while (true) { + AuditEventBase event = null; + try { + event = dequeueEvent(); + + if (event != null) { + super.log(event); + } else { + lastFlushTime = System.currentTimeMillis(); + flush(); + } + } catch (InterruptedException excp) { + LOG.info("AsyncAuditProvider.run - Interrupted! Breaking out of while loop."); + break; + } catch (Exception excp) { + logFailedEvent(event, excp); + } + } + + try { + lastFlushTime = System.currentTimeMillis(); + flush(); + } catch (Exception excp) { + LOG.error("AsyncAuditProvider.run()", excp); + } + + LOG.info("<== AsyncAuditProvider.run()"); + } + + private void queueEvent(AuditEventBase event) { + // Increase counts + lifeTimeInLogCount.incrementAndGet(); + intervalInLogCount.incrementAndGet(); + + if(! mQueue.offer(event)) { + lifeTimeDropCount.incrementAndGet(); + intervalDropCount.incrementAndGet(); + } + } + + private AuditEventBase dequeueEvent() throws InterruptedException { + AuditEventBase ret = mQueue.poll(); + + while(ret == null) { + logSummaryIfRequired(); + + if (mMaxFlushInterval > 0 ) { + long timeTillNextFlush = getTimeTillNextFlush(); + + if (timeTillNextFlush <= 0) { + break; // force flush + } + + ret = mQueue.poll(timeTillNextFlush, TimeUnit.MILLISECONDS); + } else { + // Let's wake up for summary logging + long waitTime = intervalLogDurationMS - (System.currentTimeMillis() - lastIntervalLogTime); + waitTime = waitTime <= 0 ? intervalLogDurationMS : waitTime; + + ret = mQueue.poll(waitTime, TimeUnit.MILLISECONDS); + } + } + + if(ret != null) { + lifeTimeOutLogCount.incrementAndGet(); + intervalOutLogCount.incrementAndGet(); + } + + logSummaryIfRequired(); + + return ret; + } + + private void logSummaryIfRequired() { + long intervalSinceLastLog = System.currentTimeMillis() - lastIntervalLogTime; + + if (intervalSinceLastLog > intervalLogDurationMS) { + if (intervalInLogCount.get() > 0 || intervalOutLogCount.get() > 0 ) { + long queueSize = mQueue.size(); + + LOG.info("AsyncAuditProvider-stats:" + mName + ": past " + formatIntervalForLog(intervalSinceLastLog) + + ": inLogs=" + intervalInLogCount.get() + + ", outLogs=" + intervalOutLogCount.get() + + ", dropped=" + intervalDropCount.get() + + ", currentQueueSize=" + queueSize); + + LOG.info("AsyncAuditProvider-stats:" + mName + ": process lifetime" + + ": inLogs=" + lifeTimeInLogCount.get() + + ", outLogs=" + lifeTimeOutLogCount.get() + + ", dropped=" + lifeTimeDropCount.get()); + } + + lastIntervalLogTime = System.currentTimeMillis(); + intervalInLogCount.set(0); + intervalOutLogCount.set(0); + intervalDropCount.set(0); + } + } + + private boolean isEmpty() { + return mQueue.isEmpty(); + } + + public void waitToComplete(long maxWaitSeconds) { + LOG.debug("==> AsyncAuditProvider.waitToComplete()"); + + try { + for (long waitTime = 0; !isEmpty() + && (maxWaitSeconds <= 0 || maxWaitSeconds > waitTime); waitTime += mWaitToCompleteLoopIntervalSecs) { + try { + LOG.info(String.format("%d messages yet to be flushed by %s. Sleeoping for %d sec", mQueue.size(), mName, mWaitToCompleteLoopIntervalSecs)); + Thread.sleep(mWaitToCompleteLoopIntervalSecs * 1000); + } catch (InterruptedException excp) { + // someone really wants service to exit, abandon unwritten audits and exit. + LOG.warn("Caught interrupted exception! " + mQueue.size() + " messages still unflushed! Won't wait for queue to flush, exiting...", excp); + break; + } + } + } finally { + LOG.debug("<== AsyncAuditProvider.waitToComplete()"); + } + } + + private long getTimeTillNextFlush() { + long timeTillNextFlush = mMaxFlushInterval; + + if (mMaxFlushInterval > 0) { + + if (lastFlushTime != 0) { + long timeSinceLastFlush = System.currentTimeMillis() + - lastFlushTime; + + if (timeSinceLastFlush >= mMaxFlushInterval) { + timeTillNextFlush = 0; + } else { + timeTillNextFlush = mMaxFlushInterval - timeSinceLastFlush; + } + } + } + + return timeTillNextFlush; + } +} diff --git a/auth-audits/src/main/java/org/apache/atlas/audit/provider/AuditFileCacheProvider.java b/auth-audits/src/main/java/org/apache/atlas/audit/provider/AuditFileCacheProvider.java new file mode 100644 index 00000000000..3d6499d13c9 --- /dev/null +++ b/auth-audits/src/main/java/org/apache/atlas/audit/provider/AuditFileCacheProvider.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.atlas.audit.provider; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.audit.model.AuditEventBase; +import org.apache.atlas.audit.queue.AuditFileCacheProviderSpool; + +import java.util.Collection; +import java.util.Properties; + +/* + AuditFileCacheProvider class does the work of stashing the audit logs into Local Filesystem before sending it to the AuditBatchQueue Consumer +*/ + +public class AuditFileCacheProvider extends BaseAuditHandler { + private static final Log logger = LogFactory.getLog(AuditFileCacheProvider.class); + + AuditFileCacheProviderSpool fileSpooler = null; + AuditHandler consumer = null; + + AuditFileCacheProvider(AuditHandler consumer) { + this.consumer = consumer; + } + + public void init(Properties prop, String basePropertyName) { + String propPrefix = "xasecure.audit.filecache"; + if (basePropertyName != null) { + propPrefix = basePropertyName; + } + super.init(prop, propPrefix); + //init Consumer + if (consumer != null) { + consumer.init(prop, propPrefix); + } + //init AuditFileCacheSpooler + fileSpooler = new AuditFileCacheProviderSpool(consumer); + fileSpooler.init(prop,propPrefix); + } + @Override + public boolean log(AuditEventBase event) { + boolean ret = false; + if ( event != null) { + fileSpooler.stashLogs(event); + if ( fileSpooler.isSpoolingSuccessful()) { + ret = true; + } + } + return ret; + } + + @Override + public boolean log(Collection events) { + boolean ret = true; + if ( events != null) { + for (AuditEventBase event : events) { + ret = log(event); + } + } + return ret; + } + + @Override + public void start() { + // Start the consumer thread + if (consumer != null) { + consumer.start(); + } + if (fileSpooler != null) { + // start AuditFileSpool thread + fileSpooler.start(); + } + } + + @Override + public void stop() { + logger.info("Stop called. name=" + getName()); + if (consumer != null) { + consumer.stop(); + } + } + + @Override + public void waitToComplete() { + logger.info("waitToComplete called. name=" + getName()); + if ( consumer != null) { + consumer.waitToComplete(); + } + } + + @Override + public void waitToComplete(long timeout) { + logger.info("waitToComplete called. name=" + getName()); + if ( consumer != null) { + consumer.waitToComplete(timeout); + } + } + + @Override + public void flush() { + logger.info("waitToComplete. name=" + getName()); + if ( consumer != null) { + consumer.flush(); + } + } +} diff --git a/auth-audits/src/main/java/org/apache/atlas/audit/provider/AuditHandler.java b/auth-audits/src/main/java/org/apache/atlas/audit/provider/AuditHandler.java new file mode 100644 index 00000000000..ed4feb18bf1 --- /dev/null +++ b/auth-audits/src/main/java/org/apache/atlas/audit/provider/AuditHandler.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.audit.provider; + +import org.apache.atlas.audit.model.AuditEventBase; + +import java.io.File; +import java.util.Collection; +import java.util.Properties; + +public interface AuditHandler { + boolean log(AuditEventBase event); + boolean log(Collection events); + + boolean logJSON(String event); + boolean logJSON(Collection events); + boolean logFile(File file); + + void init(Properties prop); + void init(Properties prop, String basePropertyName); + void start(); + void stop(); + void waitToComplete(); + void waitToComplete(long timeout); + + /** + * Name for this provider. Used only during logging. Uniqueness is not guaranteed + */ + String getName(); + + void flush(); +} diff --git a/auth-audits/src/main/java/org/apache/atlas/audit/provider/AuditMessageException.java b/auth-audits/src/main/java/org/apache/atlas/audit/provider/AuditMessageException.java new file mode 100644 index 00000000000..041d443e7e6 --- /dev/null +++ b/auth-audits/src/main/java/org/apache/atlas/audit/provider/AuditMessageException.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.audit.provider; + +/** + * This exception should be thrown only when there is an error in the message + * itself. E.g. invalid field type, etc. Don't throw this exception if there is + * a transient error + */ +public class AuditMessageException extends Exception { + + private static final long serialVersionUID = 1L; + + public AuditMessageException() { + } + + /** + * @param message + */ + public AuditMessageException(String message) { + super(message); + } + + /** + * @param cause + */ + public AuditMessageException(Throwable cause) { + super(cause); + } + + /** + * @param message + * @param cause + */ + public AuditMessageException(String message, Throwable cause) { + super(message, cause); + } + + /** + * @param message + * @param cause + * @param enableSuppression + * @param writableStackTrace + */ + public AuditMessageException(String message, Throwable cause, + boolean enableSuppression, boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } + +} diff --git a/auth-audits/src/main/java/org/apache/atlas/audit/provider/AuditProviderFactory.java b/auth-audits/src/main/java/org/apache/atlas/audit/provider/AuditProviderFactory.java new file mode 100644 index 00000000000..269da054232 --- /dev/null +++ b/auth-audits/src/main/java/org/apache/atlas/audit/provider/AuditProviderFactory.java @@ -0,0 +1,554 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.audit.provider; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.util.ShutdownHookManager; +import org.apache.atlas.audit.destination.*; +import org.apache.atlas.audit.provider.hdfs.HdfsAuditProvider; +import org.apache.atlas.audit.provider.solr.SolrAuditProvider; +import org.apache.atlas.audit.queue.AuditAsyncQueue; +import org.apache.atlas.audit.queue.AuditBatchQueue; +import org.apache.atlas.audit.queue.AuditFileQueue; +import org.apache.atlas.audit.queue.AuditQueue; +import org.apache.atlas.audit.queue.AuditSummaryQueue; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Properties; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +/* + * TODO: + * 1) Flag to enable/disable audit logging + * 2) Failed path to be recorded + * 3) Repo name, repo type from configuration + */ + +public class AuditProviderFactory { + private static final Log LOG = LogFactory + .getLog(AuditProviderFactory.class); + + public static final String AUDIT_IS_ENABLED_PROP = "xasecure.audit.is.enabled"; + public static final String AUDIT_HDFS_IS_ENABLED_PROP = "xasecure.audit.hdfs.is.enabled"; + public static final String AUDIT_LOG4J_IS_ENABLED_PROP = "xasecure.audit.log4j.is.enabled"; + public static final String AUDIT_KAFKA_IS_ENABLED_PROP = "xasecure.audit.kafka.is.enabled"; + public static final String AUDIT_SOLR_IS_ENABLED_PROP = "xasecure.audit.solr.is.enabled"; + + public static final String AUDIT_DEST_BASE = "xasecure.audit.destination"; + public static final String AUDIT_SHUTDOWN_HOOK_MAX_WAIT_SEC = "xasecure.audit.shutdown.hook.max.wait.seconds"; + public static final String AUDIT_IS_FILE_CACHE_PROVIDER_ENABLE_PROP = "xasecure.audit.provider.filecache.is.enabled"; + public static final String FILE_QUEUE_TYPE = "filequeue"; + public static final String DEFAULT_QUEUE_TYPE = "memoryqueue"; + public static final int AUDIT_SHUTDOWN_HOOK_MAX_WAIT_SEC_DEFAULT = 30; + + public static final int AUDIT_ASYNC_MAX_QUEUE_SIZE_DEFAULT = 10 * 1024; + public static final int AUDIT_ASYNC_MAX_FLUSH_INTERVAL_DEFAULT = 5 * 1000; + + private static final int RANGER_AUDIT_SHUTDOWN_HOOK_PRIORITY = 30; + + private volatile static AuditProviderFactory sFactory = null; + + private AuditHandler mProvider = null; + private String componentAppType = ""; + private boolean mInitDone = false; + private JVMShutdownHook jvmShutdownHook = null; + private ArrayList hbaseAppTypes = new ArrayList<>(Arrays.asList("hbaseMaster","hbaseRegional")); + + public AuditProviderFactory() { + LOG.info("AuditProviderFactory: creating.."); + + mProvider = getDefaultProvider(); + } + + public static AuditProviderFactory getInstance() { + AuditProviderFactory ret = sFactory; + if(ret == null) { + synchronized(AuditProviderFactory.class) { + ret = sFactory; + if(ret == null) { + ret = sFactory = new AuditProviderFactory(); + } + } + } + + return ret; + } + + public AuditHandler getAuditProvider() { + return mProvider; + } + + public boolean isInitDone() { + return mInitDone; + } + + /** + * call shutdown hook to provide a way to + * shutdown gracefully in addition to the ShutdownHook mechanism + */ + public void shutdown() { + if (isInitDone() && jvmShutdownHook != null) { + jvmShutdownHook.run(); + } + } + + public synchronized void init(Properties props, String appType) { + LOG.info("AuditProviderFactory: initializing.."); + + if (mInitDone) { + LOG.warn("AuditProviderFactory.init(): already initialized! Will try to re-initialize"); + } + mInitDone = true; + componentAppType = appType; + MiscUtil.setApplicationType(appType); + + boolean isEnabled = MiscUtil.getBooleanProperty(props, + AUDIT_IS_ENABLED_PROP, true); + if (!isEnabled) { + LOG.info("AuditProviderFactory: Audit not enabled.."); + mProvider = getDefaultProvider(); + return; + } + + boolean isAuditToHdfsEnabled = MiscUtil.getBooleanProperty(props, + AUDIT_HDFS_IS_ENABLED_PROP, false); + boolean isAuditToLog4jEnabled = MiscUtil.getBooleanProperty(props, + AUDIT_LOG4J_IS_ENABLED_PROP, false); + boolean isAuditToKafkaEnabled = MiscUtil.getBooleanProperty(props, + AUDIT_KAFKA_IS_ENABLED_PROP, false); + boolean isAuditToSolrEnabled = MiscUtil.getBooleanProperty(props, + AUDIT_SOLR_IS_ENABLED_PROP, false); + + boolean isAuditFileCacheProviderEnabled = MiscUtil.getBooleanProperty(props, + AUDIT_IS_FILE_CACHE_PROVIDER_ENABLE_PROP, false); + + List providers = new ArrayList(); + + for (Object propNameObj : props.keySet()) { + LOG.info("AUDIT PROPERTY: " + propNameObj.toString() + "=" + + props.getProperty(propNameObj.toString())); + } + + // Process new audit configurations + List destNameList = new ArrayList(); + + for (Object propNameObj : props.keySet()) { + String propName = propNameObj.toString(); + if (!propName.startsWith(AUDIT_DEST_BASE)) { + continue; + } + String destName = propName.substring(AUDIT_DEST_BASE.length() + 1); + List splits = MiscUtil.toArray(destName, "."); + if (splits.size() > 1) { + continue; + } + String value = props.getProperty(propName); + if (value.equalsIgnoreCase("enable") + || value.equalsIgnoreCase("enabled") + || value.equalsIgnoreCase("true")) { + destNameList.add(destName); + LOG.info("Audit destination " + propName + " is set to " + + value); + } + } + + for (String destName : destNameList) { + String destPropPrefix = AUDIT_DEST_BASE + "." + destName; + AuditHandler destProvider = getProviderFromConfig(props, + destPropPrefix, destName, null); + + if (destProvider != null) { + destProvider.init(props, destPropPrefix); + + String queueName = MiscUtil.getStringProperty(props, + destPropPrefix + "." + AuditQueue.PROP_QUEUE); + if (queueName == null || queueName.isEmpty()) { + LOG.info(destPropPrefix + "." + AuditQueue.PROP_QUEUE + + " is not set. Setting queue to batch for " + + destName); + queueName = "batch"; + } + LOG.info("queue for " + destName + " is " + queueName); + if (queueName != null && !queueName.isEmpty() + && !queueName.equalsIgnoreCase("none")) { + String queuePropPrefix = destPropPrefix + "." + queueName; + AuditHandler queueProvider = getProviderFromConfig(props, + queuePropPrefix, queueName, destProvider); + if (queueProvider != null) { + if (queueProvider instanceof AuditQueue) { + AuditQueue qProvider = (AuditQueue) queueProvider; + qProvider.init(props, queuePropPrefix); + providers.add(queueProvider); + } else { + LOG.fatal("Provider queue doesn't extend AuditQueue. Destination=" + + destName + + " can't be created. queueName=" + + queueName); + } + } else { + LOG.fatal("Queue provider for destination " + destName + + " can't be created. queueName=" + queueName); + } + } else { + LOG.info("Audit destination " + destProvider.getName() + + " added to provider list"); + providers.add(destProvider); + } + } + } + if (providers.size() > 0) { + LOG.info("Using v3 audit configuration"); + AuditHandler consumer = providers.get(0); + + // Possible pipeline is: + // async_queue -> summary_queue -> multidestination -> batch_queue + // -> hdfs_destination + // -> batch_queue -> solr_destination + // -> batch_queue -> kafka_destination + // Above, up to multidestination, the providers are same, then it + // branches out in parallel. + + // Set the providers in the reverse order e.g. + + if (providers.size() > 1) { + // If there are more than one destination, then we need multi + // destination to process it in parallel + LOG.info("MultiDestAuditProvider is used. Destination count=" + + providers.size()); + MultiDestAuditProvider multiDestProvider = new MultiDestAuditProvider(); + multiDestProvider.init(props); + multiDestProvider.addAuditProviders(providers); + consumer = multiDestProvider; + } + + // Let's see if Summary is enabled, then summarize before sending it + // downstream + String propPrefix = BaseAuditHandler.PROP_DEFAULT_PREFIX; + boolean summaryEnabled = MiscUtil.getBooleanProperty(props, + propPrefix + "." + "summary" + "." + "enabled", false); + AuditSummaryQueue summaryQueue = null; + if (summaryEnabled) { + LOG.info("AuditSummaryQueue is enabled"); + summaryQueue = new AuditSummaryQueue(consumer); + summaryQueue.init(props, propPrefix); + consumer = summaryQueue; + } else { + LOG.info("AuditSummaryQueue is disabled"); + } + + if (!isAuditFileCacheProviderEnabled) { + // Create the AsysnQueue + AuditAsyncQueue asyncQueue = new AuditAsyncQueue(consumer); + propPrefix = BaseAuditHandler.PROP_DEFAULT_PREFIX + "." + "async"; + asyncQueue.init(props, propPrefix); + asyncQueue.setParentPath(componentAppType); + mProvider = asyncQueue; + LOG.info("Starting audit queue " + mProvider.getName()); + mProvider.start(); + } else { + // Assign AsyncQueue to AuditFileCacheProvider + AuditFileCacheProvider auditFileCacheProvider = new AuditFileCacheProvider(consumer); + propPrefix = BaseAuditHandler.PROP_DEFAULT_PREFIX + "." + "filecache"; + auditFileCacheProvider.init(props, propPrefix); + auditFileCacheProvider.setParentPath(componentAppType); + mProvider = auditFileCacheProvider; + LOG.info("Starting Audit File Cache Provider " + mProvider.getName()); + mProvider.start(); + } + } else { + LOG.info("No v3 audit configuration found. Trying v2 audit configurations"); + if (!isEnabled + || !(isAuditToHdfsEnabled + || isAuditToKafkaEnabled || isAuditToLog4jEnabled + || isAuditToSolrEnabled || providers.size() == 0)) { + LOG.info("AuditProviderFactory: Audit not enabled.."); + + mProvider = getDefaultProvider(); + + return; + } + + if (isAuditToHdfsEnabled) { + LOG.info("HdfsAuditProvider is enabled"); + + HdfsAuditProvider hdfsProvider = new HdfsAuditProvider(); + + boolean isAuditToHdfsAsync = MiscUtil.getBooleanProperty(props, + HdfsAuditProvider.AUDIT_HDFS_IS_ASYNC_PROP, false); + + if (isAuditToHdfsAsync) { + int maxQueueSize = MiscUtil.getIntProperty(props, + HdfsAuditProvider.AUDIT_HDFS_MAX_QUEUE_SIZE_PROP, + AUDIT_ASYNC_MAX_QUEUE_SIZE_DEFAULT); + int maxFlushInterval = MiscUtil + .getIntProperty( + props, + HdfsAuditProvider.AUDIT_HDFS_MAX_FLUSH_INTERVAL_PROP, + AUDIT_ASYNC_MAX_FLUSH_INTERVAL_DEFAULT); + + AsyncAuditProvider asyncProvider = new AsyncAuditProvider( + "HdfsAuditProvider", maxQueueSize, + maxFlushInterval, hdfsProvider); + + providers.add(asyncProvider); + } else { + providers.add(hdfsProvider); + } + } + + /*if (isAuditToKafkaEnabled) { + LOG.info("KafkaAuditProvider is enabled"); + KafkaAuditProvider kafkaProvider = new KafkaAuditProvider(); + kafkaProvider.init(props); + + if (kafkaProvider.isAsync()) { + AsyncAuditProvider asyncProvider = new AsyncAuditProvider( + "MyKafkaAuditProvider", 1000, 1000, kafkaProvider); + providers.add(asyncProvider); + } else { + providers.add(kafkaProvider); + } + }*/ + + if (isAuditToSolrEnabled) { + LOG.info("SolrAuditProvider is enabled"); + SolrAuditProvider solrProvider = new SolrAuditProvider(); + solrProvider.init(props); + + if (solrProvider.isAsync()) { + AsyncAuditProvider asyncProvider = new AsyncAuditProvider( + "MySolrAuditProvider", 1000, 1000, solrProvider); + providers.add(asyncProvider); + } else { + providers.add(solrProvider); + } + } + + if (isAuditToLog4jEnabled) { + Log4jAuditProvider log4jProvider = new Log4jAuditProvider(); + + boolean isAuditToLog4jAsync = MiscUtil.getBooleanProperty( + props, Log4jAuditProvider.AUDIT_LOG4J_IS_ASYNC_PROP, + false); + + if (isAuditToLog4jAsync) { + int maxQueueSize = MiscUtil.getIntProperty(props, + Log4jAuditProvider.AUDIT_LOG4J_MAX_QUEUE_SIZE_PROP, + AUDIT_ASYNC_MAX_QUEUE_SIZE_DEFAULT); + int maxFlushInterval = MiscUtil + .getIntProperty( + props, + Log4jAuditProvider.AUDIT_LOG4J_MAX_FLUSH_INTERVAL_PROP, + AUDIT_ASYNC_MAX_FLUSH_INTERVAL_DEFAULT); + + AsyncAuditProvider asyncProvider = new AsyncAuditProvider( + "Log4jAuditProvider", maxQueueSize, + maxFlushInterval, log4jProvider); + + providers.add(asyncProvider); + } else { + providers.add(log4jProvider); + } + } + if (providers.size() == 0) { + mProvider = getDefaultProvider(); + } else if (providers.size() == 1) { + mProvider = providers.get(0); + } else { + MultiDestAuditProvider multiDestProvider = new MultiDestAuditProvider(); + + multiDestProvider.addAuditProviders(providers); + + mProvider = multiDestProvider; + } + + mProvider.init(props); + mProvider.start(); + } + + installJvmSutdownHook(props); + } + + private AuditHandler getProviderFromConfig(Properties props, + String propPrefix, String providerName, AuditHandler consumer) { + AuditHandler provider = null; + String className = MiscUtil.getStringProperty(props, propPrefix + "." + + BaseAuditHandler.PROP_CLASS_NAME); + if (className != null && !className.isEmpty()) { + try { + Class handlerClass = Class.forName(className); + if (handlerClass.isAssignableFrom(AuditQueue.class)) { + // Queue class needs consumer + handlerClass.getDeclaredConstructor(AuditHandler.class) + .newInstance(consumer); + } else { + provider = (AuditHandler) Class.forName(className) + .newInstance(); + } + } catch (Exception e) { + LOG.fatal("Can't instantiate audit class for providerName=" + + providerName + ", className=" + className + + ", propertyPrefix=" + propPrefix, e); + } + } else { + if (providerName.equalsIgnoreCase("file")) { + provider = new FileAuditDestination(); + } else if (providerName.equalsIgnoreCase("hdfs")) { + provider = new HDFSAuditDestination(); + } else if (providerName.equalsIgnoreCase("solr")) { + provider = new SolrAuditDestination(); + } else if (providerName.equalsIgnoreCase("elasticsearch")) { + provider = new ElasticSearchAuditDestination(); + } /*else if (providerName.equalsIgnoreCase("kafka")) { + provider = new KafkaAuditProvider(); + }*/ else if (providerName.equalsIgnoreCase("log4j")) { + provider = new Log4JAuditDestination(); + } else if (providerName.equalsIgnoreCase("batch")) { + provider = getAuditProvider(props, propPrefix, consumer); + } else if (providerName.equalsIgnoreCase("async")) { + provider = new AuditAsyncQueue(consumer); + } else { + LOG.error("Provider name doesn't have any class associated with it. providerName=" + + providerName + ", propertyPrefix=" + propPrefix); + } + } + if (provider != null && provider instanceof AuditQueue) { + if (consumer == null) { + LOG.fatal("consumer can't be null for AuditQueue. queue=" + + provider.getName() + ", propertyPrefix=" + propPrefix); + provider = null; + } + } + return provider; + } + + private AuditHandler getAuditProvider(Properties props, String propPrefix, AuditHandler consumer) { + AuditHandler ret = null; + String queueType = MiscUtil.getStringProperty(props, propPrefix + "." + "queuetype", DEFAULT_QUEUE_TYPE); + + if (LOG.isDebugEnabled()) { + LOG.debug("==> AuditProviderFactory.getAuditProvider() propPerfix= " + propPrefix + ", " + " queueType= " + queueType); + } + + if (FILE_QUEUE_TYPE.equalsIgnoreCase(queueType)) { + AuditFileQueue auditFileQueue = new AuditFileQueue(consumer); + String propPrefixFileQueue = propPrefix + "." + FILE_QUEUE_TYPE; + auditFileQueue.init(props, propPrefixFileQueue); + ret = new AuditBatchQueue(auditFileQueue); + } else { + ret = new AuditBatchQueue(consumer); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== AuditProviderFactory.getAuditProvider()"); + } + + return ret; + } + + private AuditHandler getDefaultProvider() { + return new DummyAuditProvider(); + } + + private void installJvmSutdownHook(Properties props) { + int shutdownHookMaxWaitSeconds = MiscUtil.getIntProperty(props, AUDIT_SHUTDOWN_HOOK_MAX_WAIT_SEC, AUDIT_SHUTDOWN_HOOK_MAX_WAIT_SEC_DEFAULT); + jvmShutdownHook = new JVMShutdownHook(mProvider, shutdownHookMaxWaitSeconds); + String appType = this.componentAppType; + if (appType != null && !hbaseAppTypes.contains(appType)) { + ShutdownHookManager.get().addShutdownHook(jvmShutdownHook, RANGER_AUDIT_SHUTDOWN_HOOK_PRIORITY); + } + } + + private static class RangerAsyncAuditCleanup implements Runnable { + + final Semaphore startCleanup; + final Semaphore doneCleanup; + final AuditHandler mProvider; + + RangerAsyncAuditCleanup(AuditHandler provider, Semaphore startCleanup, Semaphore doneCleanup) { + this.startCleanup = startCleanup; + this.doneCleanup = doneCleanup; + this.mProvider = provider; + } + + @Override + public void run() { + while (true) { + LOG.info("RangerAsyncAuditCleanup: Waiting to audit cleanup start signal"); + try { + startCleanup.acquire(); + } catch (InterruptedException e) { + LOG.info("RangerAsyncAuditCleanup: Interrupted while waiting for audit startCleanup signal! Exiting the thread...", e); + break; + } + LOG.info("RangerAsyncAuditCleanup: Starting cleanup"); + mProvider.waitToComplete(); + mProvider.stop(); + doneCleanup.release(); + LOG.info("RangerAsyncAuditCleanup: Done cleanup"); + } + } + } + + private static class JVMShutdownHook extends Thread { + final Semaphore startCleanup = new Semaphore(0); + final Semaphore doneCleanup = new Semaphore(0); + final Thread cleanupThread; + final int maxWait; + final AtomicBoolean done = new AtomicBoolean(false); + + public JVMShutdownHook(AuditHandler provider, int maxWait) { + this.maxWait = maxWait; + Runnable runnable = new RangerAsyncAuditCleanup(provider, startCleanup, doneCleanup); + cleanupThread = new Thread(runnable, "Ranger async Audit cleanup"); + cleanupThread.setDaemon(true); + cleanupThread.start(); + } + + public void run() { + if (!done.compareAndSet(false, true)) { + LOG.info("==> JVMShutdownHook.run() already done by another thread"); + return; + } + LOG.info("==> JVMShutdownHook.run()"); + LOG.info("JVMShutdownHook: Signalling async audit cleanup to start."); + startCleanup.release(); + try { + Long start = System.currentTimeMillis(); + LOG.info("JVMShutdownHook: Waiting up to " + maxWait + " seconds for audit cleanup to finish."); + boolean cleanupFinishedInTime = doneCleanup.tryAcquire(maxWait, TimeUnit.SECONDS); + if (cleanupFinishedInTime) { + LOG.info("JVMShutdownHook: Audit cleanup finished after " + (System.currentTimeMillis() - start) + " milli seconds"); + } else { + LOG.warn("JVMShutdownHook: could not detect finishing of audit cleanup even after waiting for " + maxWait + " seconds!"); + } + } catch (InterruptedException e) { + LOG.info("JVMShutdownHook: Interrupted while waiting for completion of Async executor!", e); + } + LOG.info("JVMShutdownHook: Interrupting ranger async audit cleanup thread"); + cleanupThread.interrupt(); + LOG.info("<== JVMShutdownHook.run()"); + } + } +} diff --git a/auth-audits/src/main/java/org/apache/atlas/audit/provider/AuditWriterFactory.java b/auth-audits/src/main/java/org/apache/atlas/audit/provider/AuditWriterFactory.java new file mode 100644 index 00000000000..b9a98d5b4e9 --- /dev/null +++ b/auth-audits/src/main/java/org/apache/atlas/audit/provider/AuditWriterFactory.java @@ -0,0 +1,117 @@ +package org.apache.atlas.audit.provider; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.audit.utils.RangerAuditWriter; + +import java.util.Map; +import java.util.Properties; + +public class AuditWriterFactory { + private static final Log logger = LogFactory.getLog(AuditWriterFactory.class); + public static final String AUDIT_FILETYPE_DEFAULT = "json"; + public static final String AUDIT_JSON_FILEWRITER_IMPL = "org.apache.ranger.audit.utils.RangerJSONAuditWriter"; + public static final String AUDIT_ORC_FILEWRITER_IMPL = "org.apache.ranger.audit.utils.RangerORCAuditWriter"; + + public Map auditConfigs = null; + public Properties props = null; + public String propPrefix = null; + public String auditProviderName = null; + public RangerAuditWriter auditWriter = null; + private static volatile AuditWriterFactory me = null; + + public static AuditWriterFactory getInstance() { + AuditWriterFactory auditWriter = me; + if (auditWriter == null) { + synchronized (AuditWriterFactory.class) { + auditWriter = me; + if (auditWriter == null) { + me = auditWriter = new AuditWriterFactory(); + } + } + } + return auditWriter; + } + + public void init(Properties props, String propPrefix, String auditProviderName, Map auditConfigs) throws Exception { + if (logger.isDebugEnabled()) { + logger.debug("==> AuditWriterFactory.init()"); + } + this.props = props; + this.propPrefix = propPrefix; + this.auditProviderName = auditProviderName; + this.auditConfigs = auditConfigs; + String auditFileType = MiscUtil.getStringProperty(props, propPrefix + ".filetype", AUDIT_FILETYPE_DEFAULT); + String writerClass = MiscUtil.getStringProperty(props, propPrefix + ".filewriter.impl"); + + auditWriter = StringUtils.isEmpty(writerClass) ? createWriter(getDefaultWriter(auditFileType)) : createWriter(writerClass); + + if (auditWriter != null) { + auditWriter.init(props, propPrefix, auditProviderName, auditConfigs); + } + + if (logger.isDebugEnabled()) { + logger.debug("<== AuditWriterFactory.init() :" + auditWriter.getClass().getName()); + } + } + + public RangerAuditWriter createWriter(String writerClass) throws Exception { + if (logger.isDebugEnabled()) { + logger.debug("==> AuditWriterFactory.createWriter()"); + } + RangerAuditWriter ret = null; + try { + Class cls = (Class) Class.forName(writerClass); + ret = cls.newInstance(); + } catch (Exception e) { + throw e; + } + if (logger.isDebugEnabled()) { + logger.debug("<== AuditWriterFactory.createWriter()"); + } + return ret; + } + + public String getDefaultWriter(String auditFileType) { + if (logger.isDebugEnabled()) { + logger.debug("==> AuditWriterFactory.getDefaultWriter()"); + } + String ret = null; + switch (auditFileType) { + case "orc": + ret = AUDIT_ORC_FILEWRITER_IMPL; + break; + case "json": + ret = AUDIT_JSON_FILEWRITER_IMPL; + break; + } + if (logger.isDebugEnabled()) { + logger.debug("<== AuditWriterFactory.getDefaultWriter() :" + ret); + } + return ret; + } + + public RangerAuditWriter getAuditWriter(){ + return this.auditWriter; + } +} \ No newline at end of file diff --git a/auth-audits/src/main/java/org/apache/atlas/audit/provider/BaseAuditHandler.java b/auth-audits/src/main/java/org/apache/atlas/audit/provider/BaseAuditHandler.java new file mode 100644 index 00000000000..2cf7a43f46a --- /dev/null +++ b/auth-audits/src/main/java/org/apache/atlas/audit/provider/BaseAuditHandler.java @@ -0,0 +1,494 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.atlas.audit.provider; + +import com.google.gson.GsonBuilder; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.audit.model.AuditEventBase; +import org.apache.atlas.audit.model.AuthzAuditEvent; + +import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.TrustManagerFactory; +import java.io.File; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.atomic.AtomicLong; + +public abstract class BaseAuditHandler implements AuditHandler { + private static final Log LOG = LogFactory.getLog(BaseAuditHandler.class); + + static final String AUDIT_LOG_FAILURE_REPORT_MIN_INTERVAL_PROP = "xasecure.audit.log.failure.report.min.interval.ms"; + + public static final String RANGER_POLICYMGR_CLIENT_KEY_FILE = "xasecure.policymgr.clientssl.keystore"; + public static final String RANGER_POLICYMGR_CLIENT_KEY_FILE_TYPE = "xasecure.policymgr.clientssl.keystore.type"; + public static final String RANGER_POLICYMGR_CLIENT_KEY_FILE_CREDENTIAL = "xasecure.policymgr.clientssl.keystore.credential.file"; + public static final String RANGER_POLICYMGR_CLIENT_KEY_FILE_CREDENTIAL_ALIAS = "sslKeyStore"; + public static final String RANGER_POLICYMGR_CLIENT_KEY_FILE_TYPE_DEFAULT = "jks"; + + public static final String RANGER_POLICYMGR_TRUSTSTORE_FILE = "xasecure.policymgr.clientssl.truststore"; + public static final String RANGER_POLICYMGR_TRUSTSTORE_FILE_TYPE = "xasecure.policymgr.clientssl.truststore.type"; + public static final String RANGER_POLICYMGR_TRUSTSTORE_FILE_CREDENTIAL = "xasecure.policymgr.clientssl.truststore.credential.file"; + public static final String RANGER_POLICYMGR_TRUSTSTORE_FILE_CREDENTIAL_ALIAS = "sslTrustStore"; + public static final String RANGER_POLICYMGR_TRUSTSTORE_FILE_TYPE_DEFAULT = "jks"; + + public static final String RANGER_SSL_KEYMANAGER_ALGO_TYPE = KeyManagerFactory.getDefaultAlgorithm(); + public static final String RANGER_SSL_TRUSTMANAGER_ALGO_TYPE = TrustManagerFactory.getDefaultAlgorithm(); + public static final String RANGER_SSL_CONTEXT_ALGO_TYPE = "TLS"; + + public static final String PROP_CONFIG = "config"; + + private int mLogFailureReportMinIntervalInMs = 60 * 1000; + + private AtomicLong mFailedLogLastReportTime = new AtomicLong(0); + private AtomicLong mFailedLogCountSinceLastReport = new AtomicLong(0); + private AtomicLong mFailedLogCountLifeTime = new AtomicLong(0); + + public static final String PROP_NAME = "name"; + public static final String PROP_CLASS_NAME = "classname"; + + public static final String PROP_DEFAULT_PREFIX = "xasecure.audit.provider"; + + protected String propPrefix = PROP_DEFAULT_PREFIX; + + protected String providerName = null; + protected String parentPath = null; + + protected int failedRetryTimes = 3; + protected int failedRetrySleep = 3 * 1000; + + int errorLogIntervalMS = 30 * 1000; // Every 30 seconds + long lastErrorLogMS = 0; + + long totalCount = 0; + long totalSuccessCount = 0; + long totalFailedCount = 0; + long totalStashedCount = 0; + long totalDeferredCount = 0; + + long lastIntervalCount = 0; + long lastIntervalSuccessCount = 0; + long lastIntervalFailedCount = 0; + long lastStashedCount = 0; + long lastDeferredCount = 0; + + long lastStatusLogTime = System.currentTimeMillis(); + long statusLogIntervalMS = 1 * 60 * 1000; + + protected Properties props = null; + protected Map configProps = new HashMap(); + + @Override + public void init(Properties props) { + init(props, null); + } + + @Override + public void init(Properties props, String basePropertyName) { + LOG.info("BaseAuditProvider.init()"); + this.props = props; + if (basePropertyName != null) { + propPrefix = basePropertyName; + } + LOG.info("propPrefix=" + propPrefix); + + String name = MiscUtil.getStringProperty(props, basePropertyName + "." + + PROP_NAME); + if (name != null && !name.isEmpty()) { + setName(name); + } + // Get final token + if (providerName == null) { + List tokens = MiscUtil.toArray(propPrefix, "."); + if (!tokens.isEmpty()) { + String finalToken = tokens.get(tokens.size() - 1); + setName(finalToken); + LOG.info("Using providerName from property prefix. providerName=" + + getName()); + } + } + LOG.info("providerName=" + getName()); + + try { + new GsonBuilder().setDateFormat("yyyyMMdd-HH:mm:ss.SSS-Z").create(); + } catch (Throwable excp) { + LOG.warn( + "Log4jAuditProvider.init(): failed to create GsonBuilder object. events will be formated using toString(), instead of Json", + excp); + } + + mLogFailureReportMinIntervalInMs = MiscUtil.getIntProperty(props, + AUDIT_LOG_FAILURE_REPORT_MIN_INTERVAL_PROP, 60 * 1000); + + String configPropsNamePrefix = propPrefix + "." + PROP_CONFIG + "."; + for (Object propNameObj : props.keySet()) { + String propName = propNameObj.toString(); + + if (!propName.startsWith(configPropsNamePrefix)) { + continue; + } + String configName = propName.substring(configPropsNamePrefix.length()); + String configValue = props.getProperty(propName); + configProps.put(configName, configValue); + LOG.info("Found Config property: " + configName + " => " + configValue); + } + } + + /* + * (non-Javadoc) + * + * @see + * org.apache.ranger.audit.provider.AuditProvider#log(org.apache.ranger. + * audit.model.AuditEventBase) + */ + @Override + public boolean log(AuditEventBase event) { + return log(Collections.singletonList(event)); + } + + /* + * (non-Javadoc) + * + * @see + * org.apache.ranger.audit.provider.AuditProvider#logJSON(java.lang.String) + */ + @Override + public boolean logJSON(String event) { + AuditEventBase eventObj = MiscUtil.fromJson(event, + AuthzAuditEvent.class); + return log(eventObj); + } + + /* + * (non-Javadoc) + * + * @see + * org.apache.ranger.audit.provider.AuditProvider#logJSON(java.util.Collection + * ) + */ + @Override + public boolean logJSON(Collection events) { + List eventList = new ArrayList(events.size()); + for (String event : events) { + eventList.add(MiscUtil.fromJson(event, AuthzAuditEvent.class)); + } + return log(eventList); + } + + @Override + public boolean logFile(File file) { + return logFile(file); + } + + public String getParentPath() { + return parentPath; + } + + public void setParentPath(String parentPath) { + this.parentPath = parentPath; + } + + public String getFinalPath() { + return getName(); + } + + public void setName(String name) { + providerName = name; + } + + @Override + public String getName() { + if (parentPath != null) { + return parentPath + "." + providerName; + } + return providerName; + } + + public long addTotalCount(int count) { + totalCount += count; + return totalCount; + } + + public long addSuccessCount(int count) { + totalSuccessCount += count; + return totalSuccessCount; + } + + public long addFailedCount(int count) { + totalFailedCount += count; + return totalFailedCount; + } + + public long addStashedCount(int count) { + totalStashedCount += count; + return totalStashedCount; + } + + public long addDeferredCount(int count) { + totalDeferredCount += count; + return totalDeferredCount; + } + + public long getTotalCount() { + return totalCount; + } + + public long getTotalSuccessCount() { + return totalSuccessCount; + } + + public long getTotalFailedCount() { + return totalFailedCount; + } + + public long getTotalStashedCount() { + return totalStashedCount; + } + + public long getLastStashedCount() { + return lastStashedCount; + } + + public long getTotalDeferredCount() { + return totalDeferredCount; + } + + public long getLastDeferredCount() { + return lastDeferredCount; + } + + public void logStatusIfRequired() { + long currTime = System.currentTimeMillis(); + if ((currTime - lastStatusLogTime) > statusLogIntervalMS) { + logStatus(); + } + } + + public void logStatus() { + try { + long currTime = System.currentTimeMillis(); + + long diffTime = currTime - lastStatusLogTime; + lastStatusLogTime = currTime; + + long diffCount = totalCount - lastIntervalCount; + long diffSuccess = totalSuccessCount - lastIntervalSuccessCount; + long diffFailed = totalFailedCount - lastIntervalFailedCount; + long diffStashed = totalStashedCount - lastStashedCount; + long diffDeferred = totalDeferredCount - lastDeferredCount; + + if (diffCount == 0 && diffSuccess == 0 && diffFailed == 0 + && diffStashed == 0 && diffDeferred == 0) { + return; + } + + lastIntervalCount = totalCount; + lastIntervalSuccessCount = totalSuccessCount; + lastIntervalFailedCount = totalFailedCount; + lastStashedCount = totalStashedCount; + lastDeferredCount = totalDeferredCount; + + String finalPath = ""; + String tFinalPath = getFinalPath(); + if (!getName().equals(tFinalPath)) { + finalPath = ", finalDestination=" + tFinalPath; + } + + String msg = "Audit Status Log: name=" + + getName() + + finalPath + + ", interval=" + + formatIntervalForLog(diffTime) + + ", events=" + + diffCount + + (diffSuccess > 0 ? (", succcessCount=" + diffSuccess) + : "") + + (diffFailed > 0 ? (", failedCount=" + diffFailed) : "") + + (diffStashed > 0 ? (", stashedCount=" + diffStashed) : "") + + (diffDeferred > 0 ? (", deferredCount=" + diffDeferred) + : "") + + ", totalEvents=" + + totalCount + + (totalSuccessCount > 0 ? (", totalSuccessCount=" + totalSuccessCount) + : "") + + (totalFailedCount > 0 ? (", totalFailedCount=" + totalFailedCount) + : "") + + (totalStashedCount > 0 ? (", totalStashedCount=" + totalStashedCount) + : "") + + (totalDeferredCount > 0 ? (", totalDeferredCount=" + totalDeferredCount) + : ""); + LOG.info(msg); + } catch (Throwable t) { + LOG.error("Error while printing stats. auditProvider=" + getName()); + } + } + + public void logError(String msg) { + long currTimeMS = System.currentTimeMillis(); + if (currTimeMS - lastErrorLogMS > errorLogIntervalMS) { + LOG.error(msg); + lastErrorLogMS = currTimeMS; + } + } + + public void logError(String msg, Throwable ex) { + long currTimeMS = System.currentTimeMillis(); + if (currTimeMS - lastErrorLogMS > errorLogIntervalMS) { + LOG.error(msg, ex); + lastErrorLogMS = currTimeMS; + } + } + + public String getTimeDiffStr(long time1, long time2) { + long timeInMs = Math.abs(time1 - time2); + return formatIntervalForLog(timeInMs); + } + + public String formatIntervalForLog(long timeInMs) { + long hours = timeInMs / (60 * 60 * 1000); + long minutes = (timeInMs / (60 * 1000)) % 60; + long seconds = (timeInMs % (60 * 1000)) / 1000; + long mSeconds = (timeInMs % (1000)); + + if (hours > 0) + return String.format("%02d:%02d:%02d.%03d hours", hours, minutes, + seconds, mSeconds); + else if (minutes > 0) + return String.format("%02d:%02d.%03d minutes", minutes, seconds, + mSeconds); + else if (seconds > 0) + return String.format("%02d.%03d seconds", seconds, mSeconds); + else + return String.format("%03d milli-seconds", mSeconds); + } + + public void logFailedEvent(AuditEventBase event) { + logFailedEvent(event, ""); + } + + public void logFailedEvent(AuditEventBase event, Throwable excp) { + long now = System.currentTimeMillis(); + + long timeSinceLastReport = now - mFailedLogLastReportTime.get(); + long countSinceLastReport = mFailedLogCountSinceLastReport + .incrementAndGet(); + long countLifeTime = mFailedLogCountLifeTime.incrementAndGet(); + + if (timeSinceLastReport >= mLogFailureReportMinIntervalInMs) { + mFailedLogLastReportTime.set(now); + mFailedLogCountSinceLastReport.set(0); + + if (excp != null) { + LOG.warn( + "failed to log audit event: " + + MiscUtil.stringify(event), excp); + } else { + LOG.warn("failed to log audit event: " + + MiscUtil.stringify(event)); + } + + if (countLifeTime > 1) { // no stats to print for the 1st failure + LOG.warn("Log failure count: " + countSinceLastReport + + " in past " + + formatIntervalForLog(timeSinceLastReport) + "; " + + countLifeTime + " during process lifetime"); + } + } + } + + public void logFailedEvent(Collection events) { + logFailedEvent(events, ""); + } + + public void logFailedEvent(Collection events, Throwable excp) { + for (AuditEventBase event : events) { + logFailedEvent(event, excp); + } + } + + public void logFailedEvent(AuditEventBase event, String message) { + long now = System.currentTimeMillis(); + + long timeSinceLastReport = now - mFailedLogLastReportTime.get(); + long countSinceLastReport = mFailedLogCountSinceLastReport + .incrementAndGet(); + long countLifeTime = mFailedLogCountLifeTime.incrementAndGet(); + + if (timeSinceLastReport >= mLogFailureReportMinIntervalInMs) { + mFailedLogLastReportTime.set(now); + mFailedLogCountSinceLastReport.set(0); + + LOG.warn("failed to log audit event: " + MiscUtil.stringify(event) + + ", errorMessage=" + message); + + if (countLifeTime > 1) { // no stats to print for the 1st failure + LOG.warn("Log failure count: " + countSinceLastReport + + " in past " + + formatIntervalForLog(timeSinceLastReport) + "; " + + countLifeTime + " during process lifetime"); + } + } + } + + public void logFailedEvent(Collection events, + String errorMessage) { + for (AuditEventBase event : events) { + logFailedEvent(event, errorMessage); + } + } + + public void logFailedEventJSON(String event, Throwable excp) { + long now = System.currentTimeMillis(); + + long timeSinceLastReport = now - mFailedLogLastReportTime.get(); + long countSinceLastReport = mFailedLogCountSinceLastReport + .incrementAndGet(); + long countLifeTime = mFailedLogCountLifeTime.incrementAndGet(); + + if (timeSinceLastReport >= mLogFailureReportMinIntervalInMs) { + mFailedLogLastReportTime.set(now); + mFailedLogCountSinceLastReport.set(0); + + if (excp != null) { + LOG.warn("failed to log audit event: " + event, excp); + } else { + LOG.warn("failed to log audit event: " + event); + } + + if (countLifeTime > 1) { // no stats to print for the 1st failure + LOG.warn("Log failure count: " + countSinceLastReport + + " in past " + + formatIntervalForLog(timeSinceLastReport) + "; " + + countLifeTime + " during process lifetime"); + } + } + } + + public void logFailedEventJSON(Collection events, Throwable excp) { + for (String event : events) { + logFailedEventJSON(event, excp); + } + } + +} diff --git a/auth-audits/src/main/java/org/apache/atlas/audit/provider/BufferedAuditProvider.java b/auth-audits/src/main/java/org/apache/atlas/audit/provider/BufferedAuditProvider.java new file mode 100644 index 00000000000..eeee82b3a3c --- /dev/null +++ b/auth-audits/src/main/java/org/apache/atlas/audit/provider/BufferedAuditProvider.java @@ -0,0 +1,120 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.audit.provider; + +import org.apache.atlas.audit.model.AuditEventBase; +import org.apache.atlas.audit.model.AuthzAuditEvent; + +import java.util.Collection; + +public abstract class BufferedAuditProvider extends BaseAuditHandler { + private LogBuffer mBuffer = null; + private LogDestination mDestination = null; + + @Override + public boolean log(AuditEventBase event) { + if (event instanceof AuthzAuditEvent) { + AuthzAuditEvent authzEvent = (AuthzAuditEvent) event; + + if (authzEvent.getAgentHostname() == null) { + authzEvent.setAgentHostname(MiscUtil.getHostname()); + } + + if (authzEvent.getLogType() == null) { + authzEvent.setLogType("RangerAudit"); + } + + if (authzEvent.getEventId() == null) { + authzEvent.setEventId(MiscUtil.generateUniqueId()); + } + } + + if (!mBuffer.add(event)) { + logFailedEvent(event); + return false; + } + return true; + } + + @Override + public boolean log(Collection events) { + boolean ret = true; + for (AuditEventBase event : events) { + ret = log(event); + if (!ret) { + break; + } + } + return ret; + } + + @Override + public boolean logJSON(String event) { + AuditEventBase eventObj = MiscUtil.fromJson(event, + AuthzAuditEvent.class); + return log(eventObj); + } + + @Override + public boolean logJSON(Collection events) { + boolean ret = true; + for (String event : events) { + ret = logJSON(event); + if (!ret) { + break; + } + } + return ret; + } + + @Override + public void start() { + mBuffer.start(mDestination); + } + + @Override + public void stop() { + mBuffer.stop(); + } + + @Override + public void waitToComplete() { + } + + @Override + public void waitToComplete(long timeout) { + } + + @Override + public void flush() { + } + + protected LogBuffer getBuffer() { + return mBuffer; + } + + protected LogDestination getDestination() { + return mDestination; + } + + protected void setBufferAndDestination(LogBuffer buffer, + LogDestination destination) { + mBuffer = buffer; + mDestination = destination; + } +} diff --git a/auth-audits/src/main/java/org/apache/atlas/audit/provider/DebugTracer.java b/auth-audits/src/main/java/org/apache/atlas/audit/provider/DebugTracer.java new file mode 100644 index 00000000000..914d56b6388 --- /dev/null +++ b/auth-audits/src/main/java/org/apache/atlas/audit/provider/DebugTracer.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.audit.provider; + +public interface DebugTracer { + void debug(String msg); + void debug(String msg, Throwable excp); + void info(String msg); + void info(String msg, Throwable excp); + void warn(String msg); + void warn(String msg, Throwable excp); + void error(String msg); + void error(String msg, Throwable excp); +} diff --git a/auth-audits/src/main/java/org/apache/atlas/audit/provider/DummyAuditProvider.java b/auth-audits/src/main/java/org/apache/atlas/audit/provider/DummyAuditProvider.java new file mode 100644 index 00000000000..fb7631d05ec --- /dev/null +++ b/auth-audits/src/main/java/org/apache/atlas/audit/provider/DummyAuditProvider.java @@ -0,0 +1,115 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.audit.provider; + +import org.apache.atlas.audit.model.AuditEventBase; +import org.apache.atlas.audit.model.AuthzAuditEvent; + +import java.io.File; +import java.util.Collection; +import java.util.Properties; + + +public class DummyAuditProvider implements AuditHandler { + @Override + public void init(Properties prop) { + // intentionally left empty + } + + @Override + public boolean log(AuditEventBase event) { + // intentionally left empty + return true; + } + + @Override + public boolean log(Collection events) { + for (AuditEventBase event : events) { + log(event); + } + return true; + } + + @Override + public boolean logJSON(String event) { + AuditEventBase eventObj = MiscUtil.fromJson(event, + AuthzAuditEvent.class); + return log(eventObj); + } + + @Override + public boolean logJSON(Collection events) { + for (String event : events) { + logJSON(event); + } + return false; + } + + @Override + public void start() { + // intentionally left empty + } + + @Override + public void stop() { + // intentionally left empty + } + + @Override + public void waitToComplete() { + // intentionally left empty + } + + @Override + public void flush() { + // intentionally left empty + } + + /* (non-Javadoc) + * @see org.apache.ranger.audit.provider.AuditProvider#init(java.util.Properties, java.lang.String) + */ + @Override + public void init(Properties prop, String basePropertyName) { + // intentionally left empty + } + + /* (non-Javadoc) + * @see org.apache.ranger.audit.provider.AuditProvider#waitToComplete(long) + */ + @Override + public void waitToComplete(long timeout) { + // intentionally left empty + } + + /* (non-Javadoc) + * @see org.apache.ranger.audit.provider.AuditProvider#getName() + */ + @Override + public String getName() { + return this.getClass().getName(); + } + + /* (non-Javadoc) + * @see org.apache.ranger.audit.provider.AuditProvider#getAuditFileType() + */ + @Override + public boolean logFile(File file) { + return logFile(file); + } + +} diff --git a/auth-audits/src/main/java/org/apache/atlas/audit/provider/LocalFileLogBuffer.java b/auth-audits/src/main/java/org/apache/atlas/audit/provider/LocalFileLogBuffer.java new file mode 100644 index 00000000000..291203ea5d9 --- /dev/null +++ b/auth-audits/src/main/java/org/apache/atlas/audit/provider/LocalFileLogBuffer.java @@ -0,0 +1,695 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.atlas.audit.provider; + +import org.apache.hadoop.security.UserGroupInformation; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileFilter; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.UnsupportedEncodingException; +import java.io.Writer; +import java.security.PrivilegedAction; +import java.util.Arrays; +import java.util.Comparator; +import java.util.TreeSet; + + +public class LocalFileLogBuffer implements LogBuffer { + private String mDirectory = null; + private String mFile = null; + private int mFlushIntervalSeconds = 1 * 60; + private int mFileBufferSizeBytes = 8 * 1024; + private String mEncoding = null; + private boolean mIsAppend = true; + private int mRolloverIntervalSeconds = 10 * 60; + private String mArchiveDirectory = null; + private int mArchiveFileCount = 10; + private DebugTracer mLogger = null; + + private Writer mWriter = null; + private String mBufferFilename = null; + private long mNextRolloverTime = 0; + private long mNextFlushTime = 0; + private int mFileOpenRetryIntervalInMs = 60 * 1000; + private long mNextFileOpenRetryTime = 0; + + private DestinationDispatcherThread mDispatcherThread = null; + + public LocalFileLogBuffer(DebugTracer tracer) { + mLogger = tracer; + } + + public String getDirectory() { + return mDirectory; + } + + public void setDirectory(String directory) { + mDirectory = directory; + } + + public String getFile() { + return mFile; + } + + public void setFile(String file) { + mFile = file; + } + + public int getFileBufferSizeBytes() { + return mFileBufferSizeBytes; + } + + public void setFileBufferSizeBytes(int fileBufferSizeBytes) { + mFileBufferSizeBytes = fileBufferSizeBytes; + } + + public int getFlushIntervalSeconds() { + return mFlushIntervalSeconds; + } + + public void setFlushIntervalSeconds(int flushIntervalSeconds) { + mFlushIntervalSeconds = flushIntervalSeconds; + } + + public String getEncoding() { + return mEncoding; + } + + public void setEncoding(String encoding) { + mEncoding = encoding; + } + + public boolean getIsAppend() { + return mIsAppend; + } + + public void setIsAppend(boolean isAppend) { + mIsAppend = isAppend; + } + + public int getRolloverIntervalSeconds() { + return mRolloverIntervalSeconds; + } + + public void setRolloverIntervalSeconds(int rolloverIntervalSeconds) { + mRolloverIntervalSeconds = rolloverIntervalSeconds; + } + + public String getArchiveDirectory() { + return mArchiveDirectory; + } + + public void setArchiveDirectory(String archiveDirectory) { + mArchiveDirectory = archiveDirectory; + } + + public int getArchiveFileCount() { + return mArchiveFileCount; + } + + public void setArchiveFileCount(int archiveFileCount) { + mArchiveFileCount = archiveFileCount; + } + + + @Override + public void start(LogDestination destination) { + mLogger.debug("==> LocalFileLogBuffer.start()"); + + mDispatcherThread = new DestinationDispatcherThread(this, destination, mLogger); + + mDispatcherThread.setDaemon(true); + + mDispatcherThread.start(); + + mLogger.debug("<== LocalFileLogBuffer.start()"); + } + + @Override + public void stop() { + mLogger.debug("==> LocalFileLogBuffer.stop()"); + + DestinationDispatcherThread dispatcherThread = mDispatcherThread; + mDispatcherThread = null; + + if(dispatcherThread != null && dispatcherThread.isAlive()) { + dispatcherThread.stopThread(); + + try { + dispatcherThread.join(); + } catch (InterruptedException e) { + mLogger.warn("LocalFileLogBuffer.stop(): failed in waiting for DispatcherThread", e); + } + } + + closeFile(); + + mLogger.debug("<== LocalFileLogBuffer.stop()"); + } + + @Override + public boolean isAvailable() { + return mWriter != null; + } + + @Override + public boolean add(T log) { + boolean ret = false; + + String msg = MiscUtil.stringify(log); + + if(msg.contains(MiscUtil.LINE_SEPARATOR)) { + msg = msg.replace(MiscUtil.LINE_SEPARATOR, MiscUtil.ESCAPE_STR + MiscUtil.LINE_SEPARATOR); + } + + synchronized(this) { + checkFileStatus(); + + Writer writer = mWriter; + + if(writer != null) { + try { + writer.write(msg + MiscUtil.LINE_SEPARATOR); + + if(mFileBufferSizeBytes == 0) { + writer.flush(); + } + + ret = true; + } catch(IOException excp) { + mLogger.warn("LocalFileLogBuffer.add(): write failed", excp); + + closeFile(); + } + } + } + + return ret; + } + + @Override + public boolean isEmpty() { + return mDispatcherThread == null || mDispatcherThread.isIdle(); + } + + private synchronized void openFile() { + mLogger.debug("==> LocalFileLogBuffer.openFile()"); + + long now = System.currentTimeMillis(); + + closeFile(); + + if(mNextFileOpenRetryTime <= now) { + try { + mNextRolloverTime = MiscUtil.getNextRolloverTime(mNextRolloverTime, (mRolloverIntervalSeconds * 1000L)); + + long startTime = MiscUtil.getRolloverStartTime(mNextRolloverTime, (mRolloverIntervalSeconds * 1000L)); + + mBufferFilename = MiscUtil.replaceTokens(mDirectory + File.separator + mFile, startTime); + + MiscUtil.createParents(new File(mBufferFilename)); + + FileOutputStream ostream = null; + try { + ostream = new FileOutputStream(mBufferFilename, mIsAppend); + } catch(Exception excp) { + mLogger.warn("LocalFileLogBuffer.openFile(): failed to open file " + mBufferFilename, excp); + } + + if(ostream != null) { + mWriter = createWriter(ostream); + + if(mWriter != null) { + mLogger.debug("LocalFileLogBuffer.openFile(): opened file " + mBufferFilename); + + mNextFlushTime = System.currentTimeMillis() + (mFlushIntervalSeconds * 1000L); + } else { + mLogger.warn("LocalFileLogBuffer.openFile(): failed to open file for write " + mBufferFilename); + + mBufferFilename = null; + } + } + } finally { + if(mWriter == null) { + mNextFileOpenRetryTime = now + mFileOpenRetryIntervalInMs; + } + } + } + + mLogger.debug("<== LocalFileLogBuffer.openFile()"); + } + + private synchronized void closeFile() { + mLogger.debug("==> LocalFileLogBuffer.closeFile()"); + + Writer writer = mWriter; + + mWriter = null; + + if(writer != null) { + try { + writer.flush(); + writer.close(); + } catch(IOException excp) { + mLogger.warn("LocalFileLogBuffer: failed to close file " + mBufferFilename, excp); + } + + if(mDispatcherThread != null) { + mDispatcherThread.addLogfile(mBufferFilename); + } + } + + mLogger.debug("<== LocalFileLogBuffer.closeFile()"); + } + + private void rollover() { + mLogger.debug("==> LocalFileLogBuffer.rollover()"); + + closeFile(); + + openFile(); + + mLogger.debug("<== LocalFileLogBuffer.rollover()"); + } + + private void checkFileStatus() { + long now = System.currentTimeMillis(); + + if(now > mNextRolloverTime) { + rollover(); + } else if(mWriter == null) { + openFile(); + } else if(now > mNextFlushTime) { + try { + mNextFlushTime = now + (mFlushIntervalSeconds * 1000L); + + mWriter.flush(); + } catch (IOException excp) { + mLogger.warn("LocalFileLogBuffer: failed to flush to file " + mBufferFilename, excp); + } + } + } + + private Writer createWriter(OutputStream os ) { + Writer writer = null; + + if(os != null) { + if(mEncoding != null) { + try { + writer = new OutputStreamWriter(os, mEncoding); + } catch(UnsupportedEncodingException excp) { + mLogger.warn("LocalFileLogBuffer: failed to create output writer for file " + mBufferFilename, excp); + } + } + + if(writer == null) { + writer = new OutputStreamWriter(os); + } + + if(mFileBufferSizeBytes > 0 && writer != null) { + writer = new BufferedWriter(writer, mFileBufferSizeBytes); + } + } + + return writer; + } + + boolean isCurrentFilename(String filename) { + return filename != null && filename.equals(mBufferFilename); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + + sb.append("LocalFileLogBuffer {"); + sb.append("Directory=").append(mDirectory).append("; "); + sb.append("File=").append(mFile).append("; "); + sb.append("RolloverIntervaSeconds=").append(mRolloverIntervalSeconds).append("; "); + sb.append("ArchiveDirectory=").append(mArchiveDirectory).append("; "); + sb.append("ArchiveFileCount=").append(mArchiveFileCount); + sb.append("}"); + + return sb.toString(); + } + +} + +class DestinationDispatcherThread extends Thread { + private TreeSet mCompletedLogfiles = new TreeSet(); + private boolean mStopThread = false; + private LocalFileLogBuffer mFileLogBuffer = null; + private LogDestination mDestination = null; + private DebugTracer mLogger = null; + + private String mCurrentLogfile = null; + + public DestinationDispatcherThread(LocalFileLogBuffer fileLogBuffer, LogDestination destination, DebugTracer tracer) { + super(DestinationDispatcherThread.class.getSimpleName() + "-" + System.currentTimeMillis()); + + mLogger = tracer; + + mFileLogBuffer = fileLogBuffer; + mDestination = destination; + + setDaemon(true); + } + + public void addLogfile(String filename) { + mLogger.debug("==> DestinationDispatcherThread.addLogfile(" + filename + ")"); + + if(filename != null) { + synchronized(mCompletedLogfiles) { + mCompletedLogfiles.add(filename); + mCompletedLogfiles.notify(); + } + } + + mLogger.debug("<== DestinationDispatcherThread.addLogfile(" + filename + ")"); + } + + public void stopThread() { + mStopThread = true; + } + + public boolean isIdle() { + synchronized(mCompletedLogfiles) { + return mCompletedLogfiles.isEmpty() && mCurrentLogfile == null; + } + } + + @Override + public void run() { + UserGroupInformation loginUser = null; + + try { + loginUser = UserGroupInformation.getLoginUser(); + } catch (IOException excp) { + mLogger.error("DestinationDispatcherThread.run(): failed to get login user details. Audit files will not be sent to HDFS destination", excp); + } + + if(loginUser == null) { + mLogger.error("DestinationDispatcherThread.run(): failed to get login user. Audit files will not be sent to HDFS destination"); + + return; + } + + loginUser.doAs(new PrivilegedAction() { + @Override + public Integer run() { + doRun(); + + return 0; + } + }); + } + + private void doRun() { + init(); + + mDestination.start(); + + long pollIntervalInMs = 1000L; + + while(! mStopThread) { + synchronized(mCompletedLogfiles) { + while(mCompletedLogfiles.isEmpty() && !mStopThread) { + try { + mCompletedLogfiles.wait(pollIntervalInMs); + } catch(InterruptedException excp) { + throw new RuntimeException("DestinationDispatcherThread.run(): failed to wait for log file", excp); + } + } + + mCurrentLogfile = mCompletedLogfiles.pollFirst(); + } + + if(mCurrentLogfile != null) { + sendCurrentFile(); + } + } + + mDestination.stop(); + } + + private void init() { + mLogger.debug("==> DestinationDispatcherThread.init()"); + + String dirName = MiscUtil.replaceTokens(mFileLogBuffer.getDirectory(), 0); + + if(dirName != null) { + File directory = new File(dirName); + + if(directory.exists() && directory.isDirectory()) { + File[] files = directory.listFiles(); + + if(files != null) { + for(File file : files) { + if(file.exists() && file.isFile() && file.canRead()) { + String filename = file.getAbsolutePath(); + if(! mFileLogBuffer.isCurrentFilename(filename)) { + addLogfile(filename); + } + } + } + } + } + } + + mLogger.debug("<== DestinationDispatcherThread.init()"); + } + + private boolean sendCurrentFile() { + mLogger.debug("==> DestinationDispatcherThread.sendCurrentFile()"); + + boolean ret = false; + + long destinationPollIntervalInMs = 1000L; + + BufferedReader reader = openCurrentFile(); + try { + while(!mStopThread) { + String log = getNextStringifiedLog(reader); + + if(log == null) { // reached end-of-file + ret = true; + + break; + } + + try { + // loop until log is sent successfully + while(!mStopThread && !mDestination.sendStringified(log)) { + try { + Thread.sleep(destinationPollIntervalInMs); + } catch(InterruptedException excp) { + throw new RuntimeException("LocalFileLogBuffer.sendCurrentFile(" + mCurrentLogfile + "): failed while waiting for destination to be available", excp); + } + } + } catch ( AuditMessageException msgError) { + mLogger.error("Error in log message:" + log); + //If there is error in log message, then it will be skipped + } + } + } finally { + closeCurrentFile(reader); + } + + if(!mStopThread) { + mDestination.flush(); + archiveCurrentFile(); + } + + mLogger.debug("<== DestinationDispatcherThread.sendCurrentFile()"); + + return ret; + } + + private String getNextStringifiedLog(BufferedReader mReader) { + String log = null; + + if(mReader != null) { + try { + while(true) { + String line = mReader.readLine(); + + if(line == null) { // reached end-of-file + break; + } + + if(line.endsWith(MiscUtil.ESCAPE_STR)) { + line = line.substring(0, line.length() - MiscUtil.ESCAPE_STR.length()); + + if(log == null) { + log = line; + } else { + log += MiscUtil.LINE_SEPARATOR; + log += line; + } + + continue; + } else { + if(log == null) { + log = line; + } else { + log += line; + } + break; + } + } + } catch (IOException excp) { + mLogger.warn("getNextStringifiedLog.getNextLog(): failed to read from file " + mCurrentLogfile, excp); + } + } + + return log; + } + + private BufferedReader openCurrentFile() { + mLogger.debug("==> openCurrentFile(" + mCurrentLogfile + ")"); + BufferedReader mReader = null; + + if(mCurrentLogfile != null) { + try { + FileInputStream inStr = new FileInputStream(mCurrentLogfile); + + InputStreamReader strReader = createReader(inStr); + + if(strReader != null) { + mReader = new BufferedReader(strReader); + } + } catch(FileNotFoundException excp) { + mLogger.warn("openNextFile(): error while opening file " + mCurrentLogfile, excp); + } + } + + mLogger.debug("<== openCurrentFile(" + mCurrentLogfile + ")"); + return mReader; + } + + private void closeCurrentFile(BufferedReader mReader) { + mLogger.debug("==> closeCurrentFile(" + mCurrentLogfile + ")"); + + if(mReader != null) { + try { + mReader.close(); + } catch(IOException excp) { + // ignore + } + } + + mLogger.debug("<== closeCurrentFile(" + mCurrentLogfile + ")"); + } + + private void archiveCurrentFile() { + if(mCurrentLogfile != null) { + File logFile = new File(mCurrentLogfile); + String archiveDirName = MiscUtil.replaceTokens(mFileLogBuffer.getArchiveDirectory(), 0); + String archiveFilename = archiveDirName + File.separator +logFile.getName(); + + try { + if(logFile.exists()) { + File archiveFile = new File(archiveFilename); + + MiscUtil.createParents(archiveFile); + + if(! logFile.renameTo(archiveFile)) { + // TODO: renameTo() does not work in all cases. in case of failure, copy the file contents to the destination and delete the file + mLogger.warn("archiving failed to move file: " + mCurrentLogfile + " ==> " + archiveFilename); + } + + File archiveDir = new File(archiveDirName); + File[] files = archiveDir.listFiles(new FileFilter() { + @Override + public boolean accept(File f) { + return f.isFile(); + } + }); + + int numOfFilesToDelete = files == null ? 0 : (files.length - mFileLogBuffer.getArchiveFileCount()); + + if(numOfFilesToDelete > 0) { + Arrays.sort(files, new Comparator() { + @Override + public int compare(File f1, File f2) { + return (int)(f1.lastModified() - f2.lastModified()); + } + }); + + for(int i = 0; i < numOfFilesToDelete; i++) { + if(! files[i].delete()) { + mLogger.warn("archiving failed to delete file: " + files[i].getAbsolutePath()); + } + } + } + } + } catch(Exception excp) { + mLogger.warn("archiveCurrentFile(): faile to move " + mCurrentLogfile + " to archive location " + archiveFilename, excp); + } + } + mCurrentLogfile = null; + } + + private InputStreamReader createReader(InputStream iStr) { + InputStreamReader reader = null; + + if(iStr != null) { + String encoding = mFileLogBuffer.getEncoding(); + + if(encoding != null) { + try { + reader = new InputStreamReader(iStr, encoding); + } catch(UnsupportedEncodingException excp) { + mLogger.warn("createReader(): failed to create input reader.", excp); + } + } + + if(reader == null) { + reader = new InputStreamReader(iStr); + } + } + + return reader; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + + sb.append("DestinationDispatcherThread {"); + sb.append("ThreadName=").append(this.getName()).append("; "); + sb.append("CompletedLogfiles.size()=").append(mCompletedLogfiles.size()).append("; "); + sb.append("StopThread=").append(mStopThread).append("; "); + sb.append("CurrentLogfile=").append(mCurrentLogfile); + sb.append("}"); + + return sb.toString(); + } +} + diff --git a/auth-audits/src/main/java/org/apache/atlas/audit/provider/Log4jAuditProvider.java b/auth-audits/src/main/java/org/apache/atlas/audit/provider/Log4jAuditProvider.java new file mode 100644 index 00000000000..97a8a0d2460 --- /dev/null +++ b/auth-audits/src/main/java/org/apache/atlas/audit/provider/Log4jAuditProvider.java @@ -0,0 +1,100 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.audit.provider; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.audit.destination.AuditDestination; +import org.apache.atlas.audit.model.AuditEventBase; +import org.apache.atlas.audit.model.AuthzAuditEvent; + +import java.util.Collection; +import java.util.Properties; + + +public class Log4jAuditProvider extends AuditDestination { + + private static final Log LOG = LogFactory.getLog(Log4jAuditProvider.class); + private static final Log AUDITLOG = LogFactory.getLog("xaaudit." + Log4jAuditProvider.class.getName()); + + public static final String AUDIT_LOG4J_IS_ASYNC_PROP = "xasecure.audit.log4j.is.async"; + public static final String AUDIT_LOG4J_MAX_QUEUE_SIZE_PROP = "xasecure.audit.log4j.async.max.queue.size"; + public static final String AUDIT_LOG4J_MAX_FLUSH_INTERVAL_PROP = "xasecure.audit.log4j.async.max.flush.interval.ms"; + + + public Log4jAuditProvider() { + LOG.info("Log4jAuditProvider: creating.."); + } + + @Override + public void init(Properties props) { + LOG.info("Log4jAuditProvider.init()"); + + super.init(props); + } + + @Override + public boolean log(AuditEventBase event) { + if(! AUDITLOG.isInfoEnabled()) + return true; + + if(event != null) { + String eventStr = MiscUtil.stringify(event); + AUDITLOG.info(eventStr); + } + return true; + } + + @Override + public boolean log(Collection events) { + for (AuditEventBase event : events) { + log(event); + } + return true; + } + + @Override + public boolean logJSON(String event) { + AuditEventBase eventObj = MiscUtil.fromJson(event, + AuthzAuditEvent.class); + return log(eventObj); + } + + @Override + public boolean logJSON(Collection events) { + for (String event : events) { + logJSON(event); + } + return true; + } + + @Override + public void start() { + // intentionally left empty + } + + @Override + public void stop() { + // intentionally left empty + } + + + + +} diff --git a/auth-audits/src/main/java/org/apache/atlas/audit/provider/Log4jTracer.java b/auth-audits/src/main/java/org/apache/atlas/audit/provider/Log4jTracer.java new file mode 100644 index 00000000000..32767287af8 --- /dev/null +++ b/auth-audits/src/main/java/org/apache/atlas/audit/provider/Log4jTracer.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.audit.provider; + +import org.apache.commons.logging.Log; + +public class Log4jTracer implements DebugTracer { + private Log mLogger = null; + + public Log4jTracer(Log logger) { + mLogger = logger; + } + + public void debug(String msg) { + mLogger.debug(msg); + } + + public void debug(String msg, Throwable excp) { + mLogger.debug(msg, excp); + } + + public void info(String msg) { + mLogger.info(msg); + } + + public void info(String msg, Throwable excp) { + mLogger.info(msg, excp); + } + + public void warn(String msg) { + mLogger.warn(msg); + } + + public void warn(String msg, Throwable excp) { + mLogger.warn(msg, excp); + } + + public void error(String msg) { + mLogger.error(msg); + } + + public void error(String msg, Throwable excp) { + mLogger.error(msg, excp); + } +} diff --git a/auth-audits/src/main/java/org/apache/atlas/audit/provider/LogBuffer.java b/auth-audits/src/main/java/org/apache/atlas/audit/provider/LogBuffer.java new file mode 100644 index 00000000000..305d6141439 --- /dev/null +++ b/auth-audits/src/main/java/org/apache/atlas/audit/provider/LogBuffer.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.atlas.audit.provider; + + +public interface LogBuffer { + void start(LogDestination destination); + + void stop(); + + boolean isAvailable(); + + boolean isEmpty(); + + boolean add(T log); +} diff --git a/auth-audits/src/main/java/org/apache/atlas/audit/provider/LogDestination.java b/auth-audits/src/main/java/org/apache/atlas/audit/provider/LogDestination.java new file mode 100644 index 00000000000..644200a27ad --- /dev/null +++ b/auth-audits/src/main/java/org/apache/atlas/audit/provider/LogDestination.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.atlas.audit.provider; + +import org.apache.atlas.audit.model.AuditEventBase; + +public interface LogDestination { + void start(); + + void stop(); + + boolean isAvailable(); + + boolean send(AuditEventBase log) throws AuditMessageException; + + boolean send(AuditEventBase[] logs) throws AuditMessageException; + + boolean sendStringified(String log) throws AuditMessageException; + + boolean sendStringified(String[] logs) throws AuditMessageException; + + boolean flush(); + + /** + * Name for the destination + * + * @return + */ + String getName(); +} diff --git a/auth-audits/src/main/java/org/apache/atlas/audit/provider/MiscUtil.java b/auth-audits/src/main/java/org/apache/atlas/audit/provider/MiscUtil.java new file mode 100644 index 00000000000..93974cb92b0 --- /dev/null +++ b/auth-audits/src/main/java/org/apache/atlas/audit/provider/MiscUtil.java @@ -0,0 +1,885 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.audit.provider; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import org.apache.commons.lang.ArrayUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authentication.util.KerberosName; +import org.apache.hadoop.security.authentication.util.KerberosUtil; +import org.apache.atlas.authorization.hadoop.utils.RangerCredentialProvider; + +import javax.security.auth.Subject; +import javax.security.auth.login.AppConfigurationEntry; +import javax.security.auth.login.Configuration; +import javax.security.auth.login.LoginContext; +import java.io.File; +import java.io.IOException; +import java.net.InetAddress; +import java.rmi.dgc.VMID; +import java.security.Principal; +import java.security.PrivilegedAction; +import java.security.PrivilegedExceptionAction; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Collections; +import java.util.Date; +import java.util.GregorianCalendar; +import java.util.HashMap; +import java.util.Hashtable; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Random; +import java.util.Set; +import java.util.StringTokenizer; +import java.util.TimeZone; +import java.util.UUID; +import java.util.regex.Pattern; + +import static org.apache.hadoop.util.PlatformName.IBM_JAVA; + +public class MiscUtil { + private static final Log logger = LogFactory.getLog(MiscUtil.class); + + public static final String TOKEN_START = "%"; + public static final String TOKEN_END = "%"; + public static final String TOKEN_HOSTNAME = "hostname"; + public static final String TOKEN_APP_TYPE = "app-type"; + public static final String TOKEN_JVM_INSTANCE = "jvm-instance"; + public static final String TOKEN_TIME = "time:"; + public static final String TOKEN_PROPERTY = "property:"; + public static final String TOKEN_ENV = "env:"; + public static final String ESCAPE_STR = "\\"; + + private static final VMID sJvmID = new VMID(); + + public static String LINE_SEPARATOR = System.getProperty("line.separator"); + + private static Gson sGsonBuilder = null; + private static String sApplicationType = null; + private static UserGroupInformation ugiLoginUser = null; + private static Subject subjectLoginUser = null; + private static String local_hostname = null; + + private static Map logHistoryList = new Hashtable(); + private static int logInterval = 30000; // 30 seconds + + static { + try { + sGsonBuilder = new GsonBuilder().setDateFormat( + "yyyy-MM-dd HH:mm:ss.SSS").create(); + } catch (Throwable excp) { + logger.warn( + "failed to create GsonBuilder object. stringify() will return obj.toString(), instead of Json", + excp); + } + + initLocalHost(); + } + + public static String replaceTokens(String str, long time) { + if (str == null) { + return str; + } + + if (time <= 0) { + time = System.currentTimeMillis(); + } + + for (int startPos = 0; startPos < str.length();) { + int tagStartPos = str.indexOf(TOKEN_START, startPos); + + if (tagStartPos == -1) { + break; + } + + int tagEndPos = str.indexOf(TOKEN_END, + tagStartPos + TOKEN_START.length()); + + if (tagEndPos == -1) { + break; + } + + String tag = str.substring(tagStartPos, + tagEndPos + TOKEN_END.length()); + String token = tag.substring(TOKEN_START.length(), + tag.lastIndexOf(TOKEN_END)); + String val = ""; + + if (token != null) { + if (token.equals(TOKEN_HOSTNAME)) { + val = getHostname(); + } else if (token.equals(TOKEN_APP_TYPE)) { + val = getApplicationType(); + } else if (token.equals(TOKEN_JVM_INSTANCE)) { + val = getJvmInstanceId(); + } else if (token.startsWith(TOKEN_PROPERTY)) { + String propertyName = token.substring(TOKEN_PROPERTY + .length()); + + val = getSystemProperty(propertyName); + } else if (token.startsWith(TOKEN_ENV)) { + String envName = token.substring(TOKEN_ENV.length()); + + val = getEnv(envName); + } else if (token.startsWith(TOKEN_TIME)) { + String dtFormat = token.substring(TOKEN_TIME.length()); + + val = getFormattedTime(time, dtFormat); + } + } + + if (val == null) { + val = ""; + } + + str = str.substring(0, tagStartPos) + val + + str.substring(tagEndPos + TOKEN_END.length()); + startPos = tagStartPos + val.length(); + } + + return str; + } + + public static String getHostname() { + String ret = local_hostname; + + if (ret == null) { + initLocalHost(); + + ret = local_hostname; + + if (ret == null) { + ret = "unknown"; + } + } + + return ret; + } + + public static void setApplicationType(String applicationType) { + sApplicationType = applicationType; + } + + public static String getApplicationType() { + return sApplicationType; + } + + public static String getJvmInstanceId() { + Integer val = Integer.valueOf(sJvmID.toString().hashCode()); + long longVal = val.longValue(); + String ret = Long.toString(Math.abs(longVal)); + + return ret; + } + + public static String getSystemProperty(String propertyName) { + String ret = null; + + try { + ret = propertyName != null ? System.getProperty(propertyName) + : null; + } catch (Exception excp) { + logger.warn("getSystemProperty(" + propertyName + ") failed", excp); + } + + return ret; + } + + public static String getEnv(String envName) { + String ret = null; + + try { + ret = envName != null ? System.getenv(envName) : null; + } catch (Exception excp) { + logger.warn("getenv(" + envName + ") failed", excp); + } + + return ret; + } + + public static String getFormattedTime(long time, String format) { + String ret = null; + + try { + SimpleDateFormat sdf = new SimpleDateFormat(format); + + ret = sdf.format(time); + } catch (Exception excp) { + logger.warn("SimpleDateFormat.format() failed: " + format, excp); + } + + return ret; + } + + public static void createParents(File file) { + if (file != null) { + String parentName = file.getParent(); + + if (parentName != null) { + File parentDir = new File(parentName); + + if (!parentDir.exists()) { + if (!parentDir.mkdirs()) { + logger.warn("createParents(): failed to create " + + parentDir.getAbsolutePath()); + } + } + } + } + } + + public static long getNextRolloverTime(long lastRolloverTime, long interval) { + long now = System.currentTimeMillis() / 1000 * 1000; // round to second + + if (lastRolloverTime <= 0) { + // should this be set to the next multiple-of-the-interval from + // start of the day? + return now + interval; + } else if (lastRolloverTime <= now) { + long nextRolloverTime = now + interval; + + // keep it at 'interval' boundary + long trimInterval = (nextRolloverTime - lastRolloverTime) + % interval; + + return nextRolloverTime - trimInterval; + } else { + return lastRolloverTime; + } + } + + public static long getRolloverStartTime(long nextRolloverTime, long interval) { + return (nextRolloverTime <= interval) ? System.currentTimeMillis() + : nextRolloverTime - interval; + } + + public static int parseInteger(String str, int defValue) { + int ret = defValue; + + if (str != null) { + try { + ret = Integer.parseInt(str); + } catch (Exception excp) { + // ignore + } + } + + return ret; + } + + public static String generateUniqueId() { + return UUID.randomUUID().toString(); + } + + // UUID.randomUUID() uses SecureRandom, which is seen to be slow in some environments; this method uses Random + public static String generateGuid() { + byte[] randomBytes = new byte[16]; + + RandomHolder.random.nextBytes(randomBytes); + + UUID uuid = UUID.nameUUIDFromBytes(randomBytes); + + return uuid.toString(); + } + + public static String stringify(T log) { + String ret = null; + + if (log != null) { + if (log instanceof String) { + ret = (String) log; + } else if (MiscUtil.sGsonBuilder != null) { + ret = MiscUtil.sGsonBuilder.toJson(log); + } else { + ret = log.toString(); + } + } + + return ret; + } + + static public T fromJson(String jsonStr, Class clazz) { + return sGsonBuilder.fromJson(jsonStr, clazz); + } + + public static String getStringProperty(Properties props, String propName) { + String ret = null; + + if (props != null && propName != null) { + String val = props.getProperty(propName); + if (val != null) { + ret = val; + } + } + + return ret; + } + + public static String getStringProperty(Properties props, String propName, String defValue) { + String ret = defValue; + + if (props != null && propName != null) { + String val = props.getProperty(propName); + if (val != null) { + ret = val; + } + } + + return ret; + } + + public static boolean getBooleanProperty(Properties props, String propName, + boolean defValue) { + boolean ret = defValue; + + if (props != null && propName != null) { + String val = props.getProperty(propName); + + if (val != null) { + ret = Boolean.valueOf(val); + } + } + + return ret; + } + + public static int getIntProperty(Properties props, String propName, + int defValue) { + int ret = defValue; + + if (props != null && propName != null) { + String val = props.getProperty(propName); + if (val != null) { + try { + ret = Integer.parseInt(val); + } catch (NumberFormatException excp) { + ret = defValue; + } + } + } + + return ret; + } + + public static long getLongProperty(Properties props, String propName, + long defValue) { + long ret = defValue; + + if (props != null && propName != null) { + String val = props.getProperty(propName); + if (val != null) { + try { + ret = Long.parseLong(val); + } catch (NumberFormatException excp) { + ret = defValue; + } + } + } + + return ret; + } + + public static Map getPropertiesWithPrefix(Properties props, + String prefix) { + Map prefixedProperties = new HashMap(); + + if (props != null && prefix != null) { + for (String key : props.stringPropertyNames()) { + if (key == null) { + continue; + } + + String val = props.getProperty(key); + + if (key.startsWith(prefix)) { + key = key.substring(prefix.length()); + + if (key == null) { + continue; + } + + prefixedProperties.put(key, val); + } + } + } + + return prefixedProperties; + } + + /** + * @param destListStr + * @param delim + * @return + */ + public static List toArray(String destListStr, String delim) { + List list = new ArrayList(); + if (destListStr != null && !destListStr.isEmpty()) { + StringTokenizer tokenizer = new StringTokenizer(destListStr, + delim.trim()); + while (tokenizer.hasMoreTokens()) { + list.add(tokenizer.nextToken()); + } + } + return list; + } + + public static String getCredentialString(String url, String alias) { + if (url != null && alias != null) { + return RangerCredentialProvider.getInstance() + .getCredentialString(url, alias); + } + return null; + } + + public static UserGroupInformation createUGIFromSubject(Subject subject) + throws IOException { + logger.info("SUBJECT " + (subject == null ? "not found" : "found")); + UserGroupInformation ugi = null; + if (subject != null) { + logger.info("SUBJECT.PRINCIPALS.size()=" + + subject.getPrincipals().size()); + Set principals = subject.getPrincipals(); + for (Principal principal : principals) { + logger.info("SUBJECT.PRINCIPAL.NAME=" + principal.getName()); + } + try { + // Do not remove the below statement. The default + // getLoginUser does some initialization which is needed + // for getUGIFromSubject() to work. + UserGroupInformation.getLoginUser(); + logger.info("Default UGI before using new Subject:" + + UserGroupInformation.getLoginUser()); + } catch (Throwable t) { + logger.error(t); + } + ugi = UserGroupInformation.getUGIFromSubject(subject); + logger.info("SUBJECT.UGI.NAME=" + ugi.getUserName() + ", ugi=" + + ugi); + } else { + logger.info("Server username is not available"); + } + return ugi; + } + + /** + * @param newUGI + * @param newSubject + */ + public static void setUGILoginUser(UserGroupInformation newUGI, + Subject newSubject) { + if (newUGI != null) { + UserGroupInformation.setLoginUser(newUGI); + ugiLoginUser = newUGI; + logger.info("Setting UGI=" + newUGI); + } else { + logger.error("UGI is null. Not setting it."); + } + if (newSubject != null) { + logger.info("Setting SUBJECT"); + subjectLoginUser = newSubject; + } + } + + public static UserGroupInformation getUGILoginUser() { + UserGroupInformation ret = ugiLoginUser; + + if (ret == null) { + try { + // Do not cache ugiLoginUser if it is not explicitly set with + // setUGILoginUser. + // It appears that the user represented by + // the returned object is periodically logged out and logged back + // in when the token is scheduled to expire. So it is better + // to get the user object every time from UserGroupInformation class and + // not cache it + ret = getLoginUser(); + } catch (IOException e) { + logger.error("Error getting UGI.", e); + } + } + + if(ret != null) { + try { + ret.checkTGTAndReloginFromKeytab(); + } catch(IOException ioe) { + logger.error("Error renewing TGT and relogin. Ignoring Exception, and continuing with the old TGT", ioe); + } + } + + return ret; + } + + /** + * Execute the {@link PrivilegedExceptionAction} on the {@link UserGroupInformation} if it's set, otherwise call it directly + */ + public static X executePrivilegedAction(final PrivilegedExceptionAction action) throws Exception { + final UserGroupInformation ugi = getUGILoginUser(); + if (ugi != null) { + return ugi.doAs(action); + } else { + return action.run(); + } + } + + /** + * Execute the {@link PrivilegedAction} on the {@link UserGroupInformation} if it's set, otherwise call it directly. + */ + public static X executePrivilegedAction(final PrivilegedAction action) { + final UserGroupInformation ugi = getUGILoginUser(); + if (ugi != null) { + return ugi.doAs(action); + } else { + return action.run(); + } + } + + public static Subject getSubjectLoginUser() { + return subjectLoginUser; + } + + public static String getKerberosNamesRules() { + return KerberosName.getRules(); + } + /** + * + * @param principal + * This could be in the format abc/host@domain.com + * @return + */ + static public String getShortNameFromPrincipalName(String principal) { + if (principal == null) { + return null; + } + try { + // Assuming it is kerberos name for now + KerberosName kerbrosName = new KerberosName(principal); + String userName = kerbrosName.getShortName(); + userName = StringUtils.substringBefore(userName, "/"); + userName = StringUtils.substringBefore(userName, "@"); + return userName; + } catch (Throwable t) { + logger.error("Error converting kerberos name. principal=" + + principal + ", KerberosName.rules=" + KerberosName.getRules()); + } + return principal; + } + + /** + * @param userName + * @return + */ + static public Set getGroupsForRequestUser(String userName) { + if (userName != null) { + try { + UserGroupInformation ugi = UserGroupInformation + .createRemoteUser(userName); + String[] groups = ugi.getGroupNames(); + if (groups != null && groups.length > 0) { + Set groupsSet = new java.util.HashSet(); + for (String group : groups) { + groupsSet.add(group); + } + return groupsSet; + } + } catch (Throwable e) { + logErrorMessageByInterval(logger, + "Error getting groups for users. userName=" + userName, e); + } + } + return Collections.emptySet(); + } + + static public boolean logErrorMessageByInterval(Log useLogger, + String message) { + return logErrorMessageByInterval(useLogger, message, null); + } + + /** + * @param useLogger + * @param message + * @param e + */ + static public boolean logErrorMessageByInterval(Log useLogger, + String message, Throwable e) { + if (message == null) { + return false; + } + + LogHistory log = logHistoryList.get(message); + if (log == null) { + log = new LogHistory(); + logHistoryList.put(message, log); + } + if ((System.currentTimeMillis() - log.lastLogTime) > logInterval) { + log.lastLogTime = System.currentTimeMillis(); + int counter = log.counter; + log.counter = 0; + if (counter > 0) { + message += ". Messages suppressed before: " + counter; + } + if (e == null) { + useLogger.error(message); + } else { + useLogger.error(message, e); + } + + return true; + } else { + log.counter++; + } + return false; + + } + + public static void setUGIFromJAASConfig(String jaasConfigAppName) throws Exception { + String keytabFile = null; + String principal = null; + UserGroupInformation ugi = null; + if (logger.isDebugEnabled()){ + logger.debug("===> MiscUtil.setUGIFromJAASConfig() jaasConfigAppName: " + jaasConfigAppName); + } + try { + AppConfigurationEntry entries[] = Configuration.getConfiguration().getAppConfigurationEntry(jaasConfigAppName); + if(!ArrayUtils.isEmpty(entries)) { + for (AppConfigurationEntry entry : entries) { + if (entry.getOptions().get("keyTab") != null) { + keytabFile = (String) entry.getOptions().get("keyTab"); + } + if (entry.getOptions().get("principal") != null) { + principal = (String) entry.getOptions().get("principal"); + } + if (!StringUtils.isEmpty(principal) && !StringUtils.isEmpty(keytabFile)) { + break; + } + } + if (!StringUtils.isEmpty(principal) && !StringUtils.isEmpty(keytabFile)) { + // This will login and set the UGI + UserGroupInformation.loginUserFromKeytab(principal, keytabFile); + ugi = UserGroupInformation.getLoginUser(); + } else { + String error_mesage = "Unable to get the principal/keytab from jaasConfigAppName: " + jaasConfigAppName; + logger.error(error_mesage); + throw new Exception(error_mesage); + } + logger.info("MiscUtil.setUGIFromJAASConfig() UGI: " + ugi + " principal: " + principal + " keytab: " + keytabFile); + } else { + logger.warn("JAASConfig file not found! Ranger Plugin will not working in a Secure Cluster..."); + } + } catch ( Exception e) { + logger.error("Unable to set UGI for Principal: " + principal + " keytab: " + keytabFile ); + throw e; + } + if (logger.isDebugEnabled()) { + logger.debug("<=== MiscUtil.setUGIFromJAASConfig() jaasConfigAppName: " + jaasConfigAppName + " UGI: " + ugi + " principal: " + principal + " keytab: " + keytabFile); + } + } + + public static void authWithKerberos(String keytab, String principal, + String nameRules) { + + if (keytab == null || principal == null) { + return; + } + Subject serverSubject = new Subject(); + int successLoginCount = 0; + String[] spnegoPrincipals = null; + + try { + if (principal.equals("*")) { + spnegoPrincipals = KerberosUtil.getPrincipalNames(keytab, + Pattern.compile("HTTP/.*")); + if (spnegoPrincipals.length == 0) { + logger.error("No principals found in keytab=" + keytab); + } + } else { + spnegoPrincipals = new String[] { principal }; + } + + if (nameRules != null) { + KerberosName.setRules(nameRules); + } + + boolean useKeytab = true; + if (!useKeytab) { + logger.info("Creating UGI with subject"); + LoginContext loginContext = null; + List loginContexts = new ArrayList(); + for (String spnegoPrincipal : spnegoPrincipals) { + try { + logger.info("Login using keytab " + keytab + + ", for principal " + spnegoPrincipal); + final KerberosConfiguration kerberosConfiguration = new KerberosConfiguration( + keytab, spnegoPrincipal); + loginContext = new LoginContext("", + serverSubject, null, kerberosConfiguration); + loginContext.login(); + successLoginCount++; + logger.info("Login success keytab " + keytab + + ", for principal " + spnegoPrincipal); + loginContexts.add(loginContext); + } catch (Throwable t) { + logger.error("Login failed keytab " + keytab + + ", for principal " + spnegoPrincipal, t); + } + if (successLoginCount > 0) { + logger.info("Total login success count=" + + successLoginCount); + try { + UserGroupInformation + .loginUserFromSubject(serverSubject); + // UserGroupInformation ugi = + // createUGIFromSubject(serverSubject); + // if (ugi != null) { + // setUGILoginUser(ugi, serverSubject); + // } + } catch (Throwable e) { + logger.error("Error creating UGI from subject. subject=" + + serverSubject); + } finally { + if (loginContext != null) { + loginContext.logout(); + } + } + } else { + logger.error("Total logins were successfull from keytab=" + + keytab + ", principal=" + principal); + } + } + } else { + logger.info("Creating UGI from keytab directly. keytab=" + + keytab + ", principal=" + spnegoPrincipals[0]); + UserGroupInformation ugi = UserGroupInformation + .loginUserFromKeytabAndReturnUGI(spnegoPrincipals[0], + keytab); + MiscUtil.setUGILoginUser(ugi, null); + } + + } catch (Throwable t) { + logger.error("Failed to login with given keytab and principal", t); + } + + } + + static class LogHistory { + long lastLogTime = 0; + int counter = 0; + } + + /** + * Kerberos context configuration for the JDK GSS library. + */ + private static class KerberosConfiguration extends Configuration { + private String keytab; + private String principal; + + public KerberosConfiguration(String keytab, String principal) { + this.keytab = keytab; + this.principal = principal; + } + + @Override + public AppConfigurationEntry[] getAppConfigurationEntry(String name) { + Map options = new HashMap(); + if (IBM_JAVA) { + options.put("useKeytab", keytab.startsWith("file://") ? keytab + : "file://" + keytab); + options.put("principal", principal); + options.put("credsType", "acceptor"); + } else { + options.put("keyTab", keytab); + options.put("principal", principal); + options.put("useKeyTab", "true"); + options.put("storeKey", "true"); + options.put("doNotPrompt", "true"); + options.put("useTicketCache", "true"); + options.put("renewTGT", "true"); + options.put("isInitiator", "false"); + } + options.put("refreshKrb5Config", "true"); + String ticketCache = System.getenv("KRB5CCNAME"); + if (ticketCache != null) { + if (IBM_JAVA) { + options.put("useDefaultCcache", "true"); + // The first value searched when "useDefaultCcache" is used. + System.setProperty("KRB5CCNAME", ticketCache); + options.put("renewTGT", "true"); + options.put("credsType", "both"); + } else { + options.put("ticketCache", ticketCache); + } + } + if (logger.isDebugEnabled()) { + options.put("debug", "true"); + } + + return new AppConfigurationEntry[] { new AppConfigurationEntry( + KerberosUtil.getKrb5LoginModuleName(), + AppConfigurationEntry.LoginModuleControlFlag.REQUIRED, + options), }; + } + } + + public static UserGroupInformation getLoginUser() throws IOException { + return UserGroupInformation.getLoginUser(); + } + + private static void initLocalHost() { + if ( logger.isDebugEnabled() ) { + logger.debug("==> MiscUtil.initLocalHost()"); + } + + try { + local_hostname = InetAddress.getLocalHost().getHostName(); + } catch (Throwable excp) { + logger.warn("getHostname()", excp); + } + if ( logger.isDebugEnabled() ) { + logger.debug("<== MiscUtil.initLocalHost()"); + } + } + public static Date getUTCDateForLocalDate(Date date) { + TimeZone gmtTimeZone = TimeZone.getTimeZone("GMT+0"); + Calendar local = Calendar.getInstance(); + int offset = local.getTimeZone().getOffset(local.getTimeInMillis()); + GregorianCalendar utc = new GregorianCalendar(gmtTimeZone); + utc.setTimeInMillis(date.getTime()); + utc.add(Calendar.MILLISECOND, -offset); + return utc.getTime(); + } + public static Date getUTCDate() { + TimeZone gmtTimeZone = TimeZone.getTimeZone("GMT+0"); + Calendar local = Calendar.getInstance(); + int offset = local.getTimeZone().getOffset(local.getTimeInMillis()); + GregorianCalendar utc = new GregorianCalendar(gmtTimeZone); + utc.setTimeInMillis(local.getTimeInMillis()); + utc.add(Calendar.MILLISECOND, -offset); + return utc.getTime(); + } + + // use Holder class to defer initialization until needed + private static class RandomHolder { + static final Random random = new Random(); + } + +} diff --git a/auth-audits/src/main/java/org/apache/atlas/audit/provider/MultiDestAuditProvider.java b/auth-audits/src/main/java/org/apache/atlas/audit/provider/MultiDestAuditProvider.java new file mode 100644 index 00000000000..91acde58e53 --- /dev/null +++ b/auth-audits/src/main/java/org/apache/atlas/audit/provider/MultiDestAuditProvider.java @@ -0,0 +1,233 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.audit.provider; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.audit.model.AuditEventBase; + +import java.io.File; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Properties; + +public class MultiDestAuditProvider extends BaseAuditHandler { + + private static final Log LOG = LogFactory + .getLog(MultiDestAuditProvider.class); + + protected List mProviders = new ArrayList(); + static final String DEFAULT_NAME = "multi_dest"; + + public MultiDestAuditProvider() { + LOG.info("MultiDestAuditProvider: creating.."); + setName(DEFAULT_NAME); + } + + public MultiDestAuditProvider(AuditHandler provider) { + LOG.info("MultiDestAuditProvider(): provider=" + + (provider == null ? null : provider.getName())); + setName(DEFAULT_NAME); + addAuditProvider(provider); + } + + @Override + public void init(Properties props) { + LOG.info("MultiDestAuditProvider.init()"); + + super.init(props); + + for (AuditHandler provider : mProviders) { + try { + provider.init(props); + } catch (Throwable excp) { + LOG.info("MultiDestAuditProvider.init(): failed " + + provider.getClass().getCanonicalName() + ")", excp); + } + } + } + + @Override + public void setParentPath(String parentPath) { + super.setParentPath(parentPath); + for (AuditHandler provider : mProviders) { + if (provider instanceof BaseAuditHandler) { + BaseAuditHandler baseAuditHander = (BaseAuditHandler) provider; + baseAuditHander.setParentPath(getName()); + } + } + } + + @Override + public void setName(String name) { + super.setName(name); + for (AuditHandler provider : mProviders) { + if (provider instanceof BaseAuditHandler) { + BaseAuditHandler baseAuditHander = (BaseAuditHandler) provider; + baseAuditHander.setParentPath(getName()); + } + } + } + + public void addAuditProvider(AuditHandler provider) { + if (provider != null) { + LOG.info("MultiDestAuditProvider.addAuditProvider(providerType=" + + provider.getClass().getCanonicalName() + ")"); + + mProviders.add(provider); + if (provider instanceof BaseAuditHandler) { + BaseAuditHandler baseAuditHander = (BaseAuditHandler) provider; + baseAuditHander.setParentPath(getName()); + } + } + } + + public void addAuditProviders(List providers) { + if (providers != null) { + for (AuditHandler provider : providers) { + LOG.info("Adding " + provider.getName() + + " as consumer to MultiDestination " + getName()); + addAuditProvider(provider); + } + } + } + + @Override + public boolean log(AuditEventBase event) { + for (AuditHandler provider : mProviders) { + try { + provider.log(event); + } catch (Throwable excp) { + logFailedEvent(event, excp); + } + } + return true; + } + + @Override + public boolean log(Collection events) { + for (AuditHandler provider : mProviders) { + try { + provider.log(events); + } catch (Throwable excp) { + logFailedEvent(events, excp); + } + } + return true; + } + + @Override + public boolean logJSON(String event) { + for (AuditHandler provider : mProviders) { + try { + provider.logJSON(event); + } catch (Throwable excp) { + logFailedEventJSON(event, excp); + } + } + return true; + } + + @Override + public boolean logJSON(Collection events) { + for (AuditHandler provider : mProviders) { + try { + provider.logJSON(events); + } catch (Throwable excp) { + logFailedEventJSON(events, excp); + } + } + return true; + } + + + @Override + public boolean logFile(File file) { + for (AuditHandler provider : mProviders) { + try { + provider.logFile(file); + } catch (Throwable excp) { + logFailedEventJSON(file.getAbsolutePath(), excp); + } + } + return true; + } + + @Override + public void start() { + for (AuditHandler provider : mProviders) { + try { + provider.start(); + } catch (Throwable excp) { + LOG.error("MultiDestAuditProvider.start(): failed for provider { " + + provider.getClass().getName() + " }", excp); + } + } + } + + @Override + public void stop() { + for (AuditHandler provider : mProviders) { + try { + provider.stop(); + } catch (Throwable excp) { + LOG.error("MultiDestAuditProvider.stop(): failed for provider { " + + provider.getClass().getName() + " }", excp); + } + } + } + + @Override + public void waitToComplete() { + for (AuditHandler provider : mProviders) { + try { + provider.waitToComplete(); + } catch (Throwable excp) { + LOG.error( + "MultiDestAuditProvider.waitToComplete(): failed for provider { " + + provider.getClass().getName() + " }", excp); + } + } + } + + @Override + public void waitToComplete(long timeout) { + for (AuditHandler provider : mProviders) { + try { + provider.waitToComplete(timeout); + } catch (Throwable excp) { + LOG.error( + "MultiDestAuditProvider.waitToComplete(): failed for provider { " + + provider.getClass().getName() + " }", excp); + } + } + } + + @Override + public void flush() { + for (AuditHandler provider : mProviders) { + try { + provider.flush(); + } catch (Throwable excp) { + LOG.error("MultiDestAuditProvider.flush(): failed for provider { " + + provider.getClass().getName() + " }", excp); + } + } + } +} diff --git a/auth-audits/src/main/java/org/apache/atlas/audit/provider/StandAloneAuditProviderFactory.java b/auth-audits/src/main/java/org/apache/atlas/audit/provider/StandAloneAuditProviderFactory.java new file mode 100644 index 00000000000..2ab6d63d9d4 --- /dev/null +++ b/auth-audits/src/main/java/org/apache/atlas/audit/provider/StandAloneAuditProviderFactory.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.audit.provider; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +public class StandAloneAuditProviderFactory extends AuditProviderFactory { + private static final Log LOG = LogFactory.getLog(StandAloneAuditProviderFactory.class); + + private volatile static StandAloneAuditProviderFactory sFactory = null; + + public static StandAloneAuditProviderFactory getInstance() { + StandAloneAuditProviderFactory ret = sFactory; + if(ret == null) { + synchronized(StandAloneAuditProviderFactory.class) { + ret = sFactory; + if(ret == null) { + ret = sFactory = new StandAloneAuditProviderFactory(); + } + } + } + return ret; + } + + private StandAloneAuditProviderFactory() { + super(); + LOG.info("StandAloneAuditProviderFactory: created.."); + } +} diff --git a/auth-audits/src/main/java/org/apache/atlas/audit/provider/hdfs/HdfsAuditProvider.java b/auth-audits/src/main/java/org/apache/atlas/audit/provider/hdfs/HdfsAuditProvider.java new file mode 100644 index 00000000000..87ce5b312eb --- /dev/null +++ b/auth-audits/src/main/java/org/apache/atlas/audit/provider/hdfs/HdfsAuditProvider.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.audit.provider.hdfs; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.audit.model.AuditEventBase; +import org.apache.atlas.audit.provider.BufferedAuditProvider; +import org.apache.atlas.audit.provider.DebugTracer; +import org.apache.atlas.audit.provider.LocalFileLogBuffer; +import org.apache.atlas.audit.provider.Log4jTracer; +import org.apache.atlas.audit.provider.MiscUtil; + +import java.util.Map; +import java.util.Properties; + +public class HdfsAuditProvider extends BufferedAuditProvider { + private static final Log LOG = LogFactory.getLog(HdfsAuditProvider.class); + + public static final String AUDIT_HDFS_IS_ASYNC_PROP = "xasecure.audit.hdfs.is.async"; + public static final String AUDIT_HDFS_MAX_QUEUE_SIZE_PROP = "xasecure.audit.hdfs.async.max.queue.size"; + public static final String AUDIT_HDFS_MAX_FLUSH_INTERVAL_PROP = "xasecure.audit.hdfs.async.max.flush.interval.ms"; + + public HdfsAuditProvider() { + } + + public void init(Properties props) { + LOG.info("HdfsAuditProvider.init()"); + + super.init(props); + + Map hdfsProps = MiscUtil.getPropertiesWithPrefix(props, "xasecure.audit.hdfs.config."); + + String encoding = hdfsProps.get("encoding"); + + String hdfsDestinationDirectory = hdfsProps.get("destination.directory"); + String hdfsDestinationFile = hdfsProps.get("destination.file"); + int hdfsDestinationFlushIntervalSeconds = MiscUtil.parseInteger(hdfsProps.get("destination.flush.interval.seconds"), 15 * 60); + int hdfsDestinationRolloverIntervalSeconds = MiscUtil.parseInteger(hdfsProps.get("destination.rollover.interval.seconds"), 24 * 60 * 60); + int hdfsDestinationOpenRetryIntervalSeconds = MiscUtil.parseInteger(hdfsProps.get("destination.open.retry.interval.seconds"), 60); + + String localFileBufferDirectory = hdfsProps.get("local.buffer.directory"); + String localFileBufferFile = hdfsProps.get("local.buffer.file"); + int localFileBufferFlushIntervalSeconds = MiscUtil.parseInteger(hdfsProps.get("local.buffer.flush.interval.seconds"), 1 * 60); + int localFileBufferFileBufferSizeBytes = MiscUtil.parseInteger(hdfsProps.get("local.buffer.file.buffer.size.bytes"), 8 * 1024); + int localFileBufferRolloverIntervalSeconds = MiscUtil.parseInteger(hdfsProps.get("local.buffer.rollover.interval.seconds"), 10 * 60); + String localFileBufferArchiveDirectory = hdfsProps.get("local.archive.directory"); + int localFileBufferArchiveFileCount = MiscUtil.parseInteger(hdfsProps.get("local.archive.max.file.count"), 10); + // Added for Azure. Note that exact name of these properties is not known as it contains the variable account name in it. + Map configProps = MiscUtil.getPropertiesWithPrefix(props, "xasecure.audit.destination.hdfs.config."); + + DebugTracer tracer = new Log4jTracer(LOG); + + HdfsLogDestination mHdfsDestination = new HdfsLogDestination(tracer); + + mHdfsDestination.setDirectory(hdfsDestinationDirectory); + mHdfsDestination.setFile(hdfsDestinationFile); + mHdfsDestination.setFlushIntervalSeconds(hdfsDestinationFlushIntervalSeconds); + mHdfsDestination.setEncoding(encoding); + mHdfsDestination.setRolloverIntervalSeconds(hdfsDestinationRolloverIntervalSeconds); + mHdfsDestination.setOpenRetryIntervalSeconds(hdfsDestinationOpenRetryIntervalSeconds); + mHdfsDestination.setConfigProps(configProps); + + LocalFileLogBuffer mLocalFileBuffer = new LocalFileLogBuffer(tracer); + + mLocalFileBuffer.setDirectory(localFileBufferDirectory); + mLocalFileBuffer.setFile(localFileBufferFile); + mLocalFileBuffer.setFlushIntervalSeconds(localFileBufferFlushIntervalSeconds); + mLocalFileBuffer.setFileBufferSizeBytes(localFileBufferFileBufferSizeBytes); + mLocalFileBuffer.setEncoding(encoding); + mLocalFileBuffer.setRolloverIntervalSeconds(localFileBufferRolloverIntervalSeconds); + mLocalFileBuffer.setArchiveDirectory(localFileBufferArchiveDirectory); + mLocalFileBuffer.setArchiveFileCount(localFileBufferArchiveFileCount); + + setBufferAndDestination(mLocalFileBuffer, mHdfsDestination); + } +} + + + diff --git a/auth-audits/src/main/java/org/apache/atlas/audit/provider/hdfs/HdfsLogDestination.java b/auth-audits/src/main/java/org/apache/atlas/audit/provider/hdfs/HdfsLogDestination.java new file mode 100644 index 00000000000..3e4b998e480 --- /dev/null +++ b/auth-audits/src/main/java/org/apache/atlas/audit/provider/hdfs/HdfsLogDestination.java @@ -0,0 +1,517 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.atlas.audit.provider.hdfs; + + +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.atlas.audit.model.AuditEventBase; +import org.apache.atlas.audit.provider.DebugTracer; +import org.apache.atlas.audit.provider.LogDestination; +import org.apache.atlas.audit.provider.MiscUtil; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.UnsupportedEncodingException; +import java.net.URI; +import java.util.Map; + +public class HdfsLogDestination implements LogDestination { + public final static String EXCP_MSG_FILESYSTEM_CLOSED = "Filesystem closed"; + + private String name = getClass().getName(); + + private String mDirectory = null; + private String mFile = null; + private int mFlushIntervalSeconds = 1 * 60; + private String mEncoding = null; + private boolean mIsAppend = false; + private int mRolloverIntervalSeconds = 24 * 60 * 60; + private int mOpenRetryIntervalSeconds = 60; + private DebugTracer mLogger = null; + + private FSDataOutputStream mFsDataOutStream = null; + private OutputStreamWriter mWriter = null; + private String mHdfsFilename = null; + private long mNextRolloverTime = 0; + private long mNextFlushTime = 0; + private long mLastOpenFailedTime = 0; + private boolean mIsStopInProgress = false; + private Map configProps = null; + + public HdfsLogDestination(DebugTracer tracer) { + mLogger = tracer; + } + + + public void setName(String name) { + this.name = name; + } + + + /* (non-Javadoc) + * @see org.apache.ranger.audit.provider.LogDestination#getName() + */ + @Override + public String getName() { + return name; + } + + public String getDirectory() { + return mDirectory; + } + + public void setDirectory(String directory) { + this.mDirectory = directory; + } + + public String getFile() { + return mFile; + } + + public void setFile(String file) { + this.mFile = file; + } + + public int getFlushIntervalSeconds() { + return mFlushIntervalSeconds; + } + + public void setFlushIntervalSeconds(int flushIntervalSeconds) { + mFlushIntervalSeconds = flushIntervalSeconds; + } + + public String getEncoding() { + return mEncoding; + } + + public void setEncoding(String encoding) { + mEncoding = encoding; + } + + public int getRolloverIntervalSeconds() { + return mRolloverIntervalSeconds; + } + + public void setRolloverIntervalSeconds(int rolloverIntervalSeconds) { + this.mRolloverIntervalSeconds = rolloverIntervalSeconds; + } + + public int getOpenRetryIntervalSeconds() { + return mOpenRetryIntervalSeconds; + } + + public void setOpenRetryIntervalSeconds(int minIntervalOpenRetrySeconds) { + this.mOpenRetryIntervalSeconds = minIntervalOpenRetrySeconds; + } + + @Override + public void start() { + mLogger.debug("==> HdfsLogDestination.start()"); + + openFile(); + + mLogger.debug("<== HdfsLogDestination.start()"); + } + + @Override + public void stop() { + mLogger.debug("==> HdfsLogDestination.stop()"); + + mIsStopInProgress = true; + + closeFile(); + + mIsStopInProgress = false; + + mLogger.debug("<== HdfsLogDestination.stop()"); + } + + @Override + public boolean isAvailable() { + return mWriter != null; + } + + @Override + public boolean send(AuditEventBase log) { + boolean ret = true; + + if(log != null) { + String msg = MiscUtil.stringify(log); + + ret = sendStringified(msg); + } + + return ret; + } + + + @Override + public boolean send(AuditEventBase[] logs) { + for (AuditEventBase log : logs) { + boolean ret = send(log); + if(!ret) { + return ret; + } + } + return true; + } + + @Override + public boolean sendStringified(String log) { + boolean ret = false; + + checkFileStatus(); + + OutputStreamWriter writer = mWriter; + + if(writer != null) { + try { + writer.write(log + MiscUtil.LINE_SEPARATOR); + + ret = true; + } catch (IOException excp) { + mLogger.warn("HdfsLogDestination.sendStringified(): write failed", excp); + + closeFile(); + } + } + + return ret; + } + + @Override + public boolean sendStringified(String[] logs) { + for (String log : logs) { + boolean ret = sendStringified(log); + if(!ret) { + return ret; + } + } + return true; + } + + + @Override + public boolean flush() { + mLogger.debug("==> HdfsLogDestination.flush()"); + + boolean ret = false; + + OutputStreamWriter writer = mWriter; + + if(writer != null) { + try { + writer.flush(); + + ret = true; + } catch (IOException excp) { + logException("HdfsLogDestination: flush() failed", excp); + } + } + + FSDataOutputStream ostream = mFsDataOutStream; + + if(ostream != null) { + try { + ostream.hflush(); + + ret = true; + } catch (IOException excp) { + logException("HdfsLogDestination: hflush() failed", excp); + } + } + + if(ret) { + mNextFlushTime = System.currentTimeMillis() + (mFlushIntervalSeconds * 1000L); + } + + mLogger.debug("<== HdfsLogDestination.flush()"); + + return ret; + } + + private void openFile() { + mLogger.debug("==> HdfsLogDestination.openFile()"); + + closeFile(); + + mNextRolloverTime = MiscUtil.getNextRolloverTime(mNextRolloverTime, (mRolloverIntervalSeconds * 1000L)); + + long startTime = MiscUtil.getRolloverStartTime(mNextRolloverTime, (mRolloverIntervalSeconds * 1000L)); + + mHdfsFilename = MiscUtil.replaceTokens(mDirectory + Path.SEPARATOR + mFile, startTime); + + FSDataOutputStream ostream = null; + FileSystem fileSystem = null; + Path pathLogfile = null; + Configuration conf = null; + boolean bOverwrite = false; + + try { + mLogger.debug("HdfsLogDestination.openFile(): opening file " + mHdfsFilename); + + URI uri = URI.create(mHdfsFilename); + + // TODO: mechanism to XA-HDFS plugin to disable auditing of access checks to the current HDFS file + + conf = createConfiguration(); + pathLogfile = new Path(mHdfsFilename); + fileSystem = FileSystem.get(uri, conf); + + try { + if(fileSystem.exists(pathLogfile)) { // file already exists. either append to the file or write to a new file + if(mIsAppend) { + mLogger.info("HdfsLogDestination.openFile(): opening file for append " + mHdfsFilename); + + ostream = fileSystem.append(pathLogfile); + } else { + mHdfsFilename = getNewFilename(mHdfsFilename, fileSystem); + pathLogfile = new Path(mHdfsFilename); + } + } + + // if file does not exist or if mIsAppend==false, create the file + if(ostream == null) { + mLogger.info("HdfsLogDestination.openFile(): opening file for write " + mHdfsFilename); + + createParents(pathLogfile, fileSystem); + ostream = fileSystem.create(pathLogfile, bOverwrite); + } + } catch(IOException excp) { + // append may not be supported by the filesystem; or the file might already be open by another application. Try a different filename + String failedFilename = mHdfsFilename; + + mHdfsFilename = getNewFilename(mHdfsFilename, fileSystem); + pathLogfile = new Path(mHdfsFilename); + + mLogger.info("HdfsLogDestination.openFile(): failed in opening file " + failedFilename + ". Will try opening " + mHdfsFilename); + } + + if(ostream == null){ + mLogger.info("HdfsLogDestination.openFile(): opening file for write " + mHdfsFilename); + + createParents(pathLogfile, fileSystem); + ostream = fileSystem.create(pathLogfile, bOverwrite); + } + } catch(Throwable ex) { + mLogger.warn("HdfsLogDestination.openFile() failed", ex); +// } finally { + // TODO: unset the property set above to exclude auditing of logfile opening + // System.setProperty(hdfsCurrentFilenameProperty, null); + } + + mWriter = createWriter(ostream); + + if(mWriter != null) { + mLogger.debug("HdfsLogDestination.openFile(): opened file " + mHdfsFilename); + + mFsDataOutStream = ostream; + mNextFlushTime = System.currentTimeMillis() + (mFlushIntervalSeconds * 1000L); + mLastOpenFailedTime = 0; + } else { + mLogger.warn("HdfsLogDestination.openFile(): failed to open file for write " + mHdfsFilename); + + mHdfsFilename = null; + mLastOpenFailedTime = System.currentTimeMillis(); + } + + mLogger.debug("<== HdfsLogDestination.openFile(" + mHdfsFilename + ")"); + } + + private void closeFile() { + mLogger.debug("==> HdfsLogDestination.closeFile()"); + + flush(); + + OutputStreamWriter writer = mWriter; + + mWriter = null; + mFsDataOutStream = null; + + if(writer != null) { + try { + mLogger.info("HdfsLogDestination.closeFile(): closing file " + mHdfsFilename); + + writer.close(); + } catch(IOException excp) { + logException("HdfsLogDestination: failed to close file " + mHdfsFilename, excp); + } + } + + mLogger.debug("<== HdfsLogDestination.closeFile()"); + } + + private void rollover() { + mLogger.debug("==> HdfsLogDestination.rollover()"); + + closeFile(); + + openFile(); + + mLogger.debug("<== HdfsLogDestination.rollover()"); + } + + private void checkFileStatus() { + long now = System.currentTimeMillis(); + + if(mWriter == null) { + if(now > (mLastOpenFailedTime + (mOpenRetryIntervalSeconds * 1000L))) { + openFile(); + } + } else if(now > mNextRolloverTime) { + rollover(); + } else if(now > mNextFlushTime) { + flush(); + } + } + + private OutputStreamWriter createWriter(OutputStream os ) { + OutputStreamWriter writer = null; + + if(os != null) { + if(mEncoding != null) { + try { + writer = new OutputStreamWriter(os, mEncoding); + } catch(UnsupportedEncodingException excp) { + mLogger.warn("HdfsLogDestination.createWriter(): failed to create output writer.", excp); + } + } + + if(writer == null) { + writer = new OutputStreamWriter(os); + } + } + + return writer; + } + + private void createParents(Path pathLogfile, FileSystem fileSystem) { + try { + Path parentPath = pathLogfile != null ? pathLogfile.getParent() : null; + + if(parentPath != null && fileSystem != null && !fileSystem.exists(parentPath)) { + fileSystem.mkdirs(parentPath); + } + } catch (IOException e) { + logException("HdfsLogDestination.createParents() failed", e); + } catch (Throwable e) { + mLogger.warn("HdfsLogDestination.createParents() failed", e); + } + } + + private String getNewFilename(String fileName, FileSystem fileSystem) { + if(fileName == null) { + return ""; + } + + for(int i = 1;; i++) { + String ret = fileName; + + String strToAppend = "-" + Integer.toString(i); + + int extnPos = ret.lastIndexOf("."); + + if(extnPos < 0) { + ret += strToAppend; + } else { + String extn = ret.substring(extnPos); + + ret = ret.substring(0, extnPos) + strToAppend + extn; + } + + if(fileSystem != null && fileExists(ret, fileSystem)) { + continue; + } else { + return ret; + } + } + } + + private boolean fileExists(String fileName, FileSystem fileSystem) { + boolean ret = false; + + if(fileName != null && fileSystem != null) { + Path path = new Path(fileName); + + try { + ret = fileSystem.exists(path); + } catch(IOException excp) { + // ignore + } + } + + return ret; + } + + private void logException(String msg, IOException excp) { + // during shutdown, the underlying FileSystem might already be closed; so don't print error details + + if(mIsStopInProgress) { + return; + } + + String excpMsgToExclude = EXCP_MSG_FILESYSTEM_CLOSED; + String excpMsg = excp != null ? excp.getMessage() : null; + boolean excpExcludeLogging = (excpMsg != null && excpMsg.contains(excpMsgToExclude)); + + if(! excpExcludeLogging) { + mLogger.warn(msg, excp); + } + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + + sb.append("HdfsLogDestination {"); + sb.append("Directory=").append(mDirectory).append("; "); + sb.append("File=").append(mFile).append("; "); + sb.append("RolloverIntervalSeconds=").append(mRolloverIntervalSeconds); + sb.append("}"); + + return sb.toString(); + } + + public void setConfigProps(Map configProps) { + this.configProps = configProps; + } + + Configuration createConfiguration() { + Configuration conf = new Configuration(); + if (configProps != null) { + for (Map.Entry entry : configProps.entrySet()) { + String key = entry.getKey(); + String value = entry.getValue(); + // for ease of install config file may contain properties with empty value, skip those + if (StringUtils.isNotEmpty(value)) { + conf.set(key, value); + } + mLogger.info("Adding property to HDFS config: " + key + " => " + value); + } + } + + mLogger.info("Returning HDFS Filesystem Config: " + conf.toString()); + return conf; + } +} diff --git a/auth-audits/src/main/java/org/apache/atlas/audit/provider/solr/SolrAuditProvider.java b/auth-audits/src/main/java/org/apache/atlas/audit/provider/solr/SolrAuditProvider.java new file mode 100644 index 00000000000..914f45685e3 --- /dev/null +++ b/auth-audits/src/main/java/org/apache/atlas/audit/provider/solr/SolrAuditProvider.java @@ -0,0 +1,303 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.audit.provider.solr; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.audit.destination.AuditDestination; +import org.apache.atlas.audit.model.AuditEventBase; +import org.apache.atlas.audit.model.AuthzAuditEvent; +import org.apache.atlas.audit.provider.MiscUtil; +import org.apache.atlas.audit.utils.SolrAppUtil; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.client.solrj.response.UpdateResponse; +import org.apache.solr.common.SolrInputDocument; + +import java.io.IOException; +import java.security.PrivilegedExceptionAction; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.Properties; + +public class SolrAuditProvider extends AuditDestination { + private static final Log LOG = LogFactory.getLog(SolrAuditProvider.class); + + public static final String AUDIT_MAX_QUEUE_SIZE_PROP = "xasecure.audit.solr.async.max.queue.size"; + public static final String AUDIT_MAX_FLUSH_INTERVAL_PROP = "xasecure.audit.solr.async.max.flush.interval.ms"; + public static final String AUDIT_RETRY_WAIT_PROP = "xasecure.audit.solr.retry.ms"; + + static final Object lock = new Object(); + volatile SolrClient solrClient = null; + Date lastConnectTime = null; + long lastFailTime = 0; + + int retryWaitTime = 30000; + + public SolrAuditProvider() { + } + + @Override + public void init(Properties props) { + LOG.info("init() called"); + super.init(props); + + retryWaitTime = MiscUtil.getIntProperty(props, + AUDIT_RETRY_WAIT_PROP, retryWaitTime); + } + + void connect() { + SolrClient me = solrClient; + if (me == null) { + synchronized (lock) { + me = solrClient; + if (me == null) { + final String solrURL = MiscUtil.getStringProperty(props, + "xasecure.audit.solr.solr_url"); + + if (lastConnectTime != null) { + // Let's wait for enough time before retrying + long diff = System.currentTimeMillis() + - lastConnectTime.getTime(); + if (diff < retryWaitTime) { + if (LOG.isDebugEnabled()) { + LOG.debug("Ignore connecting to solr url=" + + solrURL + ", lastConnect=" + diff + + "ms"); + } + return; + } + } + lastConnectTime = new Date(); + + if (solrURL == null || solrURL.isEmpty()) { + LOG.fatal("Solr URL for Audit is empty"); + return; + } + + try { + // TODO: Need to support SolrCloud also + solrClient = MiscUtil.executePrivilegedAction(new PrivilegedExceptionAction() { + @Override + public SolrClient run() throws Exception { + HttpSolrClient.Builder builder = new HttpSolrClient.Builder(); + builder.withBaseSolrUrl(solrURL); + builder.allowCompression(true); + builder.withConnectionTimeout(1000); + HttpSolrClient httpSolrClient = builder.build(); + return httpSolrClient; + }; + }); + + me = solrClient; + } catch (Throwable t) { + LOG.fatal("Can't connect to Solr server. URL=" + + solrURL, t); + } + } + } + } + } + + /* + * (non-Javadoc) + * + * @see + * org.apache.ranger.audit.provider.AuditProvider#log(org.apache.ranger. + * audit.model.AuditEventBase) + */ + @Override + public boolean log(AuditEventBase event) { + if (!(event instanceof AuthzAuditEvent)) { + LOG.error(event.getClass().getName() + + " audit event class type is not supported"); + return false; + } + AuthzAuditEvent authzEvent = (AuthzAuditEvent) event; + // TODO: This should be done at a higher level + + if (authzEvent.getAgentHostname() == null) { + authzEvent.setAgentHostname(MiscUtil.getHostname()); + } + + if (authzEvent.getLogType() == null) { + authzEvent.setLogType("RangerAudit"); + } + + if (authzEvent.getEventId() == null) { + authzEvent.setEventId(MiscUtil.generateUniqueId()); + } + + try { + if (solrClient == null) { + connect(); + if (solrClient == null) { + // Solr is still not initialized. So need to throw error + return false; + } + } + + if (lastFailTime > 0) { + long diff = System.currentTimeMillis() - lastFailTime; + if (diff < retryWaitTime) { + if (LOG.isDebugEnabled()) { + LOG.debug("Ignore sending audit. lastConnect=" + diff + + " ms"); + } + return false; + } + } + // Convert AuditEventBase to Solr document + final SolrInputDocument document = toSolrDoc(authzEvent); + final Collection docs = Collections.singletonList(document); + final UpdateResponse response = SolrAppUtil.addDocsToSolr(solrClient, docs); + + if (response.getStatus() != 0) { + lastFailTime = System.currentTimeMillis(); + + // System.out.println("Response=" + response.toString() + // + ", status= " + response.getStatus() + ", event=" + // + event); + // throw new Exception("Aborting. event=" + event + + // ", response=" + // + response.toString()); + } else { + lastFailTime = 0; + } + + } catch (Throwable t) { + LOG.error("Error sending message to Solr", t); + return false; + } + return true; + } + + @Override + public boolean log(Collection events) { + for (AuditEventBase event : events) { + log(event); + } + return true; + } + + @Override + public boolean logJSON(String event) { + AuditEventBase eventObj = MiscUtil.fromJson(event, + AuthzAuditEvent.class); + return log(eventObj); + } + + @Override + public boolean logJSON(Collection events) { + for (String event : events) { + logJSON(event); + } + return false; + } + + /* + * (non-Javadoc) + * + * @see org.apache.ranger.audit.provider.AuditProvider#start() + */ + @Override + public void start() { + connect(); + } + + /* + * (non-Javadoc) + * + * @see org.apache.ranger.audit.provider.AuditProvider#stop() + */ + @Override + public void stop() { + LOG.info("SolrAuditProvider.stop() called.."); + try { + if (solrClient != null) { + solrClient.close(); + } + } catch (IOException ioe) { + LOG.error("Error while stopping slor!", ioe); + } finally { + solrClient = null; + } + } + + /* + * (non-Javadoc) + * + * @see org.apache.ranger.audit.provider.AuditProvider#waitToComplete() + */ + @Override + public void waitToComplete() { + + } + + + @Override + public void waitToComplete(long timeout) { + + } + + /* + * (non-Javadoc) + * + * @see org.apache.ranger.audit.provider.AuditProvider#flush() + */ + @Override + public void flush() { + // TODO Auto-generated method stub + + } + + SolrInputDocument toSolrDoc(AuthzAuditEvent auditEvent) { + SolrInputDocument doc = new SolrInputDocument(); + doc.addField("id", auditEvent.getEventId()); + doc.addField("access", auditEvent.getAccessType()); + doc.addField("enforcer", auditEvent.getAclEnforcer()); + doc.addField("agent", auditEvent.getAgentId()); + doc.addField("repo", auditEvent.getRepositoryName()); + doc.addField("sess", auditEvent.getSessionId()); + doc.addField("reqUser", auditEvent.getUser()); + doc.addField("reqData", auditEvent.getRequestData()); + doc.addField("resource", auditEvent.getResourcePath()); + doc.addField("cliIP", auditEvent.getClientIP()); + doc.addField("logType", auditEvent.getLogType()); + doc.addField("result", auditEvent.getAccessResult()); + doc.addField("policy", auditEvent.getPolicyId()); + doc.addField("repoType", auditEvent.getRepositoryType()); + doc.addField("resType", auditEvent.getResourceType()); + doc.addField("reason", auditEvent.getResultReason()); + doc.addField("action", auditEvent.getAction()); + doc.addField("evtTime", auditEvent.getEventTime()); + doc.addField("tags", auditEvent.getTags()); + doc.addField("cluster", auditEvent.getClusterName()); + doc.addField("zone", auditEvent.getZoneName()); + doc.addField("agentHost", auditEvent.getAgentHostname()); + return doc; + } + + public boolean isAsync() { + return true; + } + +} diff --git a/auth-audits/src/main/java/org/apache/atlas/audit/queue/AuditAsyncQueue.java b/auth-audits/src/main/java/org/apache/atlas/audit/queue/AuditAsyncQueue.java new file mode 100644 index 00000000000..b807f57be57 --- /dev/null +++ b/auth-audits/src/main/java/org/apache/atlas/audit/queue/AuditAsyncQueue.java @@ -0,0 +1,186 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.audit.queue; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +//import org.apache.log4j.MDC; +import org.apache.atlas.audit.model.AuditEventBase; +import org.apache.atlas.audit.provider.AuditHandler; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.concurrent.LinkedBlockingQueue; + +/** + * This is a non-blocking queue with no limit on capacity. + */ +public class AuditAsyncQueue extends AuditQueue implements Runnable { + private static final Log logger = LogFactory.getLog(AuditAsyncQueue.class); + + LinkedBlockingQueue queue = new LinkedBlockingQueue(); + Thread consumerThread = null; + + static final int MAX_DRAIN = 1000; + static int threadCount = 0; + static final String DEFAULT_NAME = "async"; + + public AuditAsyncQueue(AuditHandler consumer) { + super(consumer); + setName(DEFAULT_NAME); + } + + /* + * (non-Javadoc) + * + * @see + * org.apache.ranger.audit.provider.AuditProvider#log(org.apache.ranger. + * audit.model.AuditEventBase) + */ + @Override + public boolean log(AuditEventBase event) { + // Add to the queue and return ASAP + if (queue.size() >= getMaxQueueSize()) { + return false; + } + queue.add(event); + return true; + } + + @Override + public boolean log(Collection events) { + boolean ret = true; + for (AuditEventBase event : events) { + ret = log(event); + if (!ret) { + break; + } + } + return ret; + } + + /* + * (non-Javadoc) + * + * @see org.apache.ranger.audit.provider.AuditProvider#start() + */ + @Override + public void start() { + if (consumer != null) { + consumer.start(); + } else { + logger.error("consumer is not set. Nothing will be sent to any consumer. name=" + + getName()); + } + + consumerThread = new Thread(this, this.getClass().getName() + + (threadCount++)); + consumerThread.setDaemon(true); + consumerThread.start(); + } + + /* + * (non-Javadoc) + * + * @see org.apache.ranger.audit.provider.AuditProvider#stop() + */ + @Override + public void stop() { + logger.info("Stop called. name=" + getName()); + setDrain(true); + try { + if (consumerThread != null) { + logger.info("Interrupting consumerThread. name=" + getName() + + ", consumer=" + + (consumer == null ? null : consumer.getName())); + consumerThread.interrupt(); + } + } catch (Throwable t) { + // ignore any exception + } + consumerThread = null; + } + + /* + * (non-Javadoc) + * + * @see java.lang.Runnable#run() + */ + @Override + public void run() { + try { + //This is done to clear the MDC context to avoid issue with Ranger Auditing for Knox + //MDC.clear(); + runLogAudit(); + } catch (Throwable t) { + logger.fatal("Exited thread abnormaly. queue=" + getName(), t); + } + } + + public void runLogAudit() { + while (true) { + try { + AuditEventBase event = null; + if (!isDrain()) { + // For Transfer queue take() is blocking + event = queue.take(); + } else { + // For Transfer queue poll() is non blocking + event = queue.poll(); + } + if (event != null) { + Collection eventList = new ArrayList(); + eventList.add(event); + queue.drainTo(eventList, MAX_DRAIN - 1); + consumer.log(eventList); + } + } catch (InterruptedException e) { + logger.info("Caught exception in consumer thread. Shutdown might be in progress"); + } catch (Throwable t) { + logger.error("Caught error during processing request.", t); + } + if (isDrain()) { + if (queue.isEmpty()) { + break; + } + if (isDrainMaxTimeElapsed()) { + logger.warn("Exiting polling loop because max time allowed reached. name=" + + getName() + + ", waited for " + + (stopTime - System.currentTimeMillis()) + " ms"); + } + } + } + logger.info("Exiting polling loop. name=" + getName()); + + try { + // Call stop on the consumer + logger.info("Calling to stop consumer. name=" + getName() + + ", consumer.name=" + consumer.getName()); + + // Call stop on the consumer + consumer.stop(); + } catch (Throwable t) { + logger.error("Error while calling stop on consumer.", t); + } + logger.info("Exiting consumerThread.run() method. name=" + getName()); + } + +} diff --git a/auth-audits/src/main/java/org/apache/atlas/audit/queue/AuditBatchQueue.java b/auth-audits/src/main/java/org/apache/atlas/audit/queue/AuditBatchQueue.java new file mode 100644 index 00000000000..b990188675f --- /dev/null +++ b/auth-audits/src/main/java/org/apache/atlas/audit/queue/AuditBatchQueue.java @@ -0,0 +1,364 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.audit.queue; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +//import org.apache.log4j.MDC; +import org.apache.atlas.audit.model.AuditEventBase; +import org.apache.atlas.audit.provider.AuditHandler; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Properties; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.TimeUnit; + +public class AuditBatchQueue extends AuditQueue implements Runnable { + private static final Log logger = LogFactory.getLog(AuditBatchQueue.class); + + private BlockingQueue queue = null; + private Collection localBatchBuffer = new ArrayList(); + + Thread consumerThread = null; + static int threadCount = 0; + static final String DEFAULT_NAME = "batch"; + + public AuditBatchQueue(AuditHandler consumer) { + super(consumer); + setName(DEFAULT_NAME); + } + + /* + * (non-Javadoc) + * + * @see + * org.apache.ranger.audit.provider.AuditProvider#log(org.apache.ranger. + * audit.model.AuditEventBase) + */ + @Override + public boolean log(AuditEventBase event) { + // Add to batchQueue. Block if full + queue.add(event); + return true; + } + + @Override + public boolean log(Collection events) { + boolean ret = true; + for (AuditEventBase event : events) { + ret = log(event); + if (!ret) { + break; + } + } + return ret; + } + + @Override + public void init(Properties prop, String basePropertyName) { + String propPrefix = "xasecure.audit.batch"; + if (basePropertyName != null) { + propPrefix = basePropertyName; + } + + super.init(prop, propPrefix); + + } + + /* + * (non-Javadoc) + * + * @see org.apache.ranger.audit.provider.AuditProvider#start() + */ + @Override + synchronized public void start() { + if (consumerThread != null) { + logger.error("Provider is already started. name=" + getName()); + return; + } + logger.info("Creating ArrayBlockingQueue with maxSize=" + + getMaxQueueSize()); + queue = new ArrayBlockingQueue(getMaxQueueSize()); + + // Start the consumer first + consumer.start(); + + // Then the FileSpooler + if (fileSpoolerEnabled) { + fileSpooler.start(); + } + + // Finally the queue listener + consumerThread = new Thread(this, this.getClass().getName() + + (threadCount++)); + consumerThread.setDaemon(true); + consumerThread.start(); + + } + + /* + * (non-Javadoc) + * + * @see org.apache.ranger.audit.provider.AuditProvider#stop() + */ + @Override + public void stop() { + logger.info("Stop called. name=" + getName()); + setDrain(true); + flush(); + try { + if (consumerThread != null) { + logger.info("Interrupting consumerThread. name=" + getName() + + ", consumer=" + + (consumer == null ? null : consumer.getName())); + + consumerThread.interrupt(); + } + } catch (Throwable t) { + // ignore any exception + } + consumerThread = null; + } + + /* + * (non-Javadoc) + * + * @see org.apache.ranger.audit.provider.AuditProvider#waitToComplete() + */ + @Override + public void waitToComplete() { + int defaultTimeOut = -1; + waitToComplete(defaultTimeOut); + consumer.waitToComplete(defaultTimeOut); + } + + @Override + public void waitToComplete(long timeout) { + setDrain(true); + flush(); + long sleepTime = 1000; + long startTime = System.currentTimeMillis(); + int prevQueueSize = -1; + int staticLoopCount = 0; + while ((queue.size() > 0 || localBatchBuffer.size() > 0)) { + if (prevQueueSize == queue.size()) { + logger.error("Queue size is not changing. " + getName() + + ".size=" + queue.size()); + staticLoopCount++; + if (staticLoopCount > 5) { + logger.error("Aborting writing to consumer. Some logs will be discarded." + + getName() + ".size=" + queue.size()); + break; + } + } else { + staticLoopCount = 0; + prevQueueSize = queue.size(); + } + if (consumerThread != null) { + consumerThread.interrupt(); + } + try { + Thread.sleep(sleepTime); + if (timeout > 0 + && (System.currentTimeMillis() - startTime > timeout)) { + break; + } + } catch (InterruptedException e) { + break; + } + } + consumer.waitToComplete(timeout); + } + + /* + * (non-Javadoc) + * + * @see org.apache.ranger.audit.provider.AuditProvider#flush() + */ + @Override + public void flush() { + if (fileSpoolerEnabled) { + fileSpooler.flush(); + } + consumer.flush(); + } + + /* + * (non-Javadoc) + * + * @see java.lang.Runnable#run() + */ + @Override + public void run() { + try { + //This is done to clear the MDC context to avoid issue with Ranger Auditing for Knox + //MDC.clear(); + runLogAudit(); + } catch (Throwable t) { + logger.fatal("Exited thread abnormaly. queue=" + getName(), t); + } + } + + public void runLogAudit() { + long lastDispatchTime = System.currentTimeMillis(); + boolean isDestActive = true; + while (true) { + logStatusIfRequired(); + + // Time to next dispatch + long nextDispatchDuration = lastDispatchTime + - System.currentTimeMillis() + getMaxBatchInterval(); + + boolean isToSpool = false; + boolean fileSpoolDrain = false; + try { + if (fileSpoolerEnabled && fileSpooler.isPending()) { + int percentUsed = queue.size() * 100 + / getMaxQueueSize(); + long lastAttemptDelta = fileSpooler + .getLastAttemptTimeDelta(); + + fileSpoolDrain = lastAttemptDelta > fileSpoolMaxWaitTime; + // If we should even read from queue? + if (!isDrain() && !fileSpoolDrain + && percentUsed < fileSpoolDrainThresholdPercent) { + // Since some files are still under progress and it is + // not in drain mode, lets wait and retry + if (nextDispatchDuration > 0) { + Thread.sleep(nextDispatchDuration); + } + lastDispatchTime = System.currentTimeMillis(); + continue; + } + isToSpool = true; + } + + AuditEventBase event = null; + + if (!isToSpool && !isDrain() && !fileSpoolDrain + && nextDispatchDuration > 0) { + event = queue.poll(nextDispatchDuration, + TimeUnit.MILLISECONDS); + } else { + // For poll() is non blocking + event = queue.poll(); + } + + if (event != null) { + localBatchBuffer.add(event); + if (getMaxBatchSize() >= localBatchBuffer.size()) { + queue.drainTo(localBatchBuffer, getMaxBatchSize() + - localBatchBuffer.size()); + } + } else { + // poll returned due to timeout, so reseting clock + nextDispatchDuration = lastDispatchTime + - System.currentTimeMillis() + + getMaxBatchInterval(); + + lastDispatchTime = System.currentTimeMillis(); + } + } catch (InterruptedException e) { + logger.info("Caught exception in consumer thread. Shutdown might be in progress"); + setDrain(true); + } catch (Throwable t) { + logger.error("Caught error during processing request.", t); + } + + addTotalCount(localBatchBuffer.size()); + if (localBatchBuffer.size() > 0 && isToSpool) { + // Let spool to the file directly + if (isDestActive) { + logger.info("Switching to file spool. Queue=" + getName() + + ", dest=" + consumer.getName()); + } + isDestActive = false; + // Just before stashing + lastDispatchTime = System.currentTimeMillis(); + fileSpooler.stashLogs(localBatchBuffer); + addStashedCount(localBatchBuffer.size()); + localBatchBuffer.clear(); + } else if (localBatchBuffer.size() > 0 + && (isDrain() + || localBatchBuffer.size() >= getMaxBatchSize() || nextDispatchDuration <= 0)) { + if (fileSpoolerEnabled && !isDestActive) { + logger.info("Switching to writing to destination. Queue=" + + getName() + ", dest=" + consumer.getName()); + } + // Reset time just before sending the logs + lastDispatchTime = System.currentTimeMillis(); + boolean ret = consumer.log(localBatchBuffer); + if (!ret) { + if (fileSpoolerEnabled) { + logger.info("Switching to file spool. Queue=" + + getName() + ", dest=" + consumer.getName()); + // Transient error. Stash and move on + fileSpooler.stashLogs(localBatchBuffer); + isDestActive = false; + addStashedCount(localBatchBuffer.size()); + } else { + // We need to drop this event + addFailedCount(localBatchBuffer.size()); + logFailedEvent(localBatchBuffer); + } + } else { + isDestActive = true; + addSuccessCount(localBatchBuffer.size()); + } + localBatchBuffer.clear(); + } + + if (isDrain()) { + if (!queue.isEmpty() || localBatchBuffer.size() > 0) { + logger.info("Queue is not empty. Will retry. queue.size)=" + + queue.size() + ", localBatchBuffer.size()=" + + localBatchBuffer.size()); + } else { + break; + } + if (isDrainMaxTimeElapsed()) { + logger.warn("Exiting polling loop because max time allowed reached. name=" + + getName() + + ", waited for " + + (stopTime - System.currentTimeMillis()) + " ms"); + } + } + } + + logger.info("Exiting consumerThread. Queue=" + getName() + ", dest=" + + consumer.getName()); + try { + // Call stop on the consumer + logger.info("Calling to stop consumer. name=" + getName() + + ", consumer.name=" + consumer.getName()); + + consumer.stop(); + if (fileSpoolerEnabled) { + fileSpooler.stop(); + } + } catch (Throwable t) { + logger.error("Error while calling stop on consumer.", t); + } + logStatus(); + logger.info("Exiting consumerThread.run() method. name=" + getName()); + } +} diff --git a/auth-audits/src/main/java/org/apache/atlas/audit/queue/AuditFileCacheProviderSpool.java b/auth-audits/src/main/java/org/apache/atlas/audit/queue/AuditFileCacheProviderSpool.java new file mode 100644 index 00000000000..aa869eb84cc --- /dev/null +++ b/auth-audits/src/main/java/org/apache/atlas/audit/queue/AuditFileCacheProviderSpool.java @@ -0,0 +1,940 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.audit.queue; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +//import org.apache.log4j.MDC; +import org.apache.atlas.audit.model.AuditEventBase; +import org.apache.atlas.audit.model.AuthzAuditEvent; +import org.apache.atlas.audit.provider.AuditHandler; +import org.apache.atlas.audit.provider.MiscUtil; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileFilter; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.FileReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStreamWriter; +import java.io.PrintWriter; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Date; +import java.util.Iterator; +import java.util.List; +import java.util.Properties; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; + +/** + * This class temporarily stores logs in Local file system before it despatches each logs in file to the AuditBatchQueue Consumer. + * This gets instantiated only when AuditFileCacheProvider is enabled (xasecure.audit.provider.filecache.is.enabled). + * When AuditFileCacheProvider is all the logs are stored in local file system before sent to destination. + */ + +public class AuditFileCacheProviderSpool implements Runnable { + private static final Log logger = LogFactory.getLog(AuditFileCacheProviderSpool.class); + + public enum SPOOL_FILE_STATUS { + pending, write_inprogress, read_inprogress, done + } + + public static final String PROP_FILE_SPOOL_LOCAL_DIR = "filespool.dir"; + public static final String PROP_FILE_SPOOL_LOCAL_FILE_NAME = "filespool.filename.format"; + public static final String PROP_FILE_SPOOL_ARCHIVE_DIR = "filespool.archive.dir"; + public static final String PROP_FILE_SPOOL_ARCHIVE_MAX_FILES_COUNT = "filespool.archive.max.files"; + public static final String PROP_FILE_SPOOL_FILENAME_PREFIX = "filespool.file.prefix"; + public static final String PROP_FILE_SPOOL_FILE_ROLLOVER = "filespool.file.rollover.sec"; + public static final String PROP_FILE_SPOOL_INDEX_FILE = "filespool.index.filename"; + public static final String PROP_FILE_SPOOL_DEST_RETRY_MS = "filespool.destination.retry.ms"; + public static final String PROP_FILE_SPOOL_BATCH_SIZE = "filespool.buffer.size"; + + public static final String AUDIT_IS_FILE_CACHE_PROVIDER_ENABLE_PROP = "xasecure.audit.provider.filecache.is.enabled"; + public static final String FILE_CACHE_PROVIDER_NAME = "AuditFileCacheProviderSpool"; + + AuditHandler consumerProvider = null; + + BlockingQueue indexQueue = new LinkedBlockingQueue(); + List indexRecords = new ArrayList(); + + // Folder and File attributes + File logFolder = null; + String logFileNameFormat = null; + File archiveFolder = null; + String fileNamePrefix = null; + String indexFileName = null; + File indexFile = null; + String indexDoneFileName = null; + File indexDoneFile = null; + int retryDestinationMS = 30 * 1000; // Default 30 seconds + int fileRolloverSec = 24 * 60 * 60; // In seconds + int maxArchiveFiles = 100; + int errorLogIntervalMS = 30 * 1000; // Every 30 seconds + int auditBatchSize = 1000; + long lastErrorLogMS = 0; + boolean isAuditFileCacheProviderEnabled = false; + boolean closeFile = false; + boolean isPending = false; + long lastAttemptTime = 0; + boolean initDone = false; + + PrintWriter logWriter = null; + AuditIndexRecord currentWriterIndexRecord = null; + AuditIndexRecord currentConsumerIndexRecord = null; + + BufferedReader logReader = null; + Thread destinationThread = null; + + boolean isWriting = true; + boolean isDrain = false; + boolean isDestDown = false; + boolean isSpoolingSuccessful = true; + + private Gson gson = null; + + public AuditFileCacheProviderSpool(AuditHandler consumerProvider) { + this.consumerProvider = consumerProvider; + } + + public void init(Properties prop) { + init(prop, null); + } + + public boolean init(Properties props, String basePropertyName) { + logger.debug("==> AuditFileCacheProviderSpool.init()"); + + if (initDone) { + logger.error("init() called more than once. queueProvider=" + + "" + ", consumerProvider=" + + consumerProvider.getName()); + return true; + } + String propPrefix = "xasecure.audit.filespool"; + if (basePropertyName != null) { + propPrefix = basePropertyName; + } + + try { + gson = new GsonBuilder().setDateFormat("yyyy-MM-dd HH:mm:ss.SSS") + .create(); + // Initial folder and file properties + String logFolderProp = MiscUtil.getStringProperty(props, propPrefix + + "." + PROP_FILE_SPOOL_LOCAL_DIR); + logFileNameFormat = MiscUtil.getStringProperty(props, + basePropertyName + "." + PROP_FILE_SPOOL_LOCAL_FILE_NAME); + String archiveFolderProp = MiscUtil.getStringProperty(props, + propPrefix + "." + PROP_FILE_SPOOL_ARCHIVE_DIR); + fileNamePrefix = MiscUtil.getStringProperty(props, propPrefix + "." + + PROP_FILE_SPOOL_FILENAME_PREFIX); + indexFileName = MiscUtil.getStringProperty(props, propPrefix + "." + + PROP_FILE_SPOOL_INDEX_FILE); + retryDestinationMS = MiscUtil.getIntProperty(props, propPrefix + + "." + PROP_FILE_SPOOL_DEST_RETRY_MS, retryDestinationMS); + fileRolloverSec = MiscUtil.getIntProperty(props, propPrefix + "." + + PROP_FILE_SPOOL_FILE_ROLLOVER, fileRolloverSec); + maxArchiveFiles = MiscUtil.getIntProperty(props, propPrefix + "." + + PROP_FILE_SPOOL_ARCHIVE_MAX_FILES_COUNT, maxArchiveFiles); + isAuditFileCacheProviderEnabled = MiscUtil.getBooleanProperty(props, AUDIT_IS_FILE_CACHE_PROVIDER_ENABLE_PROP, false); + logger.info("retryDestinationMS=" + retryDestinationMS + + ", queueName=" + FILE_CACHE_PROVIDER_NAME); + logger.info("fileRolloverSec=" + fileRolloverSec + ", queueName=" + + FILE_CACHE_PROVIDER_NAME); + logger.info("maxArchiveFiles=" + maxArchiveFiles + ", queueName=" + + FILE_CACHE_PROVIDER_NAME); + + if (logFolderProp == null || logFolderProp.isEmpty()) { + logger.fatal("Audit spool folder is not configured. Please set " + + propPrefix + + "." + + PROP_FILE_SPOOL_LOCAL_DIR + + ". queueName=" + FILE_CACHE_PROVIDER_NAME); + return false; + } + logFolder = new File(logFolderProp); + if (!logFolder.isDirectory()) { + boolean result = logFolder.mkdirs(); + if (!logFolder.isDirectory() || !result) { + logger.fatal("File Spool folder not found and can't be created. folder=" + + logFolder.getAbsolutePath() + + ", queueName=" + + FILE_CACHE_PROVIDER_NAME); + return false; + } + } + logger.info("logFolder=" + logFolder + ", queueName=" + + FILE_CACHE_PROVIDER_NAME); + + if (logFileNameFormat == null || logFileNameFormat.isEmpty()) { + logFileNameFormat = "spool_" + "%app-type%" + "_" + + "%time:yyyyMMdd-HHmm.ss%.log"; + } + logger.info("logFileNameFormat=" + logFileNameFormat + + ", queueName=" + FILE_CACHE_PROVIDER_NAME); + + if (archiveFolderProp == null || archiveFolderProp.isEmpty()) { + archiveFolder = new File(logFolder, "archive"); + } else { + archiveFolder = new File(archiveFolderProp); + } + if (!archiveFolder.isDirectory()) { + boolean result = archiveFolder.mkdirs(); + if (!archiveFolder.isDirectory() || !result) { + logger.error("File Spool archive folder not found and can't be created. folder=" + + archiveFolder.getAbsolutePath() + + ", queueName=" + + FILE_CACHE_PROVIDER_NAME); + return false; + } + } + logger.info("archiveFolder=" + archiveFolder + ", queueName=" + + FILE_CACHE_PROVIDER_NAME); + + if (indexFileName == null || indexFileName.isEmpty()) { + if (fileNamePrefix == null || fileNamePrefix.isEmpty()) { + fileNamePrefix = FILE_CACHE_PROVIDER_NAME + "_" + + consumerProvider.getName(); + } + indexFileName = "index_" + fileNamePrefix + "_" + "%app-type%" + + ".json"; + indexFileName = MiscUtil.replaceTokens(indexFileName, + System.currentTimeMillis()); + } + + indexFile = new File(logFolder, indexFileName); + if (!indexFile.exists()) { + boolean ret = indexFile.createNewFile(); + if (!ret) { + logger.fatal("Error creating index file. fileName=" + + indexFile.getPath()); + return false; + } + } + logger.info("indexFile=" + indexFile + ", queueName=" + + FILE_CACHE_PROVIDER_NAME); + + int lastDot = indexFileName.lastIndexOf('.'); + if (lastDot < 0) { + lastDot = indexFileName.length() - 1; + } + indexDoneFileName = indexFileName.substring(0, lastDot) + + "_closed.json"; + indexDoneFile = new File(logFolder, indexDoneFileName); + if (!indexDoneFile.exists()) { + boolean ret = indexDoneFile.createNewFile(); + if (!ret) { + logger.fatal("Error creating index done file. fileName=" + + indexDoneFile.getPath()); + return false; + } + } + logger.info("indexDoneFile=" + indexDoneFile + ", queueName=" + + FILE_CACHE_PROVIDER_NAME); + + // Load index file + loadIndexFile(); + for (AuditIndexRecord auditIndexRecord : indexRecords) { + if (!auditIndexRecord.status.equals(SPOOL_FILE_STATUS.done)) { + isPending = true; + } + if (auditIndexRecord.status + .equals(SPOOL_FILE_STATUS.write_inprogress)) { + currentWriterIndexRecord = auditIndexRecord; + logger.info("currentWriterIndexRecord=" + + currentWriterIndexRecord.filePath + + ", queueName=" + FILE_CACHE_PROVIDER_NAME); + } + if (auditIndexRecord.status + .equals(SPOOL_FILE_STATUS.read_inprogress)) { + indexQueue.add(auditIndexRecord); + } + } + printIndex(); + for (int i = 0; i < indexRecords.size(); i++) { + AuditIndexRecord auditIndexRecord = indexRecords.get(i); + if (auditIndexRecord.status.equals(SPOOL_FILE_STATUS.pending)) { + File consumerFile = new File(auditIndexRecord.filePath); + if (!consumerFile.exists()) { + logger.error("INIT: Consumer file=" + + consumerFile.getPath() + " not found."); + } else { + indexQueue.add(auditIndexRecord); + } + } + } + + } catch (Throwable t) { + logger.fatal("Error initializing File Spooler. queue=" + + FILE_CACHE_PROVIDER_NAME, t); + return false; + } + + auditBatchSize = MiscUtil.getIntProperty(props, propPrefix + + "." + PROP_FILE_SPOOL_BATCH_SIZE, auditBatchSize); + + initDone = true; + + logger.debug("<== AuditFileCacheProviderSpool.init()"); + return true; + } + + /** + * Start looking for outstanding logs and update status according. + */ + public void start() { + if (!initDone) { + logger.error("Cannot start Audit File Spooler. Initilization not done yet. queueName=" + + FILE_CACHE_PROVIDER_NAME); + return; + } + + logger.info("Starting writerThread, queueName=" + + FILE_CACHE_PROVIDER_NAME + ", consumer=" + + consumerProvider.getName()); + + // Let's start the thread to read + destinationThread = new Thread(this, FILE_CACHE_PROVIDER_NAME + "_" + + consumerProvider.getName() + "_destWriter"); + destinationThread.setDaemon(true); + destinationThread.start(); + } + + public void stop() { + if (!initDone) { + logger.error("Cannot stop Audit File Spooler. Initilization not done. queueName=" + + FILE_CACHE_PROVIDER_NAME); + return; + } + logger.info("Stop called, queueName=" + FILE_CACHE_PROVIDER_NAME + + ", consumer=" + consumerProvider.getName()); + + isDrain = true; + flush(); + + PrintWriter out = getOpenLogFileStream(); + if (out != null) { + // If write is still going on, then let's give it enough time to + // complete + for (int i = 0; i < 3; i++) { + if (isWriting) { + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + // ignore + } + continue; + } + try { + logger.info("Closing open file, queueName=" + + FILE_CACHE_PROVIDER_NAME + ", consumer=" + + consumerProvider.getName()); + + out.flush(); + out.close(); + break; + } catch (Throwable t) { + logger.debug("Error closing spool out file.", t); + } + } + } + try { + if (destinationThread != null) { + destinationThread.interrupt(); + } + destinationThread = null; + } catch (Throwable e) { + // ignore + } + } + + public void flush() { + if (!initDone) { + logger.error("Cannot flush Audit File Spooler. Initilization not done. queueName=" + + FILE_CACHE_PROVIDER_NAME); + return; + } + PrintWriter out = getOpenLogFileStream(); + if (out != null) { + out.flush(); + } + } + + /** + * If any files are still not processed. Also, if the destination is not + * reachable + * + * @return + */ + public boolean isPending() { + if (!initDone) { + logError("isPending(): File Spooler not initialized. queueName=" + + FILE_CACHE_PROVIDER_NAME); + return false; + } + + return isPending; + } + + /** + * Milliseconds from last attempt time + * + * @return + */ + public long getLastAttemptTimeDelta() { + if (lastAttemptTime == 0) { + return 0; + } + return System.currentTimeMillis() - lastAttemptTime; + } + + synchronized public void stashLogs(AuditEventBase event) { + + if (isDrain) { + // Stop has been called, so this method shouldn't be called + logger.error("stashLogs() is called after stop is called. event=" + + event); + return; + } + try { + isWriting = true; + PrintWriter logOut = getLogFileStream(); + // Convert event to json + String jsonStr = MiscUtil.stringify(event); + logOut.println(jsonStr); + logOut.flush(); + isPending = true; + isSpoolingSuccessful = true; + } catch (Throwable t) { + isSpoolingSuccessful = false; + logger.error("Error writing to file. event=" + event, t); + } finally { + isWriting = false; + } + + } + + synchronized public void stashLogs(Collection events) { + for (AuditEventBase event : events) { + stashLogs(event); + } + flush(); + } + + synchronized public void stashLogsString(String event) { + if (isDrain) { + // Stop has been called, so this method shouldn't be called + logger.error("stashLogs() is called after stop is called. event=" + + event); + return; + } + try { + isWriting = true; + PrintWriter logOut = getLogFileStream(); + logOut.println(event); + } catch (Exception ex) { + logger.error("Error writing to file. event=" + event, ex); + } finally { + isWriting = false; + } + + } + + synchronized public boolean isSpoolingSuccessful() { + return isSpoolingSuccessful; + } + + synchronized public void stashLogsString(Collection events) { + for (String event : events) { + stashLogsString(event); + } + flush(); + } + + /** + * This return the current file. If there are not current open output file, + * then it will return null + * + * @return + * @throws Exception + */ + synchronized private PrintWriter getOpenLogFileStream() { + return logWriter; + } + + /** + * @return + * @throws Exception + */ + synchronized private PrintWriter getLogFileStream() throws Exception { + closeFileIfNeeded(); + // Either there are no open log file or the previous one has been rolled + // over + if (currentWriterIndexRecord == null) { + Date currentTime = new Date(); + // Create a new file + String fileName = MiscUtil.replaceTokens(logFileNameFormat, + currentTime.getTime()); + String newFileName = fileName; + File outLogFile = null; + int i = 0; + while (true) { + outLogFile = new File(logFolder, newFileName); + File archiveLogFile = new File(archiveFolder, newFileName); + if (!outLogFile.exists() && !archiveLogFile.exists()) { + break; + } + i++; + int lastDot = fileName.lastIndexOf('.'); + String baseName = fileName.substring(0, lastDot); + String extension = fileName.substring(lastDot); + newFileName = baseName + "." + i + extension; + } + fileName = newFileName; + logger.info("Creating new file. queueName=" + + FILE_CACHE_PROVIDER_NAME + ", fileName=" + fileName); + // Open the file + logWriter = new PrintWriter(new BufferedWriter(new OutputStreamWriter(new FileOutputStream( + outLogFile),"UTF-8"))); + + AuditIndexRecord tmpIndexRecord = new AuditIndexRecord(); + + tmpIndexRecord.id = MiscUtil.generateUniqueId(); + tmpIndexRecord.filePath = outLogFile.getPath(); + tmpIndexRecord.status = SPOOL_FILE_STATUS.write_inprogress; + tmpIndexRecord.fileCreateTime = currentTime; + tmpIndexRecord.lastAttempt = true; + currentWriterIndexRecord = tmpIndexRecord; + indexRecords.add(currentWriterIndexRecord); + saveIndexFile(); + + } else { + if (logWriter == null) { + // This means the process just started. We need to open the file + // in append mode. + logger.info("Opening existing file for append. queueName=" + + FILE_CACHE_PROVIDER_NAME + ", fileName=" + + currentWriterIndexRecord.filePath); + logWriter = new PrintWriter(new BufferedWriter(new OutputStreamWriter(new FileOutputStream( + currentWriterIndexRecord.filePath, true),"UTF-8"))); + } + } + return logWriter; + } + + synchronized private void closeFileIfNeeded() throws FileNotFoundException, + IOException { + // Is there file open to write or there are no pending file, then close + // the active file + if (currentWriterIndexRecord != null) { + // Check whether the file needs to rolled + rollOverSpoolFileByTime(); + + if (closeFile) { + // Roll the file + if (logWriter != null) { + logWriter.flush(); + logWriter.close(); + logWriter = null; + closeFile = false; + } + currentWriterIndexRecord.status = SPOOL_FILE_STATUS.pending; + currentWriterIndexRecord.writeCompleteTime = new Date(); + saveIndexFile(); + logger.info("Adding file to queue. queueName=" + + FILE_CACHE_PROVIDER_NAME + ", fileName=" + + currentWriterIndexRecord.filePath); + indexQueue.add(currentWriterIndexRecord); + currentWriterIndexRecord = null; + } + } + } + + private void rollOverSpoolFileByTime() { + if (System.currentTimeMillis() + - currentWriterIndexRecord.fileCreateTime.getTime() > fileRolloverSec * 1000) { + closeFile = true; + logger.info("Closing file. Rolling over. queueName=" + + FILE_CACHE_PROVIDER_NAME + ", fileName=" + + currentWriterIndexRecord.filePath); + } + } + + /** + * Load the index file + * + * @throws IOException + */ + void loadIndexFile() throws IOException { + logger.info("Loading index file. fileName=" + indexFile.getPath()); + BufferedReader br = null; + try { + br = new BufferedReader(new InputStreamReader(new FileInputStream(indexFile), "UTF-8")); + indexRecords.clear(); + String line; + while ((line = br.readLine()) != null) { + if (!line.isEmpty() && !line.startsWith("#")) { + AuditIndexRecord record = gson.fromJson(line, + AuditIndexRecord.class); + indexRecords.add(record); + } + } + } finally { + if (br!= null) { + br.close(); + } + } + } + + synchronized void printIndex() { + logger.info("INDEX printIndex() ==== START"); + Iterator iter = indexRecords.iterator(); + while (iter.hasNext()) { + AuditIndexRecord record = iter.next(); + logger.info("INDEX=" + record + ", isFileExist=" + + (new File(record.filePath).exists())); + } + logger.info("INDEX printIndex() ==== END"); + } + + synchronized void removeIndexRecord(AuditIndexRecord indexRecord) + throws FileNotFoundException, IOException { + Iterator iter = indexRecords.iterator(); + while (iter.hasNext()) { + AuditIndexRecord record = iter.next(); + if (record.id.equals(indexRecord.id)) { + logger.info("Removing file from index. file=" + record.filePath + + ", queueName=" + FILE_CACHE_PROVIDER_NAME + + ", consumer=" + consumerProvider.getName()); + + iter.remove(); + appendToDoneFile(record); + } + } + saveIndexFile(); + // If there are no more files in the index, then let's assume the + // destination is now available + if (indexRecords.size() == 0) { + isPending = false; + } + } + + synchronized void saveIndexFile() throws FileNotFoundException, IOException { + PrintWriter out = new PrintWriter(indexFile,"UTF-8"); + for (AuditIndexRecord auditIndexRecord : indexRecords) { + out.println(gson.toJson(auditIndexRecord)); + } + out.close(); + // printIndex(); + + } + + void appendToDoneFile(AuditIndexRecord indexRecord) + throws FileNotFoundException, IOException { + logger.info("Moving to done file. " + indexRecord.filePath + + ", queueName=" + FILE_CACHE_PROVIDER_NAME + ", consumer=" + + consumerProvider.getName()); + String line = gson.toJson(indexRecord); + PrintWriter out = new PrintWriter(new BufferedWriter(new OutputStreamWriter(new FileOutputStream( + indexDoneFile, true),"UTF-8"))); + out.println(line); + out.flush(); + out.close(); + + // After Each file is read and audit events are pushed into pipe, we flush to reach the destination immediate. + consumerProvider.flush(); + + // Move to archive folder + File logFile = null; + File archiveFile = null; + try { + logFile = new File(indexRecord.filePath); + String fileName = logFile.getName(); + archiveFile = new File(archiveFolder, fileName); + logger.info("Moving logFile " + logFile + " to " + archiveFile); + boolean result = logFile.renameTo(archiveFile); + if (!result) { + logger.error("Error moving log file to archive folder. Unable to rename" + + logFile + " to archiveFile=" + archiveFile); + } + } catch (Throwable t) { + logger.error("Error moving log file to archive folder. logFile=" + + logFile + ", archiveFile=" + archiveFile, t); + } + + // After archiving the file flush the pipe + consumerProvider.flush(); + + archiveFile = null; + try { + // Remove old files + File[] logFiles = archiveFolder.listFiles(new FileFilter() { + public boolean accept(File pathname) { + return pathname.getName().toLowerCase().endsWith(".log"); + } + }); + + if (logFiles != null && logFiles.length > maxArchiveFiles) { + int filesToDelete = logFiles.length - maxArchiveFiles; + BufferedReader br = new BufferedReader(new FileReader( + indexDoneFile)); + try { + int filesDeletedCount = 0; + while ((line = br.readLine()) != null) { + if (!line.isEmpty() && !line.startsWith("#")) { + AuditIndexRecord record = gson.fromJson(line, + AuditIndexRecord.class); + logFile = new File(record.filePath); + String fileName = logFile.getName(); + archiveFile = new File(archiveFolder, fileName); + if (archiveFile.exists()) { + logger.info("Deleting archive file " + + archiveFile); + boolean ret = archiveFile.delete(); + if (!ret) { + logger.error("Error deleting archive file. archiveFile=" + + archiveFile); + } + filesDeletedCount++; + if (filesDeletedCount >= filesToDelete) { + logger.info("Deleted " + filesDeletedCount + + " files"); + break; + } + } + } + } + } finally { + br.close(); + } + } + } catch (Throwable t) { + logger.error("Error deleting older archive file. archiveFile=" + + archiveFile, t); + } + + } + + void logError(String msg) { + long currTimeMS = System.currentTimeMillis(); + if (currTimeMS - lastErrorLogMS > errorLogIntervalMS) { + logger.error(msg); + lastErrorLogMS = currTimeMS; + } + } + + class AuditIndexRecord { + String id; + String filePath; + int linePosition = 0; + SPOOL_FILE_STATUS status = SPOOL_FILE_STATUS.write_inprogress; + Date fileCreateTime; + Date writeCompleteTime; + Date doneCompleteTime; + Date lastSuccessTime; + Date lastFailedTime; + int failedAttemptCount = 0; + boolean lastAttempt = false; + + @Override + public String toString() { + return "AuditIndexRecord [id=" + id + ", filePath=" + filePath + + ", linePosition=" + linePosition + ", status=" + status + + ", fileCreateTime=" + fileCreateTime + + ", writeCompleteTime=" + writeCompleteTime + + ", doneCompleteTime=" + doneCompleteTime + + ", lastSuccessTime=" + lastSuccessTime + + ", lastFailedTime=" + lastFailedTime + + ", failedAttemptCount=" + failedAttemptCount + + ", lastAttempt=" + lastAttempt + "]"; + } + + } + + /* + * (non-Javadoc) + * + * @see java.lang.Runnable#run() + */ + @Override + public void run() { + try { + //This is done to clear the MDC context to avoid issue with Ranger Auditing for Knox + //MDC.clear(); + runLogAudit(); + } catch (Throwable t) { + logger.fatal("Exited thread without abnormaly. queue=" + + consumerProvider.getName(), t); + } + } + + public void runLogAudit() { + // boolean isResumed = false; + while (true) { + try { + if (isDestDown) { + logger.info("Destination is down. sleeping for " + + retryDestinationMS + + " milli seconds. indexQueue=" + indexQueue.size() + + ", queueName=" + FILE_CACHE_PROVIDER_NAME + + ", consumer=" + consumerProvider.getName()); + Thread.sleep(retryDestinationMS); + } + // Let's pause between each iteration + if (currentConsumerIndexRecord == null) { + currentConsumerIndexRecord = indexQueue.poll( + retryDestinationMS, TimeUnit.MILLISECONDS); + } else { + Thread.sleep(retryDestinationMS); + } + + if (isDrain) { + // Need to exit + break; + } + if (currentConsumerIndexRecord == null) { + closeFileIfNeeded(); + continue; + } + + boolean isRemoveIndex = false; + File consumerFile = new File( + currentConsumerIndexRecord.filePath); + if (!consumerFile.exists()) { + logger.error("Consumer file=" + consumerFile.getPath() + + " not found."); + printIndex(); + isRemoveIndex = true; + } else { + // Let's open the file to write + BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream( + currentConsumerIndexRecord.filePath),"UTF-8")); + try { + int startLine = currentConsumerIndexRecord.linePosition; + String line; + int currLine = 0; + List events = new ArrayList<>(); + while ((line = br.readLine()) != null) { + currLine++; + if (currLine < startLine) { + continue; + } + AuditEventBase event = MiscUtil.fromJson(line, AuthzAuditEvent.class); + events.add(event); + + if (events.size() == auditBatchSize) { + boolean ret = sendEvent(events, + currentConsumerIndexRecord, currLine); + if (!ret) { + throw new Exception("Destination down"); + } + events.clear(); + } + } + if (events.size() > 0) { + boolean ret = sendEvent(events, + currentConsumerIndexRecord, currLine); + if (!ret) { + throw new Exception("Destination down"); + } + events.clear(); + } + logger.info("Done reading file. file=" + + currentConsumerIndexRecord.filePath + + ", queueName=" + FILE_CACHE_PROVIDER_NAME + + ", consumer=" + consumerProvider.getName()); + // The entire file is read + currentConsumerIndexRecord.status = SPOOL_FILE_STATUS.done; + currentConsumerIndexRecord.doneCompleteTime = new Date(); + currentConsumerIndexRecord.lastAttempt = true; + + isRemoveIndex = true; + } catch (Exception ex) { + isDestDown = true; + logError("Destination down. queueName=" + + FILE_CACHE_PROVIDER_NAME + ", consumer=" + + consumerProvider.getName()); + lastAttemptTime = System.currentTimeMillis(); + // Update the index file + currentConsumerIndexRecord.lastFailedTime = new Date(); + currentConsumerIndexRecord.failedAttemptCount++; + currentConsumerIndexRecord.lastAttempt = false; + saveIndexFile(); + } finally { + br.close(); + } + } + if (isRemoveIndex) { + // Remove this entry from index + removeIndexRecord(currentConsumerIndexRecord); + currentConsumerIndexRecord = null; + closeFileIfNeeded(); + } + } catch (InterruptedException e) { + logger.info("Caught exception in consumer thread. Shutdown might be in progress"); + } catch (Throwable t) { + logger.error("Exception in destination writing thread.", t); + } + } + logger.info("Exiting file spooler. provider=" + FILE_CACHE_PROVIDER_NAME + + ", consumer=" + consumerProvider.getName()); + } + + private boolean sendEvent(List events, AuditIndexRecord indexRecord, + int currLine) { + boolean ret = true; + try { + ret = consumerProvider.log(events); + if (!ret) { + // Need to log error after fixed interval + logError("Error sending logs to consumer. provider=" + + FILE_CACHE_PROVIDER_NAME + ", consumer=" + + consumerProvider.getName()); + } else { + // Update index and save + indexRecord.linePosition = currLine; + indexRecord.status = SPOOL_FILE_STATUS.read_inprogress; + indexRecord.lastSuccessTime = new Date(); + indexRecord.lastAttempt = true; + saveIndexFile(); + + if (isDestDown) { + isDestDown = false; + logger.info("Destination up now. " + indexRecord.filePath + + ", queueName=" + FILE_CACHE_PROVIDER_NAME + + ", consumer=" + consumerProvider.getName()); + } + } + } catch (Throwable t) { + logger.error("Error while sending logs to consumer. provider=" + + FILE_CACHE_PROVIDER_NAME + ", consumer=" + + consumerProvider.getName() + ", log=" + events, t); + } + + return ret; + } + +} diff --git a/auth-audits/src/main/java/org/apache/atlas/audit/queue/AuditFileQueue.java b/auth-audits/src/main/java/org/apache/atlas/audit/queue/AuditFileQueue.java new file mode 100644 index 00000000000..281f9df7bd1 --- /dev/null +++ b/auth-audits/src/main/java/org/apache/atlas/audit/queue/AuditFileQueue.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.atlas.audit.queue; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.audit.model.AuditEventBase; +import org.apache.atlas.audit.provider.AuditHandler; +import org.apache.atlas.audit.provider.BaseAuditHandler; + +import java.util.Collection; +import java.util.Properties; + +/* + AuditFileQueue class does the work of stashing the audit logs into Local Filesystem before sending it to the AuditBatchQueue Consumer +*/ + +public class AuditFileQueue extends BaseAuditHandler { + private static final Log logger = LogFactory.getLog(AuditFileQueue.class); + + AuditFileQueueSpool fileSpooler = null; + AuditHandler consumer = null; + + static final String DEFAULT_NAME = "batch"; + + public AuditFileQueue(AuditHandler consumer) { + this.consumer = consumer; + } + + public void init(Properties prop, String basePropertyName) { + String propPrefix = "xasecure.audit.batch"; + if (basePropertyName != null) { + propPrefix = basePropertyName; + } + super.init(prop, propPrefix); + + //init AuditFileQueueSpooler thread to send Local logs to destination + fileSpooler = new AuditFileQueueSpool(consumer); + fileSpooler.init(prop,propPrefix); + } + + @Override + public boolean log(AuditEventBase event) { + boolean ret = false; + if ( event != null) { + fileSpooler.stashLogs(event); + if (fileSpooler.isSpoolingSuccessful()) { + ret = true; + } + } + return ret; + } + + @Override + public boolean log(Collection events) { + boolean ret = true; + if ( events != null) { + for (AuditEventBase event : events) { + ret = log(event); + } + } + return ret; + } + + + @Override + public void start() { + // Start the consumer thread + if (consumer != null) { + consumer.start(); + } + if (fileSpooler != null) { + // start AuditFileSpool thread + fileSpooler.start(); + } + } + + @Override + public void stop() { + logger.info("Stop called. name=" + getName()); + if (consumer != null) { + consumer.stop(); + } + } + + @Override + public void waitToComplete() { + logger.info("waitToComplete called. name=" + getName()); + if ( consumer != null) { + consumer.waitToComplete(); + } + } + + @Override + public void waitToComplete(long timeout) { + logger.info("waitToComplete called. name=" + getName()); + if ( consumer != null) { + consumer.waitToComplete(timeout); + } + } + + @Override + public void flush() { + logger.info("waitToComplete. name=" + getName()); + if ( consumer != null) { + consumer.flush(); + } + } + +} \ No newline at end of file diff --git a/auth-audits/src/main/java/org/apache/atlas/audit/queue/AuditFileQueueSpool.java b/auth-audits/src/main/java/org/apache/atlas/audit/queue/AuditFileQueueSpool.java new file mode 100644 index 00000000000..2cf53542b78 --- /dev/null +++ b/auth-audits/src/main/java/org/apache/atlas/audit/queue/AuditFileQueueSpool.java @@ -0,0 +1,1017 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +package org.apache.atlas.audit.queue; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +//import org.apache.log4j.MDC; +import org.apache.atlas.audit.model.AuditEventBase; +import org.apache.atlas.audit.model.AuthzAuditEvent; +import org.apache.atlas.audit.provider.AuditHandler; +import org.apache.atlas.audit.provider.MiscUtil; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileFilter; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.FileReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStreamWriter; +import java.io.PrintWriter; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Date; +import java.util.Iterator; +import java.util.List; +import java.util.Properties; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; + +/** + * This class temporarily stores logs in Local file system before it despatches each logs in file to the AuditBatchQueue Consumer. + * This gets instantiated only when AuditFileCacheProvider is enabled (xasecure.audit.provider.filecache.is.enabled). + * When AuditFileCacheProvider is enabled all the logs are stored in local file system before sent to destination. + */ + +public class AuditFileQueueSpool implements Runnable { + private static final Log logger = LogFactory.getLog(AuditFileQueueSpool.class); + + public enum SPOOL_FILE_STATUS { + pending, write_inprogress, read_inprogress, done + } + + public static final String PROP_FILE_SPOOL_LOCAL_DIR = "filespool.dir"; + public static final String PROP_FILE_SPOOL_LOCAL_FILE_NAME = "filespool.filename.format"; + public static final String PROP_FILE_SPOOL_ARCHIVE_DIR = "filespool.archive.dir"; + public static final String PROP_FILE_SPOOL_ARCHIVE_MAX_FILES_COUNT = "filespool.archive.max.files"; + public static final String PROP_FILE_SPOOL_FILENAME_PREFIX = "filespool.file.prefix"; + public static final String PROP_FILE_SPOOL_FILE_ROLLOVER = "filespool.file.rollover.sec"; + public static final String PROP_FILE_SPOOL_INDEX_FILE = "filespool.index.filename"; + public static final String PROP_FILE_SPOOL_DEST_RETRY_MS = "filespool.destination.retry.ms"; + public static final String PROP_FILE_SPOOL_BATCH_SIZE = "filespool.buffer.size"; + public static final String FILE_QUEUE_PROVIDER_NAME = "AuditFileQueueSpool"; + public static final String DEFAULT_AUDIT_FILE_TYPE = "json"; + + AuditHandler consumerProvider = null; + BlockingQueue indexQueue = new LinkedBlockingQueue(); + List indexRecords = new ArrayList(); + + // Folder and File attributes + File logFolder = null; + String logFileNameFormat = null; + File archiveFolder = null; + String fileNamePrefix = null; + String indexFileName = null; + File indexFile = null; + String indexDoneFileName = null; + String auditFileType = null; + File indexDoneFile = null; + int retryDestinationMS = 30 * 1000; // Default 30 seconds + int fileRolloverSec = 24 * 60 * 60; // In seconds + int maxArchiveFiles = 100; + int errorLogIntervalMS = 30 * 1000; // Every 30 seconds + long lastErrorLogMS = 0; + boolean isAuditFileCacheProviderEnabled = false; + boolean closeFile = false; + boolean isPending = false; + long lastAttemptTime = 0; + long bufferSize = 1000; + boolean initDone = false; + + PrintWriter logWriter = null; + AuditIndexRecord currentWriterIndexRecord = null; + AuditIndexRecord currentConsumerIndexRecord = null; + + BufferedReader logReader = null; + Thread destinationThread = null; + + boolean isWriting = true; + boolean isDrain = false; + boolean isDestDown = false; + boolean isSpoolingSuccessful = true; + + private Gson gson = null; + + public AuditFileQueueSpool(AuditHandler consumerProvider) { + this.consumerProvider = consumerProvider; + } + + public void init(Properties prop) { + init(prop, null); + } + + public boolean init(Properties props, String basePropertyName) { + logger.debug("==> AuditFileQueueSpool.init()"); + + if (initDone) { + logger.error("init() called more than once. queueProvider=" + + "" + ", consumerProvider=" + + consumerProvider.getName()); + return true; + } + String propPrefix = "xasecure.audit.filespool"; + if (basePropertyName != null) { + propPrefix = basePropertyName; + } + + try { + gson = new GsonBuilder().setDateFormat("yyyy-MM-dd HH:mm:ss.SSS") + .create(); + // Initial folder and file properties + String logFolderProp = MiscUtil.getStringProperty(props, propPrefix + + "." + PROP_FILE_SPOOL_LOCAL_DIR); + logFileNameFormat = MiscUtil.getStringProperty(props, + basePropertyName + "." + PROP_FILE_SPOOL_LOCAL_FILE_NAME); + String archiveFolderProp = MiscUtil.getStringProperty(props, + propPrefix + "." + PROP_FILE_SPOOL_ARCHIVE_DIR); + fileNamePrefix = MiscUtil.getStringProperty(props, propPrefix + "." + + PROP_FILE_SPOOL_FILENAME_PREFIX); + indexFileName = MiscUtil.getStringProperty(props, propPrefix + "." + + PROP_FILE_SPOOL_INDEX_FILE); + retryDestinationMS = MiscUtil.getIntProperty(props, propPrefix + + "." + PROP_FILE_SPOOL_DEST_RETRY_MS, retryDestinationMS); + fileRolloverSec = MiscUtil.getIntProperty(props, propPrefix + "." + + PROP_FILE_SPOOL_FILE_ROLLOVER, fileRolloverSec); + maxArchiveFiles = MiscUtil.getIntProperty(props, propPrefix + "." + + PROP_FILE_SPOOL_ARCHIVE_MAX_FILES_COUNT, maxArchiveFiles); + logger.info("retryDestinationMS=" + retryDestinationMS + + ", queueName=" + FILE_QUEUE_PROVIDER_NAME); + logger.info("fileRolloverSec=" + fileRolloverSec + ", queueName=" + + FILE_QUEUE_PROVIDER_NAME); + logger.info("maxArchiveFiles=" + maxArchiveFiles + ", queueName=" + + FILE_QUEUE_PROVIDER_NAME); + + if (logFolderProp == null || logFolderProp.isEmpty()) { + logger.fatal("Audit spool folder is not configured. Please set " + + propPrefix + + "." + + PROP_FILE_SPOOL_LOCAL_DIR + + ". queueName=" + FILE_QUEUE_PROVIDER_NAME); + return false; + } + logFolder = new File(logFolderProp); + if (!logFolder.isDirectory()) { + boolean result = logFolder.mkdirs(); + if (!logFolder.isDirectory() || !result) { + logger.fatal("File Spool folder not found and can't be created. folder=" + + logFolder.getAbsolutePath() + + ", queueName=" + + FILE_QUEUE_PROVIDER_NAME); + return false; + } + } + logger.info("logFolder=" + logFolder + ", queueName=" + + FILE_QUEUE_PROVIDER_NAME); + + if (logFileNameFormat == null || logFileNameFormat.isEmpty()) { + logFileNameFormat = "spool_" + "%app-type%" + "_" + + "%time:yyyyMMdd-HHmm.ss%.log"; + } + logger.info("logFileNameFormat=" + logFileNameFormat + + ", queueName=" + FILE_QUEUE_PROVIDER_NAME); + + if (archiveFolderProp == null || archiveFolderProp.isEmpty()) { + archiveFolder = new File(logFolder, "archive"); + } else { + archiveFolder = new File(archiveFolderProp); + } + if (!archiveFolder.isDirectory()) { + boolean result = archiveFolder.mkdirs(); + if (!archiveFolder.isDirectory() || !result) { + logger.error("File Spool archive folder not found and can't be created. folder=" + + archiveFolder.getAbsolutePath() + + ", queueName=" + + FILE_QUEUE_PROVIDER_NAME); + return false; + } + } + logger.info("archiveFolder=" + archiveFolder + ", queueName=" + + FILE_QUEUE_PROVIDER_NAME); + + if (indexFileName == null || indexFileName.isEmpty()) { + if (fileNamePrefix == null || fileNamePrefix.isEmpty()) { + fileNamePrefix = FILE_QUEUE_PROVIDER_NAME + "_" + + consumerProvider.getName(); + } + indexFileName = "index_" + fileNamePrefix + "_" + "%app-type%" + + ".json"; + indexFileName = MiscUtil.replaceTokens(indexFileName, + System.currentTimeMillis()); + } + + indexFile = new File(logFolder, indexFileName); + if (!indexFile.exists()) { + boolean ret = indexFile.createNewFile(); + if (!ret) { + logger.fatal("Error creating index file. fileName=" + + indexFile.getPath()); + return false; + } + } + logger.info("indexFile=" + indexFile + ", queueName=" + + FILE_QUEUE_PROVIDER_NAME); + + int lastDot = indexFileName.lastIndexOf('.'); + if (lastDot < 0) { + lastDot = indexFileName.length() - 1; + } + indexDoneFileName = indexFileName.substring(0, lastDot) + + "_closed.json"; + indexDoneFile = new File(logFolder, indexDoneFileName); + if (!indexDoneFile.exists()) { + boolean ret = indexDoneFile.createNewFile(); + if (!ret) { + logger.fatal("Error creating index done file. fileName=" + + indexDoneFile.getPath()); + return false; + } + } + logger.info("indexDoneFile=" + indexDoneFile + ", queueName=" + + FILE_QUEUE_PROVIDER_NAME); + + // Load index file + loadIndexFile(); + for (AuditIndexRecord auditIndexRecord : indexRecords) { + if (!auditIndexRecord.status.equals(SPOOL_FILE_STATUS.done)) { + isPending = true; + } + if (auditIndexRecord.status + .equals(SPOOL_FILE_STATUS.write_inprogress)) { + currentWriterIndexRecord = auditIndexRecord; + logger.info("currentWriterIndexRecord=" + + currentWriterIndexRecord.filePath + + ", queueName=" + FILE_QUEUE_PROVIDER_NAME); + } + if (auditIndexRecord.status + .equals(SPOOL_FILE_STATUS.read_inprogress)) { + indexQueue.add(auditIndexRecord); + } + } + printIndex(); + for (int i = 0; i < indexRecords.size(); i++) { + AuditIndexRecord auditIndexRecord = indexRecords.get(i); + if (auditIndexRecord.status.equals(SPOOL_FILE_STATUS.pending)) { + File consumerFile = new File(auditIndexRecord.filePath); + if (!consumerFile.exists()) { + logger.error("INIT: Consumer file=" + + consumerFile.getPath() + " not found."); + } else { + indexQueue.add(auditIndexRecord); + } + } + } + + auditFileType = MiscUtil.getStringProperty(props, propPrefix + ".filetype", DEFAULT_AUDIT_FILE_TYPE); + if (auditFileType == null) { + auditFileType = DEFAULT_AUDIT_FILE_TYPE; + } + + } catch (Throwable t) { + logger.fatal("Error initializing File Spooler. queue=" + + FILE_QUEUE_PROVIDER_NAME, t); + return false; + } + + bufferSize = MiscUtil.getLongProperty(props, propPrefix + + "." + PROP_FILE_SPOOL_BATCH_SIZE, bufferSize); + + initDone = true; + + logger.debug("<== AuditFileQueueSpool.init()"); + return true; + } + + /** + * Start looking for outstanding logs and update status according. + */ + public void start() { + if (!initDone) { + logger.error("Cannot start Audit File Spooler. Initilization not done yet. queueName=" + + FILE_QUEUE_PROVIDER_NAME); + return; + } + + logger.info("Starting writerThread, queueName=" + + FILE_QUEUE_PROVIDER_NAME + ", consumer=" + + consumerProvider.getName()); + + // Let's start the thread to read + destinationThread = new Thread(this, FILE_QUEUE_PROVIDER_NAME + "_" + + consumerProvider.getName() + "_destWriter"); + destinationThread.setDaemon(true); + destinationThread.start(); + } + + public void stop() { + if (!initDone) { + logger.error("Cannot stop Audit File Spooler. Initilization not done. queueName=" + + FILE_QUEUE_PROVIDER_NAME); + return; + } + logger.info("Stop called, queueName=" + FILE_QUEUE_PROVIDER_NAME + + ", consumer=" + consumerProvider.getName()); + + isDrain = true; + flush(); + + PrintWriter out = getOpenLogFileStream(); + if (out != null) { + // If write is still going on, then let's give it enough time to + // complete + for (int i = 0; i < 3; i++) { + if (isWriting) { + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + // ignore + } + continue; + } + try { + logger.info("Closing open file, queueName=" + + FILE_QUEUE_PROVIDER_NAME + ", consumer=" + + consumerProvider.getName()); + + out.flush(); + out.close(); + break; + } catch (Throwable t) { + logger.debug("Error closing spool out file.", t); + } + } + } + try { + if (destinationThread != null) { + destinationThread.interrupt(); + } + destinationThread = null; + } catch (Throwable e) { + // ignore + } + } + + public void flush() { + if (!initDone) { + logger.error("Cannot flush Audit File Spooler. Initilization not done. queueName=" + + FILE_QUEUE_PROVIDER_NAME); + return; + } + PrintWriter out = getOpenLogFileStream(); + if (out != null) { + out.flush(); + } + } + + /** + * If any files are still not processed. Also, if the destination is not + * reachable + * + * @return + */ + public boolean isPending() { + if (!initDone) { + logError("isPending(): File Spooler not initialized. queueName=" + + FILE_QUEUE_PROVIDER_NAME); + return false; + } + + return isPending; + } + + /** + * Milliseconds from last attempt time + * + * @return + */ + public long getLastAttemptTimeDelta() { + if (lastAttemptTime == 0) { + return 0; + } + return System.currentTimeMillis() - lastAttemptTime; + } + + synchronized public void stashLogs(AuditEventBase event) { + + if (isDrain) { + // Stop has been called, so this method shouldn't be called + logger.error("stashLogs() is called after stop is called. event=" + + event); + return; + } + try { + isWriting = true; + PrintWriter logOut = getLogFileStream(); + // Convert event to json + String jsonStr = MiscUtil.stringify(event); + logOut.println(jsonStr); + logOut.flush(); + isPending = true; + isSpoolingSuccessful = true; + } catch (Throwable t) { + isSpoolingSuccessful = false; + logger.error("Error writing to file. event=" + event, t); + } finally { + isWriting = false; + } + + } + + synchronized public void stashLogs(Collection events) { + for (AuditEventBase event : events) { + stashLogs(event); + } + flush(); + } + + synchronized public void stashLogsString(String event) { + if (isDrain) { + // Stop has been called, so this method shouldn't be called + logger.error("stashLogs() is called after stop is called. event=" + + event); + return; + } + try { + isWriting = true; + PrintWriter logOut = getLogFileStream(); + logOut.println(event); + } catch (Exception ex) { + logger.error("Error writing to file. event=" + event, ex); + } finally { + isWriting = false; + } + + } + + synchronized public boolean isSpoolingSuccessful() { + return isSpoolingSuccessful; + } + + synchronized public void stashLogsString(Collection events) { + for (String event : events) { + stashLogsString(event); + } + flush(); + } + + /** + * This return the current file. If there are not current open output file, + * then it will return null + * + * @return + * @throws Exception + */ + synchronized private PrintWriter getOpenLogFileStream() { + return logWriter; + } + + /** + * @return + * @throws Exception + */ + synchronized private PrintWriter getLogFileStream() throws Exception { + closeFileIfNeeded(); + // Either there are no open log file or the previous one has been rolled + // over + if (currentWriterIndexRecord == null) { + Date currentTime = new Date(); + // Create a new file + String fileName = MiscUtil.replaceTokens(logFileNameFormat, + currentTime.getTime()); + String newFileName = fileName; + File outLogFile = null; + int i = 0; + while (true) { + outLogFile = new File(logFolder, newFileName); + File archiveLogFile = new File(archiveFolder, newFileName); + if (!outLogFile.exists() && !archiveLogFile.exists()) { + break; + } + i++; + int lastDot = fileName.lastIndexOf('.'); + String baseName = fileName.substring(0, lastDot); + String extension = fileName.substring(lastDot); + newFileName = baseName + "." + i + extension; + } + fileName = newFileName; + logger.info("Creating new file. queueName=" + + FILE_QUEUE_PROVIDER_NAME + ", fileName=" + fileName); + // Open the file + logWriter = new PrintWriter(new BufferedWriter(new OutputStreamWriter(new FileOutputStream( + outLogFile),"UTF-8"))); + + AuditIndexRecord tmpIndexRecord = new AuditIndexRecord(); + + tmpIndexRecord.id = MiscUtil.generateUniqueId(); + tmpIndexRecord.filePath = outLogFile.getPath(); + tmpIndexRecord.status = SPOOL_FILE_STATUS.write_inprogress; + tmpIndexRecord.fileCreateTime = currentTime; + tmpIndexRecord.lastAttempt = true; + currentWriterIndexRecord = tmpIndexRecord; + indexRecords.add(currentWriterIndexRecord); + saveIndexFile(); + + } else { + if (logWriter == null) { + // This means the process just started. We need to open the file + // in append mode. + logger.info("Opening existing file for append. queueName=" + + FILE_QUEUE_PROVIDER_NAME + ", fileName=" + + currentWriterIndexRecord.filePath); + logWriter = new PrintWriter(new BufferedWriter(new OutputStreamWriter(new FileOutputStream( + currentWriterIndexRecord.filePath, true),"UTF-8"))); + } + } + return logWriter; + } + + synchronized private void closeFileIfNeeded() throws FileNotFoundException, + IOException { + // Is there file open to write or there are no pending file, then close + // the active file + if (currentWriterIndexRecord != null) { + // Check whether the file needs to rolled + rollOverSpoolFileByTime(); + + if (closeFile) { + // Roll the file + if (logWriter != null) { + logWriter.flush(); + logWriter.close(); + logWriter = null; + closeFile = false; + } + currentWriterIndexRecord.status = SPOOL_FILE_STATUS.pending; + currentWriterIndexRecord.writeCompleteTime = new Date(); + saveIndexFile(); + logger.info("Adding file to queue. queueName=" + + FILE_QUEUE_PROVIDER_NAME + ", fileName=" + + currentWriterIndexRecord.filePath); + indexQueue.add(currentWriterIndexRecord); + currentWriterIndexRecord = null; + } + } + } + + private void rollOverSpoolFileByTime() { + if (System.currentTimeMillis() + - currentWriterIndexRecord.fileCreateTime.getTime() > fileRolloverSec * 1000) { + closeFile = true; + logger.info("Closing file. Rolling over. queueName=" + + FILE_QUEUE_PROVIDER_NAME + ", fileName=" + + currentWriterIndexRecord.filePath); + } + } + + /** + * Load the index file + * + * @throws IOException + */ + void loadIndexFile() throws IOException { + logger.info("Loading index file. fileName=" + indexFile.getPath()); + BufferedReader br = null; + try { + br = new BufferedReader(new InputStreamReader(new FileInputStream(indexFile), "UTF-8")); + indexRecords.clear(); + String line; + while ((line = br.readLine()) != null) { + if (!line.isEmpty() && !line.startsWith("#")) { + AuditIndexRecord record = gson.fromJson(line, + AuditIndexRecord.class); + indexRecords.add(record); + } + } + } finally { + if (br!= null) { + br.close(); + } + } + } + + synchronized void printIndex() { + logger.info("INDEX printIndex() ==== START"); + Iterator iter = indexRecords.iterator(); + while (iter.hasNext()) { + AuditIndexRecord record = iter.next(); + logger.info("INDEX=" + record + ", isFileExist=" + + (new File(record.filePath).exists())); + } + logger.info("INDEX printIndex() ==== END"); + } + + synchronized void removeIndexRecord(AuditIndexRecord indexRecord) + throws FileNotFoundException, IOException { + Iterator iter = indexRecords.iterator(); + while (iter.hasNext()) { + AuditIndexRecord record = iter.next(); + if (record.id.equals(indexRecord.id)) { + logger.info("Removing file from index. file=" + record.filePath + + ", queueName=" + FILE_QUEUE_PROVIDER_NAME + + ", consumer=" + consumerProvider.getName()); + + iter.remove(); + appendToDoneFile(record); + } + } + saveIndexFile(); + // If there are no more files in the index, then let's assume the + // destination is now available + if (indexRecords.size() == 0) { + isPending = false; + } + } + + synchronized void saveIndexFile() throws FileNotFoundException, IOException { + PrintWriter out = new PrintWriter(indexFile,"UTF-8"); + for (AuditIndexRecord auditIndexRecord : indexRecords) { + out.println(gson.toJson(auditIndexRecord)); + } + out.close(); + // printIndex(); + + } + + void appendToDoneFile(AuditIndexRecord indexRecord) + throws FileNotFoundException, IOException { + logger.info("Moving to done file. " + indexRecord.filePath + + ", queueName=" + FILE_QUEUE_PROVIDER_NAME + ", consumer=" + + consumerProvider.getName()); + String line = gson.toJson(indexRecord); + PrintWriter out = new PrintWriter(new BufferedWriter(new OutputStreamWriter(new FileOutputStream( + indexDoneFile, true),"UTF-8"))); + out.println(line); + out.flush(); + out.close(); + + // After Each file is read and audit events are pushed into pipe, we flush to reach the destination immediate. + consumerProvider.flush(); + + // Move to archive folder + File logFile = null; + File archiveFile = null; + try { + logFile = new File(indexRecord.filePath); + String fileName = logFile.getName(); + archiveFile = new File(archiveFolder, fileName); + logger.info("Moving logFile " + logFile + " to " + archiveFile); + boolean result = logFile.renameTo(archiveFile); + if (!result) { + logger.error("Error moving log file to archive folder. Unable to rename" + + logFile + " to archiveFile=" + archiveFile); + } + } catch (Throwable t) { + logger.error("Error moving log file to archive folder. logFile=" + + logFile + ", archiveFile=" + archiveFile, t); + } + + // After archiving the file flush the pipe + consumerProvider.flush(); + + archiveFile = null; + try { + // Remove old files + File[] logFiles = archiveFolder.listFiles(new FileFilter() { + public boolean accept(File pathname) { + return pathname.getName().toLowerCase().endsWith(".log"); + } + }); + + if (logFiles != null && logFiles.length > maxArchiveFiles) { + int filesToDelete = logFiles.length - maxArchiveFiles; + BufferedReader br = new BufferedReader(new FileReader( + indexDoneFile)); + try { + int filesDeletedCount = 0; + while ((line = br.readLine()) != null) { + if (!line.isEmpty() && !line.startsWith("#")) { + AuditIndexRecord record = gson.fromJson(line, + AuditIndexRecord.class); + logFile = new File(record.filePath); + String fileName = logFile.getName(); + archiveFile = new File(archiveFolder, fileName); + if (archiveFile.exists()) { + logger.info("Deleting archive file " + + archiveFile); + boolean ret = archiveFile.delete(); + if (!ret) { + logger.error("Error deleting archive file. archiveFile=" + + archiveFile); + } + filesDeletedCount++; + if (filesDeletedCount >= filesToDelete) { + logger.info("Deleted " + filesDeletedCount + + " files"); + break; + } + } + } + } + } finally { + br.close(); + } + } + } catch (Throwable t) { + logger.error("Error deleting older archive file. archiveFile=" + + archiveFile, t); + } + + } + + void logError(String msg) { + long currTimeMS = System.currentTimeMillis(); + if (currTimeMS - lastErrorLogMS > errorLogIntervalMS) { + logger.error(msg); + lastErrorLogMS = currTimeMS; + } + } + + class AuditIndexRecord { + String id; + String filePath; + int linePosition = 0; + SPOOL_FILE_STATUS status = SPOOL_FILE_STATUS.write_inprogress; + Date fileCreateTime; + Date writeCompleteTime; + Date doneCompleteTime; + Date lastSuccessTime; + Date lastFailedTime; + int failedAttemptCount = 0; + boolean lastAttempt = false; + + @Override + public String toString() { + return "AuditIndexRecord [id=" + id + ", filePath=" + filePath + + ", linePosition=" + linePosition + ", status=" + status + + ", fileCreateTime=" + fileCreateTime + + ", writeCompleteTime=" + writeCompleteTime + + ", doneCompleteTime=" + doneCompleteTime + + ", lastSuccessTime=" + lastSuccessTime + + ", lastFailedTime=" + lastFailedTime + + ", failedAttemptCount=" + failedAttemptCount + + ", lastAttempt=" + lastAttempt + "]"; + } + + } + + /* + * (non-Javadoc) + * + * @see java.lang.Runnable#run() + */ + @Override + public void run() { + try { + //This is done to clear the MDC context to avoid issue with Ranger Auditing for Knox + //MDC.clear(); + runLogAudit(); + } catch (Throwable t) { + logger.fatal("Exited thread without abnormaly. queue=" + + consumerProvider.getName(), t); + } + } + + public void runLogAudit() { + // boolean isResumed = false; + while (true) { + try { + if (isDestDown) { + logger.info("Destination is down. sleeping for " + + retryDestinationMS + + " milli seconds. indexQueue=" + indexQueue.size() + + ", queueName=" + FILE_QUEUE_PROVIDER_NAME + + ", consumer=" + consumerProvider.getName()); + Thread.sleep(retryDestinationMS); + } + // Let's pause between each iteration + if (currentConsumerIndexRecord == null) { + currentConsumerIndexRecord = indexQueue.poll( + retryDestinationMS, TimeUnit.MILLISECONDS); + } else { + Thread.sleep(retryDestinationMS); + } + + if (isDrain) { + // Need to exit + break; + } + if (currentConsumerIndexRecord == null) { + closeFileIfNeeded(); + continue; + } + + boolean isRemoveIndex = false; + File consumerFile = new File( + currentConsumerIndexRecord.filePath); + if (!consumerFile.exists()) { + logger.error("Consumer file=" + consumerFile.getPath() + + " not found."); + printIndex(); + isRemoveIndex = true; + } else { + // Let's open the file to write + BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream( + currentConsumerIndexRecord.filePath),"UTF-8")); + try { + if (auditFileType.equalsIgnoreCase(DEFAULT_AUDIT_FILE_TYPE)) { + // if Audit File format is JSON each audit file in the Local Spool Location will be copied + // to HDFS location as JSON + File srcFile = new File(currentConsumerIndexRecord.filePath); + logFile(srcFile); + } else { + // If Audit File format is ORC, each records in audit files in the Local Spool Location will be + // read and converted into ORC format and pushed into an ORC file. + logEvent(br); + } + logger.info("Done reading file. file=" + + currentConsumerIndexRecord.filePath + + ", queueName=" + FILE_QUEUE_PROVIDER_NAME + + ", consumer=" + consumerProvider.getName()); + // The entire file is read + currentConsumerIndexRecord.status = SPOOL_FILE_STATUS.done; + currentConsumerIndexRecord.doneCompleteTime = new Date(); + currentConsumerIndexRecord.lastAttempt = true; + + isRemoveIndex = true; + } catch (Exception ex) { + isDestDown = true; + logError("Destination down. queueName=" + + FILE_QUEUE_PROVIDER_NAME + ", consumer=" + + consumerProvider.getName()); + lastAttemptTime = System.currentTimeMillis(); + // Update the index file + currentConsumerIndexRecord.lastFailedTime = new Date(); + currentConsumerIndexRecord.failedAttemptCount++; + currentConsumerIndexRecord.lastAttempt = false; + saveIndexFile(); + } finally { + br.close(); + } + } + if (isRemoveIndex) { + // Remove this entry from index + removeIndexRecord(currentConsumerIndexRecord); + currentConsumerIndexRecord = null; + closeFileIfNeeded(); + } + } catch (InterruptedException e) { + logger.info("Caught exception in consumer thread. Shutdown might be in progress"); + } catch (Throwable t) { + logger.error("Exception in destination writing thread.", t); + } + } + logger.info("Exiting file spooler. provider=" + FILE_QUEUE_PROVIDER_NAME + + ", consumer=" + consumerProvider.getName()); + } + + private void logEvent(BufferedReader br) throws Exception { + String line; + int currLine = 0; + int startLine = currentConsumerIndexRecord.linePosition; + List events = new ArrayList<>(); + while ((line = br.readLine()) != null) { + currLine++; + if (currLine < startLine) { + continue; + } + AuditEventBase event = MiscUtil.fromJson(line, AuthzAuditEvent.class); + events.add(event); + + if (events.size() == bufferSize) { + boolean ret = sendEvent(events, + currentConsumerIndexRecord, currLine); + if (!ret) { + throw new Exception("Destination down"); + } + events.clear(); + } + } + if (events.size() > 0) { + boolean ret = sendEvent(events, + currentConsumerIndexRecord, currLine); + if (!ret) { + throw new Exception("Destination down"); + } + events.clear(); + } + } + + private boolean sendEvent(List events, AuditIndexRecord indexRecord, + int currLine) { + boolean ret = true; + try { + ret = consumerProvider.log(events); + if (!ret) { + // Need to log error after fixed interval + logError("Error sending logs to consumer. provider=" + + FILE_QUEUE_PROVIDER_NAME + ", consumer=" + + consumerProvider.getName()); + } else { + // Update index and save + indexRecord.linePosition = currLine; + indexRecord.status = SPOOL_FILE_STATUS.read_inprogress; + indexRecord.lastSuccessTime = new Date(); + indexRecord.lastAttempt = true; + saveIndexFile(); + + if (isDestDown) { + isDestDown = false; + logger.info("Destination up now. " + indexRecord.filePath + + ", queueName=" + FILE_QUEUE_PROVIDER_NAME + + ", consumer=" + consumerProvider.getName()); + } + } + } catch (Throwable t) { + logger.error("Error while sending logs to consumer. provider=" + + FILE_QUEUE_PROVIDER_NAME + ", consumer=" + + consumerProvider.getName() + ", log=" + events, t); + } + + return ret; + } + + private void logFile(File file) throws Exception { + if (logger.isDebugEnabled()) { + logger.debug("==> AuditFileQueueSpool.logFile()"); + } + int currLine = 0; + int startLine = currentConsumerIndexRecord.linePosition; + + if (currLine < startLine) { + currLine++; + } + + boolean ret = sendFile(file,currentConsumerIndexRecord, currLine); + if (!ret) { + throw new Exception("Destination down"); + } + + if (logger.isDebugEnabled()) { + logger.debug("<== AuditFileQueueSpool.logFile()"); + } + } + + private boolean sendFile(File file, AuditIndexRecord indexRecord, + int currLine) { + boolean ret = true; + if (logger.isDebugEnabled()) { + logger.debug("==> AuditFileQueueSpool.sendFile()"); + } + + try { + ret = consumerProvider.logFile(file); + if (!ret) { + // Need to log error after fixed interval + logError("Error sending log file to consumer. provider=" + + FILE_QUEUE_PROVIDER_NAME + ", consumer=" + + consumerProvider.getName()+ ", logFile=" + file.getName()); + } else { + // Update index and save + indexRecord.linePosition = currLine; + indexRecord.status = SPOOL_FILE_STATUS.read_inprogress; + indexRecord.lastSuccessTime = new Date(); + indexRecord.lastAttempt = true; + saveIndexFile(); + + if (isDestDown) { + isDestDown = false; + logger.info("Destination up now. " + indexRecord.filePath + + ", queueName=" + FILE_QUEUE_PROVIDER_NAME + + ", consumer=" + consumerProvider.getName()); + } + } + } catch (Throwable t) { + logger.error("Error sending log file to consumer. provider=" + + FILE_QUEUE_PROVIDER_NAME + ", consumer=" + + consumerProvider.getName() + ", logFile=" + file.getName(), t); + } + + if (logger.isDebugEnabled()) { + logger.debug("<== AuditFileQueueSpool.sendFile() " + ret ); + } + return ret; + } +} \ No newline at end of file diff --git a/auth-audits/src/main/java/org/apache/atlas/audit/queue/AuditFileSpool.java b/auth-audits/src/main/java/org/apache/atlas/audit/queue/AuditFileSpool.java new file mode 100644 index 00000000000..039f7ccb24d --- /dev/null +++ b/auth-audits/src/main/java/org/apache/atlas/audit/queue/AuditFileSpool.java @@ -0,0 +1,909 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.audit.queue; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +//import org.apache.log4j.MDC; +import org.apache.atlas.audit.model.AuditEventBase; +import org.apache.atlas.audit.provider.AuditHandler; +import org.apache.atlas.audit.provider.MiscUtil; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileFilter; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.io.PrintWriter; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Date; +import java.util.Iterator; +import java.util.List; +import java.util.Properties; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; + +/** + * This class temporarily stores logs in file system if the destination is + * overloaded or down + */ +public class AuditFileSpool implements Runnable { + private static final Log logger = LogFactory.getLog(AuditFileSpool.class); + + public enum SPOOL_FILE_STATUS { + pending, write_inprogress, read_inprogress, done + } + + public static final String PROP_FILE_SPOOL_LOCAL_DIR = "filespool.dir"; + public static final String PROP_FILE_SPOOL_LOCAL_FILE_NAME = "filespool.filename.format"; + public static final String PROP_FILE_SPOOL_ARCHIVE_DIR = "filespool.archive.dir"; + public static final String PROP_FILE_SPOOL_ARCHIVE_MAX_FILES_COUNT = "filespool.archive.max.files"; + public static final String PROP_FILE_SPOOL_FILENAME_PREFIX = "filespool.file.prefix"; + public static final String PROP_FILE_SPOOL_FILE_ROLLOVER = "filespool.file.rollover.sec"; + public static final String PROP_FILE_SPOOL_INDEX_FILE = "filespool.index.filename"; + // public static final String PROP_FILE_SPOOL_INDEX_DONE_FILE = + // "filespool.index.done_filename"; + public static final String PROP_FILE_SPOOL_DEST_RETRY_MS = "filespool.destination.retry.ms"; + + AuditQueue queueProvider = null; + AuditHandler consumerProvider = null; + + BlockingQueue indexQueue = new LinkedBlockingQueue(); + + // Folder and File attributes + File logFolder = null; + String logFileNameFormat = null; + File archiveFolder = null; + String fileNamePrefix = null; + String indexFileName = null; + File indexFile = null; + String indexDoneFileName = null; + File indexDoneFile = null; + int retryDestinationMS = 30 * 1000; // Default 30 seconds + int fileRolloverSec = 24 * 60 * 60; // In seconds + int maxArchiveFiles = 100; + + int errorLogIntervalMS = 30 * 1000; // Every 30 seconds + long lastErrorLogMS = 0; + + List indexRecords = new ArrayList(); + + boolean isPending = false; + long lastAttemptTime = 0; + boolean initDone = false; + + PrintWriter logWriter = null; + AuditIndexRecord currentWriterIndexRecord = null; + AuditIndexRecord currentConsumerIndexRecord = null; + + BufferedReader logReader = null; + + Thread destinationThread = null; + + boolean isWriting = true; + boolean isDrain = false; + boolean isDestDown = false; + + private Gson gson = null; + + public AuditFileSpool(AuditQueue queueProvider, + AuditHandler consumerProvider) { + this.queueProvider = queueProvider; + this.consumerProvider = consumerProvider; + } + + public void init(Properties prop) { + init(prop, null); + } + + public boolean init(Properties props, String basePropertyName) { + if (initDone) { + logger.error("init() called more than once. queueProvider=" + + queueProvider.getName() + ", consumerProvider=" + + consumerProvider.getName()); + return true; + } + String propPrefix = "xasecure.audit.filespool"; + if (basePropertyName != null) { + propPrefix = basePropertyName; + } + + try { + gson = new GsonBuilder().setDateFormat("yyyy-MM-dd HH:mm:ss.SSS") + .create(); + + // Initial folder and file properties + String logFolderProp = MiscUtil.getStringProperty(props, propPrefix + + "." + PROP_FILE_SPOOL_LOCAL_DIR); + logFileNameFormat = MiscUtil.getStringProperty(props, + basePropertyName + "." + PROP_FILE_SPOOL_LOCAL_FILE_NAME); + String archiveFolderProp = MiscUtil.getStringProperty(props, + propPrefix + "." + PROP_FILE_SPOOL_ARCHIVE_DIR); + fileNamePrefix = MiscUtil.getStringProperty(props, propPrefix + "." + + PROP_FILE_SPOOL_FILENAME_PREFIX); + indexFileName = MiscUtil.getStringProperty(props, propPrefix + "." + + PROP_FILE_SPOOL_INDEX_FILE); + retryDestinationMS = MiscUtil.getIntProperty(props, propPrefix + + "." + PROP_FILE_SPOOL_DEST_RETRY_MS, retryDestinationMS); + fileRolloverSec = MiscUtil.getIntProperty(props, propPrefix + "." + + PROP_FILE_SPOOL_FILE_ROLLOVER, fileRolloverSec); + maxArchiveFiles = MiscUtil.getIntProperty(props, propPrefix + "." + + PROP_FILE_SPOOL_ARCHIVE_MAX_FILES_COUNT, maxArchiveFiles); + + logger.info("retryDestinationMS=" + retryDestinationMS + + ", queueName=" + queueProvider.getName()); + logger.info("fileRolloverSec=" + fileRolloverSec + ", queueName=" + + queueProvider.getName()); + logger.info("maxArchiveFiles=" + maxArchiveFiles + ", queueName=" + + queueProvider.getName()); + + if (logFolderProp == null || logFolderProp.isEmpty()) { + logger.fatal("Audit spool folder is not configured. Please set " + + propPrefix + + "." + + PROP_FILE_SPOOL_LOCAL_DIR + + ". queueName=" + queueProvider.getName()); + return false; + } + logFolder = new File(logFolderProp); + if (!logFolder.isDirectory()) { + logFolder.mkdirs(); + if (!logFolder.isDirectory()) { + logger.fatal("File Spool folder not found and can't be created. folder=" + + logFolder.getAbsolutePath() + + ", queueName=" + + queueProvider.getName()); + return false; + } + } + logger.info("logFolder=" + logFolder + ", queueName=" + + queueProvider.getName()); + + if (logFileNameFormat == null || logFileNameFormat.isEmpty()) { + logFileNameFormat = "spool_" + "%app-type%" + "_" + + "%time:yyyyMMdd-HHmm.ss%.log"; + } + logger.info("logFileNameFormat=" + logFileNameFormat + + ", queueName=" + queueProvider.getName()); + + if (archiveFolderProp == null || archiveFolderProp.isEmpty()) { + archiveFolder = new File(logFolder, "archive"); + } else { + archiveFolder = new File(archiveFolderProp); + } + if (!archiveFolder.isDirectory()) { + archiveFolder.mkdirs(); + if (!archiveFolder.isDirectory()) { + logger.error("File Spool archive folder not found and can't be created. folder=" + + archiveFolder.getAbsolutePath() + + ", queueName=" + + queueProvider.getName()); + return false; + } + } + logger.info("archiveFolder=" + archiveFolder + ", queueName=" + + queueProvider.getName()); + + if (indexFileName == null || indexFileName.isEmpty()) { + if (fileNamePrefix == null || fileNamePrefix.isEmpty()) { + fileNamePrefix = queueProvider.getName() + "_" + + consumerProvider.getName(); + } + indexFileName = "index_" + fileNamePrefix + "_" + "%app-type%" + + ".json"; + indexFileName = MiscUtil.replaceTokens(indexFileName, + System.currentTimeMillis()); + } + + indexFile = new File(logFolder, indexFileName); + if (!indexFile.exists()) { + boolean ret = indexFile.createNewFile(); + if (!ret) { + logger.fatal("Error creating index file. fileName=" + + indexDoneFile.getPath()); + return false; + } + } + logger.info("indexFile=" + indexFile + ", queueName=" + + queueProvider.getName()); + + int lastDot = indexFileName.lastIndexOf('.'); + if (lastDot < 0) { + lastDot = indexFileName.length() - 1; + } + indexDoneFileName = indexFileName.substring(0, lastDot) + + "_closed.json"; + indexDoneFile = new File(logFolder, indexDoneFileName); + if (!indexDoneFile.exists()) { + boolean ret = indexDoneFile.createNewFile(); + if (!ret) { + logger.fatal("Error creating index done file. fileName=" + + indexDoneFile.getPath()); + return false; + } + } + logger.info("indexDoneFile=" + indexDoneFile + ", queueName=" + + queueProvider.getName()); + + // Load index file + loadIndexFile(); + for (AuditIndexRecord auditIndexRecord : indexRecords) { + if (!auditIndexRecord.status.equals(SPOOL_FILE_STATUS.done)) { + isPending = true; + } + if (auditIndexRecord.status + .equals(SPOOL_FILE_STATUS.write_inprogress)) { + currentWriterIndexRecord = auditIndexRecord; + logger.info("currentWriterIndexRecord=" + + currentWriterIndexRecord.filePath + + ", queueName=" + queueProvider.getName()); + } + if (auditIndexRecord.status + .equals(SPOOL_FILE_STATUS.read_inprogress)) { + indexQueue.add(auditIndexRecord); + } + } + printIndex(); + for (AuditIndexRecord auditIndexRecord : indexRecords) { + if (auditIndexRecord.status.equals(SPOOL_FILE_STATUS.pending)) { + File consumerFile = new File(auditIndexRecord.filePath); + if (!consumerFile.exists()) { + logger.error("INIT: Consumer file=" + + consumerFile.getPath() + " not found."); + } else { + indexQueue.add(auditIndexRecord); + } + } + } + + } catch (Throwable t) { + logger.fatal("Error initializing File Spooler. queue=" + + queueProvider.getName(), t); + return false; + } + initDone = true; + return true; + } + + /** + * Start looking for outstanding logs and update status according. + */ + public void start() { + if (!initDone) { + logger.error("Cannot start Audit File Spooler. Initilization not done yet. queueName=" + + queueProvider.getName()); + return; + } + + logger.info("Starting writerThread, queueName=" + + queueProvider.getName() + ", consumer=" + + consumerProvider.getName()); + + // Let's start the thread to read + destinationThread = new Thread(this, queueProvider.getName() + "_" + + consumerProvider.getName() + "_destWriter"); + destinationThread.setDaemon(true); + destinationThread.start(); + } + + public void stop() { + if (!initDone) { + logger.error("Cannot stop Audit File Spooler. Initilization not done. queueName=" + + queueProvider.getName()); + return; + } + logger.info("Stop called, queueName=" + queueProvider.getName() + + ", consumer=" + consumerProvider.getName()); + + isDrain = true; + flush(); + + PrintWriter out = getOpenLogFileStream(); + if (out != null) { + // If write is still going on, then let's give it enough time to + // complete + for (int i = 0; i < 3; i++) { + if (isWriting) { + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + // ignore + } + continue; + } + try { + logger.info("Closing open file, queueName=" + + queueProvider.getName() + ", consumer=" + + consumerProvider.getName()); + + out.flush(); + out.close(); + break; + } catch (Throwable t) { + logger.debug("Error closing spool out file.", t); + } + } + } + try { + if (destinationThread != null) { + destinationThread.interrupt(); + } + destinationThread = null; + } catch (Throwable e) { + // ignore + } + } + + public void flush() { + if (!initDone) { + logger.error("Cannot flush Audit File Spooler. Initilization not done. queueName=" + + queueProvider.getName()); + return; + } + PrintWriter out = getOpenLogFileStream(); + if (out != null) { + out.flush(); + } + } + + /** + * If any files are still not processed. Also, if the destination is not + * reachable + * + * @return + */ + public boolean isPending() { + if (!initDone) { + logError("isPending(): File Spooler not initialized. queueName=" + + queueProvider.getName()); + return false; + } + + return isPending; + } + + /** + * Milliseconds from last attempt time + * + * @return + */ + public long getLastAttemptTimeDelta() { + if (lastAttemptTime == 0) { + return 0; + } + return System.currentTimeMillis() - lastAttemptTime; + } + + synchronized public void stashLogs(AuditEventBase event) { + if (isDrain) { + // Stop has been called, so this method shouldn't be called + logger.error("stashLogs() is called after stop is called. event=" + + event); + return; + } + try { + isWriting = true; + PrintWriter logOut = getLogFileStream(); + // Convert event to json + String jsonStr = MiscUtil.stringify(event); + logOut.println(jsonStr); + isPending = true; + } catch (Exception ex) { + logger.error("Error writing to file. event=" + event, ex); + } finally { + isWriting = false; + } + + } + + synchronized public void stashLogs(Collection events) { + for (AuditEventBase event : events) { + stashLogs(event); + } + flush(); + } + + synchronized public void stashLogsString(String event) { + if (isDrain) { + // Stop has been called, so this method shouldn't be called + logger.error("stashLogs() is called after stop is called. event=" + + event); + return; + } + try { + isWriting = true; + PrintWriter logOut = getLogFileStream(); + logOut.println(event); + } catch (Exception ex) { + logger.error("Error writing to file. event=" + event, ex); + } finally { + isWriting = false; + } + + } + + synchronized public void stashLogsString(Collection events) { + for (String event : events) { + stashLogsString(event); + } + flush(); + } + + /** + * This return the current file. If there are not current open output file, + * then it will return null + * + * @return + * @throws Exception + */ + synchronized private PrintWriter getOpenLogFileStream() { + return logWriter; + } + + /** + * @return + * @throws Exception + */ + synchronized private PrintWriter getLogFileStream() throws Exception { + closeFileIfNeeded(); + + // Either there are no open log file or the previous one has been rolled + // over + if (currentWriterIndexRecord == null) { + Date currentTime = new Date(); + // Create a new file + String fileName = MiscUtil.replaceTokens(logFileNameFormat, + currentTime.getTime()); + String newFileName = fileName; + File outLogFile = null; + int i = 0; + while (true) { + outLogFile = new File(logFolder, newFileName); + File archiveLogFile = new File(archiveFolder, newFileName); + if (!outLogFile.exists() && !archiveLogFile.exists()) { + break; + } + i++; + int lastDot = fileName.lastIndexOf('.'); + String baseName = fileName.substring(0, lastDot); + String extension = fileName.substring(lastDot); + newFileName = baseName + "." + i + extension; + } + fileName = newFileName; + logger.info("Creating new file. queueName=" + + queueProvider.getName() + ", fileName=" + fileName); + // Open the file + logWriter = new PrintWriter(new BufferedWriter(new FileWriter( + outLogFile))); + + AuditIndexRecord tmpIndexRecord = new AuditIndexRecord(); + + tmpIndexRecord.id = MiscUtil.generateUniqueId(); + tmpIndexRecord.filePath = outLogFile.getPath(); + tmpIndexRecord.status = SPOOL_FILE_STATUS.write_inprogress; + tmpIndexRecord.fileCreateTime = currentTime; + tmpIndexRecord.lastAttempt = true; + currentWriterIndexRecord = tmpIndexRecord; + indexRecords.add(currentWriterIndexRecord); + saveIndexFile(); + + } else { + if (logWriter == null) { + // This means the process just started. We need to open the file + // in append mode. + logger.info("Opening existing file for append. queueName=" + + queueProvider.getName() + ", fileName=" + + currentWriterIndexRecord.filePath); + logWriter = new PrintWriter(new BufferedWriter(new FileWriter( + currentWriterIndexRecord.filePath, true))); + } + } + return logWriter; + } + + synchronized private void closeFileIfNeeded() throws FileNotFoundException, + IOException { + // Is there file open to write or there are no pending file, then close + // the active file + if (currentWriterIndexRecord != null) { + // Check whether the file needs to rolled + boolean closeFile = false; + if (indexRecords.size() == 1) { + closeFile = true; + logger.info("Closing file. Only one open file. queueName=" + + queueProvider.getName() + ", fileName=" + + currentWriterIndexRecord.filePath); + } else if (System.currentTimeMillis() + - currentWriterIndexRecord.fileCreateTime.getTime() > fileRolloverSec * 1000) { + closeFile = true; + logger.info("Closing file. Rolling over. queueName=" + + queueProvider.getName() + ", fileName=" + + currentWriterIndexRecord.filePath); + } + if (closeFile) { + // Roll the file + if (logWriter != null) { + logWriter.flush(); + logWriter.close(); + logWriter = null; + } + currentWriterIndexRecord.status = SPOOL_FILE_STATUS.pending; + currentWriterIndexRecord.writeCompleteTime = new Date(); + saveIndexFile(); + logger.info("Adding file to queue. queueName=" + + queueProvider.getName() + ", fileName=" + + currentWriterIndexRecord.filePath); + indexQueue.add(currentWriterIndexRecord); + currentWriterIndexRecord = null; + } + } + } + + /** + * Load the index file + * + * @throws IOException + */ + void loadIndexFile() throws IOException { + logger.info("Loading index file. fileName=" + indexFile.getPath()); + BufferedReader br = new BufferedReader(new FileReader(indexFile)); + indexRecords.clear(); + String line; + while ((line = br.readLine()) != null) { + if (!line.isEmpty() && !line.startsWith("#")) { + AuditIndexRecord record = gson.fromJson(line, + AuditIndexRecord.class); + indexRecords.add(record); + } + } + br.close(); + } + + synchronized void printIndex() { + logger.info("INDEX printIndex() ==== START"); + Iterator iter = indexRecords.iterator(); + while (iter.hasNext()) { + AuditIndexRecord record = iter.next(); + logger.info("INDEX=" + record + ", isFileExist=" + + (new File(record.filePath).exists())); + } + logger.info("INDEX printIndex() ==== END"); + } + + synchronized void removeIndexRecord(AuditIndexRecord indexRecord) + throws FileNotFoundException, IOException { + Iterator iter = indexRecords.iterator(); + while (iter.hasNext()) { + AuditIndexRecord record = iter.next(); + if (record.id.equals(indexRecord.id)) { + logger.info("Removing file from index. file=" + record.filePath + + ", queueName=" + queueProvider.getName() + + ", consumer=" + consumerProvider.getName()); + + iter.remove(); + appendToDoneFile(record); + } + } + saveIndexFile(); + // If there are no more files in the index, then let's assume the + // destination is now available + if (indexRecords.size() == 0) { + isPending = false; + } + } + + synchronized void saveIndexFile() throws FileNotFoundException, IOException { + PrintWriter out = new PrintWriter(indexFile); + for (AuditIndexRecord auditIndexRecord : indexRecords) { + out.println(gson.toJson(auditIndexRecord)); + } + out.close(); + // printIndex(); + + } + + void appendToDoneFile(AuditIndexRecord indexRecord) + throws FileNotFoundException, IOException { + logger.info("Moving to done file. " + indexRecord.filePath + + ", queueName=" + queueProvider.getName() + ", consumer=" + + consumerProvider.getName()); + String line = gson.toJson(indexRecord); + PrintWriter out = new PrintWriter(new BufferedWriter(new FileWriter( + indexDoneFile, true))); + out.println(line); + out.flush(); + out.close(); + + // Move to archive folder + File logFile = null; + File archiveFile = null; + try { + logFile = new File(indexRecord.filePath); + String fileName = logFile.getName(); + archiveFile = new File(archiveFolder, fileName); + logger.info("Moving logFile " + logFile + " to " + archiveFile); + logFile.renameTo(archiveFile); + } catch (Throwable t) { + logger.error("Error moving log file to archive folder. logFile=" + + logFile + ", archiveFile=" + archiveFile, t); + } + + archiveFile = null; + try { + // Remove old files + File[] logFiles = archiveFolder.listFiles(new FileFilter() { + public boolean accept(File pathname) { + return pathname.getName().toLowerCase().endsWith(".log"); + } + }); + + if (logFiles != null && logFiles.length > maxArchiveFiles) { + int filesToDelete = logFiles.length - maxArchiveFiles; + BufferedReader br = new BufferedReader(new FileReader( + indexDoneFile)); + try { + int filesDeletedCount = 0; + while ((line = br.readLine()) != null) { + if (!line.isEmpty() && !line.startsWith("#")) { + AuditIndexRecord record = gson.fromJson(line, + AuditIndexRecord.class); + logFile = new File(record.filePath); + String fileName = logFile.getName(); + archiveFile = new File(archiveFolder, fileName); + if (archiveFile.exists()) { + logger.info("Deleting archive file " + + archiveFile); + boolean ret = archiveFile.delete(); + if (!ret) { + logger.error("Error deleting archive file. archiveFile=" + + archiveFile); + } + filesDeletedCount++; + if (filesDeletedCount >= filesToDelete) { + logger.info("Deleted " + filesDeletedCount + + " files"); + break; + } + } + } + } + } finally { + br.close(); + } + } + } catch (Throwable t) { + logger.error("Error deleting older archive file. archiveFile=" + + archiveFile, t); + } + + } + + void logError(String msg) { + long currTimeMS = System.currentTimeMillis(); + if (currTimeMS - lastErrorLogMS > errorLogIntervalMS) { + logger.error(msg); + lastErrorLogMS = currTimeMS; + } + } + + class AuditIndexRecord { + String id; + String filePath; + int linePosition = 0; + SPOOL_FILE_STATUS status = SPOOL_FILE_STATUS.write_inprogress; + Date fileCreateTime; + Date writeCompleteTime; + Date doneCompleteTime; + Date lastSuccessTime; + Date lastFailedTime; + int failedAttemptCount = 0; + boolean lastAttempt = false; + + @Override + public String toString() { + return "AuditIndexRecord [id=" + id + ", filePath=" + filePath + + ", linePosition=" + linePosition + ", status=" + status + + ", fileCreateTime=" + fileCreateTime + + ", writeCompleteTime=" + writeCompleteTime + + ", doneCompleteTime=" + doneCompleteTime + + ", lastSuccessTime=" + lastSuccessTime + + ", lastFailedTime=" + lastFailedTime + + ", failedAttemptCount=" + failedAttemptCount + + ", lastAttempt=" + lastAttempt + "]"; + } + + } + + class AuditFileSpoolAttempt { + Date attemptTime; + String status; + } + + /* + * (non-Javadoc) + * + * @see java.lang.Runnable#run() + */ + @Override + public void run() { + try { + //This is done to clear the MDC context to avoid issue with Ranger Auditing for Knox + //MDC.clear(); + runLogAudit(); + } catch (Throwable t) { + logger.fatal("Exited thread without abnormaly. queue=" + + consumerProvider.getName(), t); + } + } + + public void runLogAudit() { + // boolean isResumed = false; + while (true) { + try { + if (isDestDown) { + logger.info("Destination is down. sleeping for " + + retryDestinationMS + + " milli seconds. indexQueue=" + indexQueue.size() + + ", queueName=" + queueProvider.getName() + + ", consumer=" + consumerProvider.getName()); + Thread.sleep(retryDestinationMS); + } + + // Let's pause between each iteration + if (currentConsumerIndexRecord == null) { + currentConsumerIndexRecord = indexQueue.poll( + retryDestinationMS, TimeUnit.MILLISECONDS); + } else { + Thread.sleep(retryDestinationMS); + } + + if (isDrain) { + // Need to exit + break; + } + if (currentConsumerIndexRecord == null) { + closeFileIfNeeded(); + continue; + } + + boolean isRemoveIndex = false; + File consumerFile = new File( + currentConsumerIndexRecord.filePath); + if (!consumerFile.exists()) { + logger.error("Consumer file=" + consumerFile.getPath() + + " not found."); + printIndex(); + isRemoveIndex = true; + } else { + // Let's open the file to write + BufferedReader br = new BufferedReader(new FileReader( + currentConsumerIndexRecord.filePath)); + try { + int startLine = currentConsumerIndexRecord.linePosition; + String line; + int currLine = 0; + List lines = new ArrayList(); + while ((line = br.readLine()) != null) { + currLine++; + if (currLine < startLine) { + continue; + } + lines.add(line); + if (lines.size() == queueProvider.getMaxBatchSize()) { + boolean ret = sendEvent(lines, + currentConsumerIndexRecord, currLine); + if (!ret) { + throw new Exception("Destination down"); + } + lines.clear(); + } + } + if (lines.size() > 0) { + boolean ret = sendEvent(lines, + currentConsumerIndexRecord, currLine); + if (!ret) { + throw new Exception("Destination down"); + } + lines.clear(); + } + logger.info("Done reading file. file=" + + currentConsumerIndexRecord.filePath + + ", queueName=" + queueProvider.getName() + + ", consumer=" + consumerProvider.getName()); + // The entire file is read + currentConsumerIndexRecord.status = SPOOL_FILE_STATUS.done; + currentConsumerIndexRecord.doneCompleteTime = new Date(); + currentConsumerIndexRecord.lastAttempt = true; + + isRemoveIndex = true; + } catch (Exception ex) { + isDestDown = true; + logError("Destination down. queueName=" + + queueProvider.getName() + ", consumer=" + + consumerProvider.getName()); + lastAttemptTime = System.currentTimeMillis(); + // Update the index file + currentConsumerIndexRecord.lastFailedTime = new Date(); + currentConsumerIndexRecord.failedAttemptCount++; + currentConsumerIndexRecord.lastAttempt = false; + saveIndexFile(); + } finally { + br.close(); + } + } + if (isRemoveIndex) { + // Remove this entry from index + removeIndexRecord(currentConsumerIndexRecord); + currentConsumerIndexRecord = null; + closeFileIfNeeded(); + } + } catch (InterruptedException e) { + logger.info("Caught exception in consumer thread. Shutdown might be in progress"); + break; + } catch (Throwable t) { + logger.error("Exception in destination writing thread.", t); + } + } + logger.info("Exiting file spooler. provider=" + queueProvider.getName() + + ", consumer=" + consumerProvider.getName()); + } + + private boolean sendEvent(List lines, AuditIndexRecord indexRecord, + int currLine) { + boolean ret = true; + try { + ret = consumerProvider.logJSON(lines); + if (!ret) { + // Need to log error after fixed interval + logError("Error sending logs to consumer. provider=" + + queueProvider.getName() + ", consumer=" + + consumerProvider.getName()); + } else { + // Update index and save + indexRecord.linePosition = currLine; + indexRecord.status = SPOOL_FILE_STATUS.read_inprogress; + indexRecord.lastSuccessTime = new Date(); + indexRecord.lastAttempt = true; + saveIndexFile(); + + if (isDestDown) { + isDestDown = false; + logger.info("Destination up now. " + indexRecord.filePath + + ", queueName=" + queueProvider.getName() + + ", consumer=" + consumerProvider.getName()); + } + } + } catch (Throwable t) { + logger.error("Error while sending logs to consumer. provider=" + + queueProvider.getName() + ", consumer=" + + consumerProvider.getName() + ", log=" + lines, t); + } + + return ret; + } + +} diff --git a/auth-audits/src/main/java/org/apache/atlas/audit/queue/AuditQueue.java b/auth-audits/src/main/java/org/apache/atlas/audit/queue/AuditQueue.java new file mode 100644 index 00000000000..d27ab300358 --- /dev/null +++ b/auth-audits/src/main/java/org/apache/atlas/audit/queue/AuditQueue.java @@ -0,0 +1,230 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.audit.queue; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.audit.destination.AuditDestination; +import org.apache.atlas.audit.provider.AuditHandler; +import org.apache.atlas.audit.provider.BaseAuditHandler; +import org.apache.atlas.audit.provider.MiscUtil; + +import java.util.Properties; + +public abstract class AuditQueue extends BaseAuditHandler { + private static final Log LOG = LogFactory.getLog(AuditQueue.class); + + public static final int AUDIT_MAX_QUEUE_SIZE_DEFAULT = 1024 * 1024; + public static final int AUDIT_BATCH_INTERVAL_DEFAULT_MS = 3000; + public static final int AUDIT_BATCH_SIZE_DEFAULT = 1000; + + // This is the max time the consumer thread will wait before exiting the + // loop + public static final int AUDIT_CONSUMER_THREAD_WAIT_MS = 5000; + + private int maxQueueSize = AUDIT_MAX_QUEUE_SIZE_DEFAULT; + private int maxBatchInterval = AUDIT_BATCH_INTERVAL_DEFAULT_MS; + private int maxBatchSize = AUDIT_BATCH_SIZE_DEFAULT; + + public static final String PROP_QUEUE = "queue"; + + public static final String PROP_BATCH_SIZE = "batch.size"; + public static final String PROP_QUEUE_SIZE = "queue.size"; + public static final String PROP_BATCH_INTERVAL = "batch.interval.ms"; + + public static final String PROP_FILE_SPOOL_ENABLE = "filespool.enable"; + public static final String PROP_FILE_SPOOL_WAIT_FOR_FULL_DRAIN = "filespool.drain.full.wait.ms"; + public static final String PROP_FILE_SPOOL_QUEUE_THRESHOLD = "filespool.drain.threshold.percent"; + + final protected AuditHandler consumer; + protected AuditFileSpool fileSpooler = null; + + private boolean isDrain = false; + + protected boolean fileSpoolerEnabled = false; + protected int fileSpoolMaxWaitTime = 5 * 60 * 1000; // Default 5 minutes + protected int fileSpoolDrainThresholdPercent = 80; + + boolean isConsumerDestination = false; + // This is set when the first time stop is called. + protected long stopTime = 0; + + /** + * @param consumer + */ + public AuditQueue(AuditHandler consumer) { + this.consumer = consumer; + if (consumer instanceof BaseAuditHandler) { + BaseAuditHandler baseAuditHander = (BaseAuditHandler) consumer; + baseAuditHander.setParentPath(getName()); + } + + if (consumer != null && consumer instanceof AuditDestination) { + // If consumer is destination, then the thread should run as server + // user + isConsumerDestination = true; + } + } + + @Override + public void init(Properties props, String basePropertyName) { + LOG.info("BaseAuditProvider.init()"); + super.init(props, basePropertyName); + + setMaxBatchSize(MiscUtil.getIntProperty(props, propPrefix + "." + + PROP_BATCH_SIZE, getMaxBatchSize())); + setMaxQueueSize(MiscUtil.getIntProperty(props, propPrefix + "." + + PROP_QUEUE_SIZE, getMaxQueueSize())); + setMaxBatchInterval(MiscUtil.getIntProperty(props, propPrefix + "." + + PROP_BATCH_INTERVAL, getMaxBatchInterval())); + + fileSpoolerEnabled = MiscUtil.getBooleanProperty(props, propPrefix + + "." + PROP_FILE_SPOOL_ENABLE, false); + String logFolderProp = MiscUtil.getStringProperty(props, propPrefix + + "." + AuditFileSpool.PROP_FILE_SPOOL_LOCAL_DIR); + if (fileSpoolerEnabled || logFolderProp != null) { + LOG.info("File spool is enabled for " + getName() + + ", logFolderProp=" + logFolderProp + ", " + propPrefix + + "." + AuditFileSpool.PROP_FILE_SPOOL_LOCAL_DIR + "=" + + fileSpoolerEnabled); + fileSpoolerEnabled = true; + fileSpoolMaxWaitTime = MiscUtil.getIntProperty(props, propPrefix + + "." + PROP_FILE_SPOOL_WAIT_FOR_FULL_DRAIN, + fileSpoolMaxWaitTime); + fileSpoolDrainThresholdPercent = MiscUtil.getIntProperty(props, + propPrefix + "." + PROP_FILE_SPOOL_QUEUE_THRESHOLD, + fileSpoolDrainThresholdPercent); + fileSpooler = new AuditFileSpool(this, consumer); + if (!fileSpooler.init(props, basePropertyName)) { + fileSpoolerEnabled = false; + LOG.fatal("Couldn't initialize file spooler. Disabling it. queue=" + + getName() + ", consumer=" + consumer.getName()); + } + } else { + LOG.info("File spool is disabled for " + getName()); + } + + } + + @Override + public void setParentPath(String parentPath) { + super.setParentPath(parentPath); + if (consumer != null && consumer instanceof BaseAuditHandler) { + BaseAuditHandler base = (BaseAuditHandler) consumer; + base.setParentPath(getName()); + } + } + + @Override + public String getFinalPath() { + if (consumer != null) { + if (consumer instanceof BaseAuditHandler) { + return ((BaseAuditHandler) consumer).getFinalPath(); + } else { + return consumer.getName(); + } + } + return getName(); + } + + @Override + public void setName(String name) { + super.setName(name); + if (consumer != null && consumer instanceof BaseAuditHandler) { + BaseAuditHandler base = (BaseAuditHandler) consumer; + base.setParentPath(getName()); + } + } + + public AuditHandler getConsumer() { + return consumer; + } + + public boolean isDrainMaxTimeElapsed() { + return (stopTime - System.currentTimeMillis()) > AUDIT_CONSUMER_THREAD_WAIT_MS; + } + + public boolean isDrain() { + return isDrain; + } + + public void setDrain(boolean isDrain) { + if (isDrain && stopTime != 0) { + stopTime = System.currentTimeMillis(); + } + this.isDrain = isDrain; + } + + public int getMaxQueueSize() { + return maxQueueSize; + } + + public void setMaxQueueSize(int maxQueueSize) { + this.maxQueueSize = maxQueueSize; + } + + public int getMaxBatchInterval() { + return maxBatchInterval; + } + + public void setMaxBatchInterval(int maxBatchInterval) { + this.maxBatchInterval = maxBatchInterval; + } + + public int getMaxBatchSize() { + return maxBatchSize; + } + + public void setMaxBatchSize(int maxBatchSize) { + this.maxBatchSize = maxBatchSize; + } + + /* + * (non-Javadoc) + * + * @see org.apache.ranger.audit.provider.AuditProvider#waitToComplete() + */ + @Override + public void waitToComplete() { + if (consumer != null) { + consumer.waitToComplete(-1); + } + } + + @Override + public void waitToComplete(long timeout) { + if (consumer != null) { + consumer.waitToComplete(timeout); + } + } + + /* + * (non-Javadoc) + * + * @see org.apache.ranger.audit.provider.AuditProvider#flush() + */ + @Override + public void flush() { + if (consumer != null) { + consumer.flush(); + } + } + +} diff --git a/auth-audits/src/main/java/org/apache/atlas/audit/queue/AuditSummaryQueue.java b/auth-audits/src/main/java/org/apache/atlas/audit/queue/AuditSummaryQueue.java new file mode 100644 index 00000000000..54911829ded --- /dev/null +++ b/auth-audits/src/main/java/org/apache/atlas/audit/queue/AuditSummaryQueue.java @@ -0,0 +1,265 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.audit.queue; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +//import org.apache.log4j.MDC; +import org.apache.atlas.audit.model.AuditEventBase; +import org.apache.atlas.audit.provider.AuditHandler; +import org.apache.atlas.audit.provider.MiscUtil; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; + +/** + * This is a non-blocking queue with no limit on capacity. + */ +public class AuditSummaryQueue extends AuditQueue implements Runnable { + private static final Log logger = LogFactory + .getLog(AuditSummaryQueue.class); + + public static final String PROP_SUMMARY_INTERVAL = "summary.interval.ms"; + + LinkedBlockingQueue queue = new LinkedBlockingQueue(); + Thread consumerThread = null; + + static int threadCount = 0; + static final String DEFAULT_NAME = "summary"; + + private static final int MAX_DRAIN = 100000; + + private int maxSummaryIntervalMs = 5000; + + HashMap summaryMap = new HashMap(); + + public AuditSummaryQueue(AuditHandler consumer) { + super(consumer); + setName(DEFAULT_NAME); + } + + @Override + public void init(Properties props, String propPrefix) { + super.init(props, propPrefix); + maxSummaryIntervalMs = MiscUtil.getIntProperty(props, propPrefix + "." + + PROP_SUMMARY_INTERVAL, maxSummaryIntervalMs); + logger.info("maxSummaryInterval=" + maxSummaryIntervalMs + ", name=" + + getName()); + } + + /* + * (non-Javadoc) + * + * @see + * org.apache.ranger.audit.provider.AuditProvider#log(org.apache.ranger. + * audit.model.AuditEventBase) + */ + @Override + public boolean log(AuditEventBase event) { + // Add to the queue and return ASAP + if (queue.size() >= getMaxQueueSize()) { + return false; + } + queue.add(event); + return true; + } + + @Override + public boolean log(Collection events) { + boolean ret = true; + for (AuditEventBase event : events) { + ret = log(event); + if (!ret) { + break; + } + } + return ret; + } + + /* + * (non-Javadoc) + * + * @see org.apache.ranger.audit.provider.AuditProvider#start() + */ + @Override + public void start() { + if (consumer != null) { + consumer.start(); + } + + consumerThread = new Thread(this, this.getClass().getName() + + (threadCount++)); + consumerThread.setDaemon(true); + consumerThread.start(); + } + + /* + * (non-Javadoc) + * + * @see org.apache.ranger.audit.provider.AuditProvider#stop() + */ + @Override + public void stop() { + logger.info("Stop called. name=" + getName()); + setDrain(true); + try { + if (consumerThread != null) { + logger.info("Interrupting consumerThread. name=" + getName() + + ", consumer=" + + (consumer == null ? null : consumer.getName())); + + consumerThread.interrupt(); + } + } catch (Throwable t) { + // ignore any exception + } + consumerThread = null; + } + + /* + * (non-Javadoc) + * + * @see java.lang.Runnable#run() + */ + @Override + public void run() { + try { + //This is done to clear the MDC context to avoid issue with Ranger Auditing for Knox + //MDC.clear(); + runLogAudit(); + } catch (Throwable t) { + logger.fatal("Exited thread without abnormaly. queue=" + getName(), + t); + } + } + + public void runLogAudit() { + + long lastDispatchTime = System.currentTimeMillis(); + + while (true) { + // Time to next dispatch + long nextDispatchDuration = lastDispatchTime + - System.currentTimeMillis() + maxSummaryIntervalMs; + + Collection eventList = new ArrayList(); + + try { + AuditEventBase event = null; + if (!isDrain() && nextDispatchDuration > 0) { + event = queue.poll(nextDispatchDuration, + TimeUnit.MILLISECONDS); + } else { + // For poll() is non blocking + event = queue.poll(); + } + + if (event != null) { + eventList.add(event); + queue.drainTo(eventList, MAX_DRAIN - 1); + } else { + // poll returned due to timeout, so reseting clock + nextDispatchDuration = lastDispatchTime + - System.currentTimeMillis() + maxSummaryIntervalMs; + lastDispatchTime = System.currentTimeMillis(); + } + } catch (InterruptedException e) { + logger.info("Caught exception in consumer thread. Shutdown might be in progress"); + } catch (Throwable t) { + logger.error("Caught error during processing request.", t); + } + + for (AuditEventBase event : eventList) { + // Add to hash map + String key = event.getEventKey(); + AuditSummary auditSummary = summaryMap.get(key); + if (auditSummary == null) { + auditSummary = new AuditSummary(); + auditSummary.event = event; + auditSummary.startTime = event.getEventTime(); + auditSummary.endTime = event.getEventTime(); + auditSummary.count = 1; + summaryMap.put(key, auditSummary); + } else { + auditSummary.endTime = event.getEventTime(); + auditSummary.count++; + } + } + + if (isDrain() || nextDispatchDuration <= 0) { + // Reset time just before sending the logs + lastDispatchTime = System.currentTimeMillis(); + + for (Map.Entry entry : summaryMap + .entrySet()) { + AuditSummary auditSummary = entry.getValue(); + auditSummary.event.setEventCount(auditSummary.count); + long timeDiff = auditSummary.endTime.getTime() + - auditSummary.startTime.getTime(); + timeDiff = timeDiff > 0 ? timeDiff : 1; + auditSummary.event.setEventDurationMS(timeDiff); + boolean ret = consumer.log(auditSummary.event); + if (!ret) { + // We need to drop this event + logFailedEvent(auditSummary.event); + } + } + summaryMap.clear(); + } + + if (isDrain()) { + if (summaryMap.isEmpty() && queue.isEmpty()) { + break; + } + if (isDrainMaxTimeElapsed()) { + logger.warn("Exiting polling loop because max time allowed reached. name=" + + getName() + + ", waited for " + + (stopTime - System.currentTimeMillis()) + " ms"); + } + } + + } + + logger.info("Exiting polling loop. name=" + getName()); + try { + // Call stop on the consumer + logger.info("Calling to stop consumer. name=" + getName() + + ", consumer.name=" + consumer.getName()); + consumer.stop(); + } catch (Throwable t) { + logger.error("Error while calling stop on consumer.", t); + } + logger.info("Exiting consumerThread.run() method. name=" + getName()); + } + + class AuditSummary { + Date startTime = null; + Date endTime = null; + int count = 0; + AuditEventBase event; + } +} diff --git a/auth-audits/src/main/java/org/apache/atlas/audit/utils/AbstractKerberosUser.java b/auth-audits/src/main/java/org/apache/atlas/audit/utils/AbstractKerberosUser.java new file mode 100644 index 00000000000..44528b003dd --- /dev/null +++ b/auth-audits/src/main/java/org/apache/atlas/audit/utils/AbstractKerberosUser.java @@ -0,0 +1,246 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.audit.utils; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.security.auth.Subject; +import javax.security.auth.kerberos.KerberosPrincipal; +import javax.security.auth.kerberos.KerberosTicket; +import javax.security.auth.login.LoginContext; +import javax.security.auth.login.LoginException; +import java.security.PrivilegedAction; +import java.security.PrivilegedActionException; +import java.security.PrivilegedExceptionAction; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; + +public abstract class AbstractKerberosUser implements KerberosUser { + + private static final Logger LOG = LoggerFactory.getLogger(AbstractKerberosUser.class); + + static final String DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss'Z'"; + + /** + * Percentage of the ticket window to use before we renew the TGT. + */ + static final float TICKET_RENEW_WINDOW = 0.80f; + + protected final AtomicBoolean loggedIn = new AtomicBoolean(false); + + protected Subject subject; + protected LoginContext loginContext; + + public AbstractKerberosUser() { + } + + /** + * Performs a login using the specified principal and keytab. + * + * @throws LoginException if the login fails + */ + @Override + public synchronized void login() throws LoginException { + if (isLoggedIn()) { + return; + } + + try { + // If it's the first time ever calling login then we need to initialize a new context + if (loginContext == null) { + if (LOG.isDebugEnabled()) { + LOG.debug("Initializing new login context..."); + } + if (this.subject == null) { + // only create a new subject if a current one does not exist + // other classes may be referencing an existing subject and replacing it may break functionality of those other classes after relogin + this.subject = new Subject(); + } + this.loginContext = createLoginContext(subject); + } + + loginContext.login(); + loggedIn.set(true); + if (LOG.isDebugEnabled()) { + LOG.debug("Successful login for {}", new Object[]{getPrincipal()}); + } + } catch (LoginException le) { + LoginException loginException = new LoginException("Unable to login with " + getPrincipal() + " due to: " + le.getMessage()); + loginException.setStackTrace(le.getStackTrace()); + throw loginException; + } + } + + protected abstract LoginContext createLoginContext(final Subject subject) throws LoginException; + + /** + * Performs a logout of the current user. + * + * @throws LoginException if the logout fails + */ + @Override + public synchronized void logout() throws LoginException { + if (!isLoggedIn()) { + return; + } + + try { + loginContext.logout(); + loggedIn.set(false); + LOG.debug("Successful logout for {}", new Object[]{getPrincipal()}); + + loginContext = null; + } catch (LoginException e) { + throw new LoginException("Logout failed due to: " + e.getMessage()); + } + } + + /** + * Executes the PrivilegedAction as this user. + * + * @param action the action to execute + * @param the type of result + * @return the result of the action + * @throws IllegalStateException if this method is called while not logged in + */ + @Override + public T doAs(final PrivilegedAction action) throws IllegalStateException { + if (!isLoggedIn()) { + throw new IllegalStateException("Must login before executing actions"); + } + + return Subject.doAs(subject, action); + } + + /** + * Executes the PrivilegedAction as this user. + * + * @param action the action to execute + * @param the type of result + * @return the result of the action + * @throws IllegalStateException if this method is called while not logged in + * @throws PrivilegedActionException if an exception is thrown from the action + */ + @Override + public T doAs(final PrivilegedExceptionAction action) + throws IllegalStateException, PrivilegedActionException { + if (!isLoggedIn()) { + throw new IllegalStateException("Must login before executing actions"); + } + + return Subject.doAs(subject, action); + } + + /** + * Re-login a user from keytab if TGT is expired or is close to expiry. + * + * @throws LoginException if an error happens performing the re-login + */ + @Override + public synchronized boolean checkTGTAndRelogin() throws LoginException { + final KerberosTicket tgt = getTGT(); + if (tgt == null) { + LOG.debug("TGT was not found"); + } + + if (tgt != null && System.currentTimeMillis() < getRefreshTime(tgt)) { + LOG.debug("TGT was found, but has not reached expiration window"); + return false; + } + + LOG.debug("Performing relogin for {}", new Object[]{getPrincipal()}); + logout(); + login(); + return true; + } + + /** + * Get the Kerberos TGT. + * + * @return the user's TGT or null if none was found + */ + private synchronized KerberosTicket getTGT() { + final Set tickets = subject.getPrivateCredentials(KerberosTicket.class); + + for (KerberosTicket ticket : tickets) { + if (isTGSPrincipal(ticket.getServer())) { + return ticket; + } + } + + return null; + } + + /** + * TGS must have the server principal of the form "krbtgt/FOO@FOO". + * + * @param principal the principal to check + * @return true if the principal is the TGS, false otherwise + */ + private boolean isTGSPrincipal(final KerberosPrincipal principal) { + if (principal == null) { + return false; + } + + if (principal.getName().equals("krbtgt/" + principal.getRealm() + "@" + principal.getRealm())) { + if (LOG.isTraceEnabled()) { + LOG.trace("Found TGT principal: " + principal.getName()); + } + return true; + } + + return false; + } + + private long getRefreshTime(final KerberosTicket tgt) { + long start = tgt.getStartTime().getTime(); + long end = tgt.getEndTime().getTime(); + + if (LOG.isTraceEnabled()) { + final SimpleDateFormat dateFormat = new SimpleDateFormat(DATE_FORMAT); + final String startDate = dateFormat.format(new Date(start)); + final String endDate = dateFormat.format(new Date(end)); + LOG.trace("TGT valid starting at: " + startDate); + LOG.trace("TGT expires at: " + endDate); + } + + return start + (long) ((end - start) * TICKET_RENEW_WINDOW); + } + + /** + * @return true if this user is currently logged in, false otherwise + */ + @Override + public boolean isLoggedIn() { + return loggedIn.get(); + } + + @Override + public String toString() { + return "KerberosUser{" + + "principal='" + getPrincipal() + '\'' + + ", loggedIn=" + loggedIn + + '}'; + } +} + diff --git a/auth-audits/src/main/java/org/apache/atlas/audit/utils/InMemoryJAASConfiguration.java b/auth-audits/src/main/java/org/apache/atlas/audit/utils/InMemoryJAASConfiguration.java new file mode 100644 index 00000000000..0baf32ad4d4 --- /dev/null +++ b/auth-audits/src/main/java/org/apache/atlas/audit/utils/InMemoryJAASConfiguration.java @@ -0,0 +1,378 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.audit.utils; + +import org.apache.commons.collections.MapUtils; +import org.apache.hadoop.security.SecurityUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.security.auth.login.AppConfigurationEntry; +import javax.security.auth.login.Configuration; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.SortedSet; +import java.util.StringTokenizer; +import java.util.TreeSet; + +/** + * InMemoryJAASConfiguration + * + * An utility class - which has a static method init to load all JAAS configuration from Application properties file (eg: kafka.properties) and + * set it as part of the default lookup configuration for all JAAS configuration lookup. + * + * Example settings in application.properties: + * + * xasecure.audit.jaas.KafkaClient.loginModuleName = com.sun.security.auth.module.Krb5LoginModule + * xasecure.audit.jaas.KafkaClient.loginModuleControlFlag = required + * xasecure.audit.jaas.KafkaClient.option.useKeyTab = true + * xasecure.audit.jaas.KafkaClient.option.storeKey = true + * xasecure.audit.jaas.KafkaClient.option.serviceName = kafka + * xasecure.audit.jaas.KafkaClient.option.keyTab = /etc/security/keytabs/kafka_client.keytab + * xasecure.audit.jaas.KafkaClient.option.principal = kafka-client-1@EXAMPLE.COM + + * xasecure.audit.jaas.MyClient.0.loginModuleName = com.sun.security.auth.module.Krb5LoginModule + * xasecure.audit.jaas.MyClient.0.loginModuleControlFlag = required + * xasecure.audit.jaas.MyClient.0.option.useKeyTab = true + * xasecure.audit.jaas.MyClient.0.option.storeKey = true + * xasecure.audit.jaas.MyClient.0.option.serviceName = kafka + * xasecure.audit.jaas.MyClient.0.option.keyTab = /etc/security/keytabs/kafka_client.keytab + * xasecure.audit.jaas.MyClient.0.option.principal = kafka-client-1@EXAMPLE.COM + * + * xasecure.audit.jaas.MyClient.1.loginModuleName = com.sun.security.auth.module.Krb5LoginModule + * xasecure.audit.jaas.MyClient.1.loginModuleControlFlag = optional + * xasecure.audit.jaas.MyClient.1.option.useKeyTab = true + * xasecure.audit.jaas.MyClient.1.option.storeKey = true + * xasecure.audit.jaas.MyClient.1.option.serviceName = kafka + * xasecure.audit.jaas.MyClient.1.option.keyTab = /etc/security/keytabs/kafka_client.keytab + * xasecure.audit.jaas.MyClient.1.option.principal = kafka-client-1@EXAMPLE.COM + + * This will set the JAAS configuration - equivalent to the jaas.conf file entries: + * KafkaClient { + * com.sun.security.auth.module.Krb5LoginModule required + * useKeyTab=true + * storeKey=true + * serviceName=kafka + * keyTab="/etc/security/keytabs/kafka_client.keytab" + * principal="kafka-client-1@EXAMPLE.COM"; + * }; + * MyClient { + * com.sun.security.auth.module.Krb5LoginModule required + * useKeyTab=true + * storeKey=true + * serviceName=kafka keyTab="/etc/security/keytabs/kafka_client.keytab" + * principal="kafka-client-1@EXAMPLE.COM"; + * }; + * MyClient { + * com.sun.security.auth.module.Krb5LoginModule optional + * useKeyTab=true + * storeKey=true + * serviceName=kafka + * keyTab="/etc/security/keytabs/kafka_client.keytab" + * principal="kafka-client-1@EXAMPLE.COM"; + * }; + * + * Here is the syntax for atlas.properties to add JAAS configuration: + * + * The property name has to begin with 'xasecure.audit.jaas.' + clientId (in case of Kafka client, + * it expects the clientId to be KafkaClient). + * The following property must be there to specify the JAAS loginModule name + * 'xasecure.audit.jaas.' +' + clientId + '.loginModuleName' + * The following optional property should be set to specify the loginModuleControlFlag + * 'xasecure.audit.jaas.' +' + clientId + '.loginModuleControlFlag' + * Default value : required , Possible values: required, optional, sufficient, requisite + * Then you can add additional optional parameters as options for the configuration using the following + * syntax: + * 'xasecure.audit.jaas.' +' + clientId + '.option.' + = + * + * The current setup will lookup JAAS configration from the atlas-application.properties first, if not available, + * it will delegate to the original configuration + * + */ + +public final class InMemoryJAASConfiguration extends Configuration { + + private static final Logger LOG = LoggerFactory.getLogger(InMemoryJAASConfiguration.class); + + public static final String JAAS_CONFIG_PREFIX_PARAM = "xasecure.audit.jaas."; + public static final String JAAS_CONFIG_LOGIN_MODULE_NAME_PARAM = "loginModuleName"; + public static final String JAAS_CONFIG_LOGIN_MODULE_CONTROL_FLAG_PARAM = "loginModuleControlFlag"; + public static final String JAAS_CONFIG_LOGIN_OPTIONS_PREFIX = "option"; + public static final String JAAS_PRINCIPAL_PROP = "principal"; + + private final Configuration parent; + private final Map> applicationConfigEntryMap = new HashMap<>(); + + public static InMemoryJAASConfiguration init(String propFile) throws Exception { + LOG.debug("==> InMemoryJAASConfiguration.init( {} ) ", propFile); + + InMemoryJAASConfiguration ret = null; + InputStream in = null; + + try { + Properties properties = new Properties(); + + in = ClassLoader.getSystemResourceAsStream(propFile); + + if (in == null) { + if (!propFile.startsWith("/")) { + in = ClassLoader.getSystemResourceAsStream("/" + propFile); + } + if (in == null) { + in = new FileInputStream(new File(propFile)); + } + } + + properties.load(in); + + ret = init(properties); + } catch (IOException e) { + throw new Exception("Failed to load JAAS application properties", e); + } finally { + if ( in != null) { + try { + in.close(); + } catch ( Exception e) { + //Ignore + } + } + } + + LOG.debug("<== InMemoryJAASConfiguration.init( {} ) ", propFile); + + return ret; + } + + public static InMemoryJAASConfiguration init(Properties properties) throws Exception { + LOG.debug("==> InMemoryJAASConfiguration.init()"); + + InMemoryJAASConfiguration ret = null; + + if (properties != null && MapUtils.isNotEmpty(properties)) { + ret = new InMemoryJAASConfiguration(properties); + } else { + throw new Exception("Failed to load JAAS application properties: properties NULL or empty!"); + } + + LOG.debug("<== InMemoryJAASConfiguration.init()"); + + return ret; + } + + @Override + public AppConfigurationEntry[] getAppConfigurationEntry(String name) { + LOG.debug("==> InMemoryJAASConfiguration.getAppConfigurationEntry( {} )", name); + + AppConfigurationEntry[] ret = null; + + if (parent != null) { + ret = parent.getAppConfigurationEntry(name); + } + + if (ret == null || ret.length == 0) { + List retList = applicationConfigEntryMap.get(name); + + if (retList != null && retList.size() > 0) { + ret = retList.toArray(new AppConfigurationEntry[retList.size()]); + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== InMemoryJAASConfiguration.getAppConfigurationEntry( {} ) : {}", name, toString(ret)); + } + + return ret; + } + + private InMemoryJAASConfiguration(Properties prop) { + parent = Configuration.getConfiguration(); + + initialize(prop); + } + + private void initialize(Properties properties) { + LOG.debug("==> InMemoryJAASConfiguration.initialize()"); + + int prefixLen = JAAS_CONFIG_PREFIX_PARAM.length(); + Map> jaasClients = new HashMap<>(); + + for(String key : properties.stringPropertyNames()) { + if (key.startsWith(JAAS_CONFIG_PREFIX_PARAM)) { + String jaasKey = key.substring(prefixLen); + StringTokenizer tokenizer = new StringTokenizer(jaasKey, "."); + int tokenCount = tokenizer.countTokens(); + + if (tokenCount > 0) { + String clientId = tokenizer.nextToken(); + SortedSet indexList = jaasClients.get(clientId); + + if (indexList == null) { + indexList = new TreeSet<>(); + + jaasClients.put(clientId, indexList); + } + + String indexStr = tokenizer.nextToken(); + int indexId = isNumeric(indexStr) ? Integer.parseInt(indexStr) : -1; + Integer clientIdIndex = Integer.valueOf(indexId); + + if (!indexList.contains(clientIdIndex)) { + indexList.add(clientIdIndex); + } + } + } + } + + for(String jaasClient : jaasClients.keySet()) { + for(Integer index : jaasClients.get(jaasClient)) { + String keyPrefix = JAAS_CONFIG_PREFIX_PARAM + jaasClient + "."; + + if (index > -1) { + keyPrefix = keyPrefix + String.valueOf(index) + "."; + } + + String keyParam = keyPrefix + JAAS_CONFIG_LOGIN_MODULE_NAME_PARAM; + String loginModuleName = properties.getProperty(keyParam); + + if (loginModuleName == null) { + LOG.error("Unable to add JAAS configuration for " + + "client [" + jaasClient + "] as it is missing param [" + keyParam + "]." + + " Skipping JAAS config for [" + jaasClient + "]"); + continue; + } else { + loginModuleName = loginModuleName.trim(); + } + + keyParam = keyPrefix + JAAS_CONFIG_LOGIN_MODULE_CONTROL_FLAG_PARAM; + + String controlFlag = properties.getProperty(keyParam); + + AppConfigurationEntry.LoginModuleControlFlag loginControlFlag = null; + + if (controlFlag != null) { + controlFlag = controlFlag.trim().toLowerCase(); + + if (controlFlag.equals("optional")) { + loginControlFlag = AppConfigurationEntry.LoginModuleControlFlag.OPTIONAL; + } else if (controlFlag.equals("requisite")) { + loginControlFlag = AppConfigurationEntry.LoginModuleControlFlag.REQUISITE; + } else if (controlFlag.equals("sufficient")) { + loginControlFlag = AppConfigurationEntry.LoginModuleControlFlag.SUFFICIENT; + } else if (controlFlag.equals("required")) { + loginControlFlag = AppConfigurationEntry.LoginModuleControlFlag.REQUIRED; + } else { + String validValues = "optional|requisite|sufficient|required"; + LOG.warn("Unknown JAAS configuration value for (" + keyParam + + ") = [" + controlFlag + "], valid value are [" + validValues + + "] using the default value, REQUIRED"); + loginControlFlag = AppConfigurationEntry.LoginModuleControlFlag.REQUIRED; + } + } else { + LOG.warn("Unable to find JAAS configuration (" + + keyParam + "); using the default value, REQUIRED"); + loginControlFlag = AppConfigurationEntry.LoginModuleControlFlag.REQUIRED; + } + + Map options = new HashMap<>(); + String optionPrefix = keyPrefix + JAAS_CONFIG_LOGIN_OPTIONS_PREFIX + "."; + int optionPrefixLen = optionPrefix.length(); + + for(String key : properties.stringPropertyNames()) { + if (key.startsWith(optionPrefix)) { + String optionKey = key.substring(optionPrefixLen); + String optionVal = properties.getProperty(key); + + if (optionVal != null) { + optionVal = optionVal.trim(); + + try { + if (optionKey.equalsIgnoreCase(JAAS_PRINCIPAL_PROP)) { + optionVal = SecurityUtil.getServerPrincipal(optionVal, (String) null); + } + } catch (IOException e) { + LOG.warn("Failed to build serverPrincipal. Using provided value:[" + + optionVal + "]"); + } + } + + options.put(optionKey, optionVal); + } + } + + AppConfigurationEntry entry = new AppConfigurationEntry(loginModuleName, loginControlFlag, options); + + if (LOG.isDebugEnabled()) { + StringBuilder sb = new StringBuilder(); + + sb.append("Adding client: [").append(jaasClient).append("{").append(index).append("}]\n"); + sb.append("\tloginModule: [").append(loginModuleName).append("]\n"); + sb.append("\tcontrolFlag: [").append(loginControlFlag).append("]\n"); + + for (String key : options.keySet()) { + String val = options.get(key); + + sb.append("\tOptions: [").append(key).append("] => [").append(val).append("]\n"); + } + + LOG.debug(sb.toString()); + } + + List retList = applicationConfigEntryMap.get(jaasClient); + + if (retList == null) { + retList = new ArrayList<>(); + + applicationConfigEntryMap.put(jaasClient, retList); + } + + retList.add(entry); + } + } + + LOG.debug("<== InMemoryJAASConfiguration.initialize()"); + } + + private static boolean isNumeric(String str) { + return str.matches("-?\\d+(\\.\\d+)?"); //match a number with optional '-' and decimal. + } + + private String toString(AppConfigurationEntry[] entries) { + StringBuilder sb = new StringBuilder(); + + sb.append('['); + if (entries != null) { + for (AppConfigurationEntry entry : entries) { + sb.append("{ loginModuleName=").append(entry.getLoginModuleName()) + .append(", controlFlag=").append(entry.getControlFlag()) + .append(", options=").append(entry.getOptions()) + .append("}"); + } + } + sb.append(']'); + + return sb.toString(); + } +} diff --git a/auth-audits/src/main/java/org/apache/atlas/audit/utils/KerberosAction.java b/auth-audits/src/main/java/org/apache/atlas/audit/utils/KerberosAction.java new file mode 100644 index 00000000000..7af54484ed5 --- /dev/null +++ b/auth-audits/src/main/java/org/apache/atlas/audit/utils/KerberosAction.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.audit.utils; + +import org.apache.commons.lang3.Validate; +import org.apache.commons.logging.Log; + +import javax.security.auth.login.LoginException; +import java.security.PrivilegedActionException; +import java.security.PrivilegedExceptionAction; + +/** + * Helper class for processors to perform an action as a KerberosUser. + */ +public class KerberosAction { + + private final KerberosUser kerberosUser; + private final PrivilegedExceptionAction action; + private final Log logger; + + public KerberosAction(final KerberosUser kerberosUser, + final PrivilegedExceptionAction action, + final Log logger) { + this.kerberosUser = kerberosUser; + this.action = action; + this.logger = logger; + Validate.notNull(this.kerberosUser); + Validate.notNull(this.action); + Validate.notNull(this.logger); + } + + public T execute() throws Exception { + T result; + // lazily login the first time the processor executes + if (!kerberosUser.isLoggedIn()) { + try { + kerberosUser.login(); + logger.info("Successful login for " + kerberosUser.getPrincipal()); + } catch (LoginException e) { + throw new Exception("Login failed due to: " + e.getMessage(), e); + } + } + + // check if we need to re-login, will only happen if re-login window is reached (80% of TGT life) + try { + kerberosUser.checkTGTAndRelogin(); + } catch (LoginException e) { + throw new Exception("Relogin check failed due to: " + e.getMessage(), e); + } + + // attempt to execute the action, if an exception is caught attempt to logout/login and retry + try { + result = kerberosUser.doAs(action); + } catch (SecurityException se) { + logger.info("Privileged action failed, attempting relogin and retrying..."); + logger.debug("", se); + + try { + kerberosUser.logout(); + kerberosUser.login(); + result = kerberosUser.doAs(action); + } catch (Exception e) { + throw new Exception("Retrying privileged action failed due to: " + e.getMessage(), e); + } + } catch (PrivilegedActionException pae) { + final Exception cause = pae.getException(); + throw new Exception("Privileged action failed due to: " + cause.getMessage(), cause); + } + + return result; + } +} diff --git a/auth-audits/src/main/java/org/apache/atlas/audit/utils/KerberosJAASConfigUser.java b/auth-audits/src/main/java/org/apache/atlas/audit/utils/KerberosJAASConfigUser.java new file mode 100644 index 00000000000..5b0ff718e19 --- /dev/null +++ b/auth-audits/src/main/java/org/apache/atlas/audit/utils/KerberosJAASConfigUser.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.audit.utils; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.security.auth.Subject; +import javax.security.auth.login.AppConfigurationEntry; +import javax.security.auth.login.Configuration; +import javax.security.auth.login.LoginContext; +import javax.security.auth.login.LoginException; + +/** + * Used to authenticate and execute actions when Kerberos is enabled and a keytab is being used. + * + * */ +public class KerberosJAASConfigUser extends AbstractKerberosUser { + private static final Logger LOG = LoggerFactory.getLogger(KerberosJAASConfigUser.class); + + private final String configName; + private final Configuration config; + + public KerberosJAASConfigUser(final String configName, final Configuration config) { + this.configName = configName; + this.config = config; + } + + + @Override + public String getPrincipal() { + String ret = null; + AppConfigurationEntry[] entries = config.getAppConfigurationEntry(configName); + + if (entries != null) { + for (AppConfigurationEntry entry : entries) { + if (entry.getOptions().containsKey(InMemoryJAASConfiguration.JAAS_PRINCIPAL_PROP)) { + ret = (String) entry.getOptions().get(InMemoryJAASConfiguration.JAAS_PRINCIPAL_PROP); + + break; + } + } + } + + return ret; + } + + @Override + protected LoginContext createLoginContext(Subject subject) throws LoginException { + if (LOG.isDebugEnabled()) { + LOG.debug("==> KerberosJAASConfigUser.createLoginContext()"); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== KerberosJAASConfigUser.createLoginContext(), Subject: " + subject); + } + + return new LoginContext(configName, subject, null, config); + } +} + diff --git a/auth-audits/src/main/java/org/apache/atlas/audit/utils/KerberosUser.java b/auth-audits/src/main/java/org/apache/atlas/audit/utils/KerberosUser.java new file mode 100644 index 00000000000..8e6a95bc8f5 --- /dev/null +++ b/auth-audits/src/main/java/org/apache/atlas/audit/utils/KerberosUser.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.audit.utils; + +import javax.security.auth.login.LoginException; +import java.security.PrivilegedAction; +import java.security.PrivilegedActionException; +import java.security.PrivilegedExceptionAction; + +/** + * A keytab-based user that can login/logout and perform actions as the given user. + */ +public interface KerberosUser { + + /** + * Performs a login for the given user. + * + * @throws LoginException if the login fails + */ + void login() throws LoginException; + + /** + * Performs a logout for the given user. + * + * @throws LoginException if the logout fails + */ + void logout() throws LoginException; + + /** + * Executes the given action as the given user. + * + * @param action the action to execute + * @param the type of response + * @return the result of the action + * @throws IllegalStateException if attempting to execute an action before performing a login + */ + T doAs(PrivilegedAction action) throws IllegalStateException; + + /** + * Executes the given action as the given user. + * + * @param action the action to execute + * @param the type of response + * @return the result of the action + * @throws IllegalStateException if attempting to execute an action before performing a login + * @throws PrivilegedActionException if the action itself threw an exception + */ + T doAs(PrivilegedExceptionAction action) + throws IllegalStateException, PrivilegedActionException; + + /** + * Performs a re-login if the TGT is close to expiration. + * + * @return true if a relogin was performed, false otherwise + * @throws LoginException if the relogin fails + */ + boolean checkTGTAndRelogin() throws LoginException; + + /** + * @return true if this user is currently logged in, false otherwise + */ + boolean isLoggedIn(); + + /** + * @return the principal for this user + */ + String getPrincipal(); + +} + diff --git a/auth-audits/src/main/java/org/apache/atlas/audit/utils/RangerAuditWriter.java b/auth-audits/src/main/java/org/apache/atlas/audit/utils/RangerAuditWriter.java new file mode 100644 index 00000000000..4a34ff54ded --- /dev/null +++ b/auth-audits/src/main/java/org/apache/atlas/audit/utils/RangerAuditWriter.java @@ -0,0 +1,39 @@ +package org.apache.atlas.audit.utils; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.io.File; +import java.util.Collection; +import java.util.Map; +import java.util.Properties; + +public interface RangerAuditWriter { + void init(Properties prop, String propPrefix, String auditProviderName, Map auditConfigs); + + boolean log(Collection events) throws Exception; + + boolean logFile(File file) throws Exception; + + void start(); + + void flush(); + + void stop(); +} diff --git a/auth-audits/src/main/java/org/apache/atlas/audit/utils/RollingTimeUtil.java b/auth-audits/src/main/java/org/apache/atlas/audit/utils/RollingTimeUtil.java new file mode 100644 index 00000000000..505591e9f9e --- /dev/null +++ b/auth-audits/src/main/java/org/apache/atlas/audit/utils/RollingTimeUtil.java @@ -0,0 +1,269 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.audit.utils; + +import org.apache.commons.lang.StringUtils; + +import java.util.Calendar; +import java.util.Date; + +public class RollingTimeUtil { + public static final String MINUTES ="m"; //minutes + public static final String HOURS ="h"; //hours + public static final String DAYS ="d"; //days + public static final String WEEKS ="w"; //weeks + public static final String MONTHS ="M"; //months + public static final String YEARS ="y"; //years + + private static volatile RollingTimeUtil me = null; + + public static RollingTimeUtil getInstance() { + RollingTimeUtil result = me; + if ( result == null) { + synchronized(RollingTimeUtil.class) { + result = me; + if ( result == null){ + me = result = new RollingTimeUtil(); + } + } + } + return result; + } + + public RollingTimeUtil() { + } + + public Date computeNextRollingTime(String rollingTimePeriod) throws Exception{ + Date ret = null; + + if (!StringUtils.isEmpty(rollingTimePeriod)) { + String computePeriod = getTimeLiteral(rollingTimePeriod); + int timeNumeral = getTimeNumeral(rollingTimePeriod,computePeriod); + switch(computePeriod) { + case MINUTES: + ret = computeTopOfMinuteDate(timeNumeral); + break; + case HOURS: + ret = computeTopOfHourDate(timeNumeral); + break; + case DAYS: + ret = computeTopOfDayDate(timeNumeral); + break; + case WEEKS: + ret = computeTopOfWeekDate(timeNumeral); + break; + case MONTHS: + ret = computeTopofMonthDate(timeNumeral); + break; + case YEARS: + ret = computeTopOfYearDate(timeNumeral); + break; + } + } else { + throw new Exception("Unable to compute Next Rolling using the given Rollover period"); + } + return ret; + } + + public String convertRolloverSecondsToRolloverPeriod(long duration) { + final int SECONDS_IN_MINUTE = 60; + final int SECONDS_IN_HOUR = 60 * SECONDS_IN_MINUTE; + final int SECONDS_IN_DAY = 24 * SECONDS_IN_HOUR; + + String ret = null; + int days = (int) (duration / SECONDS_IN_DAY); + duration %= SECONDS_IN_DAY; + int hours = (int) (duration / SECONDS_IN_HOUR); + duration %= SECONDS_IN_HOUR; + int minutes = (int) (duration / SECONDS_IN_MINUTE); + + if(days != 0) { + if(hours == 0 && minutes == 0) { + ret = (days + DAYS); + } + } else if(hours != 0) { + if(minutes == 0) { + ret = (hours + HOURS); + } + } else if(minutes != 0) { + ret = (minutes + MINUTES); + } + return ret; + } + + public long computeNextRollingTime(long durationSeconds, Date previousRolloverTime) { + long now = System.currentTimeMillis(); + long nextRolloverTime = (previousRolloverTime == null) ? now : previousRolloverTime.getTime(); + long durationMillis = (durationSeconds < 1 ? 1 : durationSeconds) * 1000; + + while( nextRolloverTime <= now ) { + nextRolloverTime += durationMillis; + } + + return nextRolloverTime; + } + + private Date computeTopOfYearDate( int years){ + Date ret = null; + + Calendar calendarStart=Calendar.getInstance(); + calendarStart.add(Calendar.YEAR,years); + calendarStart.set(Calendar.MONTH,0); + calendarStart.set(Calendar.DAY_OF_MONTH,1); + calendarStart.set(Calendar.HOUR_OF_DAY,0); + calendarStart.clear(Calendar.MINUTE); + calendarStart.clear(Calendar.SECOND); + calendarStart.clear(Calendar.MILLISECOND); + + ret = calendarStart.getTime(); + + return ret; + } + + private Date computeTopofMonthDate(int months){ + + Date ret = null; + + Calendar calendarMonth=Calendar.getInstance(); + calendarMonth.set(Calendar.DAY_OF_MONTH,1); + calendarMonth.add(Calendar.MONTH, months); + calendarMonth.set(Calendar.HOUR_OF_DAY, 0); + calendarMonth.clear(Calendar.MINUTE); + calendarMonth.clear(Calendar.SECOND); + calendarMonth.clear(Calendar.MILLISECOND); + + ret = calendarMonth.getTime(); + + return ret; + } + + private Date computeTopOfWeekDate(int weeks) { + Date ret = null; + + Calendar calendarWeek=Calendar.getInstance(); + calendarWeek.set(Calendar.DAY_OF_WEEK,calendarWeek.getFirstDayOfWeek()); + calendarWeek.add(Calendar.WEEK_OF_YEAR,weeks); + calendarWeek.set(Calendar.HOUR_OF_DAY,0); + calendarWeek.clear(Calendar.MINUTE); + calendarWeek.clear(Calendar.SECOND); + calendarWeek.clear(Calendar.MILLISECOND); + + ret=calendarWeek.getTime(); + + return ret; + } + + private Date computeTopOfDayDate(int days){ + + Date ret = null; + + Calendar calendarDay=Calendar.getInstance(); + calendarDay.add(Calendar.DAY_OF_MONTH, days); + calendarDay.set(Calendar.HOUR_OF_DAY, 0); + calendarDay.clear(Calendar.MINUTE); + calendarDay.clear(Calendar.SECOND); + calendarDay.clear(Calendar.MILLISECOND); + + ret = calendarDay.getTime(); + + return ret; + + } + + private Date computeTopOfHourDate(int hours) { + Date ret = null; + + Calendar calendarHour=Calendar.getInstance(); + calendarHour.add(Calendar.HOUR_OF_DAY, hours); + calendarHour.clear(Calendar.MINUTE); + calendarHour.clear(Calendar.SECOND); + calendarHour.clear(Calendar.MILLISECOND); + + ret = calendarHour.getTime(); + + return ret; + } + + private Date computeTopOfMinuteDate(int mins) { + Date ret = null; + + Calendar calendarMin=Calendar.getInstance(); + calendarMin.add(Calendar.MINUTE,mins); + calendarMin.clear(Calendar.SECOND); + calendarMin.clear(Calendar.MILLISECOND); + + ret = calendarMin.getTime(); + + return ret; + } + + private int getTimeNumeral(String rollOverPeriod, String timeLiteral) throws Exception { + + int ret = Integer.valueOf(rollOverPeriod.substring(0, rollOverPeriod.length() - (rollOverPeriod.length() - rollOverPeriod.indexOf(timeLiteral)))); + + return ret; + } + + private String getTimeLiteral(String rollOverPeriod) throws Exception { + String ret = null; + if(StringUtils.isEmpty(rollOverPeriod)) { + throw new Exception("empty rollover period"); + } else if(rollOverPeriod.endsWith(MINUTES)) { + ret = MINUTES; + } else if(rollOverPeriod.endsWith(HOURS)) { + ret = HOURS; + } else if(rollOverPeriod.endsWith(DAYS)) { + ret = DAYS; + } else if(rollOverPeriod.endsWith(WEEKS)) { + ret = WEEKS; + } else if(rollOverPeriod.endsWith(MONTHS)) { + ret = MONTHS; + } else if(rollOverPeriod.endsWith(YEARS)) { + ret = YEARS; + } else { + throw new Exception(rollOverPeriod + ": invalid rollover period"); + } + return ret; + } + + public static void main(String[] args) { + // Test Method for RolloverTime calculation + // Set rollOverPeriod 10m,30m..,1h,2h,..1d,2d..,1w,2w..,1M,2M..1y..2y + // If nothing is set for rollOverPeriod or Duration default rollOverPeriod is 1 day + String rollOverPeriod = ""; + RollingTimeUtil rollingTimeUtil = new RollingTimeUtil(); + int duration = 86400; + Date nextRollOvertime = null; + + try { + nextRollOvertime = rollingTimeUtil.computeNextRollingTime(rollOverPeriod); + } catch (Exception e) { + rollOverPeriod = rollingTimeUtil.convertRolloverSecondsToRolloverPeriod(duration); + System.out.println(rollOverPeriod); + try { + nextRollOvertime = rollingTimeUtil.computeNextRollingTime(rollOverPeriod); + System.out.println(nextRollOvertime); + } catch (Exception e1) { + e1.printStackTrace(); + } + long rollOverTime = rollingTimeUtil.computeNextRollingTime(duration, null); + nextRollOvertime = new Date(rollOverTime); + } + } +} diff --git a/auth-audits/src/main/java/org/apache/atlas/audit/utils/SolrAppUtil.java b/auth-audits/src/main/java/org/apache/atlas/audit/utils/SolrAppUtil.java new file mode 100644 index 00000000000..c232b945fcf --- /dev/null +++ b/auth-audits/src/main/java/org/apache/atlas/audit/utils/SolrAppUtil.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.audit.utils; + +import org.apache.atlas.audit.provider.MiscUtil; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.response.UpdateResponse; +import org.apache.solr.common.SolrInputDocument; + +import java.security.PrivilegedExceptionAction; +import java.util.Collection; + +public class SolrAppUtil { + public static UpdateResponse addDocsToSolr(final SolrClient solrClient, final Collection docs) throws Exception { + return MiscUtil.executePrivilegedAction(new PrivilegedExceptionAction() { + @Override + public UpdateResponse run() throws Exception { + return solrClient.add(docs); + } + }); + } +} diff --git a/auth-plugin-atlas/pom.xml b/auth-plugin-atlas/pom.xml new file mode 100644 index 00000000000..01386ce3bff --- /dev/null +++ b/auth-plugin-atlas/pom.xml @@ -0,0 +1,51 @@ + + + + + apache-atlas + org.apache.atlas + 3.0.0-SNAPSHOT + + 4.0.0 + + auth-plugin-atlas + + + 8 + 8 + + + + + + org.apache.atlas + auth-agents-common + ${project.version} + + + + org.apache.atlas + atlas-authorization + ${project.version} + + + + \ No newline at end of file diff --git a/auth-plugin-atlas/src/main/java/org/apache/atlas/authorization/atlas/authorizer/RangerAtlasAuthorizer.java b/auth-plugin-atlas/src/main/java/org/apache/atlas/authorization/atlas/authorizer/RangerAtlasAuthorizer.java new file mode 100644 index 00000000000..dd645b473b6 --- /dev/null +++ b/auth-plugin-atlas/src/main/java/org/apache/atlas/authorization/atlas/authorizer/RangerAtlasAuthorizer.java @@ -0,0 +1,949 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.authorization.atlas.authorizer; + +import org.apache.atlas.authorize.AtlasAccessRequest; +import org.apache.atlas.authorize.AtlasAccessorResponse; +import org.apache.atlas.authorize.AtlasAdminAccessRequest; +import org.apache.atlas.authorize.AtlasAuthorizationException; +import org.apache.atlas.authorize.AtlasAuthorizer; +import org.apache.atlas.authorize.AtlasEntityAccessRequest; +import org.apache.atlas.authorize.AtlasPrivilege; +import org.apache.atlas.authorize.AtlasRelationshipAccessRequest; +import org.apache.atlas.authorize.AtlasSearchResultScrubRequest; +import org.apache.atlas.authorize.AtlasTypeAccessRequest; +import org.apache.atlas.authorize.AtlasTypesDefFilterRequest; +import org.apache.atlas.model.discovery.AtlasSearchResult; +import org.apache.atlas.model.instance.AtlasClassification; +import org.apache.atlas.model.instance.AtlasEntityHeader; +import org.apache.atlas.model.typedef.AtlasBaseTypeDef; +import org.apache.atlas.model.typedef.AtlasTypesDef; +import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.audit.model.AuthzAuditEvent; +import org.apache.atlas.plugin.audit.RangerDefaultAuditHandler; +import org.apache.atlas.plugin.contextenricher.RangerTagForEval; +import org.apache.atlas.plugin.model.RangerServiceDef; +import org.apache.atlas.plugin.model.RangerTag; +import org.apache.atlas.plugin.policyengine.RangerAccessRequestImpl; +import org.apache.atlas.plugin.policyengine.RangerAccessResourceImpl; +import org.apache.atlas.plugin.policyengine.RangerAccessResult; +import org.apache.atlas.plugin.policyresourcematcher.RangerPolicyResourceMatcher; +import org.apache.atlas.plugin.service.RangerBasePlugin; +import org.apache.atlas.plugin.util.RangerPerfTracer; + +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.ListIterator; +import java.util.Map; +import java.util.Set; + +import static org.apache.atlas.authorization.atlas.authorizer.RangerAtlasAuthorizerUtil.*; +import static org.apache.atlas.authorize.AtlasAuthorizationUtils.getCurrentUserGroups; +import static org.apache.atlas.authorize.AtlasAuthorizationUtils.getCurrentUserName; +import static org.apache.atlas.services.atlas.RangerServiceAtlas.*; + + +public class RangerAtlasAuthorizer implements AtlasAuthorizer { + private static final Log LOG = LogFactory.getLog(RangerAtlasAuthorizer.class); + private static final Log PERF_LOG = RangerPerfTracer.getPerfLogger("atlasauth.request"); + + private static volatile RangerBasePlugin atlasPlugin = null; + private static volatile RangerGroupUtil groupUtil = null; + + static final Set CLASSIFICATION_PRIVILEGES = new HashSet() {{ + add(AtlasPrivilege.ENTITY_ADD_CLASSIFICATION); + add(AtlasPrivilege.ENTITY_REMOVE_CLASSIFICATION); + add(AtlasPrivilege.ENTITY_UPDATE_CLASSIFICATION); + }}; + + @Override + public void init() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerAtlasPlugin.init()"); + } + + RangerBasePlugin plugin = atlasPlugin; + + if (plugin == null) { + synchronized (RangerAtlasPlugin.class) { + plugin = atlasPlugin; + + if (plugin == null) { + plugin = new RangerAtlasPlugin(); + plugin.init(); + + plugin.setResultProcessor(new RangerDefaultAuditHandler(plugin.getConfig())); + + atlasPlugin = plugin; + groupUtil = new RangerGroupUtil(atlasPlugin.getUserStore()); + } + } + } + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerAtlasPlugin.init()"); + } + } + + @Override + public void init(AtlasTypeRegistry typeRegistry) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerAtlasPlugin.init(typeRegistry)"); + } + + RangerBasePlugin plugin = atlasPlugin; + + if (plugin == null) { + synchronized (RangerAtlasPlugin.class) { + plugin = atlasPlugin; + + if (plugin == null) { + plugin = new RangerAtlasPlugin(typeRegistry); + + plugin.init(); + + plugin.setResultProcessor(new RangerDefaultAuditHandler(plugin.getConfig())); + + atlasPlugin = plugin; + groupUtil = new RangerGroupUtil(atlasPlugin.getUserStore()); + } + } + } + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerAtlasPlugin.init(typeRegistry)"); + } + } + + @Override + public void cleanUp() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> cleanUp "); + } + } + + @Override + public boolean isAccessAllowed(AtlasAdminAccessRequest request) throws AtlasAuthorizationException { + if (LOG.isDebugEnabled()) { + LOG.debug("==> isAccessAllowed(" + request + ")"); + } + + final boolean ret; + RangerPerfTracer perf = null; + + try { + if (RangerPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_LOG, "RangerAtlasAuthorizer.isAccessAllowed(" + request + ")"); + } + + String action = request.getAction() != null ? request.getAction().getType() : null; + RangerAccessResourceImpl rangerResource = new RangerAccessResourceImpl(Collections.singletonMap(RESOURCE_SERVICE, "*")); + RangerAccessRequestImpl rangerRequest = new RangerAccessRequestImpl(rangerResource, action, request.getUser(), request.getUserGroups(), null); + + rangerRequest.setClientIPAddress(request.getClientIPAddress()); + rangerRequest.setAccessTime(request.getAccessTime()); + rangerRequest.setAction(action); + rangerRequest.setForwardedAddresses(request.getForwardedAddresses()); + rangerRequest.setRemoteIPAddress(request.getRemoteIPAddress()); + + + ret = checkAccess(rangerRequest); + } finally { + RangerPerfTracer.log(perf); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== isAccessAllowed(" + request + "): " + ret); + } + + return ret; + } + + @Override + public boolean isAccessAllowed(AtlasEntityAccessRequest request) throws AtlasAuthorizationException { + if (LOG.isDebugEnabled()) { + LOG.debug("==> isAccessAllowed(" + request + ")"); + } + + boolean ret = true; + RangerPerfTracer perf = null; + RangerAtlasAuditHandler auditHandler = null; + + try { + if (RangerPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_LOG, "RangerAtlasAuthorizer.isAccessAllowed(" + request + ")"); + } + + // not initializing audit handler, so that audits are not logged when entity details are NULL or EMPTY STRING + if (!(StringUtils.isEmpty(request.getEntityId()) && request.getClassification() == null && request.getEntity() == null)) { + auditHandler = new RangerAtlasAuditHandler(request, getServiceDef()); + } + + ret = isAccessAllowed(request, auditHandler); + } finally { + RangerPerfTracer.log(perf); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== isAccessAllowed(" + request + "): " + ret); + } + + return ret; + } + + @Override + public boolean isAccessAllowed(AtlasTypeAccessRequest request) throws AtlasAuthorizationException { + if (LOG.isDebugEnabled()) { + LOG.debug("==> isAccessAllowed(" + request + ")"); + } + + final boolean ret; + RangerPerfTracer perf = null; + + try { + if (RangerPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_LOG, "RangerAtlasAuthorizer.isAccessAllowed(" + request + ")"); + } + + final String typeName = request.getTypeDef() != null ? request.getTypeDef().getName() : null; + final String typeCategory = request.getTypeDef() != null && request.getTypeDef().getCategory() != null ? request.getTypeDef().getCategory().name() : null; + final String action = request.getAction() != null ? request.getAction().getType() : null; + + RangerAccessResourceImpl rangerResource = new RangerAccessResourceImpl(); + + rangerResource.setValue(RESOURCE_TYPE_NAME, typeName); + rangerResource.setValue(RESOURCE_TYPE_CATEGORY, typeCategory); + + RangerAccessRequestImpl rangerRequest = new RangerAccessRequestImpl(rangerResource, action, request.getUser(), request.getUserGroups(), null); + rangerRequest.setClientIPAddress(request.getClientIPAddress()); + rangerRequest.setAccessTime(request.getAccessTime()); + rangerRequest.setAction(action); + rangerRequest.setForwardedAddresses(request.getForwardedAddresses()); + rangerRequest.setRemoteIPAddress(request.getRemoteIPAddress()); + + boolean isAuditDisabled = ACCESS_TYPE_TYPE_READ.equalsIgnoreCase(action); + + if (isAuditDisabled) { + ret = checkAccess(rangerRequest, null); + } else { + ret = checkAccess(rangerRequest); + } + + } finally { + RangerPerfTracer.log(perf); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== isAccessAllowed(" + request + "): " + ret); + } + + return ret; + } + + @Override + public boolean isAccessAllowed(AtlasRelationshipAccessRequest request) throws AtlasAuthorizationException { + if (LOG.isDebugEnabled()) { + LOG.debug("==> isAccessAllowed(" + request + ")"); + } + + boolean ret; + RangerPerfTracer perf = null; + + try { + if (RangerPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_LOG, "RangerAtlasAuthorizer.isAccessAllowed(" + request + ")"); + } + + final String action = request.getAction() != null ? request.getAction().getType() : null; + final Set end1EntityTypeAndSuperTypes = request.getEnd1EntityTypeAndAllSuperTypes(); + final Set end1Classifications = new HashSet(request.getEnd1EntityClassifications()); + final String end1EntityId = request.getEnd1EntityId(); + + final Set end2EntityTypeAndSuperTypes = request.getEnd2EntityTypeAndAllSuperTypes(); + final Set end2Classifications = new HashSet(request.getEnd2EntityClassifications()); + final String end2EntityId = request.getEnd2EntityId(); + + + String relationShipType = request.getRelationshipType(); + + RangerAccessResourceImpl rangerResource = new RangerAccessResourceImpl(); + + RangerAccessRequestImpl rangerRequest = new RangerAccessRequestImpl(rangerResource, action, request.getUser(), request.getUserGroups(), null); + rangerRequest.setClientIPAddress(request.getClientIPAddress()); + rangerRequest.setAccessTime(request.getAccessTime()); + rangerRequest.setAction(action); + rangerRequest.setForwardedAddresses(request.getForwardedAddresses()); + rangerRequest.setRemoteIPAddress(request.getRemoteIPAddress()); + + rangerResource.setValue(RESOURCE_RELATIONSHIP_TYPE, relationShipType); + + + Set classificationsWithSuperTypesEnd1 = new HashSet(); + + for (AtlasClassification classificationToAuthorize : end1Classifications) { + classificationsWithSuperTypesEnd1.addAll(request.getClassificationTypeAndAllSuperTypes(classificationToAuthorize.getTypeName())); + } + + rangerResource.setValue(RESOURCE_END_ONE_ENTITY_TYPE, end1EntityTypeAndSuperTypes); + rangerResource.setValue(RESOURCE_END_ONE_ENTITY_CLASSIFICATION, classificationsWithSuperTypesEnd1); + rangerResource.setValue(RESOURCE_END_ONE_ENTITY_ID, end1EntityId); + + + Set classificationsWithSuperTypesEnd2 = new HashSet(); + + for (AtlasClassification classificationToAuthorize : end2Classifications) { + classificationsWithSuperTypesEnd2.addAll(request.getClassificationTypeAndAllSuperTypes(classificationToAuthorize.getTypeName())); + } + + rangerResource.setValue(RESOURCE_END_TWO_ENTITY_TYPE, end2EntityTypeAndSuperTypes); + rangerResource.setValue(RESOURCE_END_TWO_ENTITY_CLASSIFICATION, classificationsWithSuperTypesEnd2); + rangerResource.setValue(RESOURCE_END_TWO_ENTITY_ID, end2EntityId); + + ret = checkAccess(rangerRequest); + + if (!ret) { // if resource based policy access not available fallback to check tag-based access. + setClassificationsToRequestContext(end1Classifications, rangerRequest); + ret = checkAccess(rangerRequest); // tag-based check with end1 classification + LOG.info("End1 checkAccess " + ret); + if (ret) { // + setClassificationsToRequestContext(end2Classifications, rangerRequest); + ret = checkAccess(rangerRequest); // tag-based check with end2 classification + LOG.info("End2 checkAccess " + ret); + } + } + + } finally { + RangerPerfTracer.log(perf); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== isAccessAllowed(" + request + "): " + ret); + } + + return ret; + } + + @Override + public AtlasAccessorResponse getAccessors(AtlasEntityAccessRequest request) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> getAccessors(" + request + ")"); + } + + AtlasAccessorResponse ret = new AtlasAccessorResponse(); + RangerPerfTracer perf = null; + + try { + if (RangerPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_LOG, "RangerAtlasAuthorizer.getAccessors(" + request + ")"); + } + + final RangerAccessResourceImpl rangerResource = new RangerAccessResourceImpl(); + final RangerAccessRequestImpl rangerRequest = new RangerAccessRequestImpl(); + + toRangerRequest(request, rangerRequest, rangerResource); + rangerRequest.setAccessorsRequested(true); + + RangerAccessResult result = null; + Set tagNames = request.getEntityClassifications(); + if (CollectionUtils.isNotEmpty(tagNames)) { + setClassificationsToRequestContext(tagNames, rangerRequest); + + // check authorization for each classification + for (AtlasClassification classificationToAuthorize : tagNames) { + rangerResource.setValue(RESOURCE_ENTITY_CLASSIFICATION, request.getClassificationTypeAndAllSuperTypes(classificationToAuthorize.getTypeName())); + + result = getAccessors(rangerRequest); + collectAccessors(result, ret); + } + } else { + rangerResource.setValue(RESOURCE_ENTITY_CLASSIFICATION, ENTITY_NOT_CLASSIFIED); + + result = getAccessors(rangerRequest); + collectAccessors(result, ret); + } + } finally { + RangerPerfTracer.log(perf); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== getAccessors(" + request + "): " + ret); + } + return ret; + } + + @Override + public AtlasAccessorResponse getAccessors(AtlasRelationshipAccessRequest request) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> getAccessors(" + request + ")"); + } + + AtlasAccessorResponse ret = new AtlasAccessorResponse(); + RangerPerfTracer perf = null; + RangerAccessResult result = null; + + try { + if (RangerPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_LOG, "RangerAtlasAuthorizer.getAccessors(" + request + ")"); + } + + final String action = request.getAction() != null ? request.getAction().getType() : null; + RangerAccessResourceImpl rangerResource = new RangerAccessResourceImpl(); + RangerAccessRequestImpl rangerRequest = new RangerAccessRequestImpl(rangerResource, action, request.getUser(), + request.getUserGroups(), null); + rangerRequest.setAccessorsRequested(true); + + + final Set end1EntityTypeAndSuperTypes = request.getEnd1EntityTypeAndAllSuperTypes(); + final Set end1Classifications = new HashSet(request.getEnd1EntityClassifications()); + final String end1EntityId = request.getEnd1EntityId(); + + final Set end2EntityTypeAndSuperTypes = request.getEnd2EntityTypeAndAllSuperTypes(); + final Set end2Classifications = new HashSet(request.getEnd2EntityClassifications()); + final String end2EntityId = request.getEnd2EntityId(); + + + String relationShipType = request.getRelationshipType(); + + + rangerRequest.setClientIPAddress(request.getClientIPAddress()); + rangerRequest.setAccessTime(request.getAccessTime()); + rangerRequest.setAction(action); + rangerRequest.setForwardedAddresses(request.getForwardedAddresses()); + rangerRequest.setRemoteIPAddress(request.getRemoteIPAddress()); + + rangerResource.setValue(RESOURCE_RELATIONSHIP_TYPE, relationShipType); + + + Set classificationsWithSuperTypesEnd1 = new HashSet(); + + for (AtlasClassification classificationToAuthorize : end1Classifications) { + classificationsWithSuperTypesEnd1.addAll(request.getClassificationTypeAndAllSuperTypes(classificationToAuthorize.getTypeName())); + } + + rangerResource.setValue(RESOURCE_END_ONE_ENTITY_TYPE, end1EntityTypeAndSuperTypes); + rangerResource.setValue(RESOURCE_END_ONE_ENTITY_CLASSIFICATION, classificationsWithSuperTypesEnd1); + rangerResource.setValue(RESOURCE_END_ONE_ENTITY_ID, end1EntityId); + + + Set classificationsWithSuperTypesEnd2 = new HashSet(); + + for (AtlasClassification classificationToAuthorize : end2Classifications) { + classificationsWithSuperTypesEnd2.addAll(request.getClassificationTypeAndAllSuperTypes(classificationToAuthorize.getTypeName())); + } + + rangerResource.setValue(RESOURCE_END_TWO_ENTITY_TYPE, end2EntityTypeAndSuperTypes); + rangerResource.setValue(RESOURCE_END_TWO_ENTITY_CLASSIFICATION, classificationsWithSuperTypesEnd2); + rangerResource.setValue(RESOURCE_END_TWO_ENTITY_ID, end2EntityId); + + result = getAccessors(rangerRequest); + collectAccessors(result, ret); + + // Check tag-based access. + setClassificationsToRequestContext(end1Classifications, rangerRequest); + RangerAccessResult resultEnd1 = getAccessors(rangerRequest); // tag-based accessors with end1 classification + + setClassificationsToRequestContext(end2Classifications, rangerRequest); + RangerAccessResult resultEnd2 = getAccessors(rangerRequest); // tag-based accessors with end2 classification + collectAccessors(resultEnd1, resultEnd2, ret); + } finally { + RangerPerfTracer.log(perf); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== getAccessors(" + request + "): " + ret); + } + + return ret; + } + + @Override + public AtlasAccessorResponse getAccessors(AtlasTypeAccessRequest request) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> getAccessors(" + request + ")"); + } + + AtlasAccessorResponse ret = new AtlasAccessorResponse(); + RangerPerfTracer perf = null; + + try { + if (RangerPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_LOG, "RangerAtlasAuthorizer.getAccessors(" + request + ")"); + } + + final String action = request.getAction() != null ? request.getAction().getType() : null; + + RangerAccessResourceImpl rangerResource = new RangerAccessResourceImpl(); + RangerAccessRequestImpl rangerRequest = new RangerAccessRequestImpl(rangerResource, action, request.getUser(), request.getUserGroups(), null); + rangerRequest.setAccessorsRequested(true); + + + final String typeName = request.getTypeDef() != null ? request.getTypeDef().getName() : null; + final String typeCategory = request.getTypeDef() != null && request.getTypeDef().getCategory() != null ? request.getTypeDef().getCategory().name() : null; + + rangerResource.setValue(RESOURCE_TYPE_NAME, typeName); + rangerResource.setValue(RESOURCE_TYPE_CATEGORY, typeCategory); + + rangerRequest.setClientIPAddress(request.getClientIPAddress()); + rangerRequest.setAccessTime(request.getAccessTime()); + rangerRequest.setAction(action); + rangerRequest.setForwardedAddresses(request.getForwardedAddresses()); + rangerRequest.setRemoteIPAddress(request.getRemoteIPAddress()); + + + RangerAccessResult result = getAccessors(rangerRequest); + collectAccessors(result, ret); + + } finally { + RangerPerfTracer.log(perf); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== getAccessors(" + request + "): " + ret); + } + return ret; + } + + @Override + public Set getRolesForCurrentUser(String userName, Set groups) { + Set ret = new HashSet<>(); + + RangerBasePlugin plugin = atlasPlugin; + + ret = plugin.getRolesFromUserAndGroups(userName, groups); + + return ret; + } + + @Override + public void scrubSearchResults(AtlasSearchResultScrubRequest request) throws AtlasAuthorizationException { + if (LOG.isDebugEnabled()) { + LOG.debug("==> scrubSearchResults(" + request + ")"); + } + + RangerPerfTracer perf = null; + + try { + if (RangerPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_LOG, "RangerAtlasAuthorizer.scrubSearchResults(" + request + ")"); + } + + final AtlasSearchResult result = request.getSearchResult(); + + if (CollectionUtils.isNotEmpty(result.getEntities())) { + for (AtlasEntityHeader entity : result.getEntities()) { + checkAccessAndScrub(entity, request); + } + } + + if (CollectionUtils.isNotEmpty(result.getFullTextResult())) { + for (AtlasSearchResult.AtlasFullTextResult fullTextResult : result.getFullTextResult()) { + if (fullTextResult != null) { + checkAccessAndScrub(fullTextResult.getEntity(), request); + } + } + } + + if (MapUtils.isNotEmpty(result.getReferredEntities())) { + for (AtlasEntityHeader entity : result.getReferredEntities().values()) { + checkAccessAndScrub(entity, request); + } + } + } finally { + RangerPerfTracer.log(perf); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== scrubSearchResults(): " + request); + } + } + + @Override + public void scrubSearchResults(AtlasSearchResultScrubRequest request, boolean isScrubAuditEnabled) throws AtlasAuthorizationException { + if (LOG.isDebugEnabled()) + LOG.debug("==> scrubSearchResults(" + request + " " + isScrubAuditEnabled); + RangerPerfTracer perf = null; + try { + if (RangerPerfTracer.isPerfTraceEnabled(PERF_LOG)) + perf = RangerPerfTracer.getPerfTracer(PERF_LOG, "RangerAtlasAuthorizer.scrubSearchResults(" + request + ")"); + AtlasSearchResult result = request.getSearchResult(); + if (CollectionUtils.isNotEmpty(result.getEntities())) { + for (AtlasEntityHeader entity : result.getEntities()) { + checkAccessAndScrub(entity, request, isScrubAuditEnabled); + } + } + if (CollectionUtils.isNotEmpty(result.getFullTextResult())) { + for (AtlasSearchResult.AtlasFullTextResult fullTextResult : result.getFullTextResult()) { + if (fullTextResult != null) + checkAccessAndScrub(fullTextResult.getEntity(), request, isScrubAuditEnabled); + } + } + if (MapUtils.isNotEmpty(result.getReferredEntities())) { + for (AtlasEntityHeader entity : result.getReferredEntities().values()) { + checkAccessAndScrub(entity, request, isScrubAuditEnabled); + } + } + } finally { + RangerPerfTracer.log(perf); + } + if (LOG.isDebugEnabled()) + LOG.debug("<== scrubSearchResults(): " + request + " " + isScrubAuditEnabled); + } + + @Override + public void filterTypesDef(AtlasTypesDefFilterRequest request) throws AtlasAuthorizationException { + + AtlasTypesDef typesDef = request.getTypesDef(); + + filterTypes(request, typesDef.getEnumDefs()); + filterTypes(request, typesDef.getStructDefs()); + filterTypes(request, typesDef.getEntityDefs()); + filterTypes(request, typesDef.getClassificationDefs()); + filterTypes(request, typesDef.getRelationshipDefs()); + filterTypes(request, typesDef.getBusinessMetadataDefs()); + + } + + private void filterTypes(AtlasAccessRequest request, List typeDefs)throws AtlasAuthorizationException { + if (typeDefs != null) { + for (ListIterator iter = typeDefs.listIterator(); iter.hasNext();) { + AtlasBaseTypeDef typeDef = iter.next(); + AtlasTypeAccessRequest typeRequest = new AtlasTypeAccessRequest(request.getAction(), typeDef, request.getUser(), request.getUserGroups()); + + typeRequest.setClientIPAddress(request.getClientIPAddress()); + typeRequest.setForwardedAddresses(request.getForwardedAddresses()); + typeRequest.setRemoteIPAddress(request.getRemoteIPAddress()); + + if (!isAccessAllowed(typeRequest)) { + iter.remove(); + } + } + } + } + + + private RangerServiceDef getServiceDef() { + RangerBasePlugin plugin = atlasPlugin; + + return plugin != null ? plugin.getServiceDef() : null; + } + + private boolean isAccessAllowed(AtlasEntityAccessRequest request, RangerAtlasAuditHandler auditHandler) throws AtlasAuthorizationException { + if (LOG.isDebugEnabled()) { + LOG.debug("==> isAccessAllowed(" + request + ")"); + } + boolean ret = false; + + try { + final String action = request.getAction() != null ? request.getAction().getType() : null; + final Set entityTypes = request.getEntityTypeAndAllSuperTypes(); + final String entityId = request.getEntityId(); + final String classification = request.getClassification() != null ? request.getClassification().getTypeName() : null; + final RangerAccessRequestImpl rangerRequest = new RangerAccessRequestImpl(); + final RangerAccessResourceImpl rangerResource = new RangerAccessResourceImpl(); + final String ownerUser = request.getEntity() != null ? (String) request.getEntity().getAttribute(RESOURCE_ENTITY_OWNER) : null; + + rangerResource.setValue(RESOURCE_ENTITY_TYPE, entityTypes); + rangerResource.setValue(RESOURCE_ENTITY_ID, entityId); + rangerResource.setOwnerUser(ownerUser); + rangerRequest.setAccessType(action); + rangerRequest.setAction(action); + rangerRequest.setUser(request.getUser()); + rangerRequest.setUserGroups(request.getUserGroups()); + rangerRequest.setClientIPAddress(request.getClientIPAddress()); + rangerRequest.setAccessTime(request.getAccessTime()); + rangerRequest.setResource(rangerResource); + rangerRequest.setForwardedAddresses(request.getForwardedAddresses()); + rangerRequest.setRemoteIPAddress(request.getRemoteIPAddress()); + + if (AtlasPrivilege.ENTITY_ADD_LABEL.equals(request.getAction()) || AtlasPrivilege.ENTITY_REMOVE_LABEL.equals(request.getAction())) { + rangerResource.setValue(RESOURCE_ENTITY_LABEL, request.getLabel()); + } else if (AtlasPrivilege.ENTITY_UPDATE_BUSINESS_METADATA.equals(request.getAction())) { + rangerResource.setValue(RESOURCE_ENTITY_BUSINESS_METADATA, request.getBusinessMetadata()); + } else if (StringUtils.isNotEmpty(classification) && CLASSIFICATION_PRIVILEGES.contains(request.getAction())) { + rangerResource.setValue(RESOURCE_CLASSIFICATION, request.getClassificationTypeAndAllSuperTypes(classification)); + } + + if (CollectionUtils.isNotEmpty(request.getEntityClassifications())) { + Set entityClassifications = request.getEntityClassifications(); + Map contextOjb = rangerRequest.getContext(); + + Set rangerTagForEval = getRangerServiceTag(entityClassifications); + + if (contextOjb == null) { + Map contextOjb1 = new HashMap(); + contextOjb1.put("CLASSIFICATIONS", rangerTagForEval); + rangerRequest.setContext(contextOjb1); + } else { + contextOjb.put("CLASSIFICATIONS", rangerTagForEval); + rangerRequest.setContext(contextOjb); + } + + // check authorization for each classification + for (AtlasClassification classificationToAuthorize : request.getEntityClassifications()) { + rangerResource.setValue(RESOURCE_ENTITY_CLASSIFICATION, request.getClassificationTypeAndAllSuperTypes(classificationToAuthorize.getTypeName())); + + ret = checkAccess(rangerRequest, auditHandler); + + if (!ret) { + break; + } + } + } else { + rangerResource.setValue(RESOURCE_ENTITY_CLASSIFICATION, ENTITY_NOT_CLASSIFIED ); + + ret = checkAccess(rangerRequest, auditHandler); + } + + } finally { + if(auditHandler != null) { + auditHandler.flushAudit(); + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== isAccessAllowed(" + request + "): " + ret); + } + + return ret; + } + + + private void setClassificationsToRequestContext(Set entityClassifications, RangerAccessRequestImpl rangerRequest) { + Map contextOjb = rangerRequest.getContext(); + + Set rangerTagForEval = getRangerServiceTag(entityClassifications); + + if (contextOjb == null) { + Map contextOjb1 = new HashMap(); + contextOjb1.put("CLASSIFICATIONS", rangerTagForEval); + rangerRequest.setContext(contextOjb1); + } else { + contextOjb.put("CLASSIFICATIONS", rangerTagForEval); + rangerRequest.setContext(contextOjb); + } + } + + Set getRangerServiceTag(Set classifications) { + Set atlasClassificationSet = new HashSet<>(); + for (AtlasClassification classification : classifications) { + RangerTag rangerTag = new RangerTag(null, classification.getTypeName(), getClassificationAttributes(classification), RangerTag.OWNER_SERVICERESOURCE); + RangerTagForEval tagForEval = new RangerTagForEval(rangerTag, RangerPolicyResourceMatcher.MatchType.SELF); + atlasClassificationSet.add(tagForEval); + } + return atlasClassificationSet; + } + + private Map getClassificationAttributes(AtlasClassification classification) { + Map attributes = classification.getAttributes(); + final Map result = new HashMap(); + if(attributes!=null) { + for (final Map.Entry entry : attributes.entrySet()) { + result.put(entry.getKey(), String.valueOf(entry.getValue())); + } + } + return result; + } + + private boolean checkAccess(RangerAccessRequestImpl request) { + boolean ret = false; + String userName = request.getUser(); + RangerBasePlugin plugin = atlasPlugin; + + if (plugin != null) { + + groupUtil.setUserStore(atlasPlugin.getUserStore()); + + request.setUserGroups(groupUtil.getContainedGroups(userName)); + + if (LOG.isDebugEnabled()) { + LOG.debug("Setting UserGroup for user: " + userName + " Groups: " + groupUtil.getContainedGroups(userName)); + } + + RangerAccessResult result = plugin.isAccessAllowed(request); + + ret = result != null && result.getIsAllowed(); + + } else { + LOG.warn("RangerAtlasPlugin not initialized. Access blocked!!!"); + } + + return ret; + } + + private boolean checkAccess(RangerAccessRequestImpl request, RangerAtlasAuditHandler auditHandler) { + boolean ret = false; + + RangerBasePlugin plugin = atlasPlugin; + String userName = request.getUser(); + + if (plugin != null) { + + groupUtil.setUserStore(atlasPlugin.getUserStore()); + + request.setUserGroups(groupUtil.getContainedGroups(userName)); + + if (LOG.isDebugEnabled()) { + LOG.debug("Setting UserGroup for user :" + userName + " Groups: " + groupUtil.getContainedGroups(userName)); + } + + RangerAccessResult result = plugin.isAccessAllowed(request, auditHandler); + + ret = result != null && result.getIsAllowed(); + + } else { + LOG.warn("RangerAtlasPlugin not initialized. Access blocked!!!"); + } + + return ret; + } + + private RangerAccessResult getAccessors(RangerAccessRequestImpl request) { + RangerAccessResult result = null; + + RangerBasePlugin plugin = atlasPlugin; + String userName = request.getUser(); + + if (plugin != null) { + + groupUtil.setUserStore(atlasPlugin.getUserStore()); + request.setUserGroups(groupUtil.getContainedGroups(userName)); + + if (LOG.isDebugEnabled()) { + LOG.debug("Setting UserGroup for user :" + userName + " Groups: " + groupUtil.getContainedGroups(userName)); + } + + result = plugin.getAssetAccessors(request); + + } else { + LOG.warn("RangerAtlasPlugin not initialized. Could not find Accessors!!!"); + } + + return result; + } + + private void checkAccessAndScrub(AtlasEntityHeader entity, AtlasSearchResultScrubRequest request) throws AtlasAuthorizationException { + if (entity != null && request != null) { + final AtlasEntityAccessRequest entityAccessRequest = new AtlasEntityAccessRequest(request.getTypeRegistry(), AtlasPrivilege.ENTITY_READ, entity, request.getUser(), request.getUserGroups()); + + entityAccessRequest.setClientIPAddress(request.getClientIPAddress()); + entityAccessRequest.setForwardedAddresses(request.getForwardedAddresses()); + entityAccessRequest.setRemoteIPAddress(request.getRemoteIPAddress()); + + if (!isAccessAllowed(entityAccessRequest, null)) { + scrubEntityHeader(entity, request.getTypeRegistry()); + } + } + } + + private void checkAccessAndScrub(AtlasEntityHeader entity, AtlasSearchResultScrubRequest request, boolean isScrubAuditEnabled) throws AtlasAuthorizationException { + if (entity != null && request != null) { + final AtlasEntityAccessRequest entityAccessRequest = new AtlasEntityAccessRequest(request.getTypeRegistry(), AtlasPrivilege.ENTITY_READ, entity, request.getUser(), request.getUserGroups()); + + entityAccessRequest.setClientIPAddress(request.getClientIPAddress()); + entityAccessRequest.setForwardedAddresses(request.getForwardedAddresses()); + entityAccessRequest.setRemoteIPAddress(request.getRemoteIPAddress()); + + boolean isEntityAccessAllowed = isScrubAuditEnabled ? isAccessAllowed(entityAccessRequest) : isAccessAllowed(entityAccessRequest, null); + if (!isEntityAccessAllowed) { + scrubEntityHeader(entity, request.getTypeRegistry()); + } + } + } + + class RangerAtlasPlugin extends RangerBasePlugin { + RangerAtlasPlugin() { + super("atlas", "atlas"); + } + + RangerAtlasPlugin(AtlasTypeRegistry typeRegistry) { + super("atlas", "atlas", typeRegistry); + } + } + + class RangerAtlasAuditHandler extends RangerDefaultAuditHandler { + private final Map auditEvents; + private final String resourcePath; + private boolean denyExists = false; + + public RangerAtlasAuditHandler(AtlasEntityAccessRequest request, RangerServiceDef serviceDef) { + Collection classifications = request.getEntityClassifications(); + String strClassifications = classifications == null ? "[]" : classifications.toString(); + + if (request.getClassification() != null) { + strClassifications += ("," + request.getClassification().getTypeName()); + } + + RangerAccessResourceImpl rangerResource = new RangerAccessResourceImpl(); + + rangerResource.setServiceDef(serviceDef); + rangerResource.setValue(RESOURCE_ENTITY_TYPE, request.getEntityType()); + rangerResource.setValue(RESOURCE_ENTITY_CLASSIFICATION, strClassifications); + rangerResource.setValue(RESOURCE_ENTITY_ID, request.getEntityId()); + + if (AtlasPrivilege.ENTITY_ADD_LABEL.equals(request.getAction()) || AtlasPrivilege.ENTITY_REMOVE_LABEL.equals(request.getAction())) { + rangerResource.setValue(RESOURCE_ENTITY_LABEL, "label=" + request.getLabel()); + } else if (AtlasPrivilege.ENTITY_UPDATE_BUSINESS_METADATA.equals(request.getAction())) { + rangerResource.setValue(RESOURCE_ENTITY_BUSINESS_METADATA, "business-metadata=" + request.getBusinessMetadata()); + } + + auditEvents = new HashMap<>(); + resourcePath = rangerResource.getAsString(); + } + + @Override + public void processResult(RangerAccessResult result) { + if (denyExists) { // nothing more to do, if a deny already encountered + return; + } + + AuthzAuditEvent auditEvent = super.getAuthzEvents(result); + + if (auditEvent != null) { + // audit event might have list of entity-types and classification-types; overwrite with the values in original request + if (resourcePath != null) { + auditEvent.setResourcePath(resourcePath); + } + + if (!result.getIsAllowed()) { + denyExists = true; + + auditEvents.clear(); + } + + auditEvents.put(auditEvent.getPolicyId() + auditEvent.getAccessType(), auditEvent); + } + } + + + public void flushAudit() { + if (auditEvents != null) { + for (AuthzAuditEvent auditEvent : auditEvents.values()) { + logAuthzAudit(auditEvent); + } + } + } + } +} diff --git a/auth-plugin-atlas/src/main/java/org/apache/atlas/authorization/atlas/authorizer/RangerAtlasAuthorizerUtil.java b/auth-plugin-atlas/src/main/java/org/apache/atlas/authorization/atlas/authorizer/RangerAtlasAuthorizerUtil.java new file mode 100644 index 00000000000..c78e986581a --- /dev/null +++ b/auth-plugin-atlas/src/main/java/org/apache/atlas/authorization/atlas/authorizer/RangerAtlasAuthorizerUtil.java @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.authorization.atlas.authorizer; + +import org.apache.atlas.authorize.AtlasAccessorResponse; +import org.apache.atlas.authorize.AtlasEntityAccessRequest; +import org.apache.atlas.authorize.AtlasPrivilege; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.atlas.plugin.model.RangerPolicy; +import org.apache.atlas.plugin.policyengine.RangerAccessRequestImpl; +import org.apache.atlas.plugin.policyengine.RangerAccessResourceImpl; +import org.apache.atlas.plugin.policyengine.RangerAccessResult; +import org.apache.atlas.plugin.policyevaluator.RangerPolicyItemEvaluator; + +import java.util.Set; + +import static org.apache.atlas.authorization.atlas.authorizer.RangerAtlasAuthorizer.CLASSIFICATION_PRIVILEGES; +import static org.apache.atlas.services.atlas.RangerServiceAtlas.RESOURCE_CLASSIFICATION; +import static org.apache.atlas.services.atlas.RangerServiceAtlas.RESOURCE_ENTITY_BUSINESS_METADATA; +import static org.apache.atlas.services.atlas.RangerServiceAtlas.RESOURCE_ENTITY_ID; +import static org.apache.atlas.services.atlas.RangerServiceAtlas.RESOURCE_ENTITY_LABEL; +import static org.apache.atlas.services.atlas.RangerServiceAtlas.RESOURCE_ENTITY_OWNER; +import static org.apache.atlas.services.atlas.RangerServiceAtlas.RESOURCE_ENTITY_TYPE; + + +public class RangerAtlasAuthorizerUtil { + + static void toRangerRequest(AtlasEntityAccessRequest request, RangerAccessRequestImpl rangerRequest, RangerAccessResourceImpl rangerResource){ + + final String action = request.getAction() != null ? request.getAction().getType() : null; + final Set entityTypes = request.getEntityTypeAndAllSuperTypes(); + final String entityId = request.getEntityId(); + final String classification = request.getClassification() != null ? request.getClassification().getTypeName() : null; + final String ownerUser = request.getEntity() != null ? (String) request.getEntity().getAttribute(RESOURCE_ENTITY_OWNER) : null; + + rangerResource.setValue(RESOURCE_ENTITY_TYPE, entityTypes); + rangerResource.setValue(RESOURCE_ENTITY_ID, entityId); + rangerResource.setOwnerUser(ownerUser); + rangerRequest.setAccessType(action); + rangerRequest.setAction(action); + rangerRequest.setUser(request.getUser()); + rangerRequest.setUserGroups(request.getUserGroups()); + rangerRequest.setClientIPAddress(request.getClientIPAddress()); + rangerRequest.setAccessTime(request.getAccessTime()); + rangerRequest.setResource(rangerResource); + rangerRequest.setForwardedAddresses(request.getForwardedAddresses()); + rangerRequest.setRemoteIPAddress(request.getRemoteIPAddress()); + + if (AtlasPrivilege.ENTITY_ADD_LABEL.equals(request.getAction()) || AtlasPrivilege.ENTITY_REMOVE_LABEL.equals(request.getAction())) { + rangerResource.setValue(RESOURCE_ENTITY_LABEL, request.getLabel()); + } else if (AtlasPrivilege.ENTITY_UPDATE_BUSINESS_METADATA.equals(request.getAction())) { + rangerResource.setValue(RESOURCE_ENTITY_BUSINESS_METADATA, request.getBusinessMetadata()); + } else if (StringUtils.isNotEmpty(classification) && CLASSIFICATION_PRIVILEGES.contains(request.getAction())) { + rangerResource.setValue(RESOURCE_CLASSIFICATION, request.getClassificationTypeAndAllSuperTypes(classification)); + } + } + + static void collectAccessors(RangerAccessResult result, AtlasAccessorResponse response) { + if (result != null && CollectionUtils.isNotEmpty(result.getMatchedItemEvaluators())) { + + result.getMatchedItemEvaluators().forEach(x -> { + collectSubjects(response, x); + }); + } + } + + static private void collectSubjects(AtlasAccessorResponse response, RangerPolicyItemEvaluator evaluator) { + + RangerPolicy.RangerPolicyItem policyItem = evaluator.getPolicyItem(); + + if (evaluator.getPolicyItemType() == RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_ALLOW) { + response.getUsers().addAll(policyItem.getUsers()); + response.getRoles().addAll(policyItem.getRoles()); + response.getGroups().addAll(policyItem.getGroups()); + + } else if (evaluator.getPolicyItemType() == RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_DENY) { + response.getDenyUsers().addAll(policyItem.getUsers()); + response.getDenyRoles().addAll(policyItem.getRoles()); + response.getDenyGroups().addAll(policyItem.getGroups()); + } + } + + static void collectAccessors(RangerAccessResult resultEnd1, RangerAccessResult resultEnd2, AtlasAccessorResponse accessorResponse) { + + if (resultEnd2 == null || CollectionUtils.isEmpty(resultEnd2.getMatchedItemEvaluators())) { + return; + } + + final AtlasAccessorResponse accessorsEnd1 = new AtlasAccessorResponse(); + final AtlasAccessorResponse accessorsEnd2 = new AtlasAccessorResponse(); + + // Collect lists of accessors for both results + resultEnd1.getMatchedItemEvaluators().forEach(x -> { + collectSubjects(accessorsEnd1, x); + }); + + resultEnd2.getMatchedItemEvaluators().forEach(x -> { + collectSubjects(accessorsEnd2, x); + }); + + // Retain only common accessors + accessorsEnd1.getUsers().retainAll(accessorsEnd2.getUsers()); + accessorsEnd1.getRoles().retainAll(accessorsEnd2.getRoles()); + accessorsEnd1.getGroups().retainAll(accessorsEnd2.getGroups()); + + accessorsEnd1.getDenyUsers().addAll(accessorsEnd2.getDenyUsers()); + accessorsEnd1.getDenyRoles().addAll(accessorsEnd2.getDenyRoles()); + accessorsEnd1.getDenyGroups().addAll(accessorsEnd2.getDenyGroups()); + + // add accessors to the response + accessorResponse.getUsers().addAll(accessorsEnd1.getUsers()); + accessorResponse.getRoles().addAll(accessorsEnd1.getRoles()); + accessorResponse.getGroups().addAll(accessorsEnd1.getGroups()); + + accessorResponse.getDenyUsers().addAll(accessorsEnd1.getDenyUsers()); + accessorResponse.getDenyRoles().addAll(accessorsEnd1.getDenyRoles()); + accessorResponse.getDenyGroups().addAll(accessorsEnd1.getDenyGroups()); + } + + static boolean hasAccessors(RangerAccessResult result) { + if (result == null) { + return false; + } + + for (RangerPolicyItemEvaluator itemEvaluator : result.getMatchedItemEvaluators()) { + RangerPolicy.RangerPolicyItem item = itemEvaluator.getPolicyItem(); + if (CollectionUtils.isNotEmpty(item.getUsers()) || CollectionUtils.isNotEmpty(item.getRoles()) && CollectionUtils.isNotEmpty(item.getGroups())) { + return true; + } + } + return false; + } +} diff --git a/auth-plugin-atlas/src/main/java/org/apache/atlas/authorization/atlas/authorizer/RangerGroupUtil.java b/auth-plugin-atlas/src/main/java/org/apache/atlas/authorization/atlas/authorizer/RangerGroupUtil.java new file mode 100644 index 00000000000..811bce4bbc0 --- /dev/null +++ b/auth-plugin-atlas/src/main/java/org/apache/atlas/authorization/atlas/authorizer/RangerGroupUtil.java @@ -0,0 +1,62 @@ +package org.apache.atlas.authorization.atlas.authorizer; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.atlas.plugin.util.RangerUserStore; + +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Set; + +public class RangerGroupUtil { + private long UserStoreVersion; + private Map> userGroupMapping; + + public enum GROUP_FOR {USER} + + public RangerGroupUtil(RangerUserStore userStore) { + if (userStore != null) { + UserStoreVersion = userStore.getUserStoreVersion(); + userGroupMapping = userStore.getUserGroupMapping(); + } else { + UserStoreVersion = -1L; + } + } + + public void setUserStore(RangerUserStore userStore) { + this.userGroupMapping = userStore.getUserGroupMapping(); + this.UserStoreVersion = userStore.getUserStoreVersion(); + } + + public long getUserStoreVersion() { return UserStoreVersion; } + + public Set getContainedGroups(String userName) { + Set data = new LinkedHashSet(); + Set tmpData = new LinkedHashSet(); + tmpData = userGroupMapping.get(userName); + if (tmpData != null){ + data = tmpData; + } + return data; + } + +} + + diff --git a/auth-plugin-atlas/src/main/java/org/apache/atlas/services/atlas/RangerServiceAtlas.java b/auth-plugin-atlas/src/main/java/org/apache/atlas/services/atlas/RangerServiceAtlas.java new file mode 100644 index 00000000000..cf12333b9ae --- /dev/null +++ b/auth-plugin-atlas/src/main/java/org/apache/atlas/services/atlas/RangerServiceAtlas.java @@ -0,0 +1,693 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.atlas.services.atlas; + +import com.google.gson.Gson; +import com.sun.jersey.api.client.Client; +import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.WebResource; +import com.sun.jersey.core.util.MultivaluedMapImpl; +import org.apache.atlas.model.discovery.AtlasSearchResult; +import org.apache.atlas.model.instance.AtlasEntityHeader; +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.io.IOCase; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.atlas.plugin.client.BaseClient; +import org.apache.atlas.plugin.client.HadoopException; +import org.apache.atlas.plugin.model.RangerPolicy; +import org.apache.atlas.plugin.model.RangerPolicy.RangerPolicyItem; +import org.apache.atlas.plugin.model.RangerPolicy.RangerPolicyItemAccess; +import org.apache.atlas.plugin.model.RangerPolicy.RangerPolicyResource; +import org.apache.atlas.plugin.model.RangerService; +import org.apache.atlas.plugin.model.RangerServiceDef; +import org.apache.atlas.plugin.policyengine.RangerPolicyEngine; +import org.apache.atlas.plugin.service.RangerBaseService; +import org.apache.atlas.plugin.service.ResourceLookupContext; +import org.apache.atlas.plugin.util.PasswordUtils; + +import javax.security.auth.Subject; +import javax.ws.rs.core.MultivaluedMap; +import javax.ws.rs.core.NewCookie; +import java.security.PrivilegedAction; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class RangerServiceAtlas extends RangerBaseService { + private static final Log LOG = LogFactory.getLog(RangerServiceAtlas.class); + + public static final String RESOURCE_SERVICE = "atlas-service"; + public static final String RESOURCE_TYPE_CATEGORY = "type-category"; + public static final String RESOURCE_TYPE_NAME = "type"; + public static final String RESOURCE_ENTITY_TYPE = "entity-type"; + public static final String RESOURCE_ENTITY_CLASSIFICATION = "entity-classification"; + public static final String RESOURCE_CLASSIFICATION = "classification"; + public static final String RESOURCE_ENTITY_ID = "entity"; + public static final String RESOURCE_ENTITY_LABEL = "entity-label"; + public static final String RESOURCE_ENTITY_BUSINESS_METADATA = "entity-business-metadata"; + public static final String RESOURCE_ENTITY_OWNER = "owner"; + public static final String RESOURCE_RELATIONSHIP_TYPE = "relationship-type"; + public static final String RESOURCE_END_ONE_ENTITY_TYPE = "end-one-entity-type"; + public static final String RESOURCE_END_ONE_ENTITY_CLASSIFICATION = "end-one-entity-classification"; + public static final String RESOURCE_END_ONE_ENTITY_ID = "end-one-entity"; + public static final String RESOURCE_END_TWO_ENTITY_TYPE = "end-two-entity-type"; + public static final String RESOURCE_END_TWO_ENTITY_CLASSIFICATION = "end-two-entity-classification"; + public static final String RESOURCE_END_TWO_ENTITY_ID = "end-two-entity"; + public static final String SEARCH_FEATURE_POLICY_NAME = "Allow users to manage favorite searches"; + + public static final String ACCESS_TYPE_ENTITY_READ = "entity-read"; + public static final String ACCESS_TYPE_TYPE_READ = "type-read"; + public static final String ACCESS_TYPE_ENTITY_CREATE = "entity-create"; + public static final String ACCESS_TYPE_ENTITY_UPDATE = "entity-update"; + public static final String ACCESS_TYPE_ENTITY_DELETE = "entity-delete"; + public static final String ADMIN_USERNAME_DEFAULT = "admin"; + public static final String TAGSYNC_USERNAME_DEFAULT = "rangertagsync"; + public static final String ENTITY_TYPE_USER_PROFILE = "__AtlasUserProfile"; + public static final String ENTITY_TYPE_SAVED_SEARCH = "__AtlasUserSavedSearch"; + public static final String ENTITY_ID_USER_PROFILE = RangerPolicyEngine.USER_CURRENT; + public static final String ENTITY_ID_USER_SAVED_SEARCH= RangerPolicyEngine.USER_CURRENT + ":*"; + + + public static final String CONFIG_REST_ADDRESS = "atlas.rest.address"; + public static final String CONFIG_USERNAME = "username"; + public static final String CONFIG_PASSWORD = "password"; + public static final String ENTITY_NOT_CLASSIFIED = "_NOT_CLASSIFIED"; + + private static final String TYPE_ENTITY = "entity"; + private static final String TYPE_CLASSIFICATION = "classification"; + private static final String TYPE_STRUCT = "struct"; + private static final String TYPE_ENUM = "enum"; + private static final String TYPE_RELATIONSHIP = "relationship"; + private static final String TYPE_BUSINESS_METADATA = "business_metadata"; + + private static final String URL_LOGIN = "/j_spring_security_check"; + private static final String URL_GET_TYPESDEF_HEADERS = "/api/atlas/v2/types/typedefs/headers"; + private static final String URl_ENTITY_SEARCH = "v2/search/attribute?attrName=qualifiedName"; + + private static final String WEB_RESOURCE_CONTENT_TYPE = "application/x-www-form-urlencoded"; + private static final String CONNECTION_ERROR_MSG = " You can still save the repository and start creating" + + " policies, but you would not be able to use autocomplete for" + + " resource names. Check ranger_admin.log for more info."; + + public RangerServiceAtlas() { + super(); + } + + @Override + public void init(RangerServiceDef serviceDef, RangerService service) { + super.init(serviceDef, service); + } + + @Override + public Map validateConfig() throws Exception { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerServiceAtlas.validateConfig()"); + } + + AtlasServiceClient client = new AtlasServiceClient(getServiceName(), configs); + Map ret = client.validateConfig(); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerServiceAtlas.validateConfig(): " + ret ); + } + + return ret; + } + + @Override + public List lookupResource(ResourceLookupContext context)throws Exception { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerServiceAtlas.lookupResource(" + context + ")"); + } + + AtlasServiceClient client = new AtlasServiceClient(getServiceName(), configs); + List ret = client.lookupResource(context); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerServiceAtlas.lookupResource("+ context + "): " + ret); + } + + return ret; + } + + @Override + public List getDefaultRangerPolicies() throws Exception { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerServiceAtlas.getDefaultRangerPolicies()"); + } + + List ret = super.getDefaultRangerPolicies(); + String adminUser = getStringConfig("atlas.admin.user", ADMIN_USERNAME_DEFAULT); + String tagSyncUser = getStringConfig("atlas.rangertagsync.user", TAGSYNC_USERNAME_DEFAULT); + + boolean relationshipTypeAllowPublic = getBooleanConfig("atlas.default-policy.relationship-type.allow.public", true); + + + for (RangerPolicy defaultPolicy : ret) { + final Map policyResources = defaultPolicy.getResources(); + + // 1. add adminUser to every policyItem + for (RangerPolicyItem defaultPolicyItem : defaultPolicy.getPolicyItems()) { + defaultPolicyItem.getUsers().add(adminUser); + } + + // 2. add a policy-item for rangertagsync user with 'entity-read' permission in the policy for 'entity-type' + if (policyResources.containsKey(RESOURCE_ENTITY_TYPE) && !policyResources.containsKey(RESOURCE_CLASSIFICATION)) { + RangerPolicyItem policyItemForTagSyncUser = new RangerPolicyItem(); + + policyItemForTagSyncUser.setUsers(Collections.singletonList(tagSyncUser)); + policyItemForTagSyncUser.setGroups(Collections.singletonList(RangerPolicyEngine.GROUP_PUBLIC)); + policyItemForTagSyncUser.setAccesses(Collections.singletonList(new RangerPolicyItemAccess(ACCESS_TYPE_ENTITY_READ))); + + defaultPolicy.getPolicyItems().add(policyItemForTagSyncUser); + } + + if (relationshipTypeAllowPublic) { + // 3. add 'public' group in the policy for 'relationship-type', + if (policyResources.containsKey(RangerServiceAtlas.RESOURCE_RELATIONSHIP_TYPE)) { + for (RangerPolicyItem defaultPolicyItem : defaultPolicy.getPolicyItems()) { + defaultPolicyItem.getGroups().add(RangerPolicyEngine.GROUP_PUBLIC); + } + } + } + + if (defaultPolicy.getName().contains("all") + && policyResources.containsKey(RangerServiceAtlas.RESOURCE_ENTITY_TYPE) + && StringUtils.isNotBlank(lookUpUser) && !policyResources.containsKey(RESOURCE_CLASSIFICATION)) { + RangerPolicyItem policyItemForLookupUser = new RangerPolicyItem(); + policyItemForLookupUser.setUsers(Collections.singletonList(lookUpUser)); + policyItemForLookupUser.setAccesses(Collections.singletonList(new RangerPolicyItemAccess(ACCESS_TYPE_ENTITY_READ))); + policyItemForLookupUser.setDelegateAdmin(false); + defaultPolicy.getPolicyItems().add(policyItemForLookupUser); + } + + // add a policy-item for rangertagsync user with 'type-read' permission in the policy for 'type-category' + if (policyResources.containsKey(RangerServiceAtlas.RESOURCE_TYPE_CATEGORY)) { + RangerPolicyItem policyItemTypeReadForAll = new RangerPolicyItem(); + policyItemTypeReadForAll.setGroups(Collections.singletonList(RangerPolicyEngine.GROUP_PUBLIC)); + policyItemTypeReadForAll.setAccesses(Collections.singletonList(new RangerPolicyItemAccess(ACCESS_TYPE_TYPE_READ))); + defaultPolicy.getPolicyItems().add(policyItemTypeReadForAll); + } + } + + //4.add new policy for public group with entity-read, entity-create, entity-update, entity-delete for __AtlasUserProfile, __AtlasUserSavedSearch entity type + RangerPolicy searchFeaturePolicy = getSearchFeaturePolicy(); + ret.add(searchFeaturePolicy); + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerServiceAtlas.getDefaultRangerPolicies()"); + } + + return ret; + } + + private RangerPolicy getSearchFeaturePolicy() { + RangerPolicy searchFeaturePolicy = new RangerPolicy(); + + searchFeaturePolicy.setName(SEARCH_FEATURE_POLICY_NAME); + searchFeaturePolicy.setService(serviceName); + searchFeaturePolicy.setResources(getSearchFeaturePolicyResource()); + searchFeaturePolicy.setPolicyItems(getSearchFeaturePolicyItem()); + + return searchFeaturePolicy; + } + + private List getSearchFeaturePolicyItem() { + List accesses = new ArrayList(); + + accesses.add(new RangerPolicyItemAccess(ACCESS_TYPE_ENTITY_READ)); + accesses.add(new RangerPolicyItemAccess(ACCESS_TYPE_ENTITY_CREATE)); + accesses.add(new RangerPolicyItemAccess(ACCESS_TYPE_ENTITY_UPDATE)); + accesses.add(new RangerPolicyItemAccess(ACCESS_TYPE_ENTITY_DELETE)); + + RangerPolicyItem item = new RangerPolicyItem(accesses, Arrays.asList(RangerPolicyEngine.USER_CURRENT), null, null, null, false); + + return Collections.singletonList(item); + } + + private Map getSearchFeaturePolicyResource() { + Map resources = new HashMap<>(); + + resources.put(RESOURCE_ENTITY_TYPE, new RangerPolicyResource(Arrays.asList(ENTITY_TYPE_USER_PROFILE, ENTITY_TYPE_SAVED_SEARCH), false, false)); + resources.put(RESOURCE_ENTITY_CLASSIFICATION, new RangerPolicyResource("*")); + resources.put(RESOURCE_ENTITY_ID, new RangerPolicyResource(Arrays.asList(ENTITY_ID_USER_PROFILE, ENTITY_ID_USER_SAVED_SEARCH), false, false)); + + return resources; + } + + private static class AtlasServiceClient extends BaseClient { + private static final String[] TYPE_CATEGORIES = new String[] { "classification", "enum", "entity", "relationship", "struct" ,"business_metadata" }; + + Map> typesDef = new HashMap<>(); + + public AtlasServiceClient(String serviceName, Map serviceConfig) { + super(serviceName, serviceConfig); + } + + public Map validateConfig() { + Map ret = new HashMap<>(); + + loginToAtlas(Client.create()); + + BaseClient.generateResponseDataMap(true, "ConnectionTest Successful", "ConnectionTest Successful", null, null, ret); + + return ret; + } + + public List lookupResource(ResourceLookupContext lookupContext) { + final List ret = new ArrayList<>(); + final String userInput = lookupContext.getUserInput(); + final List currentValues = lookupContext.getResources().get(lookupContext.getResourceName()); + + switch(lookupContext.getResourceName()) { + case RESOURCE_TYPE_CATEGORY: { + for (String typeCategory : TYPE_CATEGORIES) { + addIfStartsWithAndNotExcluded(ret, typeCategory, userInput, currentValues); + } + } + break; + + case RESOURCE_TYPE_NAME: { + refreshTypesDefs(); + + final List typeCategories = lookupContext.getResources().get(RESOURCE_TYPE_CATEGORY); + + if (emptyOrContainsMatch(typeCategories, TYPE_CLASSIFICATION)) { + addIfStartsWithAndNotExcluded(ret, typesDef.get(TYPE_CLASSIFICATION), userInput, currentValues); + } + + if (emptyOrContainsMatch(typeCategories, TYPE_ENTITY)) { + addIfStartsWithAndNotExcluded(ret, typesDef.get(TYPE_ENTITY), userInput, currentValues); + } + + if (emptyOrContainsMatch(typeCategories, TYPE_ENUM)) { + addIfStartsWithAndNotExcluded(ret, typesDef.get(TYPE_ENUM), userInput, currentValues); + } + + if (emptyOrContainsMatch(typeCategories, TYPE_STRUCT)) { + addIfStartsWithAndNotExcluded(ret, typesDef.get(TYPE_STRUCT), userInput, currentValues); + } + + if (emptyOrContainsMatch(typeCategories, TYPE_RELATIONSHIP)) { + addIfStartsWithAndNotExcluded(ret, typesDef.get(TYPE_RELATIONSHIP), userInput, currentValues); + } + + if (emptyOrContainsMatch(typeCategories, TYPE_BUSINESS_METADATA)) { + addIfStartsWithAndNotExcluded(ret, typesDef.get(TYPE_BUSINESS_METADATA), userInput, currentValues); + } + } + break; + + case RESOURCE_END_ONE_ENTITY_TYPE: + case RESOURCE_END_TWO_ENTITY_TYPE: + case RESOURCE_ENTITY_TYPE: { + refreshTypesDefs(); + + addIfStartsWithAndNotExcluded(ret, typesDef.get(TYPE_ENTITY), userInput, currentValues); + } + break; + + case RESOURCE_END_ONE_ENTITY_CLASSIFICATION: + case RESOURCE_END_TWO_ENTITY_CLASSIFICATION: + case RESOURCE_ENTITY_CLASSIFICATION: { + refreshTypesDefs(); + + addIfStartsWithAndNotExcluded(ret, typesDef.get(TYPE_CLASSIFICATION), userInput, currentValues); + } + break; + + case RESOURCE_ENTITY_ID: { + List searchTypes = lookupContext.getResources().get("entity-type"); + + if (searchTypes != null && searchTypes.size() == 1) { + List values = searchEntities(userInput, searchTypes.get(0)); + + addIfStartsWithAndNotExcluded(ret, values, userInput, currentValues); + } + } + break; + + case RESOURCE_RELATIONSHIP_TYPE: { + refreshTypesDefs(); + addIfStartsWithAndNotExcluded(ret, typesDef.get(TYPE_RELATIONSHIP), userInput, currentValues); + + } + break; + + case RESOURCE_END_ONE_ENTITY_ID: { + + List searchTypes = lookupContext.getResources().get(RESOURCE_END_ONE_ENTITY_TYPE); + + if (searchTypes != null && searchTypes.size() == 1) { + List values = searchEntities(userInput, searchTypes.get(0)); + + addIfStartsWithAndNotExcluded(ret, values, userInput, currentValues); + } + + } + break; + + case RESOURCE_END_TWO_ENTITY_ID: { + List searchTypes = lookupContext.getResources().get(RESOURCE_END_TWO_ENTITY_TYPE); + + if (searchTypes != null && searchTypes.size() == 1) { + List values = searchEntities(userInput, searchTypes.get(0)); + + addIfStartsWithAndNotExcluded(ret, values, userInput, currentValues); + } + } + break; + + default: { + ret.add(lookupContext.getResourceName()); + } + } + + return ret; + } + + private ClientResponse loginToAtlas(Client client) { + ClientResponse ret = null; + HadoopException excp = null; + String loginUrl = null; + + for (String atlasUrl : getAtlasUrls()) { + try { + loginUrl = atlasUrl + URL_LOGIN; + + WebResource webResource = client.resource(loginUrl); + MultivaluedMap formData = new MultivaluedMapImpl(); + String password = null; + + try { + password = PasswordUtils.decryptPassword(getPassword()); + } catch (Exception ex) { + LOG.info("Password decryption failed; trying Atlas connection with received password string"); + } + + if (password == null) { + password = getPassword(); + } + + formData.add("j_username", getUserName()); + formData.add("j_password", password); + + try { + ret = webResource.type(WEB_RESOURCE_CONTENT_TYPE).post(ClientResponse.class, formData); + } catch (Exception e) { + LOG.error("failed to login to Atlas at " + loginUrl, e); + } + + if (ret != null) { + break; + } + } catch (Throwable t) { + String msgDesc = "Exception while login to Atlas at : " + loginUrl; + + LOG.error(msgDesc, t); + + excp = new HadoopException(msgDesc, t); + + excp.generateResponseDataMap(false, BaseClient.getMessage(t), msgDesc + CONNECTION_ERROR_MSG, null, null); + } + } + + if (ret == null) { + if (excp == null) { + String msgDesc = "Exception while login to Atlas at : " + loginUrl; + + excp = new HadoopException(msgDesc); + + excp.generateResponseDataMap(false, "", msgDesc + CONNECTION_ERROR_MSG, null, null); + } + + throw excp; + } + + return ret; + } + + private boolean refreshTypesDefs() { + boolean ret = false; + Subject subj = getLoginSubject(); + + if (subj == null) { + return ret; + } + + Map> typesDef = Subject.doAs(subj, new PrivilegedAction>>() { + @Override + public Map> run() { + Map> ret = null; + + for (String atlasUrl : getAtlasUrls()) { + Client client = null; + + try { + client = Client.create(); + + ClientResponse loginResponse = loginToAtlas(client); + WebResource webResource = client.resource(atlasUrl + URL_GET_TYPESDEF_HEADERS); + WebResource.Builder builder = webResource.getRequestBuilder(); + + for (NewCookie cook : loginResponse.getCookies()) { + builder = builder.cookie(cook); + } + + ClientResponse response = builder.get(ClientResponse.class); + + if (response != null) { + String jsonString = response.getEntity(String.class); + Gson gson = new Gson(); + List types = gson.fromJson(jsonString, List.class); + + ret = new HashMap<>(); + + for (Object type : types) { + if (type instanceof Map) { + Map typeDef = (Map) type; + Object name = typeDef.get("name"); + Object category = typeDef.get("category"); + + if (name != null && category != null) { + String strCategory = category.toString().toLowerCase(); + List categoryList = ret.get(strCategory); + + if (categoryList == null) { + categoryList = new ArrayList<>(); + + ret.put(strCategory, categoryList); + } + + categoryList.add(name.toString()); + } + } + } + + break; + } + } catch (Throwable t) { + String msgDesc = "Exception while getting Atlas Resource List."; + + LOG.error(msgDesc, t); + } finally { + if (client != null) { + client.destroy(); + } + } + } + + return ret; + } + }); + + if (typesDef != null) { + this.typesDef = typesDef; + + ret = true; + } + + return ret; + } + + private List searchEntities(String userInput, String entityType) { + if( LOG.isDebugEnabled()) { + LOG.debug("==> RangerServiceAtlas.searchEntities(userInput=" + userInput + ", entityType=" + entityType + ")"); + } + + Subject subj = getLoginSubject(); + + if (subj == null) { + return null; + } + + List list = Subject.doAs(subj, new PrivilegedAction>() { + @Override + public List run() { + List ret = null; + + for (String atlasUrl : getAtlasUrls()) { + Client client = null; + + try { + client = Client.create(); + + ClientResponse loginResponse = loginToAtlas(client); + String entitySearcApiUrl = atlasUrl + "/api/atlas/" + URl_ENTITY_SEARCH; + StringBuilder searchUrl = new StringBuilder(); + + searchUrl.append(entitySearcApiUrl) + .append("&typeName=") + .append(entityType) + .append("&attrValuePrefix=" + userInput + "&limit=25"); + + + WebResource webResource = client.resource(searchUrl.toString()); + WebResource.Builder builder = webResource.getRequestBuilder(); + + for (NewCookie cook : loginResponse.getCookies()) { + builder = builder.cookie(cook); + } + + ClientResponse response = builder.get(ClientResponse.class); + + if (response != null) { + String jsonString = response.getEntity(String.class); + Gson gson = new Gson(); + AtlasSearchResult searchResult = gson.fromJson(jsonString, AtlasSearchResult.class); + + ret = new ArrayList<>(); + + if (searchResult != null) { + List entityHeaderList = searchResult.getEntities(); + + for (AtlasEntityHeader entity : entityHeaderList) { + ret.add((String) entity.getAttribute("qualifiedName")); + } + } + } + } catch (Throwable t) { + String msgDesc = "Exception while getting Atlas Entity Resource List."; + + LOG.error(msgDesc, t); + } finally { + if (client != null) { + client.destroy(); + } + } + } + + return ret; + } + }); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerServiceAtlas.searchEntities(userInput=" + userInput + ", entityType=" + entityType + "): " + list); + } + + return list; + } + + String[] getAtlasUrls() { + String urlString = connectionProperties.get(CONFIG_REST_ADDRESS); + String[] ret = urlString == null ? new String[0] : urlString.split(","); + + // remove separator at the end + for (int i = 0; i < ret.length; i++) { + String url = ret[i]; + + while (url.length() > 0 && url.charAt(url.length() - 1) == '/') { + url = url.substring(0, url.length() - 1); + } + + ret[i] = url; + } + + return ret; + } + + String getUserName() { + return connectionProperties.get(CONFIG_USERNAME); + } + + String getPassword() { + return connectionProperties.get(CONFIG_PASSWORD); + } + + boolean emptyOrContainsMatch(List list, String value) { + if (list == null || list.isEmpty()) { + return true; + } + + for (String item : list) { + if (StringUtils.equalsIgnoreCase(item, value) || FilenameUtils.wildcardMatch(value, item, IOCase.INSENSITIVE)) { + return true; + } + } + + return false; + } + + void addIfStartsWithAndNotExcluded(List list, List values, String prefix, List excludeList) { + if (list == null) { + return; + } + + if (values == null) { + addIfStartsWithAndNotExcluded(list, ENTITY_NOT_CLASSIFIED, prefix, excludeList); + } else { + for (String value : values) { + addIfStartsWithAndNotExcluded(list, value, prefix, excludeList); + } + } + } + + void addIfStartsWithAndNotExcluded(List list, String value, String prefix, List excludeList) { + if (value == null || list == null) { + return; + } + + if (prefix != null && !value.startsWith(prefix)) { + return; + } + + if (excludeList != null && excludeList.contains(value)) { + return; + } + + list.add(value); + } + } + + String getStringConfig(String configName, String defaultValue) { + String val = service.getConfigs().get(configName); + + return StringUtils.isBlank(val) ? defaultValue : val; + } + + boolean getBooleanConfig(String configName, boolean defaultValue) { + String val = service.getConfigs().get(configName); + + return StringUtils.isBlank(val) ? defaultValue : Boolean.parseBoolean(val); + } +} diff --git a/auth-plugin-atlas/src/main/java/org/apache/atlas/services/atlas/json/model/ResourceEntityResponse.java b/auth-plugin-atlas/src/main/java/org/apache/atlas/services/atlas/json/model/ResourceEntityResponse.java new file mode 100644 index 00000000000..46a80e1675b --- /dev/null +++ b/auth-plugin-atlas/src/main/java/org/apache/atlas/services/atlas/json/model/ResourceEntityResponse.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.atlas.services.atlas.json.model; + +public class ResourceEntityResponse { + + private String href; + private String name; + private String id; + private String type; + + public String getHref() { + return href; + } + + public void setHref(String href) { + this.href = href; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + +} diff --git a/auth-plugin-atlas/src/main/java/org/apache/atlas/services/atlas/json/model/ResourceOperationResponse.java b/auth-plugin-atlas/src/main/java/org/apache/atlas/services/atlas/json/model/ResourceOperationResponse.java new file mode 100644 index 00000000000..e63a033f81c --- /dev/null +++ b/auth-plugin-atlas/src/main/java/org/apache/atlas/services/atlas/json/model/ResourceOperationResponse.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.atlas.services.atlas.json.model; + +import java.util.List; + +public class ResourceOperationResponse { + private String requestId; + private String query; + private String queryType; + private List results; + + public String getRequestId() { + return requestId; + } + + public void setRequestId(String requestId) { + this.requestId = requestId; + } + + public String getQuery() { + return query; + } + + public void setQuery(String query) { + this.query = query; + } + + public String getQueryType() { + return queryType; + } + + public void setQueryType(String queryType) { + this.queryType = queryType; + } + + public List getResults() { + return results; + } + + public void setResults(List results) { + this.results = results; + } + + public class Results { + + private String result; + private String count; + + public String getResult() { + return result; + } + + public void setResult(String result) { + this.result = result; + } + + public String getCount() { + return count; + } + + public void setCount(String count) { + this.count = count; + } + } + +} diff --git a/auth-plugin-atlas/src/main/java/org/apache/atlas/services/atlas/json/model/ResourceTaxonomyResponse.java b/auth-plugin-atlas/src/main/java/org/apache/atlas/services/atlas/json/model/ResourceTaxonomyResponse.java new file mode 100644 index 00000000000..dad84b21846 --- /dev/null +++ b/auth-plugin-atlas/src/main/java/org/apache/atlas/services/atlas/json/model/ResourceTaxonomyResponse.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.atlas.services.atlas.json.model; + +public class ResourceTaxonomyResponse { + private String href; + private String name; + private String description; + + public String getHref() { + return href; + } + + public void setHref(String href) { + this.href = href; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + +} diff --git a/auth-plugin-atlas/src/main/java/org/apache/atlas/services/atlas/json/model/ResourceTermResponse.java b/auth-plugin-atlas/src/main/java/org/apache/atlas/services/atlas/json/model/ResourceTermResponse.java new file mode 100644 index 00000000000..c625a93cd1d --- /dev/null +++ b/auth-plugin-atlas/src/main/java/org/apache/atlas/services/atlas/json/model/ResourceTermResponse.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.atlas.services.atlas.json.model; + +public class ResourceTermResponse { + + private String href; + private String name; + private String description; + + public String getHref() { + return href; + } + + public void setHref(String href) { + this.href = href; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + +} diff --git a/auth-plugin-atlas/src/main/java/org/apache/atlas/services/atlas/json/model/ResourceTypeResponse.java b/auth-plugin-atlas/src/main/java/org/apache/atlas/services/atlas/json/model/ResourceTypeResponse.java new file mode 100644 index 00000000000..bf6b3fc7580 --- /dev/null +++ b/auth-plugin-atlas/src/main/java/org/apache/atlas/services/atlas/json/model/ResourceTypeResponse.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.atlas.services.atlas.json.model; + +import java.util.List; + +public class ResourceTypeResponse { + + private List results; + private String count; + private String requestId; + + public String getCount() { + return count; + } + + public void setCount(String count) { + this.count = count; + } + + public String getRequestId() { + return requestId; + } + + public void setRequestId(String requestId) { + this.requestId = requestId; + } + + public List getResults() { + return results; + } + + public void setResults(List results) { + this.results = results; + } + +} diff --git a/authorization/src/main/java/org/apache/atlas/authorize/AtlasAccessRequest.java b/authorization/src/main/java/org/apache/atlas/authorize/AtlasAccessRequest.java index c76a8715b76..f133e1268f3 100644 --- a/authorization/src/main/java/org/apache/atlas/authorize/AtlasAccessRequest.java +++ b/authorization/src/main/java/org/apache/atlas/authorize/AtlasAccessRequest.java @@ -176,16 +176,16 @@ public String getEntityId(AtlasEntityHeader entity, AtlasTypeRegistry typeRegist return ret == null ? "" : ret.toString(); } - public Set getClassificationNames(AtlasEntityHeader entity) { - final Set ret; + public Set getClassificationNames(AtlasEntityHeader entity) { + final Set ret; if (entity == null || entity.getClassifications() == null) { ret = Collections.emptySet(); } else { ret = new HashSet<>(); - for (AtlasClassification classify : entity.getClassifications()) { - ret.add(classify.getTypeName()); + for (AtlasClassification classification : entity.getClassifications()) { + ret.add(classification); } } diff --git a/authorization/src/main/java/org/apache/atlas/authorize/AtlasAccessorRequest.java b/authorization/src/main/java/org/apache/atlas/authorize/AtlasAccessorRequest.java new file mode 100644 index 00000000000..b3d25fa4df6 --- /dev/null +++ b/authorization/src/main/java/org/apache/atlas/authorize/AtlasAccessorRequest.java @@ -0,0 +1,163 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.authorize; + + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import org.apache.atlas.model.instance.AtlasEntityHeader; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE; +import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ONLY; + +@JsonAutoDetect(getterVisibility=PUBLIC_ONLY, setterVisibility=PUBLIC_ONLY, fieldVisibility=NONE) +@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown=true) +@XmlRootElement +@XmlAccessorType(XmlAccessType.PROPERTY) +public class AtlasAccessorRequest { + + private String action; + + private String guid; + private String typeName; + private String qualifiedName; + private String label; + private String classification; + private String businessMetadata; + + private String relationshipTypeName; + private String entityQualifiedNameEnd1; + private String entityQualifiedNameEnd2; + private String entityGuidEnd1; + private String entityGuidEnd2; + private String entityTypeEnd1; + private String entityTypeEnd2; + + private AtlasEntityHeader entity = null; + + public String getGuid() { + return guid; + } + + public void setGuid(String guid) { + this.guid = guid; + } + + public String getTypeName() { + return typeName; + } + + public void setTypeName(String typeName) { + this.typeName = typeName; + } + + public String getQualifiedName() { + return qualifiedName; + } + + public void setQualifiedName(String qualifiedName) { + this.qualifiedName = qualifiedName; + } + + public String getAction() { + return action; + } + + public void setAction(String action) { + this.action = action; + } + + public String getClassification() { + return classification; + } + + public void setClassification(String classification) { + this.classification = classification; + } + + public String getLabel() { + return label; + } + + public void setLabel(String label) { + this.label = label; + } + + public String getBusinessMetadata() { + return businessMetadata; + } + + public void setBusinessMetadata(String businessMetadata) { + this.businessMetadata = businessMetadata; + } + + public AtlasEntityHeader getEntity() { + return entity; + } + + public void setEntity(AtlasEntityHeader entity) { + this.entity = entity; + } + + public String getRelationshipTypeName() { + return relationshipTypeName; + } + + public String getEntityQualifiedNameEnd1() { + return entityQualifiedNameEnd1; + } + + public String getEntityQualifiedNameEnd2() { + return entityQualifiedNameEnd2; + } + + public String getEntityGuidEnd1() { + return entityGuidEnd1; + } + + public String getEntityGuidEnd2() { + return entityGuidEnd2; + } + + public String getEntityTypeEnd1() { + return entityTypeEnd1; + } + + public String getEntityTypeEnd2() { + return entityTypeEnd2; + } + + @Override + public String toString() { + return "AtlasAccessor{" + + "guid='" + guid + '\'' + + ", typeName='" + typeName + '\'' + + ", qualifiedName='" + qualifiedName + '\'' + + ", label='" + label + '\'' + + ", classification='" + classification + '\'' + + ", businessMetadata='" + businessMetadata + '\'' + + '}'; + } +} + diff --git a/authorization/src/main/java/org/apache/atlas/authorize/AtlasAccessorResponse.java b/authorization/src/main/java/org/apache/atlas/authorize/AtlasAccessorResponse.java new file mode 100644 index 00000000000..73e8023c090 --- /dev/null +++ b/authorization/src/main/java/org/apache/atlas/authorize/AtlasAccessorResponse.java @@ -0,0 +1,196 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.authorize; + +import com.fasterxml.jackson.annotation.JsonInclude; + +import java.util.HashSet; +import java.util.Set; + +@JsonInclude(JsonInclude.Include.NON_NULL) +public class AtlasAccessorResponse { + + private String action; + + private String guid; + private String typeName; + private String qualifiedName; + private String label; + private String classification; + private String businessMetadata; + + private String relationshipTypeName; + private String entityGuidEnd1; + private String entityQualifiedNameEnd1; + private String entityTypeEnd1; + private String entityGuidEnd2; + private String entityQualifiedNameEnd2; + private String entityTypeEnd2; + + private Set users = new HashSet<>(); + private Set groups = new HashSet<>(); + private Set roles = new HashSet<>(); + + private Set denyUsers = new HashSet<>(); + private Set denyGroups = new HashSet<>(); + private Set denyRoles = new HashSet<>(); + + public AtlasAccessorResponse() { + + } + + public void populateRequestDetails(AtlasAccessorRequest accessorRequest) { + this.action = accessorRequest.getAction(); + this.guid = accessorRequest.getGuid(); + this.typeName = accessorRequest.getTypeName(); + this.qualifiedName = accessorRequest.getQualifiedName(); + this.label = accessorRequest.getLabel(); + this.classification = accessorRequest.getClassification(); + this.businessMetadata = accessorRequest.getBusinessMetadata(); + + this.relationshipTypeName = accessorRequest.getRelationshipTypeName(); + this.entityGuidEnd1 = accessorRequest.getEntityGuidEnd1(); + this.entityQualifiedNameEnd1 = accessorRequest.getEntityQualifiedNameEnd1(); + this.entityTypeEnd1 = accessorRequest.getEntityTypeEnd1(); + this.entityGuidEnd2 = accessorRequest.getEntityGuidEnd2(); + this.entityQualifiedNameEnd2 = accessorRequest.getEntityQualifiedNameEnd2(); + this.entityTypeEnd2 = accessorRequest.getEntityTypeEnd2(); + } + + public String getAction() { + return action; + } + + public String getGuid() { + return guid; + } + + public String getTypeName() { + return typeName; + } + + public String getQualifiedName() { + return qualifiedName; + } + + public String getLabel() { + return label; + } + + public String getClassification() { + return classification; + } + + public String getBusinessMetadata() { + return businessMetadata; + } + + public String getRelationshipTypeName() { + return relationshipTypeName; + } + + public String getEntityGuidEnd1() { + return entityGuidEnd1; + } + + public String getEntityQualifiedNameEnd1() { + return entityQualifiedNameEnd1; + } + + public String getEntityTypeEnd1() { + return entityTypeEnd1; + } + + public String getEntityGuidEnd2() { + return entityGuidEnd2; + } + + public String getEntityQualifiedNameEnd2() { + return entityQualifiedNameEnd2; + } + + public String getEntityTypeEnd2() { + return entityTypeEnd2; + } + + public Set getUsers() { + return users; + } + + public void setUsers(Set users) { + this.users = users; + } + + public Set getGroups() { + return groups; + } + + public void setGroups(Set groups) { + this.groups = groups; + } + + public Set getRoles() { + return roles; + } + + public void setRoles(Set roles) { + this.roles = roles; + } + + public Set getDenyUsers() { + return denyUsers; + } + + public void setDenyUsers(Set denyUsers) { + this.denyUsers = denyUsers; + } + + public Set getDenyGroups() { + return denyGroups; + } + + public void setDenyGroups(Set denyGroups) { + this.denyGroups = denyGroups; + } + + public Set getDenyRoles() { + return denyRoles; + } + + public void setDenyRoles(Set denyRoles) { + this.denyRoles = denyRoles; + } + + @Override + public String toString() { + return "AtlasAccessorResponse{" + + "guid='" + guid + '\'' + + ", typeName='" + typeName + '\'' + + ", qualifiedName='" + qualifiedName + '\'' + + ", label='" + label + '\'' + + ", classification='" + classification + '\'' + + ", businessMetadata='" + businessMetadata + '\'' + + ", users=" + users + + ", groups=" + groups + + ", roles=" + roles + + ", denyUsers=" + denyUsers + + ", denyGroups=" + denyGroups + + ", denyRoles=" + denyRoles + + '}'; + } +} diff --git a/authorization/src/main/java/org/apache/atlas/authorize/AtlasAuthorizationUtils.java b/authorization/src/main/java/org/apache/atlas/authorize/AtlasAuthorizationUtils.java index 8a1635abca1..0a0659acb4b 100644 --- a/authorization/src/main/java/org/apache/atlas/authorize/AtlasAuthorizationUtils.java +++ b/authorization/src/main/java/org/apache/atlas/authorize/AtlasAuthorizationUtils.java @@ -23,6 +23,7 @@ import org.apache.atlas.RequestContext; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.instance.AtlasEntityHeader; +import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.utils.AtlasPerfMetrics.MetricRecorder; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; @@ -34,10 +35,10 @@ import javax.servlet.http.HttpServletRequest; import java.net.InetAddress; import java.net.UnknownHostException; -import java.util.HashSet; -import java.util.Set; -import java.util.List; -import java.util.Arrays; +import java.util.*; + +import static org.apache.atlas.repository.Constants.SKIP_DELETE_AUTH_CHECK_TYPES; +import static org.apache.atlas.repository.Constants.SKIP_UPDATE_AUTH_CHECK_TYPES; public class AtlasAuthorizationUtils { private static final Logger LOG = LoggerFactory.getLogger(AtlasAuthorizationUtils.class); @@ -58,6 +59,20 @@ public static void verifyAccess(AtlasTypeAccessRequest request, Object... errorM } } + public static void verifyUpdateEntityAccess(AtlasTypeRegistry typeRegistry, AtlasEntityHeader entityHeader, String message) throws AtlasBaseException { + if (!SKIP_UPDATE_AUTH_CHECK_TYPES.contains(entityHeader.getTypeName())) { + AtlasEntityAccessRequest request = new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_UPDATE, entityHeader); + verifyAccess(request, message); + } + } + + public static void verifyDeleteEntityAccess(AtlasTypeRegistry typeRegistry, AtlasEntityHeader entityHeader, String message) throws AtlasBaseException { + if (!SKIP_DELETE_AUTH_CHECK_TYPES.contains(entityHeader.getTypeName())) { + AtlasEntityAccessRequest request = new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_DELETE, entityHeader); + verifyAccess(request, message); + } + } + public static void verifyAccess(AtlasEntityAccessRequest request, Object... errorMsgParams) throws AtlasBaseException { if (! isAccessAllowed(request)) { String message = (errorMsgParams != null && errorMsgParams.length > 0) ? StringUtils.join(errorMsgParams) : ""; @@ -69,11 +84,21 @@ public static void verifyAccess(AtlasEntityAccessRequest request, Object... erro public static void verifyAccess(AtlasRelationshipAccessRequest request, Object... errorMsgParams) throws AtlasBaseException { if (!isAccessAllowed(request)) { String message = (errorMsgParams != null && errorMsgParams.length > 0) ? StringUtils.join(errorMsgParams) : ""; - throw new AtlasBaseException(AtlasErrorCode.UNAUTHORIZED_ACCESS, request.getUser(), message); + if (StringUtils.isEmpty(message)){ + Map errorMap = new HashMap<>(); + errorMap.put("action", request.getAction().toString()); + errorMap.put("end1", request.getEnd1Entity().getGuid()); + errorMap.put("end2", request.getEnd2Entity().getGuid()); + + throw new AtlasBaseException(AtlasErrorCode.UNAUTHORIZED_ACCESS, errorMap, request.getUser(), ""); + + } else { + throw new AtlasBaseException(AtlasErrorCode.UNAUTHORIZED_ACCESS, request.getUser(), message); + } } } - public static void scrubSearchResults(AtlasSearchResultScrubRequest request) throws AtlasBaseException { + public static void scrubSearchResults(AtlasSearchResultScrubRequest request, boolean suppressLogs) throws AtlasBaseException { String userName = getCurrentUserName(); if (StringUtils.isNotEmpty(userName)) { @@ -85,7 +110,9 @@ public static void scrubSearchResults(AtlasSearchResultScrubRequest request) thr request.setForwardedAddresses(RequestContext.get().getForwardedAddresses()); request.setRemoteIPAddress(RequestContext.get().getClientIPAddress()); - authorizer.scrubSearchResults(request, true); + boolean isScrubAuditEnabled = !suppressLogs; + + authorizer.scrubSearchResults(request, isScrubAuditEnabled); } catch (AtlasAuthorizationException e) { LOG.error("Unable to obtain AtlasAuthorizer", e); } @@ -200,6 +227,68 @@ public static boolean isAccessAllowed(AtlasRelationshipAccessRequest request) { return ret; } + public static AtlasAccessorResponse getAccessors(AtlasEntityAccessRequest request) { + AtlasAccessorResponse ret = null; + + try { + AtlasAuthorizer authorizer = AtlasAuthorizerFactory.getAtlasAuthorizer(); + setAuthInfo(request); + + ret = authorizer.getAccessors(request); + } catch (AtlasAuthorizationException e) { + LOG.error("Unable to obtain AtlasAuthorizer", e); + } + + return ret; + } + + public static AtlasAccessorResponse getAccessors(AtlasRelationshipAccessRequest request) { + AtlasAccessorResponse ret = null; + + try { + AtlasAuthorizer authorizer = AtlasAuthorizerFactory.getAtlasAuthorizer(); + setAuthInfo(request); + + ret = authorizer.getAccessors(request); + } catch (AtlasAuthorizationException e) { + LOG.error("Unable to obtain AtlasAuthorizer", e); + } + + return ret; + } + + public static AtlasAccessorResponse getAccessors(AtlasTypeAccessRequest request) { + AtlasAccessorResponse ret = null; + + try { + AtlasAuthorizer authorizer = AtlasAuthorizerFactory.getAtlasAuthorizer(); + setAuthInfo(request); + + ret = authorizer.getAccessors(request); + } catch (AtlasAuthorizationException e) { + LOG.error("Unable to obtain AtlasAuthorizer", e); + } + + return ret; + } + + public static Set getRolesForCurrentUser() throws AtlasBaseException { + Set ret = new HashSet<>(); + + try { + AtlasAuthorizer authorizer = AtlasAuthorizerFactory.getAtlasAuthorizer(); + if (authorizer == null ) { + throw new AtlasAuthorizationException("Authorizer is null"); + } + + ret = authorizer.getRolesForCurrentUser(getCurrentUserName(), getCurrentUserGroups()); + } catch (AtlasAuthorizationException e) { + LOG.error("Unable to obtain AtlasAuthorizer", e); + } + + return ret; + } + public static void filterTypesDef(AtlasTypesDefFilterRequest request) { MetricRecorder metric = RequestContext.get().startMetricRecord("filterTypesDef"); String userName = getCurrentUserName(); @@ -267,4 +356,11 @@ public static Set getCurrentUserGroups() { return ret; } + + private static void setAuthInfo(AtlasAccessRequest request) { + request.setUser(getCurrentUserName(), getCurrentUserGroups()); + request.setClientIPAddress(RequestContext.get().getClientIPAddress()); + request.setForwardedAddresses(RequestContext.get().getForwardedAddresses()); + request.setRemoteIPAddress(RequestContext.get().getClientIPAddress()); + } } diff --git a/authorization/src/main/java/org/apache/atlas/authorize/AtlasAuthorizer.java b/authorization/src/main/java/org/apache/atlas/authorize/AtlasAuthorizer.java index f9de07877f4..22aea9ea6cc 100644 --- a/authorization/src/main/java/org/apache/atlas/authorize/AtlasAuthorizer.java +++ b/authorization/src/main/java/org/apache/atlas/authorize/AtlasAuthorizer.java @@ -24,6 +24,8 @@ import org.apache.atlas.type.AtlasStructType; import org.apache.atlas.type.AtlasTypeRegistry; +import java.util.Set; + public interface AtlasAuthorizer { /** * initialization of authorizer implementation @@ -59,6 +61,14 @@ public interface AtlasAuthorizer { */ boolean isAccessAllowed(AtlasTypeAccessRequest request) throws AtlasAuthorizationException; + AtlasAccessorResponse getAccessors(AtlasEntityAccessRequest request); + + AtlasAccessorResponse getAccessors(AtlasRelationshipAccessRequest request); + + AtlasAccessorResponse getAccessors(AtlasTypeAccessRequest request); + + Set getRolesForCurrentUser(String userName, Set groups); + /** * authorize relationship type * @param request @@ -133,25 +143,15 @@ default void scrubEntityHeader(AtlasEntityHeader entity, AtlasTypeRegistry typeR entity.setScrubbed(isScrubbed); - if (entity.getClassifications() != null) { - entity.getClassifications().clear(); - } - - if (entity.getClassificationNames() != null) { - entity.getClassificationNames().clear(); - } - - if (entity.getMeanings() != null) { - entity.getMeanings().clear(); - } - - if (entity.getMeaningNames() != null) { - entity.getMeaningNames().clear(); - } } default void filterTypesDef(AtlasTypesDefFilterRequest request) throws AtlasAuthorizationException { } + + default + public void init(AtlasTypeRegistry typeRegistry) { + + } } diff --git a/authorization/src/main/java/org/apache/atlas/authorize/AtlasAuthorizerFactory.java b/authorization/src/main/java/org/apache/atlas/authorize/AtlasAuthorizerFactory.java index 72037eabbe0..0678dfd4086 100644 --- a/authorization/src/main/java/org/apache/atlas/authorize/AtlasAuthorizerFactory.java +++ b/authorization/src/main/java/org/apache/atlas/authorize/AtlasAuthorizerFactory.java @@ -21,6 +21,7 @@ import org.apache.atlas.ApplicationProperties; import org.apache.atlas.AtlasException; import org.apache.atlas.authorize.simple.AtlasSimpleAuthorizer; +import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.commons.configuration.Configuration; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; @@ -33,10 +34,14 @@ public class AtlasAuthorizerFactory { private static final String NONE_AUTHORIZER = AtlasNoneAuthorizer.class.getName(); private static final String SIMPLE_AUTHORIZER = AtlasSimpleAuthorizer.class.getName(); private static final String RANGER_AUTHORIZER = "org.apache.ranger.authorization.atlas.authorizer.RangerAtlasAuthorizer"; + private static final String ATLAS_AUTHORIZER = "org.apache.atlas.authorization.atlas.authorizer.RangerAtlasAuthorizer"; private static volatile AtlasAuthorizer INSTANCE = null; - public static AtlasAuthorizer getAtlasAuthorizer() throws AtlasAuthorizationException { + public static String CURRENT_AUTHORIZER_IMPL; + public static final String ATLAS_AUTHORIZER_IMPL = "atlas"; + + public static AtlasAuthorizer getAtlasAuthorizer(AtlasTypeRegistry typeRegistry) throws AtlasAuthorizationException { AtlasAuthorizer ret = INSTANCE; if (ret == null) { @@ -50,14 +55,17 @@ public static AtlasAuthorizer getAtlasAuthorizer() throws AtlasAuthorizationExce LOG.error("Exception while fetching configuration", e); } - String authorizerClass = configuration != null ? configuration.getString("atlas.authorizer.impl") : "SIMPLE"; + CURRENT_AUTHORIZER_IMPL = configuration != null ? configuration.getString("atlas.authorizer.impl") : "SIMPLE"; + String authorizerClass = RANGER_AUTHORIZER; - if (StringUtils.isNotEmpty(authorizerClass)) { - if (StringUtils.equalsIgnoreCase(authorizerClass, "SIMPLE")) { + if (StringUtils.isNotEmpty(CURRENT_AUTHORIZER_IMPL)) { + if (StringUtils.equalsIgnoreCase(CURRENT_AUTHORIZER_IMPL, "SIMPLE")) { authorizerClass = SIMPLE_AUTHORIZER; - } else if (StringUtils.equalsIgnoreCase(authorizerClass, "RANGER")) { + } else if (StringUtils.equalsIgnoreCase(CURRENT_AUTHORIZER_IMPL, "RANGER")) { authorizerClass = RANGER_AUTHORIZER; - } else if (StringUtils.equalsIgnoreCase(authorizerClass, "NONE")) { + } else if (StringUtils.equalsIgnoreCase(CURRENT_AUTHORIZER_IMPL, ATLAS_AUTHORIZER_IMPL)) { + authorizerClass = ATLAS_AUTHORIZER; + } else if (StringUtils.equalsIgnoreCase(CURRENT_AUTHORIZER_IMPL, "NONE")) { authorizerClass = NONE_AUTHORIZER; } } else { @@ -72,7 +80,11 @@ public static AtlasAuthorizer getAtlasAuthorizer() throws AtlasAuthorizationExce if (authorizerMetaObject != null) { INSTANCE = (AtlasAuthorizer) authorizerMetaObject.newInstance(); - INSTANCE.init(); + if (StringUtils.equalsIgnoreCase(CURRENT_AUTHORIZER_IMPL, ATLAS_AUTHORIZER_IMPL)) { + INSTANCE.init(typeRegistry); + } else { + INSTANCE.init(); + } } } catch (Exception e) { LOG.error("Error while creating authorizer of type {}", authorizerClass, e); @@ -87,4 +99,8 @@ public static AtlasAuthorizer getAtlasAuthorizer() throws AtlasAuthorizationExce return ret; } + + public static AtlasAuthorizer getAtlasAuthorizer() throws AtlasAuthorizationException { + return INSTANCE; + } } diff --git a/authorization/src/main/java/org/apache/atlas/authorize/AtlasEntityAccessRequest.java b/authorization/src/main/java/org/apache/atlas/authorize/AtlasEntityAccessRequest.java index cfa8703454b..df164864e74 100644 --- a/authorization/src/main/java/org/apache/atlas/authorize/AtlasEntityAccessRequest.java +++ b/authorization/src/main/java/org/apache/atlas/authorize/AtlasEntityAccessRequest.java @@ -32,7 +32,7 @@ public class AtlasEntityAccessRequest extends AtlasAccessRequest { private final String businessMetadata; private final String attributeName; private final AtlasTypeRegistry typeRegistry; - private final Set entityClassifications; + private final Set entityClassifications; private final boolean auditEnabled; @@ -116,7 +116,7 @@ public String getEntityType() { return entity == null ? StringUtils.EMPTY : entity.getTypeName(); } - public Set getEntityClassifications() { + public Set getEntityClassifications() { return entityClassifications; } diff --git a/authorization/src/main/java/org/apache/atlas/authorize/AtlasNoneAuthorizer.java b/authorization/src/main/java/org/apache/atlas/authorize/AtlasNoneAuthorizer.java index a371049441b..02993c7291e 100644 --- a/authorization/src/main/java/org/apache/atlas/authorize/AtlasNoneAuthorizer.java +++ b/authorization/src/main/java/org/apache/atlas/authorize/AtlasNoneAuthorizer.java @@ -21,6 +21,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Set; + public class AtlasNoneAuthorizer implements AtlasAuthorizer { private static final Logger LOG = LoggerFactory.getLogger(AtlasNoneAuthorizer.class); @@ -45,6 +47,26 @@ public boolean isAccessAllowed(AtlasTypeAccessRequest request) throws AtlasAutho return true; } + @Override + public AtlasAccessorResponse getAccessors(AtlasEntityAccessRequest request) { + return null; + } + + @Override + public AtlasAccessorResponse getAccessors(AtlasRelationshipAccessRequest request) { + return null; + } + + @Override + public AtlasAccessorResponse getAccessors(AtlasTypeAccessRequest request) { + return null; + } + + @Override + public Set getRolesForCurrentUser(String userName, Set groups) { + return null; + } + @Override public boolean isAccessAllowed(AtlasRelationshipAccessRequest request) throws AtlasAuthorizationException { return true; diff --git a/authorization/src/main/java/org/apache/atlas/authorize/AtlasPrivilege.java b/authorization/src/main/java/org/apache/atlas/authorize/AtlasPrivilege.java index 41a0c4d4aad..aaf02cbe7a7 100644 --- a/authorization/src/main/java/org/apache/atlas/authorize/AtlasPrivilege.java +++ b/authorization/src/main/java/org/apache/atlas/authorize/AtlasPrivilege.java @@ -48,8 +48,10 @@ public enum AtlasPrivilege { ADMIN_AUDITS("admin-audits"), - ADMIN_ENTITY_AUDITS("admin-entity-audits"); + ADMIN_ENTITY_AUDITS("admin-entity-audits"), + ADMIN_REPAIR_INDEX("admin-repair-index"), + ADMIN_TASK_CUD("admin-task-cud"); private final String type; AtlasPrivilege(String actionType){ diff --git a/authorization/src/main/java/org/apache/atlas/authorize/AtlasRelationshipAccessRequest.java b/authorization/src/main/java/org/apache/atlas/authorize/AtlasRelationshipAccessRequest.java index b530c01dd4b..bd1a0634c70 100644 --- a/authorization/src/main/java/org/apache/atlas/authorize/AtlasRelationshipAccessRequest.java +++ b/authorization/src/main/java/org/apache/atlas/authorize/AtlasRelationshipAccessRequest.java @@ -18,6 +18,7 @@ package org.apache.atlas.authorize; +import org.apache.atlas.model.instance.AtlasClassification; import org.apache.atlas.model.instance.AtlasEntityHeader; import org.apache.atlas.type.AtlasTypeRegistry; @@ -60,7 +61,7 @@ public Set getEnd1EntityTypeAndAllSuperTypes() { return super.getEntityTypeAndAllSuperTypes(end1Entity == null ? null : end1Entity.getTypeName(), typeRegistry); } - public Set getEnd1EntityClassifications() { + public Set getEnd1EntityClassifications() { return super.getClassificationNames(end1Entity); } @@ -72,7 +73,7 @@ public Set getEnd2EntityTypeAndAllSuperTypes() { return super.getEntityTypeAndAllSuperTypes(end2Entity == null ? null : end2Entity.getTypeName(), typeRegistry); } - public Set getEnd2EntityClassifications() { + public Set getEnd2EntityClassifications() { return super.getClassificationNames(end2Entity); } diff --git a/authorization/src/main/java/org/apache/atlas/authorize/simple/AtlasSimpleAuthorizer.java b/authorization/src/main/java/org/apache/atlas/authorize/simple/AtlasSimpleAuthorizer.java index 8b377e4dbe8..16dd9f68de8 100644 --- a/authorization/src/main/java/org/apache/atlas/authorize/simple/AtlasSimpleAuthorizer.java +++ b/authorization/src/main/java/org/apache/atlas/authorize/simple/AtlasSimpleAuthorizer.java @@ -32,10 +32,11 @@ import org.apache.atlas.authorize.simple.AtlasSimpleAuthzPolicy.*; import org.apache.atlas.model.discovery.AtlasSearchResult; import org.apache.atlas.model.discovery.AtlasSearchResult.AtlasFullTextResult; +import org.apache.atlas.authorize.AtlasAccessorResponse; +import org.apache.atlas.model.instance.AtlasClassification; import org.apache.atlas.model.instance.AtlasEntityHeader; import org.apache.atlas.model.typedef.AtlasBaseTypeDef; import org.apache.atlas.model.typedef.AtlasTypesDef; -import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.utils.AtlasJson; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; @@ -177,15 +178,38 @@ public boolean isAccessAllowed(AtlasTypeAccessRequest request) throws AtlasAutho return ret; } + @Override + public AtlasAccessorResponse getAccessors(AtlasEntityAccessRequest request) { + //TODO: implement method + return null; + } + + @Override + public AtlasAccessorResponse getAccessors(AtlasRelationshipAccessRequest request) { + //TODO: implement method + return null; + } + + @Override + public AtlasAccessorResponse getAccessors(AtlasTypeAccessRequest request) { + //TODO: implement method + return null; + } + + @Override + public Set getRolesForCurrentUser(String userName, Set groups) { + return null; + } + @Override public boolean isAccessAllowed(AtlasRelationshipAccessRequest request) throws AtlasAuthorizationException { final Set roles = getRoles(request.getUser(), request.getUserGroups()); final String relationShipType = request.getRelationshipType(); final Set end1EntityTypeAndSuperTypes = request.getEnd1EntityTypeAndAllSuperTypes(); - final Set end1Classifications = new HashSet<>(request.getEnd1EntityClassifications()); + final Set end1Classifications = new HashSet<>(request.getEnd1EntityClassifications()); final String end1EntityId = request.getEnd1EntityId(); final Set end2EntityTypeAndSuperTypes = request.getEnd2EntityTypeAndAllSuperTypes(); - final Set end2Classifications = new HashSet<>(request.getEnd2EntityClassifications()); + final Set end2Classifications = new HashSet<>(request.getEnd2EntityClassifications()); final String end2EntityId = request.getEnd2EntityId(); final String action = request.getAction() != null ? request.getAction().getType() : null; @@ -204,10 +228,10 @@ public boolean isAccessAllowed(AtlasRelationshipAccessRequest request) throws At //End1 permission check if (!hasEnd1EntityAccess) { if (isMatchAny(end1EntityTypeAndSuperTypes, permission.getEnd1EntityType()) && isMatch(end1EntityId, permission.getEnd1EntityId())) { - for (Iterator iter = end1Classifications.iterator(); iter.hasNext();) { - String entityClassification = iter.next(); + for (Iterator iter = end1Classifications.iterator(); iter.hasNext();) { + AtlasClassification entityClassification = iter.next(); - if (isMatchAny(request.getClassificationTypeAndAllSuperTypes(entityClassification), permission.getEnd1EntityClassification())) { + if (isMatchAny(request.getClassificationTypeAndAllSuperTypes(entityClassification.getTypeName()), permission.getEnd1EntityClassification())) { iter.remove(); } } @@ -219,10 +243,10 @@ public boolean isAccessAllowed(AtlasRelationshipAccessRequest request) throws At //End2 permission chech if (!hasEnd2EntityAccess) { if (isMatchAny(end2EntityTypeAndSuperTypes, permission.getEnd2EntityType()) && isMatch(end2EntityId, permission.getEnd2EntityId())) { - for (Iterator iter = end2Classifications.iterator(); iter.hasNext();) { - String entityClassification = iter.next(); + for (Iterator iter = end2Classifications.iterator(); iter.hasNext();) { + AtlasClassification entityClassification = iter.next(); - if (isMatchAny(request.getClassificationTypeAndAllSuperTypes(entityClassification), permission.getEnd2EntityClassification())) { + if (isMatchAny(request.getClassificationTypeAndAllSuperTypes(entityClassification.getTypeName()), permission.getEnd2EntityClassification())) { iter.remove(); } } @@ -248,7 +272,7 @@ public boolean isAccessAllowed(AtlasEntityAccessRequest request) throws AtlasAut final Set entityTypes = request.getEntityTypeAndAllSuperTypes(); final String entityId = request.getEntityId(); final String attribute = request.getAttributeName(); - final Set entClsToAuthz = new HashSet<>(request.getEntityClassifications()); + final Set entClsToAuthz = new HashSet<>(request.getEntityClassifications()); final Set roles = getRoles(request.getUser(), request.getUserGroups()); for (String role : roles) { @@ -264,10 +288,10 @@ public boolean isAccessAllowed(AtlasEntityAccessRequest request) throws AtlasAut // 2. access for these classifications could be granted by multiple AtlasEntityPermission entries // 3. classifications allowed by the current permission will be removed from entClsToAuthz // 4. request will be allowed once entClsToAuthz is empty i.e. user has permission for all classifications - for (Iterator iter = entClsToAuthz.iterator(); iter.hasNext(); ) { - String entityClassification = iter.next(); + for (Iterator iter = entClsToAuthz.iterator(); iter.hasNext(); ) { + AtlasClassification entityClassification = iter.next(); - if (isMatchAny(request.getClassificationTypeAndAllSuperTypes(entityClassification), permission.getEntityClassifications())) { + if (isMatchAny(request.getClassificationTypeAndAllSuperTypes(entityClassification.getTypeName()), permission.getEntityClassifications())) { iter.remove(); } } diff --git a/build.sh b/build.sh index 862319cf099..d3f77f56476 100755 --- a/build.sh +++ b/build.sh @@ -22,7 +22,12 @@ wget https://atlan-public.s3.eu-west-1.amazonaws.com/artifact/keycloak-15.0.2.1 unzip -o keycloak-15.0.2.1.zip -d ~/.m2/repository/org echo "Maven Building" -mvn -T 100 -pl '!addons/hdfs-model,!addons/hive-bridge,!addons/hive-bridge-shim,!addons/falcon-bridge-shim,!addons/falcon-bridge,!addons/sqoop-bridge,!addons/sqoop-bridge-shim,!addons/hbase-bridge,!addons/hbase-bridge-shim' -Dmaven.test.skip -DskipTests -Drat.skip=true package -Pdist + +if [ "$1" == "build_without_dashboard" ]; then + mvn -pl '!addons/hdfs-model,!addons/hive-bridge,!addons/hive-bridge-shim,!addons/falcon-bridge-shim,!addons/falcon-bridge,!addons/sqoop-bridge,!addons/sqoop-bridge-shim,!addons/hbase-bridge,!addons/hbase-bridge-shim,!addons/hbase-testing-util,!addons/kafka-bridge,!addons/impala-hook-api,!addons/impala-bridge-shim,!addons/impala-bridge,!dashboardv2,!dashboardv3' -Dmaven.test.skip -DskipTests -Drat.skip=true -DskipOverlay -DskipEnunciate=true package -Pdist +else + mvn -pl '!addons/hdfs-model,!addons/hive-bridge,!addons/hive-bridge-shim,!addons/falcon-bridge-shim,!addons/falcon-bridge,!addons/sqoop-bridge,!addons/sqoop-bridge-shim,!addons/hbase-bridge,!addons/hbase-bridge-shim,!addons/hbase-testing-util,!addons/kafka-bridge,!addons/impala-hook-api,!addons/impala-bridge-shim,!addons/impala-bridge' -Dmaven.test.skip -DskipTests -Drat.skip=true -DskipEnunciate=true package -Pdist +fi echo "[DEBUG listing distro/target" ls distro/target diff --git a/client-keycloak/pom.xml b/client-keycloak/pom.xml new file mode 100644 index 00000000000..b0a231b0c34 --- /dev/null +++ b/client-keycloak/pom.xml @@ -0,0 +1,77 @@ + + + + + + apache-atlas + org.apache.atlas + 3.0.0-SNAPSHOT + + 4.0.0 + + client-keycloak + + + 8 + 8 + 15.1.0 + + + + + org.keycloak + keycloak-core + ${keycloak-admin-client.version} + + + * + * + + + + + + com.squareup.okhttp3 + okhttp + ${okhttp3.version} + + + com.squareup.retrofit2 + retrofit + ${retrofit.version} + + + com.squareup.retrofit2 + converter-jackson + ${retrofit.version} + + + com.squareup.okhttp3 + logging-interceptor + ${okhttp3.version} + + + org.apache.atlas + atlas-common + + + + \ No newline at end of file diff --git a/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/AbstractKeycloakClient.java b/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/AbstractKeycloakClient.java new file mode 100644 index 00000000000..86bdf6fbf7d --- /dev/null +++ b/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/AbstractKeycloakClient.java @@ -0,0 +1,146 @@ +package org.apache.atlas.keycloak.client; + +import com.fasterxml.jackson.databind.ObjectMapper; +import io.micrometer.core.instrument.Timer; +import okhttp3.*; +import okhttp3.logging.HttpLoggingInterceptor; +import org.apache.atlas.AtlasErrorCode; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.keycloak.client.config.KeycloakConfig; +import org.apache.atlas.keycloak.client.service.AtlasKeycloakAuthService; +import org.apache.atlas.service.metrics.MetricUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.lang.NonNull; +import retrofit2.Retrofit; +import retrofit2.converter.jackson.JacksonConverterFactory; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.TimeUnit; + +import static java.net.HttpURLConnection.HTTP_BAD_REQUEST; +import static java.net.HttpURLConnection.HTTP_NOT_FOUND; +import static org.apache.atlas.AtlasErrorCode.BAD_REQUEST; +import static org.apache.atlas.AtlasErrorCode.RESOURCE_NOT_FOUND; + +abstract class AbstractKeycloakClient { + + private final static Logger LOG = LoggerFactory.getLogger(AbstractKeycloakClient.class); + private static final Map ERROR_CODE_MAP = new HashMap<>(); + + private static final int DEFAULT_KEYCLOAK_RETRY = 3; + private static final String AUTHORIZATION = "Authorization"; + private static final String BEARER = "Bearer "; + private static final int TIMEOUT_IN_SEC = 60; + private static final String INTEGRATION = "integration"; + private static final String KEYCLOAK = "keycloak"; + + protected final KeycloakConfig keycloakConfig; + protected final RetrofitKeycloakClient retrofit; + + private final AtlasKeycloakAuthService authService; + private MetricUtils metricUtils = null; + + static { + ERROR_CODE_MAP.put(HTTP_NOT_FOUND, RESOURCE_NOT_FOUND); + ERROR_CODE_MAP.put(HTTP_BAD_REQUEST, BAD_REQUEST); + } + + public AbstractKeycloakClient(KeycloakConfig keycloakConfig) { + this.keycloakConfig = keycloakConfig; + this.metricUtils = new MetricUtils(); + HttpLoggingInterceptor httpInterceptor = new HttpLoggingInterceptor(); + httpInterceptor.setLevel(HttpLoggingInterceptor.Level.BODY); + OkHttpClient okHttpClient = new OkHttpClient.Builder() + .addInterceptor(accessTokenInterceptor) + .addInterceptor(httpInterceptor) + .addInterceptor(responseLoggingInterceptor) + .authenticator(authInterceptor) + .connectTimeout(TIMEOUT_IN_SEC, TimeUnit.SECONDS) + .callTimeout(TIMEOUT_IN_SEC, TimeUnit.SECONDS) + .writeTimeout(TIMEOUT_IN_SEC, TimeUnit.SECONDS) + .readTimeout(TIMEOUT_IN_SEC, TimeUnit.SECONDS) + .build(); + this.retrofit = new Retrofit.Builder().client(okHttpClient) + .baseUrl(this.keycloakConfig.getAuthServerUrl()) + .addConverterFactory(JacksonConverterFactory.create(new ObjectMapper())).build() + .create(RetrofitKeycloakClient.class); + authService = new AtlasKeycloakAuthService(keycloakConfig); + } + + /** + * Basic interceptor for logging. + */ + Interceptor responseLoggingInterceptor = chain -> { + Request request = chain.request(); + String rawPath = request.url().uri().getRawPath(); + Timer.Sample timerSample = this.metricUtils.start(rawPath); + okhttp3.Response response = chain.proceed(request); + this.metricUtils.recordHttpTimer(timerSample, request.method(), rawPath, response.code(), + INTEGRATION, KEYCLOAK); + return response; + }; + + /** + * Called for every request made to keycloak + */ + Interceptor accessTokenInterceptor = new Interceptor() { + @NonNull + @Override + public Response intercept(@NonNull Chain chain) throws IOException { + Request request = chain.request().newBuilder() + .header(AUTHORIZATION, BEARER + authService.getAuthToken()) + .build(); + return chain.proceed(request); + } + }; + + /** + * Called only during auth failures. + */ + Authenticator authInterceptor = new Authenticator() { + @Override + public Request authenticate(Route route, @NonNull Response response) { + if (responseCount(response) > DEFAULT_KEYCLOAK_RETRY) { + LOG.warn("Keycloak: Falling back, retried {} times", DEFAULT_KEYCLOAK_RETRY); + return null; + } + LOG.info("Keycloak: Current keycloak token status, Expired: {}", authService.isTokenExpired()); + return response.request().newBuilder() + .addHeader(AUTHORIZATION, BEARER + authService.getAuthToken()) + .build(); + } + + private int responseCount(Response response) { + int retryCount = 1; + while ((response = response.priorResponse()) != null) { + retryCount++; + } + return retryCount; + } + + }; + + /** + * Process incoming request, and error handling. + */ + protected retrofit2.Response processResponse(retrofit2.Call req) throws AtlasBaseException { + try { + retrofit2.Response response = req.execute(); + if (Objects.isNull(response.errorBody())) { + return response; + } + String errMsg = response.errorBody().string(); + LOG.error("Keycloak: Client request processing failed code {} message:{}, request: {} {}", + response.code(), errMsg, req.request().method(), req.request().url()); + throw new AtlasBaseException(ERROR_CODE_MAP.getOrDefault(response.code(), BAD_REQUEST), errMsg); + } catch (Exception e) { + LOG.error("Keycloak: request failed, request: {} {}, Exception: {}", req.request().method(), req.request().url(), e); + throw new AtlasBaseException(BAD_REQUEST, "Keycloak request failed"); + } + } + +} diff --git a/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/AtlasKeycloakClient.java b/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/AtlasKeycloakClient.java new file mode 100644 index 00000000000..fe723bbce1b --- /dev/null +++ b/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/AtlasKeycloakClient.java @@ -0,0 +1,238 @@ +package org.apache.atlas.keycloak.client; + +import org.apache.atlas.AtlasErrorCode; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.keycloak.client.config.KeycloakConfig; +import org.apache.atlas.keycloak.client.config.KeycloakConfigBuilder; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; +import org.codehaus.jettison.json.JSONException; +import org.codehaus.jettison.json.JSONObject; +import org.keycloak.representations.idm.*; +import org.keycloak.representations.oidc.TokenMetadataRepresentation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.Set; + +import static org.apache.atlas.ApplicationProperties.ATLAS_CONFIGURATION_DIRECTORY_PROPERTY; + +/** + * Keycloak client, deals with token creation refresh. + */ +public final class AtlasKeycloakClient { + + public final static Logger LOG = LoggerFactory.getLogger(AtlasKeycloakClient.class); + + private final static String KEYCLOAK_PROPERTIES = "keycloak.json"; + private final static String DEFAULT_GRANT_TYPE = "client_credentials"; + private final static String KEY_REALM_ID = "realm"; + private final static String KEY_AUTH_SERVER_URL = "auth-server-url"; + private final static String KEY_CLIENT_ID = "resource"; + private final static String KEY_CREDENTIALS = "credentials"; + private final static String KEY_SECRET = "secret"; + + private static KeycloakRestClient KEYCLOAK; + private static AtlasKeycloakClient KEYCLOAK_CLIENT; + + private AtlasKeycloakClient() { + } + + public List searchUserByUserName(String username) throws AtlasBaseException { + return KEYCLOAK.searchUserByUserName(username).body(); + } + + public Set getRoleUserMembers(String roleName, int start, int size) throws AtlasBaseException { + return KEYCLOAK.getRoleUserMembers(roleName, start, size).body(); + } + + public List getRoleUserMembers(String roleName) throws AtlasBaseException { + return KEYCLOAK.getRoleUserMembers(roleName).body(); + } + + public List getAllUsers() throws AtlasBaseException { + int start = 0; + int size = 500; + boolean found = true; + + List ret = new ArrayList<>(0); + + do { + List userRepresentations = KEYCLOAK.getAllUsers(start, size).body(); + if (CollectionUtils.isNotEmpty(userRepresentations)) { + ret.addAll(userRepresentations); + start += size; + } else { + found = false; + } + + } while (found && ret.size() % size == 0); + + return ret; + } + + public List searchGroupByName(String groupName, Integer start, Integer size) throws AtlasBaseException { + return KEYCLOAK.searchGroupByName(groupName, start, size).body(); + } + + public List getRoleGroupMembers(String roleName) throws AtlasBaseException { + return KEYCLOAK.getRoleGroupMembers(roleName).body(); + } + + public Set getRoleGroupMembers(String roleName, Integer start, Integer size) throws AtlasBaseException { + return KEYCLOAK.getRoleGroupMembers(roleName, start, size).body(); + } + + public List getGroupsForUserById(String userId) throws AtlasBaseException { + return KEYCLOAK.getGroupsForUserById(userId).body(); + } + + public void addRealmLevelRoleMappingsForGroup(String groupId, List roles) throws AtlasBaseException { + KEYCLOAK.addRealmLevelRoleMappingsForGroup(groupId, roles); + } + + public void deleteRealmLevelRoleMappingsForGroup(String groupId, List roles) throws AtlasBaseException { + KEYCLOAK.deleteRealmLevelRoleMappingsForGroup(groupId, roles); + } + + public List getAllRoles() throws AtlasBaseException { + int start = 0; + int size = 500; + boolean found = true; + + List ret = new ArrayList<>(0); + do { + List roleRepresentations = KEYCLOAK.getAllRoles(start, size).body(); + if (CollectionUtils.isNotEmpty(roleRepresentations)) { + ret.addAll(roleRepresentations); + start += size; + } else { + found = false; + } + } while (found && ret.size() % size == 0); + + return ret; + } + + public void deleteRoleById(String roleId) throws AtlasBaseException { + KEYCLOAK.deleteRoleById(roleId); + } + + public void deleteRoleByName(String roleName) throws AtlasBaseException { + KEYCLOAK.deleteRoleByName(roleName); + } + + + public List addRealmLevelRoleMappingsForUser(String userId, List roles) throws AtlasBaseException { + return KEYCLOAK.addRealmLevelRoleMappingsForUser(userId, roles).body(); + } + + public void deleteRealmLevelRoleMappingsForUser(String userId, List roles) throws AtlasBaseException { + KEYCLOAK.deleteRealmLevelRoleMappingsForUser(userId, roles); + } + + public void createRole(RoleRepresentation roleRepresentation) throws AtlasBaseException { + KEYCLOAK.createRole(roleRepresentation); + } + + public void updateRole(String roleId, RoleRepresentation roleRepresentation) throws AtlasBaseException { + KEYCLOAK.updateRole(roleId, roleRepresentation); + } + + + public RoleRepresentation getRoleById(String roleId) throws AtlasBaseException { + return KEYCLOAK.getRoleById(roleId).body(); + } + + public RoleRepresentation getRoleByName(String roleName) throws AtlasBaseException { + return KEYCLOAK.getRoleByName(roleName).body(); + } + + public Set getRoleComposites(String roleName) throws AtlasBaseException { + return KEYCLOAK.getRoleComposites(roleName).body(); + } + + public void addComposites(String roleName, List roles) throws AtlasBaseException { + KEYCLOAK.addComposites(roleName, roles); + } + + public void deleteComposites(String roleName, List roles) throws AtlasBaseException { + KEYCLOAK.deleteComposites(roleName, roles); + } + + public List getAdminEvents(List operationTypes, String authRealm, String authClient, String authUser, String authIpAddress, String resourcePath, String dateFrom, String dateTo, Integer first, Integer max) throws AtlasBaseException { + return KEYCLOAK.getAdminEvents(operationTypes, authRealm, authClient, authUser, authIpAddress, resourcePath, dateFrom, dateTo, first, max).body(); + } + + public List getEvents(List type, String client, String user, String dateFrom, String dateTo, String ipAddress, Integer first, Integer max) throws AtlasBaseException { + return KEYCLOAK.getEvents(type, client, user, dateFrom, dateTo, ipAddress, first, max).body(); + } + + public TokenMetadataRepresentation introspectToken(String token) throws AtlasBaseException { + return KEYCLOAK.introspectToken(token).body(); + } + + public static AtlasKeycloakClient getKeycloakClient() throws AtlasBaseException { + if (Objects.isNull(KEYCLOAK_CLIENT)) { + LOG.info("Initializing Keycloak client.."); + try { + init(getConfig()); + } catch (IOException e) { + LOG.error("Failed to fetch Keycloak conf {}", e.getMessage()); + throw new AtlasBaseException(AtlasErrorCode.KEYCLOAK_INIT_FAILED, e.getMessage()); + } catch (JSONException e) { + LOG.error("Failed to parse Keycloak conf {}", e.getMessage()); + throw new AtlasBaseException(AtlasErrorCode.KEYCLOAK_INIT_FAILED, e.getMessage()); + } catch (Exception e) { + LOG.error("Failed to connect to Keycloak {}", e.getMessage()); + throw new AtlasBaseException(AtlasErrorCode.KEYCLOAK_INIT_FAILED, e.getMessage()); + } + + LOG.info("Initialized Keycloak client.."); + } + + return KEYCLOAK_CLIENT; + } + + private static void init(KeycloakConfig config) { + synchronized (AtlasKeycloakClient.class) { + if (KEYCLOAK_CLIENT == null) { + KEYCLOAK = new KeycloakRestClient(config); + KEYCLOAK_CLIENT = new AtlasKeycloakClient(); + } + } + } + + private static KeycloakConfig getConfig() throws Exception { + String confLocation = System.getProperty(ATLAS_CONFIGURATION_DIRECTORY_PROPERTY); + File confFile; + if (StringUtils.isNotEmpty(confLocation)) { + confFile = new File(confLocation, KEYCLOAK_PROPERTIES); + + if (confFile.exists()) { + String keyConf = new String(Files.readAllBytes(confFile.toPath()), StandardCharsets.UTF_8); + JSONObject object = new JSONObject(keyConf); + + String REALM_ID = object.getString(KEY_REALM_ID); + String AUTH_SERVER_URL = object.getString(KEY_AUTH_SERVER_URL) + "/"; + String CLIENT_ID = object.getString(KEY_CLIENT_ID); + String GRANT_TYPE = DEFAULT_GRANT_TYPE; + String CLIENT_SECRET = object.getJSONObject(KEY_CREDENTIALS).getString(KEY_SECRET); + + LOG.info("Keycloak conf: REALM_ID:{}, AUTH_SERVER_URL:{}", REALM_ID, AUTH_SERVER_URL); + return KeycloakConfigBuilder.builder().realId(REALM_ID).authServerUrl(AUTH_SERVER_URL).clientId(CLIENT_ID).grantType(GRANT_TYPE).clientSecret(CLIENT_SECRET).build(); + } else { + throw new AtlasBaseException(AtlasErrorCode.KEYCLOAK_INIT_FAILED, "Keycloak configuration file not found in location " + confLocation); + } + } else { + throw new AtlasBaseException(AtlasErrorCode.KEYCLOAK_INIT_FAILED, "Configuration location not found " + confLocation); + } + } +} diff --git a/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/KeycloakRestClient.java b/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/KeycloakRestClient.java new file mode 100644 index 00000000000..7cc1eea22f6 --- /dev/null +++ b/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/KeycloakRestClient.java @@ -0,0 +1,142 @@ +package org.apache.atlas.keycloak.client; + +import okhttp3.FormBody; +import okhttp3.RequestBody; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.keycloak.client.config.KeycloakConfig; +import org.keycloak.representations.idm.*; +import org.keycloak.representations.oidc.TokenMetadataRepresentation; +import retrofit2.Response; + +import java.util.List; +import java.util.Set; + +/** + * Keycloak Rest client wrapper used in atlas metastore + */ +public final class KeycloakRestClient extends AbstractKeycloakClient { + + private static final String TOKEN = "token"; + private static final String CLIENT_ID = "client_id"; + private static final String CLIENT_SECRET = "client_secret"; + public KeycloakRestClient(final KeycloakConfig keycloakConfig) { + super(keycloakConfig); + } + + public Response> searchUserByUserName(String username) throws AtlasBaseException { + return processResponse(this.retrofit.searchUserByUserName(this.keycloakConfig.getRealmId(), username)); + } + + public Response> getAllUsers(int start, int size) throws AtlasBaseException { + return processResponse(this.retrofit.getAllUsers(this.keycloakConfig.getRealmId(), start, size)); + } + + public Response> getRoleUserMembers(String roleName) throws AtlasBaseException { + return processResponse(this.retrofit.getRoleUserMembers(this.keycloakConfig.getRealmId(), roleName)); + } + + public Response> getRoleUserMembers(String roleName, Integer start, Integer size) throws AtlasBaseException { + return processResponse(this.retrofit.getRoleUserMembers(this.keycloakConfig.getRealmId(), roleName, start, size)); + } + + public Response> searchGroupByName(String groupName, Integer start, Integer size) throws AtlasBaseException { + return processResponse(this.retrofit.searchGroupByName(this.keycloakConfig.getRealmId(), groupName, start, size)); + } + + public Response> getRoleGroupMembers(String roleName) throws AtlasBaseException { + return processResponse(this.retrofit.getRoleGroupMembers(this.keycloakConfig.getRealmId(), roleName)); + } + + public Response> getRoleGroupMembers(String roleName, Integer first, Integer size) throws AtlasBaseException { + return processResponse(this.retrofit.getRoleGroupMembers(this.keycloakConfig.getRealmId(), roleName, first, size)); + } + + public Response> getGroupsForUserById(String userId) throws AtlasBaseException { + return processResponse(this.retrofit.getGroupsForUserById(this.keycloakConfig.getRealmId(), userId)); + } + + public void addRealmLevelRoleMappingsForGroup(String groupId, List roles) throws AtlasBaseException { + processResponse(this.retrofit.addRealmLevelRoleMappingsForGroup(this.keycloakConfig.getRealmId(), groupId, roles)); + } + + public void deleteRealmLevelRoleMappingsForGroup(String groupId, List roles) throws AtlasBaseException { + processResponse(this.retrofit.deleteRealmLevelRoleMappingsForGroup(this.keycloakConfig.getRealmId(), groupId, roles)); + } + + public Response> getAllRoles(int start, int size) throws AtlasBaseException { + return processResponse(this.retrofit.getAllRoles(this.keycloakConfig.getRealmId(), start, size)); + } + + public void deleteRoleById(String roleId) throws AtlasBaseException { + processResponse(this.retrofit.deleteRoleById(this.keycloakConfig.getRealmId(), roleId)); + } + + public void deleteRoleByName(String roleName) throws AtlasBaseException { + processResponse(this.retrofit.deleteRoleByName(this.keycloakConfig.getRealmId(), roleName)); + } + + public Response> addRealmLevelRoleMappingsForUser(String userId, List roles) throws AtlasBaseException { + return processResponse(this.retrofit.addRealmLevelRoleMappingsForUser(this.keycloakConfig.getRealmId(), userId, roles)); + } + + public void deleteRealmLevelRoleMappingsForUser(String userId, List roles) throws AtlasBaseException { + processResponse(this.retrofit.deleteRealmLevelRoleMappingsForUser(this.keycloakConfig.getRealmId(), userId, roles)); + } + + public void createRole(RoleRepresentation roleRepresentation) throws AtlasBaseException { + processResponse(this.retrofit.createRole(this.keycloakConfig.getRealmId(), roleRepresentation)); + } + + public void updateRole(String roleId, RoleRepresentation roleRepresentation) throws AtlasBaseException { + processResponse(this.retrofit.updateRole(this.keycloakConfig.getRealmId(), roleId, roleRepresentation)); + } + + public Response getRoleById(String roleId) throws AtlasBaseException { + return processResponse(this.retrofit.getRoleById(this.keycloakConfig.getRealmId(), roleId)); + } + + public Response getRoleByName(String roleName) throws AtlasBaseException { + return processResponse(this.retrofit.getRoleByName(this.keycloakConfig.getRealmId(), roleName)); + } + + public Response> getAllRoles(Integer first, Integer max) throws AtlasBaseException { + return processResponse(this.retrofit.getAllRoles(this.keycloakConfig.getRealmId(), first, max)); + } + + public Response> getRoleComposites(String roleName) throws AtlasBaseException { + return processResponse(this.retrofit.getRoleComposites(this.keycloakConfig.getRealmId(), roleName)); + } + + public void addComposites(String roleName, List roles) throws AtlasBaseException { + processResponse(this.retrofit.addComposites(this.keycloakConfig.getRealmId(), roleName, roles)); + } + + public void deleteComposites(String roleName, List roles) throws AtlasBaseException { + processResponse(this.retrofit.deleteComposites(this.keycloakConfig.getRealmId(), roleName, roles)); + } + + public Response> getAdminEvents(List operationTypes, String authRealm, + String authClient, String authUser, String authIpAddress, + String resourcePath, String dateFrom, String dateTo, + Integer first, Integer max) throws AtlasBaseException { + return processResponse(this.retrofit.getAdminEvents(this.keycloakConfig.getRealmId(), operationTypes, + authRealm, authClient, authUser, authIpAddress, resourcePath, dateFrom, dateTo, first, max)); + } + + public Response> getEvents(List type, String client, String user, String dateFrom, + String dateTo, String ipAddress, Integer first, Integer max) throws AtlasBaseException { + return processResponse(this.retrofit.getEvents(this.keycloakConfig.getRealmId(), type, client, user, dateFrom, dateTo, ipAddress, first, max)); + } + + public Response introspectToken(String token) throws AtlasBaseException { + return processResponse(this.retrofit.introspectToken(this.keycloakConfig.getRealmId(), getIntrospectTokenRequest(token))); + } + + private RequestBody getIntrospectTokenRequest(String token) { + return new FormBody.Builder() + .addEncoded(TOKEN, token) + .addEncoded(CLIENT_ID, this.keycloakConfig.getClientId()) + .addEncoded(CLIENT_SECRET, this.keycloakConfig.getClientSecret()) + .build(); + } +} diff --git a/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/RetrofitKeycloakClient.java b/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/RetrofitKeycloakClient.java new file mode 100644 index 00000000000..c396c9368dc --- /dev/null +++ b/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/RetrofitKeycloakClient.java @@ -0,0 +1,150 @@ +package org.apache.atlas.keycloak.client; + +import okhttp3.RequestBody; +import org.keycloak.representations.AccessTokenResponse; +import org.keycloak.representations.idm.*; +import org.keycloak.representations.oidc.TokenMetadataRepresentation; +import retrofit2.Call; +import retrofit2.http.*; + +import java.util.List; +import java.util.Set; + +public interface RetrofitKeycloakClient { + + /* Keycloak Users */ + + @Headers({"Accept: application/json", "Cache-Control: no-store", "Cache-Control: no-cache"}) + @GET("admin/realms/{realmId}/users") + Call> searchUserByUserName(@Path("realmId") String realmId, @Query("username") String username); + + @Headers({"Accept: application/json", "Cache-Control: no-store", "Cache-Control: no-cache"}) + @GET("admin/realms/{realmId}/users") + Call> getAllUsers(@Path("realmId") String realmId, @Query("first") Integer first, + @Query("max") Integer max); + + @Headers({"Accept: application/json", "Cache-Control: no-store", "Cache-Control: no-cache"}) + @GET("admin/realms/{realmId}/roles/{role-name}/users") + Call> getRoleUserMembers(@Path("realmId") String realmId, @Path("role-name") String roleName); + + @Headers({"Accept: application/json", "Cache-Control: no-store", "Cache-Control: no-cache"}) + @GET("admin/realms/{realmId}/roles/{role-name}/users") + Call> getRoleUserMembers(@Path("realmId") String realmId, @Path("role-name") String roleName, + @Query("first") Integer first, @Query("max") Integer max); + + + /* Keycloak Groups */ + + @Headers({"Accept: application/json", "Cache-Control: no-store", "Cache-Control: no-cache"}) + @GET("admin/realms/{realmId}/groups") + Call> searchGroupByName(@Path("realmId") String realmId, @Query("search") String groupName, + @Query("first") Integer first, @Query("max") Integer max); + + @Headers({"Accept: application/json", "Cache-Control: no-store", "Cache-Control: no-cache"}) + @GET("admin/realms/{realmId}/roles/{role-name}/groups") + Call> getRoleGroupMembers(@Path("realmId") String realmId, @Path("role-name") String roleName); + + @Headers({"Accept: application/json", "Cache-Control: no-store", "Cache-Control: no-cache"}) + @GET("admin/realms/{realmId}/roles/{role-name}/groups") + Call> getRoleGroupMembers(@Path("realmId") String realmId, @Path("role-name") String roleName, + @Query("first") Integer first, @Query("max") Integer max); + + @Headers({"Accept: application/json", "Cache-Control: no-store", "Cache-Control: no-cache"}) + @GET("admin/realms/{realmId}/users/{id}/groups") + Call> getGroupsForUserById(@Path("realmId") String realmId, @Path("id") String userId); + + + @Headers({"Accept: application/json", "Cache-Control: no-store", "Cache-Control: no-cache"}) + @POST("admin/realms/{realmId}/groups/{id}/role-mappings/realm") + Call addRealmLevelRoleMappingsForGroup(@Path("realmId") String realmId, @Path("id") String groupId, @Body List roles); + + @Headers({"Accept: application/json", "Cache-Control: no-store", "Cache-Control: no-cache"}) + @HTTP( method = "DELETE", path = "admin/realms/{realmId}/groups/{id}/role-mappings/realm", hasBody = true) + Call deleteRealmLevelRoleMappingsForGroup(@Path("realmId") String realmId, @Path("id") String groupId, @Body List roles); + + + /* Keycloak Roles */ + + @Headers({"Accept: application/json", "Cache-Control: no-store", "Cache-Control: no-cache"}) + @POST("admin/realms/{realmId}/users/{id}/role-mappings/realm") + Call> addRealmLevelRoleMappingsForUser(@Path("realmId") String realmId, @Path("id") String userId, + @Body List roles); + + @Headers({"Accept: application/json", "Cache-Control: no-store", "Cache-Control: no-cache"}) + @HTTP( method = "DELETE", path = "admin/realms/{realmId}/users/{id}/role-mappings/realm", hasBody = true) + Call deleteRealmLevelRoleMappingsForUser(@Path("realmId") String realmId, @Path("id") String userId, + @Body List roles); + + + @Headers({"Accept: application/json", "Cache-Control: no-store", "Cache-Control: no-cache"}) + @POST("admin/realms/{realmId}/roles") + Call createRole(@Path("realmId") String realmId, @Body RoleRepresentation role); + + @Headers({"Accept: application/json", "Cache-Control: no-store", "Cache-Control: no-cache"}) + @PUT("admin/realms/{realmId}/roles-by-id/{role-id}") + Call updateRole(@Path("realmId") String realmId, @Path("role-id") String roleId, @Body RoleRepresentation role); + + + @Headers({"Accept: application/json", "Cache-Control: no-store", "Cache-Control: no-cache"}) + @GET("admin/realms/{realmId}/roles-by-id/{role-id}") + Call getRoleById(@Path("realmId") String realmId, @Path("role-id") String roleId); + + @Headers({"Accept: application/json", "Cache-Control: no-store", "Cache-Control: no-cache"}) + @GET("admin/realms/{realmId}/roles/{role-name}") + Call getRoleByName(@Path("realmId") String realmId, @Path("role-name") String roleName); + + @Headers({"Accept: application/json", "Cache-Control: no-store", "Cache-Control: no-cache"}) + @GET("admin/realms/{realmId}/roles") + Call> getAllRoles(@Path("realmId") String realmId, @Query("first") Integer first, + @Query("max") Integer max); + + + @Headers({"Accept: application/json", "Cache-Control: no-store", "Cache-Control: no-cache"}) + @DELETE("admin/realms/{realmId}/roles-by-id/{role-id}") + Call deleteRoleById(@Path("realmId") String realmId, @Path("role-id") String roleId); + + @Headers({"Accept: application/json", "Cache-Control: no-store", "Cache-Control: no-cache"}) + @DELETE("admin/realms/{realmId}/roles/{role-name}") + Call deleteRoleByName(@Path("realmId") String realmId, @Path("role-name") String roleName); + + /* Keycloak composites */ + + @Headers({"Accept: application/json", "Cache-Control: no-store", "Cache-Control: no-cache"}) + @GET("admin/realms/{realmId}/roles/{role-name}/composites") + Call> getRoleComposites(@Path("realmId") String realmId, @Path("role-name") String roleName); + + @Headers({"Accept: application/json", "Cache-Control: no-store", "Cache-Control: no-cache"}) + @POST("admin/realms/{realmId}/roles/{role-name}/composites") + Call addComposites(@Path("realmId") String realmId, @Path("role-name") String roleName, @Body List roles); + + @Headers({"Accept: application/json", "Cache-Control: no-store", "Cache-Control: no-cache"}) + @HTTP( method = "DELETE", path = "admin/realms/{realmId}/roles/{role-name}/composites", hasBody = true) + Call deleteComposites(@Path("realmId") String realmId, @Path("role-name") String roleName, @Body List roles); + + @Headers({"Accept: application/json", "Cache-Control: no-store", "Cache-Control: no-cache"}) + @GET("admin/realms/{realmId}/admin-events") + Call> getAdminEvents(@Path("realmId") String realmId, @Query("operationTypes") List operationTypes, + @Query("authRealm") String authRealm, @Query("authClient") String authClient, + @Query("authUser") String authUser, @Query("authIpAddress") String authIpAddress, + @Query("resourcePath") String resourcePath, @Query("dateFrom") String dateFrom, + @Query("dateTo") String dateTo, @Query("first") Integer first, @Query("max") Integer max); + + @Headers({"Accept: application/json", "Cache-Control: no-store", "Cache-Control: no-cache"}) + @GET("admin/realms/{realmId}/events") + Call> getEvents(@Path("realmId") String realmId, @Query("type") List types, + @Query("client") String client, @Query("user") String user, + @Query("dateFrom") String dateFrom, @Query("dateTo") String dateTo, + @Query("ipAddress") String ipAddress, @Query("first") Integer first, + @Query("max") Integer max); + + /* Access token */ + + @Headers({"Accept: application/json", "Content-Type: application/x-www-form-urlencoded", "Cache-Control: no-store", "Cache-Control: no-cache"}) + @POST("realms/{realmId}/protocol/openid-connect/token") + Call grantToken(@Path("realmId") String realmId, @Body RequestBody request); + + @Headers({"Accept: application/json", "Content-Type: application/x-www-form-urlencoded", "Cache-Control: no-store", "Cache-Control: no-cache"}) + @POST("realms/{realmId}/protocol/openid-connect/token/introspect") + Call introspectToken(@Path("realmId") String realmId, @Body RequestBody request); + +} diff --git a/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/config/KeycloakConfig.java b/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/config/KeycloakConfig.java new file mode 100644 index 00000000000..048d2fac713 --- /dev/null +++ b/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/config/KeycloakConfig.java @@ -0,0 +1,31 @@ +package org.apache.atlas.keycloak.client.config; + +public final class KeycloakConfig { + + String authServerUrl; + String realmId; + String clientId; + String clientSecret; + String grantType; + + public String getAuthServerUrl() { + return authServerUrl; + } + + public String getRealmId() { + return realmId; + } + + public String getClientId() { + return clientId; + } + + public String getClientSecret() { + return clientSecret; + } + + public String getGrantType() { + return grantType; + } + +} diff --git a/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/config/KeycloakConfigBuilder.java b/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/config/KeycloakConfigBuilder.java new file mode 100644 index 00000000000..44583c59f3d --- /dev/null +++ b/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/config/KeycloakConfigBuilder.java @@ -0,0 +1,52 @@ +package org.apache.atlas.keycloak.client.config; + +public final class KeycloakConfigBuilder { + + private String authServerUrl; + private String realmId; + private String clientId; + private String clientSecret; + private String grantType = "client_credentials"; + + private KeycloakConfigBuilder() { + } + + public static KeycloakConfigBuilder builder() { + return new KeycloakConfigBuilder(); + } + + public KeycloakConfigBuilder authServerUrl(String authServerUrl) { + this.authServerUrl = authServerUrl; + return this; + } + + public KeycloakConfigBuilder realId(String realId) { + this.realmId = realId; + return this; + } + + public KeycloakConfigBuilder clientId(String clientId) { + this.clientId = clientId; + return this; + } + + public KeycloakConfigBuilder clientSecret(String clientSecret) { + this.clientSecret = clientSecret; + return this; + } + + public KeycloakConfigBuilder grantType(String grantType) { + this.grantType = grantType; + return this; + } + + public KeycloakConfig build() { + KeycloakConfig keycloakConfig = new KeycloakConfig(); + keycloakConfig.authServerUrl = authServerUrl; + keycloakConfig.realmId = realmId; + keycloakConfig.clientId = clientId; + keycloakConfig.clientSecret = clientSecret; + keycloakConfig.grantType = grantType; + return keycloakConfig; + } +} \ No newline at end of file diff --git a/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/service/AtlasKeycloakAuthService.java b/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/service/AtlasKeycloakAuthService.java new file mode 100644 index 00000000000..fc4d40df7e7 --- /dev/null +++ b/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/service/AtlasKeycloakAuthService.java @@ -0,0 +1,107 @@ +package org.apache.atlas.keycloak.client.service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import okhttp3.*; +import okhttp3.logging.HttpLoggingInterceptor; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.keycloak.client.RetrofitKeycloakClient; +import org.apache.atlas.keycloak.client.config.KeycloakConfig; +import org.jetbrains.annotations.NotNull; +import org.keycloak.representations.AccessTokenResponse; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import retrofit2.Retrofit; +import retrofit2.converter.jackson.JacksonConverterFactory; + +import java.time.OffsetDateTime; +import java.util.Objects; +import java.util.concurrent.TimeUnit; + +import static org.apache.atlas.AtlasErrorCode.BAD_REQUEST; + +public final class AtlasKeycloakAuthService { + + public final static Logger LOG = LoggerFactory.getLogger(AtlasKeycloakAuthService.class); + + private final static String GRANT_TYPE = "grant_type"; + private static final String CLIENT_ID = "client_id"; + private static final String CLIENT_SECRET = "client_secret"; + private static final int EXPIRY_OFFSET_SEC = 600; + private static final int TIMEOUT_IN_SECS = 60; + + private final RetrofitKeycloakClient retrofit; + private final KeycloakConfig keycloakConfig; + private AccessTokenResponse currentAccessToken; + private long expirationTime = -1; + + public AtlasKeycloakAuthService(KeycloakConfig keycloakConfig) { + this.keycloakConfig = keycloakConfig; + this.retrofit = new Retrofit.Builder().client(getOkHttpClient()) + .baseUrl(this.keycloakConfig.getAuthServerUrl()) + .addConverterFactory(JacksonConverterFactory.create(new ObjectMapper())).build() + .create(RetrofitKeycloakClient.class); + } + + @NotNull + private OkHttpClient getOkHttpClient() { + HttpLoggingInterceptor interceptor = new HttpLoggingInterceptor(); + interceptor.setLevel(HttpLoggingInterceptor.Level.BODY); + return new OkHttpClient.Builder() + .addInterceptor(interceptor) + .addInterceptor(responseLoggingInterceptor) + .connectTimeout(TIMEOUT_IN_SECS, TimeUnit.SECONDS) + .callTimeout(TIMEOUT_IN_SECS, TimeUnit.SECONDS) + .writeTimeout(TIMEOUT_IN_SECS, TimeUnit.SECONDS) + .readTimeout(TIMEOUT_IN_SECS, TimeUnit.SECONDS) + .build(); + } + + Interceptor responseLoggingInterceptor = chain -> { + Request request = chain.request(); + okhttp3.Response response = chain.proceed(request); + LOG.info("Keycloak: Auth Request for url {} Status: {}", request.url(), response.code()); + return response; + }; + + public String getAuthToken() { + if (!isTokenExpired()) { + return currentAccessToken.getToken(); + } + synchronized (this) { + if (isTokenExpired()) { + try { + retrofit2.Response resp = this.retrofit.grantToken(this.keycloakConfig.getRealmId(), getTokenRequest()).execute(); + if (resp.isSuccessful()) { + currentAccessToken = resp.body(); + expirationTime = currentTime() + currentAccessToken.getExpiresIn() - EXPIRY_OFFSET_SEC; + LOG.info("Keycloak: Auth token fetched with expiry:{} sec", expirationTime); + } else { + throw new AtlasBaseException(BAD_REQUEST, resp.errorBody().string()); + } + } catch (Exception e) { + LOG.error("Keycloak: Error while fetching access token for keycloak client.", e); + throw new RuntimeException(e); + } + } + } + return currentAccessToken.getToken(); + } + + public boolean isTokenExpired() { + synchronized (this) { + if (Objects.isNull(currentAccessToken)) { + return true; + } + return currentTime() >= expirationTime; + } + } + + private RequestBody getTokenRequest() { + return new FormBody.Builder().addEncoded(CLIENT_ID, this.keycloakConfig.getClientId()).addEncoded(CLIENT_SECRET, this.keycloakConfig.getClientSecret()).addEncoded(GRANT_TYPE, this.keycloakConfig.getGrantType()).build(); + } + + private long currentTime() { + return OffsetDateTime.now().toEpochSecond(); + } + +} diff --git a/common/pom.xml b/common/pom.xml index 616f66c533c..47c274ca4d1 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -104,6 +104,12 @@ compile + + io.micrometer + micrometer-registry-prometheus + ${micrometer.version} + + com.google.guava guava @@ -128,6 +134,18 @@ ${kafka.version} + + org.redisson + redisson + ${redis.client.version} + + + io.netty + * + + + + diff --git a/common/src/main/java/org/apache/atlas/AtlasConstants.java b/common/src/main/java/org/apache/atlas/AtlasConstants.java index ce0dbd0b4e3..d5e196d15e6 100644 --- a/common/src/main/java/org/apache/atlas/AtlasConstants.java +++ b/common/src/main/java/org/apache/atlas/AtlasConstants.java @@ -38,4 +38,5 @@ private AtlasConstants() { public static final String DEFAULT_ATLAS_REST_ADDRESS = "http://localhost:21000"; public static final String DEFAULT_TYPE_VERSION = "1.0"; public static final int ATLAS_SHUTDOWN_HOOK_PRIORITY = 30; + public static final int TASK_WAIT_TIME_MS = 180_000; } \ No newline at end of file diff --git a/common/src/main/java/org/apache/atlas/ICuratorFactory.java b/common/src/main/java/org/apache/atlas/ICuratorFactory.java new file mode 100644 index 00000000000..32d4e09f8fb --- /dev/null +++ b/common/src/main/java/org/apache/atlas/ICuratorFactory.java @@ -0,0 +1,7 @@ +package org.apache.atlas; + +import org.apache.curator.framework.recipes.locks.InterProcessMutex; + +public interface ICuratorFactory { + InterProcessMutex lockInstance(String zkRoot, String lockName); +} diff --git a/common/src/main/java/org/apache/atlas/annotation/EnableConditional.java b/common/src/main/java/org/apache/atlas/annotation/EnableConditional.java new file mode 100644 index 00000000000..6e1a1adeb76 --- /dev/null +++ b/common/src/main/java/org/apache/atlas/annotation/EnableConditional.java @@ -0,0 +1,19 @@ +package org.apache.atlas.annotation; + +import org.apache.atlas.utils.OnAtlasEnableCondition; +import org.springframework.context.annotation.Conditional; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Target({ ElementType.TYPE, ElementType.METHOD }) +@Retention(RetentionPolicy.RUNTIME) +@Conditional(OnAtlasEnableCondition.class) +public @interface EnableConditional { + // Configured atlas property + String property(); + // The default interface implementation should declare this as true + boolean isDefault() default false; +} \ No newline at end of file diff --git a/common/src/main/java/org/apache/atlas/ha/HAConfiguration.java b/common/src/main/java/org/apache/atlas/ha/HAConfiguration.java index 91fd4e10769..565923db86a 100644 --- a/common/src/main/java/org/apache/atlas/ha/HAConfiguration.java +++ b/common/src/main/java/org/apache/atlas/ha/HAConfiguration.java @@ -74,6 +74,15 @@ public static boolean isHAEnabled(Configuration configuration) { return ret; } + /** + * Return whether HS (Active-Active HA) is enabled or not. + * @param configuration underlying configuration instance + * @return + */ + public static boolean isActiveActiveHAEnabled(Configuration configuration) { + return !isHAEnabled(configuration); + } + /** * Get the web server address that a server instance with the passed ID is bound to. * diff --git a/common/src/main/java/org/apache/atlas/pc/WorkItemManager.java b/common/src/main/java/org/apache/atlas/pc/WorkItemManager.java index 351421e8efb..9fd8a683586 100644 --- a/common/src/main/java/org/apache/atlas/pc/WorkItemManager.java +++ b/common/src/main/java/org/apache/atlas/pc/WorkItemManager.java @@ -115,7 +115,7 @@ public void drain() { } public void shutdown() throws InterruptedException { - int avgCommitTimeSeconds = getAvgCommitTimeSeconds() * 2; + long avgCommitTimeSeconds = getAvgCommitTimeSeconds() * 2; LOG.info("WorkItemManager: Shutdown started. Will wait for: {} minutes...", avgCommitTimeSeconds); @@ -129,8 +129,8 @@ public Queue getResults() { return this.resultsQueue; } - private int getAvgCommitTimeSeconds() { - int commitTimeSeconds = 0; + private long getAvgCommitTimeSeconds() { + long commitTimeSeconds = 0; for (U c : consumers) { commitTimeSeconds += c.getMaxCommitTimeInMs(); diff --git a/common/src/main/java/org/apache/atlas/repository/Constants.java b/common/src/main/java/org/apache/atlas/repository/Constants.java index 259f4d50295..4a2cd5905c7 100644 --- a/common/src/main/java/org/apache/atlas/repository/Constants.java +++ b/common/src/main/java/org/apache/atlas/repository/Constants.java @@ -21,6 +21,21 @@ import org.apache.atlas.ApplicationProperties; import org.apache.atlas.AtlasException; import org.apache.commons.configuration.Configuration; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Set; import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.encodePropertyKey; @@ -29,6 +44,7 @@ * */ public final class Constants { + private static final Logger LOG = LoggerFactory.getLogger(Constants.class); /** * Globally Unique identifier property key. @@ -41,6 +57,7 @@ public final class Constants { public static final String HISTORICAL_GUID_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "historicalGuids"); public static final String FREETEXT_REQUEST_HANDLER = "/freetext"; public static final String TERMS_REQUEST_HANDLER = "/terms"; + public static final String ES_API_ALIASES = "/_aliases"; /** * Entity type name property key. @@ -65,7 +82,7 @@ public final class Constants { */ public static final String TYPE_CATEGORY_PROPERTY_KEY = getEncodedTypePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "type.category"); public static final String VERTEX_TYPE_PROPERTY_KEY = getEncodedTypePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "type"); - public static final String TYPENAME_PROPERTY_KEY = getEncodedTypePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "type.name"); + public static final String TYPENAME_PROPERTY_KEY = getEncodedTypePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "type.name"); public static final String TYPE_DISPLAYNAME_PROPERTY_KEY= getEncodedTypePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "type.displayName"); public static final String TYPEDESCRIPTION_PROPERTY_KEY = getEncodedTypePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "type.description"); public static final String TYPEVERSION_PROPERTY_KEY = getEncodedTypePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "type.version"); @@ -123,9 +140,28 @@ public final class Constants { */ public static final String CONNECTION_ENTITY_TYPE = "Connection"; public static final String QUERY_ENTITY_TYPE = "Query"; - public static final String QUERY_FOLDER_ENTITY_TYPE = "QueryFolder"; - public static final String QUERY_COLLECTION_ENTITY_TYPE = "QueryCollection"; + public static final String QUERY_FOLDER_ENTITY_TYPE = "Folder"; + public static final String QUERY_COLLECTION_ENTITY_TYPE = "Collection"; + /* + * Purpose / Persona + */ + public static final String ACCESS_CONTROL_ENTITY_TYPE = "AccessControl"; + public static final String PERSONA_ENTITY_TYPE = "Persona"; + public static final String PURPOSE_ENTITY_TYPE = "Purpose"; + public static final String POLICY_ENTITY_TYPE = "AuthPolicy"; + public static final String SERVICE_ENTITY_TYPE = "AuthService"; + + /** + * Resource + */ + public static final String LINK_ENTITY_TYPE = "Link"; + public static final String README_ENTITY_TYPE = "Readme"; + + public static final String ASSET_RELATION_ATTR = "asset"; + + public static final String ASSET_README_EDGE_LABEL = "__Asset.readme"; + public static final String ASSET_LINK_EDGE_LABEL = "__Asset.links"; /** * Lineage relations. @@ -133,6 +169,8 @@ public final class Constants { public static final String PROCESS_OUTPUTS = "__Process.outputs"; public static final String PROCESS_INPUTS = "__Process.inputs"; + public static String[] PROCESS_EDGE_LABELS = {PROCESS_OUTPUTS, PROCESS_INPUTS}; + /** * The homeId field is used when saving into Atlas a copy of an object that is being imported from another * repository. The homeId will be set to a String that identifies the other repository. The specific format @@ -190,6 +228,8 @@ public final class Constants { */ public static final String INDEX_PREFIX = "janusgraph_"; + public static final String VERTEX_INDEX_NAME = INDEX_PREFIX + VERTEX_INDEX; + public static final String NAME = "name"; public static final String QUALIFIED_NAME = "qualifiedName"; public static final String TYPE_NAME_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "typeName"; @@ -199,6 +239,9 @@ public final class Constants { public static final String INDEX_SEARCH_VERTEX_PREFIX_PROPERTY = "atlas.graph.index.search.vertex.prefix"; public static final String INDEX_SEARCH_VERTEX_PREFIX_DEFAULT = "$v$"; + public static final String ATTR_TENANT_ID = "tenantId"; + public static final String DEFAULT_TENANT_ID = "default"; + public static final String MAX_FULLTEXT_QUERY_STR_LENGTH = "atlas.graph.fulltext-max-query-str-length"; public static final String MAX_DSL_QUERY_STR_LENGTH = "atlas.graph.dsl-max-query-str-length"; @@ -207,6 +250,7 @@ public final class Constants { public static final String ATTRIBUTE_NAME_SUPERTYPENAMES = "superTypeNames"; public static final String ATTRIBUTE_NAME_STATE = "state"; public static final String ATTRIBUTE_NAME_VERSION = "version"; + public static final String ATTRIBUTE_LINK = "link"; public static final String TEMP_STRUCT_NAME_PREFIX = "__tempQueryResultStruct"; public static final String CLASSIFICATION_ENTITY_GUID = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "entityGuid"); @@ -214,6 +258,7 @@ public final class Constants { public static final String CLASSIFICATION_VALIDITY_PERIODS_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "validityPeriods"); public static final String CLASSIFICATION_VERTEX_PROPAGATE_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "propagate"); public static final String CLASSIFICATION_VERTEX_REMOVE_PROPAGATIONS_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "removePropagations"); + public static final String CLASSIFICATION_VERTEX_RESTRICT_PROPAGATE_THROUGH_LINEAGE= encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "restrictPropagationThroughLineage"); public static final String CLASSIFICATION_VERTEX_NAME_KEY = encodePropertyKey(TYPE_NAME_PROPERTY_KEY); public static final String CLASSIFICATION_EDGE_NAME_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "name"); public static final String CLASSIFICATION_EDGE_IS_PROPAGATED_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "isPropagated"); @@ -268,7 +313,11 @@ public final class Constants { public static final String TASK_ERROR_MESSAGE = encodePropertyKey(TASK_PREFIX + "errorMessage"); public static final String TASK_START_TIME = encodePropertyKey(TASK_PREFIX + "startTime"); public static final String TASK_END_TIME = encodePropertyKey(TASK_PREFIX + "endTime"); - public static final String ACTIVE_STATE_VALUE = "ACTIVE"; + public static final String TASK_TIME_TAKEN_IN_SECONDS = encodePropertyKey(TASK_PREFIX + "timeTakenInSeconds"); + public static final String TASK_CLASSIFICATION_ID = encodePropertyKey(TASK_PREFIX + "classificationId"); + public static final String TASK_ENTITY_GUID = encodePropertyKey(TASK_PREFIX + "entityGuid"); + public static final String TASK_CLASSIFICATION_NAME = encodePropertyKey(TASK_PREFIX + "classificationName"); + public static final String ACTIVE_STATE_VALUE = "ACTIVE"; /** * Index Recovery vertex property keys. @@ -317,6 +366,57 @@ public enum SupportedFileExtensions { XLSX, XLS, CSV } public static final String CLASSIFICATION_PROPAGATION_TIME_METRIC = "classification.propagation.time"; public static final String CLASSIFICATION_PROPAGATION_JOB_COUNT_METRIC = "classification.propagation.job.count"; + public static final int ELASTICSEARCH_PAGINATION_SIZE = 50; + + public static final String CATALOG_PROCESS_INPUT_RELATIONSHIP_LABEL = "__Process.inputs"; + public static final String CATALOG_PROCESS_OUTPUT_RELATIONSHIP_LABEL = "__Process.outputs"; + public static final String COLUMN_LINEAGE_RELATIONSHIP_LABEL = "__Process.columnProcesses"; + public static final String CLASSIFICATION_PROPAGATION_MODE_DEFAULT ="DEFAULT"; + public static final String CLASSIFICATION_PROPAGATION_MODE_RESTRICT_LINEAGE ="RESTRICT_LINEAGE"; + + public static final HashMap> CLASSIFICATION_PROPAGATION_EXCLUSION_MAP = new HashMap>(){{ + put(CLASSIFICATION_PROPAGATION_MODE_RESTRICT_LINEAGE, new ArrayList<>( + Arrays.asList(CATALOG_PROCESS_INPUT_RELATIONSHIP_LABEL, + CATALOG_PROCESS_OUTPUT_RELATIONSHIP_LABEL, + COLUMN_LINEAGE_RELATIONSHIP_LABEL + ))); + put(CLASSIFICATION_PROPAGATION_MODE_DEFAULT, null); + }}; + + public static final String ATTR_ADMIN_USERS = "adminUsers"; + public static final String ATTR_ADMIN_GROUPS = "adminGroups"; + public static final String ATTR_ADMIN_ROLES = "adminRoles"; + public static final String ATTR_VIEWER_USERS = "viewerUsers"; + public static final String ATTR_VIEWER_GROUPS = "viewerGroups"; + + public static final String ATTR_STARRED_BY = "starredBy"; + public static final String ATTR_STARRED_COUNT = "starredCount"; + public static final String ATTR_STARRED_DETAILS_LIST = "starredDetailsList"; + public static final String ATTR_ASSET_STARRED_BY = "assetStarredBy"; + public static final String ATTR_ASSET_STARRED_AT = "assetStarredAt"; + + public static final String STRUCT_STARRED_DETAILS = "StarredDetails"; + + public static final String KEYCLOAK_ROLE_ADMIN = "$admin"; + public static final String KEYCLOAK_ROLE_MEMBER = "$member"; + public static final String KEYCLOAK_ROLE_GUEST = "$guest"; + public static final String KEYCLOAK_ROLE_DEFAULT = "default-roles-default"; + public static final String KEYCLOAK_ROLE_API_TOKEN = "$api-token-default-access"; + + public static final String REQUEST_HEADER_USER_AGENT = "User-Agent"; + public static final String REQUEST_HEADER_HOST = "Host"; + + public static final Set SKIP_UPDATE_AUTH_CHECK_TYPES = new HashSet() {{ + add(README_ENTITY_TYPE); + add(LINK_ENTITY_TYPE); + }}; + + public static final Set SKIP_DELETE_AUTH_CHECK_TYPES = new HashSet() {{ + add(README_ENTITY_TYPE); + add(LINK_ENTITY_TYPE); + add(POLICY_ENTITY_TYPE); + }}; + private Constants() { } @@ -335,4 +435,20 @@ private static String getEncodedTypePropertyKey(String defaultKey) { return encodePropertyKey(defaultKey); } } + + public static String getStaticFileAsString(String fileName) throws IOException { + String atlasHomeDir = System.getProperty("atlas.home"); + atlasHomeDir = StringUtils.isEmpty(atlasHomeDir) ? "." : atlasHomeDir; + + Path path = Paths.get(atlasHomeDir, "static", fileName); + String resourceAsString = null; + try { + resourceAsString = new String(Files.readAllBytes(path), StandardCharsets.UTF_8); + } catch (IOException e) { + LOG.error("Failed to get static file as string: {}", fileName); + throw e; + } + + return resourceAsString; + } } diff --git a/common/src/main/java/org/apache/atlas/service/metrics/MetricUtils.java b/common/src/main/java/org/apache/atlas/service/metrics/MetricUtils.java new file mode 100644 index 00000000000..c309190391e --- /dev/null +++ b/common/src/main/java/org/apache/atlas/service/metrics/MetricUtils.java @@ -0,0 +1,97 @@ +package org.apache.atlas.service.metrics; + +import io.micrometer.core.instrument.Metrics; +import io.micrometer.core.instrument.Tags; +import io.micrometer.core.instrument.Timer; +import io.micrometer.prometheus.PrometheusConfig; +import io.micrometer.prometheus.PrometheusMeterRegistry; +import org.apache.atlas.ApplicationProperties; +import org.apache.atlas.AtlasException; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import java.util.*; +import java.util.stream.Collectors; + +import static org.apache.commons.lang.StringUtils.EMPTY; + +@Component +public class MetricUtils { + private final static Logger LOG = LoggerFactory.getLogger(MetricUtils.class); + + private static final String URI = "uri"; + private static final String LOCAL = "local"; + private static final String STATUS = "status"; + private static final String METHOD = "method"; + private static final String SERVICE = "service"; + private static final String INTEGRATION = "integration"; + private static final String ATLAS_METASTORE = "atlas-metastore"; + private static final String REGEX_URI_PLACEHOLDER = "\\[\\^/\\]\\+"; + private static final String HTTP_SERVER_REQUESTS = "http.server.requests"; + private static final String ATLAS_METRICS_URI_PATTERNS = "atlas.metrics.uri_patterns"; + private static final double[] PERCENTILES = {0.5, 0.90, 0.99}; + + private static Map METRIC_URI_PATTERNS_MAP; + private static final PrometheusMeterRegistry METER_REGISTRY; + + static { + try { + METRIC_URI_PATTERNS_MAP = Arrays.stream(ApplicationProperties.get().getStringArray(ATLAS_METRICS_URI_PATTERNS)) + .distinct().collect(Collectors.toMap(uri->uri, uri->uri.replaceAll(REGEX_URI_PLACEHOLDER, "*"))); + } catch (Exception e) { + LOG.error("Failed to load 'atlas.metrics.uri_patterns from properties"); + } + METER_REGISTRY = new PrometheusMeterRegistry(PrometheusConfig.DEFAULT); + METER_REGISTRY.config().withHighCardinalityTagsDetector().commonTags(SERVICE, ATLAS_METASTORE, INTEGRATION, LOCAL); + Metrics.globalRegistry.add(METER_REGISTRY); + } + + public Timer.Sample start(String uri) { + return matchCanonicalPattern(uri).isPresent() ? Timer.start(getMeterRegistry()) : null; + } + + public void recordHttpTimer(Timer.Sample sample, String method, String rawPath, int code, String... additionalTags) { + if (Objects.isNull(sample)) { + return; + } + sample.stop(getTimer(HTTP_SERVER_REQUESTS, method, code, rawPath, additionalTags)); + } + + private Timer getTimer(String timerName, String method, int code, String rawPath, String... additionalTags) { + Tags tags = getTags(method, code, rawPath); + if (Objects.nonNull(additionalTags) && additionalTags.length > 0) { + tags = tags.and(additionalTags); + } + return Timer.builder(timerName) + .publishPercentiles(PERCENTILES) + .tags(tags) + .register(getMeterRegistry()); + } + + private Tags getTags(String httpMethod, int httpResponseStatus, String uri) { + return Tags.of(METHOD, httpMethod, + STATUS, String.valueOf(httpResponseStatus), + URI, matchCanonicalPattern(uri).get()); + } + + public static Optional matchCanonicalPattern(String uri) { + if (Objects.isNull(uri) || uri.isEmpty()) { + return Optional.empty(); + } + if (uri.endsWith("/")) { + uri = uri.substring(0, uri.lastIndexOf("/")); + } + String updatedUrl = uri; + Optional patternOp = METRIC_URI_PATTERNS_MAP.keySet().stream() + .filter(pattern -> updatedUrl.matches(pattern + "$")) + .findFirst(); + return Optional.ofNullable(METRIC_URI_PATTERNS_MAP.get(patternOp.orElse(EMPTY))); + } + + public static PrometheusMeterRegistry getMeterRegistry() { + return METER_REGISTRY; + } + +} diff --git a/common/src/main/java/org/apache/atlas/service/metrics/MetricsRegistry.java b/common/src/main/java/org/apache/atlas/service/metrics/MetricsRegistry.java new file mode 100644 index 00000000000..1730fb50de1 --- /dev/null +++ b/common/src/main/java/org/apache/atlas/service/metrics/MetricsRegistry.java @@ -0,0 +1,14 @@ +package org.apache.atlas.service.metrics; + +import org.apache.atlas.utils.AtlasPerfMetrics; + +import java.io.IOException; +import java.io.PrintWriter; + +public interface MetricsRegistry { + + void collect(String requestId, String requestUri, AtlasPerfMetrics metrics); + + void scrape(PrintWriter writer) throws IOException; + +} diff --git a/common/src/main/java/org/apache/atlas/service/metrics/MetricsRegistryServiceImpl.java b/common/src/main/java/org/apache/atlas/service/metrics/MetricsRegistryServiceImpl.java new file mode 100644 index 00000000000..65a5be31f98 --- /dev/null +++ b/common/src/main/java/org/apache/atlas/service/metrics/MetricsRegistryServiceImpl.java @@ -0,0 +1,77 @@ +package org.apache.atlas.service.metrics; + +import io.micrometer.core.instrument.Metrics; +import io.micrometer.core.instrument.Tags; +import io.micrometer.core.instrument.Timer; +import io.micrometer.prometheus.PrometheusMeterRegistry; +import org.apache.atlas.ApplicationProperties; +import org.apache.atlas.AtlasException; +import org.apache.atlas.utils.AtlasPerfMetrics; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import javax.inject.Inject; +import java.io.IOException; +import java.io.PrintWriter; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import static org.apache.atlas.service.metrics.MetricUtils.getMeterRegistry; + + +@Component +public class MetricsRegistryServiceImpl implements MetricsRegistry { + + private static final Logger LOG = LoggerFactory.getLogger(MetricsRegistryServiceImpl.class); + + private static final String NAME = "name"; + private static final String URI = "uri"; + private static final String METHOD_DIST_SUMMARY = "method_dist_summary"; + private static final double[] PERCENTILES = {0.99}; + private static final String METHOD_LEVEL_METRICS_ENABLE = "atlas.metrics.method_level.enable"; + private static final String ATLAS_METRICS_METHOD_PATTERNS = "atlas.metrics.method_patterns"; + private final List filteredMethods; + + @Inject + public MetricsRegistryServiceImpl() throws AtlasException { + this.filteredMethods = Arrays.stream(ApplicationProperties.get().getStringArray(ATLAS_METRICS_METHOD_PATTERNS)).collect(Collectors.toList()); + } + + @Override + public void collect(String requestId, String requestUri, AtlasPerfMetrics metrics) { + try { + if (!ApplicationProperties.get().getBoolean(METHOD_LEVEL_METRICS_ENABLE, false)) { + return; + } + + for (String name : this.filteredMethods) { + if(metrics.hasMetric(name)) { + AtlasPerfMetrics.Metric metric = metrics.getMetric(name); + Timer.builder(METHOD_DIST_SUMMARY).tags(Tags.of(NAME, metric.getName(), URI, requestUri)).publishPercentiles(PERCENTILES) + .register(getMeterRegistry()).record(metric.getTotalTimeMSecs(), TimeUnit.MILLISECONDS); + } + } + } catch (Exception e) { + LOG.error("Failed to collect metrics", e); + return; + } + } + + @Override + public void scrape(PrintWriter writer) { + Metrics.globalRegistry.getRegistries().forEach(r -> { + try { + ((PrometheusMeterRegistry) r).scrape(writer); + writer.flush(); + } catch (IOException e) { + LOG.warn("Failed to write metrics while scraping", e); + } finally { + writer.close(); + } + }); + } + +} diff --git a/common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java b/common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java new file mode 100644 index 00000000000..ed3ab375791 --- /dev/null +++ b/common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java @@ -0,0 +1,128 @@ +package org.apache.atlas.service.redis; + +import org.apache.atlas.ApplicationProperties; +import org.apache.atlas.AtlasException; +import org.apache.commons.configuration.Configuration; +import org.apache.commons.lang.ArrayUtils; +import org.redisson.api.RLock; +import org.redisson.api.RedissonClient; +import org.redisson.config.Config; +import org.redisson.config.ReadMode; + +import javax.annotation.PreDestroy; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.Arrays; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; + +public abstract class AbstractRedisService implements RedisService { + + private static final String REDIS_URL_PREFIX = "redis://"; + private static final String ATLAS_REDIS_URL = "atlas.redis.url"; + private static final String ATLAS_REDIS_SENTINEL_URLS = "atlas.redis.sentinel.urls"; + private static final String ATLAS_REDIS_USERNAME = "atlas.redis.username"; + private static final String ATLAS_REDIS_PASSWORD = "atlas.redis.password"; + private static final String ATLAS_REDIS_MASTER_NAME = "atlas.redis.master_name"; + private static final String ATLAS_REDIS_LOCK_WAIT_TIME_MS = "atlas.redis.lock.wait_time.ms"; + private static final String ATLAS_REDIS_LOCK_WATCHDOG_TIMEOUT_MS = "atlas.redis.lock.watchdog_timeout.ms"; + private static final int DEFAULT_REDIS_WAIT_TIME_MS = 15_000; + private static final int DEFAULT_REDIS_LOCK_WATCHDOG_TIMEOUT_MS = 600_000; + + RedissonClient redisClient; + Map keyLockMap; + Configuration atlasConfig; + long waitTimeInMS; + long watchdogTimeoutInMS; + + @Override + public boolean acquireDistributedLock(String key) throws Exception { + getLogger().info("Attempting to acquire distributed lock for {}, host:{}", key, getHostAddress()); + boolean isLockAcquired; + try { + RLock lock = redisClient.getFairLock(key); + isLockAcquired = lock.tryLock(waitTimeInMS, TimeUnit.MILLISECONDS); + if (isLockAcquired) { + keyLockMap.put(key, lock); + getLogger().info("Successfully acquired distributed lock for {}, host:{}", key, getHostAddress()); + } else { + getLogger().info("Attempt failed as lock {} is already acquired, host: {}.", key, getHostAddress()); + } + } catch (InterruptedException e) { + getLogger().error("Failed to acquire distributed lock for {}, host: {}", key, getHostAddress(), e); + throw new AtlasException(e); + } + return isLockAcquired; + } + + @Override + public void releaseDistributedLock(String key) { + if (!keyLockMap.containsKey(key)) { + return; + } + try { + RLock lock = keyLockMap.get(key); + if (lock.isHeldByCurrentThread()) { + getLogger().info("Attempt to release distributed lock for {}, host: {}", key, getHostAddress()); + lock.unlock(); + getLogger().info("Successfully released distributed lock for {}, host: {}", key, getHostAddress()); + } + } catch (Exception e) { + getLogger().error("Failed to release distributed lock for {}", key, e); + } + } + + private String getHostAddress() throws UnknownHostException { + return InetAddress.getLocalHost().getHostAddress(); + } + + private Config initAtlasConfig() throws AtlasException { + keyLockMap = new ConcurrentHashMap<>(); + atlasConfig = ApplicationProperties.get(); + waitTimeInMS = atlasConfig.getLong(ATLAS_REDIS_LOCK_WAIT_TIME_MS, DEFAULT_REDIS_WAIT_TIME_MS); + watchdogTimeoutInMS = atlasConfig.getLong(ATLAS_REDIS_LOCK_WATCHDOG_TIMEOUT_MS, DEFAULT_REDIS_LOCK_WATCHDOG_TIMEOUT_MS); + Config redisConfig = new Config(); + redisConfig.setLockWatchdogTimeout(watchdogTimeoutInMS); + return redisConfig; + } + + Config getLocalConfig() throws AtlasException { + Config config = initAtlasConfig(); + config.useSingleServer() + .setAddress(formatUrls(atlasConfig.getStringArray(ATLAS_REDIS_URL))[0]) + .setUsername(atlasConfig.getString(ATLAS_REDIS_USERNAME)) + .setPassword(atlasConfig.getString(ATLAS_REDIS_PASSWORD)); + return config; + } + + Config getProdConfig() throws AtlasException { + Config config = initAtlasConfig(); + config.useSentinelServers() + .setReadMode(ReadMode.MASTER_SLAVE) + .setCheckSentinelsList(false) + .setMasterName(atlasConfig.getString(ATLAS_REDIS_MASTER_NAME)) + .addSentinelAddress(formatUrls(atlasConfig.getStringArray(ATLAS_REDIS_SENTINEL_URLS))) + .setUsername(atlasConfig.getString(ATLAS_REDIS_USERNAME)) + .setPassword(atlasConfig.getString(ATLAS_REDIS_PASSWORD)); + return config; + } + + private String[] formatUrls(String[] urls) throws IllegalArgumentException { + if (ArrayUtils.isEmpty(urls)) { + getLogger().error("Invalid redis cluster urls"); + throw new IllegalArgumentException("Invalid redis cluster urls"); + } + return Arrays.stream(urls).map(url -> { + if (url.startsWith(REDIS_URL_PREFIX)) { + return url; + } + return REDIS_URL_PREFIX + url; + }).toArray(String[]::new); + } + + @PreDestroy + public void flushLocks(){ + keyLockMap.keySet().stream().forEach(k->keyLockMap.get(k).unlock()); + } +} diff --git a/common/src/main/java/org/apache/atlas/service/redis/NoRedisServiceImpl.java b/common/src/main/java/org/apache/atlas/service/redis/NoRedisServiceImpl.java new file mode 100644 index 00000000000..96a8fadc99f --- /dev/null +++ b/common/src/main/java/org/apache/atlas/service/redis/NoRedisServiceImpl.java @@ -0,0 +1,37 @@ +package org.apache.atlas.service.redis; + +import org.apache.atlas.annotation.ConditionalOnAtlasProperty; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; + +@Component +@ConditionalOnAtlasProperty(property = "atlas.redis.service.impl", isDefault = true) +public class NoRedisServiceImpl extends AbstractRedisService { + + private static final Logger LOG = LoggerFactory.getLogger(NoRedisServiceImpl.class); + + @PostConstruct + public void init() { + LOG.info("Enabled no redis implementation."); + } + + @Override + public boolean acquireDistributedLock(String key) { + //do nothing + return true; + } + + @Override + public void releaseDistributedLock(String key) { + //do nothing + } + + @Override + public Logger getLogger() { + return LOG; + } + +} diff --git a/common/src/main/java/org/apache/atlas/service/redis/RedisService.java b/common/src/main/java/org/apache/atlas/service/redis/RedisService.java new file mode 100644 index 00000000000..1475f93e832 --- /dev/null +++ b/common/src/main/java/org/apache/atlas/service/redis/RedisService.java @@ -0,0 +1,13 @@ +package org.apache.atlas.service.redis; + +import org.slf4j.Logger; + +public interface RedisService { + + boolean acquireDistributedLock(String key) throws Exception; + + void releaseDistributedLock(String key); + + Logger getLogger(); + +} diff --git a/common/src/main/java/org/apache/atlas/service/redis/RedisServiceImpl.java b/common/src/main/java/org/apache/atlas/service/redis/RedisServiceImpl.java new file mode 100644 index 00000000000..42dec6fa783 --- /dev/null +++ b/common/src/main/java/org/apache/atlas/service/redis/RedisServiceImpl.java @@ -0,0 +1,29 @@ +package org.apache.atlas.service.redis; + +import org.apache.atlas.AtlasException; +import org.apache.atlas.annotation.ConditionalOnAtlasProperty; +import org.redisson.Redisson; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; + +@Component +@ConditionalOnAtlasProperty(property = "atlas.redis.service.impl") +public class RedisServiceImpl extends AbstractRedisService{ + + private static final Logger LOG = LoggerFactory.getLogger(RedisServiceImpl.class); + + @PostConstruct + public void init() throws AtlasException { + redisClient = Redisson.create(getProdConfig()); + LOG.info("Sentinel redis client created successfully."); + } + + @Override + public Logger getLogger() { + return LOG; + } + +} diff --git a/common/src/main/java/org/apache/atlas/service/redis/RedisServiceLocalImpl.java b/common/src/main/java/org/apache/atlas/service/redis/RedisServiceLocalImpl.java new file mode 100644 index 00000000000..2eb774920ef --- /dev/null +++ b/common/src/main/java/org/apache/atlas/service/redis/RedisServiceLocalImpl.java @@ -0,0 +1,28 @@ +package org.apache.atlas.service.redis; + +import org.apache.atlas.AtlasException; +import org.apache.atlas.annotation.ConditionalOnAtlasProperty; +import org.redisson.Redisson; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; + +@Component +@ConditionalOnAtlasProperty(property = "atlas.redis.service.impl") +public class RedisServiceLocalImpl extends AbstractRedisService { + + private static final Logger LOG = LoggerFactory.getLogger(RedisServiceLocalImpl.class); + + @PostConstruct + public void init() throws AtlasException { + redisClient = Redisson.create(getLocalConfig()); + LOG.info("Local redis client created successfully."); + } + + @Override + public Logger getLogger() { + return LOG; + } +} diff --git a/common/src/main/java/org/apache/atlas/utils/AtlasPerfMetrics.java b/common/src/main/java/org/apache/atlas/utils/AtlasPerfMetrics.java index c72b2c3e25a..382afb04db5 100644 --- a/common/src/main/java/org/apache/atlas/utils/AtlasPerfMetrics.java +++ b/common/src/main/java/org/apache/atlas/utils/AtlasPerfMetrics.java @@ -33,8 +33,8 @@ public MetricRecorder getMetricRecorder(String name) { public void recordMetric(MetricRecorder recorder) { if (recorder != null) { - final String name = recorder.name; - final long timeTaken = recorder.getElapsedTime(); + final String name = recorder.name; + final long timeTaken = recorder.getElapsedTime(); Metric metric = metrics.get(name); @@ -65,6 +65,10 @@ public Metric getMetric(String name) { return metrics.get(name); } + public boolean hasMetric(String name) { + return metrics.containsKey(name); + } + @Override public String toString() { StringBuilder sb = new StringBuilder(); @@ -86,7 +90,7 @@ public String toString() { public class MetricRecorder { private final String name; - private final long startTimeMs = System.currentTimeMillis(); + private final long startTimeMs = System.currentTimeMillis(); MetricRecorder(String name) { this.name = name; @@ -99,8 +103,8 @@ long getElapsedTime() { public static class Metric { private final String name; - private short invocations = 0; - private long totalTimeMSecs = 0; + private long invocations = 0; + private long totalTimeMSecs = 0; public Metric(String name) { this.name = name; @@ -110,7 +114,7 @@ public String getName() { return name; } - public short getInvocations() { + public long getInvocations() { return invocations; } diff --git a/common/src/main/java/org/apache/atlas/utils/OnAtlasEnableCondition.java b/common/src/main/java/org/apache/atlas/utils/OnAtlasEnableCondition.java new file mode 100644 index 00000000000..b91510c5ad5 --- /dev/null +++ b/common/src/main/java/org/apache/atlas/utils/OnAtlasEnableCondition.java @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.utils; + +import org.apache.atlas.ApplicationProperties; +import org.apache.atlas.AtlasException; +import org.apache.atlas.annotation.ConditionalOnAtlasProperty; +import org.apache.atlas.annotation.EnableConditional; +import org.apache.commons.configuration.Configuration; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.context.annotation.Condition; +import org.springframework.context.annotation.ConditionContext; +import org.springframework.core.type.AnnotatedTypeMetadata; +import org.springframework.core.type.AnnotationMetadata; + +public class OnAtlasEnableCondition implements Condition { + private final Logger LOG = LoggerFactory.getLogger(OnAtlasEnableCondition.class); + + @Override + public boolean matches(ConditionContext context, AnnotatedTypeMetadata metadata) { + boolean matches = false; + String propertyName = (String) metadata.getAnnotationAttributes(EnableConditional.class.getName()).get("property"); + if (metadata instanceof AnnotatedTypeMetadata) { + + try { + Configuration configuration = ApplicationProperties.get(); + boolean enabled = configuration.getBoolean(propertyName, true); + return enabled; + } catch (AtlasException e) { + LOG.error("Unable to load atlas properties. Dependent bean configuration may fail"); + } + } + return false; + } +} \ No newline at end of file diff --git a/dashboardv2/package-lock.json b/dashboardv2/package-lock.json index 61fd944c622..b21a0a4df0f 100644 --- a/dashboardv2/package-lock.json +++ b/dashboardv2/package-lock.json @@ -396,6 +396,16 @@ "integrity": "sha1-NWnt6Lo0MV+rmcPpLLBMciDeH6g=", "dev": true }, + "call-bind": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", + "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==", + "dev": true, + "requires": { + "function-bind": "^1.1.1", + "get-intrinsic": "^1.0.2" + } + }, "camel-case": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/camel-case/-/camel-case-3.0.0.tgz", @@ -691,9 +701,9 @@ "integrity": "sha512-ii0/r5f4sjKNTfh84Di+DpztYwqKhEyUlKoPrzUFfeSkWxjW49xU2QzO9qrPrNkpdI0XJkfzvmTu8V2Zylln6A==" }, "d3-color": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/d3-color/-/d3-color-1.4.1.tgz", - "integrity": "sha512-p2sTHSLCJI2QKunbGb7ocOh7DgTAn8IrLx21QRc/BSnodXM4sv6aLQlnfpvehFMLZEfBc6g9pH9SWQccFYfJ9Q==" + "version" : "3.1.0", + "resolved" :"https://registry.yarnpkg.com/d3-color/-/d3-color-3.1.0.tgz#395b2833dfac71507f12ac2f7af23bf819de24e2", + "integrity" : "sha512-zg/chbXyeBtMQ1LbD/WSoW2DpC3I0mpmPdW+ynRTj/x2DAWYrIY7qeZIHidozwV24m4iavr15lNwIwLxRmOxhA==" }, "d3-contour": { "version": "1.3.2", @@ -1523,6 +1533,25 @@ "integrity": "sha512-3t6rVToeoZfYSGd8YoLFR2DJkiQrIiUrGcjvFX2mDw3bn6k2OtwHN0TNCLbBO+w8qTvimhDkv+LSscbJY1vE6w==", "dev": true }, + "get-intrinsic": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.3.tgz", + "integrity": "sha512-QJVz1Tj7MS099PevUG5jvnt9tSkXN8K14dxQlikJuPt4uD9hHAHjLyLBiLR5zELelBdD9QNRAXZzsJx0WaDL9A==", + "dev": true, + "requires": { + "function-bind": "^1.1.1", + "has": "^1.0.3", + "has-symbols": "^1.0.3" + }, + "dependencies": { + "has-symbols": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", + "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==", + "dev": true + } + } + }, "get-stdin": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-4.0.1.tgz", @@ -1952,6 +1981,15 @@ "har-schema": "^2.0.0" } }, + "has": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", + "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", + "dev": true, + "requires": { + "function-bind": "^1.1.1" + } + }, "has-ansi": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz", @@ -2344,9 +2382,9 @@ } }, "lodash": { - "version": "4.17.20", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz", - "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==" + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, "longest": { "version": "1.0.1", @@ -2475,9 +2513,9 @@ } }, "minimatch": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", - "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "version": "3.0.8", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.8.tgz", + "integrity": "sha512-6FsRAQsxQ61mw+qP1ZzbL9Bc78x2p5OqNgNpnoAFLTrX8n5Kxph0CsnhmKKNXTWjXqU5L0pGPR7hYk+XWZr60Q==", "dev": true, "requires": { "brace-expansion": "^1.1.7" @@ -2703,6 +2741,12 @@ "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=", "dev": true }, + "object-inspect": { + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.2.tgz", + "integrity": "sha512-z+cPxW0QGUp0mcqcsgQyLVRDoXFQbXOwBaqyF7VIgI4TWNQsDHrBpUQslRmIfAoYWdYzs6UlKJtB2XJpTaNSpQ==", + "dev": true + }, "object-keys": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", @@ -2946,10 +2990,13 @@ "dev": true }, "qs": { - "version": "6.9.4", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.9.4.tgz", - "integrity": "sha512-A1kFqHekCTM7cz0udomYUoYNWjBebHm/5wzU/XqrBRBNWectVH0QIiN+NEcZ0Dte5hvzHwbr8+XQmguPhJ6WdQ==", - "dev": true + "version": "6.11.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz", + "integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==", + "dev": true, + "requires": { + "side-channel": "^1.0.4" + } }, "range-parser": { "version": "1.2.1", @@ -3076,9 +3123,9 @@ }, "dependencies": { "qs": { - "version": "6.5.2", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz", - "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==", + "version": "6.5.3", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.3.tgz", + "integrity": "sha512-qxXIEh4pCGfHICj1mAJQ2/2XVZkjCDTcEgfoSQxc/fYivUZxTkk7L3bDBJSoNrEzXI17oUO5Dp07ktqE5KzczA==", "dev": true } } @@ -3371,6 +3418,17 @@ "integrity": "sha512-BvE/TwpZX4FXExxOxZyRGQQv651MSwmWKZGqvmPcRIjDqWub67kTKuIMx43cZZrS/cBBzwBcNDWoFxt2XEFIpQ==", "dev": true }, + "side-channel": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz", + "integrity": "sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==", + "dev": true, + "requires": { + "call-bind": "^1.0.0", + "get-intrinsic": "^1.0.2", + "object-inspect": "^1.9.0" + } + }, "signal-exit": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.3.tgz", @@ -3879,4 +3937,4 @@ } } } -} \ No newline at end of file +} diff --git a/dashboardv2/public/img/entity-icon/expandBtn.svg b/dashboardv2/public/img/entity-icon/expandBtn.svg new file mode 100644 index 00000000000..88693e74b3a --- /dev/null +++ b/dashboardv2/public/img/entity-icon/expandBtn.svg @@ -0,0 +1,24 @@ + + + + + + + + diff --git a/dashboardv2/public/js/collection/VLineageList.js b/dashboardv2/public/js/collection/VLineageList.js index bda649220ee..4f980a737ef 100644 --- a/dashboardv2/public/js/collection/VLineageList.js +++ b/dashboardv2/public/js/collection/VLineageList.js @@ -42,7 +42,11 @@ define(['require', dataType: 'json' }, options); - return this.constructor.nonCrudOperation.call(this, url, 'GET', options); + if (options.compactLineageEnabled) { + return this.constructor.nonCrudOperation.call(this, url, 'POST', options); + } else { + return this.constructor.nonCrudOperation.call(this, url, 'GET', options); + } } }, //Static Class Members diff --git a/dashboardv2/public/js/external_lib/atlas-lineage/dist/index.js b/dashboardv2/public/js/external_lib/atlas-lineage/dist/index.js index b31ee491136..6a184ee6e8a 100644 --- a/dashboardv2/public/js/external_lib/atlas-lineage/dist/index.js +++ b/dashboardv2/public/js/external_lib/atlas-lineage/dist/index.js @@ -1 +1 @@ -!function(t,e){"object"==typeof exports&&"object"==typeof module?module.exports=e(require("platform"),require("dagreD3")):"function"==typeof define&&define.amd?define(["platform","dagreD3"],e):"object"==typeof exports?exports.LineageHelper=e(require("platform"),require("dagreD3")):t.LineageHelper=e(t.platform,t.dagreD3)}(window,(function(t,e){return function(t){var e={};function n(r){if(e[r])return e[r].exports;var i=e[r]={i:r,l:!1,exports:{}};return t[r].call(i.exports,i,i.exports,n),i.l=!0,i.exports}return n.m=t,n.c=e,n.d=function(t,e,r){n.o(t,e)||Object.defineProperty(t,e,{enumerable:!0,get:r})},n.r=function(t){"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(t,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(t,"__esModule",{value:!0})},n.t=function(t,e){if(1&e&&(t=n(t)),8&e)return t;if(4&e&&"object"==typeof t&&t&&t.__esModule)return t;var r=Object.create(null);if(n.r(r),Object.defineProperty(r,"default",{enumerable:!0,value:t}),2&e&&"string"!=typeof t)for(var i in t)n.d(r,i,function(e){return t[e]}.bind(null,i));return r},n.n=function(t){var e=t&&t.__esModule?function(){return t.default}:function(){return t};return n.d(e,"a",e),e},n.o=function(t,e){return Object.prototype.hasOwnProperty.call(t,e)},n.p="",n(n.s=10)}([,function(e,n){e.exports=t},function(t,n){t.exports=e},,,,,,,function(t,e,n){},function(t,e,n){"use strict";n.r(e),n.d(e,"default",(function(){return Br}));var r=n(2),i=n.n(r);function o(){}var a=function(t){return null==t?o:function(){return this.querySelector(t)}};function s(){return[]}var u=function(t){return null==t?s:function(){return this.querySelectorAll(t)}},l=function(t){return function(){return this.matches(t)}},c=function(t){return new Array(t.length)};function h(t,e){this.ownerDocument=t.ownerDocument,this.namespaceURI=t.namespaceURI,this._next=null,this._parent=t,this.__data__=e}h.prototype={constructor:h,appendChild:function(t){return this._parent.insertBefore(t,this._next)},insertBefore:function(t,e){return this._parent.insertBefore(t,e)},querySelector:function(t){return this._parent.querySelector(t)},querySelectorAll:function(t){return this._parent.querySelectorAll(t)}};function f(t,e,n,r,i,o){for(var a,s=0,u=e.length,l=o.length;se?1:t>=e?0:NaN}var g="http://www.w3.org/1999/xhtml",v={svg:"http://www.w3.org/2000/svg",xhtml:g,xlink:"http://www.w3.org/1999/xlink",xml:"http://www.w3.org/XML/1998/namespace",xmlns:"http://www.w3.org/2000/xmlns/"},y=function(t){var e=t+="",n=e.indexOf(":");return n>=0&&"xmlns"!==(e=t.slice(0,n))&&(t=t.slice(n+1)),v.hasOwnProperty(e)?{space:v[e],local:t}:t};function m(t){return function(){this.removeAttribute(t)}}function w(t){return function(){this.removeAttributeNS(t.space,t.local)}}function b(t,e){return function(){this.setAttribute(t,e)}}function x(t,e){return function(){this.setAttributeNS(t.space,t.local,e)}}function k(t,e){return function(){var n=e.apply(this,arguments);null==n?this.removeAttribute(t):this.setAttribute(t,n)}}function E(t,e){return function(){var n=e.apply(this,arguments);null==n?this.removeAttributeNS(t.space,t.local):this.setAttributeNS(t.space,t.local,n)}}var T=function(t){return t.ownerDocument&&t.ownerDocument.defaultView||t.document&&t||t.defaultView};function N(t){return function(){this.style.removeProperty(t)}}function O(t,e,n){return function(){this.style.setProperty(t,e,n)}}function S(t,e,n){return function(){var r=e.apply(this,arguments);null==r?this.style.removeProperty(t):this.style.setProperty(t,r,n)}}function P(t,e){return t.style.getPropertyValue(e)||T(t).getComputedStyle(t,null).getPropertyValue(e)}function A(t){return function(){delete this[t]}}function M(t,e){return function(){this[t]=e}}function z(t,e){return function(){var n=e.apply(this,arguments);null==n?delete this[t]:this[t]=n}}function j(t){return t.trim().split(/^|\s+/)}function D(t){return t.classList||new B(t)}function B(t){this._node=t,this._names=j(t.getAttribute("class")||"")}function C(t,e){for(var n=D(t),r=-1,i=e.length;++r=0&&(this._names.splice(e,1),this._node.setAttribute("class",this._names.join(" ")))},contains:function(t){return this._names.indexOf(t)>=0}};function q(){this.textContent=""}function U(t){return function(){this.textContent=t}}function F(t){return function(){var e=t.apply(this,arguments);this.textContent=null==e?"":e}}function H(){this.innerHTML=""}function V(t){return function(){this.innerHTML=t}}function X(t){return function(){var e=t.apply(this,arguments);this.innerHTML=null==e?"":e}}function Y(){this.nextSibling&&this.parentNode.appendChild(this)}function Z(){this.previousSibling&&this.parentNode.insertBefore(this,this.parentNode.firstChild)}function W(t){return function(){var e=this.ownerDocument,n=this.namespaceURI;return n===g&&e.documentElement.namespaceURI===g?e.createElement(t):e.createElementNS(n,t)}}function K(t){return function(){return this.ownerDocument.createElementNS(t.space,t.local)}}var Q=function(t){var e=y(t);return(e.local?K:W)(e)};function J(){return null}function tt(){var t=this.parentNode;t&&t.removeChild(this)}function et(){var t=this.cloneNode(!1),e=this.parentNode;return e?e.insertBefore(t,this.nextSibling):t}function nt(){var t=this.cloneNode(!0),e=this.parentNode;return e?e.insertBefore(t,this.nextSibling):t}var rt={},it=null;"undefined"!=typeof document&&("onmouseenter"in document.documentElement||(rt={mouseenter:"mouseover",mouseleave:"mouseout"}));function ot(t,e,n){return t=at(t,e,n),function(e){var n=e.relatedTarget;n&&(n===this||8&n.compareDocumentPosition(this))||t.call(this,e)}}function at(t,e,n){return function(r){var i=it;it=r;try{t.call(this,this.__data__,e,n)}finally{it=i}}}function st(t){return t.trim().split(/^|\s+/).map((function(t){var e="",n=t.indexOf(".");return n>=0&&(e=t.slice(n+1),t=t.slice(0,n)),{type:t,name:e}}))}function ut(t){return function(){var e=this.__on;if(e){for(var n,r=0,i=-1,o=e.length;r=k&&(k=x+1);!(b=_[k])&&++k=0;)(r=i[o])&&(a&&4^r.compareDocumentPosition(a)&&a.parentNode.insertBefore(r,a),a=r);return this},sort:function(t){function e(e,n){return e&&n?t(e.__data__,n.__data__):!e-!n}t||(t=d);for(var n=this._groups,r=n.length,i=new Array(r),o=0;o1?this.each((null==e?N:"function"==typeof e?S:O)(t,e,null==n?"":n)):P(this.node(),t)},property:function(t,e){return arguments.length>1?this.each((null==e?A:"function"==typeof e?z:M)(t,e)):this.node()[t]},classed:function(t,e){var n=j(t+"");if(arguments.length<2){for(var r=D(this.node()),i=-1,o=n.length;++i=0&&(n=t.slice(r+1),t=t.slice(0,r)),t&&!e.hasOwnProperty(t))throw new Error("unknown type: "+t);return{type:t,name:n}}))}function Nt(t,e){for(var n,r=0,i=t.length;r0)for(var n,r,i=new Array(n),o=0;o=0&&e._call.call(null,t),e=e._next;--Ut}()}finally{Ut=0,function(){var t,e,n=Bt,r=1/0;for(;n;)n._call?(r>n._time&&(r=n._time),t=n,n=n._next):(e=n._next,n._next=null,n=t?t._next=e:Bt=e);Ct=t,ne(r)}(),Vt=0}}function ee(){var t=Yt.now(),e=t-$t;e>1e3&&(Xt-=e,$t=t)}function ne(t){Ut||(Ft&&(Ft=clearTimeout(Ft)),t-Vt>24?(t<1/0&&(Ft=setTimeout(te,t-Yt.now()-Xt)),Ht&&(Ht=clearInterval(Ht))):(Ht||($t=Yt.now(),Ht=setInterval(ee,1e3)),Ut=1,Zt(te)))}Qt.prototype=Jt.prototype={constructor:Qt,restart:function(t,e,n){if("function"!=typeof t)throw new TypeError("callback is not a function");n=(null==n?Wt():+n)+(null==e?0:+e),this._next||Ct===this||(Ct?Ct._next=this:Bt=this,Ct=this),this._call=t,this._time=n,ne()},stop:function(){this._call&&(this._call=null,this._time=1/0,ne())}};var re=function(t,e,n){var r=new Qt;return e=null==e?0:+e,r.restart((function(n){r.stop(),t(n+e)}),e,n),r},ie=St("start","end","cancel","interrupt"),oe=[],ae=function(t,e,n,r,i,o){var a=t.__transition;if(a){if(n in a)return}else t.__transition={};!function(t,e,n){var r,i=t.__transition;function o(u){var l,c,h,f;if(1!==n.state)return s();for(l in i)if((f=i[l]).name===n.name){if(3===f.state)return re(o);4===f.state?(f.state=6,f.timer.stop(),f.on.call("interrupt",t,t.__data__,f.index,f.group),delete i[l]):+l0)throw new Error("too late; already scheduled");return n}function ue(t,e){var n=le(t,e);if(n.state>3)throw new Error("too late; already running");return n}function le(t,e){var n=t.__transition;if(!n||!(n=n[e]))throw new Error("transition not found");return n}var ce,he,fe,pe,de=function(t,e){var n,r,i,o=t.__transition,a=!0;if(o){for(i in e=null==e?null:e+"",o)(n=o[i]).name===e?(r=n.state>2&&n.state<5,n.state=6,n.timer.stop(),n.on.call(r?"interrupt":"cancel",t,t.__data__,n.index,n.group),delete o[i]):a=!1;a&&delete t.__transition}},ge=function(t,e){return t=+t,e=+e,function(n){return t*(1-n)+e*n}},ve=180/Math.PI,ye={translateX:0,translateY:0,rotate:0,skewX:0,scaleX:1,scaleY:1},me=function(t,e,n,r,i,o){var a,s,u;return(a=Math.sqrt(t*t+e*e))&&(t/=a,e/=a),(u=t*n+e*r)&&(n-=t*u,r-=e*u),(s=Math.sqrt(n*n+r*r))&&(n/=s,r/=s,u/=s),t*r180?e+=360:e-t>180&&(t+=360),o.push({i:n.push(i(n)+"rotate(",null,r)-2,x:ge(t,e)})):e&&n.push(i(n)+"rotate("+e+r)}(o.rotate,a.rotate,s,u),function(t,e,n,o){t!==e?o.push({i:n.push(i(n)+"skewX(",null,r)-2,x:ge(t,e)}):e&&n.push(i(n)+"skewX("+e+r)}(o.skewX,a.skewX,s,u),function(t,e,n,r,o,a){if(t!==n||e!==r){var s=o.push(i(o)+"scale(",null,",",null,")");a.push({i:s-4,x:ge(t,n)},{i:s-2,x:ge(e,r)})}else 1===n&&1===r||o.push(i(o)+"scale("+n+","+r+")")}(o.scaleX,o.scaleY,a.scaleX,a.scaleY,s,u),o=a=null,function(t){for(var e,n=-1,r=u.length;++n>8&15|e>>4&240,e>>4&15|240&e,(15&e)<<4|15&e,1):8===n?Fe(e>>24&255,e>>16&255,e>>8&255,(255&e)/255):4===n?Fe(e>>12&15|e>>8&240,e>>8&15|e>>4&240,e>>4&15|240&e,((15&e)<<4|15&e)/255):null):(e=ze.exec(t))?new Ve(e[1],e[2],e[3],1):(e=je.exec(t))?new Ve(255*e[1]/100,255*e[2]/100,255*e[3]/100,1):(e=De.exec(t))?Fe(e[1],e[2],e[3],e[4]):(e=Be.exec(t))?Fe(255*e[1]/100,255*e[2]/100,255*e[3]/100,e[4]):(e=Ce.exec(t))?We(e[1],e[2]/100,e[3]/100,1):(e=Ie.exec(t))?We(e[1],e[2]/100,e[3]/100,e[4]):Le.hasOwnProperty(t)?Ue(Le[t]):"transparent"===t?new Ve(NaN,NaN,NaN,0):null}function Ue(t){return new Ve(t>>16&255,t>>8&255,255&t,1)}function Fe(t,e,n,r){return r<=0&&(t=e=n=NaN),new Ve(t,e,n,r)}function He(t){return t instanceof Oe||(t=qe(t)),t?new Ve((t=t.rgb()).r,t.g,t.b,t.opacity):new Ve}function $e(t,e,n,r){return 1===arguments.length?He(t):new Ve(t,e,n,null==r?1:r)}function Ve(t,e,n,r){this.r=+t,this.g=+e,this.b=+n,this.opacity=+r}function Xe(){return"#"+Ze(this.r)+Ze(this.g)+Ze(this.b)}function Ye(){var t=this.opacity;return(1===(t=isNaN(t)?1:Math.max(0,Math.min(1,t)))?"rgb(":"rgba(")+Math.max(0,Math.min(255,Math.round(this.r)||0))+", "+Math.max(0,Math.min(255,Math.round(this.g)||0))+", "+Math.max(0,Math.min(255,Math.round(this.b)||0))+(1===t?")":", "+t+")")}function Ze(t){return((t=Math.max(0,Math.min(255,Math.round(t)||0)))<16?"0":"")+t.toString(16)}function We(t,e,n,r){return r<=0?t=e=n=NaN:n<=0||n>=1?t=e=NaN:e<=0&&(t=NaN),new Qe(t,e,n,r)}function Ke(t){if(t instanceof Qe)return new Qe(t.h,t.s,t.l,t.opacity);if(t instanceof Oe||(t=qe(t)),!t)return new Qe;if(t instanceof Qe)return t;var e=(t=t.rgb()).r/255,n=t.g/255,r=t.b/255,i=Math.min(e,n,r),o=Math.max(e,n,r),a=NaN,s=o-i,u=(o+i)/2;return s?(a=e===o?(n-r)/s+6*(n0&&u<1?0:a,new Qe(a,s,u,t.opacity)}function Qe(t,e,n,r){this.h=+t,this.s=+e,this.l=+n,this.opacity=+r}function Je(t,e,n){return 255*(t<60?e+(n-e)*t/60:t<180?n:t<240?e+(n-e)*(240-t)/60:e)}function tn(t,e,n,r,i){var o=t*t,a=o*t;return((1-3*t+3*o-a)*e+(4-6*o+3*a)*n+(1+3*t+3*o-3*a)*r+a*i)/6}Te(Oe,qe,{copy:function(t){return Object.assign(new this.constructor,this,t)},displayable:function(){return this.rgb().displayable()},hex:Re,formatHex:Re,formatHsl:function(){return Ke(this).formatHsl()},formatRgb:Ge,toString:Ge}),Te(Ve,$e,Ne(Oe,{brighter:function(t){return t=null==t?1/.7:Math.pow(1/.7,t),new Ve(this.r*t,this.g*t,this.b*t,this.opacity)},darker:function(t){return t=null==t?.7:Math.pow(.7,t),new Ve(this.r*t,this.g*t,this.b*t,this.opacity)},rgb:function(){return this},displayable:function(){return-.5<=this.r&&this.r<255.5&&-.5<=this.g&&this.g<255.5&&-.5<=this.b&&this.b<255.5&&0<=this.opacity&&this.opacity<=1},hex:Xe,formatHex:Xe,formatRgb:Ye,toString:Ye})),Te(Qe,(function(t,e,n,r){return 1===arguments.length?Ke(t):new Qe(t,e,n,null==r?1:r)}),Ne(Oe,{brighter:function(t){return t=null==t?1/.7:Math.pow(1/.7,t),new Qe(this.h,this.s,this.l*t,this.opacity)},darker:function(t){return t=null==t?.7:Math.pow(.7,t),new Qe(this.h,this.s,this.l*t,this.opacity)},rgb:function(){var t=this.h%360+360*(this.h<0),e=isNaN(t)||isNaN(this.s)?0:this.s,n=this.l,r=n+(n<.5?n:1-n)*e,i=2*n-r;return new Ve(Je(t>=240?t-240:t+120,i,r),Je(t,i,r),Je(t<120?t+240:t-120,i,r),this.opacity)},displayable:function(){return(0<=this.s&&this.s<=1||isNaN(this.s))&&0<=this.l&&this.l<=1&&0<=this.opacity&&this.opacity<=1},formatHsl:function(){var t=this.opacity;return(1===(t=isNaN(t)?1:Math.max(0,Math.min(1,t)))?"hsl(":"hsla(")+(this.h||0)+", "+100*(this.s||0)+"%, "+100*(this.l||0)+"%"+(1===t?")":", "+t+")")}}));var en=function(t){return function(){return t}};function nn(t,e){return function(n){return t+n*e}}function rn(t){return 1==(t=+t)?on:function(e,n){return n-e?function(t,e,n){return t=Math.pow(t,n),e=Math.pow(e,n)-t,n=1/n,function(r){return Math.pow(t+r*e,n)}}(e,n,t):en(isNaN(e)?n:e)}}function on(t,e){var n=e-t;return n?nn(t,n):en(isNaN(t)?e:t)}var an=function t(e){var n=rn(e);function r(t,e){var r=n((t=$e(t)).r,(e=$e(e)).r),i=n(t.g,e.g),o=n(t.b,e.b),a=on(t.opacity,e.opacity);return function(e){return t.r=r(e),t.g=i(e),t.b=o(e),t.opacity=a(e),t+""}}return r.gamma=t,r}(1);function sn(t){return function(e){var n,r,i=e.length,o=new Array(i),a=new Array(i),s=new Array(i);for(n=0;n=1?(n=1,e-1):Math.floor(n*e),i=t[r],o=t[r+1],a=r>0?t[r-1]:2*i-o,s=ro&&(i=e.slice(o,i),s[a]?s[a]+=i:s[++a]=i),(n=n[0])===(r=r[0])?s[a]?s[a]+=r:s[++a]=r:(s[++a]=null,u.push({i:a,x:ge(n,r)})),o=ln.lastIndex;return o=0&&(t=t.slice(0,e)),!t||"start"===t}))}(e)?se:ue;return function(){var a=o(this,t),s=a.on;s!==r&&(i=(r=s).copy()).on(e,n),a.on=i}}var Sn=yt.prototype.constructor;function Pn(t){return function(){this.style.removeProperty(t)}}function An(t,e,n){return function(r){this.style.setProperty(t,e.call(this,r),n)}}function Mn(t,e,n){var r,i;function o(){var o=e.apply(this,arguments);return o!==i&&(r=(i=o)&&An(t,o,n)),r}return o._value=e,o}function zn(t){return function(e){this.textContent=t.call(this,e)}}function jn(t){var e,n;function r(){var r=t.apply(this,arguments);return r!==n&&(e=(n=r)&&zn(r)),e}return r._value=t,r}var Dn=0;function Bn(t,e,n,r){this._groups=t,this._parents=e,this._name=n,this._id=r}function Cn(){return++Dn}var In=yt.prototype;Bn.prototype=function(t){return yt().transition(t)}.prototype={constructor:Bn,select:function(t){var e=this._name,n=this._id;"function"!=typeof t&&(t=a(t));for(var r=this._groups,i=r.length,o=new Array(i),s=0;sr?(r+i)/2:Math.min(0,r)||Math.max(0,i),a>o?(o+a)/2:Math.min(0,o)||Math.max(0,a))}var Qn=function(t){return function(){return t}};function Jn(t,e,n,r,i,o,a,s,u,l){this.target=t,this.type=e,this.subject=n,this.identifier=r,this.active=i,this.x=o,this.y=a,this.dx=s,this.dy=u,this._=l}function tr(){return!it.ctrlKey&&!it.button}function er(){return this.parentNode}function nr(t){return null==t?{x:it.x,y:it.y}:t}function rr(){return navigator.maxTouchPoints||"ontouchstart"in this}Jn.prototype.on=function(){var t=this._.on.apply(this._,arguments);return t===this._?this:t};var ir=function(){var t,e,n,r,i=tr,o=er,a=nr,s=rr,u={},l=St("start","drag","end"),c=0,h=0;function f(t){t.on("mousedown.drag",p).filter(s).on("touchstart.drag",v).on("touchmove.drag",y).on("touchend.drag touchcancel.drag",m).style("touch-action","none").style("-webkit-tap-highlight-color","rgba(0,0,0,0)")}function p(){if(!r&&i.apply(this,arguments)){var a=_("mouse",o.apply(this,arguments),Gt,this,arguments);a&&(mt(it.view).on("mousemove.drag",d,!0).on("mouseup.drag",g,!0),Mt(it.view),Pt(),n=!1,t=it.clientX,e=it.clientY,a("start"))}}function d(){if(At(),!n){var r=it.clientX-t,i=it.clientY-e;n=r*r+i*i>h}u.mouse("drag")}function g(){mt(it.view).on("mousemove.drag mouseup.drag",null),zt(it.view,n),At(),u.mouse("end")}function v(){if(i.apply(this,arguments)){var t,e,n=it.changedTouches,r=o.apply(this,arguments),a=n.length;for(t=0;t1e-6)if(Math.abs(c*s-u*l)>1e-6&&i){var f=n-o,p=r-a,d=s*s+u*u,g=f*f+p*p,v=Math.sqrt(d),y=Math.sqrt(h),m=i*Math.tan((or-Math.acos((d+h-g)/(2*v*y)))/2),_=m/y,w=m/v;Math.abs(_-1)>1e-6&&(this._+="L"+(t+_*l)+","+(e+_*c)),this._+="A"+i+","+i+",0,0,"+ +(c*f>l*p)+","+(this._x1=t+w*s)+","+(this._y1=e+w*u)}else this._+="L"+(this._x1=t)+","+(this._y1=e);else;},arc:function(t,e,n,r,i,o){t=+t,e=+e,o=!!o;var a=(n=+n)*Math.cos(r),s=n*Math.sin(r),u=t+a,l=e+s,c=1^o,h=o?r-i:i-r;if(n<0)throw new Error("negative radius: "+n);null===this._x1?this._+="M"+u+","+l:(Math.abs(this._x1-u)>1e-6||Math.abs(this._y1-l)>1e-6)&&(this._+="L"+u+","+l),n&&(h<0&&(h=h%ar+ar),h>sr?this._+="A"+n+","+n+",0,1,"+c+","+(t-a)+","+(e-s)+"A"+n+","+n+",0,1,"+c+","+(this._x1=u)+","+(this._y1=l):h>1e-6&&(this._+="A"+n+","+n+",0,"+ +(h>=or)+","+c+","+(this._x1=t+n*Math.cos(i))+","+(this._y1=e+n*Math.sin(i))))},rect:function(t,e,n,r){this._+="M"+(this._x0=this._x1=+t)+","+(this._y0=this._y1=+e)+"h"+ +n+"v"+ +r+"h"+-n+"Z"},toString:function(){return this._}};var cr=lr,hr=function(t){return function(){return t}};function fr(t){this._context=t}fr.prototype={areaStart:function(){this._line=0},areaEnd:function(){this._line=NaN},lineStart:function(){this._point=0},lineEnd:function(){(this._line||0!==this._line&&1===this._point)&&this._context.closePath(),this._line=1-this._line},point:function(t,e){switch(t=+t,e=+e,this._point){case 0:this._point=1,this._line?this._context.lineTo(t,e):this._context.moveTo(t,e);break;case 1:this._point=2;default:this._context.lineTo(t,e)}}};var pr=function(t){return new fr(t)};function dr(t){return t[0]}function gr(t){return t[1]}var vr=n(1),yr=n.n(vr),mr={entityStateReadOnly:{ACTIVE:!1,DELETED:!0,STATUS_ACTIVE:!1,STATUS_DELETED:!0}},_r={nodeArrowDistance:24,refreshGraphForSafari:function(t){t.edgeEl.each((function(t){var e=this,n=$(this).find("pattern");setTimeout((function(t){$(e).find("defs").append(n)}),500)}))},refreshGraphForIE:function(t){var e=t.edgePathEl,n=0;e.each((function(t){var e=$(this).find("marker");$(this).find("marker").remove();var r=this;++n,setTimeout((function(t){$(r).find("defs").append(e),0===--n&&(this.$(".fontLoader").hide(),this.$("svg").fadeTo(1e3,1))}),1e3)}))},dragNode:function(t){var e=this,n=t.g,r=t.svg,i=t.guid,o=(t.edgePathEl,{dragmove:function(t,e){var r=this,i=mt(t),o=n.node(e),a=o.x,s=o.y;o.x+=it.dx,o.y+=it.dy,i.attr("transform","translate("+o.x+","+o.y+")");var u=o.x-a,l=o.y-s;n.edges().forEach((function(t){if(t.v==e||t.w==e){var i=n.edge(t.v,t.w);r.translateEdge(i,u,l),mt(i.elem).select("path").attr("d",r.calcPoints(t))}}))},translateEdge:function(t,e,n){t.points.forEach((function(t){t.x=t.x+e,t.y=t.y+n}))},calcPoints:function(t){var e=n.edge(t.v,t.w),r=n.node(t.v),i=n.node(t.w),o=e.points.slice(1,e.points.length-1);e.points.slice(1,e.points.length-1);return o.unshift(this.intersectRect(r,o[0])),o.push(this.intersectRect(i,o[o.length-1])),function(){var t=dr,e=gr,n=hr(!0),r=null,i=pr,o=null;function a(a){var s,u,l,c=a.length,h=!1;for(null==r&&(o=i(l=cr())),s=0;s<=c;++s)!(sMath.abs(a)*c?(s<0&&(c=-c),h=0===s?0:c*a/s,f=c):(a<0&&(l=-l),h=l,f=0===a?0:l*s/a),{x:r+h,y:o+f}}}),a=ir().on("drag",(function(t){o.dragmove.call(o,this,t)})),s=ir().on("drag",(function(t){o.translateEdge(n.edge(t.v,t.w),it.dx,it.dy);var e=n.edge(t.v,t.w);mt(e.elem).select("path").attr("d",o.calcPoints(t))}));a(r.selectAll("g.node")),s(r.selectAll("g.edgePath"))},zoomIn:function(t){var e=t.svg,n=t.scaleFactor,r=void 0===n?1.3:n;this.d3Zoom.scaleBy(e.transition().duration(750),r)},zoomOut:function(t){var e=t.svg,n=t.scaleFactor,r=void 0===n?.8:n;this.d3Zoom.scaleBy(e.transition().duration(750),r)},zoom:function(t){var e=t.svg,n=t.xa,r=t.ya,i=t.scale;e.transition().duration(750).call(this.d3Zoom.transform,Fn.translate(n,r).scale(i))},fitToScreen:function(t){var e=t.svg,n=e.node(),r=n.getBBox(),i=n.parentElement,o=i.clientWidth,a=i.clientHeight,s=r.width,u=r.height,l=r.x+s/2,c=r.y+u/2,h=(h||.95)/Math.max(s/o,u/a),f=o/2-h*l,p=a/2-h*c;this.zoom({svg:e,xa:f,ya:p,scale:h})},centerNode:function(t){var e=t.guid,n=t.g,r=t.svg,i=t.svgGroupEl,o=(t.edgePathEl,t.width),a=t.height,s=t.fitToScreen,u=t.onCenterZoomed,l=t.isSelected;this.d3Zoom=function(){var t,e,n=Vn,r=Xn,i=Kn,o=Zn,a=Wn,s=[0,1/0],u=[[-1/0,-1/0],[1/0,1/0]],l=250,c=It,h=St("start","zoom","end"),f=0;function p(t){t.property("__zoom",Yn).on("wheel.zoom",w).on("mousedown.zoom",b).on("dblclick.zoom",x).filter(a).on("touchstart.zoom",k).on("touchmove.zoom",E).on("touchend.zoom touchcancel.zoom",T).style("touch-action","none").style("-webkit-tap-highlight-color","rgba(0,0,0,0)")}function d(t,e){return(e=Math.max(s[0],Math.min(s[1],e)))===t.k?t:new Un(e,t.x,t.y)}function g(t,e,n){var r=e[0]-n[0]*t.k,i=e[1]-n[1]*t.k;return r===t.x&&i===t.y?t:new Un(t.k,r,i)}function v(t){return[(+t[0][0]+ +t[1][0])/2,(+t[0][1]+ +t[1][1])/2]}function y(t,e,n){t.on("start.zoom",(function(){m(this,arguments).start()})).on("interrupt.zoom end.zoom",(function(){m(this,arguments).end()})).tween("zoom",(function(){var t=this,i=arguments,o=m(t,i),a=r.apply(t,i),s=null==n?v(a):"function"==typeof n?n.apply(t,i):n,u=Math.max(a[1][0]-a[0][0],a[1][1]-a[0][1]),l=t.__zoom,h="function"==typeof e?e.apply(t,i):e,f=c(l.invert(s).concat(u/l.k),h.invert(s).concat(u/h.k));return function(t){if(1===t)t=h;else{var e=f(t),n=u/e[2];t=new Un(n,s[0]-e[0]*n,s[1]-e[1]*n)}o.zoom(null,t)}}))}function m(t,e,n){return!n&&t.__zooming||new _(t,e)}function _(t,e){this.that=t,this.args=e,this.active=0,this.extent=r.apply(t,e),this.taps=0}function w(){if(n.apply(this,arguments)){var t=m(this,arguments),e=this.__zoom,r=Math.max(s[0],Math.min(s[1],e.k*Math.pow(2,o.apply(this,arguments)))),a=Gt(this);if(t.wheel)t.mouse[0][0]===a[0]&&t.mouse[0][1]===a[1]||(t.mouse[1]=e.invert(t.mouse[0]=a)),clearTimeout(t.wheel);else{if(e.k===r)return;t.mouse=[a,e.invert(a)],de(this),t.start()}$n(),t.wheel=setTimeout(l,150),t.zoom("mouse",i(g(d(e,r),t.mouse[0],t.mouse[1]),t.extent,u))}function l(){t.wheel=null,t.end()}}function b(){if(!e&&n.apply(this,arguments)){var t=m(this,arguments,!0),r=mt(it.view).on("mousemove.zoom",l,!0).on("mouseup.zoom",c,!0),o=Gt(this),a=it.clientX,s=it.clientY;Mt(it.view),Hn(),t.mouse=[o,this.__zoom.invert(o)],de(this),t.start()}function l(){if($n(),!t.moved){var e=it.clientX-a,n=it.clientY-s;t.moved=e*e+n*n>f}t.zoom("mouse",i(g(t.that.__zoom,t.mouse[0]=Gt(t.that),t.mouse[1]),t.extent,u))}function c(){r.on("mousemove.zoom mouseup.zoom",null),zt(it.view,t.moved),$n(),t.end()}}function x(){if(n.apply(this,arguments)){var t=this.__zoom,e=Gt(this),o=t.invert(e),a=t.k*(it.shiftKey?.5:2),s=i(g(d(t,a),e,o),r.apply(this,arguments),u);$n(),l>0?mt(this).transition().duration(l).call(y,s,e):mt(this).call(p.transform,s)}}function k(){if(n.apply(this,arguments)){var e,r,i,o,a=it.touches,s=a.length,u=m(this,arguments,it.changedTouches.length===s);for(Hn(),r=0;rg[id='"+e+"']"),h=(this.d3Zoom.scaleExtent([.01,50]).on("zoom",(function(){i.attr("transform",it.transform)})),null),f=null;if(c.empty()){if(s)return void this.fitToScreen({svg:r});h=n.graph().width/2,f=n.graph().height/2}else{var p=c.attr("transform").replace(/[^0-9\-.,]/g,"").split(",");h=p[0],f=p[1]}var d=-(1.2*h-o/2),g=-(1.2*f-a/2);this.zoom({svg:r,xa:d,ya:g,scale:1.2}),l?r.transition().duration(750).call(this.d3Zoom.transform,Fn.translate(d,g).scale(1.2)):r.call(this.d3Zoom.transform,Fn.translate(d,g).scale(1.2)),u&&u({newScale:1.2,newTranslate:[d,g],d3Zoom:this.d3Zoom,selectedNodeEl:c})},getToolTipDirection:function(t){var e=t.el,n=mt("body").node().getBoundingClientRect().width,r=mt(e).node().getBoundingClientRect(),i="e";return n-r.left<330?(i=n-r.left<330&&r.top<400?"sw":"w",n-r.left<330&&r.top>600&&(i="nw")):r.top>600?(i=n-r.left<330&&r.top>600?"nw":"n",r.left<50&&(i="ne")):r.top<400&&(i=r.left<50?"se":"s"),i},onHoverFade:function(t){var e=t.svg,n=t.g,r=t.mouseenter,i=t.nodesToHighlight,o=t.hoveredNode;return function(t){var i=e.selectAll(".node"),a=e.selectAll(".edgePath");if(r){e.classed("hover",!0);var s=n.successors(o),u=n.predecessors(o);t=s.concat(u);i.classed("hover-active-node",(function(e,n,r){return!!function(t,e,n){if(t===n||e&&e.length&&-1!=e.indexOf(n))return!0}(o,t,e)})),a.classed("hover-active-path",(function(t){return!!(t.v===o||t.w===o?1:0)}))}else e.classed("hover",!1),i.classed("hover-active-node",!1),a.classed("hover-active-path",!1)}(i)},getBaseUrl:function(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:window.location.pathname;return t.replace(/\/[\w-]+.(jsp|html)|\/+$/gi,"")},getEntityIconPath:function(t){var e=t.entityData,n=t.errorUrl,r=t.imgBasePath,i=this.getBaseUrl()+(r||"/img/entity-icon/");if(e){var o=function(t){return i+(mr.entityStateReadOnly[l]?"disabled/"+t:t)},a=function(){return c?mr.entityStateReadOnly[l]?i+"disabled/process.png":i+"process.png":mr.entityStateReadOnly[l]?i+"disabled/table.png":i+"table.png"},s=e.typeName,u=e.serviceType,l=e.status,c=e.isProcess;if(n){if(n.indexOf("table.png")>-1||n.indexOf("process.png")>-1)return null;var h=!(!n||!n.match("entity-icon/"+s+".png|disabled/"+s+".png"));return u&&h?o(u+".png"):a()}return s?o(s+".png"):u?o(u+".png"):a()}},base64Encode:function(t,e){var n=new FileReader;n.addEventListener("load",(function(){return e(n.result)})),n.readAsDataURL(t)},imgShapeRender:function(t,e,n,r){var i=r.dagreD3,o=r.defsEl,a=r.imgBasePath,s=r.guid,u=r.isRankdirToBottom,l=this,c=s,h=this.getEntityIconPath({entityData:n,imgBasePath:a}),f=h.split("/").pop();if(void 0===this.imageObject&&(this.imageObject={}),n.isDeleted&&(f="deleted_"+f),n.id==c)var p=!0;var d=t.append("circle").attr("fill","url(#img_"+f+")").attr("r",u?"30px":"24px").attr("data-stroke",n.id).attr("stroke-width","2px").attr("class","nodeImage "+(p?"currentNode":n.isProcess?"process":"node"));if(p&&d.attr("stroke","#fb4200"),!0===n.isIncomplete){t.attr("class","node isIncomplete show"),t.insert("rect").attr("x","-5").attr("y","-23").attr("width","14").attr("height","16").attr("fill","url(#img_hourglass.svg)").attr("data-stroke",n.id).attr("stroke-width","2px");g({imgName:"hourglass.svg",imageIconPath:"/img/entity-icon/hourglass.svg",leftPosition:"0",topPosition:"0",width:"12",height:"14"})}function g(t){o.select('pattern[id="img_'+t.imgName+'"]').empty()&&o.append("pattern").attr("x","0%").attr("y","0%").attr("patternUnits","objectBoundingBox").attr("id","img_"+t.imgName).attr("width","100%").attr("height","100%").append("image").attr("href",(function(e){var r=this;if(n){!function e(i){var o=i.imagePath,a={url:o,method:"GET",cache:!0};d.attr("data-iconpath",o);var s=new XMLHttpRequest;s.onreadystatechange=function(){if(4===s.readyState)if(200===s.status)"IE"!==yr.a.name?l.base64Encode(this.response,(function(e){l.imageObject[t.imageIconPath]=e,mt(r).attr("xlink:href",e)})):l.imageObject[t.imageIconPath]=o,t.imageIconPath!==d.attr("data-iconpath")&&d.attr("data-iconpathorigin",t.imageIconPath);else if(404===s.status){var i=l.getEntityIconPath({entityData:n,errorUrl:o});if(null===i){var a=mt(r.parentElement);a.select("image").remove(),a.attr("patternContentUnits","objectBoundingBox").append("circle").attr("r","24px").attr("fill","#e8e8e8")}else e({imagePath:i})}},s.responseType="blob",s.open(a.method,a.url,!0),s.send(null)}({imagePath:t.imageIconPath})}})).attr("x",t.leftPosition).attr("y",t.topPosition).attr("width",t.width).attr("height",t.height)}return g({imgName:f,imageIconPath:h,leftPosition:u?"11":"4",topPosition:u?"20":p?"3":"4",width:"40",height:"40"}),n.intersect=function(t){return i.intersect.circle(n,p?l.nodeArrowDistance+3:l.nodeArrowDistance,t)},d},arrowPointRender:function(t,e,n,r,i){var o=i.dagreD3,a=t.node(),s=a?a.parentNode:t;mt(s).select("path.path").attr("marker-end","url(#"+e+")");var u=t.append("marker").attr("id",e).attr("viewBox","0 0 10 10").attr("refX",8).attr("refY",5).attr("markerUnits","strokeWidth").attr("markerWidth",4).attr("markerHeight",4).attr("orient","auto").append("path").attr("d","M 0 0 L 10 5 L 0 10 z").style("fill",n.styleObj.stroke);o.util.applyStyle(u,n[r+"Style"])},saveSvg:function(t){var e=t.svg,n=t.width,r=t.height,i=t.downloadFileName,o=t.onExportLineage,a=this,s=e.clone(!0).node();setTimeout((function(){"Firefox"===yr.a.name&&(s.setAttribute("width",n),s.setAttribute("height",r));var t=mt("body").append("div");t.classed("hidden-svg",!0),t.node().appendChild(s);var e=mt(".hidden-svg svg");e.select("g").attr("transform","scale(1)"),e.select("foreignObject").remove();var u=150,l=150,c=s.getBBox().width+u,h=s.getBBox().height+l,f=s.getBBox().x,p=s.getBBox().y;s.attributes.viewBox.value=f+","+p+","+c+","+h;var d=document.createElement("canvas");d.id="canvas",d.style.display="none",d.width=1*s.getBBox().width+u,d.height=1*s.getBBox().height+l,mt("body").node().appendChild(d);var g=d.getContext("2d"),v=(new XMLSerializer).serializeToString(s),y=window.URL||window.webkitURL||window;g.fillStyle="#FFFFFF",g.fillRect(0,0,d.width,d.height),g.strokeRect(0,0,d.width,d.height),g.restore();var m=new Image(d.width,d.height),_=new Blob([v],{type:"image/svg+xml;base64"});"Safari"===yr.a.name&&(_=new Blob([v],{type:"image/svg+xml"})),g.drawImage(m,50,50,d.width,d.height);var w=y.createObjectURL(_);m.onload=function(){try{var e=document.createElement("a");e.download=i,document.body.appendChild(e),g.drawImage(m,50,50,d.width,d.height),d.toBlob((function(t){t?(e.href=y.createObjectURL(t),t.size>1e7&&o({status:"failed",message:"The Image size is huge, please open the image in a browser!"}),e.click(),o({status:"Success",message:"Successful"}),"Safari"===yr.a.name&&a.refreshGraphForSafari({edgeEl:a.$("svg g.node")})):o({status:"failed",message:"There was an error in downloading Lineage!"})}),"image/png"),t.remove(),d.remove()}catch(t){o({status:"failed",message:"There was an error in downloading Lineage!"})}},m.src=w}),0)}};function wr(t,e){var n=Object.keys(t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(t);e&&(r=r.filter((function(e){return Object.getOwnPropertyDescriptor(t,e).enumerable}))),n.push.apply(n,r)}return n}function br(t){for(var e=1;e0&&void 0!==arguments[0]?arguments[0]:{},i=r.entityData,o=r.errorUrl,a=this.getBaseUrl(window.location.pathname)+Globals.entityImgPath;function s(t){return a+(mr.entityStateReadOnly[e]?"disabled/"+t:t)}function u(){return i.isProcess?mr.entityStateReadOnly[e]?a+"disabled/process.png":a+"process.png":mr.entityStateReadOnly[e]?a+"disabled/table.png":a+"table.png"}if(i&&(n=i.typeName,t=i&&i.serviceType,e=i&&i.status),i){if(o){var l=!(!o||!o.match("entity-icon/"+n+".png|disabled/"+n+".png"));return t&&l?s(t+".png"):u()}return i.typeName?s(i.typeName+".png"):u()}},isProcess:function(t){var e=t.typeName,n=t.superTypes;t.entityDef;return"Process"==e||n.indexOf("Process")>-1},isDeleted:function(t){if(void 0!==t)return mr.entityStateReadOnly[t.status]},isNodeToBeUpdated:function(t,e){var n=e.isProcessHideCheck,r=e.isDeletedEntityHideCheck,i={isProcess:n&&t.isProcess,isDeleted:r&&t.isDeleted};return i.update=i.isProcess||i.isDeleted,i},getServiceType:function(t){var e=t.typeName,n=t.entityDef,r=null;return e&&n&&(r=n.serviceType||null),r},getEntityDef:function(t){var e=t.typeName,n=t.entityDefCollection,r=null;return e&&(r=n.find((function(t){return t.name==e}))),r},getNestedSuperTypes:function(t){var e=t.entityDef,n=t.entityDefCollection,r=new Set;return function t(e,n){e&&e.superTypes&&e.superTypes.length&&e.superTypes.forEach((function(e){r.add(e);var i=n.find((function(t){t.name}));i&&t(i,n)}))}(e,n),Array.from(r)},generateData:function(t){var e=this,n=t.data,r=void 0===n?{}:n,i=t.filterObj,o=t.entityDefCollection,a=t.g,s=t.guid,u=t.setGraphEdge,l=t.setGraphNode;return new Promise((function(t,n){try{var c=r.relations||{},h=r.guidEntityMap||{},f=i.isProcessHideCheck||i.isDeletedEntityHideCheck,p={fill:"none",stroke:"#ffb203",width:3},d=function(t){if(t){if(t.updatedValues)return t;var n=t.displayText?t.displayText:" ",r=Object.assign(t,{shape:"img",updatedValues:!0,label:n.trunc(18),toolTipLabel:n,id:t.guid,isLineage:!0,isIncomplete:t.isIncomplete,entityDef:e.getEntityDef({typeName:t.typeName,entityDefCollection:o})});return r.serviceType=e.getServiceType(r),r.superTypes=e.getNestedSuperTypes(br(br({},r),{},{entityDefCollection:o})),r.isProcess=e.isProcess(r),r.isDeleted=e.isDeleted(r),r}},g=function(t){return"fill:"+t.fill+";stroke:"+t.stroke+";stroke-width:"+t.width},v=function(t,n,r){var i=[];return t.forEach((function(t){if(e.isNodeToBeUpdated(d(h[t]),r).update)if(w[t])i=i.concat(w[t]);else{var n=function t(n,r){if(n&&b[n]){var i=[];return b[n].forEach((function(n){if(e.isNodeToBeUpdated(d(h[n]),r).update){var o=t(n,r);o&&(i=i.concat(o))}else i.push(n)})),i}return null}(t,r);n&&(i=i.concat(n))}else i.push(t)})),i},y=function(t){if(a._nodes[t])return a._nodes[t];var e=d(h[t]);return l(t,e),e},m=function(t,e){var n=arguments.length>2&&void 0!==arguments[2]?arguments[2]:{};u(t,e,br({arrowhead:"arrowPoint",curve:bt,style:g(p),styleObj:p},n))},_=function(t,e){y(t),y(e),m(t,e)},w={};if(f){var b=function(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{},e=t.relations,n={};return e.forEach((function(t){n[t.fromEntityId]?n[t.fromEntityId].push(t.toEntityId):n[t.fromEntityId]=[t.toEntityId]})),n}(r);Object.keys(b).forEach((function(t){var n=b[t],r=e.isNodeToBeUpdated(d(h[t]),i),o=v(n,0,i);r.update?w[t]?w[t]=w[t].concat(o):w[t]=o:o.forEach((function(e){_(t,e)}))}))}else c.length?c.forEach((function(t){_(t.fromEntityId,t.toEntityId)})):y(s);a._nodes[s]&&(a._nodes[s]&&(a._nodes[s].isLineage=!1),e.findImpactNodeAndUpdateData({guid:s,g:a,setEdge:m,getStyleObjStr:g})),t(a)}catch(t){n(t)}}))},findImpactNodeAndUpdateData:function(t){var e=t.guid,n=t.getStyleObjStr,r=t.g,i=t.setEdge,o={},a={fill:"none",stroke:"#fb4200",width:3};!function t(){var e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{},s=arguments.length>1?arguments[1]:void 0,u=Object.keys(e);u.length&&(o[s]||(o[s]=!0,u.forEach((function(e){r._nodes[e]&&(r._nodes[e].isLineage=!1),i(s,e,{style:n(a),styleObj:a}),t(r._sucs[e],e)}))))}(r._sucs[e],e)}};String.prototype.trunc=String.prototype.trunc||function(t){return this.length>t?this.substr(0,t-1)+"...":this};function Er(){}function Tr(t,e){var n=new Er;if(t instanceof Er)t.each((function(t,e){n.set(e,t)}));else if(Array.isArray(t)){var r,i=-1,o=t.length;if(null==e)for(;++i0&&void 0!==arguments[0]?arguments[0]:{};return n._createGraph(n.options,n.graphOptions,t)},clear:function(t){return n.clear(t)},refresh:function(t){return n.refresh(t)},centerAlign:function(t){return n.centerAlign(t)},exportLineage:function(t){return n.exportLineage(t)},zoomIn:function(t){return n.zoomIn(t)},zoomOut:function(t){return n.zoomOut(t)},zoom:function(t){return n.zoom(t)},fullScreen:function(t){return n.fullScreen(t)},searchNode:function(t){return n.searchNode(t)},displayFullName:function(t){return n.displayFullName(t)},removeNodeSelection:function(t){return n.removeNodeSelection(t)},getGraphOptions:function(){return n.graphOptions},getNode:function(t,e){var r=null;return(r=e?n.actualData[t]:n.g._nodes[t])&&(r=Object.assign({},r)),r},getNodes:function(t,e){var r=null;return(r=e?n.actualData:n.g._nodes)&&(r=Object.assign({},r)),r},setNode:this._setGraphNode,setEdge:this._setGraphEdge},!1===a&&this.init(),this.initReturnObj}var e,n,r;return e=t,(n=[{key:"_updateAllOptions",value:function(t){Object.assign(this.options,t);var e=this.svg.node().getBoundingClientRect();this.graphOptions.width=this.options.width||e.width,this.graphOptions.height=this.options.height||e.height;var n=this.graphOptions,r=n.svg,i=n.width,o=n.height,a=n.guid,s=this.options.fitToScreen;r.select("g").node().removeAttribute("transform"),r.attr("viewBox","0 0 "+i+" "+o).attr("enable-background","new 0 0 "+i+" "+o),this.centerAlign({fitToScreen:s,guid:a})}},{key:"_updateOptions",value:function(t){Object.assign(this.options,{filterObj:{isProcessHideCheck:!1,isDeletedEntityHideCheck:!1}},t)}},{key:"init",value:function(){var t=this.options.data,e=void 0===t?{}:t;e.baseEntityGuid&&(this.guid=e.baseEntityGuid),this._initializeGraph(),this._initGraph()}},{key:"clear",value:function(){this.options.el||(this.svg.remove(),this.svg=null),this.g=null,this.graphOptions={}}},{key:"centerAlign",value:function(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{},e=this.svg.select("g"),n=e.selectAll("g.edgePath");_r.centerNode(zr(zr({},this.graphOptions),{},{svgGroupEl:e,edgePathEl:n},t))}},{key:"zoomIn",value:function(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{};_r.zoomIn(zr(zr({},this.graphOptions),t))}},{key:"zoomOut",value:function(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{};_r.zoomOut(zr(zr({},this.graphOptions),t))}},{key:"zoom",value:function(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{};_r.zoom(zr(zr({},this.graphOptions),t))}},{key:"displayFullName",value:function(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{},e=this;this.g.nodes().forEach((function(n){var r=e.svg.selectAll("g.nodes>g[id='"+n+"']"),i=e.g.node(n).toolTipLabel;1==t.bLabelFullText?r.select("tspan").text(i):r.select("tspan").text(i.trunc(18))})),this.selectedNode&&this.searchNode({guid:this.selectedNode})}},{key:"refresh",value:function(){this.clear(),this._initializeGraph(),this._initGraph({refresh:!0}),this.selectedNode=""}},{key:"removeNodeSelection",value:function(){this.svg.selectAll("g.node>circle").classed("node-detail-highlight",!1)}},{key:"searchNode",value:function(t){var e=t.guid,n=t.onSearchNode;this.svg.selectAll(".serach-rect").remove(),this.svg.selectAll(".label").attr("stroke","none"),this.selectedNode=e,this.centerAlign({guid:e,onCenterZoomed:function(t){var e=t.selectedNodeEl,r=e.node().getBBox(),i=r.width+10,o=r.x-5;e.select(".label").attr("stroke","#316132"),e.select("circle").classed("wobble",!0),e.insert("rect","circle").attr("class","serach-rect").attr("stroke","#37bb9b").attr("stroke-width","2.5px").attr("fill","none").attr("x",o).attr("y",-27.5).attr("width",i).attr("height",60),n&&"function"==typeof n&&n(t)},isSelected:!0})}},{key:"exportLineage",value:function(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{},e=t.downloadFileName;if(void 0===e){var n=this.g._nodes[this.guid];e=n&&n.attributes?"".concat(n.attributes.qualifiedName||n.attributes.name||"lineage_export",".png"):"export.png"}_r.saveSvg(zr(zr({},this.graphOptions),{},{downloadFileName:e,onExportLineage:function(t){function e(e){return t.apply(this,arguments)}return e.toString=function(){return t.toString()},e}((function(e){t.onExportLineage&&onExportLineage(e)}))}))}},{key:"fullScreen",value:function(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{},e=t.el;if(void 0===e)throw new Error("LineageHelper requires el propety to apply fullScreen class");var n=mt(e);return n.classed("fullscreen-mode")?(n.classed("fullscreen-mode",!1),!1):(n.classed("fullscreen-mode",!0),!0)}},{key:"_getValueFromUser",value:function(t){if(void 0!==t)return"function"==typeof t?t():t}},{key:"_initializeGraph",value:function(){var t=this.options,e=t.width,n=void 0===e?"100%":e,r=t.height,o=void 0===r?"100%":r,a=t.el;this.svg=mt(a),a instanceof SVGElement||(this.svg.selectAll("*").remove(),this.svg=this.svg.append("svg").attr("xmlns","http://www.w3.org/2000/svg").attr(" xmlns:xlink","http://www.w3.org/1999/xlink").attr("version","1.1").attr("width",n).attr("height",o)),this.g=(new i.a.graphlib.Graph).setGraph(Object.assign({nodesep:50,ranksep:90,rankdir:"LR",marginx:20,marginy:20,transition:function(t){return t.transition().duration(500)}},this.options.dagreOptions)).setDefaultEdgeLabel((function(){return{}}));var s=this.svg.node().getBoundingClientRect();this.actualData={},this.graphOptions={svg:this.svg,g:this.g,dagreD3:i.a,guid:this.guid,width:this.options.width||s.width,height:this.options.height||s.height}}},{key:"_initGraph",value:function(){var t=this,e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{},n=e.refresh;this.svg&&this.svg.select("g").remove();var r=this.options.filterObj;if(this.options.getFilterObj){var i=this.options.getFilterObj();if(void 0!==i||null!==i){if("object"!==Ar(i))throw new Error("getFilterObj expect return type `object`,`null` or `Undefined`");r=i}}if(!0!==this.options.setDataManually)return void 0===this.options.data||this.options.data&&0===this.options.data.relations.length&&_.isEmpty(this.options.data.guidEntityMap)?(this.options.beforeRender&&this.options.beforeRender(),this.svg.append("text").attr("x","50%").attr("y","50%").attr("alignment-baseline","middle").attr("text-anchor","middle").text("No lineage data found"),void(this.options.afterRender&&this.options.afterRender())):kr.generateData(zr(zr(zr({},this.options),{},{filterObj:r},this.graphOptions),{},{setGraphNode:this._setGraphNode,setGraphEdge:this._setGraphEdge})).then((function(e){t._createGraph(t.options,t.graphOptions,{refresh:n})}))}},{key:"_createGraph",value:function(t,e,n){var r=this,o=t.data,a=void 0===o?{}:o,s=t.imgBasePath,u=t.isShowTooltip,l=t.isShowHoverPath,c=t.onLabelClick,h=t.onPathClick,f=t.onNodeClick,p=t.zoom,d=t.fitToScreen,g=t.getToolTipContent,v=t.toolTipTitle,y=n.refresh;this.options.beforeRender&&this.options.beforeRender(),this.selectedNode="";var m=this,_=e.svg,w=e.g,b=e.width,x=e.height,k=this.options.dagreOptions&&"tb"===this.options.dagreOptions.rankdir;if(_ instanceof yt==0)throw new Error("svg is not initialized or something went wrong while creatig graph instance");if(void 0!==w._nodes&&0!==w._nodes.length){w.nodes().forEach((function(t){var e=w.node(t);e&&(e.rx=e.ry=5)})),_.attr("viewBox","0 0 "+b+" "+x).attr("enable-background","new 0 0 "+b+" "+x);var E=_.append("g"),T=_.append("defs"),N=new i.a.render;N.arrows().arrowPoint=function(){return _r.arrowPointRender.apply(_r,Array.prototype.slice.call(arguments).concat([zr({},e)]))},N.shapes().img=function(){return _r.imgShapeRender.apply(_r,Array.prototype.slice.call(arguments).concat([zr(zr({},e),{},{isRankdirToBottom:k,imgBasePath:m._getValueFromUser(s),defsEl:T})]))};var O=function(){var t=function(){return"n"},e=function(){return[0,0]},n=function(){return" "},r=document.body,i=h(),o=null,a=null,s=null;function u(t){(o=function(t){var e=t.node();return e?"svg"===e.tagName.toLowerCase()?e:e.ownerSVGElement:null}(t))&&(a=o.createSVGPoint(),r.appendChild(i))}u.show=function(){var i=Array.prototype.slice.call(arguments);i[i.length-1]instanceof SVGElement&&(s=i.pop());var o,a=n.apply(this,i),h=e.apply(this,i),p=t.apply(this,i),d=f(),g=c.length,v=document.documentElement.scrollTop||r.scrollTop,y=document.documentElement.scrollLeft||r.scrollLeft;for(d.html(a).style("opacity",1).style("pointer-events","all");g--;)d.classed(c[g],!1);return o=l.get(p).apply(this),d.classed(p,!0).style("top",o.top+h[0]+v+"px").style("left",o.left+h[1]+y+"px"),u},u.hide=function(){return f().style("opacity",0).style("pointer-events","none"),u},u.attr=function(t,e){if(arguments.length<2&&"string"==typeof t)return f().attr(t);var n=Array.prototype.slice.call(arguments);return yt.prototype.attr.apply(f(),n),u},u.style=function(t,e){if(arguments.length<2&&"string"==typeof t)return f().style(t);var n=Array.prototype.slice.call(arguments);return yt.prototype.style.apply(f(),n),u},u.direction=function(e){return arguments.length?(t=null==e?e:d(e),u):t},u.offset=function(t){return arguments.length?(e=null==t?t:d(t),u):e},u.html=function(t){return arguments.length?(n=null==t?t:d(t),u):n},u.rootElement=function(t){return arguments.length?(r=null==t?t:d(t),u):r},u.destroy=function(){return i&&(f().remove(),i=null),u};var l=Nr({n:function(){var t=p(this);return{top:t.n.y-i.offsetHeight,left:t.n.x-i.offsetWidth/2}},s:function(){var t=p(this);return{top:t.s.y,left:t.s.x-i.offsetWidth/2}},e:function(){var t=p(this);return{top:t.e.y-i.offsetHeight/2,left:t.e.x}},w:function(){var t=p(this);return{top:t.w.y-i.offsetHeight/2,left:t.w.x-i.offsetWidth}},nw:function(){var t=p(this);return{top:t.nw.y-i.offsetHeight,left:t.nw.x-i.offsetWidth}},ne:function(){var t=p(this);return{top:t.ne.y-i.offsetHeight,left:t.ne.x}},sw:function(){var t=p(this);return{top:t.sw.y,left:t.sw.x-i.offsetWidth}},se:function(){var t=p(this);return{top:t.se.y,left:t.se.x}}}),c=l.keys();function h(){var t=mt(document.createElement("div"));return t.style("position","absolute").style("top",0).style("opacity",0).style("pointer-events","none").style("box-sizing","border-box"),t.node()}function f(){return null==i&&(i=h(),r.appendChild(i)),mt(i)}function p(t){for(var e=s||t;null==e.getScreenCTM&&null!=e.parentNode;)e=e.parentNode;var n={},r=e.getScreenCTM(),i=e.getBBox(),o=i.width,u=i.height,l=i.x,c=i.y;return a.x=l,a.y=c,n.nw=a.matrixTransform(r),a.x+=o,n.ne=a.matrixTransform(r),a.y+=u,n.se=a.matrixTransform(r),a.x-=o,n.sw=a.matrixTransform(r),a.y-=u/2,n.w=a.matrixTransform(r),a.x+=o,n.e=a.matrixTransform(r),a.x-=o/2,a.y-=u/2,n.n=a.matrixTransform(r),a.y+=u,n.s=a.matrixTransform(r),n}function d(t){return"function"==typeof t?t:function(){return t}}return u}().attr("class","d3-tip").offset([10,0]).html((function(t){if(g&&"function"==typeof g)return g(t,w.node(t));var e=w.node(t),n="";return v?n="

"+v+"
":e.id!==r.guid&&(n="
"+(e.isLineage?"Lineage":"Impact")+"
"),n+="
"+e.toolTipLabel+"
",e.typeName&&(n+="
("+e.typeName+")
"),e.queryText&&(n+="
Query: "+e.queryText+"
"),"
"+n+"
"}));_.call(O),N(E,w),E.selectAll("g.nodes g.label").attr("transform",(function(){return k?"translate(2,-20)":"translate(2,-38)"})).attr("font-size","10px").on("mouseenter",(function(t){it.preventDefault(),mt(this).classed("highlight",!0)})).on("mouseleave",(function(t){it.preventDefault(),mt(this).classed("highlight",!1)})).on("click",(function(t){it.preventDefault(),c&&"function"==typeof c&&c({clickedData:t}),O.hide(t)})),E.selectAll("g.nodes g.node circle").on("mouseenter",(function(t,n,r){if(m.activeNode=!0,this.getScreenCTM().translate(+this.getAttribute("cx"),+this.getAttribute("cy")),m.svg.selectAll(".node").classed("active",!1),mt(this).classed("active",!0),m._getValueFromUser(u)){var i=_r.getToolTipDirection({el:this});O.direction(i).show(t,this)}!1!==m._getValueFromUser(l)&&_r.onHoverFade(zr({opacity:.3,mouseenter:!0,hoveredNode:t},e))})).on("mouseleave",(function(t){m.activeNode=!1;var n=this;setTimeout((function(e){m.activeTip||m.activeNode||(mt(n).classed("active",!1),m._getValueFromUser(u)&&O.hide(t))}),150),!1!==m._getValueFromUser(l)&&_r.onHoverFade(zr({mouseenter:!1,hoveredNode:t},e))})).on("click",(function(t){it.defaultPrevented||(it.preventDefault(),O.hide(t),_.selectAll("g.node>circle").classed("node-detail-highlight",!1),mt(this).classed("node-detail-highlight",!0),f&&"function"==typeof f&&f({clickedData:t,el:this}))}));var S=E.selectAll("g.edgePath");S.selectAll("path.path").on("click",(function(t){if(h&&"function"==typeof h){var e=a.relations.find((function(e){if(e.fromEntityId===t.v&&e.toEntityId===t.w)return!0}));h({pathRelationObj:e,clickedData:t})}})),!1!==p&&_r.centerNode(zr(zr({},e),{},{fitToScreen:d,svgGroupEl:E,edgePathEl:S})),_r.dragNode(zr(zr({},e),{},{edgePathEl:S})),!0!==y&&this._addLegend(),this.options.afterRender&&this.options.afterRender()}else _.html('No relations to display')}},{key:"_addLegend",value:function(){if(!1!==this.options.legends){var t=mt(this.options.legendsEl||this.options.el).insert("div",":first-child").classed("legends",!0),e=t.append("span").style("color","#fb4200");e.append("i").classed("fa fa-circle-o fa-fw",!0),e.append("span").html("Current Entity"),(e=t.append("span").style("color","#686868")).append("i").classed("fa fa-hourglass-half fa-fw",!0),e.append("span").html("In Progress"),(e=t.append("span").style("color","#df9b00")).append("i").classed("fa fa-long-arrow-right fa-fw",!0),e.append("span").html("Lineage"),(e=t.append("span").style("color","#fb4200")).append("i").classed("fa fa-long-arrow-right fa-fw",!0),e.append("span").html("Impact")}}}])&&Dr(e.prototype,n),r&&Dr(e,r),t}()}])})); \ No newline at end of file +!function(t,e){"object"==typeof exports&&"object"==typeof module?module.exports=e(require("platform"),require("dagreD3")):"function"==typeof define&&define.amd?define(["platform","dagreD3"],e):"object"==typeof exports?exports.LineageHelper=e(require("platform"),require("dagreD3")):t.LineageHelper=e(t.platform,t.dagreD3)}(window,(function(t,e){return function(t){var e={};function n(r){if(e[r])return e[r].exports;var i=e[r]={i:r,l:!1,exports:{}};return t[r].call(i.exports,i,i.exports,n),i.l=!0,i.exports}return n.m=t,n.c=e,n.d=function(t,e,r){n.o(t,e)||Object.defineProperty(t,e,{enumerable:!0,get:r})},n.r=function(t){"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(t,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(t,"__esModule",{value:!0})},n.t=function(t,e){if(1&e&&(t=n(t)),8&e)return t;if(4&e&&"object"==typeof t&&t&&t.__esModule)return t;var r=Object.create(null);if(n.r(r),Object.defineProperty(r,"default",{enumerable:!0,value:t}),2&e&&"string"!=typeof t)for(var i in t)n.d(r,i,function(e){return t[e]}.bind(null,i));return r},n.n=function(t){var e=t&&t.__esModule?function(){return t.default}:function(){return t};return n.d(e,"a",e),e},n.o=function(t,e){return Object.prototype.hasOwnProperty.call(t,e)},n.p="",n(n.s=10)}([,function(e,n){e.exports=t},function(t,n){t.exports=e},,,,,,,function(t,e,n){},function(t,e,n){"use strict";n.r(e),n.d(e,"default",(function(){return Cr}));var r=n(2),i=n.n(r);function o(){}var a=function(t){return null==t?o:function(){return this.querySelector(t)}};function s(){return[]}var u=function(t){return null==t?s:function(){return this.querySelectorAll(t)}},l=function(t){return function(){return this.matches(t)}},c=function(t){return new Array(t.length)};function h(t,e){this.ownerDocument=t.ownerDocument,this.namespaceURI=t.namespaceURI,this._next=null,this._parent=t,this.__data__=e}h.prototype={constructor:h,appendChild:function(t){return this._parent.insertBefore(t,this._next)},insertBefore:function(t,e){return this._parent.insertBefore(t,e)},querySelector:function(t){return this._parent.querySelector(t)},querySelectorAll:function(t){return this._parent.querySelectorAll(t)}};function f(t,e,n,r,i,o){for(var a,s=0,u=e.length,l=o.length;se?1:t>=e?0:NaN}var g="http://www.w3.org/1999/xhtml",v={svg:"http://www.w3.org/2000/svg",xhtml:g,xlink:"http://www.w3.org/1999/xlink",xml:"http://www.w3.org/XML/1998/namespace",xmlns:"http://www.w3.org/2000/xmlns/"},y=function(t){var e=t+="",n=e.indexOf(":");return n>=0&&"xmlns"!==(e=t.slice(0,n))&&(t=t.slice(n+1)),v.hasOwnProperty(e)?{space:v[e],local:t}:t};function m(t){return function(){this.removeAttribute(t)}}function w(t){return function(){this.removeAttributeNS(t.space,t.local)}}function b(t,e){return function(){this.setAttribute(t,e)}}function x(t,e){return function(){this.setAttributeNS(t.space,t.local,e)}}function k(t,e){return function(){var n=e.apply(this,arguments);null==n?this.removeAttribute(t):this.setAttribute(t,n)}}function E(t,e){return function(){var n=e.apply(this,arguments);null==n?this.removeAttributeNS(t.space,t.local):this.setAttributeNS(t.space,t.local,n)}}var T=function(t){return t.ownerDocument&&t.ownerDocument.defaultView||t.document&&t||t.defaultView};function N(t){return function(){this.style.removeProperty(t)}}function O(t,e,n){return function(){this.style.setProperty(t,e,n)}}function S(t,e,n){return function(){var r=e.apply(this,arguments);null==r?this.style.removeProperty(t):this.style.setProperty(t,r,n)}}function P(t,e){return t.style.getPropertyValue(e)||T(t).getComputedStyle(t,null).getPropertyValue(e)}function A(t){return function(){delete this[t]}}function M(t,e){return function(){this[t]=e}}function z(t,e){return function(){var n=e.apply(this,arguments);null==n?delete this[t]:this[t]=n}}function j(t){return t.trim().split(/^|\s+/)}function D(t){return t.classList||new C(t)}function C(t){this._node=t,this._names=j(t.getAttribute("class")||"")}function B(t,e){for(var n=D(t),r=-1,i=e.length;++r=0&&(this._names.splice(e,1),this._node.setAttribute("class",this._names.join(" ")))},contains:function(t){return this._names.indexOf(t)>=0}};function q(){this.textContent=""}function F(t){return function(){this.textContent=t}}function U(t){return function(){var e=t.apply(this,arguments);this.textContent=null==e?"":e}}function H(){this.innerHTML=""}function V(t){return function(){this.innerHTML=t}}function X(t){return function(){var e=t.apply(this,arguments);this.innerHTML=null==e?"":e}}function Y(){this.nextSibling&&this.parentNode.appendChild(this)}function Z(){this.previousSibling&&this.parentNode.insertBefore(this,this.parentNode.firstChild)}function W(t){return function(){var e=this.ownerDocument,n=this.namespaceURI;return n===g&&e.documentElement.namespaceURI===g?e.createElement(t):e.createElementNS(n,t)}}function K(t){return function(){return this.ownerDocument.createElementNS(t.space,t.local)}}var Q=function(t){var e=y(t);return(e.local?K:W)(e)};function J(){return null}function tt(){var t=this.parentNode;t&&t.removeChild(this)}function et(){var t=this.cloneNode(!1),e=this.parentNode;return e?e.insertBefore(t,this.nextSibling):t}function nt(){var t=this.cloneNode(!0),e=this.parentNode;return e?e.insertBefore(t,this.nextSibling):t}var rt={},it=null;"undefined"!=typeof document&&("onmouseenter"in document.documentElement||(rt={mouseenter:"mouseover",mouseleave:"mouseout"}));function ot(t,e,n){return t=at(t,e,n),function(e){var n=e.relatedTarget;n&&(n===this||8&n.compareDocumentPosition(this))||t.call(this,e)}}function at(t,e,n){return function(r){var i=it;it=r;try{t.call(this,this.__data__,e,n)}finally{it=i}}}function st(t){return t.trim().split(/^|\s+/).map((function(t){var e="",n=t.indexOf(".");return n>=0&&(e=t.slice(n+1),t=t.slice(0,n)),{type:t,name:e}}))}function ut(t){return function(){var e=this.__on;if(e){for(var n,r=0,i=-1,o=e.length;r=k&&(k=x+1);!(b=_[k])&&++k=0;)(r=i[o])&&(a&&4^r.compareDocumentPosition(a)&&a.parentNode.insertBefore(r,a),a=r);return this},sort:function(t){function e(e,n){return e&&n?t(e.__data__,n.__data__):!e-!n}t||(t=d);for(var n=this._groups,r=n.length,i=new Array(r),o=0;o1?this.each((null==e?N:"function"==typeof e?S:O)(t,e,null==n?"":n)):P(this.node(),t)},property:function(t,e){return arguments.length>1?this.each((null==e?A:"function"==typeof e?z:M)(t,e)):this.node()[t]},classed:function(t,e){var n=j(t+"");if(arguments.length<2){for(var r=D(this.node()),i=-1,o=n.length;++i=0&&(n=t.slice(r+1),t=t.slice(0,r)),t&&!e.hasOwnProperty(t))throw new Error("unknown type: "+t);return{type:t,name:n}}))}function Nt(t,e){for(var n,r=0,i=t.length;r0)for(var n,r,i=new Array(n),o=0;o=0&&e._call.call(null,t),e=e._next;--Ft}()}finally{Ft=0,function(){var t,e,n=Ct,r=1/0;for(;n;)n._call?(r>n._time&&(r=n._time),t=n,n=n._next):(e=n._next,n._next=null,n=t?t._next=e:Ct=e);Bt=t,ne(r)}(),Vt=0}}function ee(){var t=Yt.now(),e=t-$t;e>1e3&&(Xt-=e,$t=t)}function ne(t){Ft||(Ut&&(Ut=clearTimeout(Ut)),t-Vt>24?(t<1/0&&(Ut=setTimeout(te,t-Yt.now()-Xt)),Ht&&(Ht=clearInterval(Ht))):(Ht||($t=Yt.now(),Ht=setInterval(ee,1e3)),Ft=1,Zt(te)))}Qt.prototype=Jt.prototype={constructor:Qt,restart:function(t,e,n){if("function"!=typeof t)throw new TypeError("callback is not a function");n=(null==n?Wt():+n)+(null==e?0:+e),this._next||Bt===this||(Bt?Bt._next=this:Ct=this,Bt=this),this._call=t,this._time=n,ne()},stop:function(){this._call&&(this._call=null,this._time=1/0,ne())}};var re=function(t,e,n){var r=new Qt;return e=null==e?0:+e,r.restart((function(n){r.stop(),t(n+e)}),e,n),r},ie=St("start","end","cancel","interrupt"),oe=[],ae=function(t,e,n,r,i,o){var a=t.__transition;if(a){if(n in a)return}else t.__transition={};!function(t,e,n){var r,i=t.__transition;function o(u){var l,c,h,f;if(1!==n.state)return s();for(l in i)if((f=i[l]).name===n.name){if(3===f.state)return re(o);4===f.state?(f.state=6,f.timer.stop(),f.on.call("interrupt",t,t.__data__,f.index,f.group),delete i[l]):+l0)throw new Error("too late; already scheduled");return n}function ue(t,e){var n=le(t,e);if(n.state>3)throw new Error("too late; already running");return n}function le(t,e){var n=t.__transition;if(!n||!(n=n[e]))throw new Error("transition not found");return n}var ce,he,fe,pe,de=function(t,e){var n,r,i,o=t.__transition,a=!0;if(o){for(i in e=null==e?null:e+"",o)(n=o[i]).name===e?(r=n.state>2&&n.state<5,n.state=6,n.timer.stop(),n.on.call(r?"interrupt":"cancel",t,t.__data__,n.index,n.group),delete o[i]):a=!1;a&&delete t.__transition}},ge=function(t,e){return t=+t,e=+e,function(n){return t*(1-n)+e*n}},ve=180/Math.PI,ye={translateX:0,translateY:0,rotate:0,skewX:0,scaleX:1,scaleY:1},me=function(t,e,n,r,i,o){var a,s,u;return(a=Math.sqrt(t*t+e*e))&&(t/=a,e/=a),(u=t*n+e*r)&&(n-=t*u,r-=e*u),(s=Math.sqrt(n*n+r*r))&&(n/=s,r/=s,u/=s),t*r180?e+=360:e-t>180&&(t+=360),o.push({i:n.push(i(n)+"rotate(",null,r)-2,x:ge(t,e)})):e&&n.push(i(n)+"rotate("+e+r)}(o.rotate,a.rotate,s,u),function(t,e,n,o){t!==e?o.push({i:n.push(i(n)+"skewX(",null,r)-2,x:ge(t,e)}):e&&n.push(i(n)+"skewX("+e+r)}(o.skewX,a.skewX,s,u),function(t,e,n,r,o,a){if(t!==n||e!==r){var s=o.push(i(o)+"scale(",null,",",null,")");a.push({i:s-4,x:ge(t,n)},{i:s-2,x:ge(e,r)})}else 1===n&&1===r||o.push(i(o)+"scale("+n+","+r+")")}(o.scaleX,o.scaleY,a.scaleX,a.scaleY,s,u),o=a=null,function(t){for(var e,n=-1,r=u.length;++n>8&15|e>>4&240,e>>4&15|240&e,(15&e)<<4|15&e,1):8===n?Ue(e>>24&255,e>>16&255,e>>8&255,(255&e)/255):4===n?Ue(e>>12&15|e>>8&240,e>>8&15|e>>4&240,e>>4&15|240&e,((15&e)<<4|15&e)/255):null):(e=ze.exec(t))?new Ve(e[1],e[2],e[3],1):(e=je.exec(t))?new Ve(255*e[1]/100,255*e[2]/100,255*e[3]/100,1):(e=De.exec(t))?Ue(e[1],e[2],e[3],e[4]):(e=Ce.exec(t))?Ue(255*e[1]/100,255*e[2]/100,255*e[3]/100,e[4]):(e=Be.exec(t))?We(e[1],e[2]/100,e[3]/100,1):(e=Ie.exec(t))?We(e[1],e[2]/100,e[3]/100,e[4]):Le.hasOwnProperty(t)?Fe(Le[t]):"transparent"===t?new Ve(NaN,NaN,NaN,0):null}function Fe(t){return new Ve(t>>16&255,t>>8&255,255&t,1)}function Ue(t,e,n,r){return r<=0&&(t=e=n=NaN),new Ve(t,e,n,r)}function He(t){return t instanceof Oe||(t=qe(t)),t?new Ve((t=t.rgb()).r,t.g,t.b,t.opacity):new Ve}function $e(t,e,n,r){return 1===arguments.length?He(t):new Ve(t,e,n,null==r?1:r)}function Ve(t,e,n,r){this.r=+t,this.g=+e,this.b=+n,this.opacity=+r}function Xe(){return"#"+Ze(this.r)+Ze(this.g)+Ze(this.b)}function Ye(){var t=this.opacity;return(1===(t=isNaN(t)?1:Math.max(0,Math.min(1,t)))?"rgb(":"rgba(")+Math.max(0,Math.min(255,Math.round(this.r)||0))+", "+Math.max(0,Math.min(255,Math.round(this.g)||0))+", "+Math.max(0,Math.min(255,Math.round(this.b)||0))+(1===t?")":", "+t+")")}function Ze(t){return((t=Math.max(0,Math.min(255,Math.round(t)||0)))<16?"0":"")+t.toString(16)}function We(t,e,n,r){return r<=0?t=e=n=NaN:n<=0||n>=1?t=e=NaN:e<=0&&(t=NaN),new Qe(t,e,n,r)}function Ke(t){if(t instanceof Qe)return new Qe(t.h,t.s,t.l,t.opacity);if(t instanceof Oe||(t=qe(t)),!t)return new Qe;if(t instanceof Qe)return t;var e=(t=t.rgb()).r/255,n=t.g/255,r=t.b/255,i=Math.min(e,n,r),o=Math.max(e,n,r),a=NaN,s=o-i,u=(o+i)/2;return s?(a=e===o?(n-r)/s+6*(n0&&u<1?0:a,new Qe(a,s,u,t.opacity)}function Qe(t,e,n,r){this.h=+t,this.s=+e,this.l=+n,this.opacity=+r}function Je(t,e,n){return 255*(t<60?e+(n-e)*t/60:t<180?n:t<240?e+(n-e)*(240-t)/60:e)}function tn(t,e,n,r,i){var o=t*t,a=o*t;return((1-3*t+3*o-a)*e+(4-6*o+3*a)*n+(1+3*t+3*o-3*a)*r+a*i)/6}Te(Oe,qe,{copy:function(t){return Object.assign(new this.constructor,this,t)},displayable:function(){return this.rgb().displayable()},hex:Re,formatHex:Re,formatHsl:function(){return Ke(this).formatHsl()},formatRgb:Ge,toString:Ge}),Te(Ve,$e,Ne(Oe,{brighter:function(t){return t=null==t?1/.7:Math.pow(1/.7,t),new Ve(this.r*t,this.g*t,this.b*t,this.opacity)},darker:function(t){return t=null==t?.7:Math.pow(.7,t),new Ve(this.r*t,this.g*t,this.b*t,this.opacity)},rgb:function(){return this},displayable:function(){return-.5<=this.r&&this.r<255.5&&-.5<=this.g&&this.g<255.5&&-.5<=this.b&&this.b<255.5&&0<=this.opacity&&this.opacity<=1},hex:Xe,formatHex:Xe,formatRgb:Ye,toString:Ye})),Te(Qe,(function(t,e,n,r){return 1===arguments.length?Ke(t):new Qe(t,e,n,null==r?1:r)}),Ne(Oe,{brighter:function(t){return t=null==t?1/.7:Math.pow(1/.7,t),new Qe(this.h,this.s,this.l*t,this.opacity)},darker:function(t){return t=null==t?.7:Math.pow(.7,t),new Qe(this.h,this.s,this.l*t,this.opacity)},rgb:function(){var t=this.h%360+360*(this.h<0),e=isNaN(t)||isNaN(this.s)?0:this.s,n=this.l,r=n+(n<.5?n:1-n)*e,i=2*n-r;return new Ve(Je(t>=240?t-240:t+120,i,r),Je(t,i,r),Je(t<120?t+240:t-120,i,r),this.opacity)},displayable:function(){return(0<=this.s&&this.s<=1||isNaN(this.s))&&0<=this.l&&this.l<=1&&0<=this.opacity&&this.opacity<=1},formatHsl:function(){var t=this.opacity;return(1===(t=isNaN(t)?1:Math.max(0,Math.min(1,t)))?"hsl(":"hsla(")+(this.h||0)+", "+100*(this.s||0)+"%, "+100*(this.l||0)+"%"+(1===t?")":", "+t+")")}}));var en=function(t){return function(){return t}};function nn(t,e){return function(n){return t+n*e}}function rn(t){return 1==(t=+t)?on:function(e,n){return n-e?function(t,e,n){return t=Math.pow(t,n),e=Math.pow(e,n)-t,n=1/n,function(r){return Math.pow(t+r*e,n)}}(e,n,t):en(isNaN(e)?n:e)}}function on(t,e){var n=e-t;return n?nn(t,n):en(isNaN(t)?e:t)}var an=function t(e){var n=rn(e);function r(t,e){var r=n((t=$e(t)).r,(e=$e(e)).r),i=n(t.g,e.g),o=n(t.b,e.b),a=on(t.opacity,e.opacity);return function(e){return t.r=r(e),t.g=i(e),t.b=o(e),t.opacity=a(e),t+""}}return r.gamma=t,r}(1);function sn(t){return function(e){var n,r,i=e.length,o=new Array(i),a=new Array(i),s=new Array(i);for(n=0;n=1?(n=1,e-1):Math.floor(n*e),i=t[r],o=t[r+1],a=r>0?t[r-1]:2*i-o,s=ro&&(i=e.slice(o,i),s[a]?s[a]+=i:s[++a]=i),(n=n[0])===(r=r[0])?s[a]?s[a]+=r:s[++a]=r:(s[++a]=null,u.push({i:a,x:ge(n,r)})),o=ln.lastIndex;return o=0&&(t=t.slice(0,e)),!t||"start"===t}))}(e)?se:ue;return function(){var a=o(this,t),s=a.on;s!==r&&(i=(r=s).copy()).on(e,n),a.on=i}}var Sn=yt.prototype.constructor;function Pn(t){return function(){this.style.removeProperty(t)}}function An(t,e,n){return function(r){this.style.setProperty(t,e.call(this,r),n)}}function Mn(t,e,n){var r,i;function o(){var o=e.apply(this,arguments);return o!==i&&(r=(i=o)&&An(t,o,n)),r}return o._value=e,o}function zn(t){return function(e){this.textContent=t.call(this,e)}}function jn(t){var e,n;function r(){var r=t.apply(this,arguments);return r!==n&&(e=(n=r)&&zn(r)),e}return r._value=t,r}var Dn=0;function Cn(t,e,n,r){this._groups=t,this._parents=e,this._name=n,this._id=r}function Bn(){return++Dn}var In=yt.prototype;Cn.prototype=function(t){return yt().transition(t)}.prototype={constructor:Cn,select:function(t){var e=this._name,n=this._id;"function"!=typeof t&&(t=a(t));for(var r=this._groups,i=r.length,o=new Array(i),s=0;sr?(r+i)/2:Math.min(0,r)||Math.max(0,i),a>o?(o+a)/2:Math.min(0,o)||Math.max(0,a))}var Qn=function(t){return function(){return t}};function Jn(t,e,n,r,i,o,a,s,u,l){this.target=t,this.type=e,this.subject=n,this.identifier=r,this.active=i,this.x=o,this.y=a,this.dx=s,this.dy=u,this._=l}function tr(){return!it.ctrlKey&&!it.button}function er(){return this.parentNode}function nr(t){return null==t?{x:it.x,y:it.y}:t}function rr(){return navigator.maxTouchPoints||"ontouchstart"in this}Jn.prototype.on=function(){var t=this._.on.apply(this._,arguments);return t===this._?this:t};var ir=function(){var t,e,n,r,i=tr,o=er,a=nr,s=rr,u={},l=St("start","drag","end"),c=0,h=0;function f(t){t.on("mousedown.drag",p).filter(s).on("touchstart.drag",v).on("touchmove.drag",y).on("touchend.drag touchcancel.drag",m).style("touch-action","none").style("-webkit-tap-highlight-color","rgba(0,0,0,0)")}function p(){if(!r&&i.apply(this,arguments)){var a=_("mouse",o.apply(this,arguments),Gt,this,arguments);a&&(mt(it.view).on("mousemove.drag",d,!0).on("mouseup.drag",g,!0),Mt(it.view),Pt(),n=!1,t=it.clientX,e=it.clientY,a("start"))}}function d(){if(At(),!n){var r=it.clientX-t,i=it.clientY-e;n=r*r+i*i>h}u.mouse("drag")}function g(){mt(it.view).on("mousemove.drag mouseup.drag",null),zt(it.view,n),At(),u.mouse("end")}function v(){if(i.apply(this,arguments)){var t,e,n=it.changedTouches,r=o.apply(this,arguments),a=n.length;for(t=0;t1e-6)if(Math.abs(c*s-u*l)>1e-6&&i){var f=n-o,p=r-a,d=s*s+u*u,g=f*f+p*p,v=Math.sqrt(d),y=Math.sqrt(h),m=i*Math.tan((or-Math.acos((d+h-g)/(2*v*y)))/2),_=m/y,w=m/v;Math.abs(_-1)>1e-6&&(this._+="L"+(t+_*l)+","+(e+_*c)),this._+="A"+i+","+i+",0,0,"+ +(c*f>l*p)+","+(this._x1=t+w*s)+","+(this._y1=e+w*u)}else this._+="L"+(this._x1=t)+","+(this._y1=e);else;},arc:function(t,e,n,r,i,o){t=+t,e=+e,o=!!o;var a=(n=+n)*Math.cos(r),s=n*Math.sin(r),u=t+a,l=e+s,c=1^o,h=o?r-i:i-r;if(n<0)throw new Error("negative radius: "+n);null===this._x1?this._+="M"+u+","+l:(Math.abs(this._x1-u)>1e-6||Math.abs(this._y1-l)>1e-6)&&(this._+="L"+u+","+l),n&&(h<0&&(h=h%ar+ar),h>sr?this._+="A"+n+","+n+",0,1,"+c+","+(t-a)+","+(e-s)+"A"+n+","+n+",0,1,"+c+","+(this._x1=u)+","+(this._y1=l):h>1e-6&&(this._+="A"+n+","+n+",0,"+ +(h>=or)+","+c+","+(this._x1=t+n*Math.cos(i))+","+(this._y1=e+n*Math.sin(i))))},rect:function(t,e,n,r){this._+="M"+(this._x0=this._x1=+t)+","+(this._y0=this._y1=+e)+"h"+ +n+"v"+ +r+"h"+-n+"Z"},toString:function(){return this._}};var cr=lr,hr=function(t){return function(){return t}};function fr(t){this._context=t}fr.prototype={areaStart:function(){this._line=0},areaEnd:function(){this._line=NaN},lineStart:function(){this._point=0},lineEnd:function(){(this._line||0!==this._line&&1===this._point)&&this._context.closePath(),this._line=1-this._line},point:function(t,e){switch(t=+t,e=+e,this._point){case 0:this._point=1,this._line?this._context.lineTo(t,e):this._context.moveTo(t,e);break;case 1:this._point=2;default:this._context.lineTo(t,e)}}};var pr=function(t){return new fr(t)};function dr(t){return t[0]}function gr(t){return t[1]}var vr=n(1),yr=n.n(vr),mr={entityStateReadOnly:{ACTIVE:!1,DELETED:!0,STATUS_ACTIVE:!1,STATUS_DELETED:!0}},_r={nodeArrowDistance:24,refreshGraphForSafari:function(t){t.edgeEl.each((function(t){var e=this,n=$(this).find("pattern");setTimeout((function(t){$(e).find("defs").append(n)}),500)}))},refreshGraphForIE:function(t){var e=t.edgePathEl,n=0;e.each((function(t){var e=$(this).find("marker");$(this).find("marker").remove();var r=this;++n,setTimeout((function(t){$(r).find("defs").append(e),0===--n&&(this.$(".fontLoader").hide(),this.$("svg").fadeTo(1e3,1))}),1e3)}))},dragNode:function(t){var e=this,n=t.g,r=t.svg,i=t.guid,o=(t.edgePathEl,{dragmove:function(t,e){var r=this,i=mt(t),o=n.node(e),a=o.x,s=o.y;o.x+=it.dx,o.y+=it.dy,i.attr("transform","translate("+o.x+","+o.y+")");var u=o.x-a,l=o.y-s;n.edges().forEach((function(t){if(t.v==e||t.w==e){var i=n.edge(t.v,t.w);r.translateEdge(i,u,l),mt(i.elem).select("path").attr("d",r.calcPoints(t))}}))},translateEdge:function(t,e,n){t.points.forEach((function(t){t.x=t.x+e,t.y=t.y+n}))},calcPoints:function(t){var e=n.edge(t.v,t.w),r=n.node(t.v),i=n.node(t.w),o=e.points.slice(1,e.points.length-1);e.points.slice(1,e.points.length-1);return o.unshift(this.intersectRect(r,o[0])),o.push(this.intersectRect(i,o[o.length-1])),function(){var t=dr,e=gr,n=hr(!0),r=null,i=pr,o=null;function a(a){var s,u,l,c=a.length,h=!1;for(null==r&&(o=i(l=cr())),s=0;s<=c;++s)!(sMath.abs(a)*c?(s<0&&(c=-c),h=0===s?0:c*a/s,f=c):(a<0&&(l=-l),h=l,f=0===a?0:l*s/a),{x:r+h,y:o+f}}}),a=ir().on("drag",(function(t){o.dragmove.call(o,this,t)})),s=ir().on("drag",(function(t){o.translateEdge(n.edge(t.v,t.w),it.dx,it.dy);var e=n.edge(t.v,t.w);mt(e.elem).select("path").attr("d",o.calcPoints(t))}));a(r.selectAll("g.node")),s(r.selectAll("g.edgePath"))},zoomIn:function(t){var e=t.svg,n=t.scaleFactor,r=void 0===n?1.3:n;this.d3Zoom.scaleBy(e.transition().duration(750),r)},zoomOut:function(t){var e=t.svg,n=t.scaleFactor,r=void 0===n?.8:n;this.d3Zoom.scaleBy(e.transition().duration(750),r)},zoom:function(t){var e=t.svg,n=t.xa,r=t.ya,i=t.scale;e.transition().duration(750).call(this.d3Zoom.transform,Un.translate(n,r).scale(i))},fitToScreen:function(t){var e=t.svg,n=e.node(),r=n.getBBox(),i=n.parentElement,o=i.clientWidth,a=i.clientHeight,s=r.width,u=r.height,l=r.x+s/2,c=r.y+u/2,h=(h||.95)/Math.max(s/o,u/a),f=o/2-h*l,p=a/2-h*c;this.zoom({svg:e,xa:f,ya:p,scale:h})},centerNode:function(t){var e=t.guid,n=t.g,r=t.svg,i=t.svgGroupEl,o=(t.edgePathEl,t.width),a=t.height,s=t.fitToScreen,u=t.onCenterZoomed,l=t.isSelected;this.d3Zoom=function(){var t,e,n=Vn,r=Xn,i=Kn,o=Zn,a=Wn,s=[0,1/0],u=[[-1/0,-1/0],[1/0,1/0]],l=250,c=It,h=St("start","zoom","end"),f=0;function p(t){t.property("__zoom",Yn).on("wheel.zoom",w).on("mousedown.zoom",b).on("dblclick.zoom",x).filter(a).on("touchstart.zoom",k).on("touchmove.zoom",E).on("touchend.zoom touchcancel.zoom",T).style("touch-action","none").style("-webkit-tap-highlight-color","rgba(0,0,0,0)")}function d(t,e){return(e=Math.max(s[0],Math.min(s[1],e)))===t.k?t:new Fn(e,t.x,t.y)}function g(t,e,n){var r=e[0]-n[0]*t.k,i=e[1]-n[1]*t.k;return r===t.x&&i===t.y?t:new Fn(t.k,r,i)}function v(t){return[(+t[0][0]+ +t[1][0])/2,(+t[0][1]+ +t[1][1])/2]}function y(t,e,n){t.on("start.zoom",(function(){m(this,arguments).start()})).on("interrupt.zoom end.zoom",(function(){m(this,arguments).end()})).tween("zoom",(function(){var t=this,i=arguments,o=m(t,i),a=r.apply(t,i),s=null==n?v(a):"function"==typeof n?n.apply(t,i):n,u=Math.max(a[1][0]-a[0][0],a[1][1]-a[0][1]),l=t.__zoom,h="function"==typeof e?e.apply(t,i):e,f=c(l.invert(s).concat(u/l.k),h.invert(s).concat(u/h.k));return function(t){if(1===t)t=h;else{var e=f(t),n=u/e[2];t=new Fn(n,s[0]-e[0]*n,s[1]-e[1]*n)}o.zoom(null,t)}}))}function m(t,e,n){return!n&&t.__zooming||new _(t,e)}function _(t,e){this.that=t,this.args=e,this.active=0,this.extent=r.apply(t,e),this.taps=0}function w(){if(n.apply(this,arguments)){var t=m(this,arguments),e=this.__zoom,r=Math.max(s[0],Math.min(s[1],e.k*Math.pow(2,o.apply(this,arguments)))),a=Gt(this);if(t.wheel)t.mouse[0][0]===a[0]&&t.mouse[0][1]===a[1]||(t.mouse[1]=e.invert(t.mouse[0]=a)),clearTimeout(t.wheel);else{if(e.k===r)return;t.mouse=[a,e.invert(a)],de(this),t.start()}$n(),t.wheel=setTimeout(l,150),t.zoom("mouse",i(g(d(e,r),t.mouse[0],t.mouse[1]),t.extent,u))}function l(){t.wheel=null,t.end()}}function b(){if(!e&&n.apply(this,arguments)){var t=m(this,arguments,!0),r=mt(it.view).on("mousemove.zoom",l,!0).on("mouseup.zoom",c,!0),o=Gt(this),a=it.clientX,s=it.clientY;Mt(it.view),Hn(),t.mouse=[o,this.__zoom.invert(o)],de(this),t.start()}function l(){if($n(),!t.moved){var e=it.clientX-a,n=it.clientY-s;t.moved=e*e+n*n>f}t.zoom("mouse",i(g(t.that.__zoom,t.mouse[0]=Gt(t.that),t.mouse[1]),t.extent,u))}function c(){r.on("mousemove.zoom mouseup.zoom",null),zt(it.view,t.moved),$n(),t.end()}}function x(){if(n.apply(this,arguments)){var t=this.__zoom,e=Gt(this),o=t.invert(e),a=t.k*(it.shiftKey?.5:2),s=i(g(d(t,a),e,o),r.apply(this,arguments),u);$n(),l>0?mt(this).transition().duration(l).call(y,s,e):mt(this).call(p.transform,s)}}function k(){if(n.apply(this,arguments)){var e,r,i,o,a=it.touches,s=a.length,u=m(this,arguments,it.changedTouches.length===s);for(Hn(),r=0;rg[id='"+e+"']"),h=(this.d3Zoom.scaleExtent([.01,50]).on("zoom",(function(){i.attr("transform",it.transform)})),null),f=null;if(c.empty()){if(s)return void this.fitToScreen({svg:r});h=n.graph().width/2,f=n.graph().height/2}else{var p=c.attr("transform").replace(/[^0-9\-.,]/g,"").split(",");h=p[0],f=p[1]}var d=-(1.2*h-o/2),g=-(1.2*f-a/2);this.zoom({svg:r,xa:d,ya:g,scale:1.2}),l?r.transition().duration(750).call(this.d3Zoom.transform,Un.translate(d,g).scale(1.2)):r.call(this.d3Zoom.transform,Un.translate(d,g).scale(1.2)),u&&u({newScale:1.2,newTranslate:[d,g],d3Zoom:this.d3Zoom,selectedNodeEl:c})},getToolTipDirection:function(t){var e=t.el,n=mt("body").node().getBoundingClientRect().width,r=mt(e).node().getBoundingClientRect(),i="e";return n-r.left<330?(i=n-r.left<330&&r.top<400?"sw":"w",n-r.left<330&&r.top>600&&(i="nw")):r.top>600?(i=n-r.left<330&&r.top>600?"nw":"n",r.left<50&&(i="ne")):r.top<400&&(i=r.left<50?"se":"s"),i},onHoverFade:function(t){var e=t.svg,n=t.g,r=t.mouseenter,i=t.nodesToHighlight,o=t.hoveredNode;return function(t){var i=e.selectAll(".node"),a=e.selectAll(".edgePath");if(r){e.classed("hover",!0);var s=n.successors(o),u=n.predecessors(o);t=s.concat(u);i.classed("hover-active-node",(function(e,n,r){return!!function(t,e,n){if(t===n||e&&e.length&&-1!=e.indexOf(n))return!0}(o,t,e)})),a.classed("hover-active-path",(function(t){return!!(t.v===o||t.w===o?1:0)}))}else e.classed("hover",!1),i.classed("hover-active-node",!1),a.classed("hover-active-path",!1)}(i)},getBaseUrl:function(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:window.location.pathname;return t.replace(/\/[\w-]+.(jsp|html)|\/+$/gi,"")},getEntityIconPath:function(t){var e=t.entityData,n=t.errorUrl,r=t.imgBasePath,i=this.getBaseUrl()+(r||"/img/entity-icon/");if(e){var o=function(t){return i+(mr.entityStateReadOnly[l]?"disabled/"+t:t)},a=function(){return c?mr.entityStateReadOnly[l]?i+"disabled/process.png":i+"process.png":mr.entityStateReadOnly[l]?i+"disabled/table.png":i+"table.png"},s=e.typeName,u=e.serviceType,l=e.status,c=e.isProcess;if(n){if(n.indexOf("table.png")>-1)return null;var h=!(!n||!n.match("entity-icon/"+s+".png|disabled/"+s+".png"));return u&&h?o(u+".png"):a()}return s?o(s+".png"):u?o(u+".png"):a()}},base64Encode:function(t,e){var n=new FileReader;n.addEventListener("load",(function(){return e(n.result)})),n.readAsDataURL(t)},imgShapeRender:function(t,e,n,r){var i=r.dagreD3,o=r.defsEl,a=r.imgBasePath,s=r.guid,u=r.isRankdirToBottom,l=this,c=s,h=n.btnType?"/img/entity-icon/expandBtn.svg":this.getEntityIconPath({entityData:n,imgBasePath:a}),f=h.split("/").pop();if(void 0===this.imageObject&&(this.imageObject={}),n.isDeleted&&(f="deleted_"+f),n.id==c)var p=!0;var d=t.append("circle").attr("fill","url(#img_"+f+")").attr("r",u?"30px":"24px").attr("data-stroke",n.id).attr("stroke-width","2px").attr("class","nodeImage "+(p?"currentNode":n.isProcess?"process":"node"));if(p&&d.attr("stroke","#fb4200"),!0===n.isIncomplete){t.attr("class","node isIncomplete show"),t.insert("rect").attr("x","-5").attr("y","-23").attr("width","14").attr("height","16").attr("fill","url(#img_hourglass.svg)").attr("data-stroke",n.id).attr("stroke-width","2px");g({imgName:"hourglass.svg",imageIconPath:"/img/entity-icon/hourglass.svg",leftPosition:"0",topPosition:"0",width:"12",height:"14"})}function g(t){o.select('pattern[id="img_'+t.imgName+'"]').empty()&&o.append("pattern").attr("x","0%").attr("y","0%").attr("patternUnits","objectBoundingBox").attr("id","img_"+t.imgName).attr("width","100%").attr("height","100%").append("image").attr("href",(function(e){var r=this;if(n){!function e(i){var o=i.imagePath,a={url:o,method:"GET",cache:!0};d.attr("data-iconpath",o);var s=new XMLHttpRequest;s.onreadystatechange=function(){if(4===s.readyState)if(200===s.status)"IE"!==yr.a.name?l.base64Encode(this.response,(function(e){l.imageObject[t.imageIconPath]=e,mt(r).attr("xlink:href",e)})):l.imageObject[t.imageIconPath]=o,t.imageIconPath!==d.attr("data-iconpath")&&d.attr("data-iconpathorigin",t.imageIconPath);else if(404===s.status){var i=l.getEntityIconPath({entityData:n,errorUrl:o});if(null===i){var a=mt(r.parentElement);a.select("image").remove(),a.attr("patternContentUnits","objectBoundingBox").append("circle").attr("r","24px").attr("fill","#e8e8e8")}else e({imagePath:i})}},s.responseType="blob",s.open(a.method,a.url,!0),s.send(null)}({imagePath:t.imageIconPath})}})).attr("x",t.leftPosition).attr("y",t.topPosition).attr("width",t.width).attr("height",t.height)}return g({imgName:f,imageIconPath:h,leftPosition:u?"11":"4",topPosition:u?"20":p?"3":"4",width:"40",height:"40"}),n.intersect=function(t){return i.intersect.circle(n,p?l.nodeArrowDistance+3:l.nodeArrowDistance,t)},d},arrowPointRender:function(t,e,n,r,i){var o=i.dagreD3,a=t.node(),s=a?a.parentNode:t;mt(s).select("path.path").attr("marker-end","url(#"+e+")");var u=t.append("marker").attr("id",e).attr("viewBox","0 0 10 10").attr("refX",8).attr("refY",5).attr("markerUnits","strokeWidth").attr("markerWidth",4).attr("markerHeight",4).attr("orient","auto").append("path").attr("d","M 0 0 L 10 5 L 0 10 z").style("fill",n.styleObj.stroke);o.util.applyStyle(u,n[r+"Style"])},saveSvg:function(t){var e=t.svg,n=t.width,r=t.height,i=t.downloadFileName,o=t.onExportLineage,a=this,s=e.clone(!0).node();setTimeout((function(){"Firefox"===yr.a.name&&(s.setAttribute("width",n),s.setAttribute("height",r));var t=mt("body").append("div");t.classed("hidden-svg",!0),t.node().appendChild(s);var e=mt(".hidden-svg svg");e.select("g").attr("transform","scale(1)"),e.select("foreignObject").remove();var u=150,l=150,c=s.getBBox().width+u,h=s.getBBox().height+l,f=s.getBBox().x,p=s.getBBox().y;s.attributes.viewBox.value=f+","+p+","+c+","+h;var d=document.createElement("canvas");d.id="canvas",d.style.display="none",d.width=1*s.getBBox().width+u,d.height=1*s.getBBox().height+l,mt("body").node().appendChild(d);var g=d.getContext("2d"),v=(new XMLSerializer).serializeToString(s),y=window.URL||window.webkitURL||window;g.fillStyle="#FFFFFF",g.fillRect(0,0,d.width,d.height),g.strokeRect(0,0,d.width,d.height),g.restore();var m=new Image(d.width,d.height),_=new Blob([v],{type:"image/svg+xml;base64"});"Safari"===yr.a.name&&(_=new Blob([v],{type:"image/svg+xml"})),g.drawImage(m,50,50,d.width,d.height);var w=y.createObjectURL(_);m.onload=function(){try{var e=document.createElement("a");e.download=i,document.body.appendChild(e),g.drawImage(m,50,50,d.width,d.height),d.toBlob((function(t){t?(e.href=y.createObjectURL(t),t.size>1e7&&o({status:"failed",message:"The Image size is huge, please open the image in a browser!"}),e.click(),o({status:"Success",message:"Successful"}),"Safari"===yr.a.name&&a.refreshGraphForSafari({edgeEl:a.$("svg g.node")})):o({status:"failed",message:"There was an error in downloading Lineage!"})}),"image/png"),t.remove(),d.remove()}catch(t){o({status:"failed",message:"There was an error in downloading Lineage!"})}},m.src=w}),0)}};function wr(t,e){var n=Object.keys(t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(t);e&&(r=r.filter((function(e){return Object.getOwnPropertyDescriptor(t,e).enumerable}))),n.push.apply(n,r)}return n}function br(t){for(var e=1;e0&&void 0!==arguments[0]?arguments[0]:{},i=r.entityData,o=r.errorUrl,a=this.getBaseUrl(window.location.pathname)+Globals.entityImgPath;function s(t){return a+(mr.entityStateReadOnly[e]?"disabled/"+t:t)}function u(){return i.isProcess?mr.entityStateReadOnly[e]?a+"disabled/process.png":a+"process.png":mr.entityStateReadOnly[e]?a+"disabled/table.png":a+"table.png"}if(i&&(n=i.typeName,t=i&&i.serviceType,e=i&&i.status),i){if(o){var l=!(!o||!o.match("entity-icon/"+n+".png|disabled/"+n+".png"));return t&&l?s(t+".png"):u()}return i.typeName?s(i.typeName+".png"):u()}},isProcess:function(t){var e=t.typeName,n=t.superTypes;t.entityDef;return"Process"==e||n.indexOf("Process")>-1},isDeleted:function(t){if(void 0!==t)return mr.entityStateReadOnly[t.status]},isNodeToBeUpdated:function(t,e){var n=e.isProcessHideCheck,r=e.isDeletedEntityHideCheck,i={isProcess:n&&t.isProcess,isDeleted:r&&t.isDeleted};return i.update=i.isProcess||i.isDeleted,"Expand"===t.label&&(i.update=!0),i},getServiceType:function(t){var e=t.typeName,n=t.entityDef,r=null;return e&&n&&(r=n.serviceType||null),r},getEntityDef:function(t){var e=t.typeName,n=t.entityDefCollection,r=null;return e&&(r=n.find((function(t){return t.name==e}))),r},getNestedSuperTypes:function(t){var e=t.entityDef,n=t.entityDefCollection,r=new Set;return function t(e,n){e&&e.superTypes&&e.superTypes.length&&e.superTypes.forEach((function(e){r.add(e);var i=n.find((function(t){t.name}));i&&t(i,n)}))}(e,n),Array.from(r)},generateData:function(t){var e=this,n=t.data,r=void 0===n?{}:n,i=t.filterObj,o=t.entityDefCollection,a=t.g,s=t.guid,u=t.setGraphEdge,l=t.setGraphNode;return new Promise((function(t,n){try{var c=r.relations||{},h=r.guidEntityMap||{},f=i.isProcessHideCheck||i.isDeletedEntityHideCheck,p={fill:"none",stroke:"#ffb203",width:3},d=function(t){if(t){if(t.updatedValues)return t;var n=t.displayText?t.displayText:" ",r=Object.assign(t,{shape:"img",updatedValues:!0,label:n.trunc(18),toolTipLabel:n,id:t.guid,isLineage:!0,isIncomplete:t.isIncomplete,entityDef:e.getEntityDef({typeName:t.typeName,entityDefCollection:o})});return r.serviceType=e.getServiceType(r),r.superTypes=e.getNestedSuperTypes(br(br({},r),{},{entityDefCollection:o})),r.isProcess=e.isProcess(r),r.isDeleted=e.isDeleted(r),r}},g=function(t){return"fill:"+t.fill+";stroke:"+t.stroke+";stroke-width:"+t.width},v=function(t,n,r){var i=[];return t.forEach((function(t){if(e.isNodeToBeUpdated(d(h[t]),r).update)if(w[t])i=i.concat(w[t]);else{var n=function t(n,r){if(n&&b[n]){var i=[];return b[n].forEach((function(n){if(e.isNodeToBeUpdated(d(h[n]),r).update){var o=t(n,r);o&&(i=i.concat(o))}else i.push(n)})),i}return null}(t,r);n&&(i=i.concat(n))}else i.push(t)})),i},y=function(t){if(a._nodes[t])return a._nodes[t];var e=d(h[t]);return l(t,e),e},m=function(t,e){var n=arguments.length>2&&void 0!==arguments[2]?arguments[2]:{};u(t,e,br({arrowhead:"arrowPoint",curve:bt,style:g(p),styleObj:p},n))},_=function(t,e){y(t),y(e),m(t,e)},w={};if(f){var b=function(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{},e=t.relations,n={};return e.forEach((function(t){n[t.fromEntityId]?n[t.fromEntityId].push(t.toEntityId):n[t.fromEntityId]=[t.toEntityId]})),n}(r);Object.keys(b).forEach((function(t){var n=b[t],r=e.isNodeToBeUpdated(d(h[t]),i),o=v(n,0,i);r.update?w[t]?w[t]=w[t].concat(o):w[t]=o:o.forEach((function(e){_(t,e)}))}))}else c.length?c.forEach((function(t){_(t.fromEntityId,t.toEntityId)})):y(s);a._nodes[s]&&(a._nodes[s]&&(a._nodes[s].isLineage=!1),e.findImpactNodeAndUpdateData({guid:s,g:a,setEdge:m,getStyleObjStr:g})),a._nodes[s]||y(s),t(a)}catch(t){n(t)}}))},findImpactNodeAndUpdateData:function(t){var e=t.guid,n=t.getStyleObjStr,r=t.g,i=t.setEdge,o={},a={fill:"none",stroke:"#fb4200",width:3};!function t(){var e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{},s=arguments.length>1?arguments[1]:void 0,u=Object.keys(e);u.length&&(o[s]||(o[s]=!0,u.forEach((function(e){r._nodes[e]&&(r._nodes[e].isLineage=!1),i(s,e,{style:n(a),styleObj:a}),t(r._sucs[e],e)}))))}(r._sucs[e],e)}};String.prototype.trunc=String.prototype.trunc||function(t){return this.length>t?this.substr(0,t-1)+"...":this};function Er(){}function Tr(t,e){var n=new Er;if(t instanceof Er)t.each((function(t,e){n.set(e,t)}));else if(Array.isArray(t)){var r,i=-1,o=t.length;if(null==e)for(;++i0&&void 0!==arguments[0]?arguments[0]:{};return n._createGraph(n.options,n.graphOptions,t)},clear:function(t){return n.clear(t)},refresh:function(t){return n.refresh(t)},centerAlign:function(t){return n.centerAlign(t)},exportLineage:function(t){return n.exportLineage(t)},zoomIn:function(t){return n.zoomIn(t)},zoomOut:function(t){return n.zoomOut(t)},zoom:function(t){return n.zoom(t)},fullScreen:function(t){return n.fullScreen(t)},searchNode:function(t){return n.searchNode(t)},displayFullName:function(t){return n.displayFullName(t)},removeNodeSelection:function(t){return n.removeNodeSelection(t)},getGraphOptions:function(){return n.graphOptions},getNode:function(t,e){var r=null;return(r=e?n.actualData[t]:n.g._nodes[t])&&(r=Object.assign({},r)),r},getNodes:function(t,e){var r=null;return(r=e?n.actualData:n.g._nodes)&&(r=Object.assign({},r)),r},setNode:this._setGraphNode,setEdge:this._setGraphEdge},!1===a&&this.init(),this.initReturnObj}var e,n,r;return e=t,(n=[{key:"_updateAllOptions",value:function(t){Object.assign(this.options,t);var e=this.svg.node().getBoundingClientRect();this.graphOptions.width=this.options.width||e.width,this.graphOptions.height=this.options.height||e.height;var n=this.graphOptions,r=n.svg,i=n.width,o=n.height,a=n.guid,s=this.options.fitToScreen;r.select("g").node().removeAttribute("transform"),r.attr("viewBox","0 0 "+i+" "+o).attr("enable-background","new 0 0 "+i+" "+o),this.centerAlign({fitToScreen:s,guid:a})}},{key:"_updateOptions",value:function(t){Object.assign(this.options,{filterObj:{isProcessHideCheck:!1,isDeletedEntityHideCheck:!1}},t)}},{key:"init",value:function(){var t=this.options.data,e=void 0===t?{}:t;e.baseEntityGuid&&(this.guid=e.baseEntityGuid),this._initializeGraph(),this._initGraph()}},{key:"clear",value:function(){this.options.el||(this.svg.remove(),this.svg=null),this.g=null,this.graphOptions={}}},{key:"centerAlign",value:function(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{},e=this.svg.select("g"),n=e.selectAll("g.edgePath");_r.centerNode(zr(zr({},this.graphOptions),{},{svgGroupEl:e,edgePathEl:n},t))}},{key:"zoomIn",value:function(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{};_r.zoomIn(zr(zr({},this.graphOptions),t))}},{key:"zoomOut",value:function(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{};_r.zoomOut(zr(zr({},this.graphOptions),t))}},{key:"zoom",value:function(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{};_r.zoom(zr(zr({},this.graphOptions),t))}},{key:"displayFullName",value:function(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{},e=this;this.g.nodes().forEach((function(n){var r=e.svg.selectAll("g.nodes>g[id='"+n+"']"),i=e.g.node(n).toolTipLabel;1==t.bLabelFullText?r.select("tspan").text(i):r.select("tspan").text(i.trunc(18))})),this.selectedNode&&this.searchNode({guid:this.selectedNode})}},{key:"refresh",value:function(t){if(this.clear(),this._initializeGraph(),this._initGraph({refresh:!0}),this.selectedNode="",t&&t.compactLineageEnabled&&t.filterObj){var e=t.filterObj.isProcessHideCheck,n=t.filterObj.isDeletedEntityHideCheck;this._AddFilterNotification(e,n)}}},{key:"removeNodeSelection",value:function(){this.svg.selectAll("g.node>circle").classed("node-detail-highlight",!1)}},{key:"searchNode",value:function(t){var e=t.guid,n=t.onSearchNode;this.svg.selectAll(".serach-rect").remove(),this.svg.selectAll(".label").attr("stroke","none"),this.selectedNode=e,this.centerAlign({guid:e,onCenterZoomed:function(t){var e=t.selectedNodeEl,r=e.node().getBBox(),i=r.width+10,o=r.x-5;e.select(".label").attr("stroke","#316132"),e.select("circle").classed("wobble",!0),e.insert("rect","circle").attr("class","serach-rect").attr("stroke","#37bb9b").attr("stroke-width","2.5px").attr("fill","none").attr("x",o).attr("y",-27.5).attr("width",i).attr("height",60),n&&"function"==typeof n&&n(t)},isSelected:!0})}},{key:"exportLineage",value:function(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{},e=t.downloadFileName;if(void 0===e){var n=this.g._nodes[this.guid];e=n&&n.attributes?"".concat(n.attributes.qualifiedName||n.attributes.name||"lineage_export",".png"):"export.png"}_r.saveSvg(zr(zr({},this.graphOptions),{},{downloadFileName:e,onExportLineage:function(t){function e(e){return t.apply(this,arguments)}return e.toString=function(){return t.toString()},e}((function(e){t.onExportLineage&&onExportLineage(e)}))}))}},{key:"fullScreen",value:function(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{},e=t.el;if(void 0===e)throw new Error("LineageHelper requires el propety to apply fullScreen class");var n=mt(e);return n.classed("fullscreen-mode")?(n.classed("fullscreen-mode",!1),!1):(n.classed("fullscreen-mode",!0),!0)}},{key:"_getValueFromUser",value:function(t){if(void 0!==t)return"function"==typeof t?t():t}},{key:"_initializeGraph",value:function(){var t=this.options,e=t.width,n=void 0===e?"100%":e,r=t.height,o=void 0===r?"100%":r,a=t.el;this.svg=mt(a),a instanceof SVGElement||(this.svg.selectAll("*").remove(),this.svg=this.svg.append("svg").attr("xmlns","http://www.w3.org/2000/svg").attr(" xmlns:xlink","http://www.w3.org/1999/xlink").attr("version","1.1").attr("width",n).attr("height",o)),this.g=(new i.a.graphlib.Graph).setGraph(Object.assign({nodesep:50,ranksep:90,rankdir:"LR",marginx:20,marginy:20,transition:function(t){return t.transition().duration(500)}},this.options.dagreOptions)).setDefaultEdgeLabel((function(){return{}}));var s=this.svg.node().getBoundingClientRect();this.actualData={},this.graphOptions={svg:this.svg,g:this.g,dagreD3:i.a,guid:this.guid,width:this.options.width||s.width,height:this.options.height||s.height}}},{key:"_initGraph",value:function(){var t=this,e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{},n=e.refresh;this.svg&&this.svg.select("g").remove();var r=this.options.filterObj;if(this.options.getFilterObj){var i=this.options.getFilterObj();if(void 0!==i||null!==i){if("object"!==Ar(i))throw new Error("getFilterObj expect return type `object`,`null` or `Undefined`");r=i}}if(!0!==this.options.setDataManually)return void 0===this.options.data||this.options.data&&0===this.options.data.relations.length&&_.isEmpty(this.options.data.guidEntityMap)?(this.options.beforeRender&&this.options.beforeRender(),this.svg.append("text").attr("x","50%").attr("y","50%").attr("alignment-baseline","middle").attr("text-anchor","middle").text("No lineage data found"),void(this.options.afterRender&&this.options.afterRender())):kr.generateData(zr(zr(zr({},this.options),{},{filterObj:r},this.graphOptions),{},{setGraphNode:this._setGraphNode,setGraphEdge:this._setGraphEdge})).then((function(e){t._createGraph(t.options,t.graphOptions,{refresh:n})}))}},{key:"_createGraph",value:function(t,e,n){var r=this,o=t.data,a=void 0===o?{}:o,s=t.imgBasePath,u=t.isShowTooltip,l=t.isShowHoverPath,c=t.onLabelClick,h=t.onPathClick,f=t.onNodeClick,p=t.zoom,d=t.fitToScreen,g=t.getToolTipContent,v=t.toolTipTitle,y=n.refresh;this.options.beforeRender&&this.options.beforeRender(),this.selectedNode="";var m=this,_=e.svg,w=e.g,b=e.width,x=e.height,k=this.options.dagreOptions&&"tb"===this.options.dagreOptions.rankdir;if(_ instanceof yt==0)throw new Error("svg is not initialized or something went wrong while creatig graph instance");if(void 0!==w._nodes&&0!==w._nodes.length){w.nodes().forEach((function(t){var e=w.node(t);e&&(e.rx=e.ry=5)})),_.attr("viewBox","0 0 "+b+" "+x).attr("enable-background","new 0 0 "+b+" "+x);var E=_.append("g"),T=_.append("defs"),N=new i.a.render;N.arrows().arrowPoint=function(){return _r.arrowPointRender.apply(_r,Array.prototype.slice.call(arguments).concat([zr({},e)]))},N.shapes().img=function(){return _r.imgShapeRender.apply(_r,Array.prototype.slice.call(arguments).concat([zr(zr({},e),{},{isRankdirToBottom:k,imgBasePath:m._getValueFromUser(s),defsEl:T})]))};var O=function(){var t=function(){return"n"},e=function(){return[0,0]},n=function(){return" "},r=document.body,i=h(),o=null,a=null,s=null;function u(t){(o=function(t){var e=t.node();return e?"svg"===e.tagName.toLowerCase()?e:e.ownerSVGElement:null}(t))&&(a=o.createSVGPoint(),r.appendChild(i))}u.show=function(){var i=Array.prototype.slice.call(arguments);i[i.length-1]instanceof SVGElement&&(s=i.pop());var o,a=n.apply(this,i),h=e.apply(this,i),p=t.apply(this,i),d=f(),g=c.length,v=document.documentElement.scrollTop||r.scrollTop,y=document.documentElement.scrollLeft||r.scrollLeft;for(d.html(a).style("opacity",1).style("pointer-events","all");g--;)d.classed(c[g],!1);return o=l.get(p).apply(this),d.classed(p,!0).style("top",o.top+h[0]+v+"px").style("left",o.left+h[1]+y+"px"),u},u.hide=function(){return f().style("opacity",0).style("pointer-events","none"),u},u.attr=function(t,e){if(arguments.length<2&&"string"==typeof t)return f().attr(t);var n=Array.prototype.slice.call(arguments);return yt.prototype.attr.apply(f(),n),u},u.style=function(t,e){if(arguments.length<2&&"string"==typeof t)return f().style(t);var n=Array.prototype.slice.call(arguments);return yt.prototype.style.apply(f(),n),u},u.direction=function(e){return arguments.length?(t=null==e?e:d(e),u):t},u.offset=function(t){return arguments.length?(e=null==t?t:d(t),u):e},u.html=function(t){return arguments.length?(n=null==t?t:d(t),u):n},u.rootElement=function(t){return arguments.length?(r=null==t?t:d(t),u):r},u.destroy=function(){return i&&(f().remove(),i=null),u};var l=Nr({n:function(){var t=p(this);return{top:t.n.y-i.offsetHeight,left:t.n.x-i.offsetWidth/2}},s:function(){var t=p(this);return{top:t.s.y,left:t.s.x-i.offsetWidth/2}},e:function(){var t=p(this);return{top:t.e.y-i.offsetHeight/2,left:t.e.x}},w:function(){var t=p(this);return{top:t.w.y-i.offsetHeight/2,left:t.w.x-i.offsetWidth}},nw:function(){var t=p(this);return{top:t.nw.y-i.offsetHeight,left:t.nw.x-i.offsetWidth}},ne:function(){var t=p(this);return{top:t.ne.y-i.offsetHeight,left:t.ne.x}},sw:function(){var t=p(this);return{top:t.sw.y,left:t.sw.x-i.offsetWidth}},se:function(){var t=p(this);return{top:t.se.y,left:t.se.x}}}),c=l.keys();function h(){var t=mt(document.createElement("div"));return t.style("position","absolute").style("top",0).style("opacity",0).style("pointer-events","none").style("box-sizing","border-box"),t.node()}function f(){return null==i&&(i=h(),r.appendChild(i)),mt(i)}function p(t){for(var e=s||t;null==e.getScreenCTM&&null!=e.parentNode;)e=e.parentNode;var n={},r=e.getScreenCTM(),i=e.getBBox(),o=i.width,u=i.height,l=i.x,c=i.y;return a.x=l,a.y=c,n.nw=a.matrixTransform(r),a.x+=o,n.ne=a.matrixTransform(r),a.y+=u,n.se=a.matrixTransform(r),a.x-=o,n.sw=a.matrixTransform(r),a.y-=u/2,n.w=a.matrixTransform(r),a.x+=o,n.e=a.matrixTransform(r),a.x-=o/2,a.y-=u/2,n.n=a.matrixTransform(r),a.y+=u,n.s=a.matrixTransform(r),n}function d(t){return"function"==typeof t?t:function(){return t}}return u}().attr("class","d3-tip").offset([10,0]).html((function(t){if(g&&"function"==typeof g)return g(t,w.node(t));var e=w.node(t),n="";return v?n="
"+v+"
":e.id!==r.guid&&(n="
"+(e.isLineage?"Lineage":"Impact")+"
"),n+="
"+e.toolTipLabel+"
",e.typeName&&(n+="
("+e.typeName+")
"),e.queryText&&(n+="
Query: "+e.queryText+"
"),"
"+n+"
"}));_.call(O),N(E,w),E.selectAll("g.nodes g.label").attr("transform",(function(){return k?"translate(2,-20)":"translate(2,-38)"})).attr("font-size","10px").on("mouseenter",(function(t){it.preventDefault(),mt(this).classed("highlight",!0)})).on("mouseleave",(function(t){it.preventDefault(),mt(this).classed("highlight",!1)})).on("click",(function(t){it.preventDefault(),c&&"function"==typeof c&&c({clickedData:t}),O.hide(t)})),E.selectAll("g.nodes g.node circle").on("mouseenter",(function(t,n,r){if(m.activeNode=!0,this.getScreenCTM().translate(+this.getAttribute("cx"),+this.getAttribute("cy")),m.svg.selectAll(".node").classed("active",!1),mt(this).classed("active",!0),m._getValueFromUser(u)&&0!==t.indexOf("more")){var i=_r.getToolTipDirection({el:this});O.direction(i).show(t,this)}!1!==m._getValueFromUser(l)&&_r.onHoverFade(zr({opacity:.3,mouseenter:!0,hoveredNode:t},e))})).on("mouseleave",(function(t){m.activeNode=!1;var n=this;setTimeout((function(e){m.activeTip||m.activeNode||(mt(n).classed("active",!1),m._getValueFromUser(u)&&O.hide(t))}),150),!1!==m._getValueFromUser(l)&&_r.onHoverFade(zr({mouseenter:!1,hoveredNode:t},e))})).on("click",(function(t){it.defaultPrevented||(it.preventDefault(),O.hide(t),_.selectAll("g.node>circle").classed("node-detail-highlight",!1),mt(this).classed("node-detail-highlight",!0),f&&"function"==typeof f&&f({clickedData:t,el:this}))}));var S=E.selectAll("g.edgePath");S.selectAll("path.path").on("click",(function(t){if(h&&"function"==typeof h){var e=a.relations.find((function(e){if(e.fromEntityId===t.v&&e.toEntityId===t.w)return!0}));h({pathRelationObj:e,clickedData:t})}})),!1!==p&&_r.centerNode(zr(zr({},e),{},{fitToScreen:d,svgGroupEl:E,edgePathEl:S})),_r.dragNode(zr(zr({},e),{},{edgePathEl:S})),!0!==y&&this._addLegend(),this.options.afterRender&&this.options.afterRender()}else _.html('No relations to display')}},{key:"_addLegend",value:function(){if(!1!==this.options.legends){var t=mt(this.options.legendsEl||this.options.el).insert("div",":first-child").classed("legends",!0),e=t.append("span").style("color","#fb4200");e.append("i").classed("fa fa-circle-o fa-fw",!0),e.append("span").html("Current Entity"),(e=t.append("span").style("color","#686868")).append("i").classed("fa fa-hourglass-half fa-fw",!0),e.append("span").html("In Progress"),(e=t.append("span").style("color","#df9b00")).append("i").classed("fa fa-long-arrow-right fa-fw",!0),e.append("span").html("Lineage"),(e=t.append("span").style("color","#fb4200")).append("i").classed("fa fa-long-arrow-right fa-fw",!0),e.append("span").html("Impact"),(e=t.append("span").classed("notification hide",!0).style("color","#686868")).append("i").classed("fa fa-exclamation fa-fw",!0),e.append("span").html("Filtering hides all Expand buttons.")}}},{key:"_AddFilterNotification",value:function(t,e){t||e?$(this.options.legendsEl).find(".notification").removeClass("hide"):$(this.options.legendsEl).find(".notification").addClass("hide")}}])&&Dr(e.prototype,n),r&&Dr(e,r),t}()}])})); \ No newline at end of file diff --git a/dashboardv2/public/js/external_lib/atlas-lineage/dist/styles.css b/dashboardv2/public/js/external_lib/atlas-lineage/dist/styles.css index 2d7f9d9de1a..3bcce59e583 100644 --- a/dashboardv2/public/js/external_lib/atlas-lineage/dist/styles.css +++ b/dashboardv2/public/js/external_lib/atlas-lineage/dist/styles.css @@ -1,2 +1 @@ -.node{cursor:pointer}.node text{font-size:10px;font-family:sans-serif}.node .label{fill:#868686}.node .label.highlight{cursor:pointer;fill:#4a90e2;text-decoration:underline}.node .label.highlight tspan{font-weight:400}.node circle{-moz-transition:all 0.3s;-webkit-transition:all 0.3s;transition:all 0.3s;stroke-width:1.5px}.node circle.node-detail-highlight{stroke:#4a90e2;stroke-width:2px}.node circle.nodeImage.green:hover{stroke:#ffb203}.node circle.nodeImage.blue:hover{stroke:#4b91e2}.node circle.nodeImage.currentNode{stroke:#fb4200}.node circle.nodeImage:hover{-moz-transform:scale(1.4);-webkit-transform:scale(1.4);transform:scale(1.4)}.node.active circle{-moz-transform:scale(1.4);-webkit-transform:scale(1.4);transform:scale(1.4)}.node.active circle.nodeImage.green{stroke:#ffb203}.node.active circle.nodeImage.blue{stroke:#4b91e2}.legends>span{margin-right:8px;font-family:Source Sans Pro}svg.hover g.node{opacity:0.1 !important}svg.hover g.edgePath{opacity:0 !important}svg.hover g.node.hover-active-node,svg.hover g.edgePath.hover-active-path{opacity:1 !important}.invisible .node circle{transition:all 0s}.edgePath .path{cursor:pointer}.link{fill:none;stroke:#ccc;stroke-width:1.5px}.text-center{text-align:center}.d3-tip{line-height:1;font-weight:bold;padding:12px;background:rgba(0,0,0,0.8);color:#fff;z-index:999;max-width:300px;border-radius:2px}.d3-tip .tip-inner-scroll{overflow:auto;max-height:300px}.d3-tip .tip-inner-scroll h5{margin:7px 0px}.d3-tip:after{box-sizing:border-box;display:inline;font-size:10px;width:100%;line-height:1;color:rgba(0,0,0,0.8);position:absolute}.d3-tip.n:after{content:"\25BC";margin:-1px 0 0 0;top:100%;left:0;text-align:center}.d3-tip.e:after{content:"\25C0";margin:-4px 0 0 0;top:50%;left:-8px}.d3-tip.s:after{content:"\25B2";margin:0 0 1px 0;top:-8px;left:0;text-align:center}.d3-tip.w:after{content:"\25B6";margin:-4px 0 0 -1px;top:50%;left:100%}g.type-TK>rect{fill:#00ffd0}.fullscreen-mode{position:fixed;height:100% !important;top:0;bottom:0;left:0;width:100%;right:0;padding:0 !important;z-index:9999;overflow:hidden !important}.fullscreen-mode .resizeGraph{position:fixed;height:100% !important}.fullscreen-mode .resizeGraph .ui-resizable-handle{display:none}.fullscreen-mode .lineage-box{padding:10px !important}.fullscreen-mode .box-panel{margin:10px !important}@keyframes zoominoutsinglefeatured{0%{transform:scale(1, 1)}50%{transform:scale(1.2, 1.2)}100%{transform:scale(1, 1)}}.wobble{animation:zoominoutsinglefeatured 1s 5}.hidden-svg{visibility:hidden}@-webkit-keyframes blink{from{opacity:0.2}to{opacity:0.5}} - +.node{cursor:pointer}.node text{font-size:10px;font-family:sans-serif}.node .label{fill:#868686}.node .label.highlight{cursor:pointer;fill:#4a90e2;text-decoration:underline}.node .label.highlight tspan{font-weight:400}.node circle{-moz-transition:all 0.3s;-webkit-transition:all 0.3s;transition:all 0.3s;stroke-width:1.5px}.node circle.node-detail-highlight{stroke:#4a90e2;stroke-width:2px}.node circle.nodeImage.green:hover{stroke:#ffb203}.node circle.nodeImage.blue:hover{stroke:#4b91e2}.node circle.nodeImage.currentNode{stroke:#fb4200}.node circle.nodeImage:hover{-moz-transform:scale(1.4);-webkit-transform:scale(1.4);transform:scale(1.4)}.node.active circle{-moz-transform:scale(1.4);-webkit-transform:scale(1.4);transform:scale(1.4)}.node.active circle.nodeImage.green{stroke:#ffb203}.node.active circle.nodeImage.blue{stroke:#4b91e2}.legends>span{margin-right:8px;font-family:Source Sans Pro}svg.hover g.node{opacity:0.1 !important}svg.hover g.edgePath{opacity:0 !important}svg.hover g.node.hover-active-node,svg.hover g.edgePath.hover-active-path{opacity:1 !important}.invisible .node circle{transition:all 0s}.edgePath .path{cursor:pointer}.link{fill:none;stroke:#ccc;stroke-width:1.5px}.text-center{text-align:center}.d3-tip{line-height:1;font-weight:bold;padding:12px;background:rgba(0,0,0,0.8);color:#fff;z-index:999;max-width:300px;border-radius:2px;word-break:break-all}.d3-tip .tip-inner-scroll{overflow:auto;max-height:300px}.d3-tip .tip-inner-scroll h5{margin:7px 0px}.d3-tip:after{box-sizing:border-box;display:inline;font-size:10px;width:100%;line-height:1;color:rgba(0,0,0,0.8);position:absolute}.d3-tip.n:after{content:"\25BC";margin:-1px 0 0 0;top:100%;left:0;text-align:center}.d3-tip.e:after{content:"\25C0";margin:-4px 0 0 0;top:50%;left:-8px}.d3-tip.s:after{content:"\25B2";margin:0 0 1px 0;top:-8px;left:0;text-align:center}.d3-tip.w:after{content:"\25B6";margin:-4px 0 0 -1px;top:50%;left:100%}g.type-TK>rect{fill:#00ffd0}.fullscreen-mode{position:fixed;height:100% !important;top:0;bottom:0;left:0;width:100%;right:0;padding:0 !important;z-index:9999;overflow:hidden !important}.fullscreen-mode .resizeGraph{position:fixed;height:100% !important}.fullscreen-mode .resizeGraph .ui-resizable-handle{display:none}.fullscreen-mode .lineage-box{padding:10px !important}.fullscreen-mode .box-panel{margin:10px !important}@keyframes zoominoutsinglefeatured{0%{transform:scale(1, 1)}50%{transform:scale(1.2, 1.2)}100%{transform:scale(1, 1)}}.wobble{animation:zoominoutsinglefeatured 1s 5}.hidden-svg{visibility:hidden}@-webkit-keyframes blink{from{opacity:0.2}to{opacity:0.5}} diff --git a/dashboardv2/public/js/external_lib/atlas-lineage/package-lock.json b/dashboardv2/public/js/external_lib/atlas-lineage/package-lock.json index 69f5a400691..6b9aca093e3 100644 --- a/dashboardv2/public/js/external_lib/atlas-lineage/package-lock.json +++ b/dashboardv2/public/js/external_lib/atlas-lineage/package-lock.json @@ -2333,10 +2333,10 @@ "integrity": "sha512-ii0/r5f4sjKNTfh84Di+DpztYwqKhEyUlKoPrzUFfeSkWxjW49xU2QzO9qrPrNkpdI0XJkfzvmTu8V2Zylln6A==" }, "d3-color": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/d3-color/-/d3-color-1.4.1.tgz", - "integrity": "sha512-p2sTHSLCJI2QKunbGb7ocOh7DgTAn8IrLx21QRc/BSnodXM4sv6aLQlnfpvehFMLZEfBc6g9pH9SWQccFYfJ9Q==" - }, + "version" : "3.1.0", + "resolved" :"https://registry.yarnpkg.com/d3-color/-/d3-color-3.1.0.tgz#395b2833dfac71507f12ac2f7af23bf819de24e2", + "integrity" : "sha512-zg/chbXyeBtMQ1LbD/WSoW2DpC3I0mpmPdW+ynRTj/x2DAWYrIY7qeZIHidozwV24m4iavr15lNwIwLxRmOxhA==" + }, "d3-contour": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/d3-contour/-/d3-contour-1.3.2.tgz", @@ -2567,9 +2567,9 @@ "dev": true }, "decode-uri-component": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.2.0.tgz", - "integrity": "sha1-6zkTMzRYd1y4TNGh+uBiEGu4dUU=", + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.2.2.tgz", + "integrity": "sha512-FqUYQ+8o158GyGTrMFJms9qh3CqTKvAqgqsTnkLI8sKu0028orqBhxNMFkFen0zGyg6epACD32pjVk58ngIErQ==", "dev": true }, "define-properties": { @@ -3641,9 +3641,9 @@ "dev": true }, "loader-utils": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/loader-utils/-/loader-utils-1.4.0.tgz", - "integrity": "sha512-qH0WSMBtn/oHuwjy/NucEgbx5dbxxnxup9s4PVXJUDHZBQY+s0NWA9rJf53RBnQZxfch7euUui7hpoAPvALZdA==", + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/loader-utils/-/loader-utils-1.4.2.tgz", + "integrity": "sha512-I5d00Pd/jwMD2QCduo657+YM/6L3KZu++pmX9VFncxaxvHcru9jx1lBaFft+r4Mt2jK0Yhp41XlRAihzPxHNCg==", "dev": true, "requires": { "big.js": "^5.2.2", @@ -3673,9 +3673,9 @@ } }, "lodash": { - "version": "4.17.20", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz", - "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==" + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, "loose-envify": { "version": "1.4.0", diff --git a/dashboardv2/public/js/external_lib/atlas-lineage/src/Utils/DataUtils.js b/dashboardv2/public/js/external_lib/atlas-lineage/src/Utils/DataUtils.js index 1255eb070f0..50068ab777a 100644 --- a/dashboardv2/public/js/external_lib/atlas-lineage/src/Utils/DataUtils.js +++ b/dashboardv2/public/js/external_lib/atlas-lineage/src/Utils/DataUtils.js @@ -19,343 +19,350 @@ import Enums from "../Enums"; import { curveBasis } from "d3-shape"; const DataUtils = { - /** - * [getBaseUrl description] - * @param {[type]} url [description] - * @return {[type]} [description] - */ - getBaseUrl: function(url) { - return url.replace(/\/[\w-]+.(jsp|html)|\/+$/gi, ""); - }, - /** - * [getEntityIconPath description] - * @param {[type]} options.entityData [description] - * @param {Object} options.errorUrl } [description] - * @return {[type]} [description] - */ - getEntityIconPath: function({ entityData, errorUrl } = {}) { - var serviceType, - status, - typeName, - iconBasePath = this.getBaseUrl(window.location.pathname) + Globals.entityImgPath; - if (entityData) { - typeName = entityData.typeName; - serviceType = entityData && entityData.serviceType; - status = entityData && entityData.status; - } + /** + * [getBaseUrl description] + * @param {[type]} url [description] + * @return {[type]} [description] + */ + getBaseUrl: function(url) { + return url.replace(/\/[\w-]+.(jsp|html)|\/+$/gi, ""); + }, + /** + * [getEntityIconPath description] + * @param {[type]} options.entityData [description] + * @param {Object} options.errorUrl } [description] + * @return {[type]} [description] + */ + getEntityIconPath: function({ entityData, errorUrl } = {}) { + var serviceType, + status, + typeName, + iconBasePath = this.getBaseUrl(window.location.pathname) + Globals.entityImgPath; + if (entityData) { + typeName = entityData.typeName; + serviceType = entityData && entityData.serviceType; + status = entityData && entityData.status; + } - function getImgPath(imageName) { - return iconBasePath + (Enums.entityStateReadOnly[status] ? "disabled/" + imageName : imageName); - } + function getImgPath(imageName) { + return iconBasePath + (Enums.entityStateReadOnly[status] ? "disabled/" + imageName : imageName); + } - function getDefaultImgPath() { - if (entityData.isProcess) { - if (Enums.entityStateReadOnly[status]) { - return iconBasePath + "disabled/process.png"; - } else { - return iconBasePath + "process.png"; - } - } else { - if (Enums.entityStateReadOnly[status]) { - return iconBasePath + "disabled/table.png"; - } else { - return iconBasePath + "table.png"; - } - } - } + function getDefaultImgPath() { + if (entityData.isProcess) { + if (Enums.entityStateReadOnly[status]) { + return iconBasePath + "disabled/process.png"; + } else { + return iconBasePath + "process.png"; + } + } else { + if (Enums.entityStateReadOnly[status]) { + return iconBasePath + "disabled/table.png"; + } else { + return iconBasePath + "table.png"; + } + } + } - if (entityData) { - if (errorUrl) { - var isErrorInTypeName = errorUrl && errorUrl.match("entity-icon/" + typeName + ".png|disabled/" + typeName + ".png") ? true : false; - if (serviceType && isErrorInTypeName) { - var imageName = serviceType + ".png"; - return getImgPath(imageName); - } else { - return getDefaultImgPath(); - } - } else if (entityData.typeName) { - var imageName = entityData.typeName + ".png"; - return getImgPath(imageName); - } else { - return getDefaultImgPath(); - } - } - }, - /** - * [isProcess description] - * @param {[type]} options.typeName [description] - * @param {[type]} options.superTypes [description] - * @param {[type]} options.entityDef [description] - * @return {Boolean} [description] - */ - isProcess: function({ typeName, superTypes, entityDef }) { - if (typeName == "Process") { - return true; - } - return superTypes.indexOf("Process") > -1; - }, - /** - * [isDeleted description] - * @param {[type]} node [description] - * @return {Boolean} [description] - */ - isDeleted: function(node) { - if (node === undefined) { - return; - } - return Enums.entityStateReadOnly[node.status]; - }, - isNodeToBeUpdated: function(node, filterObj) { - var isProcessHideCheck = filterObj.isProcessHideCheck, - isDeletedEntityHideCheck = filterObj.isDeletedEntityHideCheck; - var returnObj = { - isProcess: isProcessHideCheck && node.isProcess, - isDeleted: isDeletedEntityHideCheck && node.isDeleted - }; - returnObj["update"] = returnObj.isProcess || returnObj.isDeleted; - return returnObj; - }, - /** - * [getServiceType description] - * @param {[type]} options.typeName [description] - * @param {[type]} options.entityDef [description] - * @return {[type]} [description] - */ - getServiceType: function({ typeName, entityDef }) { - var serviceType = null; - if (typeName) { - if (entityDef) { - serviceType = entityDef.serviceType || null; - } - } - return serviceType; - }, - /** - * [getEntityDef description] - * @param {[type]} options.typeName [description] - * @param {[type]} options.entityDefCollection [description] - * @return {[type]} [description] - */ - getEntityDef: function({ typeName, entityDefCollection }) { - var entityDef = null; - if (typeName) { - entityDef = entityDefCollection.find(function(obj) { - return obj.name == typeName; - }); - } - return entityDef; - }, - /** - * [getNestedSuperTypes description] - * @param {[type]} options.entityDef [description] - * @param {[type]} options.entityDefCollection [description] - * @return {[type]} [description] - */ - getNestedSuperTypes: function({ entityDef, entityDefCollection }) { - var data = entityDef, - collection = entityDefCollection, - superTypes = new Set(); + if (entityData) { + if (errorUrl) { + var isErrorInTypeName = errorUrl && errorUrl.match("entity-icon/" + typeName + ".png|disabled/" + typeName + ".png") ? true : false; + if (serviceType && isErrorInTypeName) { + var imageName = serviceType + ".png"; + return getImgPath(imageName); + } else { + return getDefaultImgPath(); + } + } else if (entityData.typeName) { + var imageName = entityData.typeName + ".png"; + return getImgPath(imageName); + } else { + return getDefaultImgPath(); + } + } + }, + /** + * [isProcess description] + * @param {[type]} options.typeName [description] + * @param {[type]} options.superTypes [description] + * @param {[type]} options.entityDef [description] + * @return {Boolean} [description] + */ + isProcess: function({ typeName, superTypes, entityDef }) { + if (typeName == "Process") { + return true; + } + return superTypes.indexOf("Process") > -1; + }, + /** + * [isDeleted description] + * @param {[type]} node [description] + * @return {Boolean} [description] + */ + isDeleted: function(node) { + if (node === undefined) { + return; + } + return Enums.entityStateReadOnly[node.status]; + }, + isNodeToBeUpdated: function(node, filterObj) { + var isProcessHideCheck = filterObj.isProcessHideCheck, + isDeletedEntityHideCheck = filterObj.isDeletedEntityHideCheck; + var returnObj = { + isProcess: isProcessHideCheck && node.isProcess, + isDeleted: isDeletedEntityHideCheck && node.isDeleted + }; + returnObj["update"] = returnObj.isProcess || returnObj.isDeleted; + if (node.label === "Expand") { + returnObj["update"] = true; + } + return returnObj; + }, + /** + * [getServiceType description] + * @param {[type]} options.typeName [description] + * @param {[type]} options.entityDef [description] + * @return {[type]} [description] + */ + getServiceType: function({ typeName, entityDef }) { + var serviceType = null; + if (typeName) { + if (entityDef) { + serviceType = entityDef.serviceType || null; + } + } + return serviceType; + }, + /** + * [getEntityDef description] + * @param {[type]} options.typeName [description] + * @param {[type]} options.entityDefCollection [description] + * @return {[type]} [description] + */ + getEntityDef: function({ typeName, entityDefCollection }) { + var entityDef = null; + if (typeName) { + entityDef = entityDefCollection.find(function(obj) { + return obj.name == typeName; + }); + } + return entityDef; + }, + /** + * [getNestedSuperTypes description] + * @param {[type]} options.entityDef [description] + * @param {[type]} options.entityDefCollection [description] + * @return {[type]} [description] + */ + getNestedSuperTypes: function({ entityDef, entityDefCollection }) { + var data = entityDef, + collection = entityDefCollection, + superTypes = new Set(); - var getData = function(data, collection) { - if (data) { - if (data.superTypes && data.superTypes.length) { - data.superTypes.forEach(function(superTypeName) { - superTypes.add(superTypeName); - var collectionData = collection.find(function(obj) { - obj.name === superTypeName; - }); - if (collectionData) { - getData(collectionData, collection); - } - }); - } - } - }; - getData(data, collection); - return Array.from(superTypes); - }, - generateData: function({ data = {}, filterObj, entityDefCollection, g, guid, setGraphEdge, setGraphNode }) { - return new Promise((resolve, reject) => { - try { - var relations = data.relations || {}, - guidEntityMap = data.guidEntityMap || {}, - isHideFilterOn = filterObj.isProcessHideCheck || filterObj.isDeletedEntityHideCheck, - newHashMap = {}, - styleObj = { - fill: "none", - stroke: "#ffb203", - width: 3 - }, - makeNodeData = (relationObj) => { - if (relationObj) { - if (relationObj.updatedValues) { - return relationObj; - } - var obj = Object.assign(relationObj, { - shape: "img", - updatedValues: true, - label: relationObj.displayText.trunc(18), - toolTipLabel: relationObj.displayText, - id: relationObj.guid, - isLineage: true, - isIncomplete: relationObj.isIncomplete, - entityDef: this.getEntityDef({ typeName: relationObj.typeName, entityDefCollection }) - }); - obj["serviceType"] = this.getServiceType(obj); - obj["superTypes"] = this.getNestedSuperTypes({ - ...obj, - entityDefCollection: entityDefCollection - }); - obj["isProcess"] = this.isProcess(obj); - obj["isDeleted"] = this.isDeleted(obj); - return obj; - } - }, - crateLineageRelationshipHashMap = function({ relations } = {}) { - var newHashMap = {}; - relations.forEach(function(obj) { - if (newHashMap[obj.fromEntityId]) { - newHashMap[obj.fromEntityId].push(obj.toEntityId); - } else { - newHashMap[obj.fromEntityId] = [obj.toEntityId]; - } - }); - return newHashMap; - }, - getStyleObjStr = function(styleObj) { - return "fill:" + styleObj.fill + ";stroke:" + styleObj.stroke + ";stroke-width:" + styleObj.width; - }, - getNewToNodeRelationship = (toNodeGuid, filterObj) => { - if (toNodeGuid && relationshipMap[toNodeGuid]) { - var newRelationship = []; - relationshipMap[toNodeGuid].forEach((guid) => { - var nodeToBeUpdated = this.isNodeToBeUpdated(makeNodeData(guidEntityMap[guid]), filterObj); - if (nodeToBeUpdated.update) { - var newRelation = getNewToNodeRelationship(guid, filterObj); - if (newRelation) { - newRelationship = newRelationship.concat(newRelation); - } - } else { - newRelationship.push(guid); - } - }); - return newRelationship; - } else { - return null; - } - }, - getToNodeRelation = (toNodes, fromNodeToBeUpdated, filterObj) => { - var toNodeRelationship = []; - toNodes.forEach((toNodeGuid) => { - var toNodeToBeUpdated = this.isNodeToBeUpdated(makeNodeData(guidEntityMap[toNodeGuid]), filterObj); - if (toNodeToBeUpdated.update) { - // To node need to updated - if (pendingFromRelationship[toNodeGuid]) { - toNodeRelationship = toNodeRelationship.concat(pendingFromRelationship[toNodeGuid]); - } else { - var newToNodeRelationship = getNewToNodeRelationship(toNodeGuid, filterObj); - if (newToNodeRelationship) { - toNodeRelationship = toNodeRelationship.concat(newToNodeRelationship); - } - } - } else { - //when bothe node not to be updated. - toNodeRelationship.push(toNodeGuid); - } - }); - return toNodeRelationship; - }, - setNode = (guid) => { - if (!g._nodes[guid]) { - var nodeData = makeNodeData(guidEntityMap[guid]); - setGraphNode(guid, nodeData); - return nodeData; - } else { - return g._nodes[guid]; - } - }, - setEdge = function(fromNodeGuid, toNodeGuid, opt = {}) { - setGraphEdge(fromNodeGuid, toNodeGuid, { - arrowhead: "arrowPoint", - curve: curveBasis, - style: getStyleObjStr(styleObj), - styleObj: styleObj, - ...opt - }); - }, - setGraphData = function(fromEntityId, toEntityId) { - setNode(fromEntityId); - setNode(toEntityId); - setEdge(fromEntityId, toEntityId); - }, - pendingFromRelationship = {}; - if (isHideFilterOn) { - var relationshipMap = crateLineageRelationshipHashMap(data); - Object.keys(relationshipMap).forEach((fromNodeGuid) => { - var toNodes = relationshipMap[fromNodeGuid], - fromNodeToBeUpdated = this.isNodeToBeUpdated(makeNodeData(guidEntityMap[fromNodeGuid]), filterObj), - toNodeList = getToNodeRelation(toNodes, fromNodeToBeUpdated, filterObj); - if (fromNodeToBeUpdated.update) { - if (pendingFromRelationship[fromNodeGuid]) { - pendingFromRelationship[fromNodeGuid] = pendingFromRelationship[fromNodeGuid].concat(toNodeList); - } else { - pendingFromRelationship[fromNodeGuid] = toNodeList; - } - } else { - toNodeList.forEach(function(toNodeGuid) { - setGraphData(fromNodeGuid, toNodeGuid); - }); - } - }); - } else { - relations.forEach(function(obj) { - setGraphData(obj.fromEntityId, obj.toEntityId); - }); - } - if (g._nodes[guid]) { - if (g._nodes[guid]) { - g._nodes[guid]["isLineage"] = false; - } - this.findImpactNodeAndUpdateData({ - guid, - g, - setEdge, - getStyleObjStr - }); - } - resolve(g); - } catch (e) { - reject(e); - } - }); - }, - findImpactNodeAndUpdateData: function({ guid, getStyleObjStr, g, setEdge }) { - var that = this, - traversedMap = {}, - styleObj = { - fill: "none", - stroke: "#fb4200", - width: 3 - }, - traversed = function(toNodeList = {}, fromNodeGuid) { - let toNodeKeyList = Object.keys(toNodeList); - if (toNodeKeyList.length) { - if (!traversedMap[fromNodeGuid]) { - traversedMap[fromNodeGuid] = true; - toNodeKeyList.forEach(function(toNodeGuid) { - if (g._nodes[toNodeGuid]) { - g._nodes[toNodeGuid]["isLineage"] = false; - } - setEdge(fromNodeGuid, toNodeGuid, { - style: getStyleObjStr(styleObj), - styleObj: styleObj - }); - traversed(g._sucs[toNodeGuid], toNodeGuid); - }); - } - } - }; - traversed(g._sucs[guid], guid); - } + var getData = function(data, collection) { + if (data) { + if (data.superTypes && data.superTypes.length) { + data.superTypes.forEach(function(superTypeName) { + superTypes.add(superTypeName); + var collectionData = collection.find(function(obj) { + obj.name === superTypeName; + }); + if (collectionData) { + getData(collectionData, collection); + } + }); + } + } + }; + getData(data, collection); + return Array.from(superTypes); + }, + generateData: function({ data = {}, filterObj, entityDefCollection, g, guid, setGraphEdge, setGraphNode }) { + return new Promise((resolve, reject) => { + try { + var relations = data.relations || {}, + guidEntityMap = data.guidEntityMap || {}, + isHideFilterOn = filterObj.isProcessHideCheck || filterObj.isDeletedEntityHideCheck, + newHashMap = {}, + styleObj = { + fill: "none", + stroke: "#ffb203", + width: 3 + }, + makeNodeData = (relationObj) => { + if (relationObj) { + if (relationObj.updatedValues) { + return relationObj; + } + var nodeLabel = relationObj.displayText ? relationObj.displayText : " ", + obj = Object.assign(relationObj, { + shape: "img", + updatedValues: true, + label: nodeLabel.trunc(18), + toolTipLabel: nodeLabel, + id: relationObj.guid, + isLineage: true, + isIncomplete: relationObj.isIncomplete, + entityDef: this.getEntityDef({ typeName: relationObj.typeName, entityDefCollection }) + }); + obj["serviceType"] = this.getServiceType(obj); + obj["superTypes"] = this.getNestedSuperTypes({ + ...obj, + entityDefCollection: entityDefCollection + }); + obj["isProcess"] = this.isProcess(obj); + obj["isDeleted"] = this.isDeleted(obj); + return obj; + } + }, + crateLineageRelationshipHashMap = function({ relations } = {}) { + var newHashMap = {}; + relations.forEach(function(obj) { + if (newHashMap[obj.fromEntityId]) { + newHashMap[obj.fromEntityId].push(obj.toEntityId); + } else { + newHashMap[obj.fromEntityId] = [obj.toEntityId]; + } + }); + return newHashMap; + }, + getStyleObjStr = function(styleObj) { + return "fill:" + styleObj.fill + ";stroke:" + styleObj.stroke + ";stroke-width:" + styleObj.width; + }, + getNewToNodeRelationship = (toNodeGuid, filterObj) => { + if (toNodeGuid && relationshipMap[toNodeGuid]) { + var newRelationship = []; + relationshipMap[toNodeGuid].forEach((guid) => { + var nodeToBeUpdated = this.isNodeToBeUpdated(makeNodeData(guidEntityMap[guid]), filterObj); + if (nodeToBeUpdated.update) { + var newRelation = getNewToNodeRelationship(guid, filterObj); + if (newRelation) { + newRelationship = newRelationship.concat(newRelation); + } + } else { + newRelationship.push(guid); + } + }); + return newRelationship; + } else { + return null; + } + }, + getToNodeRelation = (toNodes, fromNodeToBeUpdated, filterObj) => { + var toNodeRelationship = []; + toNodes.forEach((toNodeGuid) => { + var toNodeToBeUpdated = this.isNodeToBeUpdated(makeNodeData(guidEntityMap[toNodeGuid]), filterObj); + if (toNodeToBeUpdated.update) { + // To node need to updated + if (pendingFromRelationship[toNodeGuid]) { + toNodeRelationship = toNodeRelationship.concat(pendingFromRelationship[toNodeGuid]); + } else { + var newToNodeRelationship = getNewToNodeRelationship(toNodeGuid, filterObj); + if (newToNodeRelationship) { + toNodeRelationship = toNodeRelationship.concat(newToNodeRelationship); + } + } + } else { + //when bothe node not to be updated. + toNodeRelationship.push(toNodeGuid); + } + }); + return toNodeRelationship; + }, + setNode = (guid) => { + if (!g._nodes[guid]) { + var nodeData = makeNodeData(guidEntityMap[guid]); + setGraphNode(guid, nodeData); + return nodeData; + } else { + return g._nodes[guid]; + } + }, + setEdge = function(fromNodeGuid, toNodeGuid, opt = {}) { + setGraphEdge(fromNodeGuid, toNodeGuid, { + arrowhead: "arrowPoint", + curve: curveBasis, + style: getStyleObjStr(styleObj), + styleObj: styleObj, + ...opt + }); + }, + setGraphData = function(fromEntityId, toEntityId) { + setNode(fromEntityId); + setNode(toEntityId); + setEdge(fromEntityId, toEntityId); + }, + pendingFromRelationship = {}; + if (isHideFilterOn) { + var relationshipMap = crateLineageRelationshipHashMap(data); + Object.keys(relationshipMap).forEach((fromNodeGuid) => { + var toNodes = relationshipMap[fromNodeGuid], + fromNodeToBeUpdated = this.isNodeToBeUpdated(makeNodeData(guidEntityMap[fromNodeGuid]), filterObj), + toNodeList = getToNodeRelation(toNodes, fromNodeToBeUpdated, filterObj); + if (fromNodeToBeUpdated.update) { + if (pendingFromRelationship[fromNodeGuid]) { + pendingFromRelationship[fromNodeGuid] = pendingFromRelationship[fromNodeGuid].concat(toNodeList); + } else { + pendingFromRelationship[fromNodeGuid] = toNodeList; + } + } else { + toNodeList.forEach(function(toNodeGuid) { + setGraphData(fromNodeGuid, toNodeGuid); + }); + } + }); + } else { + relations.forEach(function(obj) { + setGraphData(obj.fromEntityId, obj.toEntityId); + }); + } + if (g._nodes[guid]) { + if (g._nodes[guid]) { + g._nodes[guid]["isLineage"] = false; + } + this.findImpactNodeAndUpdateData({ + guid, + g, + setEdge, + getStyleObjStr + }); + } + if (!g._nodes[guid]) { + setNode(guid); + } + resolve(g); + } catch (e) { + reject(e); + } + }); + }, + findImpactNodeAndUpdateData: function({ guid, getStyleObjStr, g, setEdge }) { + var that = this, + traversedMap = {}, + styleObj = { + fill: "none", + stroke: "#fb4200", + width: 3 + }, + traversed = function(toNodeList = {}, fromNodeGuid) { + let toNodeKeyList = Object.keys(toNodeList); + if (toNodeKeyList.length) { + if (!traversedMap[fromNodeGuid]) { + traversedMap[fromNodeGuid] = true; + toNodeKeyList.forEach(function(toNodeGuid) { + if (g._nodes[toNodeGuid]) { + g._nodes[toNodeGuid]["isLineage"] = false; + } + setEdge(fromNodeGuid, toNodeGuid, { + style: getStyleObjStr(styleObj), + styleObj: styleObj + }); + traversed(g._sucs[toNodeGuid], toNodeGuid); + }); + } + } + }; + traversed(g._sucs[guid], guid); + } }; export default DataUtils; \ No newline at end of file diff --git a/dashboardv2/public/js/external_lib/atlas-lineage/src/Utils/LineageUtils.js b/dashboardv2/public/js/external_lib/atlas-lineage/src/Utils/LineageUtils.js index 448a95ae6eb..588e3ac67d0 100644 --- a/dashboardv2/public/js/external_lib/atlas-lineage/src/Utils/LineageUtils.js +++ b/dashboardv2/public/js/external_lib/atlas-lineage/src/Utils/LineageUtils.js @@ -31,25 +31,25 @@ const LineageUtils = { * @type {Number} */ nodeArrowDistance: 24, - refreshGraphForSafari: function (options) { + refreshGraphForSafari: function(options) { var edgePathEl = options.edgeEl, IEGraphRenderDone = 0; - edgePathEl.each(function (argument) { + edgePathEl.each(function(argument) { var eleRef = this, childNode = $(this).find("pattern"); - setTimeout(function (argument) { + setTimeout(function(argument) { $(eleRef).find("defs").append(childNode); }, 500); }); }, - refreshGraphForIE: function ({ edgePathEl }) { + refreshGraphForIE: function({ edgePathEl }) { var IEGraphRenderDone = 0; - edgePathEl.each(function (argument) { + edgePathEl.each(function(argument) { var childNode = $(this).find("marker"); $(this).find("marker").remove(); var eleRef = this; ++IEGraphRenderDone; - setTimeout(function (argument) { + setTimeout(function(argument) { $(eleRef).find("defs").append(childNode); --IEGraphRenderDone; if (IEGraphRenderDone === 0) { @@ -67,9 +67,9 @@ const LineageUtils = { * @param {[type]} options.edgePathEl [description] * @return {[type]} [description] */ - dragNode: function ({ g, svg, guid, edgePathEl }) { + dragNode: function({ g, svg, guid, edgePathEl }) { var dragHelper = { - dragmove: function (el, d) { + dragmove: function(el, d) { var node = select(el), selectedNode = g.node(d), prevX = selectedNode.x, @@ -91,13 +91,13 @@ const LineageUtils = { }); //LineageUtils.refreshGraphForIE({ edgePathEl: edgePathEl }); }, - translateEdge: function (e, dx, dy) { - e.points.forEach(function (p) { + translateEdge: function(e, dx, dy) { + e.points.forEach(function(p) { p.x = p.x + dx; p.y = p.y + dy; }); }, - calcPoints: function (e) { + calcPoints: function(e) { var edge = g.edge(e.v, e.w), tail = g.node(e.v), head = g.node(e.w), @@ -106,10 +106,10 @@ const LineageUtils = { points.unshift(this.intersectRect(tail, points[0])); points.push(this.intersectRect(head, points[points.length - 1])); return line() - .x(function (d) { + .x(function(d) { return d.x; }) - .y(function (d) { + .y(function(d) { return d.y; }) .curve(curveBasis)(points); @@ -146,10 +146,10 @@ const LineageUtils = { }; } }; - var dragNodeHandler = drag().on("drag", function (d) { + var dragNodeHandler = drag().on("drag", function(d) { dragHelper.dragmove.call(dragHelper, this, d); }), - dragEdgePathHandler = drag().on("drag", function (d) { + dragEdgePathHandler = drag().on("drag", function(d) { dragHelper.translateEdge(g.edge(d.v, d.w), event.dx, event.dy); var edgeObj = g.edge(d.v, d.w); select(edgeObj.elem).select("path").attr("d", dragHelper.calcPoints(d)); @@ -158,16 +158,16 @@ const LineageUtils = { dragNodeHandler(svg.selectAll("g.node")); dragEdgePathHandler(svg.selectAll("g.edgePath")); }, - zoomIn: function ({ svg, scaleFactor = 1.3 }) { + zoomIn: function({ svg, scaleFactor = 1.3 }) { this.d3Zoom.scaleBy(svg.transition().duration(750), scaleFactor); }, - zoomOut: function ({ svg, scaleFactor = 0.8 }) { + zoomOut: function({ svg, scaleFactor = 0.8 }) { this.d3Zoom.scaleBy(svg.transition().duration(750), scaleFactor); }, - zoom: function ({ svg, xa, ya, scale }) { + zoom: function({ svg, xa, ya, scale }) { svg.transition().duration(750).call(this.d3Zoom.transform, zoomIdentity.translate(xa, ya).scale(scale)); }, - fitToScreen: function ({ svg }) { + fitToScreen: function({ svg }) { var node = svg.node(); var bounds = node.getBBox(); @@ -197,14 +197,14 @@ const LineageUtils = { * @param {[type]} options.onCenterZoomed [description] * @return {[type]} [description] */ - centerNode: function ({ guid, g, svg, svgGroupEl, edgePathEl, width, height, fitToScreen, onCenterZoomed }) { + centerNode: function({ guid, g, svg, svgGroupEl, edgePathEl, width, height, fitToScreen, onCenterZoomed, isSelected }) { this.d3Zoom = zoom(); svg.call(this.d3Zoom).on("dblclick.zoom", null); // restrict events let selectedNodeEl = svg.selectAll("g.nodes>g[id='" + guid + "']"), - zoomListener = this.d3Zoom.scaleExtent([0.01, 50]).on("zoom", function () { + zoomListener = this.d3Zoom.scaleExtent([0.01, 50]).on("zoom", function() { svgGroupEl.attr("transform", event.transform); }), x = null, @@ -236,7 +236,13 @@ const LineageUtils = { var xa = -(x * scale - width / 2), ya = -(y * scale - height / 2); this.zoom({ svg, xa, ya, scale }); - svg.transition().duration(750).call(this.d3Zoom.transform, zoomIdentity.translate(xa, ya).scale(scale)); + + if (!isSelected) { + svg.call(this.d3Zoom.transform, zoomIdentity.translate(xa, ya).scale(scale)); + } else { + svg.transition().duration(750).call(this.d3Zoom.transform, zoomIdentity.translate(xa, ya).scale(scale)); + } + if (onCenterZoomed) { onCenterZoomed({ newScale: scale, newTranslate: [xa, ya], d3Zoom: this.d3Zoom, selectedNodeEl }); @@ -250,7 +256,7 @@ const LineageUtils = { * @param {[type]} options.el [description] * @return {[type]} [description] */ - getToolTipDirection: function ({ el }) { + getToolTipDirection: function({ el }) { var width = select("body").node().getBoundingClientRect().width, currentELWidth = select(el).node().getBoundingClientRect(), direction = "e"; @@ -279,10 +285,10 @@ const LineageUtils = { * @param {[type]} options.hoveredNode [description] * @return {[type]} [description] */ - onHoverFade: function ({ svg, g, mouseenter, nodesToHighlight, hoveredNode }) { + onHoverFade: function({ svg, g, mouseenter, nodesToHighlight, hoveredNode }) { var node = svg.selectAll(".node"), path = svg.selectAll(".edgePath"), - isConnected = function (a, b, o) { + isConnected = function(a, b, o) { if (a === o || (b && b.length && b.indexOf(o) != -1)) { return true; } @@ -292,14 +298,14 @@ const LineageUtils = { var nextNode = g.successors(hoveredNode), previousNode = g.predecessors(hoveredNode), nodesToHighlight = nextNode.concat(previousNode); - node.classed("hover-active-node", function (currentNode, i, nodes) { + node.classed("hover-active-node", function(currentNode, i, nodes) { if (isConnected(hoveredNode, nodesToHighlight, currentNode)) { return true; } else { return false; } }); - path.classed("hover-active-path", function (c) { + path.classed("hover-active-path", function(c) { var _thisOpacity = c.v === hoveredNode || c.w === hoveredNode ? 1 : 0; if (_thisOpacity) { return true; @@ -318,10 +324,10 @@ const LineageUtils = { * @param {[type]} path [description] * @return {[type]} [description] */ - getBaseUrl: function (url = window.location.pathname) { + getBaseUrl: function(url = window.location.pathname) { return url.replace(/\/[\w-]+.(jsp|html)|\/+$/gi, ""); }, - getEntityIconPath: function ({ entityData, errorUrl, imgBasePath }) { + getEntityIconPath: function({ entityData, errorUrl, imgBasePath }) { var iconBasePath = this.getBaseUrl() + (imgBasePath || "/img/entity-icon/"); if (entityData) { let { typeName, serviceType, status, isProcess } = entityData; @@ -348,7 +354,7 @@ const LineageUtils = { if (errorUrl) { // Check if the default img path has error, if yes then stop recursion. - if (errorUrl.indexOf("table.png") > -1 || errorUrl.indexOf("process.png") > -1) { + if (errorUrl.indexOf("table.png") > -1) { //removed condition for default process image return null; } var isErrorInTypeName = errorUrl && errorUrl.match("entity-icon/" + typeName + ".png|disabled/" + typeName + ".png") ? true : false; @@ -369,15 +375,15 @@ const LineageUtils = { } } }, - base64Encode: function (file, callback) { + base64Encode: function(file, callback) { const reader = new FileReader(); reader.addEventListener("load", () => callback(reader.result)); reader.readAsDataURL(file); }, - imgShapeRender: function (parent, bbox, node, { dagreD3, defsEl, imgBasePath, guid, isRankdirToBottom }) { + imgShapeRender: function(parent, bbox, node, { dagreD3, defsEl, imgBasePath, guid, isRankdirToBottom }) { var that = this, viewGuid = guid, - imageIconPath = this.getEntityIconPath({ entityData: node, imgBasePath }), + imageIconPath = node.btnType ? "/img/entity-icon/expandBtn.svg" : this.getEntityIconPath({ entityData: node, imgBasePath }), imgName = imageIconPath.split("/").pop(); if (this.imageObject === undefined) { this.imageObject = {}; @@ -390,7 +396,7 @@ const LineageUtils = { } var shapeSvg = parent .append("circle") - .attr("fill", "url(#img_" + encodeURI(imgName) + ")") + .attr("fill", "url(#img_" + imgName + ")") .attr("r", isRankdirToBottom ? "30px" : "24px") .attr("data-stroke", node.id) .attr("stroke-width", "2px") @@ -401,90 +407,112 @@ const LineageUtils = { if (node.isIncomplete === true) { parent.attr("class", "node isIncomplete show"); parent - .insert("foreignObject") - .attr("x", "-25") - .attr("y", "-25") - .attr("width", "50") - .attr("height", "50") - .append("xhtml:div") - .insert("i") - .attr("class", "fa fa-hourglass-half"); + .insert("rect") + .attr("x", "-5") + .attr("y", "-23") + .attr("width", "14") + .attr("height", "16") + .attr("fill", "url(#img_hourglass.svg)") + .attr("data-stroke", node.id) + .attr("stroke-width", "2px"); + + var patternConfigShellIcon = { + imgName: "hourglass.svg", + imageIconPath: "/img/entity-icon/hourglass.svg", + leftPosition: "0", + topPosition: "0", + width: "12", + height: "14" + }; + svgPattern(patternConfigShellIcon); } + var patternConfigEntityIcon = { + imgName: imgName, + imageIconPath: imageIconPath, + leftPosition: isRankdirToBottom ? "11" : "4", + topPosition: isRankdirToBottom ? "20" : currentNode ? "3" : "4", + width: "40", + height: "40" + }; + svgPattern(patternConfigEntityIcon); - if (defsEl.select('pattern[id="img_' + imgName + '"]').empty()) { - defsEl - .append("pattern") - .attr("x", "0%") - .attr("y", "0%") - .attr("patternUnits", "objectBoundingBox") - .attr("id", "img_" + imgName) - .attr("width", "100%") - .attr("height", "100%") - .append("image") - .attr("href", function (d) { - var imgEl = this; - if (node) { - var getImageData = function (options) { - var imagePath = options.imagePath, - ajaxOptions = { - url: imagePath, - method: "GET", - cache: true - }; + function svgPattern(patternConfig) { + if (defsEl.select('pattern[id="img_' + patternConfig.imgName + '"]').empty()) { + defsEl + .append("pattern") + .attr("x", "0%") + .attr("y", "0%") + .attr("patternUnits", "objectBoundingBox") + .attr("id", "img_" + patternConfig.imgName) + .attr("width", "100%") + .attr("height", "100%") + .append("image") + .attr("href", function(d) { + var imgEl = this; + if (node) { + var getImageData = function(options) { + var imagePath = options.imagePath, + ajaxOptions = { + url: imagePath, + method: "GET", + cache: true + }; - // if (platform.name !== "IE") { - // ajaxOptions["mimeType"] = "text/plain; charset=x-user-defined"; - // } - shapeSvg.attr("data-iconpath", imagePath); - var xhr = new XMLHttpRequest(); - xhr.onreadystatechange = function () { - if (xhr.readyState === 4) { - if (xhr.status === 200) { - if (platform.name !== "IE") { - that.base64Encode(this.response, (url) => { - that.imageObject[imageIconPath] = url; - select(imgEl).attr("xlink:href", url); - }); - } else { - that.imageObject[imageIconPath] = imagePath; - } - if (imageIconPath !== shapeSvg.attr("data-iconpath")) { - shapeSvg.attr("data-iconpathorigin", imageIconPath); - } - } else if (xhr.status === 404) { - const imgPath = that.getEntityIconPath({ entityData: node, errorUrl: imagePath }); - if (imgPath === null) { - const patternEL = select(imgEl.parentElement); - patternEL.select("image").remove(); - patternEL - .attr("patternContentUnits", "objectBoundingBox") - .append("circle") - .attr("r", "24px") - .attr("fill", "#e8e8e8"); - } else { - getImageData({ - imagePath: imgPath - }); + // if (platform.name !== "IE") { + // ajaxOptions["mimeType"] = "text/plain; charset=x-user-defined"; + // } + shapeSvg.attr("data-iconpath", imagePath); + var xhr = new XMLHttpRequest(); + xhr.onreadystatechange = function() { + if (xhr.readyState === 4) { + if (xhr.status === 200) { + if (platform.name !== "IE") { + that.base64Encode(this.response, (url) => { + that.imageObject[patternConfig.imageIconPath] = url; + select(imgEl).attr("xlink:href", url); + }); + } else { + that.imageObject[patternConfig.imageIconPath] = imagePath; + } + if (patternConfig.imageIconPath !== shapeSvg.attr("data-iconpath")) { + shapeSvg.attr("data-iconpathorigin", patternConfig.imageIconPath); + } + } else if (xhr.status === 404) { + const imgPath = that.getEntityIconPath({ entityData: node, errorUrl: imagePath }); + if (imgPath === null) { + const patternEL = select(imgEl.parentElement); + patternEL.select("image").remove(); + patternEL + .attr("patternContentUnits", "objectBoundingBox") + .append("circle") + .attr("r", "24px") + .attr("fill", "#e8e8e8"); + } else { + getImageData({ + imagePath: imgPath + }); + } } } - } + }; + xhr.responseType = "blob"; + xhr.open(ajaxOptions.method, ajaxOptions.url, true); + xhr.send(null); }; - xhr.responseType = "blob"; - xhr.open(ajaxOptions.method, ajaxOptions.url, true); - xhr.send(null); - }; - getImageData({ - imagePath: imageIconPath - }); - } - }) - .attr("x", isRankdirToBottom ? "11" : "4") - .attr("y", isRankdirToBottom ? "20" : currentNode ? "3" : "4") - .attr("width", "40") - .attr("height", "40"); + getImageData({ + imagePath: patternConfig.imageIconPath + }); + } + }) + .attr("x", patternConfig.leftPosition) + .attr("y", patternConfig.topPosition) + .attr("width", patternConfig.width) + .attr("height", patternConfig.height); + } } - node.intersect = function (point) { + + node.intersect = function(point) { return dagreD3.intersect.circle(node, currentNode ? that.nodeArrowDistance + 3 : that.nodeArrowDistance, point); }; return shapeSvg; @@ -494,7 +522,7 @@ const LineageUtils = { * @param {[type]} {parent, id, edge, type, viewOptions [description] * @return {[type]} [description] */ - arrowPointRender: function (parent, id, edge, type, { dagreD3 }) { + arrowPointRender: function(parent, id, edge, type, { dagreD3 }) { var node = parent.node(), parentNode = node ? node.parentNode : parent; select(parentNode) @@ -523,11 +551,11 @@ const LineageUtils = { * @param {[type]} options.onExportLineage [description] * @return {[type]} [description] */ - saveSvg: function ({ svg, width, height, downloadFileName, onExportLineage }) { + saveSvg: function({ svg, width, height, downloadFileName, onExportLineage }) { var that = this, svgClone = svg.clone(true).node(), scaleFactor = 1; - setTimeout(function () { + setTimeout(function() { if (platform.name === "Firefox") { svgClone.setAttribute("width", width); svgClone.setAttribute("height", height); @@ -570,15 +598,16 @@ const LineageUtils = { if (platform.name === "Safari") { svgBlob = new Blob([data], { type: "image/svg+xml" }); } + ctx.drawImage(img, 50, 50, canvas.width, canvas.height); var url = DOMURL.createObjectURL(svgBlob); - img.onload = function () { + img.onload = function() { try { var a = document.createElement("a"); a.download = downloadFileName; document.body.appendChild(a); ctx.drawImage(img, 50, 50, canvas.width, canvas.height); - canvas.toBlob(function (blob) { + canvas.toBlob(function(blob) { if (!blob) { onExportLineage({ status: "failed", message: "There was an error in downloading Lineage!" }); return; diff --git a/dashboardv2/public/js/external_lib/atlas-lineage/src/index.js b/dashboardv2/public/js/external_lib/atlas-lineage/src/index.js index 3ceb3d90a90..0c4702237fe 100644 --- a/dashboardv2/public/js/external_lib/atlas-lineage/src/index.js +++ b/dashboardv2/public/js/external_lib/atlas-lineage/src/index.js @@ -45,6 +45,7 @@ export default class LineageHelper { zoom: (arg) => this.zoom(arg), fullScreen: (arg) => this.fullScreen(arg), searchNode: (arg) => this.searchNode(arg), + displayFullName: (arg) => this.displayFullName(arg), removeNodeSelection: (arg) => this.removeNodeSelection(arg), getGraphOptions: () => this.graphOptions, getNode: (guid, actual) => { @@ -163,14 +164,36 @@ export default class LineageHelper { zoom(opt = {}) { LineageUtils.zoom({ ...this.graphOptions, ...opt }); } + + displayFullName(opt = {}) { + var that = this; + this.g.nodes().forEach(function(v) { + var selectedNodeEl = that.svg.selectAll("g.nodes>g[id='" + v + "']"), + label = that.g.node(v).toolTipLabel; + if (opt.bLabelFullText == true) + selectedNodeEl.select('tspan').text(label); + else + selectedNodeEl.select('tspan').text(label.trunc(18)); + }); + if (this.selectedNode) { + this.searchNode({ guid: this.selectedNode }); + } + } + /** * [refresh Allows user to rerender the lineage] * @return {[type]} [description] */ - refresh() { + refresh(options) { this.clear(); this._initializeGraph(); this._initGraph({ refresh: true }); + this.selectedNode = ""; + if (options && options.compactLineageEnabled && options.filterObj) { + var isProcessHideCheck = options.filterObj.isProcessHideCheck, + isDeletedEntityHideCheck = options.filterObj.isDeletedEntityHideCheck; + this._AddFilterNotification(isProcessHideCheck, isDeletedEntityHideCheck); + } } /** * [removeNodeSelection description] @@ -185,23 +208,32 @@ export default class LineageHelper { */ searchNode({ guid, onSearchNode }) { this.svg.selectAll(".serach-rect").remove(); + this.svg.selectAll(".label").attr("stroke", "none"); + this.selectedNode = guid; this.centerAlign({ guid: guid, - onCenterZoomed: function (opts) { + onCenterZoomed: function(opts) { const { selectedNodeEl } = opts; + var oSelectedNode = selectedNodeEl.node().getBBox(), + rectWidth = oSelectedNode.width + 10, + rectXPos = oSelectedNode.x - 5; selectedNodeEl.select(".label").attr("stroke", "#316132"); selectedNodeEl.select("circle").classed("wobble", true); selectedNodeEl .insert("rect", "circle") .attr("class", "serach-rect") - .attr("x", -50) + .attr("stroke", "#37bb9b") + .attr("stroke-width", "2.5px") + .attr("fill", "none") + .attr("x", rectXPos) .attr("y", -27.5) - .attr("width", 100) - .attr("height", 55); + .attr("width", rectWidth) + .attr("height", 60); if (onSearchNode && typeof onSearchNode === "function") { onSearchNode(opts); } - } + }, + isSelected: true }); } @@ -287,8 +319,7 @@ export default class LineageHelper { // initlize the dagreD3 graphlib this.g = new dagreD3.graphlib.Graph() .setGraph( - Object.assign( - { + Object.assign({ nodesep: 50, ranksep: 90, rankdir: "LR", @@ -301,7 +332,7 @@ export default class LineageHelper { this.options.dagreOptions ) ) - .setDefaultEdgeLabel(function () { + .setDefaultEdgeLabel(function() { return {}; }); @@ -339,7 +370,7 @@ export default class LineageHelper { if (this.options.setDataManually === true) { return; - } else if (this.options.data === undefined || (this.options.data && this.options.data.relations.length === 0)) { + } else if (this.options.data === undefined || (this.options.data && this.options.data.relations.length === 0 && _.isEmpty(this.options.data.guidEntityMap))) { if (this.options.beforeRender) { this.options.beforeRender(); } @@ -402,8 +433,7 @@ export default class LineageHelper { * @param {[type]} graphOptions [description] * @return {[type]} [description] */ - _createGraph( - { + _createGraph({ data = {}, imgBasePath, isShowTooltip, @@ -416,12 +446,12 @@ export default class LineageHelper { getToolTipContent, toolTipTitle }, - graphOptions, - { refresh } + graphOptions, { refresh } ) { if (this.options.beforeRender) { this.options.beforeRender(); } + this.selectedNode = ""; const that = this, { svg, g, width, height } = graphOptions, isRankdirToBottom = this.options.dagreOptions && this.options.dagreOptions.rankdir === "tb"; @@ -435,7 +465,7 @@ export default class LineageHelper { return; } - g.nodes().forEach(function (v) { + g.nodes().forEach(function(v) { var node = g.node(v); // Round the corners of the nodes if (node) { @@ -452,11 +482,11 @@ export default class LineageHelper { // Create the renderer var render = new dagreD3.render(); // Add our custom arrow (a hollow-point) - render.arrows().arrowPoint = function () { + render.arrows().arrowPoint = function() { return LineageUtils.arrowPointRender(...arguments, { ...graphOptions }); }; // Render custom img inside shape - render.shapes().img = function () { + render.shapes().img = function() { return LineageUtils.imgShapeRender(...arguments, { ...graphOptions, isRankdirToBottom: isRankdirToBottom, @@ -506,17 +536,18 @@ export default class LineageHelper { if (isRankdirToBottom) { return "translate(2,-20)"; } - return "translate(2,-35)"; + return "translate(2,-38)"; }) - .on("mouseenter", function (d) { + .attr("font-size", "10px") + .on("mouseenter", function(d) { event.preventDefault(); select(this).classed("highlight", true); }) - .on("mouseleave", function (d) { + .on("mouseleave", function(d) { event.preventDefault(); select(this).classed("highlight", false); }) - .on("click", function (d) { + .on("click", function(d) { event.preventDefault(); if (onLabelClick && typeof onLabelClick === "function") { onLabelClick({ clickedData: d }); @@ -526,12 +557,12 @@ export default class LineageHelper { svgGroupEl .selectAll("g.nodes g.node circle") - .on("mouseenter", function (d, index, element) { + .on("mouseenter", function(d, index, element) { that.activeNode = true; var matrix = this.getScreenCTM().translate(+this.getAttribute("cx"), +this.getAttribute("cy")); that.svg.selectAll(".node").classed("active", false); select(this).classed("active", true); - if (that._getValueFromUser(isShowTooltip)) { + if (that._getValueFromUser(isShowTooltip) && (d.indexOf("more") !== 0)) { var direction = LineageUtils.getToolTipDirection({ el: this }); tooltip.direction(direction).show(d, this); } @@ -545,10 +576,10 @@ export default class LineageHelper { ...graphOptions }); }) - .on("mouseleave", function (d) { + .on("mouseleave", function(d) { that.activeNode = false; var nodeEL = this; - setTimeout(function (argument) { + setTimeout(function(argument) { if (!(that.activeTip || that.activeNode)) { select(nodeEL).classed("active", false); if (that._getValueFromUser(isShowTooltip)) { @@ -565,7 +596,7 @@ export default class LineageHelper { ...graphOptions }); }) - .on("click", function (d) { + .on("click", function(d) { if (event.defaultPrevented) return; // ignore drag event.preventDefault(); tooltip.hide(d); @@ -578,9 +609,9 @@ export default class LineageHelper { // Bind event on edgePath var edgePathEl = svgGroupEl.selectAll("g.edgePath"); - edgePathEl.selectAll("path.path").on("click", function (d) { + edgePathEl.selectAll("path.path").on("click", function(d) { if (onPathClick && typeof onPathClick === "function") { - var pathRelationObj = data.relations.find(function (obj) { + var pathRelationObj = data.relations.find(function(obj) { if (obj.fromEntityId === d.v && obj.toEntityId === d.w) { return true; } @@ -651,5 +682,16 @@ export default class LineageHelper { span = container.append("span").style("color", "#fb4200"); span.append("i").classed("fa fa-long-arrow-right fa-fw", true); span.append("span").html("Impact"); + + span = container.append("span").classed("notification hide", true).style("color", "#686868"); + span.append("i").classed("fa fa-exclamation fa-fw", true); + span.append("span").html("Filtering hides all Expand buttons."); + } + _AddFilterNotification(isProcessHideCheck, isDeletedEntityHideCheck) { + if ((isProcessHideCheck || isDeletedEntityHideCheck)) { + $(this.options.legendsEl).find('.notification').removeClass('hide'); + } else { + $(this.options.legendsEl).find('.notification').addClass('hide'); + } } } \ No newline at end of file diff --git a/dashboardv2/public/js/external_lib/atlas-lineage/src/styles/graph.scss b/dashboardv2/public/js/external_lib/atlas-lineage/src/styles/graph.scss index ab1e82df38c..2f1322765d0 100644 --- a/dashboardv2/public/js/external_lib/atlas-lineage/src/styles/graph.scss +++ b/dashboardv2/public/js/external_lib/atlas-lineage/src/styles/graph.scss @@ -26,17 +26,17 @@ //transition: opacity 0.3s linear; - rect { - stroke: $color_mountain_mist_approx; - fill: $white; - stroke-width: 1.5px; - - &.serach-rect { - stroke: $color_keppel_approx; - fill: transparent; - stroke-width: 2.5px; - } - } + // rect { + // stroke: $color_mountain_mist_approx; + // fill: $white; + // stroke-width: 1.5px; + + // &.serach-rect { + // stroke: $color_keppel_approx; + // fill: transparent; + // stroke-width: 2.5px; + // } + // } .label { fill: $color_suva_gray_approx; @@ -104,7 +104,7 @@ } .legends { - > span { + >span { margin-right: 8px; font-family: $font_0; } @@ -156,10 +156,12 @@ svg.hover { z-index: 999; max-width: 300px; //Instead of the line below you could use @include border-radius($radius, $vertical-radius) border-radius: 2px; + word-break: break-all; .tip-inner-scroll { overflow: auto; max-height: 300px; + h5 { margin: 7px 0px; } @@ -211,7 +213,7 @@ svg.hover { } } -g.type-TK > rect { +g.type-TK>rect { fill: $color_bright_turquoise_approx; } diff --git a/dashboardv2/public/js/main.js b/dashboardv2/public/js/main.js index f2ea1d81608..10ce386f20c 100644 --- a/dashboardv2/public/js/main.js +++ b/dashboardv2/public/js/main.js @@ -289,6 +289,12 @@ require(['App', Globals.isTasksEnabled = response['atlas.tasks.enabled']; } if (response['atlas.session.timeout.secs']) { Globals.idealTimeoutSeconds = response['atlas.session.timeout.secs']; } + if(response['atlas.lineage.on.demand.enabled'] !== undefined){ + Globals.isLineageOnDemandEnabled = response['atlas.lineage.on.demand.enabled']; + } + if(response['atlas.lineage.on.demand.default.node.count'] !== undefined){ + Globals.lineageNodeCount = response['atlas.lineage.on.demand.default.node.count']; + } /* Atlas idealTimeout redirectUrl: url to redirect after timeout idealTimeLimit: timeout in seconds diff --git a/dashboardv2/public/js/templates/graph/LineageLayoutView_tmpl.html b/dashboardv2/public/js/templates/graph/LineageLayoutView_tmpl.html index 5f90b096e3e..30dd45d7fa0 100644 --- a/dashboardv2/public/js/templates/graph/LineageLayoutView_tmpl.html +++ b/dashboardv2/public/js/templates/graph/LineageLayoutView_tmpl.html @@ -90,10 +90,20 @@

Settings

- +
+ {{#if compactLineageEnabled}} +
+
+ +
+ +
+
+
+ {{/if}}
diff --git a/dashboardv2/public/js/utils/Globals.js b/dashboardv2/public/js/utils/Globals.js index 8fe8ebc4cf4..3b72d81b1ba 100644 --- a/dashboardv2/public/js/utils/Globals.js +++ b/dashboardv2/public/js/utils/Globals.js @@ -50,6 +50,10 @@ define(["require"], function(require) { Globals.isDebugMetricsEnabled = false; Globals.isTasksEnabled = false; Globals.idealTimeoutSeconds = 900; + Globals.isFullScreenView = false; + Globals.isLineageOnDemandEnabled = false; + Globals.lineageNodeCount = 3; + Globals.lineageDepth = 3; return Globals; }); \ No newline at end of file diff --git a/dashboardv2/public/js/utils/Helper.js b/dashboardv2/public/js/utils/Helper.js index f211bd055a8..0ba1b6c4e02 100644 --- a/dashboardv2/public/js/utils/Helper.js +++ b/dashboardv2/public/js/utils/Helper.js @@ -17,10 +17,11 @@ */ define(['require', 'utils/Utils', + 'utils/Globals', 'd3', 'marionette', 'jquery-ui' -], function(require, Utils, d3) { +], function(require, Utils, Globals, d3) { 'use strict'; _.mixin({ numberFormatWithComma: function(number) { @@ -372,7 +373,13 @@ define(['require', }); } //For closing the modal on browsers navigation - $(window).on('popstate', function(){ + $(window).on('popstate', function() { $('body').find('.modal-dialog .close').click(); + //To close the full-screen mode in lineage on browsers navigation. + if (!Globals.isFullScreenView) { + $('#tab-lineage').removeClass("fullscreen-mode"); + $("#r_lineageLayoutView").find('button[data-id="fullScreen-toggler"]').attr("data-original-title", "Full Screen").find("i").removeClass("fa-compress").addClass("fa-expand"); + } + Globals.isFullScreenView = false; }); }) \ No newline at end of file diff --git a/dashboardv2/public/js/views/graph/LineageLayoutView.js b/dashboardv2/public/js/views/graph/LineageLayoutView.js index e989c4719bd..94bb19db76e 100644 --- a/dashboardv2/public/js/views/graph/LineageLayoutView.js +++ b/dashboardv2/public/js/views/graph/LineageLayoutView.js @@ -52,6 +52,7 @@ define(['require', checkHideProcess: "[data-id='checkHideProcess']", checkDeletedEntity: "[data-id='checkDeletedEntity']", selectDepth: 'select[data-id="selectDepth"]', + selectNodeCount: 'select[data-id="selectNodeCount"]', filterToggler: '[data-id="filter-toggler"]', settingToggler: '[data-id="setting-toggler"]', searchToggler: '[data-id="search-toggler"]', @@ -74,7 +75,8 @@ define(['require', templateHelpers: function() { return { width: "100%", - height: "100%" + height: "100%", + compactLineageEnabled: Globals.isLineageOnDemandEnabled }; }, /** ui events hash */ @@ -83,6 +85,7 @@ define(['require', events["click " + this.ui.checkHideProcess] = 'onCheckUnwantedEntity'; events["click " + this.ui.checkDeletedEntity] = 'onCheckUnwantedEntity'; events['change ' + this.ui.selectDepth] = 'onSelectDepthChange'; + events['change ' + this.ui.selectNodeCount] = 'onSelectNodeCount'; events["click " + this.ui.filterToggler] = 'onClickFilterToggler'; events["click " + this.ui.boxClose] = 'toggleBoxPanel'; events["click " + this.ui.settingToggler] = 'onClickSettingToggler'; @@ -109,7 +112,7 @@ define(['require', this.filterObj = { isProcessHideCheck: false, isDeletedEntityHideCheck: false, - depthCount: '' + depthCount: Globals.lineageDepth }; this.searchNodeObj = { selectedNode: '' @@ -117,16 +120,31 @@ define(['require', this.labelFullText = false; }, onRender: function() { - var that = this; + var that = this, + nodeCountArray = _.uniq([3, 6, Globals.lineageNodeCount]); + this.initialQueryObj = {}; + this.initialQueryObj["constraints"] = {}; this.ui.searchToggler.prop("disabled", true); - this.$graphButtonsEl = this.$(".graph-button-group button, select[data-id='selectDepth']") - this.fetchGraphData(); + this.$graphButtonsEl = this.$(".graph-button-group button, select[data-id='selectDepth']"); + if (Globals.isLineageOnDemandEnabled) { + this.ui.resetLineage.attr("title", "Reset Lineage"); + this.initialQueryObj["constraints"][this.guid] = { + "direction": "BOTH", + "inputRelationsLimit": Globals.lineageNodeCount, + "outputRelationsLimit": Globals.lineageNodeCount, + "depth": Globals.lineageDepth, + } + } + this.fetchGraphData({ queryParam: this.initialQueryObj }); if (this.layoutRendered) { this.layoutRendered(); } if (this.processCheck) { this.hideCheckForProcess(); } + if (this.entity.status === "DELETED") { + this.hideCheckForDeletedEntity(); + } //this.initializeGraph(); this.ui.selectDepth.select2({ data: _.sortBy([3, 6, 9, 12, 15, 18, 21]), @@ -134,6 +152,13 @@ define(['require', dropdownCssClass: "number-input", multiple: false }); + this.ui.selectNodeCount.select2({ + data: _.sortBy(nodeCountArray), + tags: true, + dropdownCssClass: "number-input", + multiple: false + }); + this.ui.selectNodeCount.val(Globals.lineageNodeCount).trigger("change"); }, onShow: function() { this.$('.fontLoader').show(); @@ -150,6 +175,7 @@ define(['require', panel = $(e.target).parents('.tab-pane').first(); icon.toggleClass('fa-expand fa-compress'); if (icon.hasClass('fa-expand')) { + Globals.isFullScreenView = false; icon.parent('button').attr("data-original-title", "Full Screen"); } else { icon.parent('button').attr("data-original-title", "Default View"); @@ -165,17 +191,22 @@ define(['require', onCheckUnwantedEntity: function(e) { var that = this; //this.initializeGraph(); + this.searchNodeObj.selectedNode = ""; if ($(e.target).data("id") === "checkHideProcess") { this.filterObj.isProcessHideCheck = e.target.checked; } else { this.filterObj.isDeletedEntityHideCheck = e.target.checked; } - this.LineageHelperRef.refresh(); + this.renderLineageTypeSearch(this.data); + this.LineageHelperRef.refresh({ compactLineageEnabled: Globals.isLineageOnDemandEnabled, filterObj: this.filterObj }); }, toggleBoxPanel: function(options) { var el = options && options.el, nodeDetailToggler = options && options.nodeDetailToggler, currentTarget = options.currentTarget; + if (options.nodeDetailToggler) { + this.ui.lineageTypeSearch.select2("close"); + } this.$el.find('.show-box-panel').removeClass('show-box-panel'); if (el && el.addClass) { el.addClass('show-box-panel'); @@ -214,12 +245,33 @@ define(['require', }, onSelectDepthChange: function(e, options) { //this.initializeGraph(); - this.filterObj.depthCount = e.currentTarget.value; + Globals.lineageDepth = parseInt(e.currentTarget.value); //legends property is added in queryParam to stop the legend getting added in lineage graph whenever dept is changed. - this.fetchGraphData({ queryParam: { 'depth': this.filterObj.depthCount }, 'legends': false }); + if (!Globals.isLineageOnDemandEnabled) { + this.fetchGraphData({ queryParam: { 'depth': Globals.lineageDepth }, 'legends': false }); + } + + if (Globals.isLineageOnDemandEnabled) { + this.initialQueryObj["constraints"][this.guid].depth = Globals.lineageDepth; + this.fetchGraphData({ queryParam: this.initialQueryObj, 'legends': false }) + } + }, + onSelectNodeCount: function(e, options) { + Globals.lineageNodeCount = parseInt(e.target.value); + if (Globals.lineageNodeCount === 0) { + Utils.notifyWarn({ + content: 'Value cannot be less than 1' + }); + this.ui.selectNodeCount.val(3).trigger("change"); + } }, onClickResetLineage: function() { - this.LineageHelperRef.refresh(); + if (Globals.isLineageOnDemandEnabled) { + this.fetchGraphData({ queryParam: this.initialQueryObj, 'legends': false }); + } + if (!Globals.isLineageOnDemandEnabled) { + this.LineageHelperRef.refresh(); + } this.searchNodeObj.selectedNode = ""; this.ui.lineageTypeSearch.data({ refresh: true }).val("").trigger("change"); this.ui.labelFullName.prop("checked", false); @@ -270,13 +322,14 @@ define(['require', } _.extend(this.currentEntityData, _.pick(this.entity, 'attributes', 'guid', 'isIncomplete', 'status', 'typeName')); //End - this.collection.getLineage(this.guid, { - queryParam: queryParam, + var dataObj = { + compactLineageEnabled: Globals.isLineageOnDemandEnabled, success: function(data) { if (that.isDestroyed) { return; } data["legends"] = options ? options.legends : true; + that.lineageOnDemandPayload = data.lineageOnDemandPayload ? data.lineageOnDemandPayload : {}; // show only main part of lineage current entity is at bottom, so reverse is done var relationsReverse = data.relations ? data.relations.reverse() : null, lineageMaxRelationCount = 9000; @@ -292,6 +345,10 @@ define(['require', data.guidEntityMap[data.baseEntityGuid] = that.currentEntityData; } } + that.data = data; + var updatedData = that.updateLineageData(data); + _.extend(data.guidEntityMap, updatedData.plusBtnsObj); + data.relations = data.relations.concat(updatedData.plusBtnRelationsArray); that.createGraph(data); that.renderLineageTypeSearch(data); }, @@ -302,7 +359,68 @@ define(['require', that.$('.fontLoader').hide(); that.$('svg>g').show(); } - }) + }; + Globals.isLineageOnDemandEnabled ? dataObj.data = queryParam : dataObj.queryParam = queryParam; + + this.collection.getLineage(this.guid, dataObj) + }, + updateLineageData: function(data) { + var that = this, + rawData = data, + plusBtnsObj = {}, + plusBtnRelationsArray = []; + this.relationsOnDemand = data.relationsOnDemand ? data.relationsOnDemand : null; + this.lineageOnDemandPayload = data.lineageOnDemandPayload ? data.lineageOnDemandPayload : null; + if (this.relationsOnDemand) { + _.each(this.relationsOnDemand, function(values, nodeId) { + if (values.hasMoreInputs) { + var btnType = "Input", + moreInputBtnObj = that.createExpandButtonObj({ nodeId: nodeId, btnType: btnType }); + plusBtnsObj[moreInputBtnObj.guid] = moreInputBtnObj; + plusBtnRelationsArray.push({ fromEntityId: moreInputBtnObj.guid, toEntityId: nodeId, relationshipId: 'dummy' }); + } + if (values.hasMoreOutputs) { + var btnType = "Output", + moreOutputBtnObj = that.createExpandButtonObj({ nodeId: nodeId, btnType: btnType }); + plusBtnsObj[moreOutputBtnObj.guid] = moreOutputBtnObj; + plusBtnRelationsArray.push({ fromEntityId: nodeId, toEntityId: moreOutputBtnObj.guid, relationshipId: 'dummy' }); + } + }); + return { + plusBtnsObj: plusBtnsObj, + plusBtnRelationsArray: plusBtnRelationsArray + } + } + }, + generateAddButtonId: function(btnType) { + return btnType + Math.random().toString(16).slice(2) + }, + createExpandButtonObj: function(options) { + var defaultObj = { + attributes: { + owner: '', + createTime: 0, + qualifiedName: 'PlusBtn', + name: 'PlusBtn', + description: '' + }, + isExpandBtn: true, + classificationNames: [], + displayText: "Expand", + isIncomplete: false, + labels: [], + meaningNames: [], + meanings: [], + status: "ACTIVE", + typeName: "Table" + }, + btnObj = Object.assign({}, defaultObj), + btnType = (options.btnType === "Input") ? "more-inputs" : "more-outputs", + btnId = this.generateAddButtonId(btnType); + btnObj.guid = btnId; + btnObj.parentNodeGuid = options.nodeId; + btnObj.btnType = options.btnType; + return btnObj; }, createGraph: function(data) { var that = this; @@ -323,7 +441,6 @@ define(['require', isShowHoverPath: function() { return that.ui.showOnlyHoverPath.prop('checked') }, isShowTooltip: function() { return that.ui.showTooltip.prop('checked') }, onPathClick: function(d) { - console.log("Path Clicked"); if (d.pathRelationObj) { var relationshipId = d.pathRelationObj.relationshipId; require(['views/graph/PropagationPropertyModal'], function(PropagationPropertyModal) { @@ -338,12 +455,21 @@ define(['require', } }, onNodeClick: function(d) { + if (d.clickedData.indexOf("more") >= 0) { + that.onExpandNodeClick({ guid: d.clickedData }); + return; + } that.onClickNodeToggler(); that.updateRelationshipDetails({ guid: d.clickedData }); that.calculateLineageDetailPanelHeight(); }, onLabelClick: function(d) { var guid = d.clickedData; + Globals.isFullScreenView = true; + that.ui.lineageTypeSearch.select2("close"); + if (guid.indexOf("more") >= 0) { + return; + } if (that.guid == guid) { Utils.notifyInfo({ html: true, @@ -376,6 +502,57 @@ define(['require', } }); }, + onExpandNodeClick: function(options) { + var parentNodeData = this.LineageHelperRef.getNode(options.guid); + this.updateQueryObject(parentNodeData.parentNodeGuid, parentNodeData.btnType); + }, + updateQueryObject: function(parentId, btnType) { + var inputLimit = null, + outputLimit = null, + that = this, + queryParam; + if (_.has(that.lineageOnDemandPayload, parentId)) { + _.find(that.lineageOnDemandPayload, function(value, key) { + if (key === parentId) { + if (btnType === "Input") { + value.inputRelationsLimit = value.inputRelationsLimit + Globals.lineageNodeCount; + value.outputRelationsLimit = value.outputRelationsLimit; + } + if (btnType === "Output") { + value.inputRelationsLimit = value.inputRelationsLimit; + value.outputRelationsLimit = value.outputRelationsLimit + Globals.lineageNodeCount; + } + } + }); + } else { + var relationCount = that.validateInputOutputLimit(parentId, btnType); + if (btnType === "Input") { + inputLimit = relationCount.inputRelationCount + Globals.lineageNodeCount; + outputLimit = relationCount.outputRelationCount; + } + if (btnType === "Output") { + inputLimit = relationCount.inputRelationCount; + outputLimit = relationCount.outputRelationCount + Globals.lineageNodeCount; + } + this.lineageOnDemandPayload["constraints"][parentId] = { direction: "BOTH", inputRelationsLimit: inputLimit, outputRelationsLimit: outputLimit, depth: Globals.lineageDepth }; + } + var queryParameter = {} + queryParameter["constraints"] = this.lineageOnDemandPayload["constraints"]; + this.fetchGraphData({ queryParam: queryParameter, 'legends': false }); + }, + validateInputOutputLimit: function(parentId, btnType) { + var inputRelationCount, outputRelationCount; + for (var guid in this.relationsOnDemand) { + if (parentId === guid && (btnType === "Input" || btnType === "Output")) { + inputRelationCount = this.relationsOnDemand[guid].inputRelationsCount ? this.relationsOnDemand[guid].inputRelationsCount : Globals.lineageNodeCount; + outputRelationCount = this.relationsOnDemand[guid].outputRelationsCount ? this.relationsOnDemand[guid].outputRelationsCount : Globals.lineageNodeCount; + } + } + return { + inputRelationCount: inputRelationCount, + outputRelationCount: outputRelationCount + }; + }, noLineage: function() { this.$('.fontLoader').hide(); this.$('.depth-container').hide(); @@ -387,6 +564,10 @@ define(['require', hideCheckForProcess: function() { this.$('.hideProcessContainer').hide(); }, + hideCheckForDeletedEntity: function() { + //This has been added to handle the scenario where hideDeletedEntityCheck should hide form filters when baseEntity is deleted. + this.$('.hideDeletedContainer').hide(); + }, renderLineageTypeSearch: function(data) { var that = this; return new Promise(function(resolve, reject) { @@ -395,7 +576,7 @@ define(['require', if (!_.isEmpty(data)) { _.each(data.guidEntityMap, function(obj, index) { var nodeData = that.LineageHelperRef.getNode(obj.guid); - if ((that.filterObj.isProcessHideCheck || that.filterObj.isDeletedEntityHideCheck) && nodeData && (nodeData.isProcess || nodeData.isDeleted)) { + if ((that.filterObj.isProcessHideCheck && obj && obj.isProcess) || (that.filterObj.isDeletedEntityHideCheck && obj && obj.isDeleted) || (Globals.isLineageOnDemandEnabled && obj && _.contains(["Input", "Output"], obj.btnType))) { return; } typeStr += ''; diff --git a/dashboardv3/package-lock.json b/dashboardv3/package-lock.json index 2b00b159e41..710137f9304 100644 --- a/dashboardv3/package-lock.json +++ b/dashboardv3/package-lock.json @@ -396,6 +396,16 @@ "integrity": "sha1-NWnt6Lo0MV+rmcPpLLBMciDeH6g=", "dev": true }, + "call-bind": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", + "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==", + "dev": true, + "requires": { + "function-bind": "^1.1.1", + "get-intrinsic": "^1.0.2" + } + }, "camel-case": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/camel-case/-/camel-case-3.0.0.tgz", @@ -691,9 +701,9 @@ "integrity": "sha512-ii0/r5f4sjKNTfh84Di+DpztYwqKhEyUlKoPrzUFfeSkWxjW49xU2QzO9qrPrNkpdI0XJkfzvmTu8V2Zylln6A==" }, "d3-color": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/d3-color/-/d3-color-1.4.1.tgz", - "integrity": "sha512-p2sTHSLCJI2QKunbGb7ocOh7DgTAn8IrLx21QRc/BSnodXM4sv6aLQlnfpvehFMLZEfBc6g9pH9SWQccFYfJ9Q==" + "version" : "3.1.0", + "resolved" :"https://registry.yarnpkg.com/d3-color/-/d3-color-3.1.0.tgz#395b2833dfac71507f12ac2f7af23bf819de24e2", + "integrity" : "sha512-zg/chbXyeBtMQ1LbD/WSoW2DpC3I0mpmPdW+ynRTj/x2DAWYrIY7qeZIHidozwV24m4iavr15lNwIwLxRmOxhA==" }, "d3-contour": { "version": "1.3.2", @@ -948,10 +958,10 @@ "integrity": "sha1-NC39EoN8kJdPM/HMCnha6lcNzcI=" }, "d3-color": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/d3-color/-/d3-color-1.0.3.tgz", - "integrity": "sha1-vHZD/KjlOoNH4vva/6I2eWtYUJs=" - }, + "version" : "3.1.0", + "resolved" :"https://registry.yarnpkg.com/d3-color/-/d3-color-3.1.0.tgz#395b2833dfac71507f12ac2f7af23bf819de24e2", + "integrity" : "sha512-zg/chbXyeBtMQ1LbD/WSoW2DpC3I0mpmPdW+ynRTj/x2DAWYrIY7qeZIHidozwV24m4iavr15lNwIwLxRmOxhA==" + }, "d3-dispatch": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/d3-dispatch/-/d3-dispatch-1.0.3.tgz", @@ -1523,6 +1533,25 @@ "integrity": "sha512-3t6rVToeoZfYSGd8YoLFR2DJkiQrIiUrGcjvFX2mDw3bn6k2OtwHN0TNCLbBO+w8qTvimhDkv+LSscbJY1vE6w==", "dev": true }, + "get-intrinsic": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.3.tgz", + "integrity": "sha512-QJVz1Tj7MS099PevUG5jvnt9tSkXN8K14dxQlikJuPt4uD9hHAHjLyLBiLR5zELelBdD9QNRAXZzsJx0WaDL9A==", + "dev": true, + "requires": { + "function-bind": "^1.1.1", + "has": "^1.0.3", + "has-symbols": "^1.0.3" + }, + "dependencies": { + "has-symbols": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", + "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==", + "dev": true + } + } + }, "get-stdin": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-4.0.1.tgz", @@ -1952,6 +1981,15 @@ "har-schema": "^2.0.0" } }, + "has": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", + "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", + "dev": true, + "requires": { + "function-bind": "^1.1.1" + } + }, "has-ansi": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz", @@ -2352,9 +2390,9 @@ } }, "lodash": { - "version": "4.17.20", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz", - "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==" + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, "longest": { "version": "1.0.1", @@ -2483,9 +2521,9 @@ } }, "minimatch": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", - "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "version": "3.0.8", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.8.tgz", + "integrity": "sha512-6FsRAQsxQ61mw+qP1ZzbL9Bc78x2p5OqNgNpnoAFLTrX8n5Kxph0CsnhmKKNXTWjXqU5L0pGPR7hYk+XWZr60Q==", "dev": true, "requires": { "brace-expansion": "^1.1.7" @@ -2711,6 +2749,12 @@ "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=", "dev": true }, + "object-inspect": { + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.2.tgz", + "integrity": "sha512-z+cPxW0QGUp0mcqcsgQyLVRDoXFQbXOwBaqyF7VIgI4TWNQsDHrBpUQslRmIfAoYWdYzs6UlKJtB2XJpTaNSpQ==", + "dev": true + }, "object-keys": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", @@ -2954,10 +2998,13 @@ "dev": true }, "qs": { - "version": "6.9.4", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.9.4.tgz", - "integrity": "sha512-A1kFqHekCTM7cz0udomYUoYNWjBebHm/5wzU/XqrBRBNWectVH0QIiN+NEcZ0Dte5hvzHwbr8+XQmguPhJ6WdQ==", - "dev": true + "version": "6.11.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz", + "integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==", + "dev": true, + "requires": { + "side-channel": "^1.0.4" + } }, "range-parser": { "version": "1.2.1", @@ -3084,9 +3131,9 @@ }, "dependencies": { "qs": { - "version": "6.5.2", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz", - "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==", + "version": "6.5.3", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.3.tgz", + "integrity": "sha512-qxXIEh4pCGfHICj1mAJQ2/2XVZkjCDTcEgfoSQxc/fYivUZxTkk7L3bDBJSoNrEzXI17oUO5Dp07ktqE5KzczA==", "dev": true } } @@ -3379,6 +3426,17 @@ "integrity": "sha512-BvE/TwpZX4FXExxOxZyRGQQv651MSwmWKZGqvmPcRIjDqWub67kTKuIMx43cZZrS/cBBzwBcNDWoFxt2XEFIpQ==", "dev": true }, + "side-channel": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz", + "integrity": "sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==", + "dev": true, + "requires": { + "call-bind": "^1.0.0", + "get-intrinsic": "^1.0.2", + "object-inspect": "^1.9.0" + } + }, "signal-exit": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.3.tgz", @@ -3887,4 +3945,4 @@ } } } -} \ No newline at end of file +} diff --git a/dashboardv3/public/img/entity-icon/expandBtn.svg b/dashboardv3/public/img/entity-icon/expandBtn.svg new file mode 100644 index 00000000000..88693e74b3a --- /dev/null +++ b/dashboardv3/public/img/entity-icon/expandBtn.svg @@ -0,0 +1,24 @@ + + + + + + + + diff --git a/dashboardv3/public/js/collection/VLineageList.js b/dashboardv3/public/js/collection/VLineageList.js index bda649220ee..4f980a737ef 100644 --- a/dashboardv3/public/js/collection/VLineageList.js +++ b/dashboardv3/public/js/collection/VLineageList.js @@ -42,7 +42,11 @@ define(['require', dataType: 'json' }, options); - return this.constructor.nonCrudOperation.call(this, url, 'GET', options); + if (options.compactLineageEnabled) { + return this.constructor.nonCrudOperation.call(this, url, 'POST', options); + } else { + return this.constructor.nonCrudOperation.call(this, url, 'GET', options); + } } }, //Static Class Members diff --git a/dashboardv3/public/js/external_lib/atlas-lineage/dist/index.js b/dashboardv3/public/js/external_lib/atlas-lineage/dist/index.js index b31ee491136..6a184ee6e8a 100644 --- a/dashboardv3/public/js/external_lib/atlas-lineage/dist/index.js +++ b/dashboardv3/public/js/external_lib/atlas-lineage/dist/index.js @@ -1 +1 @@ -!function(t,e){"object"==typeof exports&&"object"==typeof module?module.exports=e(require("platform"),require("dagreD3")):"function"==typeof define&&define.amd?define(["platform","dagreD3"],e):"object"==typeof exports?exports.LineageHelper=e(require("platform"),require("dagreD3")):t.LineageHelper=e(t.platform,t.dagreD3)}(window,(function(t,e){return function(t){var e={};function n(r){if(e[r])return e[r].exports;var i=e[r]={i:r,l:!1,exports:{}};return t[r].call(i.exports,i,i.exports,n),i.l=!0,i.exports}return n.m=t,n.c=e,n.d=function(t,e,r){n.o(t,e)||Object.defineProperty(t,e,{enumerable:!0,get:r})},n.r=function(t){"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(t,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(t,"__esModule",{value:!0})},n.t=function(t,e){if(1&e&&(t=n(t)),8&e)return t;if(4&e&&"object"==typeof t&&t&&t.__esModule)return t;var r=Object.create(null);if(n.r(r),Object.defineProperty(r,"default",{enumerable:!0,value:t}),2&e&&"string"!=typeof t)for(var i in t)n.d(r,i,function(e){return t[e]}.bind(null,i));return r},n.n=function(t){var e=t&&t.__esModule?function(){return t.default}:function(){return t};return n.d(e,"a",e),e},n.o=function(t,e){return Object.prototype.hasOwnProperty.call(t,e)},n.p="",n(n.s=10)}([,function(e,n){e.exports=t},function(t,n){t.exports=e},,,,,,,function(t,e,n){},function(t,e,n){"use strict";n.r(e),n.d(e,"default",(function(){return Br}));var r=n(2),i=n.n(r);function o(){}var a=function(t){return null==t?o:function(){return this.querySelector(t)}};function s(){return[]}var u=function(t){return null==t?s:function(){return this.querySelectorAll(t)}},l=function(t){return function(){return this.matches(t)}},c=function(t){return new Array(t.length)};function h(t,e){this.ownerDocument=t.ownerDocument,this.namespaceURI=t.namespaceURI,this._next=null,this._parent=t,this.__data__=e}h.prototype={constructor:h,appendChild:function(t){return this._parent.insertBefore(t,this._next)},insertBefore:function(t,e){return this._parent.insertBefore(t,e)},querySelector:function(t){return this._parent.querySelector(t)},querySelectorAll:function(t){return this._parent.querySelectorAll(t)}};function f(t,e,n,r,i,o){for(var a,s=0,u=e.length,l=o.length;se?1:t>=e?0:NaN}var g="http://www.w3.org/1999/xhtml",v={svg:"http://www.w3.org/2000/svg",xhtml:g,xlink:"http://www.w3.org/1999/xlink",xml:"http://www.w3.org/XML/1998/namespace",xmlns:"http://www.w3.org/2000/xmlns/"},y=function(t){var e=t+="",n=e.indexOf(":");return n>=0&&"xmlns"!==(e=t.slice(0,n))&&(t=t.slice(n+1)),v.hasOwnProperty(e)?{space:v[e],local:t}:t};function m(t){return function(){this.removeAttribute(t)}}function w(t){return function(){this.removeAttributeNS(t.space,t.local)}}function b(t,e){return function(){this.setAttribute(t,e)}}function x(t,e){return function(){this.setAttributeNS(t.space,t.local,e)}}function k(t,e){return function(){var n=e.apply(this,arguments);null==n?this.removeAttribute(t):this.setAttribute(t,n)}}function E(t,e){return function(){var n=e.apply(this,arguments);null==n?this.removeAttributeNS(t.space,t.local):this.setAttributeNS(t.space,t.local,n)}}var T=function(t){return t.ownerDocument&&t.ownerDocument.defaultView||t.document&&t||t.defaultView};function N(t){return function(){this.style.removeProperty(t)}}function O(t,e,n){return function(){this.style.setProperty(t,e,n)}}function S(t,e,n){return function(){var r=e.apply(this,arguments);null==r?this.style.removeProperty(t):this.style.setProperty(t,r,n)}}function P(t,e){return t.style.getPropertyValue(e)||T(t).getComputedStyle(t,null).getPropertyValue(e)}function A(t){return function(){delete this[t]}}function M(t,e){return function(){this[t]=e}}function z(t,e){return function(){var n=e.apply(this,arguments);null==n?delete this[t]:this[t]=n}}function j(t){return t.trim().split(/^|\s+/)}function D(t){return t.classList||new B(t)}function B(t){this._node=t,this._names=j(t.getAttribute("class")||"")}function C(t,e){for(var n=D(t),r=-1,i=e.length;++r=0&&(this._names.splice(e,1),this._node.setAttribute("class",this._names.join(" ")))},contains:function(t){return this._names.indexOf(t)>=0}};function q(){this.textContent=""}function U(t){return function(){this.textContent=t}}function F(t){return function(){var e=t.apply(this,arguments);this.textContent=null==e?"":e}}function H(){this.innerHTML=""}function V(t){return function(){this.innerHTML=t}}function X(t){return function(){var e=t.apply(this,arguments);this.innerHTML=null==e?"":e}}function Y(){this.nextSibling&&this.parentNode.appendChild(this)}function Z(){this.previousSibling&&this.parentNode.insertBefore(this,this.parentNode.firstChild)}function W(t){return function(){var e=this.ownerDocument,n=this.namespaceURI;return n===g&&e.documentElement.namespaceURI===g?e.createElement(t):e.createElementNS(n,t)}}function K(t){return function(){return this.ownerDocument.createElementNS(t.space,t.local)}}var Q=function(t){var e=y(t);return(e.local?K:W)(e)};function J(){return null}function tt(){var t=this.parentNode;t&&t.removeChild(this)}function et(){var t=this.cloneNode(!1),e=this.parentNode;return e?e.insertBefore(t,this.nextSibling):t}function nt(){var t=this.cloneNode(!0),e=this.parentNode;return e?e.insertBefore(t,this.nextSibling):t}var rt={},it=null;"undefined"!=typeof document&&("onmouseenter"in document.documentElement||(rt={mouseenter:"mouseover",mouseleave:"mouseout"}));function ot(t,e,n){return t=at(t,e,n),function(e){var n=e.relatedTarget;n&&(n===this||8&n.compareDocumentPosition(this))||t.call(this,e)}}function at(t,e,n){return function(r){var i=it;it=r;try{t.call(this,this.__data__,e,n)}finally{it=i}}}function st(t){return t.trim().split(/^|\s+/).map((function(t){var e="",n=t.indexOf(".");return n>=0&&(e=t.slice(n+1),t=t.slice(0,n)),{type:t,name:e}}))}function ut(t){return function(){var e=this.__on;if(e){for(var n,r=0,i=-1,o=e.length;r=k&&(k=x+1);!(b=_[k])&&++k=0;)(r=i[o])&&(a&&4^r.compareDocumentPosition(a)&&a.parentNode.insertBefore(r,a),a=r);return this},sort:function(t){function e(e,n){return e&&n?t(e.__data__,n.__data__):!e-!n}t||(t=d);for(var n=this._groups,r=n.length,i=new Array(r),o=0;o1?this.each((null==e?N:"function"==typeof e?S:O)(t,e,null==n?"":n)):P(this.node(),t)},property:function(t,e){return arguments.length>1?this.each((null==e?A:"function"==typeof e?z:M)(t,e)):this.node()[t]},classed:function(t,e){var n=j(t+"");if(arguments.length<2){for(var r=D(this.node()),i=-1,o=n.length;++i=0&&(n=t.slice(r+1),t=t.slice(0,r)),t&&!e.hasOwnProperty(t))throw new Error("unknown type: "+t);return{type:t,name:n}}))}function Nt(t,e){for(var n,r=0,i=t.length;r0)for(var n,r,i=new Array(n),o=0;o=0&&e._call.call(null,t),e=e._next;--Ut}()}finally{Ut=0,function(){var t,e,n=Bt,r=1/0;for(;n;)n._call?(r>n._time&&(r=n._time),t=n,n=n._next):(e=n._next,n._next=null,n=t?t._next=e:Bt=e);Ct=t,ne(r)}(),Vt=0}}function ee(){var t=Yt.now(),e=t-$t;e>1e3&&(Xt-=e,$t=t)}function ne(t){Ut||(Ft&&(Ft=clearTimeout(Ft)),t-Vt>24?(t<1/0&&(Ft=setTimeout(te,t-Yt.now()-Xt)),Ht&&(Ht=clearInterval(Ht))):(Ht||($t=Yt.now(),Ht=setInterval(ee,1e3)),Ut=1,Zt(te)))}Qt.prototype=Jt.prototype={constructor:Qt,restart:function(t,e,n){if("function"!=typeof t)throw new TypeError("callback is not a function");n=(null==n?Wt():+n)+(null==e?0:+e),this._next||Ct===this||(Ct?Ct._next=this:Bt=this,Ct=this),this._call=t,this._time=n,ne()},stop:function(){this._call&&(this._call=null,this._time=1/0,ne())}};var re=function(t,e,n){var r=new Qt;return e=null==e?0:+e,r.restart((function(n){r.stop(),t(n+e)}),e,n),r},ie=St("start","end","cancel","interrupt"),oe=[],ae=function(t,e,n,r,i,o){var a=t.__transition;if(a){if(n in a)return}else t.__transition={};!function(t,e,n){var r,i=t.__transition;function o(u){var l,c,h,f;if(1!==n.state)return s();for(l in i)if((f=i[l]).name===n.name){if(3===f.state)return re(o);4===f.state?(f.state=6,f.timer.stop(),f.on.call("interrupt",t,t.__data__,f.index,f.group),delete i[l]):+l0)throw new Error("too late; already scheduled");return n}function ue(t,e){var n=le(t,e);if(n.state>3)throw new Error("too late; already running");return n}function le(t,e){var n=t.__transition;if(!n||!(n=n[e]))throw new Error("transition not found");return n}var ce,he,fe,pe,de=function(t,e){var n,r,i,o=t.__transition,a=!0;if(o){for(i in e=null==e?null:e+"",o)(n=o[i]).name===e?(r=n.state>2&&n.state<5,n.state=6,n.timer.stop(),n.on.call(r?"interrupt":"cancel",t,t.__data__,n.index,n.group),delete o[i]):a=!1;a&&delete t.__transition}},ge=function(t,e){return t=+t,e=+e,function(n){return t*(1-n)+e*n}},ve=180/Math.PI,ye={translateX:0,translateY:0,rotate:0,skewX:0,scaleX:1,scaleY:1},me=function(t,e,n,r,i,o){var a,s,u;return(a=Math.sqrt(t*t+e*e))&&(t/=a,e/=a),(u=t*n+e*r)&&(n-=t*u,r-=e*u),(s=Math.sqrt(n*n+r*r))&&(n/=s,r/=s,u/=s),t*r180?e+=360:e-t>180&&(t+=360),o.push({i:n.push(i(n)+"rotate(",null,r)-2,x:ge(t,e)})):e&&n.push(i(n)+"rotate("+e+r)}(o.rotate,a.rotate,s,u),function(t,e,n,o){t!==e?o.push({i:n.push(i(n)+"skewX(",null,r)-2,x:ge(t,e)}):e&&n.push(i(n)+"skewX("+e+r)}(o.skewX,a.skewX,s,u),function(t,e,n,r,o,a){if(t!==n||e!==r){var s=o.push(i(o)+"scale(",null,",",null,")");a.push({i:s-4,x:ge(t,n)},{i:s-2,x:ge(e,r)})}else 1===n&&1===r||o.push(i(o)+"scale("+n+","+r+")")}(o.scaleX,o.scaleY,a.scaleX,a.scaleY,s,u),o=a=null,function(t){for(var e,n=-1,r=u.length;++n>8&15|e>>4&240,e>>4&15|240&e,(15&e)<<4|15&e,1):8===n?Fe(e>>24&255,e>>16&255,e>>8&255,(255&e)/255):4===n?Fe(e>>12&15|e>>8&240,e>>8&15|e>>4&240,e>>4&15|240&e,((15&e)<<4|15&e)/255):null):(e=ze.exec(t))?new Ve(e[1],e[2],e[3],1):(e=je.exec(t))?new Ve(255*e[1]/100,255*e[2]/100,255*e[3]/100,1):(e=De.exec(t))?Fe(e[1],e[2],e[3],e[4]):(e=Be.exec(t))?Fe(255*e[1]/100,255*e[2]/100,255*e[3]/100,e[4]):(e=Ce.exec(t))?We(e[1],e[2]/100,e[3]/100,1):(e=Ie.exec(t))?We(e[1],e[2]/100,e[3]/100,e[4]):Le.hasOwnProperty(t)?Ue(Le[t]):"transparent"===t?new Ve(NaN,NaN,NaN,0):null}function Ue(t){return new Ve(t>>16&255,t>>8&255,255&t,1)}function Fe(t,e,n,r){return r<=0&&(t=e=n=NaN),new Ve(t,e,n,r)}function He(t){return t instanceof Oe||(t=qe(t)),t?new Ve((t=t.rgb()).r,t.g,t.b,t.opacity):new Ve}function $e(t,e,n,r){return 1===arguments.length?He(t):new Ve(t,e,n,null==r?1:r)}function Ve(t,e,n,r){this.r=+t,this.g=+e,this.b=+n,this.opacity=+r}function Xe(){return"#"+Ze(this.r)+Ze(this.g)+Ze(this.b)}function Ye(){var t=this.opacity;return(1===(t=isNaN(t)?1:Math.max(0,Math.min(1,t)))?"rgb(":"rgba(")+Math.max(0,Math.min(255,Math.round(this.r)||0))+", "+Math.max(0,Math.min(255,Math.round(this.g)||0))+", "+Math.max(0,Math.min(255,Math.round(this.b)||0))+(1===t?")":", "+t+")")}function Ze(t){return((t=Math.max(0,Math.min(255,Math.round(t)||0)))<16?"0":"")+t.toString(16)}function We(t,e,n,r){return r<=0?t=e=n=NaN:n<=0||n>=1?t=e=NaN:e<=0&&(t=NaN),new Qe(t,e,n,r)}function Ke(t){if(t instanceof Qe)return new Qe(t.h,t.s,t.l,t.opacity);if(t instanceof Oe||(t=qe(t)),!t)return new Qe;if(t instanceof Qe)return t;var e=(t=t.rgb()).r/255,n=t.g/255,r=t.b/255,i=Math.min(e,n,r),o=Math.max(e,n,r),a=NaN,s=o-i,u=(o+i)/2;return s?(a=e===o?(n-r)/s+6*(n0&&u<1?0:a,new Qe(a,s,u,t.opacity)}function Qe(t,e,n,r){this.h=+t,this.s=+e,this.l=+n,this.opacity=+r}function Je(t,e,n){return 255*(t<60?e+(n-e)*t/60:t<180?n:t<240?e+(n-e)*(240-t)/60:e)}function tn(t,e,n,r,i){var o=t*t,a=o*t;return((1-3*t+3*o-a)*e+(4-6*o+3*a)*n+(1+3*t+3*o-3*a)*r+a*i)/6}Te(Oe,qe,{copy:function(t){return Object.assign(new this.constructor,this,t)},displayable:function(){return this.rgb().displayable()},hex:Re,formatHex:Re,formatHsl:function(){return Ke(this).formatHsl()},formatRgb:Ge,toString:Ge}),Te(Ve,$e,Ne(Oe,{brighter:function(t){return t=null==t?1/.7:Math.pow(1/.7,t),new Ve(this.r*t,this.g*t,this.b*t,this.opacity)},darker:function(t){return t=null==t?.7:Math.pow(.7,t),new Ve(this.r*t,this.g*t,this.b*t,this.opacity)},rgb:function(){return this},displayable:function(){return-.5<=this.r&&this.r<255.5&&-.5<=this.g&&this.g<255.5&&-.5<=this.b&&this.b<255.5&&0<=this.opacity&&this.opacity<=1},hex:Xe,formatHex:Xe,formatRgb:Ye,toString:Ye})),Te(Qe,(function(t,e,n,r){return 1===arguments.length?Ke(t):new Qe(t,e,n,null==r?1:r)}),Ne(Oe,{brighter:function(t){return t=null==t?1/.7:Math.pow(1/.7,t),new Qe(this.h,this.s,this.l*t,this.opacity)},darker:function(t){return t=null==t?.7:Math.pow(.7,t),new Qe(this.h,this.s,this.l*t,this.opacity)},rgb:function(){var t=this.h%360+360*(this.h<0),e=isNaN(t)||isNaN(this.s)?0:this.s,n=this.l,r=n+(n<.5?n:1-n)*e,i=2*n-r;return new Ve(Je(t>=240?t-240:t+120,i,r),Je(t,i,r),Je(t<120?t+240:t-120,i,r),this.opacity)},displayable:function(){return(0<=this.s&&this.s<=1||isNaN(this.s))&&0<=this.l&&this.l<=1&&0<=this.opacity&&this.opacity<=1},formatHsl:function(){var t=this.opacity;return(1===(t=isNaN(t)?1:Math.max(0,Math.min(1,t)))?"hsl(":"hsla(")+(this.h||0)+", "+100*(this.s||0)+"%, "+100*(this.l||0)+"%"+(1===t?")":", "+t+")")}}));var en=function(t){return function(){return t}};function nn(t,e){return function(n){return t+n*e}}function rn(t){return 1==(t=+t)?on:function(e,n){return n-e?function(t,e,n){return t=Math.pow(t,n),e=Math.pow(e,n)-t,n=1/n,function(r){return Math.pow(t+r*e,n)}}(e,n,t):en(isNaN(e)?n:e)}}function on(t,e){var n=e-t;return n?nn(t,n):en(isNaN(t)?e:t)}var an=function t(e){var n=rn(e);function r(t,e){var r=n((t=$e(t)).r,(e=$e(e)).r),i=n(t.g,e.g),o=n(t.b,e.b),a=on(t.opacity,e.opacity);return function(e){return t.r=r(e),t.g=i(e),t.b=o(e),t.opacity=a(e),t+""}}return r.gamma=t,r}(1);function sn(t){return function(e){var n,r,i=e.length,o=new Array(i),a=new Array(i),s=new Array(i);for(n=0;n=1?(n=1,e-1):Math.floor(n*e),i=t[r],o=t[r+1],a=r>0?t[r-1]:2*i-o,s=ro&&(i=e.slice(o,i),s[a]?s[a]+=i:s[++a]=i),(n=n[0])===(r=r[0])?s[a]?s[a]+=r:s[++a]=r:(s[++a]=null,u.push({i:a,x:ge(n,r)})),o=ln.lastIndex;return o=0&&(t=t.slice(0,e)),!t||"start"===t}))}(e)?se:ue;return function(){var a=o(this,t),s=a.on;s!==r&&(i=(r=s).copy()).on(e,n),a.on=i}}var Sn=yt.prototype.constructor;function Pn(t){return function(){this.style.removeProperty(t)}}function An(t,e,n){return function(r){this.style.setProperty(t,e.call(this,r),n)}}function Mn(t,e,n){var r,i;function o(){var o=e.apply(this,arguments);return o!==i&&(r=(i=o)&&An(t,o,n)),r}return o._value=e,o}function zn(t){return function(e){this.textContent=t.call(this,e)}}function jn(t){var e,n;function r(){var r=t.apply(this,arguments);return r!==n&&(e=(n=r)&&zn(r)),e}return r._value=t,r}var Dn=0;function Bn(t,e,n,r){this._groups=t,this._parents=e,this._name=n,this._id=r}function Cn(){return++Dn}var In=yt.prototype;Bn.prototype=function(t){return yt().transition(t)}.prototype={constructor:Bn,select:function(t){var e=this._name,n=this._id;"function"!=typeof t&&(t=a(t));for(var r=this._groups,i=r.length,o=new Array(i),s=0;sr?(r+i)/2:Math.min(0,r)||Math.max(0,i),a>o?(o+a)/2:Math.min(0,o)||Math.max(0,a))}var Qn=function(t){return function(){return t}};function Jn(t,e,n,r,i,o,a,s,u,l){this.target=t,this.type=e,this.subject=n,this.identifier=r,this.active=i,this.x=o,this.y=a,this.dx=s,this.dy=u,this._=l}function tr(){return!it.ctrlKey&&!it.button}function er(){return this.parentNode}function nr(t){return null==t?{x:it.x,y:it.y}:t}function rr(){return navigator.maxTouchPoints||"ontouchstart"in this}Jn.prototype.on=function(){var t=this._.on.apply(this._,arguments);return t===this._?this:t};var ir=function(){var t,e,n,r,i=tr,o=er,a=nr,s=rr,u={},l=St("start","drag","end"),c=0,h=0;function f(t){t.on("mousedown.drag",p).filter(s).on("touchstart.drag",v).on("touchmove.drag",y).on("touchend.drag touchcancel.drag",m).style("touch-action","none").style("-webkit-tap-highlight-color","rgba(0,0,0,0)")}function p(){if(!r&&i.apply(this,arguments)){var a=_("mouse",o.apply(this,arguments),Gt,this,arguments);a&&(mt(it.view).on("mousemove.drag",d,!0).on("mouseup.drag",g,!0),Mt(it.view),Pt(),n=!1,t=it.clientX,e=it.clientY,a("start"))}}function d(){if(At(),!n){var r=it.clientX-t,i=it.clientY-e;n=r*r+i*i>h}u.mouse("drag")}function g(){mt(it.view).on("mousemove.drag mouseup.drag",null),zt(it.view,n),At(),u.mouse("end")}function v(){if(i.apply(this,arguments)){var t,e,n=it.changedTouches,r=o.apply(this,arguments),a=n.length;for(t=0;t1e-6)if(Math.abs(c*s-u*l)>1e-6&&i){var f=n-o,p=r-a,d=s*s+u*u,g=f*f+p*p,v=Math.sqrt(d),y=Math.sqrt(h),m=i*Math.tan((or-Math.acos((d+h-g)/(2*v*y)))/2),_=m/y,w=m/v;Math.abs(_-1)>1e-6&&(this._+="L"+(t+_*l)+","+(e+_*c)),this._+="A"+i+","+i+",0,0,"+ +(c*f>l*p)+","+(this._x1=t+w*s)+","+(this._y1=e+w*u)}else this._+="L"+(this._x1=t)+","+(this._y1=e);else;},arc:function(t,e,n,r,i,o){t=+t,e=+e,o=!!o;var a=(n=+n)*Math.cos(r),s=n*Math.sin(r),u=t+a,l=e+s,c=1^o,h=o?r-i:i-r;if(n<0)throw new Error("negative radius: "+n);null===this._x1?this._+="M"+u+","+l:(Math.abs(this._x1-u)>1e-6||Math.abs(this._y1-l)>1e-6)&&(this._+="L"+u+","+l),n&&(h<0&&(h=h%ar+ar),h>sr?this._+="A"+n+","+n+",0,1,"+c+","+(t-a)+","+(e-s)+"A"+n+","+n+",0,1,"+c+","+(this._x1=u)+","+(this._y1=l):h>1e-6&&(this._+="A"+n+","+n+",0,"+ +(h>=or)+","+c+","+(this._x1=t+n*Math.cos(i))+","+(this._y1=e+n*Math.sin(i))))},rect:function(t,e,n,r){this._+="M"+(this._x0=this._x1=+t)+","+(this._y0=this._y1=+e)+"h"+ +n+"v"+ +r+"h"+-n+"Z"},toString:function(){return this._}};var cr=lr,hr=function(t){return function(){return t}};function fr(t){this._context=t}fr.prototype={areaStart:function(){this._line=0},areaEnd:function(){this._line=NaN},lineStart:function(){this._point=0},lineEnd:function(){(this._line||0!==this._line&&1===this._point)&&this._context.closePath(),this._line=1-this._line},point:function(t,e){switch(t=+t,e=+e,this._point){case 0:this._point=1,this._line?this._context.lineTo(t,e):this._context.moveTo(t,e);break;case 1:this._point=2;default:this._context.lineTo(t,e)}}};var pr=function(t){return new fr(t)};function dr(t){return t[0]}function gr(t){return t[1]}var vr=n(1),yr=n.n(vr),mr={entityStateReadOnly:{ACTIVE:!1,DELETED:!0,STATUS_ACTIVE:!1,STATUS_DELETED:!0}},_r={nodeArrowDistance:24,refreshGraphForSafari:function(t){t.edgeEl.each((function(t){var e=this,n=$(this).find("pattern");setTimeout((function(t){$(e).find("defs").append(n)}),500)}))},refreshGraphForIE:function(t){var e=t.edgePathEl,n=0;e.each((function(t){var e=$(this).find("marker");$(this).find("marker").remove();var r=this;++n,setTimeout((function(t){$(r).find("defs").append(e),0===--n&&(this.$(".fontLoader").hide(),this.$("svg").fadeTo(1e3,1))}),1e3)}))},dragNode:function(t){var e=this,n=t.g,r=t.svg,i=t.guid,o=(t.edgePathEl,{dragmove:function(t,e){var r=this,i=mt(t),o=n.node(e),a=o.x,s=o.y;o.x+=it.dx,o.y+=it.dy,i.attr("transform","translate("+o.x+","+o.y+")");var u=o.x-a,l=o.y-s;n.edges().forEach((function(t){if(t.v==e||t.w==e){var i=n.edge(t.v,t.w);r.translateEdge(i,u,l),mt(i.elem).select("path").attr("d",r.calcPoints(t))}}))},translateEdge:function(t,e,n){t.points.forEach((function(t){t.x=t.x+e,t.y=t.y+n}))},calcPoints:function(t){var e=n.edge(t.v,t.w),r=n.node(t.v),i=n.node(t.w),o=e.points.slice(1,e.points.length-1);e.points.slice(1,e.points.length-1);return o.unshift(this.intersectRect(r,o[0])),o.push(this.intersectRect(i,o[o.length-1])),function(){var t=dr,e=gr,n=hr(!0),r=null,i=pr,o=null;function a(a){var s,u,l,c=a.length,h=!1;for(null==r&&(o=i(l=cr())),s=0;s<=c;++s)!(sMath.abs(a)*c?(s<0&&(c=-c),h=0===s?0:c*a/s,f=c):(a<0&&(l=-l),h=l,f=0===a?0:l*s/a),{x:r+h,y:o+f}}}),a=ir().on("drag",(function(t){o.dragmove.call(o,this,t)})),s=ir().on("drag",(function(t){o.translateEdge(n.edge(t.v,t.w),it.dx,it.dy);var e=n.edge(t.v,t.w);mt(e.elem).select("path").attr("d",o.calcPoints(t))}));a(r.selectAll("g.node")),s(r.selectAll("g.edgePath"))},zoomIn:function(t){var e=t.svg,n=t.scaleFactor,r=void 0===n?1.3:n;this.d3Zoom.scaleBy(e.transition().duration(750),r)},zoomOut:function(t){var e=t.svg,n=t.scaleFactor,r=void 0===n?.8:n;this.d3Zoom.scaleBy(e.transition().duration(750),r)},zoom:function(t){var e=t.svg,n=t.xa,r=t.ya,i=t.scale;e.transition().duration(750).call(this.d3Zoom.transform,Fn.translate(n,r).scale(i))},fitToScreen:function(t){var e=t.svg,n=e.node(),r=n.getBBox(),i=n.parentElement,o=i.clientWidth,a=i.clientHeight,s=r.width,u=r.height,l=r.x+s/2,c=r.y+u/2,h=(h||.95)/Math.max(s/o,u/a),f=o/2-h*l,p=a/2-h*c;this.zoom({svg:e,xa:f,ya:p,scale:h})},centerNode:function(t){var e=t.guid,n=t.g,r=t.svg,i=t.svgGroupEl,o=(t.edgePathEl,t.width),a=t.height,s=t.fitToScreen,u=t.onCenterZoomed,l=t.isSelected;this.d3Zoom=function(){var t,e,n=Vn,r=Xn,i=Kn,o=Zn,a=Wn,s=[0,1/0],u=[[-1/0,-1/0],[1/0,1/0]],l=250,c=It,h=St("start","zoom","end"),f=0;function p(t){t.property("__zoom",Yn).on("wheel.zoom",w).on("mousedown.zoom",b).on("dblclick.zoom",x).filter(a).on("touchstart.zoom",k).on("touchmove.zoom",E).on("touchend.zoom touchcancel.zoom",T).style("touch-action","none").style("-webkit-tap-highlight-color","rgba(0,0,0,0)")}function d(t,e){return(e=Math.max(s[0],Math.min(s[1],e)))===t.k?t:new Un(e,t.x,t.y)}function g(t,e,n){var r=e[0]-n[0]*t.k,i=e[1]-n[1]*t.k;return r===t.x&&i===t.y?t:new Un(t.k,r,i)}function v(t){return[(+t[0][0]+ +t[1][0])/2,(+t[0][1]+ +t[1][1])/2]}function y(t,e,n){t.on("start.zoom",(function(){m(this,arguments).start()})).on("interrupt.zoom end.zoom",(function(){m(this,arguments).end()})).tween("zoom",(function(){var t=this,i=arguments,o=m(t,i),a=r.apply(t,i),s=null==n?v(a):"function"==typeof n?n.apply(t,i):n,u=Math.max(a[1][0]-a[0][0],a[1][1]-a[0][1]),l=t.__zoom,h="function"==typeof e?e.apply(t,i):e,f=c(l.invert(s).concat(u/l.k),h.invert(s).concat(u/h.k));return function(t){if(1===t)t=h;else{var e=f(t),n=u/e[2];t=new Un(n,s[0]-e[0]*n,s[1]-e[1]*n)}o.zoom(null,t)}}))}function m(t,e,n){return!n&&t.__zooming||new _(t,e)}function _(t,e){this.that=t,this.args=e,this.active=0,this.extent=r.apply(t,e),this.taps=0}function w(){if(n.apply(this,arguments)){var t=m(this,arguments),e=this.__zoom,r=Math.max(s[0],Math.min(s[1],e.k*Math.pow(2,o.apply(this,arguments)))),a=Gt(this);if(t.wheel)t.mouse[0][0]===a[0]&&t.mouse[0][1]===a[1]||(t.mouse[1]=e.invert(t.mouse[0]=a)),clearTimeout(t.wheel);else{if(e.k===r)return;t.mouse=[a,e.invert(a)],de(this),t.start()}$n(),t.wheel=setTimeout(l,150),t.zoom("mouse",i(g(d(e,r),t.mouse[0],t.mouse[1]),t.extent,u))}function l(){t.wheel=null,t.end()}}function b(){if(!e&&n.apply(this,arguments)){var t=m(this,arguments,!0),r=mt(it.view).on("mousemove.zoom",l,!0).on("mouseup.zoom",c,!0),o=Gt(this),a=it.clientX,s=it.clientY;Mt(it.view),Hn(),t.mouse=[o,this.__zoom.invert(o)],de(this),t.start()}function l(){if($n(),!t.moved){var e=it.clientX-a,n=it.clientY-s;t.moved=e*e+n*n>f}t.zoom("mouse",i(g(t.that.__zoom,t.mouse[0]=Gt(t.that),t.mouse[1]),t.extent,u))}function c(){r.on("mousemove.zoom mouseup.zoom",null),zt(it.view,t.moved),$n(),t.end()}}function x(){if(n.apply(this,arguments)){var t=this.__zoom,e=Gt(this),o=t.invert(e),a=t.k*(it.shiftKey?.5:2),s=i(g(d(t,a),e,o),r.apply(this,arguments),u);$n(),l>0?mt(this).transition().duration(l).call(y,s,e):mt(this).call(p.transform,s)}}function k(){if(n.apply(this,arguments)){var e,r,i,o,a=it.touches,s=a.length,u=m(this,arguments,it.changedTouches.length===s);for(Hn(),r=0;rg[id='"+e+"']"),h=(this.d3Zoom.scaleExtent([.01,50]).on("zoom",(function(){i.attr("transform",it.transform)})),null),f=null;if(c.empty()){if(s)return void this.fitToScreen({svg:r});h=n.graph().width/2,f=n.graph().height/2}else{var p=c.attr("transform").replace(/[^0-9\-.,]/g,"").split(",");h=p[0],f=p[1]}var d=-(1.2*h-o/2),g=-(1.2*f-a/2);this.zoom({svg:r,xa:d,ya:g,scale:1.2}),l?r.transition().duration(750).call(this.d3Zoom.transform,Fn.translate(d,g).scale(1.2)):r.call(this.d3Zoom.transform,Fn.translate(d,g).scale(1.2)),u&&u({newScale:1.2,newTranslate:[d,g],d3Zoom:this.d3Zoom,selectedNodeEl:c})},getToolTipDirection:function(t){var e=t.el,n=mt("body").node().getBoundingClientRect().width,r=mt(e).node().getBoundingClientRect(),i="e";return n-r.left<330?(i=n-r.left<330&&r.top<400?"sw":"w",n-r.left<330&&r.top>600&&(i="nw")):r.top>600?(i=n-r.left<330&&r.top>600?"nw":"n",r.left<50&&(i="ne")):r.top<400&&(i=r.left<50?"se":"s"),i},onHoverFade:function(t){var e=t.svg,n=t.g,r=t.mouseenter,i=t.nodesToHighlight,o=t.hoveredNode;return function(t){var i=e.selectAll(".node"),a=e.selectAll(".edgePath");if(r){e.classed("hover",!0);var s=n.successors(o),u=n.predecessors(o);t=s.concat(u);i.classed("hover-active-node",(function(e,n,r){return!!function(t,e,n){if(t===n||e&&e.length&&-1!=e.indexOf(n))return!0}(o,t,e)})),a.classed("hover-active-path",(function(t){return!!(t.v===o||t.w===o?1:0)}))}else e.classed("hover",!1),i.classed("hover-active-node",!1),a.classed("hover-active-path",!1)}(i)},getBaseUrl:function(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:window.location.pathname;return t.replace(/\/[\w-]+.(jsp|html)|\/+$/gi,"")},getEntityIconPath:function(t){var e=t.entityData,n=t.errorUrl,r=t.imgBasePath,i=this.getBaseUrl()+(r||"/img/entity-icon/");if(e){var o=function(t){return i+(mr.entityStateReadOnly[l]?"disabled/"+t:t)},a=function(){return c?mr.entityStateReadOnly[l]?i+"disabled/process.png":i+"process.png":mr.entityStateReadOnly[l]?i+"disabled/table.png":i+"table.png"},s=e.typeName,u=e.serviceType,l=e.status,c=e.isProcess;if(n){if(n.indexOf("table.png")>-1||n.indexOf("process.png")>-1)return null;var h=!(!n||!n.match("entity-icon/"+s+".png|disabled/"+s+".png"));return u&&h?o(u+".png"):a()}return s?o(s+".png"):u?o(u+".png"):a()}},base64Encode:function(t,e){var n=new FileReader;n.addEventListener("load",(function(){return e(n.result)})),n.readAsDataURL(t)},imgShapeRender:function(t,e,n,r){var i=r.dagreD3,o=r.defsEl,a=r.imgBasePath,s=r.guid,u=r.isRankdirToBottom,l=this,c=s,h=this.getEntityIconPath({entityData:n,imgBasePath:a}),f=h.split("/").pop();if(void 0===this.imageObject&&(this.imageObject={}),n.isDeleted&&(f="deleted_"+f),n.id==c)var p=!0;var d=t.append("circle").attr("fill","url(#img_"+f+")").attr("r",u?"30px":"24px").attr("data-stroke",n.id).attr("stroke-width","2px").attr("class","nodeImage "+(p?"currentNode":n.isProcess?"process":"node"));if(p&&d.attr("stroke","#fb4200"),!0===n.isIncomplete){t.attr("class","node isIncomplete show"),t.insert("rect").attr("x","-5").attr("y","-23").attr("width","14").attr("height","16").attr("fill","url(#img_hourglass.svg)").attr("data-stroke",n.id).attr("stroke-width","2px");g({imgName:"hourglass.svg",imageIconPath:"/img/entity-icon/hourglass.svg",leftPosition:"0",topPosition:"0",width:"12",height:"14"})}function g(t){o.select('pattern[id="img_'+t.imgName+'"]').empty()&&o.append("pattern").attr("x","0%").attr("y","0%").attr("patternUnits","objectBoundingBox").attr("id","img_"+t.imgName).attr("width","100%").attr("height","100%").append("image").attr("href",(function(e){var r=this;if(n){!function e(i){var o=i.imagePath,a={url:o,method:"GET",cache:!0};d.attr("data-iconpath",o);var s=new XMLHttpRequest;s.onreadystatechange=function(){if(4===s.readyState)if(200===s.status)"IE"!==yr.a.name?l.base64Encode(this.response,(function(e){l.imageObject[t.imageIconPath]=e,mt(r).attr("xlink:href",e)})):l.imageObject[t.imageIconPath]=o,t.imageIconPath!==d.attr("data-iconpath")&&d.attr("data-iconpathorigin",t.imageIconPath);else if(404===s.status){var i=l.getEntityIconPath({entityData:n,errorUrl:o});if(null===i){var a=mt(r.parentElement);a.select("image").remove(),a.attr("patternContentUnits","objectBoundingBox").append("circle").attr("r","24px").attr("fill","#e8e8e8")}else e({imagePath:i})}},s.responseType="blob",s.open(a.method,a.url,!0),s.send(null)}({imagePath:t.imageIconPath})}})).attr("x",t.leftPosition).attr("y",t.topPosition).attr("width",t.width).attr("height",t.height)}return g({imgName:f,imageIconPath:h,leftPosition:u?"11":"4",topPosition:u?"20":p?"3":"4",width:"40",height:"40"}),n.intersect=function(t){return i.intersect.circle(n,p?l.nodeArrowDistance+3:l.nodeArrowDistance,t)},d},arrowPointRender:function(t,e,n,r,i){var o=i.dagreD3,a=t.node(),s=a?a.parentNode:t;mt(s).select("path.path").attr("marker-end","url(#"+e+")");var u=t.append("marker").attr("id",e).attr("viewBox","0 0 10 10").attr("refX",8).attr("refY",5).attr("markerUnits","strokeWidth").attr("markerWidth",4).attr("markerHeight",4).attr("orient","auto").append("path").attr("d","M 0 0 L 10 5 L 0 10 z").style("fill",n.styleObj.stroke);o.util.applyStyle(u,n[r+"Style"])},saveSvg:function(t){var e=t.svg,n=t.width,r=t.height,i=t.downloadFileName,o=t.onExportLineage,a=this,s=e.clone(!0).node();setTimeout((function(){"Firefox"===yr.a.name&&(s.setAttribute("width",n),s.setAttribute("height",r));var t=mt("body").append("div");t.classed("hidden-svg",!0),t.node().appendChild(s);var e=mt(".hidden-svg svg");e.select("g").attr("transform","scale(1)"),e.select("foreignObject").remove();var u=150,l=150,c=s.getBBox().width+u,h=s.getBBox().height+l,f=s.getBBox().x,p=s.getBBox().y;s.attributes.viewBox.value=f+","+p+","+c+","+h;var d=document.createElement("canvas");d.id="canvas",d.style.display="none",d.width=1*s.getBBox().width+u,d.height=1*s.getBBox().height+l,mt("body").node().appendChild(d);var g=d.getContext("2d"),v=(new XMLSerializer).serializeToString(s),y=window.URL||window.webkitURL||window;g.fillStyle="#FFFFFF",g.fillRect(0,0,d.width,d.height),g.strokeRect(0,0,d.width,d.height),g.restore();var m=new Image(d.width,d.height),_=new Blob([v],{type:"image/svg+xml;base64"});"Safari"===yr.a.name&&(_=new Blob([v],{type:"image/svg+xml"})),g.drawImage(m,50,50,d.width,d.height);var w=y.createObjectURL(_);m.onload=function(){try{var e=document.createElement("a");e.download=i,document.body.appendChild(e),g.drawImage(m,50,50,d.width,d.height),d.toBlob((function(t){t?(e.href=y.createObjectURL(t),t.size>1e7&&o({status:"failed",message:"The Image size is huge, please open the image in a browser!"}),e.click(),o({status:"Success",message:"Successful"}),"Safari"===yr.a.name&&a.refreshGraphForSafari({edgeEl:a.$("svg g.node")})):o({status:"failed",message:"There was an error in downloading Lineage!"})}),"image/png"),t.remove(),d.remove()}catch(t){o({status:"failed",message:"There was an error in downloading Lineage!"})}},m.src=w}),0)}};function wr(t,e){var n=Object.keys(t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(t);e&&(r=r.filter((function(e){return Object.getOwnPropertyDescriptor(t,e).enumerable}))),n.push.apply(n,r)}return n}function br(t){for(var e=1;e0&&void 0!==arguments[0]?arguments[0]:{},i=r.entityData,o=r.errorUrl,a=this.getBaseUrl(window.location.pathname)+Globals.entityImgPath;function s(t){return a+(mr.entityStateReadOnly[e]?"disabled/"+t:t)}function u(){return i.isProcess?mr.entityStateReadOnly[e]?a+"disabled/process.png":a+"process.png":mr.entityStateReadOnly[e]?a+"disabled/table.png":a+"table.png"}if(i&&(n=i.typeName,t=i&&i.serviceType,e=i&&i.status),i){if(o){var l=!(!o||!o.match("entity-icon/"+n+".png|disabled/"+n+".png"));return t&&l?s(t+".png"):u()}return i.typeName?s(i.typeName+".png"):u()}},isProcess:function(t){var e=t.typeName,n=t.superTypes;t.entityDef;return"Process"==e||n.indexOf("Process")>-1},isDeleted:function(t){if(void 0!==t)return mr.entityStateReadOnly[t.status]},isNodeToBeUpdated:function(t,e){var n=e.isProcessHideCheck,r=e.isDeletedEntityHideCheck,i={isProcess:n&&t.isProcess,isDeleted:r&&t.isDeleted};return i.update=i.isProcess||i.isDeleted,i},getServiceType:function(t){var e=t.typeName,n=t.entityDef,r=null;return e&&n&&(r=n.serviceType||null),r},getEntityDef:function(t){var e=t.typeName,n=t.entityDefCollection,r=null;return e&&(r=n.find((function(t){return t.name==e}))),r},getNestedSuperTypes:function(t){var e=t.entityDef,n=t.entityDefCollection,r=new Set;return function t(e,n){e&&e.superTypes&&e.superTypes.length&&e.superTypes.forEach((function(e){r.add(e);var i=n.find((function(t){t.name}));i&&t(i,n)}))}(e,n),Array.from(r)},generateData:function(t){var e=this,n=t.data,r=void 0===n?{}:n,i=t.filterObj,o=t.entityDefCollection,a=t.g,s=t.guid,u=t.setGraphEdge,l=t.setGraphNode;return new Promise((function(t,n){try{var c=r.relations||{},h=r.guidEntityMap||{},f=i.isProcessHideCheck||i.isDeletedEntityHideCheck,p={fill:"none",stroke:"#ffb203",width:3},d=function(t){if(t){if(t.updatedValues)return t;var n=t.displayText?t.displayText:" ",r=Object.assign(t,{shape:"img",updatedValues:!0,label:n.trunc(18),toolTipLabel:n,id:t.guid,isLineage:!0,isIncomplete:t.isIncomplete,entityDef:e.getEntityDef({typeName:t.typeName,entityDefCollection:o})});return r.serviceType=e.getServiceType(r),r.superTypes=e.getNestedSuperTypes(br(br({},r),{},{entityDefCollection:o})),r.isProcess=e.isProcess(r),r.isDeleted=e.isDeleted(r),r}},g=function(t){return"fill:"+t.fill+";stroke:"+t.stroke+";stroke-width:"+t.width},v=function(t,n,r){var i=[];return t.forEach((function(t){if(e.isNodeToBeUpdated(d(h[t]),r).update)if(w[t])i=i.concat(w[t]);else{var n=function t(n,r){if(n&&b[n]){var i=[];return b[n].forEach((function(n){if(e.isNodeToBeUpdated(d(h[n]),r).update){var o=t(n,r);o&&(i=i.concat(o))}else i.push(n)})),i}return null}(t,r);n&&(i=i.concat(n))}else i.push(t)})),i},y=function(t){if(a._nodes[t])return a._nodes[t];var e=d(h[t]);return l(t,e),e},m=function(t,e){var n=arguments.length>2&&void 0!==arguments[2]?arguments[2]:{};u(t,e,br({arrowhead:"arrowPoint",curve:bt,style:g(p),styleObj:p},n))},_=function(t,e){y(t),y(e),m(t,e)},w={};if(f){var b=function(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{},e=t.relations,n={};return e.forEach((function(t){n[t.fromEntityId]?n[t.fromEntityId].push(t.toEntityId):n[t.fromEntityId]=[t.toEntityId]})),n}(r);Object.keys(b).forEach((function(t){var n=b[t],r=e.isNodeToBeUpdated(d(h[t]),i),o=v(n,0,i);r.update?w[t]?w[t]=w[t].concat(o):w[t]=o:o.forEach((function(e){_(t,e)}))}))}else c.length?c.forEach((function(t){_(t.fromEntityId,t.toEntityId)})):y(s);a._nodes[s]&&(a._nodes[s]&&(a._nodes[s].isLineage=!1),e.findImpactNodeAndUpdateData({guid:s,g:a,setEdge:m,getStyleObjStr:g})),t(a)}catch(t){n(t)}}))},findImpactNodeAndUpdateData:function(t){var e=t.guid,n=t.getStyleObjStr,r=t.g,i=t.setEdge,o={},a={fill:"none",stroke:"#fb4200",width:3};!function t(){var e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{},s=arguments.length>1?arguments[1]:void 0,u=Object.keys(e);u.length&&(o[s]||(o[s]=!0,u.forEach((function(e){r._nodes[e]&&(r._nodes[e].isLineage=!1),i(s,e,{style:n(a),styleObj:a}),t(r._sucs[e],e)}))))}(r._sucs[e],e)}};String.prototype.trunc=String.prototype.trunc||function(t){return this.length>t?this.substr(0,t-1)+"...":this};function Er(){}function Tr(t,e){var n=new Er;if(t instanceof Er)t.each((function(t,e){n.set(e,t)}));else if(Array.isArray(t)){var r,i=-1,o=t.length;if(null==e)for(;++i0&&void 0!==arguments[0]?arguments[0]:{};return n._createGraph(n.options,n.graphOptions,t)},clear:function(t){return n.clear(t)},refresh:function(t){return n.refresh(t)},centerAlign:function(t){return n.centerAlign(t)},exportLineage:function(t){return n.exportLineage(t)},zoomIn:function(t){return n.zoomIn(t)},zoomOut:function(t){return n.zoomOut(t)},zoom:function(t){return n.zoom(t)},fullScreen:function(t){return n.fullScreen(t)},searchNode:function(t){return n.searchNode(t)},displayFullName:function(t){return n.displayFullName(t)},removeNodeSelection:function(t){return n.removeNodeSelection(t)},getGraphOptions:function(){return n.graphOptions},getNode:function(t,e){var r=null;return(r=e?n.actualData[t]:n.g._nodes[t])&&(r=Object.assign({},r)),r},getNodes:function(t,e){var r=null;return(r=e?n.actualData:n.g._nodes)&&(r=Object.assign({},r)),r},setNode:this._setGraphNode,setEdge:this._setGraphEdge},!1===a&&this.init(),this.initReturnObj}var e,n,r;return e=t,(n=[{key:"_updateAllOptions",value:function(t){Object.assign(this.options,t);var e=this.svg.node().getBoundingClientRect();this.graphOptions.width=this.options.width||e.width,this.graphOptions.height=this.options.height||e.height;var n=this.graphOptions,r=n.svg,i=n.width,o=n.height,a=n.guid,s=this.options.fitToScreen;r.select("g").node().removeAttribute("transform"),r.attr("viewBox","0 0 "+i+" "+o).attr("enable-background","new 0 0 "+i+" "+o),this.centerAlign({fitToScreen:s,guid:a})}},{key:"_updateOptions",value:function(t){Object.assign(this.options,{filterObj:{isProcessHideCheck:!1,isDeletedEntityHideCheck:!1}},t)}},{key:"init",value:function(){var t=this.options.data,e=void 0===t?{}:t;e.baseEntityGuid&&(this.guid=e.baseEntityGuid),this._initializeGraph(),this._initGraph()}},{key:"clear",value:function(){this.options.el||(this.svg.remove(),this.svg=null),this.g=null,this.graphOptions={}}},{key:"centerAlign",value:function(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{},e=this.svg.select("g"),n=e.selectAll("g.edgePath");_r.centerNode(zr(zr({},this.graphOptions),{},{svgGroupEl:e,edgePathEl:n},t))}},{key:"zoomIn",value:function(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{};_r.zoomIn(zr(zr({},this.graphOptions),t))}},{key:"zoomOut",value:function(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{};_r.zoomOut(zr(zr({},this.graphOptions),t))}},{key:"zoom",value:function(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{};_r.zoom(zr(zr({},this.graphOptions),t))}},{key:"displayFullName",value:function(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{},e=this;this.g.nodes().forEach((function(n){var r=e.svg.selectAll("g.nodes>g[id='"+n+"']"),i=e.g.node(n).toolTipLabel;1==t.bLabelFullText?r.select("tspan").text(i):r.select("tspan").text(i.trunc(18))})),this.selectedNode&&this.searchNode({guid:this.selectedNode})}},{key:"refresh",value:function(){this.clear(),this._initializeGraph(),this._initGraph({refresh:!0}),this.selectedNode=""}},{key:"removeNodeSelection",value:function(){this.svg.selectAll("g.node>circle").classed("node-detail-highlight",!1)}},{key:"searchNode",value:function(t){var e=t.guid,n=t.onSearchNode;this.svg.selectAll(".serach-rect").remove(),this.svg.selectAll(".label").attr("stroke","none"),this.selectedNode=e,this.centerAlign({guid:e,onCenterZoomed:function(t){var e=t.selectedNodeEl,r=e.node().getBBox(),i=r.width+10,o=r.x-5;e.select(".label").attr("stroke","#316132"),e.select("circle").classed("wobble",!0),e.insert("rect","circle").attr("class","serach-rect").attr("stroke","#37bb9b").attr("stroke-width","2.5px").attr("fill","none").attr("x",o).attr("y",-27.5).attr("width",i).attr("height",60),n&&"function"==typeof n&&n(t)},isSelected:!0})}},{key:"exportLineage",value:function(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{},e=t.downloadFileName;if(void 0===e){var n=this.g._nodes[this.guid];e=n&&n.attributes?"".concat(n.attributes.qualifiedName||n.attributes.name||"lineage_export",".png"):"export.png"}_r.saveSvg(zr(zr({},this.graphOptions),{},{downloadFileName:e,onExportLineage:function(t){function e(e){return t.apply(this,arguments)}return e.toString=function(){return t.toString()},e}((function(e){t.onExportLineage&&onExportLineage(e)}))}))}},{key:"fullScreen",value:function(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{},e=t.el;if(void 0===e)throw new Error("LineageHelper requires el propety to apply fullScreen class");var n=mt(e);return n.classed("fullscreen-mode")?(n.classed("fullscreen-mode",!1),!1):(n.classed("fullscreen-mode",!0),!0)}},{key:"_getValueFromUser",value:function(t){if(void 0!==t)return"function"==typeof t?t():t}},{key:"_initializeGraph",value:function(){var t=this.options,e=t.width,n=void 0===e?"100%":e,r=t.height,o=void 0===r?"100%":r,a=t.el;this.svg=mt(a),a instanceof SVGElement||(this.svg.selectAll("*").remove(),this.svg=this.svg.append("svg").attr("xmlns","http://www.w3.org/2000/svg").attr(" xmlns:xlink","http://www.w3.org/1999/xlink").attr("version","1.1").attr("width",n).attr("height",o)),this.g=(new i.a.graphlib.Graph).setGraph(Object.assign({nodesep:50,ranksep:90,rankdir:"LR",marginx:20,marginy:20,transition:function(t){return t.transition().duration(500)}},this.options.dagreOptions)).setDefaultEdgeLabel((function(){return{}}));var s=this.svg.node().getBoundingClientRect();this.actualData={},this.graphOptions={svg:this.svg,g:this.g,dagreD3:i.a,guid:this.guid,width:this.options.width||s.width,height:this.options.height||s.height}}},{key:"_initGraph",value:function(){var t=this,e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{},n=e.refresh;this.svg&&this.svg.select("g").remove();var r=this.options.filterObj;if(this.options.getFilterObj){var i=this.options.getFilterObj();if(void 0!==i||null!==i){if("object"!==Ar(i))throw new Error("getFilterObj expect return type `object`,`null` or `Undefined`");r=i}}if(!0!==this.options.setDataManually)return void 0===this.options.data||this.options.data&&0===this.options.data.relations.length&&_.isEmpty(this.options.data.guidEntityMap)?(this.options.beforeRender&&this.options.beforeRender(),this.svg.append("text").attr("x","50%").attr("y","50%").attr("alignment-baseline","middle").attr("text-anchor","middle").text("No lineage data found"),void(this.options.afterRender&&this.options.afterRender())):kr.generateData(zr(zr(zr({},this.options),{},{filterObj:r},this.graphOptions),{},{setGraphNode:this._setGraphNode,setGraphEdge:this._setGraphEdge})).then((function(e){t._createGraph(t.options,t.graphOptions,{refresh:n})}))}},{key:"_createGraph",value:function(t,e,n){var r=this,o=t.data,a=void 0===o?{}:o,s=t.imgBasePath,u=t.isShowTooltip,l=t.isShowHoverPath,c=t.onLabelClick,h=t.onPathClick,f=t.onNodeClick,p=t.zoom,d=t.fitToScreen,g=t.getToolTipContent,v=t.toolTipTitle,y=n.refresh;this.options.beforeRender&&this.options.beforeRender(),this.selectedNode="";var m=this,_=e.svg,w=e.g,b=e.width,x=e.height,k=this.options.dagreOptions&&"tb"===this.options.dagreOptions.rankdir;if(_ instanceof yt==0)throw new Error("svg is not initialized or something went wrong while creatig graph instance");if(void 0!==w._nodes&&0!==w._nodes.length){w.nodes().forEach((function(t){var e=w.node(t);e&&(e.rx=e.ry=5)})),_.attr("viewBox","0 0 "+b+" "+x).attr("enable-background","new 0 0 "+b+" "+x);var E=_.append("g"),T=_.append("defs"),N=new i.a.render;N.arrows().arrowPoint=function(){return _r.arrowPointRender.apply(_r,Array.prototype.slice.call(arguments).concat([zr({},e)]))},N.shapes().img=function(){return _r.imgShapeRender.apply(_r,Array.prototype.slice.call(arguments).concat([zr(zr({},e),{},{isRankdirToBottom:k,imgBasePath:m._getValueFromUser(s),defsEl:T})]))};var O=function(){var t=function(){return"n"},e=function(){return[0,0]},n=function(){return" "},r=document.body,i=h(),o=null,a=null,s=null;function u(t){(o=function(t){var e=t.node();return e?"svg"===e.tagName.toLowerCase()?e:e.ownerSVGElement:null}(t))&&(a=o.createSVGPoint(),r.appendChild(i))}u.show=function(){var i=Array.prototype.slice.call(arguments);i[i.length-1]instanceof SVGElement&&(s=i.pop());var o,a=n.apply(this,i),h=e.apply(this,i),p=t.apply(this,i),d=f(),g=c.length,v=document.documentElement.scrollTop||r.scrollTop,y=document.documentElement.scrollLeft||r.scrollLeft;for(d.html(a).style("opacity",1).style("pointer-events","all");g--;)d.classed(c[g],!1);return o=l.get(p).apply(this),d.classed(p,!0).style("top",o.top+h[0]+v+"px").style("left",o.left+h[1]+y+"px"),u},u.hide=function(){return f().style("opacity",0).style("pointer-events","none"),u},u.attr=function(t,e){if(arguments.length<2&&"string"==typeof t)return f().attr(t);var n=Array.prototype.slice.call(arguments);return yt.prototype.attr.apply(f(),n),u},u.style=function(t,e){if(arguments.length<2&&"string"==typeof t)return f().style(t);var n=Array.prototype.slice.call(arguments);return yt.prototype.style.apply(f(),n),u},u.direction=function(e){return arguments.length?(t=null==e?e:d(e),u):t},u.offset=function(t){return arguments.length?(e=null==t?t:d(t),u):e},u.html=function(t){return arguments.length?(n=null==t?t:d(t),u):n},u.rootElement=function(t){return arguments.length?(r=null==t?t:d(t),u):r},u.destroy=function(){return i&&(f().remove(),i=null),u};var l=Nr({n:function(){var t=p(this);return{top:t.n.y-i.offsetHeight,left:t.n.x-i.offsetWidth/2}},s:function(){var t=p(this);return{top:t.s.y,left:t.s.x-i.offsetWidth/2}},e:function(){var t=p(this);return{top:t.e.y-i.offsetHeight/2,left:t.e.x}},w:function(){var t=p(this);return{top:t.w.y-i.offsetHeight/2,left:t.w.x-i.offsetWidth}},nw:function(){var t=p(this);return{top:t.nw.y-i.offsetHeight,left:t.nw.x-i.offsetWidth}},ne:function(){var t=p(this);return{top:t.ne.y-i.offsetHeight,left:t.ne.x}},sw:function(){var t=p(this);return{top:t.sw.y,left:t.sw.x-i.offsetWidth}},se:function(){var t=p(this);return{top:t.se.y,left:t.se.x}}}),c=l.keys();function h(){var t=mt(document.createElement("div"));return t.style("position","absolute").style("top",0).style("opacity",0).style("pointer-events","none").style("box-sizing","border-box"),t.node()}function f(){return null==i&&(i=h(),r.appendChild(i)),mt(i)}function p(t){for(var e=s||t;null==e.getScreenCTM&&null!=e.parentNode;)e=e.parentNode;var n={},r=e.getScreenCTM(),i=e.getBBox(),o=i.width,u=i.height,l=i.x,c=i.y;return a.x=l,a.y=c,n.nw=a.matrixTransform(r),a.x+=o,n.ne=a.matrixTransform(r),a.y+=u,n.se=a.matrixTransform(r),a.x-=o,n.sw=a.matrixTransform(r),a.y-=u/2,n.w=a.matrixTransform(r),a.x+=o,n.e=a.matrixTransform(r),a.x-=o/2,a.y-=u/2,n.n=a.matrixTransform(r),a.y+=u,n.s=a.matrixTransform(r),n}function d(t){return"function"==typeof t?t:function(){return t}}return u}().attr("class","d3-tip").offset([10,0]).html((function(t){if(g&&"function"==typeof g)return g(t,w.node(t));var e=w.node(t),n="";return v?n="
"+v+"
":e.id!==r.guid&&(n="
"+(e.isLineage?"Lineage":"Impact")+"
"),n+="
"+e.toolTipLabel+"
",e.typeName&&(n+="
("+e.typeName+")
"),e.queryText&&(n+="
Query: "+e.queryText+"
"),"
"+n+"
"}));_.call(O),N(E,w),E.selectAll("g.nodes g.label").attr("transform",(function(){return k?"translate(2,-20)":"translate(2,-38)"})).attr("font-size","10px").on("mouseenter",(function(t){it.preventDefault(),mt(this).classed("highlight",!0)})).on("mouseleave",(function(t){it.preventDefault(),mt(this).classed("highlight",!1)})).on("click",(function(t){it.preventDefault(),c&&"function"==typeof c&&c({clickedData:t}),O.hide(t)})),E.selectAll("g.nodes g.node circle").on("mouseenter",(function(t,n,r){if(m.activeNode=!0,this.getScreenCTM().translate(+this.getAttribute("cx"),+this.getAttribute("cy")),m.svg.selectAll(".node").classed("active",!1),mt(this).classed("active",!0),m._getValueFromUser(u)){var i=_r.getToolTipDirection({el:this});O.direction(i).show(t,this)}!1!==m._getValueFromUser(l)&&_r.onHoverFade(zr({opacity:.3,mouseenter:!0,hoveredNode:t},e))})).on("mouseleave",(function(t){m.activeNode=!1;var n=this;setTimeout((function(e){m.activeTip||m.activeNode||(mt(n).classed("active",!1),m._getValueFromUser(u)&&O.hide(t))}),150),!1!==m._getValueFromUser(l)&&_r.onHoverFade(zr({mouseenter:!1,hoveredNode:t},e))})).on("click",(function(t){it.defaultPrevented||(it.preventDefault(),O.hide(t),_.selectAll("g.node>circle").classed("node-detail-highlight",!1),mt(this).classed("node-detail-highlight",!0),f&&"function"==typeof f&&f({clickedData:t,el:this}))}));var S=E.selectAll("g.edgePath");S.selectAll("path.path").on("click",(function(t){if(h&&"function"==typeof h){var e=a.relations.find((function(e){if(e.fromEntityId===t.v&&e.toEntityId===t.w)return!0}));h({pathRelationObj:e,clickedData:t})}})),!1!==p&&_r.centerNode(zr(zr({},e),{},{fitToScreen:d,svgGroupEl:E,edgePathEl:S})),_r.dragNode(zr(zr({},e),{},{edgePathEl:S})),!0!==y&&this._addLegend(),this.options.afterRender&&this.options.afterRender()}else _.html('No relations to display')}},{key:"_addLegend",value:function(){if(!1!==this.options.legends){var t=mt(this.options.legendsEl||this.options.el).insert("div",":first-child").classed("legends",!0),e=t.append("span").style("color","#fb4200");e.append("i").classed("fa fa-circle-o fa-fw",!0),e.append("span").html("Current Entity"),(e=t.append("span").style("color","#686868")).append("i").classed("fa fa-hourglass-half fa-fw",!0),e.append("span").html("In Progress"),(e=t.append("span").style("color","#df9b00")).append("i").classed("fa fa-long-arrow-right fa-fw",!0),e.append("span").html("Lineage"),(e=t.append("span").style("color","#fb4200")).append("i").classed("fa fa-long-arrow-right fa-fw",!0),e.append("span").html("Impact")}}}])&&Dr(e.prototype,n),r&&Dr(e,r),t}()}])})); \ No newline at end of file +!function(t,e){"object"==typeof exports&&"object"==typeof module?module.exports=e(require("platform"),require("dagreD3")):"function"==typeof define&&define.amd?define(["platform","dagreD3"],e):"object"==typeof exports?exports.LineageHelper=e(require("platform"),require("dagreD3")):t.LineageHelper=e(t.platform,t.dagreD3)}(window,(function(t,e){return function(t){var e={};function n(r){if(e[r])return e[r].exports;var i=e[r]={i:r,l:!1,exports:{}};return t[r].call(i.exports,i,i.exports,n),i.l=!0,i.exports}return n.m=t,n.c=e,n.d=function(t,e,r){n.o(t,e)||Object.defineProperty(t,e,{enumerable:!0,get:r})},n.r=function(t){"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(t,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(t,"__esModule",{value:!0})},n.t=function(t,e){if(1&e&&(t=n(t)),8&e)return t;if(4&e&&"object"==typeof t&&t&&t.__esModule)return t;var r=Object.create(null);if(n.r(r),Object.defineProperty(r,"default",{enumerable:!0,value:t}),2&e&&"string"!=typeof t)for(var i in t)n.d(r,i,function(e){return t[e]}.bind(null,i));return r},n.n=function(t){var e=t&&t.__esModule?function(){return t.default}:function(){return t};return n.d(e,"a",e),e},n.o=function(t,e){return Object.prototype.hasOwnProperty.call(t,e)},n.p="",n(n.s=10)}([,function(e,n){e.exports=t},function(t,n){t.exports=e},,,,,,,function(t,e,n){},function(t,e,n){"use strict";n.r(e),n.d(e,"default",(function(){return Cr}));var r=n(2),i=n.n(r);function o(){}var a=function(t){return null==t?o:function(){return this.querySelector(t)}};function s(){return[]}var u=function(t){return null==t?s:function(){return this.querySelectorAll(t)}},l=function(t){return function(){return this.matches(t)}},c=function(t){return new Array(t.length)};function h(t,e){this.ownerDocument=t.ownerDocument,this.namespaceURI=t.namespaceURI,this._next=null,this._parent=t,this.__data__=e}h.prototype={constructor:h,appendChild:function(t){return this._parent.insertBefore(t,this._next)},insertBefore:function(t,e){return this._parent.insertBefore(t,e)},querySelector:function(t){return this._parent.querySelector(t)},querySelectorAll:function(t){return this._parent.querySelectorAll(t)}};function f(t,e,n,r,i,o){for(var a,s=0,u=e.length,l=o.length;se?1:t>=e?0:NaN}var g="http://www.w3.org/1999/xhtml",v={svg:"http://www.w3.org/2000/svg",xhtml:g,xlink:"http://www.w3.org/1999/xlink",xml:"http://www.w3.org/XML/1998/namespace",xmlns:"http://www.w3.org/2000/xmlns/"},y=function(t){var e=t+="",n=e.indexOf(":");return n>=0&&"xmlns"!==(e=t.slice(0,n))&&(t=t.slice(n+1)),v.hasOwnProperty(e)?{space:v[e],local:t}:t};function m(t){return function(){this.removeAttribute(t)}}function w(t){return function(){this.removeAttributeNS(t.space,t.local)}}function b(t,e){return function(){this.setAttribute(t,e)}}function x(t,e){return function(){this.setAttributeNS(t.space,t.local,e)}}function k(t,e){return function(){var n=e.apply(this,arguments);null==n?this.removeAttribute(t):this.setAttribute(t,n)}}function E(t,e){return function(){var n=e.apply(this,arguments);null==n?this.removeAttributeNS(t.space,t.local):this.setAttributeNS(t.space,t.local,n)}}var T=function(t){return t.ownerDocument&&t.ownerDocument.defaultView||t.document&&t||t.defaultView};function N(t){return function(){this.style.removeProperty(t)}}function O(t,e,n){return function(){this.style.setProperty(t,e,n)}}function S(t,e,n){return function(){var r=e.apply(this,arguments);null==r?this.style.removeProperty(t):this.style.setProperty(t,r,n)}}function P(t,e){return t.style.getPropertyValue(e)||T(t).getComputedStyle(t,null).getPropertyValue(e)}function A(t){return function(){delete this[t]}}function M(t,e){return function(){this[t]=e}}function z(t,e){return function(){var n=e.apply(this,arguments);null==n?delete this[t]:this[t]=n}}function j(t){return t.trim().split(/^|\s+/)}function D(t){return t.classList||new C(t)}function C(t){this._node=t,this._names=j(t.getAttribute("class")||"")}function B(t,e){for(var n=D(t),r=-1,i=e.length;++r=0&&(this._names.splice(e,1),this._node.setAttribute("class",this._names.join(" ")))},contains:function(t){return this._names.indexOf(t)>=0}};function q(){this.textContent=""}function F(t){return function(){this.textContent=t}}function U(t){return function(){var e=t.apply(this,arguments);this.textContent=null==e?"":e}}function H(){this.innerHTML=""}function V(t){return function(){this.innerHTML=t}}function X(t){return function(){var e=t.apply(this,arguments);this.innerHTML=null==e?"":e}}function Y(){this.nextSibling&&this.parentNode.appendChild(this)}function Z(){this.previousSibling&&this.parentNode.insertBefore(this,this.parentNode.firstChild)}function W(t){return function(){var e=this.ownerDocument,n=this.namespaceURI;return n===g&&e.documentElement.namespaceURI===g?e.createElement(t):e.createElementNS(n,t)}}function K(t){return function(){return this.ownerDocument.createElementNS(t.space,t.local)}}var Q=function(t){var e=y(t);return(e.local?K:W)(e)};function J(){return null}function tt(){var t=this.parentNode;t&&t.removeChild(this)}function et(){var t=this.cloneNode(!1),e=this.parentNode;return e?e.insertBefore(t,this.nextSibling):t}function nt(){var t=this.cloneNode(!0),e=this.parentNode;return e?e.insertBefore(t,this.nextSibling):t}var rt={},it=null;"undefined"!=typeof document&&("onmouseenter"in document.documentElement||(rt={mouseenter:"mouseover",mouseleave:"mouseout"}));function ot(t,e,n){return t=at(t,e,n),function(e){var n=e.relatedTarget;n&&(n===this||8&n.compareDocumentPosition(this))||t.call(this,e)}}function at(t,e,n){return function(r){var i=it;it=r;try{t.call(this,this.__data__,e,n)}finally{it=i}}}function st(t){return t.trim().split(/^|\s+/).map((function(t){var e="",n=t.indexOf(".");return n>=0&&(e=t.slice(n+1),t=t.slice(0,n)),{type:t,name:e}}))}function ut(t){return function(){var e=this.__on;if(e){for(var n,r=0,i=-1,o=e.length;r=k&&(k=x+1);!(b=_[k])&&++k=0;)(r=i[o])&&(a&&4^r.compareDocumentPosition(a)&&a.parentNode.insertBefore(r,a),a=r);return this},sort:function(t){function e(e,n){return e&&n?t(e.__data__,n.__data__):!e-!n}t||(t=d);for(var n=this._groups,r=n.length,i=new Array(r),o=0;o1?this.each((null==e?N:"function"==typeof e?S:O)(t,e,null==n?"":n)):P(this.node(),t)},property:function(t,e){return arguments.length>1?this.each((null==e?A:"function"==typeof e?z:M)(t,e)):this.node()[t]},classed:function(t,e){var n=j(t+"");if(arguments.length<2){for(var r=D(this.node()),i=-1,o=n.length;++i=0&&(n=t.slice(r+1),t=t.slice(0,r)),t&&!e.hasOwnProperty(t))throw new Error("unknown type: "+t);return{type:t,name:n}}))}function Nt(t,e){for(var n,r=0,i=t.length;r0)for(var n,r,i=new Array(n),o=0;o=0&&e._call.call(null,t),e=e._next;--Ft}()}finally{Ft=0,function(){var t,e,n=Ct,r=1/0;for(;n;)n._call?(r>n._time&&(r=n._time),t=n,n=n._next):(e=n._next,n._next=null,n=t?t._next=e:Ct=e);Bt=t,ne(r)}(),Vt=0}}function ee(){var t=Yt.now(),e=t-$t;e>1e3&&(Xt-=e,$t=t)}function ne(t){Ft||(Ut&&(Ut=clearTimeout(Ut)),t-Vt>24?(t<1/0&&(Ut=setTimeout(te,t-Yt.now()-Xt)),Ht&&(Ht=clearInterval(Ht))):(Ht||($t=Yt.now(),Ht=setInterval(ee,1e3)),Ft=1,Zt(te)))}Qt.prototype=Jt.prototype={constructor:Qt,restart:function(t,e,n){if("function"!=typeof t)throw new TypeError("callback is not a function");n=(null==n?Wt():+n)+(null==e?0:+e),this._next||Bt===this||(Bt?Bt._next=this:Ct=this,Bt=this),this._call=t,this._time=n,ne()},stop:function(){this._call&&(this._call=null,this._time=1/0,ne())}};var re=function(t,e,n){var r=new Qt;return e=null==e?0:+e,r.restart((function(n){r.stop(),t(n+e)}),e,n),r},ie=St("start","end","cancel","interrupt"),oe=[],ae=function(t,e,n,r,i,o){var a=t.__transition;if(a){if(n in a)return}else t.__transition={};!function(t,e,n){var r,i=t.__transition;function o(u){var l,c,h,f;if(1!==n.state)return s();for(l in i)if((f=i[l]).name===n.name){if(3===f.state)return re(o);4===f.state?(f.state=6,f.timer.stop(),f.on.call("interrupt",t,t.__data__,f.index,f.group),delete i[l]):+l0)throw new Error("too late; already scheduled");return n}function ue(t,e){var n=le(t,e);if(n.state>3)throw new Error("too late; already running");return n}function le(t,e){var n=t.__transition;if(!n||!(n=n[e]))throw new Error("transition not found");return n}var ce,he,fe,pe,de=function(t,e){var n,r,i,o=t.__transition,a=!0;if(o){for(i in e=null==e?null:e+"",o)(n=o[i]).name===e?(r=n.state>2&&n.state<5,n.state=6,n.timer.stop(),n.on.call(r?"interrupt":"cancel",t,t.__data__,n.index,n.group),delete o[i]):a=!1;a&&delete t.__transition}},ge=function(t,e){return t=+t,e=+e,function(n){return t*(1-n)+e*n}},ve=180/Math.PI,ye={translateX:0,translateY:0,rotate:0,skewX:0,scaleX:1,scaleY:1},me=function(t,e,n,r,i,o){var a,s,u;return(a=Math.sqrt(t*t+e*e))&&(t/=a,e/=a),(u=t*n+e*r)&&(n-=t*u,r-=e*u),(s=Math.sqrt(n*n+r*r))&&(n/=s,r/=s,u/=s),t*r180?e+=360:e-t>180&&(t+=360),o.push({i:n.push(i(n)+"rotate(",null,r)-2,x:ge(t,e)})):e&&n.push(i(n)+"rotate("+e+r)}(o.rotate,a.rotate,s,u),function(t,e,n,o){t!==e?o.push({i:n.push(i(n)+"skewX(",null,r)-2,x:ge(t,e)}):e&&n.push(i(n)+"skewX("+e+r)}(o.skewX,a.skewX,s,u),function(t,e,n,r,o,a){if(t!==n||e!==r){var s=o.push(i(o)+"scale(",null,",",null,")");a.push({i:s-4,x:ge(t,n)},{i:s-2,x:ge(e,r)})}else 1===n&&1===r||o.push(i(o)+"scale("+n+","+r+")")}(o.scaleX,o.scaleY,a.scaleX,a.scaleY,s,u),o=a=null,function(t){for(var e,n=-1,r=u.length;++n>8&15|e>>4&240,e>>4&15|240&e,(15&e)<<4|15&e,1):8===n?Ue(e>>24&255,e>>16&255,e>>8&255,(255&e)/255):4===n?Ue(e>>12&15|e>>8&240,e>>8&15|e>>4&240,e>>4&15|240&e,((15&e)<<4|15&e)/255):null):(e=ze.exec(t))?new Ve(e[1],e[2],e[3],1):(e=je.exec(t))?new Ve(255*e[1]/100,255*e[2]/100,255*e[3]/100,1):(e=De.exec(t))?Ue(e[1],e[2],e[3],e[4]):(e=Ce.exec(t))?Ue(255*e[1]/100,255*e[2]/100,255*e[3]/100,e[4]):(e=Be.exec(t))?We(e[1],e[2]/100,e[3]/100,1):(e=Ie.exec(t))?We(e[1],e[2]/100,e[3]/100,e[4]):Le.hasOwnProperty(t)?Fe(Le[t]):"transparent"===t?new Ve(NaN,NaN,NaN,0):null}function Fe(t){return new Ve(t>>16&255,t>>8&255,255&t,1)}function Ue(t,e,n,r){return r<=0&&(t=e=n=NaN),new Ve(t,e,n,r)}function He(t){return t instanceof Oe||(t=qe(t)),t?new Ve((t=t.rgb()).r,t.g,t.b,t.opacity):new Ve}function $e(t,e,n,r){return 1===arguments.length?He(t):new Ve(t,e,n,null==r?1:r)}function Ve(t,e,n,r){this.r=+t,this.g=+e,this.b=+n,this.opacity=+r}function Xe(){return"#"+Ze(this.r)+Ze(this.g)+Ze(this.b)}function Ye(){var t=this.opacity;return(1===(t=isNaN(t)?1:Math.max(0,Math.min(1,t)))?"rgb(":"rgba(")+Math.max(0,Math.min(255,Math.round(this.r)||0))+", "+Math.max(0,Math.min(255,Math.round(this.g)||0))+", "+Math.max(0,Math.min(255,Math.round(this.b)||0))+(1===t?")":", "+t+")")}function Ze(t){return((t=Math.max(0,Math.min(255,Math.round(t)||0)))<16?"0":"")+t.toString(16)}function We(t,e,n,r){return r<=0?t=e=n=NaN:n<=0||n>=1?t=e=NaN:e<=0&&(t=NaN),new Qe(t,e,n,r)}function Ke(t){if(t instanceof Qe)return new Qe(t.h,t.s,t.l,t.opacity);if(t instanceof Oe||(t=qe(t)),!t)return new Qe;if(t instanceof Qe)return t;var e=(t=t.rgb()).r/255,n=t.g/255,r=t.b/255,i=Math.min(e,n,r),o=Math.max(e,n,r),a=NaN,s=o-i,u=(o+i)/2;return s?(a=e===o?(n-r)/s+6*(n0&&u<1?0:a,new Qe(a,s,u,t.opacity)}function Qe(t,e,n,r){this.h=+t,this.s=+e,this.l=+n,this.opacity=+r}function Je(t,e,n){return 255*(t<60?e+(n-e)*t/60:t<180?n:t<240?e+(n-e)*(240-t)/60:e)}function tn(t,e,n,r,i){var o=t*t,a=o*t;return((1-3*t+3*o-a)*e+(4-6*o+3*a)*n+(1+3*t+3*o-3*a)*r+a*i)/6}Te(Oe,qe,{copy:function(t){return Object.assign(new this.constructor,this,t)},displayable:function(){return this.rgb().displayable()},hex:Re,formatHex:Re,formatHsl:function(){return Ke(this).formatHsl()},formatRgb:Ge,toString:Ge}),Te(Ve,$e,Ne(Oe,{brighter:function(t){return t=null==t?1/.7:Math.pow(1/.7,t),new Ve(this.r*t,this.g*t,this.b*t,this.opacity)},darker:function(t){return t=null==t?.7:Math.pow(.7,t),new Ve(this.r*t,this.g*t,this.b*t,this.opacity)},rgb:function(){return this},displayable:function(){return-.5<=this.r&&this.r<255.5&&-.5<=this.g&&this.g<255.5&&-.5<=this.b&&this.b<255.5&&0<=this.opacity&&this.opacity<=1},hex:Xe,formatHex:Xe,formatRgb:Ye,toString:Ye})),Te(Qe,(function(t,e,n,r){return 1===arguments.length?Ke(t):new Qe(t,e,n,null==r?1:r)}),Ne(Oe,{brighter:function(t){return t=null==t?1/.7:Math.pow(1/.7,t),new Qe(this.h,this.s,this.l*t,this.opacity)},darker:function(t){return t=null==t?.7:Math.pow(.7,t),new Qe(this.h,this.s,this.l*t,this.opacity)},rgb:function(){var t=this.h%360+360*(this.h<0),e=isNaN(t)||isNaN(this.s)?0:this.s,n=this.l,r=n+(n<.5?n:1-n)*e,i=2*n-r;return new Ve(Je(t>=240?t-240:t+120,i,r),Je(t,i,r),Je(t<120?t+240:t-120,i,r),this.opacity)},displayable:function(){return(0<=this.s&&this.s<=1||isNaN(this.s))&&0<=this.l&&this.l<=1&&0<=this.opacity&&this.opacity<=1},formatHsl:function(){var t=this.opacity;return(1===(t=isNaN(t)?1:Math.max(0,Math.min(1,t)))?"hsl(":"hsla(")+(this.h||0)+", "+100*(this.s||0)+"%, "+100*(this.l||0)+"%"+(1===t?")":", "+t+")")}}));var en=function(t){return function(){return t}};function nn(t,e){return function(n){return t+n*e}}function rn(t){return 1==(t=+t)?on:function(e,n){return n-e?function(t,e,n){return t=Math.pow(t,n),e=Math.pow(e,n)-t,n=1/n,function(r){return Math.pow(t+r*e,n)}}(e,n,t):en(isNaN(e)?n:e)}}function on(t,e){var n=e-t;return n?nn(t,n):en(isNaN(t)?e:t)}var an=function t(e){var n=rn(e);function r(t,e){var r=n((t=$e(t)).r,(e=$e(e)).r),i=n(t.g,e.g),o=n(t.b,e.b),a=on(t.opacity,e.opacity);return function(e){return t.r=r(e),t.g=i(e),t.b=o(e),t.opacity=a(e),t+""}}return r.gamma=t,r}(1);function sn(t){return function(e){var n,r,i=e.length,o=new Array(i),a=new Array(i),s=new Array(i);for(n=0;n=1?(n=1,e-1):Math.floor(n*e),i=t[r],o=t[r+1],a=r>0?t[r-1]:2*i-o,s=ro&&(i=e.slice(o,i),s[a]?s[a]+=i:s[++a]=i),(n=n[0])===(r=r[0])?s[a]?s[a]+=r:s[++a]=r:(s[++a]=null,u.push({i:a,x:ge(n,r)})),o=ln.lastIndex;return o=0&&(t=t.slice(0,e)),!t||"start"===t}))}(e)?se:ue;return function(){var a=o(this,t),s=a.on;s!==r&&(i=(r=s).copy()).on(e,n),a.on=i}}var Sn=yt.prototype.constructor;function Pn(t){return function(){this.style.removeProperty(t)}}function An(t,e,n){return function(r){this.style.setProperty(t,e.call(this,r),n)}}function Mn(t,e,n){var r,i;function o(){var o=e.apply(this,arguments);return o!==i&&(r=(i=o)&&An(t,o,n)),r}return o._value=e,o}function zn(t){return function(e){this.textContent=t.call(this,e)}}function jn(t){var e,n;function r(){var r=t.apply(this,arguments);return r!==n&&(e=(n=r)&&zn(r)),e}return r._value=t,r}var Dn=0;function Cn(t,e,n,r){this._groups=t,this._parents=e,this._name=n,this._id=r}function Bn(){return++Dn}var In=yt.prototype;Cn.prototype=function(t){return yt().transition(t)}.prototype={constructor:Cn,select:function(t){var e=this._name,n=this._id;"function"!=typeof t&&(t=a(t));for(var r=this._groups,i=r.length,o=new Array(i),s=0;sr?(r+i)/2:Math.min(0,r)||Math.max(0,i),a>o?(o+a)/2:Math.min(0,o)||Math.max(0,a))}var Qn=function(t){return function(){return t}};function Jn(t,e,n,r,i,o,a,s,u,l){this.target=t,this.type=e,this.subject=n,this.identifier=r,this.active=i,this.x=o,this.y=a,this.dx=s,this.dy=u,this._=l}function tr(){return!it.ctrlKey&&!it.button}function er(){return this.parentNode}function nr(t){return null==t?{x:it.x,y:it.y}:t}function rr(){return navigator.maxTouchPoints||"ontouchstart"in this}Jn.prototype.on=function(){var t=this._.on.apply(this._,arguments);return t===this._?this:t};var ir=function(){var t,e,n,r,i=tr,o=er,a=nr,s=rr,u={},l=St("start","drag","end"),c=0,h=0;function f(t){t.on("mousedown.drag",p).filter(s).on("touchstart.drag",v).on("touchmove.drag",y).on("touchend.drag touchcancel.drag",m).style("touch-action","none").style("-webkit-tap-highlight-color","rgba(0,0,0,0)")}function p(){if(!r&&i.apply(this,arguments)){var a=_("mouse",o.apply(this,arguments),Gt,this,arguments);a&&(mt(it.view).on("mousemove.drag",d,!0).on("mouseup.drag",g,!0),Mt(it.view),Pt(),n=!1,t=it.clientX,e=it.clientY,a("start"))}}function d(){if(At(),!n){var r=it.clientX-t,i=it.clientY-e;n=r*r+i*i>h}u.mouse("drag")}function g(){mt(it.view).on("mousemove.drag mouseup.drag",null),zt(it.view,n),At(),u.mouse("end")}function v(){if(i.apply(this,arguments)){var t,e,n=it.changedTouches,r=o.apply(this,arguments),a=n.length;for(t=0;t1e-6)if(Math.abs(c*s-u*l)>1e-6&&i){var f=n-o,p=r-a,d=s*s+u*u,g=f*f+p*p,v=Math.sqrt(d),y=Math.sqrt(h),m=i*Math.tan((or-Math.acos((d+h-g)/(2*v*y)))/2),_=m/y,w=m/v;Math.abs(_-1)>1e-6&&(this._+="L"+(t+_*l)+","+(e+_*c)),this._+="A"+i+","+i+",0,0,"+ +(c*f>l*p)+","+(this._x1=t+w*s)+","+(this._y1=e+w*u)}else this._+="L"+(this._x1=t)+","+(this._y1=e);else;},arc:function(t,e,n,r,i,o){t=+t,e=+e,o=!!o;var a=(n=+n)*Math.cos(r),s=n*Math.sin(r),u=t+a,l=e+s,c=1^o,h=o?r-i:i-r;if(n<0)throw new Error("negative radius: "+n);null===this._x1?this._+="M"+u+","+l:(Math.abs(this._x1-u)>1e-6||Math.abs(this._y1-l)>1e-6)&&(this._+="L"+u+","+l),n&&(h<0&&(h=h%ar+ar),h>sr?this._+="A"+n+","+n+",0,1,"+c+","+(t-a)+","+(e-s)+"A"+n+","+n+",0,1,"+c+","+(this._x1=u)+","+(this._y1=l):h>1e-6&&(this._+="A"+n+","+n+",0,"+ +(h>=or)+","+c+","+(this._x1=t+n*Math.cos(i))+","+(this._y1=e+n*Math.sin(i))))},rect:function(t,e,n,r){this._+="M"+(this._x0=this._x1=+t)+","+(this._y0=this._y1=+e)+"h"+ +n+"v"+ +r+"h"+-n+"Z"},toString:function(){return this._}};var cr=lr,hr=function(t){return function(){return t}};function fr(t){this._context=t}fr.prototype={areaStart:function(){this._line=0},areaEnd:function(){this._line=NaN},lineStart:function(){this._point=0},lineEnd:function(){(this._line||0!==this._line&&1===this._point)&&this._context.closePath(),this._line=1-this._line},point:function(t,e){switch(t=+t,e=+e,this._point){case 0:this._point=1,this._line?this._context.lineTo(t,e):this._context.moveTo(t,e);break;case 1:this._point=2;default:this._context.lineTo(t,e)}}};var pr=function(t){return new fr(t)};function dr(t){return t[0]}function gr(t){return t[1]}var vr=n(1),yr=n.n(vr),mr={entityStateReadOnly:{ACTIVE:!1,DELETED:!0,STATUS_ACTIVE:!1,STATUS_DELETED:!0}},_r={nodeArrowDistance:24,refreshGraphForSafari:function(t){t.edgeEl.each((function(t){var e=this,n=$(this).find("pattern");setTimeout((function(t){$(e).find("defs").append(n)}),500)}))},refreshGraphForIE:function(t){var e=t.edgePathEl,n=0;e.each((function(t){var e=$(this).find("marker");$(this).find("marker").remove();var r=this;++n,setTimeout((function(t){$(r).find("defs").append(e),0===--n&&(this.$(".fontLoader").hide(),this.$("svg").fadeTo(1e3,1))}),1e3)}))},dragNode:function(t){var e=this,n=t.g,r=t.svg,i=t.guid,o=(t.edgePathEl,{dragmove:function(t,e){var r=this,i=mt(t),o=n.node(e),a=o.x,s=o.y;o.x+=it.dx,o.y+=it.dy,i.attr("transform","translate("+o.x+","+o.y+")");var u=o.x-a,l=o.y-s;n.edges().forEach((function(t){if(t.v==e||t.w==e){var i=n.edge(t.v,t.w);r.translateEdge(i,u,l),mt(i.elem).select("path").attr("d",r.calcPoints(t))}}))},translateEdge:function(t,e,n){t.points.forEach((function(t){t.x=t.x+e,t.y=t.y+n}))},calcPoints:function(t){var e=n.edge(t.v,t.w),r=n.node(t.v),i=n.node(t.w),o=e.points.slice(1,e.points.length-1);e.points.slice(1,e.points.length-1);return o.unshift(this.intersectRect(r,o[0])),o.push(this.intersectRect(i,o[o.length-1])),function(){var t=dr,e=gr,n=hr(!0),r=null,i=pr,o=null;function a(a){var s,u,l,c=a.length,h=!1;for(null==r&&(o=i(l=cr())),s=0;s<=c;++s)!(sMath.abs(a)*c?(s<0&&(c=-c),h=0===s?0:c*a/s,f=c):(a<0&&(l=-l),h=l,f=0===a?0:l*s/a),{x:r+h,y:o+f}}}),a=ir().on("drag",(function(t){o.dragmove.call(o,this,t)})),s=ir().on("drag",(function(t){o.translateEdge(n.edge(t.v,t.w),it.dx,it.dy);var e=n.edge(t.v,t.w);mt(e.elem).select("path").attr("d",o.calcPoints(t))}));a(r.selectAll("g.node")),s(r.selectAll("g.edgePath"))},zoomIn:function(t){var e=t.svg,n=t.scaleFactor,r=void 0===n?1.3:n;this.d3Zoom.scaleBy(e.transition().duration(750),r)},zoomOut:function(t){var e=t.svg,n=t.scaleFactor,r=void 0===n?.8:n;this.d3Zoom.scaleBy(e.transition().duration(750),r)},zoom:function(t){var e=t.svg,n=t.xa,r=t.ya,i=t.scale;e.transition().duration(750).call(this.d3Zoom.transform,Un.translate(n,r).scale(i))},fitToScreen:function(t){var e=t.svg,n=e.node(),r=n.getBBox(),i=n.parentElement,o=i.clientWidth,a=i.clientHeight,s=r.width,u=r.height,l=r.x+s/2,c=r.y+u/2,h=(h||.95)/Math.max(s/o,u/a),f=o/2-h*l,p=a/2-h*c;this.zoom({svg:e,xa:f,ya:p,scale:h})},centerNode:function(t){var e=t.guid,n=t.g,r=t.svg,i=t.svgGroupEl,o=(t.edgePathEl,t.width),a=t.height,s=t.fitToScreen,u=t.onCenterZoomed,l=t.isSelected;this.d3Zoom=function(){var t,e,n=Vn,r=Xn,i=Kn,o=Zn,a=Wn,s=[0,1/0],u=[[-1/0,-1/0],[1/0,1/0]],l=250,c=It,h=St("start","zoom","end"),f=0;function p(t){t.property("__zoom",Yn).on("wheel.zoom",w).on("mousedown.zoom",b).on("dblclick.zoom",x).filter(a).on("touchstart.zoom",k).on("touchmove.zoom",E).on("touchend.zoom touchcancel.zoom",T).style("touch-action","none").style("-webkit-tap-highlight-color","rgba(0,0,0,0)")}function d(t,e){return(e=Math.max(s[0],Math.min(s[1],e)))===t.k?t:new Fn(e,t.x,t.y)}function g(t,e,n){var r=e[0]-n[0]*t.k,i=e[1]-n[1]*t.k;return r===t.x&&i===t.y?t:new Fn(t.k,r,i)}function v(t){return[(+t[0][0]+ +t[1][0])/2,(+t[0][1]+ +t[1][1])/2]}function y(t,e,n){t.on("start.zoom",(function(){m(this,arguments).start()})).on("interrupt.zoom end.zoom",(function(){m(this,arguments).end()})).tween("zoom",(function(){var t=this,i=arguments,o=m(t,i),a=r.apply(t,i),s=null==n?v(a):"function"==typeof n?n.apply(t,i):n,u=Math.max(a[1][0]-a[0][0],a[1][1]-a[0][1]),l=t.__zoom,h="function"==typeof e?e.apply(t,i):e,f=c(l.invert(s).concat(u/l.k),h.invert(s).concat(u/h.k));return function(t){if(1===t)t=h;else{var e=f(t),n=u/e[2];t=new Fn(n,s[0]-e[0]*n,s[1]-e[1]*n)}o.zoom(null,t)}}))}function m(t,e,n){return!n&&t.__zooming||new _(t,e)}function _(t,e){this.that=t,this.args=e,this.active=0,this.extent=r.apply(t,e),this.taps=0}function w(){if(n.apply(this,arguments)){var t=m(this,arguments),e=this.__zoom,r=Math.max(s[0],Math.min(s[1],e.k*Math.pow(2,o.apply(this,arguments)))),a=Gt(this);if(t.wheel)t.mouse[0][0]===a[0]&&t.mouse[0][1]===a[1]||(t.mouse[1]=e.invert(t.mouse[0]=a)),clearTimeout(t.wheel);else{if(e.k===r)return;t.mouse=[a,e.invert(a)],de(this),t.start()}$n(),t.wheel=setTimeout(l,150),t.zoom("mouse",i(g(d(e,r),t.mouse[0],t.mouse[1]),t.extent,u))}function l(){t.wheel=null,t.end()}}function b(){if(!e&&n.apply(this,arguments)){var t=m(this,arguments,!0),r=mt(it.view).on("mousemove.zoom",l,!0).on("mouseup.zoom",c,!0),o=Gt(this),a=it.clientX,s=it.clientY;Mt(it.view),Hn(),t.mouse=[o,this.__zoom.invert(o)],de(this),t.start()}function l(){if($n(),!t.moved){var e=it.clientX-a,n=it.clientY-s;t.moved=e*e+n*n>f}t.zoom("mouse",i(g(t.that.__zoom,t.mouse[0]=Gt(t.that),t.mouse[1]),t.extent,u))}function c(){r.on("mousemove.zoom mouseup.zoom",null),zt(it.view,t.moved),$n(),t.end()}}function x(){if(n.apply(this,arguments)){var t=this.__zoom,e=Gt(this),o=t.invert(e),a=t.k*(it.shiftKey?.5:2),s=i(g(d(t,a),e,o),r.apply(this,arguments),u);$n(),l>0?mt(this).transition().duration(l).call(y,s,e):mt(this).call(p.transform,s)}}function k(){if(n.apply(this,arguments)){var e,r,i,o,a=it.touches,s=a.length,u=m(this,arguments,it.changedTouches.length===s);for(Hn(),r=0;rg[id='"+e+"']"),h=(this.d3Zoom.scaleExtent([.01,50]).on("zoom",(function(){i.attr("transform",it.transform)})),null),f=null;if(c.empty()){if(s)return void this.fitToScreen({svg:r});h=n.graph().width/2,f=n.graph().height/2}else{var p=c.attr("transform").replace(/[^0-9\-.,]/g,"").split(",");h=p[0],f=p[1]}var d=-(1.2*h-o/2),g=-(1.2*f-a/2);this.zoom({svg:r,xa:d,ya:g,scale:1.2}),l?r.transition().duration(750).call(this.d3Zoom.transform,Un.translate(d,g).scale(1.2)):r.call(this.d3Zoom.transform,Un.translate(d,g).scale(1.2)),u&&u({newScale:1.2,newTranslate:[d,g],d3Zoom:this.d3Zoom,selectedNodeEl:c})},getToolTipDirection:function(t){var e=t.el,n=mt("body").node().getBoundingClientRect().width,r=mt(e).node().getBoundingClientRect(),i="e";return n-r.left<330?(i=n-r.left<330&&r.top<400?"sw":"w",n-r.left<330&&r.top>600&&(i="nw")):r.top>600?(i=n-r.left<330&&r.top>600?"nw":"n",r.left<50&&(i="ne")):r.top<400&&(i=r.left<50?"se":"s"),i},onHoverFade:function(t){var e=t.svg,n=t.g,r=t.mouseenter,i=t.nodesToHighlight,o=t.hoveredNode;return function(t){var i=e.selectAll(".node"),a=e.selectAll(".edgePath");if(r){e.classed("hover",!0);var s=n.successors(o),u=n.predecessors(o);t=s.concat(u);i.classed("hover-active-node",(function(e,n,r){return!!function(t,e,n){if(t===n||e&&e.length&&-1!=e.indexOf(n))return!0}(o,t,e)})),a.classed("hover-active-path",(function(t){return!!(t.v===o||t.w===o?1:0)}))}else e.classed("hover",!1),i.classed("hover-active-node",!1),a.classed("hover-active-path",!1)}(i)},getBaseUrl:function(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:window.location.pathname;return t.replace(/\/[\w-]+.(jsp|html)|\/+$/gi,"")},getEntityIconPath:function(t){var e=t.entityData,n=t.errorUrl,r=t.imgBasePath,i=this.getBaseUrl()+(r||"/img/entity-icon/");if(e){var o=function(t){return i+(mr.entityStateReadOnly[l]?"disabled/"+t:t)},a=function(){return c?mr.entityStateReadOnly[l]?i+"disabled/process.png":i+"process.png":mr.entityStateReadOnly[l]?i+"disabled/table.png":i+"table.png"},s=e.typeName,u=e.serviceType,l=e.status,c=e.isProcess;if(n){if(n.indexOf("table.png")>-1)return null;var h=!(!n||!n.match("entity-icon/"+s+".png|disabled/"+s+".png"));return u&&h?o(u+".png"):a()}return s?o(s+".png"):u?o(u+".png"):a()}},base64Encode:function(t,e){var n=new FileReader;n.addEventListener("load",(function(){return e(n.result)})),n.readAsDataURL(t)},imgShapeRender:function(t,e,n,r){var i=r.dagreD3,o=r.defsEl,a=r.imgBasePath,s=r.guid,u=r.isRankdirToBottom,l=this,c=s,h=n.btnType?"/img/entity-icon/expandBtn.svg":this.getEntityIconPath({entityData:n,imgBasePath:a}),f=h.split("/").pop();if(void 0===this.imageObject&&(this.imageObject={}),n.isDeleted&&(f="deleted_"+f),n.id==c)var p=!0;var d=t.append("circle").attr("fill","url(#img_"+f+")").attr("r",u?"30px":"24px").attr("data-stroke",n.id).attr("stroke-width","2px").attr("class","nodeImage "+(p?"currentNode":n.isProcess?"process":"node"));if(p&&d.attr("stroke","#fb4200"),!0===n.isIncomplete){t.attr("class","node isIncomplete show"),t.insert("rect").attr("x","-5").attr("y","-23").attr("width","14").attr("height","16").attr("fill","url(#img_hourglass.svg)").attr("data-stroke",n.id).attr("stroke-width","2px");g({imgName:"hourglass.svg",imageIconPath:"/img/entity-icon/hourglass.svg",leftPosition:"0",topPosition:"0",width:"12",height:"14"})}function g(t){o.select('pattern[id="img_'+t.imgName+'"]').empty()&&o.append("pattern").attr("x","0%").attr("y","0%").attr("patternUnits","objectBoundingBox").attr("id","img_"+t.imgName).attr("width","100%").attr("height","100%").append("image").attr("href",(function(e){var r=this;if(n){!function e(i){var o=i.imagePath,a={url:o,method:"GET",cache:!0};d.attr("data-iconpath",o);var s=new XMLHttpRequest;s.onreadystatechange=function(){if(4===s.readyState)if(200===s.status)"IE"!==yr.a.name?l.base64Encode(this.response,(function(e){l.imageObject[t.imageIconPath]=e,mt(r).attr("xlink:href",e)})):l.imageObject[t.imageIconPath]=o,t.imageIconPath!==d.attr("data-iconpath")&&d.attr("data-iconpathorigin",t.imageIconPath);else if(404===s.status){var i=l.getEntityIconPath({entityData:n,errorUrl:o});if(null===i){var a=mt(r.parentElement);a.select("image").remove(),a.attr("patternContentUnits","objectBoundingBox").append("circle").attr("r","24px").attr("fill","#e8e8e8")}else e({imagePath:i})}},s.responseType="blob",s.open(a.method,a.url,!0),s.send(null)}({imagePath:t.imageIconPath})}})).attr("x",t.leftPosition).attr("y",t.topPosition).attr("width",t.width).attr("height",t.height)}return g({imgName:f,imageIconPath:h,leftPosition:u?"11":"4",topPosition:u?"20":p?"3":"4",width:"40",height:"40"}),n.intersect=function(t){return i.intersect.circle(n,p?l.nodeArrowDistance+3:l.nodeArrowDistance,t)},d},arrowPointRender:function(t,e,n,r,i){var o=i.dagreD3,a=t.node(),s=a?a.parentNode:t;mt(s).select("path.path").attr("marker-end","url(#"+e+")");var u=t.append("marker").attr("id",e).attr("viewBox","0 0 10 10").attr("refX",8).attr("refY",5).attr("markerUnits","strokeWidth").attr("markerWidth",4).attr("markerHeight",4).attr("orient","auto").append("path").attr("d","M 0 0 L 10 5 L 0 10 z").style("fill",n.styleObj.stroke);o.util.applyStyle(u,n[r+"Style"])},saveSvg:function(t){var e=t.svg,n=t.width,r=t.height,i=t.downloadFileName,o=t.onExportLineage,a=this,s=e.clone(!0).node();setTimeout((function(){"Firefox"===yr.a.name&&(s.setAttribute("width",n),s.setAttribute("height",r));var t=mt("body").append("div");t.classed("hidden-svg",!0),t.node().appendChild(s);var e=mt(".hidden-svg svg");e.select("g").attr("transform","scale(1)"),e.select("foreignObject").remove();var u=150,l=150,c=s.getBBox().width+u,h=s.getBBox().height+l,f=s.getBBox().x,p=s.getBBox().y;s.attributes.viewBox.value=f+","+p+","+c+","+h;var d=document.createElement("canvas");d.id="canvas",d.style.display="none",d.width=1*s.getBBox().width+u,d.height=1*s.getBBox().height+l,mt("body").node().appendChild(d);var g=d.getContext("2d"),v=(new XMLSerializer).serializeToString(s),y=window.URL||window.webkitURL||window;g.fillStyle="#FFFFFF",g.fillRect(0,0,d.width,d.height),g.strokeRect(0,0,d.width,d.height),g.restore();var m=new Image(d.width,d.height),_=new Blob([v],{type:"image/svg+xml;base64"});"Safari"===yr.a.name&&(_=new Blob([v],{type:"image/svg+xml"})),g.drawImage(m,50,50,d.width,d.height);var w=y.createObjectURL(_);m.onload=function(){try{var e=document.createElement("a");e.download=i,document.body.appendChild(e),g.drawImage(m,50,50,d.width,d.height),d.toBlob((function(t){t?(e.href=y.createObjectURL(t),t.size>1e7&&o({status:"failed",message:"The Image size is huge, please open the image in a browser!"}),e.click(),o({status:"Success",message:"Successful"}),"Safari"===yr.a.name&&a.refreshGraphForSafari({edgeEl:a.$("svg g.node")})):o({status:"failed",message:"There was an error in downloading Lineage!"})}),"image/png"),t.remove(),d.remove()}catch(t){o({status:"failed",message:"There was an error in downloading Lineage!"})}},m.src=w}),0)}};function wr(t,e){var n=Object.keys(t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(t);e&&(r=r.filter((function(e){return Object.getOwnPropertyDescriptor(t,e).enumerable}))),n.push.apply(n,r)}return n}function br(t){for(var e=1;e0&&void 0!==arguments[0]?arguments[0]:{},i=r.entityData,o=r.errorUrl,a=this.getBaseUrl(window.location.pathname)+Globals.entityImgPath;function s(t){return a+(mr.entityStateReadOnly[e]?"disabled/"+t:t)}function u(){return i.isProcess?mr.entityStateReadOnly[e]?a+"disabled/process.png":a+"process.png":mr.entityStateReadOnly[e]?a+"disabled/table.png":a+"table.png"}if(i&&(n=i.typeName,t=i&&i.serviceType,e=i&&i.status),i){if(o){var l=!(!o||!o.match("entity-icon/"+n+".png|disabled/"+n+".png"));return t&&l?s(t+".png"):u()}return i.typeName?s(i.typeName+".png"):u()}},isProcess:function(t){var e=t.typeName,n=t.superTypes;t.entityDef;return"Process"==e||n.indexOf("Process")>-1},isDeleted:function(t){if(void 0!==t)return mr.entityStateReadOnly[t.status]},isNodeToBeUpdated:function(t,e){var n=e.isProcessHideCheck,r=e.isDeletedEntityHideCheck,i={isProcess:n&&t.isProcess,isDeleted:r&&t.isDeleted};return i.update=i.isProcess||i.isDeleted,"Expand"===t.label&&(i.update=!0),i},getServiceType:function(t){var e=t.typeName,n=t.entityDef,r=null;return e&&n&&(r=n.serviceType||null),r},getEntityDef:function(t){var e=t.typeName,n=t.entityDefCollection,r=null;return e&&(r=n.find((function(t){return t.name==e}))),r},getNestedSuperTypes:function(t){var e=t.entityDef,n=t.entityDefCollection,r=new Set;return function t(e,n){e&&e.superTypes&&e.superTypes.length&&e.superTypes.forEach((function(e){r.add(e);var i=n.find((function(t){t.name}));i&&t(i,n)}))}(e,n),Array.from(r)},generateData:function(t){var e=this,n=t.data,r=void 0===n?{}:n,i=t.filterObj,o=t.entityDefCollection,a=t.g,s=t.guid,u=t.setGraphEdge,l=t.setGraphNode;return new Promise((function(t,n){try{var c=r.relations||{},h=r.guidEntityMap||{},f=i.isProcessHideCheck||i.isDeletedEntityHideCheck,p={fill:"none",stroke:"#ffb203",width:3},d=function(t){if(t){if(t.updatedValues)return t;var n=t.displayText?t.displayText:" ",r=Object.assign(t,{shape:"img",updatedValues:!0,label:n.trunc(18),toolTipLabel:n,id:t.guid,isLineage:!0,isIncomplete:t.isIncomplete,entityDef:e.getEntityDef({typeName:t.typeName,entityDefCollection:o})});return r.serviceType=e.getServiceType(r),r.superTypes=e.getNestedSuperTypes(br(br({},r),{},{entityDefCollection:o})),r.isProcess=e.isProcess(r),r.isDeleted=e.isDeleted(r),r}},g=function(t){return"fill:"+t.fill+";stroke:"+t.stroke+";stroke-width:"+t.width},v=function(t,n,r){var i=[];return t.forEach((function(t){if(e.isNodeToBeUpdated(d(h[t]),r).update)if(w[t])i=i.concat(w[t]);else{var n=function t(n,r){if(n&&b[n]){var i=[];return b[n].forEach((function(n){if(e.isNodeToBeUpdated(d(h[n]),r).update){var o=t(n,r);o&&(i=i.concat(o))}else i.push(n)})),i}return null}(t,r);n&&(i=i.concat(n))}else i.push(t)})),i},y=function(t){if(a._nodes[t])return a._nodes[t];var e=d(h[t]);return l(t,e),e},m=function(t,e){var n=arguments.length>2&&void 0!==arguments[2]?arguments[2]:{};u(t,e,br({arrowhead:"arrowPoint",curve:bt,style:g(p),styleObj:p},n))},_=function(t,e){y(t),y(e),m(t,e)},w={};if(f){var b=function(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{},e=t.relations,n={};return e.forEach((function(t){n[t.fromEntityId]?n[t.fromEntityId].push(t.toEntityId):n[t.fromEntityId]=[t.toEntityId]})),n}(r);Object.keys(b).forEach((function(t){var n=b[t],r=e.isNodeToBeUpdated(d(h[t]),i),o=v(n,0,i);r.update?w[t]?w[t]=w[t].concat(o):w[t]=o:o.forEach((function(e){_(t,e)}))}))}else c.length?c.forEach((function(t){_(t.fromEntityId,t.toEntityId)})):y(s);a._nodes[s]&&(a._nodes[s]&&(a._nodes[s].isLineage=!1),e.findImpactNodeAndUpdateData({guid:s,g:a,setEdge:m,getStyleObjStr:g})),a._nodes[s]||y(s),t(a)}catch(t){n(t)}}))},findImpactNodeAndUpdateData:function(t){var e=t.guid,n=t.getStyleObjStr,r=t.g,i=t.setEdge,o={},a={fill:"none",stroke:"#fb4200",width:3};!function t(){var e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{},s=arguments.length>1?arguments[1]:void 0,u=Object.keys(e);u.length&&(o[s]||(o[s]=!0,u.forEach((function(e){r._nodes[e]&&(r._nodes[e].isLineage=!1),i(s,e,{style:n(a),styleObj:a}),t(r._sucs[e],e)}))))}(r._sucs[e],e)}};String.prototype.trunc=String.prototype.trunc||function(t){return this.length>t?this.substr(0,t-1)+"...":this};function Er(){}function Tr(t,e){var n=new Er;if(t instanceof Er)t.each((function(t,e){n.set(e,t)}));else if(Array.isArray(t)){var r,i=-1,o=t.length;if(null==e)for(;++i0&&void 0!==arguments[0]?arguments[0]:{};return n._createGraph(n.options,n.graphOptions,t)},clear:function(t){return n.clear(t)},refresh:function(t){return n.refresh(t)},centerAlign:function(t){return n.centerAlign(t)},exportLineage:function(t){return n.exportLineage(t)},zoomIn:function(t){return n.zoomIn(t)},zoomOut:function(t){return n.zoomOut(t)},zoom:function(t){return n.zoom(t)},fullScreen:function(t){return n.fullScreen(t)},searchNode:function(t){return n.searchNode(t)},displayFullName:function(t){return n.displayFullName(t)},removeNodeSelection:function(t){return n.removeNodeSelection(t)},getGraphOptions:function(){return n.graphOptions},getNode:function(t,e){var r=null;return(r=e?n.actualData[t]:n.g._nodes[t])&&(r=Object.assign({},r)),r},getNodes:function(t,e){var r=null;return(r=e?n.actualData:n.g._nodes)&&(r=Object.assign({},r)),r},setNode:this._setGraphNode,setEdge:this._setGraphEdge},!1===a&&this.init(),this.initReturnObj}var e,n,r;return e=t,(n=[{key:"_updateAllOptions",value:function(t){Object.assign(this.options,t);var e=this.svg.node().getBoundingClientRect();this.graphOptions.width=this.options.width||e.width,this.graphOptions.height=this.options.height||e.height;var n=this.graphOptions,r=n.svg,i=n.width,o=n.height,a=n.guid,s=this.options.fitToScreen;r.select("g").node().removeAttribute("transform"),r.attr("viewBox","0 0 "+i+" "+o).attr("enable-background","new 0 0 "+i+" "+o),this.centerAlign({fitToScreen:s,guid:a})}},{key:"_updateOptions",value:function(t){Object.assign(this.options,{filterObj:{isProcessHideCheck:!1,isDeletedEntityHideCheck:!1}},t)}},{key:"init",value:function(){var t=this.options.data,e=void 0===t?{}:t;e.baseEntityGuid&&(this.guid=e.baseEntityGuid),this._initializeGraph(),this._initGraph()}},{key:"clear",value:function(){this.options.el||(this.svg.remove(),this.svg=null),this.g=null,this.graphOptions={}}},{key:"centerAlign",value:function(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{},e=this.svg.select("g"),n=e.selectAll("g.edgePath");_r.centerNode(zr(zr({},this.graphOptions),{},{svgGroupEl:e,edgePathEl:n},t))}},{key:"zoomIn",value:function(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{};_r.zoomIn(zr(zr({},this.graphOptions),t))}},{key:"zoomOut",value:function(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{};_r.zoomOut(zr(zr({},this.graphOptions),t))}},{key:"zoom",value:function(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{};_r.zoom(zr(zr({},this.graphOptions),t))}},{key:"displayFullName",value:function(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{},e=this;this.g.nodes().forEach((function(n){var r=e.svg.selectAll("g.nodes>g[id='"+n+"']"),i=e.g.node(n).toolTipLabel;1==t.bLabelFullText?r.select("tspan").text(i):r.select("tspan").text(i.trunc(18))})),this.selectedNode&&this.searchNode({guid:this.selectedNode})}},{key:"refresh",value:function(t){if(this.clear(),this._initializeGraph(),this._initGraph({refresh:!0}),this.selectedNode="",t&&t.compactLineageEnabled&&t.filterObj){var e=t.filterObj.isProcessHideCheck,n=t.filterObj.isDeletedEntityHideCheck;this._AddFilterNotification(e,n)}}},{key:"removeNodeSelection",value:function(){this.svg.selectAll("g.node>circle").classed("node-detail-highlight",!1)}},{key:"searchNode",value:function(t){var e=t.guid,n=t.onSearchNode;this.svg.selectAll(".serach-rect").remove(),this.svg.selectAll(".label").attr("stroke","none"),this.selectedNode=e,this.centerAlign({guid:e,onCenterZoomed:function(t){var e=t.selectedNodeEl,r=e.node().getBBox(),i=r.width+10,o=r.x-5;e.select(".label").attr("stroke","#316132"),e.select("circle").classed("wobble",!0),e.insert("rect","circle").attr("class","serach-rect").attr("stroke","#37bb9b").attr("stroke-width","2.5px").attr("fill","none").attr("x",o).attr("y",-27.5).attr("width",i).attr("height",60),n&&"function"==typeof n&&n(t)},isSelected:!0})}},{key:"exportLineage",value:function(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{},e=t.downloadFileName;if(void 0===e){var n=this.g._nodes[this.guid];e=n&&n.attributes?"".concat(n.attributes.qualifiedName||n.attributes.name||"lineage_export",".png"):"export.png"}_r.saveSvg(zr(zr({},this.graphOptions),{},{downloadFileName:e,onExportLineage:function(t){function e(e){return t.apply(this,arguments)}return e.toString=function(){return t.toString()},e}((function(e){t.onExportLineage&&onExportLineage(e)}))}))}},{key:"fullScreen",value:function(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{},e=t.el;if(void 0===e)throw new Error("LineageHelper requires el propety to apply fullScreen class");var n=mt(e);return n.classed("fullscreen-mode")?(n.classed("fullscreen-mode",!1),!1):(n.classed("fullscreen-mode",!0),!0)}},{key:"_getValueFromUser",value:function(t){if(void 0!==t)return"function"==typeof t?t():t}},{key:"_initializeGraph",value:function(){var t=this.options,e=t.width,n=void 0===e?"100%":e,r=t.height,o=void 0===r?"100%":r,a=t.el;this.svg=mt(a),a instanceof SVGElement||(this.svg.selectAll("*").remove(),this.svg=this.svg.append("svg").attr("xmlns","http://www.w3.org/2000/svg").attr(" xmlns:xlink","http://www.w3.org/1999/xlink").attr("version","1.1").attr("width",n).attr("height",o)),this.g=(new i.a.graphlib.Graph).setGraph(Object.assign({nodesep:50,ranksep:90,rankdir:"LR",marginx:20,marginy:20,transition:function(t){return t.transition().duration(500)}},this.options.dagreOptions)).setDefaultEdgeLabel((function(){return{}}));var s=this.svg.node().getBoundingClientRect();this.actualData={},this.graphOptions={svg:this.svg,g:this.g,dagreD3:i.a,guid:this.guid,width:this.options.width||s.width,height:this.options.height||s.height}}},{key:"_initGraph",value:function(){var t=this,e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{},n=e.refresh;this.svg&&this.svg.select("g").remove();var r=this.options.filterObj;if(this.options.getFilterObj){var i=this.options.getFilterObj();if(void 0!==i||null!==i){if("object"!==Ar(i))throw new Error("getFilterObj expect return type `object`,`null` or `Undefined`");r=i}}if(!0!==this.options.setDataManually)return void 0===this.options.data||this.options.data&&0===this.options.data.relations.length&&_.isEmpty(this.options.data.guidEntityMap)?(this.options.beforeRender&&this.options.beforeRender(),this.svg.append("text").attr("x","50%").attr("y","50%").attr("alignment-baseline","middle").attr("text-anchor","middle").text("No lineage data found"),void(this.options.afterRender&&this.options.afterRender())):kr.generateData(zr(zr(zr({},this.options),{},{filterObj:r},this.graphOptions),{},{setGraphNode:this._setGraphNode,setGraphEdge:this._setGraphEdge})).then((function(e){t._createGraph(t.options,t.graphOptions,{refresh:n})}))}},{key:"_createGraph",value:function(t,e,n){var r=this,o=t.data,a=void 0===o?{}:o,s=t.imgBasePath,u=t.isShowTooltip,l=t.isShowHoverPath,c=t.onLabelClick,h=t.onPathClick,f=t.onNodeClick,p=t.zoom,d=t.fitToScreen,g=t.getToolTipContent,v=t.toolTipTitle,y=n.refresh;this.options.beforeRender&&this.options.beforeRender(),this.selectedNode="";var m=this,_=e.svg,w=e.g,b=e.width,x=e.height,k=this.options.dagreOptions&&"tb"===this.options.dagreOptions.rankdir;if(_ instanceof yt==0)throw new Error("svg is not initialized or something went wrong while creatig graph instance");if(void 0!==w._nodes&&0!==w._nodes.length){w.nodes().forEach((function(t){var e=w.node(t);e&&(e.rx=e.ry=5)})),_.attr("viewBox","0 0 "+b+" "+x).attr("enable-background","new 0 0 "+b+" "+x);var E=_.append("g"),T=_.append("defs"),N=new i.a.render;N.arrows().arrowPoint=function(){return _r.arrowPointRender.apply(_r,Array.prototype.slice.call(arguments).concat([zr({},e)]))},N.shapes().img=function(){return _r.imgShapeRender.apply(_r,Array.prototype.slice.call(arguments).concat([zr(zr({},e),{},{isRankdirToBottom:k,imgBasePath:m._getValueFromUser(s),defsEl:T})]))};var O=function(){var t=function(){return"n"},e=function(){return[0,0]},n=function(){return" "},r=document.body,i=h(),o=null,a=null,s=null;function u(t){(o=function(t){var e=t.node();return e?"svg"===e.tagName.toLowerCase()?e:e.ownerSVGElement:null}(t))&&(a=o.createSVGPoint(),r.appendChild(i))}u.show=function(){var i=Array.prototype.slice.call(arguments);i[i.length-1]instanceof SVGElement&&(s=i.pop());var o,a=n.apply(this,i),h=e.apply(this,i),p=t.apply(this,i),d=f(),g=c.length,v=document.documentElement.scrollTop||r.scrollTop,y=document.documentElement.scrollLeft||r.scrollLeft;for(d.html(a).style("opacity",1).style("pointer-events","all");g--;)d.classed(c[g],!1);return o=l.get(p).apply(this),d.classed(p,!0).style("top",o.top+h[0]+v+"px").style("left",o.left+h[1]+y+"px"),u},u.hide=function(){return f().style("opacity",0).style("pointer-events","none"),u},u.attr=function(t,e){if(arguments.length<2&&"string"==typeof t)return f().attr(t);var n=Array.prototype.slice.call(arguments);return yt.prototype.attr.apply(f(),n),u},u.style=function(t,e){if(arguments.length<2&&"string"==typeof t)return f().style(t);var n=Array.prototype.slice.call(arguments);return yt.prototype.style.apply(f(),n),u},u.direction=function(e){return arguments.length?(t=null==e?e:d(e),u):t},u.offset=function(t){return arguments.length?(e=null==t?t:d(t),u):e},u.html=function(t){return arguments.length?(n=null==t?t:d(t),u):n},u.rootElement=function(t){return arguments.length?(r=null==t?t:d(t),u):r},u.destroy=function(){return i&&(f().remove(),i=null),u};var l=Nr({n:function(){var t=p(this);return{top:t.n.y-i.offsetHeight,left:t.n.x-i.offsetWidth/2}},s:function(){var t=p(this);return{top:t.s.y,left:t.s.x-i.offsetWidth/2}},e:function(){var t=p(this);return{top:t.e.y-i.offsetHeight/2,left:t.e.x}},w:function(){var t=p(this);return{top:t.w.y-i.offsetHeight/2,left:t.w.x-i.offsetWidth}},nw:function(){var t=p(this);return{top:t.nw.y-i.offsetHeight,left:t.nw.x-i.offsetWidth}},ne:function(){var t=p(this);return{top:t.ne.y-i.offsetHeight,left:t.ne.x}},sw:function(){var t=p(this);return{top:t.sw.y,left:t.sw.x-i.offsetWidth}},se:function(){var t=p(this);return{top:t.se.y,left:t.se.x}}}),c=l.keys();function h(){var t=mt(document.createElement("div"));return t.style("position","absolute").style("top",0).style("opacity",0).style("pointer-events","none").style("box-sizing","border-box"),t.node()}function f(){return null==i&&(i=h(),r.appendChild(i)),mt(i)}function p(t){for(var e=s||t;null==e.getScreenCTM&&null!=e.parentNode;)e=e.parentNode;var n={},r=e.getScreenCTM(),i=e.getBBox(),o=i.width,u=i.height,l=i.x,c=i.y;return a.x=l,a.y=c,n.nw=a.matrixTransform(r),a.x+=o,n.ne=a.matrixTransform(r),a.y+=u,n.se=a.matrixTransform(r),a.x-=o,n.sw=a.matrixTransform(r),a.y-=u/2,n.w=a.matrixTransform(r),a.x+=o,n.e=a.matrixTransform(r),a.x-=o/2,a.y-=u/2,n.n=a.matrixTransform(r),a.y+=u,n.s=a.matrixTransform(r),n}function d(t){return"function"==typeof t?t:function(){return t}}return u}().attr("class","d3-tip").offset([10,0]).html((function(t){if(g&&"function"==typeof g)return g(t,w.node(t));var e=w.node(t),n="";return v?n="
"+v+"
":e.id!==r.guid&&(n="
"+(e.isLineage?"Lineage":"Impact")+"
"),n+="
"+e.toolTipLabel+"
",e.typeName&&(n+="
("+e.typeName+")
"),e.queryText&&(n+="
Query: "+e.queryText+"
"),"
"+n+"
"}));_.call(O),N(E,w),E.selectAll("g.nodes g.label").attr("transform",(function(){return k?"translate(2,-20)":"translate(2,-38)"})).attr("font-size","10px").on("mouseenter",(function(t){it.preventDefault(),mt(this).classed("highlight",!0)})).on("mouseleave",(function(t){it.preventDefault(),mt(this).classed("highlight",!1)})).on("click",(function(t){it.preventDefault(),c&&"function"==typeof c&&c({clickedData:t}),O.hide(t)})),E.selectAll("g.nodes g.node circle").on("mouseenter",(function(t,n,r){if(m.activeNode=!0,this.getScreenCTM().translate(+this.getAttribute("cx"),+this.getAttribute("cy")),m.svg.selectAll(".node").classed("active",!1),mt(this).classed("active",!0),m._getValueFromUser(u)&&0!==t.indexOf("more")){var i=_r.getToolTipDirection({el:this});O.direction(i).show(t,this)}!1!==m._getValueFromUser(l)&&_r.onHoverFade(zr({opacity:.3,mouseenter:!0,hoveredNode:t},e))})).on("mouseleave",(function(t){m.activeNode=!1;var n=this;setTimeout((function(e){m.activeTip||m.activeNode||(mt(n).classed("active",!1),m._getValueFromUser(u)&&O.hide(t))}),150),!1!==m._getValueFromUser(l)&&_r.onHoverFade(zr({mouseenter:!1,hoveredNode:t},e))})).on("click",(function(t){it.defaultPrevented||(it.preventDefault(),O.hide(t),_.selectAll("g.node>circle").classed("node-detail-highlight",!1),mt(this).classed("node-detail-highlight",!0),f&&"function"==typeof f&&f({clickedData:t,el:this}))}));var S=E.selectAll("g.edgePath");S.selectAll("path.path").on("click",(function(t){if(h&&"function"==typeof h){var e=a.relations.find((function(e){if(e.fromEntityId===t.v&&e.toEntityId===t.w)return!0}));h({pathRelationObj:e,clickedData:t})}})),!1!==p&&_r.centerNode(zr(zr({},e),{},{fitToScreen:d,svgGroupEl:E,edgePathEl:S})),_r.dragNode(zr(zr({},e),{},{edgePathEl:S})),!0!==y&&this._addLegend(),this.options.afterRender&&this.options.afterRender()}else _.html('No relations to display')}},{key:"_addLegend",value:function(){if(!1!==this.options.legends){var t=mt(this.options.legendsEl||this.options.el).insert("div",":first-child").classed("legends",!0),e=t.append("span").style("color","#fb4200");e.append("i").classed("fa fa-circle-o fa-fw",!0),e.append("span").html("Current Entity"),(e=t.append("span").style("color","#686868")).append("i").classed("fa fa-hourglass-half fa-fw",!0),e.append("span").html("In Progress"),(e=t.append("span").style("color","#df9b00")).append("i").classed("fa fa-long-arrow-right fa-fw",!0),e.append("span").html("Lineage"),(e=t.append("span").style("color","#fb4200")).append("i").classed("fa fa-long-arrow-right fa-fw",!0),e.append("span").html("Impact"),(e=t.append("span").classed("notification hide",!0).style("color","#686868")).append("i").classed("fa fa-exclamation fa-fw",!0),e.append("span").html("Filtering hides all Expand buttons.")}}},{key:"_AddFilterNotification",value:function(t,e){t||e?$(this.options.legendsEl).find(".notification").removeClass("hide"):$(this.options.legendsEl).find(".notification").addClass("hide")}}])&&Dr(e.prototype,n),r&&Dr(e,r),t}()}])})); \ No newline at end of file diff --git a/dashboardv3/public/js/external_lib/atlas-lineage/dist/styles.css b/dashboardv3/public/js/external_lib/atlas-lineage/dist/styles.css index 2d7f9d9de1a..3bcce59e583 100644 --- a/dashboardv3/public/js/external_lib/atlas-lineage/dist/styles.css +++ b/dashboardv3/public/js/external_lib/atlas-lineage/dist/styles.css @@ -1,2 +1 @@ -.node{cursor:pointer}.node text{font-size:10px;font-family:sans-serif}.node .label{fill:#868686}.node .label.highlight{cursor:pointer;fill:#4a90e2;text-decoration:underline}.node .label.highlight tspan{font-weight:400}.node circle{-moz-transition:all 0.3s;-webkit-transition:all 0.3s;transition:all 0.3s;stroke-width:1.5px}.node circle.node-detail-highlight{stroke:#4a90e2;stroke-width:2px}.node circle.nodeImage.green:hover{stroke:#ffb203}.node circle.nodeImage.blue:hover{stroke:#4b91e2}.node circle.nodeImage.currentNode{stroke:#fb4200}.node circle.nodeImage:hover{-moz-transform:scale(1.4);-webkit-transform:scale(1.4);transform:scale(1.4)}.node.active circle{-moz-transform:scale(1.4);-webkit-transform:scale(1.4);transform:scale(1.4)}.node.active circle.nodeImage.green{stroke:#ffb203}.node.active circle.nodeImage.blue{stroke:#4b91e2}.legends>span{margin-right:8px;font-family:Source Sans Pro}svg.hover g.node{opacity:0.1 !important}svg.hover g.edgePath{opacity:0 !important}svg.hover g.node.hover-active-node,svg.hover g.edgePath.hover-active-path{opacity:1 !important}.invisible .node circle{transition:all 0s}.edgePath .path{cursor:pointer}.link{fill:none;stroke:#ccc;stroke-width:1.5px}.text-center{text-align:center}.d3-tip{line-height:1;font-weight:bold;padding:12px;background:rgba(0,0,0,0.8);color:#fff;z-index:999;max-width:300px;border-radius:2px}.d3-tip .tip-inner-scroll{overflow:auto;max-height:300px}.d3-tip .tip-inner-scroll h5{margin:7px 0px}.d3-tip:after{box-sizing:border-box;display:inline;font-size:10px;width:100%;line-height:1;color:rgba(0,0,0,0.8);position:absolute}.d3-tip.n:after{content:"\25BC";margin:-1px 0 0 0;top:100%;left:0;text-align:center}.d3-tip.e:after{content:"\25C0";margin:-4px 0 0 0;top:50%;left:-8px}.d3-tip.s:after{content:"\25B2";margin:0 0 1px 0;top:-8px;left:0;text-align:center}.d3-tip.w:after{content:"\25B6";margin:-4px 0 0 -1px;top:50%;left:100%}g.type-TK>rect{fill:#00ffd0}.fullscreen-mode{position:fixed;height:100% !important;top:0;bottom:0;left:0;width:100%;right:0;padding:0 !important;z-index:9999;overflow:hidden !important}.fullscreen-mode .resizeGraph{position:fixed;height:100% !important}.fullscreen-mode .resizeGraph .ui-resizable-handle{display:none}.fullscreen-mode .lineage-box{padding:10px !important}.fullscreen-mode .box-panel{margin:10px !important}@keyframes zoominoutsinglefeatured{0%{transform:scale(1, 1)}50%{transform:scale(1.2, 1.2)}100%{transform:scale(1, 1)}}.wobble{animation:zoominoutsinglefeatured 1s 5}.hidden-svg{visibility:hidden}@-webkit-keyframes blink{from{opacity:0.2}to{opacity:0.5}} - +.node{cursor:pointer}.node text{font-size:10px;font-family:sans-serif}.node .label{fill:#868686}.node .label.highlight{cursor:pointer;fill:#4a90e2;text-decoration:underline}.node .label.highlight tspan{font-weight:400}.node circle{-moz-transition:all 0.3s;-webkit-transition:all 0.3s;transition:all 0.3s;stroke-width:1.5px}.node circle.node-detail-highlight{stroke:#4a90e2;stroke-width:2px}.node circle.nodeImage.green:hover{stroke:#ffb203}.node circle.nodeImage.blue:hover{stroke:#4b91e2}.node circle.nodeImage.currentNode{stroke:#fb4200}.node circle.nodeImage:hover{-moz-transform:scale(1.4);-webkit-transform:scale(1.4);transform:scale(1.4)}.node.active circle{-moz-transform:scale(1.4);-webkit-transform:scale(1.4);transform:scale(1.4)}.node.active circle.nodeImage.green{stroke:#ffb203}.node.active circle.nodeImage.blue{stroke:#4b91e2}.legends>span{margin-right:8px;font-family:Source Sans Pro}svg.hover g.node{opacity:0.1 !important}svg.hover g.edgePath{opacity:0 !important}svg.hover g.node.hover-active-node,svg.hover g.edgePath.hover-active-path{opacity:1 !important}.invisible .node circle{transition:all 0s}.edgePath .path{cursor:pointer}.link{fill:none;stroke:#ccc;stroke-width:1.5px}.text-center{text-align:center}.d3-tip{line-height:1;font-weight:bold;padding:12px;background:rgba(0,0,0,0.8);color:#fff;z-index:999;max-width:300px;border-radius:2px;word-break:break-all}.d3-tip .tip-inner-scroll{overflow:auto;max-height:300px}.d3-tip .tip-inner-scroll h5{margin:7px 0px}.d3-tip:after{box-sizing:border-box;display:inline;font-size:10px;width:100%;line-height:1;color:rgba(0,0,0,0.8);position:absolute}.d3-tip.n:after{content:"\25BC";margin:-1px 0 0 0;top:100%;left:0;text-align:center}.d3-tip.e:after{content:"\25C0";margin:-4px 0 0 0;top:50%;left:-8px}.d3-tip.s:after{content:"\25B2";margin:0 0 1px 0;top:-8px;left:0;text-align:center}.d3-tip.w:after{content:"\25B6";margin:-4px 0 0 -1px;top:50%;left:100%}g.type-TK>rect{fill:#00ffd0}.fullscreen-mode{position:fixed;height:100% !important;top:0;bottom:0;left:0;width:100%;right:0;padding:0 !important;z-index:9999;overflow:hidden !important}.fullscreen-mode .resizeGraph{position:fixed;height:100% !important}.fullscreen-mode .resizeGraph .ui-resizable-handle{display:none}.fullscreen-mode .lineage-box{padding:10px !important}.fullscreen-mode .box-panel{margin:10px !important}@keyframes zoominoutsinglefeatured{0%{transform:scale(1, 1)}50%{transform:scale(1.2, 1.2)}100%{transform:scale(1, 1)}}.wobble{animation:zoominoutsinglefeatured 1s 5}.hidden-svg{visibility:hidden}@-webkit-keyframes blink{from{opacity:0.2}to{opacity:0.5}} diff --git a/dashboardv3/public/js/external_lib/atlas-lineage/package-lock.json b/dashboardv3/public/js/external_lib/atlas-lineage/package-lock.json index 69f5a400691..751a7075520 100644 --- a/dashboardv3/public/js/external_lib/atlas-lineage/package-lock.json +++ b/dashboardv3/public/js/external_lib/atlas-lineage/package-lock.json @@ -1207,9 +1207,9 @@ "dev": true }, "acorn": { - "version": "6.4.1", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.4.1.tgz", - "integrity": "sha512-ZVA9k326Nwrj3Cj9jlh3wGFutC2ZornPNARZwsNYqQYgN0EsV2d53w5RN/co65Ohn4sUAUtb1rSUAOD6XN9idA==", + "version": "6.4.2", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.4.2.tgz", + "integrity": "sha512-XtGIhXwF8YM8bJhGxG5kXgjkEuNGLTkoYqVE+KMR+aspr4KGYmKYg7yUe3KghyQ9yheNwLnjmzh/7+gfDBmHCQ==", "dev": true }, "ajv": { @@ -1304,9 +1304,9 @@ }, "dependencies": { "bn.js": { - "version": "4.11.9", - "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.9.tgz", - "integrity": "sha512-E6QoYqCKZfgatHTdHzs1RRKP7ip4vvm+EyRUeE2RF0NblwVvb0p6jSVeNTOFxPn26QXN2o6SMfNxKp6kU8zQaw==", + "version": "4.12.0", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz", + "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==", "dev": true } } @@ -1324,13 +1324,13 @@ "inherits": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz", - "integrity": "sha1-sX0I0ya0Qj5Wjv9xn5GwscvfafE=", + "integrity": "sha512-8nWq2nLTAwd02jTqJExUYFSD/fKq6VH9Y/oG2accc/kdI0V98Bag8d5a4gi3XHz73rDWa2PvTtvcWYquKqSENA==", "dev": true }, "util": { "version": "0.10.3", "resolved": "https://registry.npmjs.org/util/-/util-0.10.3.tgz", - "integrity": "sha1-evsa/lCAUkZInj23/g7TeTNqwPk=", + "integrity": "sha512-5KiHfsmkqacuKjkRkdV7SsfDJ2EGiPsK92s2MhNSY0craxjTdKTtqKsJaCWp4LW33ZZ0OPUv1WO/TFvNQRiQxQ==", "dev": true, "requires": { "inherits": "2.0.1" @@ -1571,9 +1571,9 @@ "dev": true }, "balanced-match": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", - "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", "dev": true }, "base": { @@ -1632,9 +1632,9 @@ } }, "base64-js": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz", - "integrity": "sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g==", + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", "dev": true }, "big.js": { @@ -1666,9 +1666,9 @@ "dev": true }, "bn.js": { - "version": "5.1.3", - "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-5.1.3.tgz", - "integrity": "sha512-GkTiFpjFtUzU9CbMeJ5iazkCzGL3jrhzerzZIuqLABjbwRaFt33I9tUdSNryIptM+RxDet6OKm2WnLXzW51KsQ==", + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-5.2.1.tgz", + "integrity": "sha512-eXRvHzWyYPBuB4NBy0cmYQjGitUrtqwbvlzP3G6VFnNRbsZQIxQ10PbKKHt8gZ/HW/D/747aDl+QkDqg3KQLMQ==", "dev": true }, "brace-expansion": { @@ -1693,7 +1693,7 @@ "brorand": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/brorand/-/brorand-1.1.0.tgz", - "integrity": "sha1-EsJe/kCkXjwyPrhnWgoM5XsiNx8=", + "integrity": "sha512-cKV8tMCEpQs4hK/ik71d6LrPOnpkpGBR0wzxqr68g2m/LB2GxVYQroAjMJZRVM1Y4BCjCKc3vAamxSzOY2RP+w==", "dev": true }, "browserify-aes": { @@ -1734,21 +1734,13 @@ } }, "browserify-rsa": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/browserify-rsa/-/browserify-rsa-4.0.1.tgz", - "integrity": "sha1-IeCr+vbyApzy+vsTNWenAdQTVSQ=", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/browserify-rsa/-/browserify-rsa-4.1.0.tgz", + "integrity": "sha512-AdEER0Hkspgno2aR97SAf6vi0y0k8NuOpGnVH3O99rcA5Q6sh8QxcngtHuJ6uXwnfAXNM4Gn1Gb7/MV1+Ymbog==", "dev": true, "requires": { - "bn.js": "^4.1.0", + "bn.js": "^5.0.0", "randombytes": "^2.0.1" - }, - "dependencies": { - "bn.js": { - "version": "4.11.9", - "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.9.tgz", - "integrity": "sha512-E6QoYqCKZfgatHTdHzs1RRKP7ip4vvm+EyRUeE2RF0NblwVvb0p6jSVeNTOFxPn26QXN2o6SMfNxKp6kU8zQaw==", - "dev": true - } } }, "browserify-sign": { @@ -1803,15 +1795,35 @@ } }, "browserslist": { - "version": "4.14.0", - "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.14.0.tgz", - "integrity": "sha512-pUsXKAF2lVwhmtpeA3LJrZ76jXuusrNyhduuQs7CDFf9foT4Y38aQOserd2lMe5DSSrjf3fx34oHwryuvxAUgQ==", + "version": "4.21.4", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.21.4.tgz", + "integrity": "sha512-CBHJJdDmgjl3daYjN5Cp5kbTf1mUhZoS+beLklHIvkOWscs83YAhLlF3Wsh/lciQYAcbBJgTOD44VtG31ZM4Hw==", "dev": true, "requires": { - "caniuse-lite": "^1.0.30001111", - "electron-to-chromium": "^1.3.523", - "escalade": "^3.0.2", - "node-releases": "^1.1.60" + "caniuse-lite": "^1.0.30001400", + "electron-to-chromium": "^1.4.251", + "node-releases": "^2.0.6", + "update-browserslist-db": "^1.0.9" + }, + "dependencies": { + "caniuse-lite": { + "version": "1.0.30001407", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001407.tgz", + "integrity": "sha512-4ydV+t4P7X3zH83fQWNDX/mQEzYomossfpViCOx9zHBSMV+rIe3LFqglHHtVyvNl1FhTNxPxs3jei82iqOW04w==", + "dev": true + }, + "electron-to-chromium": { + "version": "1.4.255", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.255.tgz", + "integrity": "sha512-H+mFNKow6gi2P5Gi2d1Fvd3TUEJlB9CF7zYaIV9T83BE3wP1xZ0mRPbNTm0KUjyd1QiVy7iKXuIcjlDtBQMiAQ==", + "dev": true + }, + "node-releases": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.6.tgz", + "integrity": "sha512-PiVXnNuFm5+iYkLBNeq5211hvO38y63T0i2KKh2KnUs3RpzJ+JtODFjkD8yjLwnDkTYF1eKXheUwdssR+NRZdg==", + "dev": true + } } }, "buffer": { @@ -1826,21 +1838,21 @@ } }, "buffer-from": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", - "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==", + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", + "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", "dev": true }, "buffer-xor": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/buffer-xor/-/buffer-xor-1.0.3.tgz", - "integrity": "sha1-JuYe0UIvtw3ULm42cp7VHYVf6Nk=", + "integrity": "sha512-571s0T7nZWK6vB67HI5dyUF7wXiNcfaPPPTl6zYCNApANjIvYJTg7hlud/+cJpdAhS7dVzqMLmfhfHR3rAcOjQ==", "dev": true }, "builtin-status-codes": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/builtin-status-codes/-/builtin-status-codes-3.0.0.tgz", - "integrity": "sha1-hZgoeOIbmOHGZCXgPQF0eI9Wnug=", + "integrity": "sha512-HpGFw18DgFWlncDfjTa2rcQ4W88O1mC8e8yZ2AvQY5KDaktSTwo+KRf6nHK6FRI5FyRyb/5T6+TSxfP7QyGsmQ==", "dev": true }, "cacache": { @@ -1889,12 +1901,6 @@ "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==", "dev": true }, - "caniuse-lite": { - "version": "1.0.30001118", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001118.tgz", - "integrity": "sha512-RNKPLojZo74a0cP7jFMidQI7nvLER40HgNfgKQEJ2PFm225L0ectUungNQoK3Xk3StQcFbpBPNEvoWD59436Hg==", - "dev": true - }, "chalk": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", @@ -1929,13 +1935,10 @@ "dev": true }, "chrome-trace-event": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/chrome-trace-event/-/chrome-trace-event-1.0.2.tgz", - "integrity": "sha512-9e/zx1jw7B4CO+c/RXoCsfg/x1AfUBioy4owYH0bJprEYAx5hRFLRhWBqHAG57D0ZM4H7vxbP7bPe0VwhQRYDQ==", - "dev": true, - "requires": { - "tslib": "^1.9.0" - } + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/chrome-trace-event/-/chrome-trace-event-1.0.3.tgz", + "integrity": "sha512-p3KULyQg4S7NIHixdwbGX+nFHkoBiA4YQmyWtjb8XngSKV124nJmRysgAeujbUVb15vh+RvFUfCPqU7rXk+hZg==", + "dev": true }, "cipher-base": { "version": "1.0.4", @@ -2054,7 +2057,7 @@ "concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", - "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", "dev": true }, "concat-stream": { @@ -2078,7 +2081,7 @@ "constants-browserify": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/constants-browserify/-/constants-browserify-1.0.0.tgz", - "integrity": "sha1-wguW2MYXdIqvHBYCF2DNJ/y4y3U=", + "integrity": "sha512-xFxOwqIzR/e1k1gLiWEophSCMqXcwVHIH7akf7b/vxcUeGunlj3hvZaaqxwHsTgn+IndtkQJgSztIDWeumWJDQ==", "dev": true }, "convert-source-map": { @@ -2151,9 +2154,9 @@ }, "dependencies": { "bn.js": { - "version": "4.11.9", - "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.9.tgz", - "integrity": "sha512-E6QoYqCKZfgatHTdHzs1RRKP7ip4vvm+EyRUeE2RF0NblwVvb0p6jSVeNTOFxPn26QXN2o6SMfNxKp6kU8zQaw==", + "version": "4.12.0", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz", + "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==", "dev": true } } @@ -2255,7 +2258,7 @@ "cyclist": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/cyclist/-/cyclist-1.0.1.tgz", - "integrity": "sha1-WW6WmP0MgOEgOMK4LW6xs1tiJNk=", + "integrity": "sha512-NJGVKPS81XejHcLhaLJS7plab0fK3slPh11mESeeDq2W4ZI5kUKK/LRRdVDvjJseojbPB7ZwjnyOybg3Igea/A==", "dev": true }, "d3": { @@ -2567,9 +2570,9 @@ "dev": true }, "decode-uri-component": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.2.0.tgz", - "integrity": "sha1-6zkTMzRYd1y4TNGh+uBiEGu4dUU=", + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.2.2.tgz", + "integrity": "sha512-FqUYQ+8o158GyGTrMFJms9qh3CqTKvAqgqsTnkLI8sKu0028orqBhxNMFkFen0zGyg6epACD32pjVk58ngIErQ==", "dev": true }, "define-properties": { @@ -2650,9 +2653,9 @@ }, "dependencies": { "bn.js": { - "version": "4.11.9", - "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.9.tgz", - "integrity": "sha512-E6QoYqCKZfgatHTdHzs1RRKP7ip4vvm+EyRUeE2RF0NblwVvb0p6jSVeNTOFxPn26QXN2o6SMfNxKp6kU8zQaw==", + "version": "4.12.0", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz", + "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==", "dev": true } } @@ -2675,31 +2678,31 @@ "stream-shift": "^1.0.0" } }, - "electron-to-chromium": { - "version": "1.3.549", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.3.549.tgz", - "integrity": "sha512-q09qZdginlqDH3+Y1P6ch5UDTW8nZ1ijwMkxFs15J/DAWOwqolIx8HZH1UP0vReByBigk/dPlU22xS1MaZ+kpQ==", - "dev": true - }, "elliptic": { - "version": "6.5.3", - "resolved": "https://registry.npmjs.org/elliptic/-/elliptic-6.5.3.tgz", - "integrity": "sha512-IMqzv5wNQf+E6aHeIqATs0tOLeOTwj1QKbRcS3jBbYkl5oLAserA8yJTT7/VyHUYG91PRmPyeQDObKLPpeS4dw==", + "version": "6.5.4", + "resolved": "https://registry.npmjs.org/elliptic/-/elliptic-6.5.4.tgz", + "integrity": "sha512-iLhC6ULemrljPZb+QutR5TQGB+pdW6KGD5RSegS+8sorOZT+rdQFbsQFJgvN3eRqNALqJer4oQ16YvJHlU8hzQ==", "dev": true, "requires": { - "bn.js": "^4.4.0", - "brorand": "^1.0.1", + "bn.js": "^4.11.9", + "brorand": "^1.1.0", "hash.js": "^1.0.0", - "hmac-drbg": "^1.0.0", - "inherits": "^2.0.1", - "minimalistic-assert": "^1.0.0", - "minimalistic-crypto-utils": "^1.0.0" + "hmac-drbg": "^1.0.1", + "inherits": "^2.0.4", + "minimalistic-assert": "^1.0.1", + "minimalistic-crypto-utils": "^1.0.1" }, "dependencies": { "bn.js": { - "version": "4.11.9", - "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.9.tgz", - "integrity": "sha512-E6QoYqCKZfgatHTdHzs1RRKP7ip4vvm+EyRUeE2RF0NblwVvb0p6jSVeNTOFxPn26QXN2o6SMfNxKp6kU8zQaw==", + "version": "4.12.0", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz", + "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==", + "dev": true + }, + "inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", "dev": true } } @@ -2757,12 +2760,6 @@ "prr": "~1.0.1" } }, - "escalade": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.0.2.tgz", - "integrity": "sha512-gPYAU37hYCUhW5euPeR+Y74F7BL+IBsV93j5cvGriSaD1aG6MGsqsV1yamRdrWrb2j3aiZvb0X+UBOWpx3JWtQ==", - "dev": true - }, "escape-string-regexp": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", @@ -2780,12 +2777,20 @@ } }, "esrecurse": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.2.1.tgz", - "integrity": "sha512-64RBB++fIOAXPw3P9cy89qfMlvZEXZkqqJkjqqXIvzP5ezRZjW+lPWjw35UX/3EhUPFYbg5ER4JYgDw4007/DQ==", + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", + "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", "dev": true, "requires": { - "estraverse": "^4.1.0" + "estraverse": "^5.2.0" + }, + "dependencies": { + "estraverse": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "dev": true + } } }, "estraverse": { @@ -2801,9 +2806,9 @@ "dev": true }, "events": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/events/-/events-3.2.0.tgz", - "integrity": "sha512-/46HWwbfCX2xTawVfkKLGxMifJYQBWMwY1mjywRtb4c9x8l5NP3KoJtnIOiL1hfdRkIuYhETxQlo62IF8tcnlg==", + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", + "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", "dev": true }, "evp_bytestokey": { @@ -3055,7 +3060,7 @@ "from2": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/from2/-/from2-2.3.0.tgz", - "integrity": "sha1-i/tVAr3kpNNs/e6gB/zKIdfjgq8=", + "integrity": "sha512-OMcX/4IC/uqEPVgGeyfN22LJk6AZrMkRZHxcHBMBvHScDGgwTm2GT2Wkgtocyd3JfZffjj2kYUDXXII0Fk9W0g==", "dev": true, "requires": { "inherits": "^2.0.1", @@ -3065,7 +3070,7 @@ "fs-write-stream-atomic": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/fs-write-stream-atomic/-/fs-write-stream-atomic-1.0.10.tgz", - "integrity": "sha1-tH31NJPvkR33VzHnCp3tAYnbQMk=", + "integrity": "sha512-gehEzmPn2nAwr39eay+x3X34Ra+M2QlVUTLhkXPjWdeO8RF9kszk116avgBJM3ZyNHgHXBNx+VmPaFC36k0PzA==", "dev": true, "requires": { "graceful-fs": "^4.1.2", @@ -3077,7 +3082,7 @@ "fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", - "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", "dev": true }, "fsevents": { @@ -3112,15 +3117,15 @@ "dev": true }, "glob": { - "version": "7.1.6", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", - "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==", + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", "dev": true, "requires": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", "inherits": "2", - "minimatch": "^3.0.4", + "minimatch": "^3.1.1", "once": "^1.3.0", "path-is-absolute": "^1.0.0" } @@ -3311,7 +3316,7 @@ "hmac-drbg": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/hmac-drbg/-/hmac-drbg-1.0.1.tgz", - "integrity": "sha1-0nRXAQJabHdabFRXk+1QL8DGSaE=", + "integrity": "sha512-Tti3gMqLdZfhOQY1Mzf/AanLiqh1WTiJgEj26ZuYQ9fbkLomzGchCws4FyrSd4VkpBfiNhaE1On+lOz894jvXg==", "dev": true, "requires": { "hash.js": "^1.0.3", @@ -3331,7 +3336,7 @@ "https-browserify": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/https-browserify/-/https-browserify-1.0.0.tgz", - "integrity": "sha1-7AbBDgo0wPL68Zn3/X/Hj//QPHM=", + "integrity": "sha512-J+FkSdyD+0mA0N+81tMotaRMfSL9SGi+xpD3T6YApKsc3bGSXJlfXri3VyFOeYkfLRQisDk1W+jIFFKBeUBbBg==", "dev": true }, "iconv-lite": { @@ -3352,15 +3357,15 @@ } }, "ieee754": { - "version": "1.1.13", - "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", - "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==", + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", "dev": true }, "iferr": { "version": "0.1.5", "resolved": "https://registry.npmjs.org/iferr/-/iferr-0.1.5.tgz", - "integrity": "sha1-xg7taebY/bazEEofy8ocGS3FtQE=", + "integrity": "sha512-DUNFN5j7Tln0D+TxzloUjKB+CtVu6myn0JEFak6dG18mNt9YkQ6lzGCdafwofISZ1lLF3xRHJ98VKy9ynkcFaA==", "dev": true }, "import-local": { @@ -3376,7 +3381,7 @@ "imurmurhash": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", - "integrity": "sha1-khi5srkoojixPcT7a21XbyMUU+o=", + "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", "dev": true }, "indexes-of": { @@ -3394,7 +3399,7 @@ "inflight": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", - "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", "dev": true, "requires": { "once": "^1.3.0", @@ -3559,7 +3564,7 @@ "is-wsl": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-1.1.0.tgz", - "integrity": "sha1-HxbkqiKwTRM2tmGIpmrzxgDDpm0=", + "integrity": "sha512-gfygJYZ2gLTDlmbWMI0CE2MwnFzSN/2SZfkMlItC4K/JBlsWVDB0bO6XhqcY13YXE7iMcAJnzTCJjPiTeJJ0Mw==", "dev": true }, "isarray": { @@ -3641,9 +3646,9 @@ "dev": true }, "loader-utils": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/loader-utils/-/loader-utils-1.4.0.tgz", - "integrity": "sha512-qH0WSMBtn/oHuwjy/NucEgbx5dbxxnxup9s4PVXJUDHZBQY+s0NWA9rJf53RBnQZxfch7euUui7hpoAPvALZdA==", + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/loader-utils/-/loader-utils-1.4.2.tgz", + "integrity": "sha512-I5d00Pd/jwMD2QCduo657+YM/6L3KZu++pmX9VFncxaxvHcru9jx1lBaFft+r4Mt2jK0Yhp41XlRAihzPxHNCg==", "dev": true, "requires": { "big.js": "^5.2.2", @@ -3673,9 +3678,9 @@ } }, "lodash": { - "version": "4.17.20", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz", - "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==" + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, "loose-envify": { "version": "1.4.0", @@ -3734,7 +3739,7 @@ "memory-fs": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/memory-fs/-/memory-fs-0.4.1.tgz", - "integrity": "sha1-OpoguEYlI+RHz7x+i7gO1me/xVI=", + "integrity": "sha512-cda4JKCxReDXFXRqOHPQscuIYg1PvxbE2S2GP45rnwfEK+vZaXC8C1OFvdHIbgw0DLzowXGVoxLaAmlgRy14GQ==", "dev": true, "requires": { "errno": "^0.1.3", @@ -3857,9 +3862,9 @@ }, "dependencies": { "bn.js": { - "version": "4.11.9", - "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.9.tgz", - "integrity": "sha512-E6QoYqCKZfgatHTdHzs1RRKP7ip4vvm+EyRUeE2RF0NblwVvb0p6jSVeNTOFxPn26QXN2o6SMfNxKp6kU8zQaw==", + "version": "4.12.0", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz", + "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==", "dev": true } } @@ -3898,22 +3903,22 @@ "minimalistic-crypto-utils": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/minimalistic-crypto-utils/-/minimalistic-crypto-utils-1.0.1.tgz", - "integrity": "sha1-9sAMHAsIIkblxNmd+4x8CDsrWCo=", + "integrity": "sha512-JIYlbt6g8i5jKfJ3xz7rF0LXmv2TkDxBLUkiBeZ7bAx4GnnNMr8xFpGnOxn6GhTEHx3SjRrZEoU+j04prX1ktg==", "dev": true }, "minimatch": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", - "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", "dev": true, "requires": { "brace-expansion": "^1.1.7" } }, "minimist": { - "version": "1.2.5", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", - "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==", + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz", + "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==", "dev": true }, "mississippi": { @@ -3967,7 +3972,7 @@ "move-concurrently": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/move-concurrently/-/move-concurrently-1.0.1.tgz", - "integrity": "sha1-viwAX9oy4LKa8fBdfEszIUxwH5I=", + "integrity": "sha512-hdrFxZOycD/g6A6SoI2bB5NA/5NEqD0569+S47WZhPvm46sD50ZHdYaFmnua5lndde9rCHGjmfK7Z8BuCt/PcQ==", "dev": true, "requires": { "aproba": "^1.1.1", @@ -3985,9 +3990,9 @@ "dev": true }, "nan": { - "version": "2.14.1", - "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.1.tgz", - "integrity": "sha512-isWHgVjnFjh2x2yuJ/tj3JbwoHu3UC2dX5G/88Cm24yB6YopVgxvBObDY7n5xW6ExmFhJpSEQqFPvq9zaXc8Jw==", + "version": "2.16.0", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.16.0.tgz", + "integrity": "sha512-UdAqHyFngu7TfQKsCBgAA6pWDkT8MAO7d0jyOecVhN5354xbLqdn8mV9Tat9gepAupm0bt2DbeaSC8vS52MuFA==", "dev": true, "optional": true }, @@ -4056,7 +4061,7 @@ "punycode": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", - "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=", + "integrity": "sha512-jmYNElW7yvO7TV33CjSmvSiE2yco3bV2czu/OzDKdMNVZQWfxCblURLhf+47syQRBntjfLdd/H0egrzIG+oaFQ==", "dev": true }, "util": { @@ -4070,12 +4075,6 @@ } } }, - "node-releases": { - "version": "1.1.60", - "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-1.1.60.tgz", - "integrity": "sha512-gsO4vjEdQaTusZAEebUWp2a5d7dF5DYoIpDG7WySnk7BuZDW+GPpHXoXXuYawRBr/9t5q54tirPz79kFIWg4dA==", - "dev": true - }, "normalize-path": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", @@ -4170,7 +4169,7 @@ "once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", - "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", "dev": true, "requires": { "wrappy": "1" @@ -4179,7 +4178,7 @@ "os-browserify": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/os-browserify/-/os-browserify-0.3.0.tgz", - "integrity": "sha1-hUNzx/XCMVkU/Jv8a9gjj92h7Cc=", + "integrity": "sha512-gjcpUc3clBf9+210TRaDWbf+rZZZEshZ+DlXMRCeAjp0xhTrnQsKHypIy1J3d5hKdUzj69t708EHtU8P6bUn0A==", "dev": true }, "p-limit": { @@ -4267,7 +4266,7 @@ "path-dirname": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/path-dirname/-/path-dirname-1.0.2.tgz", - "integrity": "sha1-zDPSTVJeCZpTiMAzbG4yuRYGCeA=", + "integrity": "sha512-ALzNPpyNq9AqXMBjeymIjFDAkAFH06mHJH/cSBHAgU0s4vfpBn6b2nf8tiRLvagKD8RbTpq2FKTBg7cl9l3c7Q==", "dev": true, "optional": true }, @@ -4280,7 +4279,7 @@ "path-is-absolute": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", - "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", "dev": true }, "path-key": { @@ -4290,15 +4289,15 @@ "dev": true }, "path-parse": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.6.tgz", - "integrity": "sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw==", + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", + "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", "dev": true }, "pbkdf2": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/pbkdf2/-/pbkdf2-3.1.1.tgz", - "integrity": "sha512-4Ejy1OPxi9f2tt1rRV7Go7zmfDQ+ZectEQz3VGUQhgq62HtIRPDyG/JtnwIxs6x3uNMwo2V7q1fMvKjb+Tnpqg==", + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/pbkdf2/-/pbkdf2-3.1.2.tgz", + "integrity": "sha512-iuh7L6jA7JEGu2WxDwtQP1ddOpaJNC4KlDEFfdQajSGgGPNi4OyDc2R7QnbY2bR9QjBVGwgvTdNJZoE7RaxUMA==", "dev": true, "requires": { "create-hash": "^1.1.2", @@ -4308,6 +4307,12 @@ "sha.js": "^2.4.8" } }, + "picocolors": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-0.2.1.tgz", + "integrity": "sha512-cMlDqaLEqfSaW8Z7N5Jw+lyIW869EzT73/F5lhtY9cLGoVxSXznfgfXMO0Z5K0o0Q2TkTXq+0KFsdnSe3jDViA==", + "dev": true + }, "picomatch": { "version": "2.2.2", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.2.2.tgz", @@ -4341,14 +4346,13 @@ "dev": true }, "postcss": { - "version": "7.0.32", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-7.0.32.tgz", - "integrity": "sha512-03eXong5NLnNCD05xscnGKGDZ98CyzoqPSMjOe6SuoQY7Z2hIj0Ld1g/O/UQRuOle2aRtiIRDg9tDcTGAkLfKw==", + "version": "7.0.39", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-7.0.39.tgz", + "integrity": "sha512-yioayjNbHn6z1/Bywyb2Y4s3yvDAeXGOyxqD+LnVOinq6Mdmd++SW2wUNVzavyyHxd6+DxzWGIuosg6P1Rj8uA==", "dev": true, "requires": { - "chalk": "^2.4.2", - "source-map": "^0.6.1", - "supports-color": "^6.1.0" + "picocolors": "^0.2.1", + "source-map": "^0.6.1" }, "dependencies": { "source-map": { @@ -4356,15 +4360,6 @@ "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", "dev": true - }, - "supports-color": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-6.1.0.tgz", - "integrity": "sha512-qe1jfm1Mg7Nq/NSh6XE24gPXROEVsWHxC1LIx//XNlD9iw7YZQGjZNjYN7xGaEG6iKdA8EtNFW6R0gjnVXp+wQ==", - "dev": true, - "requires": { - "has-flag": "^3.0.0" - } } } }, @@ -4447,7 +4442,7 @@ "promise-inflight": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/promise-inflight/-/promise-inflight-1.0.1.tgz", - "integrity": "sha1-mEcocL8igTL8vdhoEputEsPAKeM=", + "integrity": "sha512-6zWPyEOFaQBJYcGMHBKTKJ3u6TBsnMFOIZSa6ce1e/ZrrsOlnHRHbabMjLiBYKp+n44X9eUI6VUPaukCXHuG4g==", "dev": true }, "prr": { @@ -4471,9 +4466,9 @@ }, "dependencies": { "bn.js": { - "version": "4.11.9", - "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.9.tgz", - "integrity": "sha512-E6QoYqCKZfgatHTdHzs1RRKP7ip4vvm+EyRUeE2RF0NblwVvb0p6jSVeNTOFxPn26QXN2o6SMfNxKp6kU8zQaw==", + "version": "4.12.0", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz", + "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==", "dev": true } } @@ -4530,13 +4525,13 @@ "querystring": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", - "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=", + "integrity": "sha512-X/xY82scca2tau62i9mDyU9K+I+djTMUsvwf7xnUX5GLvVzgJybOJf4Y6o9Zx3oJK/LSXg5tTZBjwzqVPaPO2g==", "dev": true }, "querystring-es3": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/querystring-es3/-/querystring-es3-0.2.1.tgz", - "integrity": "sha1-nsYfeQSYdXB9aUFFlv2Qek1xHnM=", + "integrity": "sha512-773xhDQnZBMFobEiztv8LIl70ch5MSF/jUQVlhwFyBILqq96anmoctVIYz+ZRp0qbCKATTn6ev02M3r7Ga5vqA==", "dev": true }, "randombytes": { @@ -4662,7 +4657,7 @@ "remove-trailing-separator": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/remove-trailing-separator/-/remove-trailing-separator-1.1.0.tgz", - "integrity": "sha1-wkvOKig62tW8P1jg1IJJuSN52O8=", + "integrity": "sha512-/hS+Y0u3aOfIETiaiirUFwDBDzmXPvO+jAfKTitUngIPzdKc6Z0LoFjM/CK5PL4C+eKwHohlHAb6H0VFfmmUsw==", "dev": true, "optional": true }, @@ -4771,7 +4766,7 @@ "run-queue": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/run-queue/-/run-queue-1.0.3.tgz", - "integrity": "sha1-6Eg5bwV9Ij8kOGkkYY4laUFh7Ec=", + "integrity": "sha512-ntymy489o0/QQplUDnpYAYUsO50K9SBrIVaKCWDOJzYJts0f9WH9RFJkyagebkw5+y1oi00R7ynNW/d12GBumg==", "dev": true, "requires": { "aproba": "^1.1.1" @@ -4890,7 +4885,7 @@ "setimmediate": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz", - "integrity": "sha1-KQy7Iy4waULX1+qbg3Mqt4VvgoU=", + "integrity": "sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==", "dev": true }, "sha.js": { @@ -5084,9 +5079,9 @@ } }, "source-map-support": { - "version": "0.5.19", - "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.19.tgz", - "integrity": "sha512-Wonm7zOCIJzBGQdB+thsPar0kYuCIzYvxZwlBa87yi/Mdjv7Tip2cyVbLj5o0cFPN4EVkuTwb3GDDyUx2DGnGw==", + "version": "0.5.21", + "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.21.tgz", + "integrity": "sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==", "dev": true, "requires": { "buffer-from": "^1.0.0", @@ -5117,9 +5112,9 @@ } }, "ssri": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/ssri/-/ssri-6.0.1.tgz", - "integrity": "sha512-3Wge10hNcT1Kur4PDFwEieXSCMCJs/7WvSACcrMYrNp+b8kDL1/0wJch5Ni2WrtwEa2IO8OsVfeKIciKCDx/QA==", + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/ssri/-/ssri-6.0.2.tgz", + "integrity": "sha512-cepbSq/neFK7xB6A50KHN0xHDotYzq58wWCa5LeWqnPrHG8GzfEjO/4O8kpmcGW+oaxkvhEJCWgbgNk4/ZV93Q==", "dev": true, "requires": { "figgy-pudding": "^3.5.1" @@ -5237,15 +5232,6 @@ "ansi-regex": "^2.0.0" } }, - "supports-color": { - "version": "5.5.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", - "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", - "dev": true, - "requires": { - "has-flag": "^3.0.0" - } - }, "tapable": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/tapable/-/tapable-1.1.3.tgz", @@ -5253,9 +5239,9 @@ "dev": true }, "terser": { - "version": "4.8.0", - "resolved": "https://registry.npmjs.org/terser/-/terser-4.8.0.tgz", - "integrity": "sha512-EAPipTNeWsb/3wLPeup1tVPaXfIaU68xMnVdPafIL1TV05OhASArYyIfFvnvJCNrR2NIOvDVNNTFRa+Re2MWyw==", + "version": "4.8.1", + "resolved": "https://registry.npmjs.org/terser/-/terser-4.8.1.tgz", + "integrity": "sha512-4GnLC0x667eJG0ewJTa6z/yXrbLGv80D9Ru6HIpCQmO+Q4PfEtBFi0ObSckqwL6VyQv/7ENJieXHo2ANmdQwgw==", "dev": true, "requires": { "commander": "^2.20.0", @@ -5318,9 +5304,9 @@ } }, "timers-browserify": { - "version": "2.0.11", - "resolved": "https://registry.npmjs.org/timers-browserify/-/timers-browserify-2.0.11.tgz", - "integrity": "sha512-60aV6sgJ5YEbzUdn9c8kYGIqOubPoUdqQCul3SBAsRCZ40s6Y5cMcrW4dt3/k/EsbLVJNl9n6Vz3fTc+k2GeKQ==", + "version": "2.0.12", + "resolved": "https://registry.npmjs.org/timers-browserify/-/timers-browserify-2.0.12.tgz", + "integrity": "sha512-9phl76Cqm6FhSX9Xe1ZUAMLtm1BLkKj2Qd5ApyWkXzsMRaA7dgr81kf4wJmQf/hAvg8EEyJxDo3du/0KlhPiKQ==", "dev": true, "requires": { "setimmediate": "^1.0.4" @@ -5329,7 +5315,7 @@ "to-arraybuffer": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/to-arraybuffer/-/to-arraybuffer-1.0.1.tgz", - "integrity": "sha1-fSKbH8xjfkZsoIEYCDanqr/4P0M=", + "integrity": "sha512-okFlQcoGTi4LQBG/PgSYblw9VOyptsz2KJZqc6qtgGdes8VktzUQkj4BI2blit072iS8VODNcMA+tvnS9dnuMA==", "dev": true }, "to-fast-properties": { @@ -5379,22 +5365,16 @@ "is-number": "^7.0.0" } }, - "tslib": { - "version": "1.13.0", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.13.0.tgz", - "integrity": "sha512-i/6DQjL8Xf3be4K/E6Wgpekn5Qasl1usyw++dAA35Ue5orEn65VIxOA+YvNNl9HV3qv70T7CNwjODHZrLwvd1Q==", - "dev": true - }, "tty-browserify": { "version": "0.0.0", "resolved": "https://registry.npmjs.org/tty-browserify/-/tty-browserify-0.0.0.tgz", - "integrity": "sha1-oVe6QC2iTpv5V/mqadUk7tQpAaY=", + "integrity": "sha512-JVa5ijo+j/sOoHGjw0sxw734b1LhBkQ3bvUGNdxnVXDCX81Yx7TFgnZygxrIIWn23hbfTaMYLwRmAxFyDuFmIw==", "dev": true }, "typedarray": { "version": "0.0.6", "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", - "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=", + "integrity": "sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA==", "dev": true }, "unicode-canonical-property-names-ecmascript": { @@ -5508,6 +5488,24 @@ "dev": true, "optional": true }, + "update-browserslist-db": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.9.tgz", + "integrity": "sha512-/xsqn21EGVdXI3EXSum1Yckj3ZVZugqyOZQ/CxYPBD/R+ko9NSUScf8tFF4dOKY+2pvSSJA/S+5B8s4Zr4kyvg==", + "dev": true, + "requires": { + "escalade": "^3.1.1", + "picocolors": "^1.0.0" + }, + "dependencies": { + "escalade": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", + "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==", + "dev": true + } + } + }, "uri-js": { "version": "4.2.2", "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.2.2.tgz", @@ -5526,7 +5524,7 @@ "url": { "version": "0.11.0", "resolved": "https://registry.npmjs.org/url/-/url-0.11.0.tgz", - "integrity": "sha1-ODjpfPxgUh63PFJajlW/3Z4uKPE=", + "integrity": "sha512-kbailJa29QrtXnxgq+DdCEGlbTeYM2eJUxsz6vjZavrCYPMIFHMKQmSKYAIuUK2i7hgPm28a8piX5NTUtM/LKQ==", "dev": true, "requires": { "punycode": "1.3.2", @@ -5536,7 +5534,7 @@ "punycode": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", - "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=", + "integrity": "sha512-RofWgt/7fL5wP1Y7fxE7/EmTLzQVnB0ycyibJ0OOHIlJqTNzglYFxVwETOcIoJqJmpDXJ9xImDv+Fq34F/d4Dw==", "dev": true } } @@ -5575,21 +5573,21 @@ "dev": true }, "watchpack": { - "version": "1.7.4", - "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-1.7.4.tgz", - "integrity": "sha512-aWAgTW4MoSJzZPAicljkO1hsi1oKj/RRq/OJQh2PKI2UKL04c2Bs+MBOB+BBABHTXJpf9mCwHN7ANCvYsvY2sg==", + "version": "1.7.5", + "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-1.7.5.tgz", + "integrity": "sha512-9P3MWk6SrKjHsGkLT2KHXdQ/9SNkyoJbabxnKOoJepsvJjJG8uYTR3yTPxPQvNDI3w4Nz1xnE0TLHK4RIVe/MQ==", "dev": true, "requires": { "chokidar": "^3.4.1", "graceful-fs": "^4.1.2", "neo-async": "^2.5.0", - "watchpack-chokidar2": "^2.0.0" + "watchpack-chokidar2": "^2.0.1" } }, "watchpack-chokidar2": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/watchpack-chokidar2/-/watchpack-chokidar2-2.0.0.tgz", - "integrity": "sha512-9TyfOyN/zLUbA288wZ8IsMZ+6cbzvsNyEzSBp6e/zkifi6xxbl8SmQ/CxQq32k8NNqrdVEVUVSEf56L4rQ/ZxA==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/watchpack-chokidar2/-/watchpack-chokidar2-2.0.1.tgz", + "integrity": "sha512-nCFfBIPKr5Sh61s4LPpy1Wtfi0HE8isJ3d2Yb5/Ppw2P2B/3eVSEBjKfN0fmHJSK14+31KwMKmcrzs2GM4P0Ww==", "dev": true, "optional": true, "requires": { @@ -5610,7 +5608,7 @@ "normalize-path": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-2.1.1.tgz", - "integrity": "sha1-GrKLVW4Zg2Oowab35vogE3/mrtk=", + "integrity": "sha512-3pKJwH184Xo/lnH6oyP1q2pMd7HcypqqmRs91/6/i2CGtWwIKGCkOOMTm/zXbgTEWHw1uNpNi/igc3ePOYHb6w==", "dev": true, "optional": true, "requires": { @@ -5669,7 +5667,7 @@ "extend-shallow": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", - "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=", + "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==", "dev": true, "optional": true, "requires": { @@ -5679,7 +5677,7 @@ "fill-range": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-4.0.0.tgz", - "integrity": "sha1-1USBHUKPmOsGpj3EAtJAPDKMOPc=", + "integrity": "sha512-VcpLTWqWDiTerugjj8e3+esbg+skS3M9e54UuR3iCeIDMXCLTsAH8hTSzDQU/X6/6t3eYkOKoZSef2PlU6U1XQ==", "dev": true, "optional": true, "requires": { @@ -5703,7 +5701,7 @@ "glob-parent": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-3.1.0.tgz", - "integrity": "sha1-nmr2KZ2NO9K9QEMIMr0RPfkGxa4=", + "integrity": "sha512-E8Ak/2+dZY6fnzlR7+ueWvhsH1SjHr4jjss4YS/h4py44jY9MhK/VFdaZJAWDz6BbL21KeteKxFSFpq8OS5gVA==", "dev": true, "optional": true, "requires": { @@ -5714,7 +5712,7 @@ "is-glob": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-3.1.0.tgz", - "integrity": "sha1-e6WuJCF4BKxwcHuWkiVnSGzD6Eo=", + "integrity": "sha512-UFpDDrPgM6qpnFNI+rh/p3bUaq9hKLZN8bMUWzxmcnZVS3omf4IPK+BrewlnWjO1WmUsMYuSjKh4UJuV4+Lqmw==", "dev": true, "optional": true, "requires": { @@ -5726,7 +5724,7 @@ "is-binary-path": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-1.0.1.tgz", - "integrity": "sha1-dfFmQrSA8YenEcgUFh/TpKdlWJg=", + "integrity": "sha512-9fRVlXc0uCxEDj1nQzaWONSpbTfx0FmJfzHF7pwlI8DkWGoHBBea4Pg5Ky0ojwwxQmnSifgbKkI06Qv0Ljgj+Q==", "dev": true, "optional": true, "requires": { @@ -5736,7 +5734,7 @@ "is-number": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/is-number/-/is-number-3.0.0.tgz", - "integrity": "sha1-JP1iAaR4LPUFYcgQJ2r8fRLXEZU=", + "integrity": "sha512-4cboCqIpliH+mAvFNegjZQ4kgKc3ZUhQVr3HvWbSh5q3WH2v82ct+T2Y1hdU5Gdtorx/cLifQjqCbL7bpznLTg==", "dev": true, "optional": true, "requires": { @@ -5746,7 +5744,7 @@ "kind-of": { "version": "3.2.2", "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", - "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=", + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", "dev": true, "optional": true, "requires": { @@ -5768,7 +5766,7 @@ "to-regex-range": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-2.1.1.tgz", - "integrity": "sha1-fIDBe53+vlmeJzZ+DU3VWQFB2zg=", + "integrity": "sha512-ZZWNfCjUokXXDGXFpZehJIkZqq91BcULFq/Pi7M5i4JnxXdhMKAK682z8bCW3o8Hj1wuuzoKcW3DfVzaP6VuNg==", "dev": true, "optional": true, "requires": { @@ -5925,7 +5923,7 @@ "wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", "dev": true }, "xtend": { diff --git a/dashboardv3/public/js/external_lib/atlas-lineage/package.json b/dashboardv3/public/js/external_lib/atlas-lineage/package.json index 6e03eee3f5a..02772d11038 100644 --- a/dashboardv3/public/js/external_lib/atlas-lineage/package.json +++ b/dashboardv3/public/js/external_lib/atlas-lineage/package.json @@ -37,7 +37,7 @@ "path": "^0.12.7", "sass": "^1.26.5", "sass-loader": "^8.0.2", - "webpack": "^4.43.0", + "webpack": "^4.44.1", "webpack-cli": "^3.3.11" } } diff --git a/dashboardv3/public/js/external_lib/atlas-lineage/src/Utils/DataUtils.js b/dashboardv3/public/js/external_lib/atlas-lineage/src/Utils/DataUtils.js index 1255eb070f0..7d2acf4a72a 100644 --- a/dashboardv3/public/js/external_lib/atlas-lineage/src/Utils/DataUtils.js +++ b/dashboardv3/public/js/external_lib/atlas-lineage/src/Utils/DataUtils.js @@ -19,343 +19,354 @@ import Enums from "../Enums"; import { curveBasis } from "d3-shape"; const DataUtils = { - /** - * [getBaseUrl description] - * @param {[type]} url [description] - * @return {[type]} [description] - */ - getBaseUrl: function(url) { - return url.replace(/\/[\w-]+.(jsp|html)|\/+$/gi, ""); - }, - /** - * [getEntityIconPath description] - * @param {[type]} options.entityData [description] - * @param {Object} options.errorUrl } [description] - * @return {[type]} [description] - */ - getEntityIconPath: function({ entityData, errorUrl } = {}) { - var serviceType, - status, - typeName, - iconBasePath = this.getBaseUrl(window.location.pathname) + Globals.entityImgPath; - if (entityData) { - typeName = entityData.typeName; - serviceType = entityData && entityData.serviceType; - status = entityData && entityData.status; - } + /** + * [getBaseUrl description] + * @param {[type]} url [description] + * @return {[type]} [description] + */ + getBaseUrl: function(url) { + return url.replace(/\/[\w-]+.(jsp|html)|\/+$/gi, ""); + }, + /** + * [getEntityIconPath description] + * @param {[type]} options.entityData [description] + * @param {Object} options.errorUrl } [description] + * @return {[type]} [description] + */ + getEntityIconPath: function({ entityData, errorUrl } = {}) { + var serviceType, + status, + typeName, + iconBasePath = this.getBaseUrl(window.location.pathname) + Globals.entityImgPath; + if (entityData) { + typeName = entityData.typeName; + serviceType = entityData && entityData.serviceType; + status = entityData && entityData.status; + } - function getImgPath(imageName) { - return iconBasePath + (Enums.entityStateReadOnly[status] ? "disabled/" + imageName : imageName); - } + function getImgPath(imageName) { + return iconBasePath + (Enums.entityStateReadOnly[status] ? "disabled/" + imageName : imageName); + } - function getDefaultImgPath() { - if (entityData.isProcess) { - if (Enums.entityStateReadOnly[status]) { - return iconBasePath + "disabled/process.png"; - } else { - return iconBasePath + "process.png"; - } - } else { - if (Enums.entityStateReadOnly[status]) { - return iconBasePath + "disabled/table.png"; - } else { - return iconBasePath + "table.png"; - } - } - } + function getDefaultImgPath() { + if (entityData.isProcess) { + if (Enums.entityStateReadOnly[status]) { + return iconBasePath + "disabled/process.png"; + } else { + return iconBasePath + "process.png"; + } + } else { + if (Enums.entityStateReadOnly[status]) { + return iconBasePath + "disabled/table.png"; + } else { + return iconBasePath + "table.png"; + } + } + } - if (entityData) { - if (errorUrl) { - var isErrorInTypeName = errorUrl && errorUrl.match("entity-icon/" + typeName + ".png|disabled/" + typeName + ".png") ? true : false; - if (serviceType && isErrorInTypeName) { - var imageName = serviceType + ".png"; - return getImgPath(imageName); - } else { - return getDefaultImgPath(); - } - } else if (entityData.typeName) { - var imageName = entityData.typeName + ".png"; - return getImgPath(imageName); - } else { - return getDefaultImgPath(); - } - } - }, - /** - * [isProcess description] - * @param {[type]} options.typeName [description] - * @param {[type]} options.superTypes [description] - * @param {[type]} options.entityDef [description] - * @return {Boolean} [description] - */ - isProcess: function({ typeName, superTypes, entityDef }) { - if (typeName == "Process") { - return true; - } - return superTypes.indexOf("Process") > -1; - }, - /** - * [isDeleted description] - * @param {[type]} node [description] - * @return {Boolean} [description] - */ - isDeleted: function(node) { - if (node === undefined) { - return; - } - return Enums.entityStateReadOnly[node.status]; - }, - isNodeToBeUpdated: function(node, filterObj) { - var isProcessHideCheck = filterObj.isProcessHideCheck, - isDeletedEntityHideCheck = filterObj.isDeletedEntityHideCheck; - var returnObj = { - isProcess: isProcessHideCheck && node.isProcess, - isDeleted: isDeletedEntityHideCheck && node.isDeleted - }; - returnObj["update"] = returnObj.isProcess || returnObj.isDeleted; - return returnObj; - }, - /** - * [getServiceType description] - * @param {[type]} options.typeName [description] - * @param {[type]} options.entityDef [description] - * @return {[type]} [description] - */ - getServiceType: function({ typeName, entityDef }) { - var serviceType = null; - if (typeName) { - if (entityDef) { - serviceType = entityDef.serviceType || null; - } - } - return serviceType; - }, - /** - * [getEntityDef description] - * @param {[type]} options.typeName [description] - * @param {[type]} options.entityDefCollection [description] - * @return {[type]} [description] - */ - getEntityDef: function({ typeName, entityDefCollection }) { - var entityDef = null; - if (typeName) { - entityDef = entityDefCollection.find(function(obj) { - return obj.name == typeName; - }); - } - return entityDef; - }, - /** - * [getNestedSuperTypes description] - * @param {[type]} options.entityDef [description] - * @param {[type]} options.entityDefCollection [description] - * @return {[type]} [description] - */ - getNestedSuperTypes: function({ entityDef, entityDefCollection }) { - var data = entityDef, - collection = entityDefCollection, - superTypes = new Set(); + if (entityData) { + if (errorUrl) { + var isErrorInTypeName = errorUrl && errorUrl.match("entity-icon/" + typeName + ".png|disabled/" + typeName + ".png") ? true : false; + if (serviceType && isErrorInTypeName) { + var imageName = serviceType + ".png"; + return getImgPath(imageName); + } else { + return getDefaultImgPath(); + } + } else if (entityData.typeName) { + var imageName = entityData.typeName + ".png"; + return getImgPath(imageName); + } else { + return getDefaultImgPath(); + } + } + }, + /** + * [isProcess description] + * @param {[type]} options.typeName [description] + * @param {[type]} options.superTypes [description] + * @param {[type]} options.entityDef [description] + * @return {Boolean} [description] + */ + isProcess: function({ typeName, superTypes, entityDef }) { + if (typeName == "Process") { + return true; + } + return superTypes.indexOf("Process") > -1; + }, + /** + * [isDeleted description] + * @param {[type]} node [description] + * @return {Boolean} [description] + */ + isDeleted: function(node) { + if (node === undefined) { + return; + } + return Enums.entityStateReadOnly[node.status]; + }, + isNodeToBeUpdated: function(node, filterObj) { + var isProcessHideCheck = filterObj.isProcessHideCheck, + isDeletedEntityHideCheck = filterObj.isDeletedEntityHideCheck; + var returnObj = { + isProcess: isProcessHideCheck && node.isProcess, + isDeleted: isDeletedEntityHideCheck && node.isDeleted + }; + returnObj["update"] = returnObj.isProcess || returnObj.isDeleted; + if (node.label === "Expand") { + returnObj["update"] = true; + } + return returnObj; + }, + /** + * [getServiceType description] + * @param {[type]} options.typeName [description] + * @param {[type]} options.entityDef [description] + * @return {[type]} [description] + */ + getServiceType: function({ typeName, entityDef }) { + var serviceType = null; + if (typeName) { + if (entityDef) { + serviceType = entityDef.serviceType || null; + } + } + return serviceType; + }, + /** + * [getEntityDef description] + * @param {[type]} options.typeName [description] + * @param {[type]} options.entityDefCollection [description] + * @return {[type]} [description] + */ + getEntityDef: function({ typeName, entityDefCollection }) { + var entityDef = null; + if (typeName) { + entityDef = entityDefCollection.find(function(obj) { + return obj.name == typeName; + }); + } + return entityDef; + }, + /** + * [getNestedSuperTypes description] + * @param {[type]} options.entityDef [description] + * @param {[type]} options.entityDefCollection [description] + * @return {[type]} [description] + */ + getNestedSuperTypes: function({ entityDef, entityDefCollection }) { + var data = entityDef, + collection = entityDefCollection, + superTypes = new Set(); - var getData = function(data, collection) { - if (data) { - if (data.superTypes && data.superTypes.length) { - data.superTypes.forEach(function(superTypeName) { - superTypes.add(superTypeName); - var collectionData = collection.find(function(obj) { - obj.name === superTypeName; - }); - if (collectionData) { - getData(collectionData, collection); - } - }); - } - } - }; - getData(data, collection); - return Array.from(superTypes); - }, - generateData: function({ data = {}, filterObj, entityDefCollection, g, guid, setGraphEdge, setGraphNode }) { - return new Promise((resolve, reject) => { - try { - var relations = data.relations || {}, - guidEntityMap = data.guidEntityMap || {}, - isHideFilterOn = filterObj.isProcessHideCheck || filterObj.isDeletedEntityHideCheck, - newHashMap = {}, - styleObj = { - fill: "none", - stroke: "#ffb203", - width: 3 - }, - makeNodeData = (relationObj) => { - if (relationObj) { - if (relationObj.updatedValues) { - return relationObj; - } - var obj = Object.assign(relationObj, { - shape: "img", - updatedValues: true, - label: relationObj.displayText.trunc(18), - toolTipLabel: relationObj.displayText, - id: relationObj.guid, - isLineage: true, - isIncomplete: relationObj.isIncomplete, - entityDef: this.getEntityDef({ typeName: relationObj.typeName, entityDefCollection }) - }); - obj["serviceType"] = this.getServiceType(obj); - obj["superTypes"] = this.getNestedSuperTypes({ - ...obj, - entityDefCollection: entityDefCollection - }); - obj["isProcess"] = this.isProcess(obj); - obj["isDeleted"] = this.isDeleted(obj); - return obj; - } - }, - crateLineageRelationshipHashMap = function({ relations } = {}) { - var newHashMap = {}; - relations.forEach(function(obj) { - if (newHashMap[obj.fromEntityId]) { - newHashMap[obj.fromEntityId].push(obj.toEntityId); - } else { - newHashMap[obj.fromEntityId] = [obj.toEntityId]; - } - }); - return newHashMap; - }, - getStyleObjStr = function(styleObj) { - return "fill:" + styleObj.fill + ";stroke:" + styleObj.stroke + ";stroke-width:" + styleObj.width; - }, - getNewToNodeRelationship = (toNodeGuid, filterObj) => { - if (toNodeGuid && relationshipMap[toNodeGuid]) { - var newRelationship = []; - relationshipMap[toNodeGuid].forEach((guid) => { - var nodeToBeUpdated = this.isNodeToBeUpdated(makeNodeData(guidEntityMap[guid]), filterObj); - if (nodeToBeUpdated.update) { - var newRelation = getNewToNodeRelationship(guid, filterObj); - if (newRelation) { - newRelationship = newRelationship.concat(newRelation); - } - } else { - newRelationship.push(guid); - } - }); - return newRelationship; - } else { - return null; - } - }, - getToNodeRelation = (toNodes, fromNodeToBeUpdated, filterObj) => { - var toNodeRelationship = []; - toNodes.forEach((toNodeGuid) => { - var toNodeToBeUpdated = this.isNodeToBeUpdated(makeNodeData(guidEntityMap[toNodeGuid]), filterObj); - if (toNodeToBeUpdated.update) { - // To node need to updated - if (pendingFromRelationship[toNodeGuid]) { - toNodeRelationship = toNodeRelationship.concat(pendingFromRelationship[toNodeGuid]); - } else { - var newToNodeRelationship = getNewToNodeRelationship(toNodeGuid, filterObj); - if (newToNodeRelationship) { - toNodeRelationship = toNodeRelationship.concat(newToNodeRelationship); - } - } - } else { - //when bothe node not to be updated. - toNodeRelationship.push(toNodeGuid); - } - }); - return toNodeRelationship; - }, - setNode = (guid) => { - if (!g._nodes[guid]) { - var nodeData = makeNodeData(guidEntityMap[guid]); - setGraphNode(guid, nodeData); - return nodeData; - } else { - return g._nodes[guid]; - } - }, - setEdge = function(fromNodeGuid, toNodeGuid, opt = {}) { - setGraphEdge(fromNodeGuid, toNodeGuid, { - arrowhead: "arrowPoint", - curve: curveBasis, - style: getStyleObjStr(styleObj), - styleObj: styleObj, - ...opt - }); - }, - setGraphData = function(fromEntityId, toEntityId) { - setNode(fromEntityId); - setNode(toEntityId); - setEdge(fromEntityId, toEntityId); - }, - pendingFromRelationship = {}; - if (isHideFilterOn) { - var relationshipMap = crateLineageRelationshipHashMap(data); - Object.keys(relationshipMap).forEach((fromNodeGuid) => { - var toNodes = relationshipMap[fromNodeGuid], - fromNodeToBeUpdated = this.isNodeToBeUpdated(makeNodeData(guidEntityMap[fromNodeGuid]), filterObj), - toNodeList = getToNodeRelation(toNodes, fromNodeToBeUpdated, filterObj); - if (fromNodeToBeUpdated.update) { - if (pendingFromRelationship[fromNodeGuid]) { - pendingFromRelationship[fromNodeGuid] = pendingFromRelationship[fromNodeGuid].concat(toNodeList); - } else { - pendingFromRelationship[fromNodeGuid] = toNodeList; - } - } else { - toNodeList.forEach(function(toNodeGuid) { - setGraphData(fromNodeGuid, toNodeGuid); - }); - } - }); - } else { - relations.forEach(function(obj) { - setGraphData(obj.fromEntityId, obj.toEntityId); - }); - } - if (g._nodes[guid]) { - if (g._nodes[guid]) { - g._nodes[guid]["isLineage"] = false; - } - this.findImpactNodeAndUpdateData({ - guid, - g, - setEdge, - getStyleObjStr - }); - } - resolve(g); - } catch (e) { - reject(e); - } - }); - }, - findImpactNodeAndUpdateData: function({ guid, getStyleObjStr, g, setEdge }) { - var that = this, - traversedMap = {}, - styleObj = { - fill: "none", - stroke: "#fb4200", - width: 3 - }, - traversed = function(toNodeList = {}, fromNodeGuid) { - let toNodeKeyList = Object.keys(toNodeList); - if (toNodeKeyList.length) { - if (!traversedMap[fromNodeGuid]) { - traversedMap[fromNodeGuid] = true; - toNodeKeyList.forEach(function(toNodeGuid) { - if (g._nodes[toNodeGuid]) { - g._nodes[toNodeGuid]["isLineage"] = false; - } - setEdge(fromNodeGuid, toNodeGuid, { - style: getStyleObjStr(styleObj), - styleObj: styleObj - }); - traversed(g._sucs[toNodeGuid], toNodeGuid); - }); - } - } - }; - traversed(g._sucs[guid], guid); - } + var getData = function(data, collection) { + if (data) { + if (data.superTypes && data.superTypes.length) { + data.superTypes.forEach(function(superTypeName) { + superTypes.add(superTypeName); + var collectionData = collection.find(function(obj) { + obj.name === superTypeName; + }); + if (collectionData) { + getData(collectionData, collection); + } + }); + } + } + }; + getData(data, collection); + return Array.from(superTypes); + }, + generateData: function({ data = {}, filterObj, entityDefCollection, g, guid, setGraphEdge, setGraphNode }) { + return new Promise((resolve, reject) => { + try { + var relations = data.relations || {}, + guidEntityMap = data.guidEntityMap || {}, + isHideFilterOn = filterObj.isProcessHideCheck || filterObj.isDeletedEntityHideCheck, + newHashMap = {}, + styleObj = { + fill: "none", + stroke: "#ffb203", + width: 3 + }, + makeNodeData = (relationObj) => { + if (relationObj) { + if (relationObj.updatedValues) { + return relationObj; + } + var nodeLabel = relationObj.displayText ? relationObj.displayText : " ", + obj = Object.assign(relationObj, { + shape: "img", + updatedValues: true, + label: nodeLabel.trunc(18), + toolTipLabel: nodeLabel, + id: relationObj.guid, + isLineage: true, + isIncomplete: relationObj.isIncomplete, + entityDef: this.getEntityDef({ typeName: relationObj.typeName, entityDefCollection }) + }); + obj["serviceType"] = this.getServiceType(obj); + obj["superTypes"] = this.getNestedSuperTypes({ + ...obj, + entityDefCollection: entityDefCollection + }); + obj["isProcess"] = this.isProcess(obj); + obj["isDeleted"] = this.isDeleted(obj); + return obj; + } + }, + crateLineageRelationshipHashMap = function({ relations } = {}) { + var newHashMap = {}; + relations.forEach(function(obj) { + if (newHashMap[obj.fromEntityId]) { + newHashMap[obj.fromEntityId].push(obj.toEntityId); + } else { + newHashMap[obj.fromEntityId] = [obj.toEntityId]; + } + }); + return newHashMap; + }, + getStyleObjStr = function(styleObj) { + return "fill:" + styleObj.fill + ";stroke:" + styleObj.stroke + ";stroke-width:" + styleObj.width; + }, + getNewToNodeRelationship = (toNodeGuid, filterObj) => { + if (toNodeGuid && relationshipMap[toNodeGuid]) { + var newRelationship = []; + relationshipMap[toNodeGuid].forEach((guid) => { + var nodeToBeUpdated = this.isNodeToBeUpdated(makeNodeData(guidEntityMap[guid]), filterObj); + if (nodeToBeUpdated.update) { + var newRelation = getNewToNodeRelationship(guid, filterObj); + if (newRelation) { + newRelationship = newRelationship.concat(newRelation); + } + } else { + newRelationship.push(guid); + } + }); + return newRelationship; + } else { + return null; + } + }, + getToNodeRelation = (toNodes, fromNodeToBeUpdated, filterObj) => { + var toNodeRelationship = []; + toNodes.forEach((toNodeGuid) => { + var toNodeToBeUpdated = this.isNodeToBeUpdated(makeNodeData(guidEntityMap[toNodeGuid]), filterObj); + if (toNodeToBeUpdated.update) { + // To node need to updated + if (pendingFromRelationship[toNodeGuid]) { + toNodeRelationship = toNodeRelationship.concat(pendingFromRelationship[toNodeGuid]); + } else { + var newToNodeRelationship = getNewToNodeRelationship(toNodeGuid, filterObj); + if (newToNodeRelationship) { + toNodeRelationship = toNodeRelationship.concat(newToNodeRelationship); + } + } + } else { + //when bothe node not to be updated. + toNodeRelationship.push(toNodeGuid); + } + }); + return toNodeRelationship; + }, + setNode = (guid) => { + if (!g._nodes[guid]) { + var nodeData = makeNodeData(guidEntityMap[guid]); + setGraphNode(guid, nodeData); + return nodeData; + } else { + return g._nodes[guid]; + } + }, + setEdge = function(fromNodeGuid, toNodeGuid, opt = {}) { + setGraphEdge(fromNodeGuid, toNodeGuid, { + arrowhead: "arrowPoint", + curve: curveBasis, + style: getStyleObjStr(styleObj), + styleObj: styleObj, + ...opt + }); + }, + setGraphData = function(fromEntityId, toEntityId) { + setNode(fromEntityId); + setNode(toEntityId); + setEdge(fromEntityId, toEntityId); + }, + pendingFromRelationship = {}; + if (isHideFilterOn) { + var relationshipMap = crateLineageRelationshipHashMap(data); + Object.keys(relationshipMap).forEach((fromNodeGuid) => { + var toNodes = relationshipMap[fromNodeGuid], + fromNodeToBeUpdated = this.isNodeToBeUpdated(makeNodeData(guidEntityMap[fromNodeGuid]), filterObj), + toNodeList = getToNodeRelation(toNodes, fromNodeToBeUpdated, filterObj); + if (fromNodeToBeUpdated.update) { + if (pendingFromRelationship[fromNodeGuid]) { + pendingFromRelationship[fromNodeGuid] = pendingFromRelationship[fromNodeGuid].concat(toNodeList); + } else { + pendingFromRelationship[fromNodeGuid] = toNodeList; + } + } else { + toNodeList.forEach(function(toNodeGuid) { + setGraphData(fromNodeGuid, toNodeGuid); + }); + } + }); + } else { + if (relations.length) { + relations.forEach(function(obj) { + setGraphData(obj.fromEntityId, obj.toEntityId); + }); + } else { + setNode(guid); + } + } + if (g._nodes[guid]) { + if (g._nodes[guid]) { + g._nodes[guid]["isLineage"] = false; + } + this.findImpactNodeAndUpdateData({ + guid, + g, + setEdge, + getStyleObjStr + }); + } + if (!g._nodes[guid]) { + setNode(guid); + } + resolve(g); + } catch (e) { + reject(e); + } + }); + }, + findImpactNodeAndUpdateData: function({ guid, getStyleObjStr, g, setEdge }) { + var that = this, + traversedMap = {}, + styleObj = { + fill: "none", + stroke: "#fb4200", + width: 3 + }, + traversed = function(toNodeList = {}, fromNodeGuid) { + let toNodeKeyList = Object.keys(toNodeList); + if (toNodeKeyList.length) { + if (!traversedMap[fromNodeGuid]) { + traversedMap[fromNodeGuid] = true; + toNodeKeyList.forEach(function(toNodeGuid) { + if (g._nodes[toNodeGuid]) { + g._nodes[toNodeGuid]["isLineage"] = false; + } + setEdge(fromNodeGuid, toNodeGuid, { + style: getStyleObjStr(styleObj), + styleObj: styleObj + }); + traversed(g._sucs[toNodeGuid], toNodeGuid); + }); + } + } + }; + traversed(g._sucs[guid], guid); + } }; export default DataUtils; \ No newline at end of file diff --git a/dashboardv3/public/js/external_lib/atlas-lineage/src/Utils/LineageUtils.js b/dashboardv3/public/js/external_lib/atlas-lineage/src/Utils/LineageUtils.js index 448a95ae6eb..588e3ac67d0 100644 --- a/dashboardv3/public/js/external_lib/atlas-lineage/src/Utils/LineageUtils.js +++ b/dashboardv3/public/js/external_lib/atlas-lineage/src/Utils/LineageUtils.js @@ -31,25 +31,25 @@ const LineageUtils = { * @type {Number} */ nodeArrowDistance: 24, - refreshGraphForSafari: function (options) { + refreshGraphForSafari: function(options) { var edgePathEl = options.edgeEl, IEGraphRenderDone = 0; - edgePathEl.each(function (argument) { + edgePathEl.each(function(argument) { var eleRef = this, childNode = $(this).find("pattern"); - setTimeout(function (argument) { + setTimeout(function(argument) { $(eleRef).find("defs").append(childNode); }, 500); }); }, - refreshGraphForIE: function ({ edgePathEl }) { + refreshGraphForIE: function({ edgePathEl }) { var IEGraphRenderDone = 0; - edgePathEl.each(function (argument) { + edgePathEl.each(function(argument) { var childNode = $(this).find("marker"); $(this).find("marker").remove(); var eleRef = this; ++IEGraphRenderDone; - setTimeout(function (argument) { + setTimeout(function(argument) { $(eleRef).find("defs").append(childNode); --IEGraphRenderDone; if (IEGraphRenderDone === 0) { @@ -67,9 +67,9 @@ const LineageUtils = { * @param {[type]} options.edgePathEl [description] * @return {[type]} [description] */ - dragNode: function ({ g, svg, guid, edgePathEl }) { + dragNode: function({ g, svg, guid, edgePathEl }) { var dragHelper = { - dragmove: function (el, d) { + dragmove: function(el, d) { var node = select(el), selectedNode = g.node(d), prevX = selectedNode.x, @@ -91,13 +91,13 @@ const LineageUtils = { }); //LineageUtils.refreshGraphForIE({ edgePathEl: edgePathEl }); }, - translateEdge: function (e, dx, dy) { - e.points.forEach(function (p) { + translateEdge: function(e, dx, dy) { + e.points.forEach(function(p) { p.x = p.x + dx; p.y = p.y + dy; }); }, - calcPoints: function (e) { + calcPoints: function(e) { var edge = g.edge(e.v, e.w), tail = g.node(e.v), head = g.node(e.w), @@ -106,10 +106,10 @@ const LineageUtils = { points.unshift(this.intersectRect(tail, points[0])); points.push(this.intersectRect(head, points[points.length - 1])); return line() - .x(function (d) { + .x(function(d) { return d.x; }) - .y(function (d) { + .y(function(d) { return d.y; }) .curve(curveBasis)(points); @@ -146,10 +146,10 @@ const LineageUtils = { }; } }; - var dragNodeHandler = drag().on("drag", function (d) { + var dragNodeHandler = drag().on("drag", function(d) { dragHelper.dragmove.call(dragHelper, this, d); }), - dragEdgePathHandler = drag().on("drag", function (d) { + dragEdgePathHandler = drag().on("drag", function(d) { dragHelper.translateEdge(g.edge(d.v, d.w), event.dx, event.dy); var edgeObj = g.edge(d.v, d.w); select(edgeObj.elem).select("path").attr("d", dragHelper.calcPoints(d)); @@ -158,16 +158,16 @@ const LineageUtils = { dragNodeHandler(svg.selectAll("g.node")); dragEdgePathHandler(svg.selectAll("g.edgePath")); }, - zoomIn: function ({ svg, scaleFactor = 1.3 }) { + zoomIn: function({ svg, scaleFactor = 1.3 }) { this.d3Zoom.scaleBy(svg.transition().duration(750), scaleFactor); }, - zoomOut: function ({ svg, scaleFactor = 0.8 }) { + zoomOut: function({ svg, scaleFactor = 0.8 }) { this.d3Zoom.scaleBy(svg.transition().duration(750), scaleFactor); }, - zoom: function ({ svg, xa, ya, scale }) { + zoom: function({ svg, xa, ya, scale }) { svg.transition().duration(750).call(this.d3Zoom.transform, zoomIdentity.translate(xa, ya).scale(scale)); }, - fitToScreen: function ({ svg }) { + fitToScreen: function({ svg }) { var node = svg.node(); var bounds = node.getBBox(); @@ -197,14 +197,14 @@ const LineageUtils = { * @param {[type]} options.onCenterZoomed [description] * @return {[type]} [description] */ - centerNode: function ({ guid, g, svg, svgGroupEl, edgePathEl, width, height, fitToScreen, onCenterZoomed }) { + centerNode: function({ guid, g, svg, svgGroupEl, edgePathEl, width, height, fitToScreen, onCenterZoomed, isSelected }) { this.d3Zoom = zoom(); svg.call(this.d3Zoom).on("dblclick.zoom", null); // restrict events let selectedNodeEl = svg.selectAll("g.nodes>g[id='" + guid + "']"), - zoomListener = this.d3Zoom.scaleExtent([0.01, 50]).on("zoom", function () { + zoomListener = this.d3Zoom.scaleExtent([0.01, 50]).on("zoom", function() { svgGroupEl.attr("transform", event.transform); }), x = null, @@ -236,7 +236,13 @@ const LineageUtils = { var xa = -(x * scale - width / 2), ya = -(y * scale - height / 2); this.zoom({ svg, xa, ya, scale }); - svg.transition().duration(750).call(this.d3Zoom.transform, zoomIdentity.translate(xa, ya).scale(scale)); + + if (!isSelected) { + svg.call(this.d3Zoom.transform, zoomIdentity.translate(xa, ya).scale(scale)); + } else { + svg.transition().duration(750).call(this.d3Zoom.transform, zoomIdentity.translate(xa, ya).scale(scale)); + } + if (onCenterZoomed) { onCenterZoomed({ newScale: scale, newTranslate: [xa, ya], d3Zoom: this.d3Zoom, selectedNodeEl }); @@ -250,7 +256,7 @@ const LineageUtils = { * @param {[type]} options.el [description] * @return {[type]} [description] */ - getToolTipDirection: function ({ el }) { + getToolTipDirection: function({ el }) { var width = select("body").node().getBoundingClientRect().width, currentELWidth = select(el).node().getBoundingClientRect(), direction = "e"; @@ -279,10 +285,10 @@ const LineageUtils = { * @param {[type]} options.hoveredNode [description] * @return {[type]} [description] */ - onHoverFade: function ({ svg, g, mouseenter, nodesToHighlight, hoveredNode }) { + onHoverFade: function({ svg, g, mouseenter, nodesToHighlight, hoveredNode }) { var node = svg.selectAll(".node"), path = svg.selectAll(".edgePath"), - isConnected = function (a, b, o) { + isConnected = function(a, b, o) { if (a === o || (b && b.length && b.indexOf(o) != -1)) { return true; } @@ -292,14 +298,14 @@ const LineageUtils = { var nextNode = g.successors(hoveredNode), previousNode = g.predecessors(hoveredNode), nodesToHighlight = nextNode.concat(previousNode); - node.classed("hover-active-node", function (currentNode, i, nodes) { + node.classed("hover-active-node", function(currentNode, i, nodes) { if (isConnected(hoveredNode, nodesToHighlight, currentNode)) { return true; } else { return false; } }); - path.classed("hover-active-path", function (c) { + path.classed("hover-active-path", function(c) { var _thisOpacity = c.v === hoveredNode || c.w === hoveredNode ? 1 : 0; if (_thisOpacity) { return true; @@ -318,10 +324,10 @@ const LineageUtils = { * @param {[type]} path [description] * @return {[type]} [description] */ - getBaseUrl: function (url = window.location.pathname) { + getBaseUrl: function(url = window.location.pathname) { return url.replace(/\/[\w-]+.(jsp|html)|\/+$/gi, ""); }, - getEntityIconPath: function ({ entityData, errorUrl, imgBasePath }) { + getEntityIconPath: function({ entityData, errorUrl, imgBasePath }) { var iconBasePath = this.getBaseUrl() + (imgBasePath || "/img/entity-icon/"); if (entityData) { let { typeName, serviceType, status, isProcess } = entityData; @@ -348,7 +354,7 @@ const LineageUtils = { if (errorUrl) { // Check if the default img path has error, if yes then stop recursion. - if (errorUrl.indexOf("table.png") > -1 || errorUrl.indexOf("process.png") > -1) { + if (errorUrl.indexOf("table.png") > -1) { //removed condition for default process image return null; } var isErrorInTypeName = errorUrl && errorUrl.match("entity-icon/" + typeName + ".png|disabled/" + typeName + ".png") ? true : false; @@ -369,15 +375,15 @@ const LineageUtils = { } } }, - base64Encode: function (file, callback) { + base64Encode: function(file, callback) { const reader = new FileReader(); reader.addEventListener("load", () => callback(reader.result)); reader.readAsDataURL(file); }, - imgShapeRender: function (parent, bbox, node, { dagreD3, defsEl, imgBasePath, guid, isRankdirToBottom }) { + imgShapeRender: function(parent, bbox, node, { dagreD3, defsEl, imgBasePath, guid, isRankdirToBottom }) { var that = this, viewGuid = guid, - imageIconPath = this.getEntityIconPath({ entityData: node, imgBasePath }), + imageIconPath = node.btnType ? "/img/entity-icon/expandBtn.svg" : this.getEntityIconPath({ entityData: node, imgBasePath }), imgName = imageIconPath.split("/").pop(); if (this.imageObject === undefined) { this.imageObject = {}; @@ -390,7 +396,7 @@ const LineageUtils = { } var shapeSvg = parent .append("circle") - .attr("fill", "url(#img_" + encodeURI(imgName) + ")") + .attr("fill", "url(#img_" + imgName + ")") .attr("r", isRankdirToBottom ? "30px" : "24px") .attr("data-stroke", node.id) .attr("stroke-width", "2px") @@ -401,90 +407,112 @@ const LineageUtils = { if (node.isIncomplete === true) { parent.attr("class", "node isIncomplete show"); parent - .insert("foreignObject") - .attr("x", "-25") - .attr("y", "-25") - .attr("width", "50") - .attr("height", "50") - .append("xhtml:div") - .insert("i") - .attr("class", "fa fa-hourglass-half"); + .insert("rect") + .attr("x", "-5") + .attr("y", "-23") + .attr("width", "14") + .attr("height", "16") + .attr("fill", "url(#img_hourglass.svg)") + .attr("data-stroke", node.id) + .attr("stroke-width", "2px"); + + var patternConfigShellIcon = { + imgName: "hourglass.svg", + imageIconPath: "/img/entity-icon/hourglass.svg", + leftPosition: "0", + topPosition: "0", + width: "12", + height: "14" + }; + svgPattern(patternConfigShellIcon); } + var patternConfigEntityIcon = { + imgName: imgName, + imageIconPath: imageIconPath, + leftPosition: isRankdirToBottom ? "11" : "4", + topPosition: isRankdirToBottom ? "20" : currentNode ? "3" : "4", + width: "40", + height: "40" + }; + svgPattern(patternConfigEntityIcon); - if (defsEl.select('pattern[id="img_' + imgName + '"]').empty()) { - defsEl - .append("pattern") - .attr("x", "0%") - .attr("y", "0%") - .attr("patternUnits", "objectBoundingBox") - .attr("id", "img_" + imgName) - .attr("width", "100%") - .attr("height", "100%") - .append("image") - .attr("href", function (d) { - var imgEl = this; - if (node) { - var getImageData = function (options) { - var imagePath = options.imagePath, - ajaxOptions = { - url: imagePath, - method: "GET", - cache: true - }; + function svgPattern(patternConfig) { + if (defsEl.select('pattern[id="img_' + patternConfig.imgName + '"]').empty()) { + defsEl + .append("pattern") + .attr("x", "0%") + .attr("y", "0%") + .attr("patternUnits", "objectBoundingBox") + .attr("id", "img_" + patternConfig.imgName) + .attr("width", "100%") + .attr("height", "100%") + .append("image") + .attr("href", function(d) { + var imgEl = this; + if (node) { + var getImageData = function(options) { + var imagePath = options.imagePath, + ajaxOptions = { + url: imagePath, + method: "GET", + cache: true + }; - // if (platform.name !== "IE") { - // ajaxOptions["mimeType"] = "text/plain; charset=x-user-defined"; - // } - shapeSvg.attr("data-iconpath", imagePath); - var xhr = new XMLHttpRequest(); - xhr.onreadystatechange = function () { - if (xhr.readyState === 4) { - if (xhr.status === 200) { - if (platform.name !== "IE") { - that.base64Encode(this.response, (url) => { - that.imageObject[imageIconPath] = url; - select(imgEl).attr("xlink:href", url); - }); - } else { - that.imageObject[imageIconPath] = imagePath; - } - if (imageIconPath !== shapeSvg.attr("data-iconpath")) { - shapeSvg.attr("data-iconpathorigin", imageIconPath); - } - } else if (xhr.status === 404) { - const imgPath = that.getEntityIconPath({ entityData: node, errorUrl: imagePath }); - if (imgPath === null) { - const patternEL = select(imgEl.parentElement); - patternEL.select("image").remove(); - patternEL - .attr("patternContentUnits", "objectBoundingBox") - .append("circle") - .attr("r", "24px") - .attr("fill", "#e8e8e8"); - } else { - getImageData({ - imagePath: imgPath - }); + // if (platform.name !== "IE") { + // ajaxOptions["mimeType"] = "text/plain; charset=x-user-defined"; + // } + shapeSvg.attr("data-iconpath", imagePath); + var xhr = new XMLHttpRequest(); + xhr.onreadystatechange = function() { + if (xhr.readyState === 4) { + if (xhr.status === 200) { + if (platform.name !== "IE") { + that.base64Encode(this.response, (url) => { + that.imageObject[patternConfig.imageIconPath] = url; + select(imgEl).attr("xlink:href", url); + }); + } else { + that.imageObject[patternConfig.imageIconPath] = imagePath; + } + if (patternConfig.imageIconPath !== shapeSvg.attr("data-iconpath")) { + shapeSvg.attr("data-iconpathorigin", patternConfig.imageIconPath); + } + } else if (xhr.status === 404) { + const imgPath = that.getEntityIconPath({ entityData: node, errorUrl: imagePath }); + if (imgPath === null) { + const patternEL = select(imgEl.parentElement); + patternEL.select("image").remove(); + patternEL + .attr("patternContentUnits", "objectBoundingBox") + .append("circle") + .attr("r", "24px") + .attr("fill", "#e8e8e8"); + } else { + getImageData({ + imagePath: imgPath + }); + } } } - } + }; + xhr.responseType = "blob"; + xhr.open(ajaxOptions.method, ajaxOptions.url, true); + xhr.send(null); }; - xhr.responseType = "blob"; - xhr.open(ajaxOptions.method, ajaxOptions.url, true); - xhr.send(null); - }; - getImageData({ - imagePath: imageIconPath - }); - } - }) - .attr("x", isRankdirToBottom ? "11" : "4") - .attr("y", isRankdirToBottom ? "20" : currentNode ? "3" : "4") - .attr("width", "40") - .attr("height", "40"); + getImageData({ + imagePath: patternConfig.imageIconPath + }); + } + }) + .attr("x", patternConfig.leftPosition) + .attr("y", patternConfig.topPosition) + .attr("width", patternConfig.width) + .attr("height", patternConfig.height); + } } - node.intersect = function (point) { + + node.intersect = function(point) { return dagreD3.intersect.circle(node, currentNode ? that.nodeArrowDistance + 3 : that.nodeArrowDistance, point); }; return shapeSvg; @@ -494,7 +522,7 @@ const LineageUtils = { * @param {[type]} {parent, id, edge, type, viewOptions [description] * @return {[type]} [description] */ - arrowPointRender: function (parent, id, edge, type, { dagreD3 }) { + arrowPointRender: function(parent, id, edge, type, { dagreD3 }) { var node = parent.node(), parentNode = node ? node.parentNode : parent; select(parentNode) @@ -523,11 +551,11 @@ const LineageUtils = { * @param {[type]} options.onExportLineage [description] * @return {[type]} [description] */ - saveSvg: function ({ svg, width, height, downloadFileName, onExportLineage }) { + saveSvg: function({ svg, width, height, downloadFileName, onExportLineage }) { var that = this, svgClone = svg.clone(true).node(), scaleFactor = 1; - setTimeout(function () { + setTimeout(function() { if (platform.name === "Firefox") { svgClone.setAttribute("width", width); svgClone.setAttribute("height", height); @@ -570,15 +598,16 @@ const LineageUtils = { if (platform.name === "Safari") { svgBlob = new Blob([data], { type: "image/svg+xml" }); } + ctx.drawImage(img, 50, 50, canvas.width, canvas.height); var url = DOMURL.createObjectURL(svgBlob); - img.onload = function () { + img.onload = function() { try { var a = document.createElement("a"); a.download = downloadFileName; document.body.appendChild(a); ctx.drawImage(img, 50, 50, canvas.width, canvas.height); - canvas.toBlob(function (blob) { + canvas.toBlob(function(blob) { if (!blob) { onExportLineage({ status: "failed", message: "There was an error in downloading Lineage!" }); return; diff --git a/dashboardv3/public/js/external_lib/atlas-lineage/src/index.js b/dashboardv3/public/js/external_lib/atlas-lineage/src/index.js index 3ceb3d90a90..0c4702237fe 100644 --- a/dashboardv3/public/js/external_lib/atlas-lineage/src/index.js +++ b/dashboardv3/public/js/external_lib/atlas-lineage/src/index.js @@ -45,6 +45,7 @@ export default class LineageHelper { zoom: (arg) => this.zoom(arg), fullScreen: (arg) => this.fullScreen(arg), searchNode: (arg) => this.searchNode(arg), + displayFullName: (arg) => this.displayFullName(arg), removeNodeSelection: (arg) => this.removeNodeSelection(arg), getGraphOptions: () => this.graphOptions, getNode: (guid, actual) => { @@ -163,14 +164,36 @@ export default class LineageHelper { zoom(opt = {}) { LineageUtils.zoom({ ...this.graphOptions, ...opt }); } + + displayFullName(opt = {}) { + var that = this; + this.g.nodes().forEach(function(v) { + var selectedNodeEl = that.svg.selectAll("g.nodes>g[id='" + v + "']"), + label = that.g.node(v).toolTipLabel; + if (opt.bLabelFullText == true) + selectedNodeEl.select('tspan').text(label); + else + selectedNodeEl.select('tspan').text(label.trunc(18)); + }); + if (this.selectedNode) { + this.searchNode({ guid: this.selectedNode }); + } + } + /** * [refresh Allows user to rerender the lineage] * @return {[type]} [description] */ - refresh() { + refresh(options) { this.clear(); this._initializeGraph(); this._initGraph({ refresh: true }); + this.selectedNode = ""; + if (options && options.compactLineageEnabled && options.filterObj) { + var isProcessHideCheck = options.filterObj.isProcessHideCheck, + isDeletedEntityHideCheck = options.filterObj.isDeletedEntityHideCheck; + this._AddFilterNotification(isProcessHideCheck, isDeletedEntityHideCheck); + } } /** * [removeNodeSelection description] @@ -185,23 +208,32 @@ export default class LineageHelper { */ searchNode({ guid, onSearchNode }) { this.svg.selectAll(".serach-rect").remove(); + this.svg.selectAll(".label").attr("stroke", "none"); + this.selectedNode = guid; this.centerAlign({ guid: guid, - onCenterZoomed: function (opts) { + onCenterZoomed: function(opts) { const { selectedNodeEl } = opts; + var oSelectedNode = selectedNodeEl.node().getBBox(), + rectWidth = oSelectedNode.width + 10, + rectXPos = oSelectedNode.x - 5; selectedNodeEl.select(".label").attr("stroke", "#316132"); selectedNodeEl.select("circle").classed("wobble", true); selectedNodeEl .insert("rect", "circle") .attr("class", "serach-rect") - .attr("x", -50) + .attr("stroke", "#37bb9b") + .attr("stroke-width", "2.5px") + .attr("fill", "none") + .attr("x", rectXPos) .attr("y", -27.5) - .attr("width", 100) - .attr("height", 55); + .attr("width", rectWidth) + .attr("height", 60); if (onSearchNode && typeof onSearchNode === "function") { onSearchNode(opts); } - } + }, + isSelected: true }); } @@ -287,8 +319,7 @@ export default class LineageHelper { // initlize the dagreD3 graphlib this.g = new dagreD3.graphlib.Graph() .setGraph( - Object.assign( - { + Object.assign({ nodesep: 50, ranksep: 90, rankdir: "LR", @@ -301,7 +332,7 @@ export default class LineageHelper { this.options.dagreOptions ) ) - .setDefaultEdgeLabel(function () { + .setDefaultEdgeLabel(function() { return {}; }); @@ -339,7 +370,7 @@ export default class LineageHelper { if (this.options.setDataManually === true) { return; - } else if (this.options.data === undefined || (this.options.data && this.options.data.relations.length === 0)) { + } else if (this.options.data === undefined || (this.options.data && this.options.data.relations.length === 0 && _.isEmpty(this.options.data.guidEntityMap))) { if (this.options.beforeRender) { this.options.beforeRender(); } @@ -402,8 +433,7 @@ export default class LineageHelper { * @param {[type]} graphOptions [description] * @return {[type]} [description] */ - _createGraph( - { + _createGraph({ data = {}, imgBasePath, isShowTooltip, @@ -416,12 +446,12 @@ export default class LineageHelper { getToolTipContent, toolTipTitle }, - graphOptions, - { refresh } + graphOptions, { refresh } ) { if (this.options.beforeRender) { this.options.beforeRender(); } + this.selectedNode = ""; const that = this, { svg, g, width, height } = graphOptions, isRankdirToBottom = this.options.dagreOptions && this.options.dagreOptions.rankdir === "tb"; @@ -435,7 +465,7 @@ export default class LineageHelper { return; } - g.nodes().forEach(function (v) { + g.nodes().forEach(function(v) { var node = g.node(v); // Round the corners of the nodes if (node) { @@ -452,11 +482,11 @@ export default class LineageHelper { // Create the renderer var render = new dagreD3.render(); // Add our custom arrow (a hollow-point) - render.arrows().arrowPoint = function () { + render.arrows().arrowPoint = function() { return LineageUtils.arrowPointRender(...arguments, { ...graphOptions }); }; // Render custom img inside shape - render.shapes().img = function () { + render.shapes().img = function() { return LineageUtils.imgShapeRender(...arguments, { ...graphOptions, isRankdirToBottom: isRankdirToBottom, @@ -506,17 +536,18 @@ export default class LineageHelper { if (isRankdirToBottom) { return "translate(2,-20)"; } - return "translate(2,-35)"; + return "translate(2,-38)"; }) - .on("mouseenter", function (d) { + .attr("font-size", "10px") + .on("mouseenter", function(d) { event.preventDefault(); select(this).classed("highlight", true); }) - .on("mouseleave", function (d) { + .on("mouseleave", function(d) { event.preventDefault(); select(this).classed("highlight", false); }) - .on("click", function (d) { + .on("click", function(d) { event.preventDefault(); if (onLabelClick && typeof onLabelClick === "function") { onLabelClick({ clickedData: d }); @@ -526,12 +557,12 @@ export default class LineageHelper { svgGroupEl .selectAll("g.nodes g.node circle") - .on("mouseenter", function (d, index, element) { + .on("mouseenter", function(d, index, element) { that.activeNode = true; var matrix = this.getScreenCTM().translate(+this.getAttribute("cx"), +this.getAttribute("cy")); that.svg.selectAll(".node").classed("active", false); select(this).classed("active", true); - if (that._getValueFromUser(isShowTooltip)) { + if (that._getValueFromUser(isShowTooltip) && (d.indexOf("more") !== 0)) { var direction = LineageUtils.getToolTipDirection({ el: this }); tooltip.direction(direction).show(d, this); } @@ -545,10 +576,10 @@ export default class LineageHelper { ...graphOptions }); }) - .on("mouseleave", function (d) { + .on("mouseleave", function(d) { that.activeNode = false; var nodeEL = this; - setTimeout(function (argument) { + setTimeout(function(argument) { if (!(that.activeTip || that.activeNode)) { select(nodeEL).classed("active", false); if (that._getValueFromUser(isShowTooltip)) { @@ -565,7 +596,7 @@ export default class LineageHelper { ...graphOptions }); }) - .on("click", function (d) { + .on("click", function(d) { if (event.defaultPrevented) return; // ignore drag event.preventDefault(); tooltip.hide(d); @@ -578,9 +609,9 @@ export default class LineageHelper { // Bind event on edgePath var edgePathEl = svgGroupEl.selectAll("g.edgePath"); - edgePathEl.selectAll("path.path").on("click", function (d) { + edgePathEl.selectAll("path.path").on("click", function(d) { if (onPathClick && typeof onPathClick === "function") { - var pathRelationObj = data.relations.find(function (obj) { + var pathRelationObj = data.relations.find(function(obj) { if (obj.fromEntityId === d.v && obj.toEntityId === d.w) { return true; } @@ -651,5 +682,16 @@ export default class LineageHelper { span = container.append("span").style("color", "#fb4200"); span.append("i").classed("fa fa-long-arrow-right fa-fw", true); span.append("span").html("Impact"); + + span = container.append("span").classed("notification hide", true).style("color", "#686868"); + span.append("i").classed("fa fa-exclamation fa-fw", true); + span.append("span").html("Filtering hides all Expand buttons."); + } + _AddFilterNotification(isProcessHideCheck, isDeletedEntityHideCheck) { + if ((isProcessHideCheck || isDeletedEntityHideCheck)) { + $(this.options.legendsEl).find('.notification').removeClass('hide'); + } else { + $(this.options.legendsEl).find('.notification').addClass('hide'); + } } } \ No newline at end of file diff --git a/dashboardv3/public/js/external_lib/atlas-lineage/src/styles/graph.scss b/dashboardv3/public/js/external_lib/atlas-lineage/src/styles/graph.scss index ab1e82df38c..2f1322765d0 100644 --- a/dashboardv3/public/js/external_lib/atlas-lineage/src/styles/graph.scss +++ b/dashboardv3/public/js/external_lib/atlas-lineage/src/styles/graph.scss @@ -26,17 +26,17 @@ //transition: opacity 0.3s linear; - rect { - stroke: $color_mountain_mist_approx; - fill: $white; - stroke-width: 1.5px; - - &.serach-rect { - stroke: $color_keppel_approx; - fill: transparent; - stroke-width: 2.5px; - } - } + // rect { + // stroke: $color_mountain_mist_approx; + // fill: $white; + // stroke-width: 1.5px; + + // &.serach-rect { + // stroke: $color_keppel_approx; + // fill: transparent; + // stroke-width: 2.5px; + // } + // } .label { fill: $color_suva_gray_approx; @@ -104,7 +104,7 @@ } .legends { - > span { + >span { margin-right: 8px; font-family: $font_0; } @@ -156,10 +156,12 @@ svg.hover { z-index: 999; max-width: 300px; //Instead of the line below you could use @include border-radius($radius, $vertical-radius) border-radius: 2px; + word-break: break-all; .tip-inner-scroll { overflow: auto; max-height: 300px; + h5 { margin: 7px 0px; } @@ -211,7 +213,7 @@ svg.hover { } } -g.type-TK > rect { +g.type-TK>rect { fill: $color_bright_turquoise_approx; } diff --git a/dashboardv3/public/js/main.js b/dashboardv3/public/js/main.js index 97e177c7031..1e58f3d9b22 100644 --- a/dashboardv3/public/js/main.js +++ b/dashboardv3/public/js/main.js @@ -318,6 +318,12 @@ require(['App', Globals.isTasksEnabled = response['atlas.tasks.enabled']; } if (response['atlas.session.timeout.secs']) { Globals.idealTimeoutSeconds = response['atlas.session.timeout.secs']; } + if(response['atlas.lineage.on.demand.enabled'] !== undefined){ + Globals.isLineageOnDemandEnabled = response['atlas.lineage.on.demand.enabled']; + } + if(response['atlas.lineage.on.demand.default.node.count'] !== undefined){ + Globals.lineageNodeCount = response['atlas.lineage.on.demand.default.node.count']; + } /* Atlas idealTimeout redirectUrl: url to redirect after timeout idealTimeLimit: timeout in seconds diff --git a/dashboardv3/public/js/templates/graph/LineageLayoutView_tmpl.html b/dashboardv3/public/js/templates/graph/LineageLayoutView_tmpl.html index 5f90b096e3e..30dd45d7fa0 100644 --- a/dashboardv3/public/js/templates/graph/LineageLayoutView_tmpl.html +++ b/dashboardv3/public/js/templates/graph/LineageLayoutView_tmpl.html @@ -90,10 +90,20 @@

Settings

- +
+ {{#if compactLineageEnabled}} +
+
+ +
+ +
+
+
+ {{/if}}
diff --git a/dashboardv3/public/js/utils/Globals.js b/dashboardv3/public/js/utils/Globals.js index df17a683be5..65dda8bd481 100644 --- a/dashboardv3/public/js/utils/Globals.js +++ b/dashboardv3/public/js/utils/Globals.js @@ -51,6 +51,10 @@ define(["require"], function(require) { Globals.isTasksEnabled = false; Globals.advanceSearchData = {}; Globals.idealTimeoutSeconds = 900; + Globals.isFullScreenView = false; + Globals.isLineageOnDemandEnabled = false; + Globals.lineageNodeCount = 3; + Globals.lineageDepth = 3; return Globals; }); \ No newline at end of file diff --git a/dashboardv3/public/js/utils/Helper.js b/dashboardv3/public/js/utils/Helper.js index a242098b13b..164beac576c 100644 --- a/dashboardv3/public/js/utils/Helper.js +++ b/dashboardv3/public/js/utils/Helper.js @@ -17,11 +17,12 @@ */ define(['require', 'utils/Utils', + 'utils/Globals', 'd3', 'marionette', 'jquery-ui', 'jstree' -], function(require, Utils, d3) { +], function(require, Utils, Globals, d3) { 'use strict'; _.mixin({ numberFormatWithComma: function(number) { @@ -393,5 +394,11 @@ define(['require', //For closing the modal on browsers navigation $(window).on('popstate', function() { $('body').find('.modal-dialog .close').click(); + //To close the full-screen mode in lineage on browsers navigation. + if (!Globals.isFullScreenView) { + $('#tab-lineage').removeClass("fullscreen-mode"); + $("#r_lineageLayoutView").find('button[data-id="fullScreen-toggler"]').attr("data-original-title", "Full Screen").find("i").removeClass("fa-compress").addClass("fa-expand"); + } + Globals.isFullScreenView = false; }); }) \ No newline at end of file diff --git a/dashboardv3/public/js/views/graph/LineageLayoutView.js b/dashboardv3/public/js/views/graph/LineageLayoutView.js index e989c4719bd..94bb19db76e 100644 --- a/dashboardv3/public/js/views/graph/LineageLayoutView.js +++ b/dashboardv3/public/js/views/graph/LineageLayoutView.js @@ -52,6 +52,7 @@ define(['require', checkHideProcess: "[data-id='checkHideProcess']", checkDeletedEntity: "[data-id='checkDeletedEntity']", selectDepth: 'select[data-id="selectDepth"]', + selectNodeCount: 'select[data-id="selectNodeCount"]', filterToggler: '[data-id="filter-toggler"]', settingToggler: '[data-id="setting-toggler"]', searchToggler: '[data-id="search-toggler"]', @@ -74,7 +75,8 @@ define(['require', templateHelpers: function() { return { width: "100%", - height: "100%" + height: "100%", + compactLineageEnabled: Globals.isLineageOnDemandEnabled }; }, /** ui events hash */ @@ -83,6 +85,7 @@ define(['require', events["click " + this.ui.checkHideProcess] = 'onCheckUnwantedEntity'; events["click " + this.ui.checkDeletedEntity] = 'onCheckUnwantedEntity'; events['change ' + this.ui.selectDepth] = 'onSelectDepthChange'; + events['change ' + this.ui.selectNodeCount] = 'onSelectNodeCount'; events["click " + this.ui.filterToggler] = 'onClickFilterToggler'; events["click " + this.ui.boxClose] = 'toggleBoxPanel'; events["click " + this.ui.settingToggler] = 'onClickSettingToggler'; @@ -109,7 +112,7 @@ define(['require', this.filterObj = { isProcessHideCheck: false, isDeletedEntityHideCheck: false, - depthCount: '' + depthCount: Globals.lineageDepth }; this.searchNodeObj = { selectedNode: '' @@ -117,16 +120,31 @@ define(['require', this.labelFullText = false; }, onRender: function() { - var that = this; + var that = this, + nodeCountArray = _.uniq([3, 6, Globals.lineageNodeCount]); + this.initialQueryObj = {}; + this.initialQueryObj["constraints"] = {}; this.ui.searchToggler.prop("disabled", true); - this.$graphButtonsEl = this.$(".graph-button-group button, select[data-id='selectDepth']") - this.fetchGraphData(); + this.$graphButtonsEl = this.$(".graph-button-group button, select[data-id='selectDepth']"); + if (Globals.isLineageOnDemandEnabled) { + this.ui.resetLineage.attr("title", "Reset Lineage"); + this.initialQueryObj["constraints"][this.guid] = { + "direction": "BOTH", + "inputRelationsLimit": Globals.lineageNodeCount, + "outputRelationsLimit": Globals.lineageNodeCount, + "depth": Globals.lineageDepth, + } + } + this.fetchGraphData({ queryParam: this.initialQueryObj }); if (this.layoutRendered) { this.layoutRendered(); } if (this.processCheck) { this.hideCheckForProcess(); } + if (this.entity.status === "DELETED") { + this.hideCheckForDeletedEntity(); + } //this.initializeGraph(); this.ui.selectDepth.select2({ data: _.sortBy([3, 6, 9, 12, 15, 18, 21]), @@ -134,6 +152,13 @@ define(['require', dropdownCssClass: "number-input", multiple: false }); + this.ui.selectNodeCount.select2({ + data: _.sortBy(nodeCountArray), + tags: true, + dropdownCssClass: "number-input", + multiple: false + }); + this.ui.selectNodeCount.val(Globals.lineageNodeCount).trigger("change"); }, onShow: function() { this.$('.fontLoader').show(); @@ -150,6 +175,7 @@ define(['require', panel = $(e.target).parents('.tab-pane').first(); icon.toggleClass('fa-expand fa-compress'); if (icon.hasClass('fa-expand')) { + Globals.isFullScreenView = false; icon.parent('button').attr("data-original-title", "Full Screen"); } else { icon.parent('button').attr("data-original-title", "Default View"); @@ -165,17 +191,22 @@ define(['require', onCheckUnwantedEntity: function(e) { var that = this; //this.initializeGraph(); + this.searchNodeObj.selectedNode = ""; if ($(e.target).data("id") === "checkHideProcess") { this.filterObj.isProcessHideCheck = e.target.checked; } else { this.filterObj.isDeletedEntityHideCheck = e.target.checked; } - this.LineageHelperRef.refresh(); + this.renderLineageTypeSearch(this.data); + this.LineageHelperRef.refresh({ compactLineageEnabled: Globals.isLineageOnDemandEnabled, filterObj: this.filterObj }); }, toggleBoxPanel: function(options) { var el = options && options.el, nodeDetailToggler = options && options.nodeDetailToggler, currentTarget = options.currentTarget; + if (options.nodeDetailToggler) { + this.ui.lineageTypeSearch.select2("close"); + } this.$el.find('.show-box-panel').removeClass('show-box-panel'); if (el && el.addClass) { el.addClass('show-box-panel'); @@ -214,12 +245,33 @@ define(['require', }, onSelectDepthChange: function(e, options) { //this.initializeGraph(); - this.filterObj.depthCount = e.currentTarget.value; + Globals.lineageDepth = parseInt(e.currentTarget.value); //legends property is added in queryParam to stop the legend getting added in lineage graph whenever dept is changed. - this.fetchGraphData({ queryParam: { 'depth': this.filterObj.depthCount }, 'legends': false }); + if (!Globals.isLineageOnDemandEnabled) { + this.fetchGraphData({ queryParam: { 'depth': Globals.lineageDepth }, 'legends': false }); + } + + if (Globals.isLineageOnDemandEnabled) { + this.initialQueryObj["constraints"][this.guid].depth = Globals.lineageDepth; + this.fetchGraphData({ queryParam: this.initialQueryObj, 'legends': false }) + } + }, + onSelectNodeCount: function(e, options) { + Globals.lineageNodeCount = parseInt(e.target.value); + if (Globals.lineageNodeCount === 0) { + Utils.notifyWarn({ + content: 'Value cannot be less than 1' + }); + this.ui.selectNodeCount.val(3).trigger("change"); + } }, onClickResetLineage: function() { - this.LineageHelperRef.refresh(); + if (Globals.isLineageOnDemandEnabled) { + this.fetchGraphData({ queryParam: this.initialQueryObj, 'legends': false }); + } + if (!Globals.isLineageOnDemandEnabled) { + this.LineageHelperRef.refresh(); + } this.searchNodeObj.selectedNode = ""; this.ui.lineageTypeSearch.data({ refresh: true }).val("").trigger("change"); this.ui.labelFullName.prop("checked", false); @@ -270,13 +322,14 @@ define(['require', } _.extend(this.currentEntityData, _.pick(this.entity, 'attributes', 'guid', 'isIncomplete', 'status', 'typeName')); //End - this.collection.getLineage(this.guid, { - queryParam: queryParam, + var dataObj = { + compactLineageEnabled: Globals.isLineageOnDemandEnabled, success: function(data) { if (that.isDestroyed) { return; } data["legends"] = options ? options.legends : true; + that.lineageOnDemandPayload = data.lineageOnDemandPayload ? data.lineageOnDemandPayload : {}; // show only main part of lineage current entity is at bottom, so reverse is done var relationsReverse = data.relations ? data.relations.reverse() : null, lineageMaxRelationCount = 9000; @@ -292,6 +345,10 @@ define(['require', data.guidEntityMap[data.baseEntityGuid] = that.currentEntityData; } } + that.data = data; + var updatedData = that.updateLineageData(data); + _.extend(data.guidEntityMap, updatedData.plusBtnsObj); + data.relations = data.relations.concat(updatedData.plusBtnRelationsArray); that.createGraph(data); that.renderLineageTypeSearch(data); }, @@ -302,7 +359,68 @@ define(['require', that.$('.fontLoader').hide(); that.$('svg>g').show(); } - }) + }; + Globals.isLineageOnDemandEnabled ? dataObj.data = queryParam : dataObj.queryParam = queryParam; + + this.collection.getLineage(this.guid, dataObj) + }, + updateLineageData: function(data) { + var that = this, + rawData = data, + plusBtnsObj = {}, + plusBtnRelationsArray = []; + this.relationsOnDemand = data.relationsOnDemand ? data.relationsOnDemand : null; + this.lineageOnDemandPayload = data.lineageOnDemandPayload ? data.lineageOnDemandPayload : null; + if (this.relationsOnDemand) { + _.each(this.relationsOnDemand, function(values, nodeId) { + if (values.hasMoreInputs) { + var btnType = "Input", + moreInputBtnObj = that.createExpandButtonObj({ nodeId: nodeId, btnType: btnType }); + plusBtnsObj[moreInputBtnObj.guid] = moreInputBtnObj; + plusBtnRelationsArray.push({ fromEntityId: moreInputBtnObj.guid, toEntityId: nodeId, relationshipId: 'dummy' }); + } + if (values.hasMoreOutputs) { + var btnType = "Output", + moreOutputBtnObj = that.createExpandButtonObj({ nodeId: nodeId, btnType: btnType }); + plusBtnsObj[moreOutputBtnObj.guid] = moreOutputBtnObj; + plusBtnRelationsArray.push({ fromEntityId: nodeId, toEntityId: moreOutputBtnObj.guid, relationshipId: 'dummy' }); + } + }); + return { + plusBtnsObj: plusBtnsObj, + plusBtnRelationsArray: plusBtnRelationsArray + } + } + }, + generateAddButtonId: function(btnType) { + return btnType + Math.random().toString(16).slice(2) + }, + createExpandButtonObj: function(options) { + var defaultObj = { + attributes: { + owner: '', + createTime: 0, + qualifiedName: 'PlusBtn', + name: 'PlusBtn', + description: '' + }, + isExpandBtn: true, + classificationNames: [], + displayText: "Expand", + isIncomplete: false, + labels: [], + meaningNames: [], + meanings: [], + status: "ACTIVE", + typeName: "Table" + }, + btnObj = Object.assign({}, defaultObj), + btnType = (options.btnType === "Input") ? "more-inputs" : "more-outputs", + btnId = this.generateAddButtonId(btnType); + btnObj.guid = btnId; + btnObj.parentNodeGuid = options.nodeId; + btnObj.btnType = options.btnType; + return btnObj; }, createGraph: function(data) { var that = this; @@ -323,7 +441,6 @@ define(['require', isShowHoverPath: function() { return that.ui.showOnlyHoverPath.prop('checked') }, isShowTooltip: function() { return that.ui.showTooltip.prop('checked') }, onPathClick: function(d) { - console.log("Path Clicked"); if (d.pathRelationObj) { var relationshipId = d.pathRelationObj.relationshipId; require(['views/graph/PropagationPropertyModal'], function(PropagationPropertyModal) { @@ -338,12 +455,21 @@ define(['require', } }, onNodeClick: function(d) { + if (d.clickedData.indexOf("more") >= 0) { + that.onExpandNodeClick({ guid: d.clickedData }); + return; + } that.onClickNodeToggler(); that.updateRelationshipDetails({ guid: d.clickedData }); that.calculateLineageDetailPanelHeight(); }, onLabelClick: function(d) { var guid = d.clickedData; + Globals.isFullScreenView = true; + that.ui.lineageTypeSearch.select2("close"); + if (guid.indexOf("more") >= 0) { + return; + } if (that.guid == guid) { Utils.notifyInfo({ html: true, @@ -376,6 +502,57 @@ define(['require', } }); }, + onExpandNodeClick: function(options) { + var parentNodeData = this.LineageHelperRef.getNode(options.guid); + this.updateQueryObject(parentNodeData.parentNodeGuid, parentNodeData.btnType); + }, + updateQueryObject: function(parentId, btnType) { + var inputLimit = null, + outputLimit = null, + that = this, + queryParam; + if (_.has(that.lineageOnDemandPayload, parentId)) { + _.find(that.lineageOnDemandPayload, function(value, key) { + if (key === parentId) { + if (btnType === "Input") { + value.inputRelationsLimit = value.inputRelationsLimit + Globals.lineageNodeCount; + value.outputRelationsLimit = value.outputRelationsLimit; + } + if (btnType === "Output") { + value.inputRelationsLimit = value.inputRelationsLimit; + value.outputRelationsLimit = value.outputRelationsLimit + Globals.lineageNodeCount; + } + } + }); + } else { + var relationCount = that.validateInputOutputLimit(parentId, btnType); + if (btnType === "Input") { + inputLimit = relationCount.inputRelationCount + Globals.lineageNodeCount; + outputLimit = relationCount.outputRelationCount; + } + if (btnType === "Output") { + inputLimit = relationCount.inputRelationCount; + outputLimit = relationCount.outputRelationCount + Globals.lineageNodeCount; + } + this.lineageOnDemandPayload["constraints"][parentId] = { direction: "BOTH", inputRelationsLimit: inputLimit, outputRelationsLimit: outputLimit, depth: Globals.lineageDepth }; + } + var queryParameter = {} + queryParameter["constraints"] = this.lineageOnDemandPayload["constraints"]; + this.fetchGraphData({ queryParam: queryParameter, 'legends': false }); + }, + validateInputOutputLimit: function(parentId, btnType) { + var inputRelationCount, outputRelationCount; + for (var guid in this.relationsOnDemand) { + if (parentId === guid && (btnType === "Input" || btnType === "Output")) { + inputRelationCount = this.relationsOnDemand[guid].inputRelationsCount ? this.relationsOnDemand[guid].inputRelationsCount : Globals.lineageNodeCount; + outputRelationCount = this.relationsOnDemand[guid].outputRelationsCount ? this.relationsOnDemand[guid].outputRelationsCount : Globals.lineageNodeCount; + } + } + return { + inputRelationCount: inputRelationCount, + outputRelationCount: outputRelationCount + }; + }, noLineage: function() { this.$('.fontLoader').hide(); this.$('.depth-container').hide(); @@ -387,6 +564,10 @@ define(['require', hideCheckForProcess: function() { this.$('.hideProcessContainer').hide(); }, + hideCheckForDeletedEntity: function() { + //This has been added to handle the scenario where hideDeletedEntityCheck should hide form filters when baseEntity is deleted. + this.$('.hideDeletedContainer').hide(); + }, renderLineageTypeSearch: function(data) { var that = this; return new Promise(function(resolve, reject) { @@ -395,7 +576,7 @@ define(['require', if (!_.isEmpty(data)) { _.each(data.guidEntityMap, function(obj, index) { var nodeData = that.LineageHelperRef.getNode(obj.guid); - if ((that.filterObj.isProcessHideCheck || that.filterObj.isDeletedEntityHideCheck) && nodeData && (nodeData.isProcess || nodeData.isDeleted)) { + if ((that.filterObj.isProcessHideCheck && obj && obj.isProcess) || (that.filterObj.isDeletedEntityHideCheck && obj && obj.isDeleted) || (Globals.isLineageOnDemandEnabled && obj && _.contains(["Input", "Output"], obj.btnType))) { return; } typeStr += ''; diff --git a/distro/pom.xml b/distro/pom.xml index 08a9071f422..6f30eecabd1 100644 --- a/distro/pom.xml +++ b/distro/pom.xml @@ -128,7 +128,7 @@ atlas.graph.storage.hbase.regions-per-server=1 - src/main/assemblies/atlas-kafka-hook-package.xml + src/main/assemblies/atlas-server-package.xml src/main/assemblies/standalone-package.xml src/main/assemblies/src-package.xml @@ -545,10 +545,4 @@ atlas.graph.storage.conf-file=${sys:atlas.home}/conf/cassandra.yml - - - org.apache.atlas - storm-bridge - - diff --git a/distro/src/bin/atlas_config.py b/distro/src/bin/atlas_config.py index 31e6fd0439c..46ef42a34f9 100755 --- a/distro/src/bin/atlas_config.py +++ b/distro/src/bin/atlas_config.py @@ -500,7 +500,7 @@ def get_topics_to_create(confdir): if topic_list is not None: topics = topic_list.split(",") else: - topics = [getConfigWithDefault("atlas.notification.hook.topic.name", "ATLAS_HOOK"), getConfigWithDefault("atlas.notification.entities.topic.name", "ATLAS_ENTITIES")] + topics = [getConfigWithDefault("atlas.notification.hook.topic.name", "ATLAS_HOOK"), getConfigWithDefault("atlas.notification.entities.topic.name", "ATLAS_ENTITIES"), getConfigWithDefault("atlas.notification.relationships.topic.name", "ATLAS_RELATIONSHIPS")] return topics def get_atlas_url_port(confdir): diff --git a/distro/src/main/assemblies/atlas-server-package.xml b/distro/src/main/assemblies/atlas-server-package.xml index 8e8b86c425f..9eb71ef7cf7 100755 --- a/distro/src/main/assemblies/atlas-server-package.xml +++ b/distro/src/main/assemblies/atlas-server-package.xml @@ -151,6 +151,16 @@ ../addons/elasticsearch elasticsearch + + + ../addons/static + static + + + + ../addons/policies + policies + diff --git a/distro/src/main/assemblies/standalone-package.xml b/distro/src/main/assemblies/standalone-package.xml index 28eafac87f8..3ef91c047e3 100755 --- a/distro/src/main/assemblies/standalone-package.xml +++ b/distro/src/main/assemblies/standalone-package.xml @@ -131,6 +131,16 @@ elasticsearch + + ../addons/static + static + + + + ../addons/policies + policies + + ../addons/hive-bridge/src/bin diff --git a/docs/package.json b/docs/package.json index f38a965edfd..1160c10e4cb 100644 --- a/docs/package.json +++ b/docs/package.json @@ -17,7 +17,7 @@ "@babel/core": "7.4.5", "@babel/plugin-syntax-dynamic-import": "7.2.0", "@babel/preset-react": "7.0.0", - "axios": "0.19.0", + "axios": "0.21.1", "babel-loader": "8.0.6", "babel-plugin-lodash": "3.3.4", "babel-plugin-react-transform": "3.0.0", diff --git a/ecrorgcrossaccountpolicy.json b/ecrorgcrossaccountpolicy.json new file mode 100644 index 00000000000..b9ae8a923db --- /dev/null +++ b/ecrorgcrossaccountpolicy.json @@ -0,0 +1,27 @@ +{ + "__comment": "Policy is currently not in use by main-ecr.yaml, deprecated as of Jan 11, 2023", + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "AllowPull", + "Effect": "Allow", + "Principal": "*", + "Action": [ + "ecr:BatchCheckLayerAvailability", + "ecr:BatchGetImage", + "ecr:DescribeImages", + "ecr:DescribeRepositories", + "ecr:GetDownloadUrlForLayer" + ], + "Condition": { + "ForAnyValue:StringLike": { + "aws:PrincipalOrgPaths": [ + "o-c64vo42aib/*", + "o-zjrs0debz8/*", + "o-jgyaklzq1t/*" + ] + } + } + } + ] +} diff --git a/graphdb/api/pom.xml b/graphdb/api/pom.xml index ee6ed7b50bc..4f502911a72 100644 --- a/graphdb/api/pom.xml +++ b/graphdb/api/pom.xml @@ -54,7 +54,7 @@ org.elasticsearch elasticsearch - 7.6.2 + ${elasticsearch.version} compile diff --git a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasElement.java b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasElement.java index d03aecad333..fbed380a69f 100644 --- a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasElement.java +++ b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasElement.java @@ -73,6 +73,8 @@ public interface AtlasElement { List getMultiValuedProperty(String propertyName, Class elementType); + Set getMultiValuedSetProperty(String propertyName, Class elementType); + /** * Gets the value of a multiplicity one property whose value is a list. It * attempts to convert the elements in the list to the specified type. Currently diff --git a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraph.java b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraph.java index 7d06965c95a..2b41b1c07dc 100644 --- a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraph.java +++ b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraph.java @@ -18,6 +18,7 @@ package org.apache.atlas.repository.graphdb; import org.apache.atlas.AtlasException; +import org.apache.atlas.ESAliasRequestBuilder; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.groovy.GroovyExpression; import org.apache.atlas.model.discovery.SearchParams; @@ -205,7 +206,14 @@ public interface AtlasGraph { */ AtlasIndexQuery elasticsearchQuery(String indexName, SearchSourceBuilder sourceBuilder); - AtlasIndexQuery elasticsearchQuery(String indexName, SearchParams searchParams); + AtlasIndexQuery elasticsearchQuery(String indexName, SearchParams searchParams)throws AtlasBaseException; + + void createOrUpdateESAlias(ESAliasRequestBuilder aliasRequestBuilder) throws AtlasBaseException; + + void deleteESAlias(String indexName, String aliasName) throws AtlasBaseException; + + AtlasIndexQuery elasticsearchQuery(String indexName) throws AtlasBaseException; + /** * Gets the management object associated with this graph and opens a transaction diff --git a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasIndexQuery.java b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasIndexQuery.java index 54ea668982b..a4435ccff36 100644 --- a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasIndexQuery.java +++ b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasIndexQuery.java @@ -23,6 +23,8 @@ import org.apache.tinkerpop.gremlin.process.traversal.Order; import java.util.Iterator; +import java.util.Map; +import java.util.Set; /** * A graph query that runs directly against a particular index. @@ -40,6 +42,13 @@ public interface AtlasIndexQuery { DirectIndexQueryResult vertices(SearchParams searchParams) throws AtlasBaseException ; + /** + * Gets the query results form index + * + * @return Map of indexQuery result + */ + Map directIndexQuery(String query) throws AtlasBaseException; + /** * Gets the query results. * @@ -88,6 +97,10 @@ interface Result { */ double getScore(); + Set getCollapseKeys(); + + DirectIndexQueryResult getCollapseVertices(String key); + } } diff --git a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasVertexQuery.java b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasVertexQuery.java index 1b69f74f2eb..b0ef59d48bd 100644 --- a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasVertexQuery.java +++ b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasVertexQuery.java @@ -76,6 +76,14 @@ public interface AtlasVertexQuery { */ AtlasVertexQuery label(String label); + /** + * Specifies the edge label that should be queried. + * + * @param labels + * @return + */ + AtlasVertexQuery label(String... labels); + /** * Returns edges that matches property key and value. * diff --git a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/DirectIndexQueryResult.java b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/DirectIndexQueryResult.java index 9bc46ba52c1..8e7dd0e0edf 100644 --- a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/DirectIndexQueryResult.java +++ b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/DirectIndexQueryResult.java @@ -8,6 +8,7 @@ public class DirectIndexQueryResult { private Iterator> iterator; private Map aggregationMap; + private Integer approximateCount; public Iterator> getIterator() { return iterator; @@ -24,4 +25,12 @@ public Map getAggregationMap() { public void setAggregationMap(Map aggregationMap) { this.aggregationMap = aggregationMap; } + + public Integer getApproximateCount() { + return approximateCount; + } + + public void setApproximateCount(Integer approximateCount) { + this.approximateCount = approximateCount; + } } \ No newline at end of file diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchDatabase.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchDatabase.java index 0b8ca5b21bd..aac96cba484 100644 --- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchDatabase.java +++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchDatabase.java @@ -18,6 +18,7 @@ package org.apache.atlas.repository.graphdb.janus; import org.apache.atlas.ApplicationProperties; +import org.apache.atlas.AtlasConfiguration; import org.apache.atlas.AtlasException; import org.apache.commons.configuration.Configuration; import org.apache.http.HttpHost; @@ -64,8 +65,9 @@ public static RestHighLevelClient getClient() { List httpHosts = getHttpHosts(); RestClientBuilder restClientBuilder = RestClient.builder(httpHosts.toArray(new HttpHost[0])) - .setRequestConfigCallback(requestConfigBuilder -> requestConfigBuilder.setConnectTimeout(900000) - .setSocketTimeout(900000)); + .setRequestConfigCallback(requestConfigBuilder -> requestConfigBuilder + .setConnectTimeout(AtlasConfiguration.INDEX_CLIENT_CONNECTION_TIMEOUT.getInt()) + .setSocketTimeout(AtlasConfiguration.INDEX_CLIENT_SOCKET_TIMEOUT.getInt())); searchClient = new RestHighLevelClient(restClientBuilder); } catch (AtlasException e) { @@ -85,9 +87,10 @@ public static RestClient getLowLevelClient() { List httpHosts = getHttpHosts(); RestClientBuilder builder = RestClient.builder(httpHosts.get(0)); + builder.setHttpClientConfigCallback(httpAsyncClientBuilder -> httpAsyncClientBuilder.setKeepAliveStrategy(((httpResponse, httpContext) -> 3600000))); builder.setRequestConfigCallback(requestConfigBuilder -> requestConfigBuilder - .setConnectTimeout(900000) - .setSocketTimeout(900000)); + .setConnectTimeout(AtlasConfiguration.INDEX_CLIENT_CONNECTION_TIMEOUT.getInt()) + .setSocketTimeout(AtlasConfiguration.INDEX_CLIENT_SOCKET_TIMEOUT.getInt())); lowLevelClient = builder.build(); } catch (AtlasException e) { diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java index 2857923d4fc..3755c6dc3a2 100644 --- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java +++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java @@ -19,7 +19,6 @@ import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.model.discovery.IndexSearchParams; import org.apache.atlas.model.discovery.SearchParams; import org.apache.atlas.repository.graphdb.AtlasIndexQuery; import org.apache.atlas.repository.graphdb.AtlasVertex; @@ -36,6 +35,7 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.HttpAsyncResponseConsumerFactory; import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.RestClient; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; @@ -47,13 +47,12 @@ import org.slf4j.LoggerFactory; import java.io.IOException; -import java.util.Arrays; -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.stream.Stream; +import static org.apache.atlas.AtlasErrorCode.INDEX_NOT_FOUND; + + public class AtlasElasticsearchQuery implements AtlasIndexQuery { private static final Logger LOG = LoggerFactory.getLogger(AtlasElasticsearchQuery.class); @@ -84,6 +83,15 @@ private AtlasElasticsearchQuery(AtlasJanusGraph graph, String index) { searchResponse = null; } + public AtlasElasticsearchQuery(AtlasJanusGraph graph, String index, RestClient restClient) { + this(graph, restClient, index, null); + } + + public AtlasElasticsearchQuery(String index, RestClient restClient) { + this.lowLevelRestClient = restClient; + this.index = index; + } + private SearchRequest getSearchRequest(String index, SearchSourceBuilder sourceBuilder) { SearchRequest searchRequest = new SearchRequest(index); searchRequest.source(sourceBuilder); @@ -116,7 +124,7 @@ private DirectIndexQueryResult runQueryWithLowLevelClient(SearchParams searchPar try { - String responseString = perfromDirectIndexQuery(searchParams.getQuery()); + String responseString = performDirectIndexQuery(searchParams.getQuery(), false); if (LOG.isDebugEnabled()) { LOG.debug("runQueryWithLowLevelClient.response : {}", responseString); } @@ -131,15 +139,57 @@ private DirectIndexQueryResult runQueryWithLowLevelClient(SearchParams searchPar return result; } - private String perfromDirectIndexQuery(String query) throws IOException { + private Map runQueryWithLowLevelClient(String query) throws AtlasBaseException { + Map ret = new HashMap<>(); + try { + String responseString = performDirectIndexQuery(query, true); + + Map responseMap = AtlasType.fromJson(responseString, Map.class); + Map hits_0 = AtlasType.fromJson(AtlasType.toJson(responseMap.get("hits")), Map.class); + + ret.put("total", hits_0.get("total").get("value")); + + List hits_1 = AtlasType.fromJson(AtlasType.toJson(hits_0.get("hits")), List.class); + ret.put("data", hits_1); + + Map aggregationsMap = (Map) responseMap.get("aggregations"); + + if (MapUtils.isNotEmpty(aggregationsMap)) { + ret.put("aggregations", aggregationsMap); + } + + return ret; + + } catch (IOException e) { + LOG.error("Failed to execute direct query on ES {}", e.getMessage()); + throw new AtlasBaseException(AtlasErrorCode.INDEX_SEARCH_FAILED, e.getMessage()); + } + } + + private String performDirectIndexQuery(String query, boolean source) throws AtlasBaseException, IOException { HttpEntity entity = new NStringEntity(query, ContentType.APPLICATION_JSON); + String endPoint; - String endPoint = index + "/_search?_source=false"; + if (source) { + endPoint = index + "/_search"; + } else { + endPoint = index + "/_search?_source=false"; + } Request request = new Request("GET", endPoint); request.setEntity(entity); - Response response = lowLevelRestClient.performRequest(request); + Response response; + try { + response = lowLevelRestClient.performRequest(request); + } catch (ResponseException rex) { + if (rex.getResponse().getStatusLine().getStatusCode() == 404) { + LOG.warn(String.format("ES index with name %s not found", index)); + throw new AtlasBaseException(INDEX_NOT_FOUND, index); + } else { + throw new AtlasBaseException(rex); + } + } return EntityUtils.toString(response.getEntity()); } @@ -171,6 +221,11 @@ public DirectIndexQueryResult vertices(SearchP return runQueryWithLowLevelClient(searchParams); } + @Override + public Map directIndexQuery(String query) throws AtlasBaseException { + return runQueryWithLowLevelClient(query); + } + @Override public Iterator> vertices() { SearchRequest searchRequest = getSearchRequest(index, sourceBuilder); @@ -215,14 +270,28 @@ public AtlasVertex getVertex() { public double getScore() { return hit.getScore(); } + + @Override + public Set getCollapseKeys() { + return null; + } + + @Override + public DirectIndexQueryResult getCollapseVertices(String key) { + return null; + } } public final class ResultImplDirect implements AtlasIndexQuery.Result { private LinkedHashMap hit; + Map innerHitsMap; public ResultImplDirect(LinkedHashMap hit) { this.hit = hit; + if (hit.get("inner_hits") != null) { + innerHitsMap = AtlasType.fromJson(AtlasType.toJson(hit.get("inner_hits")), Map.class); + } } @Override @@ -231,6 +300,34 @@ public AtlasVertex getVertex() { return graph.getVertex(String.valueOf(vertexId)); } + @Override + public Set getCollapseKeys() { + Set collapseKeys = new HashSet<>(); + if (innerHitsMap != null) { + collapseKeys = innerHitsMap.keySet(); + } + return collapseKeys; + } + + @Override + public DirectIndexQueryResult getCollapseVertices(String key) { + DirectIndexQueryResult result = new DirectIndexQueryResult(); + if (innerHitsMap != null) { + Map responseMap = AtlasType.fromJson(AtlasType.toJson(innerHitsMap.get(key)), Map.class); + Map hits_0 = AtlasType.fromJson(AtlasType.toJson(responseMap.get("hits")), Map.class); + + Integer approximateCount = (Integer) hits_0.get("total").get("value"); + result.setApproximateCount(approximateCount); + + List hits_1 = AtlasType.fromJson(AtlasType.toJson(hits_0.get("hits")), List.class); + Stream> resultStream = hits_1.stream().map(ResultImplDirect::new); + result.setIterator(resultStream.iterator()); + + return result; + } + return null; + } + @Override public double getScore() { Object _score = hit.get("_score"); diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusElement.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusElement.java index d6286bb2c25..080a7c8a104 100644 --- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusElement.java +++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusElement.java @@ -17,13 +17,7 @@ */ package org.apache.atlas.repository.graphdb.janus; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.Iterator; -import java.util.List; -import java.util.Objects; -import java.util.Set; +import java.util.*; import org.apache.atlas.repository.graphdb.AtlasEdge; import org.apache.atlas.repository.graphdb.AtlasElement; @@ -52,6 +46,14 @@ public class AtlasJanusElement implements AtlasElement { private T element; protected AtlasJanusGraph graph; + //excludeProperties: Getting key related issue while Migration mode when fetching few attributes from graph + //This is dirty fix to ignore getting such attributes value from graph & return null explicitly + private static final Set excludeProperties = new HashSet<>(); + static { + excludeProperties.add("replicatedTo"); + excludeProperties.add("replicatedFrom"); + } + public AtlasJanusElement(AtlasJanusGraph graph, T element) { this.element = element; this.graph = graph; @@ -59,6 +61,9 @@ public AtlasJanusElement(AtlasJanusGraph graph, T element) { @Override public T getProperty(String propertyName, Class clazz) { + if (excludeProperties.contains(propertyName)) { + return null; + } //add explicit logic to return null if the property does not exist //This is the behavior Atlas expects. Janus throws an exception @@ -210,6 +215,19 @@ public List getMultiValuedProperty(String propertyName, Class elementT return value; } + @Override + public Set getMultiValuedSetProperty(String propertyName, Class elementType) { + Set value = new HashSet<>(); + Iterator> it = getWrappedElement().properties(propertyName); + + while (it.hasNext()) { + Property currentProperty = it.next(); + Object currentPropertyValue = currentProperty.value(); + value.add((V) currentPropertyValue); + } + return value; + } + @Override public void setListProperty(String propertyName, List values) { setProperty(propertyName, values); diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java index a69a7f02b1f..3a9c9337976 100644 --- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java +++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java @@ -24,6 +24,7 @@ import org.apache.atlas.ApplicationProperties; import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.AtlasException; +import org.apache.atlas.ESAliasRequestBuilder; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.groovy.GroovyExpression; import org.apache.atlas.model.discovery.SearchParams; @@ -45,6 +46,9 @@ import org.apache.atlas.repository.graphdb.utils.IteratorToIterableAdapter; import org.apache.atlas.type.AtlasType; import org.apache.commons.configuration.Configuration; +import org.apache.http.HttpEntity; +import org.apache.http.entity.ContentType; +import org.apache.http.nio.entity.NStringEntity; import org.apache.tinkerpop.gremlin.groovy.jsr223.GremlinGroovyScriptEngine; import org.apache.tinkerpop.gremlin.jsr223.DefaultImportCustomizer; import org.apache.tinkerpop.gremlin.process.traversal.P; @@ -58,6 +62,8 @@ import org.apache.tinkerpop.gremlin.structure.io.IoCore; import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONMapper; import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONWriter; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.search.builder.SearchSourceBuilder; @@ -87,7 +93,9 @@ import java.util.Map; import java.util.Set; +import static org.apache.atlas.AtlasErrorCode.INDEX_SEARCH_FAILED; import static org.apache.atlas.AtlasErrorCode.RELATIONSHIP_CREATE_INVALID_PARAMS; +import static org.apache.atlas.ESAliasRequestBuilder.ESAliasAction.REMOVE; import static org.apache.atlas.repository.Constants.*; import static org.apache.atlas.repository.graphdb.janus.AtlasElasticsearchDatabase.getClient; import static org.apache.atlas.repository.graphdb.janus.AtlasElasticsearchDatabase.getLowLevelClient; @@ -307,13 +315,70 @@ public AtlasIndexQuery elasticsearchQuery(Stri return new AtlasElasticsearchQuery(this, elasticsearchClient, INDEX_PREFIX + indexName, sourceBuilder); } - public AtlasIndexQuery elasticsearchQuery(String indexName, SearchParams searchParams) { + public AtlasIndexQuery elasticsearchQuery(String indexName, SearchParams searchParams) throws AtlasBaseException { if (restClient == null) { LOG.error("restClient is not initiated, failed to run query on ES"); + throw new AtlasBaseException(INDEX_SEARCH_FAILED, "restClient is not initiated"); } return new AtlasElasticsearchQuery(this, restClient, INDEX_PREFIX + indexName, searchParams); } + @Override + public void createOrUpdateESAlias(ESAliasRequestBuilder builder) throws AtlasBaseException { + String aliasRequest = builder.build(); + + HttpEntity entity = new NStringEntity(aliasRequest, ContentType.APPLICATION_JSON); + Request request = new Request("POST", ES_API_ALIASES); + request.setEntity(entity); + + Response response = null; + try { + response = restClient.performRequest(request); + } catch (IOException e) { + LOG.error("Failed to execute direct query on ES {}", e.getMessage()); + throw new AtlasBaseException(AtlasErrorCode.INDEX_ALIAS_FAILED, "creating/updating", e.getMessage()); + } + + int statusCode = response.getStatusLine().getStatusCode();; + if (statusCode != 200) { + throw new AtlasBaseException(AtlasErrorCode.INDEX_ALIAS_FAILED, "creating/updating", "Status code " + statusCode); + } + } + + @Override + public void deleteESAlias(String indexName, String aliasName) throws AtlasBaseException { + ESAliasRequestBuilder builder = new ESAliasRequestBuilder(); + builder.addAction(REMOVE, new ESAliasRequestBuilder.AliasAction(indexName, aliasName)); + + String aliasRequest = builder.build(); + HttpEntity entity = new NStringEntity(aliasRequest, ContentType.APPLICATION_JSON); + + Request request = new Request("POST", ES_API_ALIASES); + request.setEntity(entity); + + Response response = null; + try { + response = restClient.performRequest(request); + } catch (IOException e) { + LOG.error("Failed to execute direct query on ES {}", e.getMessage()); + throw new AtlasBaseException(AtlasErrorCode.INDEX_ALIAS_FAILED, "deleting", e.getMessage()); + } + + int statusCode = response.getStatusLine().getStatusCode();; + if (statusCode != 200) { + throw new AtlasBaseException(AtlasErrorCode.INDEX_ALIAS_FAILED, "deleting", "Status code " + statusCode); + } + } + + @Override + public AtlasIndexQuery elasticsearchQuery(String indexName) throws AtlasBaseException { + if (restClient == null) { + LOG.error("restClient is not initiated, failed to run query on ES"); + throw new AtlasBaseException(INDEX_SEARCH_FAILED, "restClient is not initiated"); + } + return new AtlasElasticsearchQuery(this, indexName, restClient); + } + @Override public AtlasGraphManagement getManagementSystem() { return new AtlasJanusGraphManagement(this, getGraph().openManagement()); diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusIndexQuery.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusIndexQuery.java index 79e22547305..7004e5a6474 100644 --- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusIndexQuery.java +++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusIndexQuery.java @@ -18,6 +18,8 @@ package org.apache.atlas.repository.graphdb.janus; import java.util.Iterator; +import java.util.Map; +import java.util.Set; import com.google.common.base.Preconditions; import org.apache.atlas.exception.AtlasBaseException; @@ -49,6 +51,11 @@ public DirectIndexQueryResult vertices(SearchP return null; } + @Override + public Map directIndexQuery(String query) throws AtlasBaseException { + return null; + } + @Override public Iterator> vertices() { Iterator> results = query.vertices().iterator(); @@ -133,5 +140,15 @@ public AtlasVertex getVertex() { public double getScore() { return source.getScore(); } + + @Override + public Set getCollapseKeys() { + return null; + } + + @Override + public DirectIndexQueryResult getCollapseVertices(String key) { + return null; + } } } diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusVertexQuery.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusVertexQuery.java index 9b34c0fb668..004e952ab8d 100644 --- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusVertexQuery.java +++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusVertexQuery.java @@ -82,6 +82,12 @@ public AtlasVertexQuery label(String label) { return this; } + @Override + public AtlasVertexQuery label(String... labels) { + query.labels(labels); + return this; + } + @Override public AtlasVertexQuery has(String key, Object value) { query.has(key, value); diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/JanusUtils.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/JanusUtils.java new file mode 100644 index 00000000000..0de3bc71495 --- /dev/null +++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/JanusUtils.java @@ -0,0 +1,17 @@ +package org.apache.atlas.repository.graphdb.janus; + +import org.janusgraph.util.encoding.LongEncoding; + +import javax.annotation.Nonnull; +import java.util.Objects; + +public final class JanusUtils { + + private JanusUtils(){} + + public static String toLongEncoding(Object vertexId) { + Objects.requireNonNull(vertexId); + return LongEncoding.encode(Long.parseLong(vertexId.toString())); + } + +} \ No newline at end of file diff --git a/intg/pom.xml b/intg/pom.xml index 40031cb70e8..c15929de351 100644 --- a/intg/pom.xml +++ b/intg/pom.xml @@ -112,6 +112,13 @@ ${guava.version} test + + + + com.launchdarkly + launchdarkly-java-server-sdk + ${launch-darkly.version} + diff --git a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java index b63fab7cf40..2c8f74546ca 100644 --- a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java +++ b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java @@ -35,13 +35,15 @@ public enum AtlasConfiguration { QUERY_PARAM_MAX_LENGTH("atlas.query.param.max.length", 4*1024), REST_API_ENABLE_DELETE_TYPE_OVERRIDE("atlas.rest.enable.delete.type.override", false), - NOTIFICATION_RELATIONSHIPS_ENABLED("atlas.notification.relationships.enabled", false), + NOTIFICATION_RELATIONSHIPS_ENABLED("atlas.notification.relationships.enabled", true), NOTIFICATION_HOOK_TOPIC_NAME("atlas.notification.hook.topic.name", "ATLAS_HOOK"), NOTIFICATION_ENTITIES_TOPIC_NAME("atlas.notification.entities.topic.name", "ATLAS_ENTITIES"), + NOTIFICATION_RELATIONSHIPS_TOPIC_NAME("atlas.notification.relationships.topic.name", "ATLAS_RELATIONSHIPS"), NOTIFICATION_HOOK_CONSUMER_TOPIC_NAMES("atlas.notification.hook.consumer.topic.names", "ATLAS_HOOK"), // a comma separated list of topic names NOTIFICATION_ENTITIES_CONSUMER_TOPIC_NAMES("atlas.notification.entities.consumer.topic.names", "ATLAS_ENTITIES"), // a comma separated list of topic names + NOTIFICATION_RELATIONSHIPS_CONSUMER_TOPIC_NAMES("atlas.notification.relationships.consumer.topic.names", "ATLAS_RELATIONSHIPS"), // a comma separated list of topic names NOTIFICATION_MESSAGE_MAX_LENGTH_BYTES("atlas.notification.message.max.length.bytes", (1000 * 1000)), NOTIFICATION_MESSAGE_COMPRESSION_ENABLED("atlas.notification.message.compression.enabled", true), @@ -81,8 +83,35 @@ public enum AtlasConfiguration { DSL_CACHED_TRANSLATOR("atlas.dsl.cached.translator", true), DEBUG_METRICS_ENABLED("atlas.debug.metrics.enabled", false), TASKS_USE_ENABLED("atlas.tasks.enabled", true), + TASKS_REQUEUE_GRAPH_QUERY("atlas.tasks.requeue.graph.query", false), + TASKS_REQUEUE_POLL_INTERVAL("atlas.tasks.requeue.poll.interval.millis", 60000), + TASKS_QUEUE_SIZE("atlas.tasks.queue.size", 1000), SESSION_TIMEOUT_SECS("atlas.session.timeout.secs", -1), +<<<<<<< HEAD UPDATE_COMPOSITE_INDEX_STATUS("atlas.update.composite.index.status", true); +======= + UPDATE_COMPOSITE_INDEX_STATUS("atlas.update.composite.index.status", true), + TASKS_GRAPH_COMMIT_CHUNK_SIZE("atlas.tasks.graph.commit.chunk.size", 100), + MAX_NUMBER_OF_RETRIES("atlas.tasks.graph.retry.count", 3), + GRAPH_TRAVERSAL_PARALLELISM("atlas.graph.traverse.bucket.size",10), + LINEAGE_ON_DEMAND_ENABLED("atlas.lineage.on.demand.enabled", true), + LINEAGE_ON_DEMAND_DEFAULT_NODE_COUNT("atlas.lineage.on.demand.default.node.count", 3), + LINEAGE_MAX_NODE_COUNT("atlas.lineage.max.node.count", 100), + + SUPPORTED_RELATIONSHIP_EVENTS("atlas.notification.relationships.filter", "asset_readme,asset_links"), + + REST_API_XSS_FILTER_MASK_STRING("atlas.rest.xss.filter.mask.string", "map<[a-zA-Z _,:<>0-9\\x60]*>|struct<[a-zA-Z _,:<>0-9\\x60]*>|array<[a-zA-Z _,:<>0-9\\x60]*>|\\{\\{[a-zA-Z _,-:0-9\\x60\\{\\}]*\\}\\}"), + REST_API_XSS_FILTER_EXLUDE_SERVER_NAME("atlas.rest.xss.filter.exclude.server.name", "atlas-service-atlas.atlas.svc.cluster.local"), + + + INDEX_CLIENT_CONNECTION_TIMEOUT("atlas.index.client.connection.timeout.ms", 900000), + INDEX_CLIENT_SOCKET_TIMEOUT("atlas.index.client.socket.timeout.ms", 900000), + ENABLE_SEARCH_LOGGER("atlas.enable.search.logger", true), + SEARCH_LOGGER_MAX_THREADS("atlas.enable.search.logger.max.threads", 20), + + PERSONA_POLICY_ASSET_MAX_LIMIT("atlas.persona.policy.asset.maxlimit", 1000), + ENABLE_KEYCLOAK_TOKEN_INTROSPECTION("atlas.canary.keycloak.token_introspection", false); +>>>>>>> 7f3c811fb15361ba82bfb4edd7a8393798863ff0 private static final Configuration APPLICATION_PROPERTIES; diff --git a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java index 39e28b3845f..9debcba0e7f 100644 --- a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java +++ b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java @@ -174,10 +174,16 @@ public enum AtlasErrorCode { NOT_VALID_FILE(400, "ATLAS-400-00-09C", "Invalid {0} file"), ATTRIBUTE_NAME_ALREADY_EXISTS_IN_PARENT_TYPE(400, "ATLAS-400-00-09D", "Invalid attribute name: {0}.{1}. Attribute already exists in parent type: {2}"), UNAUTHORIZED_ACCESS(403, "ATLAS-403-00-001", "{0} is not authorized to perform {1}"), + UNAUTHORIZED_CONNECTION_ADMIN(403, "ATLAS-403-00-002", "{0} is not admin for connection {1}"), MISSING_CLASSIFICATION_DISPLAY_NAME(400, "ATLAS-400-00-09E", "Classification displayName is empty/null"), TYPEDEF_ATTR_DISPLAY_NAME_IS_REQUIRED(400, "ATLAS-400-00-09F", "displayName is required for typedef \"{0}\" attribute"), EMPTY_REQUEST(400, "ATLAS-400-00-100", "Empty Request or null, expects Map of List of RelatedObjects with term-id as key"), TYPEDEF_DISPLAY_NAME_IS_REQUIRED(400, "ATLAS-400-00-101", "displayName is required for typedef"), + IMPORT_INVALID_ZIP_ENTRY(400, "ATLAS-400-00-10F", "{0}: invalid zip entry. Reason: {1}"), + INVALID_PAGINATION_STATE(400, "ATLAS-400-00-102", "Invalid pagination state"), + PAGINATION_CAN_ONLY_BE_USED_WITH_DEPTH_ONE(400, "ATLAS-400-00-103", "Pagination can be used only when depth is 1"), + CANT_CALCULATE_VERTEX_COUNTS_WITHOUT_PAGINATION(400, "ATLAS-400-00-104", "Vertex counts can't be calculated without pagination"), + FORBIDDEN_TYPENAME(400,"ATLAS-400-00-107", "Forbidden type: Can not pass builtin type {0}"), // All Not found enums go here TYPE_NAME_NOT_FOUND(404, "ATLAS-404-00-001", "Given typename {0} was invalid"), @@ -204,8 +210,12 @@ public enum AtlasErrorCode { NO_TYPE_NAME_ON_VERTEX(404, "ATLAS-404-00-015", "No typename found for given entity with guid: {0}"), RELATIONSHIP_LABEL_NOT_FOUND(404, "ATLAS-404-00-016", "Given relationshipLabel {0} was invalid"), INVALID_LINEAGE_ENTITY_TYPE_HIDE_PROCESS(404, "ATLAS-404-00-017", "Given instance guid {0} with type {1} is not a valid lineage entity type with hideProcess as true."), + TASK_NOT_FOUND(404, "ATLAS-404-00-018", "Given task guid {0} is invalid/not found"), + RESOURCE_NOT_FOUND(404, "ATLAS-404-00-019", "{0} not found"), + INDEX_NOT_FOUND(404, "ATLAS-404-00-020", "ES index {0} not found"), METHOD_NOT_ALLOWED(405, "ATLAS-405-00-001", "Error 405 - The request method {0} is inappropriate for the URL: {1}"), + DELETE_TAG_PROPAGATION_NOT_ALLOWED(406, "ATLAS-406-00-001", "Classification delete is not allowed; Add/Update classification propagation is in queue for classification: {0} and entity: {1}. Please try again"), // All data conflict errors go here TYPE_ALREADY_EXISTS(409, "ATLAS-409-00-001", "Given type {0} already exists"), @@ -216,13 +226,20 @@ public enum AtlasErrorCode { SAVED_SEARCH_ALREADY_EXISTS(409, "ATLAS-409-00-006", "search named {0} already exists for user {1}"), GLOSSARY_ALREADY_EXISTS(409, "ATLAS-409-00-007", "Glossary with name {0} already exists"), GLOSSARY_TERM_ALREADY_EXISTS(409, "ATLAS-409-00-009", "Glossary term with name {0} already exists"), - GLOSSARY_CATEGORY_ALREADY_EXISTS(409, "ATLAS-409-00-00A", "Glossary category with name {0} already exists"), + GLOSSARY_CATEGORY_ALREADY_EXISTS(409, "ATLAS-409-00-00A", "Glossary category with name {0} already exists on this level"), ACHOR_UPDATION_NOT_SUPPORTED(409, "ATLAS-400-00-0010", "Anchor(glossary) change not supported"), GLOSSARY_IMPORT_FAILED(409, "ATLAS-409-00-011", "Glossary import failed"), TYPE_WITH_DISPLAY_NAME_ALREADY_EXISTS(409, "ATLAS-409-00-012", "Given type {0} already exists"), TYPE_ATTR_WITH_DISPLAY_NAME_ALREADY_EXISTS(409, "ATLAS-409-00-013", "Given attributeDef {0} for type {1} already exists"), RELATIONSHIP_CREATE_INVALID_PARAMS(409, "ATLAS-409-00-014", "Relationship create between same vertex not allowed, vertex guid: {0}"), +<<<<<<< HEAD +======= + OPERATION_NOT_SUPPORTED(409, "ATLAS-409-00-015", "Operation not supported: {0}"), + ACCESS_CONTROL_ALREADY_EXISTS(409, "ATLAS-409-00-016", "{0} with name {1} already exists"), + +>>>>>>> 7f3c811fb15361ba82bfb4edd7a8393798863ff0 CATEGORY_PARENT_FROM_OTHER_GLOSSARY(409, "ATLAS-400-00-0015", "Parent category from another Anchor(glossary) not supported"), + CLASSIFICATION_TYPE_HAS_REFERENCES(409, "ATLAS-400-00-0016", "Given classification {0} [{1}] has references"), // All internal errors go here INTERNAL_ERROR(500, "ATLAS-500-00-001", "Internal server error {0}"), @@ -248,7 +265,30 @@ public enum AtlasErrorCode { FAILED_TO_CREATE_GLOSSARY_TERM(500, "ATLAS-500-00-016", "Error occurred while creating glossary term: {0}"), FAILED_TO_UPDATE_GLOSSARY_TERM(500, "ATLAS-500-00-017", "Error occurred while updating glossary term: {0}"), REPAIR_INDEX_FAILED(500, "ATLAS-500-00-018", "Error occurred while repairing indices: {0}"), - INDEX_SEARCH_FAILED(400, "ATLAS-400-00-102", "Error occurred while running direct index query on ES: {0}"); + INDEX_SEARCH_FAILED(400, "ATLAS-400-00-102", "Error occurred while running direct index query on ES: {0}"), + DEPRECATED_API(400, "ATLAS-400-00-103", "Deprecated API. Use {0} instead"), + DISABLED_API(400, "ATLAS-400-00-104", "API temporarily disabled. Reason: {0}"), + HAS_LINEAGE_GET_EDGE_FAILED(500, "ATLAS-500-00-019", "Error occurred while getting edge between vertices for hasLineage migration: {0}"), + FAILED_TO_REFRESH_TYPE_DEF_CACHE(500, "ATLAS-500-00-20", "Failed to refresh type-def cache"), + CINV_UNHEALTHY(500, "ATLAS-500-00-21", "Unable to process type-definition operations"), + RUNTIME_EXCEPTION(500, "ATLAS-500-00-020", "Runtime exception {0}"), + KEYCLOAK_INIT_FAILED(500, "ATLAS-500-00-022", "Failed to initialize keycloak client: {0}"), + BATCH_SIZE_TOO_LARGE(406, "ATLAS-406-00-001", "Batch size is too large, please use a smaller batch size"), + + + CLASSIFICATION_CURRENTLY_BEING_PROPAGATED(400, "ATLAS-400-00-105", "Classification {0} is currently being propagated."), + TASK_STATUS_NOT_APPROPRIATE(400, "ATLAS-400-00-106", "Unable to restart the task with guid {0} whose status is {1}. "), + NO_LINEAGE_CONSTRAINTS_FOR_GUID(404, "ATLAS-404-00-016", "No lineage constraints found for requested entity with guid : {0}"), + LINEAGE_ON_DEMAND_NOT_ENABLED(400, "ATLAS-400-00-100", "Lineage on demand config: {0} is not enabled"), + + INDEX_ALIAS_FAILED(400, "ATLAS-400-00-108", "Error occurred while {0} ES alias: {1}"), + JSON_ERROR(400, "ATLAS-400-00-109", "Error occurred putting object into JSONObject: {0}"), + DISABLED_OPERATION(400, "ATLAS-400-00-110", "This operation is temporarily disabled as it is under maintenance."), + TASK_INVALID_PARAMETERS(400, "ATLAS-400-00-111", "Invalid parameters for task {0}"), + TASK_TYPE_NOT_SUPPORTED(400, "ATLAS-400-00-112", "Task type {0} is not supported"), + + PERSONA_POLICY_ASSETS_LIMIT_EXCEEDED(400, "ATLAS-400-00-113", "Exceeded limit of maximum allowed assets across policies for a Persona: Limit: {0}, assets: {1}"); + private String errorCode; private String errorMessage; diff --git a/intg/src/main/java/org/apache/atlas/DeleteType.java b/intg/src/main/java/org/apache/atlas/DeleteType.java index 57294bf7ae1..83435c61ea9 100644 --- a/intg/src/main/java/org/apache/atlas/DeleteType.java +++ b/intg/src/main/java/org/apache/atlas/DeleteType.java @@ -23,7 +23,8 @@ public enum DeleteType { DEFAULT, SOFT, - HARD; + HARD, + PURGE; public static DeleteType from(String s) { if(StringUtils.isEmpty(s)) { @@ -37,6 +38,9 @@ public static DeleteType from(String s) { case "hard": return HARD; + case "purge": + return PURGE; + default: return DEFAULT; } diff --git a/intg/src/main/java/org/apache/atlas/ESAliasRequestBuilder.java b/intg/src/main/java/org/apache/atlas/ESAliasRequestBuilder.java new file mode 100644 index 00000000000..45341325cd1 --- /dev/null +++ b/intg/src/main/java/org/apache/atlas/ESAliasRequestBuilder.java @@ -0,0 +1,143 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas; + +import com.fasterxml.jackson.annotation.JsonInclude; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.type.AtlasType; +import org.apache.commons.collections.MapUtils; +import org.codehaus.jettison.json.JSONException; +import org.codehaus.jettison.json.JSONObject; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.apache.atlas.AtlasErrorCode.JSON_ERROR; +import static org.apache.atlas.AtlasErrorCode.OPERATION_NOT_SUPPORTED; +import static org.apache.atlas.ESAliasRequestBuilder.ESAliasAction.ADD; +import static org.apache.atlas.ESAliasRequestBuilder.ESAliasAction.REMOVE; + + +@JsonInclude(JsonInclude.Include.NON_NULL) +public class ESAliasRequestBuilder { + private List jActionObject = new ArrayList<>(); + + public static enum ESAliasAction { + ADD("add"), + REMOVE("remove"); + + String type; + + ESAliasAction(String type) { + this.type = type; + } + + public String getType() { + return type; + } + } + + public ESAliasRequestBuilder addAction(ESAliasAction eSAliasAction, AliasAction action) throws AtlasBaseException { + JSONObject object = new JSONObject(); + String type = ""; + + try { + switch (eSAliasAction) { + case ADD: + type = ADD.getType(); + if (MapUtils.isNotEmpty(action.getFilter())) { + object.put("filter", new JSONObject(AtlasType.toJson(action.getFilter()))); + } + break; + + case REMOVE: + type = REMOVE.getType(); + break; + + default: + throw new AtlasBaseException(OPERATION_NOT_SUPPORTED, String.format("Action %s is not supported by ENUM ESAliasAction", eSAliasAction)); + } + + object.put("index", action.getIndex()); + object.put("alias", action.getAlias()); + + JSONObject j_1 = new JSONObject(); + j_1.put(type, object); + + jActionObject.add(j_1); + + } catch (JSONException e) { + throw new AtlasBaseException(JSON_ERROR, e.getMessage()); + } + return this; + } + + public String build() throws AtlasBaseException { + JSONObject jsonObject = new JSONObject(); + + try { + jsonObject.put("actions", jActionObject); + } catch (JSONException e) { + throw new AtlasBaseException(JSON_ERROR, e.getMessage()); + } + + return jsonObject.toString(); + } + + @JsonInclude(JsonInclude.Include.NON_NULL) + public static class AliasAction { + private String index; + private String alias; + Map filter; + + public AliasAction(String index, String alias, Map filter) { + this.index = index; + this.alias = alias; + this.filter = filter; + } + + public AliasAction(String index, String alias) { + this(index, alias, null); + } + + public String getIndex() { + return index; + } + + public void setIndex(String index) { + this.index = index; + } + + public String getAlias() { + return alias; + } + + public void setAlias(String alias) { + this.alias = alias; + } + + public Map getFilter() { + return filter; + } + + public void setFilter(Map filter) { + this.filter = filter; + } + } +} diff --git a/intg/src/main/java/org/apache/atlas/exception/AtlasBaseException.java b/intg/src/main/java/org/apache/atlas/exception/AtlasBaseException.java index 2010c23a6b6..a3aa1c32e73 100644 --- a/intg/src/main/java/org/apache/atlas/exception/AtlasBaseException.java +++ b/intg/src/main/java/org/apache/atlas/exception/AtlasBaseException.java @@ -19,13 +19,16 @@ import org.apache.atlas.AtlasErrorCode; +import java.util.HashMap; import java.util.List; +import java.util.Map; /** * Base Exception class for Atlas API. */ public class AtlasBaseException extends Exception { + private Map errorDetailsMap = new HashMap<>(); private AtlasErrorCode atlasErrorCode; private String entityGuid; @@ -45,6 +48,12 @@ public AtlasBaseException(final AtlasErrorCode errorCode, final List par this.atlasErrorCode = errorCode; } + public AtlasBaseException(AtlasErrorCode errorCode, Map errorDetailsMap, String ... params) { + super(errorCode.getFormattedErrorMessage(params)); + this.errorDetailsMap = errorDetailsMap; + this.atlasErrorCode = errorCode; + } + public AtlasBaseException() { this(AtlasErrorCode.INTERNAL_ERROR); } @@ -97,4 +106,8 @@ public String getEntityGuid() { public void setEntityGuid(String entityGuid) { this.entityGuid = entityGuid; } + + public Map getErrorDetailsMap() { + return errorDetailsMap; + } } diff --git a/intg/src/main/java/org/apache/atlas/featureflag/AtlasFeatureFlagClient.java b/intg/src/main/java/org/apache/atlas/featureflag/AtlasFeatureFlagClient.java new file mode 100644 index 00000000000..fc0e86f90c9 --- /dev/null +++ b/intg/src/main/java/org/apache/atlas/featureflag/AtlasFeatureFlagClient.java @@ -0,0 +1,61 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.featureflag; + +import com.launchdarkly.sdk.server.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + + +import javax.annotation.PreDestroy; +import java.io.IOException; +import java.util.Objects; + +@Component +final public class AtlasFeatureFlagClient { + + private static final Logger LOG = LoggerFactory.getLogger(AtlasFeatureFlagClient.class); + private final static String LAUNCH_DARKLY_SDK_KEY = Objects.toString(System.getenv("USER_LAUNCH_DARKLY_SDK_KEY"), ""); + + public final static String INSTANCE_DOMAIN_NAME = "https://" + Objects.toString(System.getenv("DOMAIN_NAME"), ""); + public final static String UNQ_CONTEXT_KEY = "context-atlas"; + public final static String CONTEXT_NAME = "Atlas"; + + private static LDClient launchDarklyClient; + + public AtlasFeatureFlagClient() { + try { + //launchDarklyClient = new LDClient(LAUNCH_DARKLY_SDK_KEY); + //not allowing initialising LD client due high number of threads issue + } catch (Exception e) { + LOG.error("Error while initializing LaunchDarkly client", e); + } + } + + public LDClient getClient() { + return launchDarklyClient; + } + + @PreDestroy + public void destroy() throws IOException { + if (launchDarklyClient != null) { + launchDarklyClient.close(); + } + } +} \ No newline at end of file diff --git a/intg/src/main/java/org/apache/atlas/featureflag/FeatureFlagStore.java b/intg/src/main/java/org/apache/atlas/featureflag/FeatureFlagStore.java new file mode 100644 index 00000000000..21f7115089e --- /dev/null +++ b/intg/src/main/java/org/apache/atlas/featureflag/FeatureFlagStore.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.featureflag; + +public interface FeatureFlagStore { + + static final String DISABLE_ACCESS_CONTROL_FEATURE_FLAG_KEY = "disable-access-control-ops"; + static final String IS_INSTANCE_MIGRATED_FEATURE_FLAG_KEY = "users_groups_list_api_migrated_response"; + static final String LINEAGE_EVENTS_FEATURE_FLAG_KEY = "metastore-enable-lineage-events-for-pipeline"; + + boolean evaluate(FeatureFlag flag, String key, boolean value); + + boolean evaluate(FeatureFlag flag, String key, String value); + + enum FeatureFlag { + DISABLE_ACCESS_CONTROL(DISABLE_ACCESS_CONTROL_FEATURE_FLAG_KEY, false), + IS_INSTANCE_MIGRATED(IS_INSTANCE_MIGRATED_FEATURE_FLAG_KEY, false), + ENABLE_LINEAGE_EVENTS(LINEAGE_EVENTS_FEATURE_FLAG_KEY, false), + ADD_CONNECTION_ROLE_IN_ADMIN_ROLE("add-connection-role-in-admin-role", true), + ALLOW_CONNECTION_ADMIN_OPS("allow-connection-admins-ops", true); + + private final String key; + private final boolean defaultValue; + + FeatureFlag(String key, boolean defaultValue) { + this.key = key; + this.defaultValue = defaultValue; + } + + public String getKey() { + return key; + } + + public boolean getDefaultValue() { + return defaultValue; + } + } +} \ No newline at end of file diff --git a/intg/src/main/java/org/apache/atlas/featureflag/FeatureFlagStoreLaunchDarklyImpl.java b/intg/src/main/java/org/apache/atlas/featureflag/FeatureFlagStoreLaunchDarklyImpl.java new file mode 100644 index 00000000000..13bdac5e6df --- /dev/null +++ b/intg/src/main/java/org/apache/atlas/featureflag/FeatureFlagStoreLaunchDarklyImpl.java @@ -0,0 +1,78 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.featureflag; + +import com.launchdarkly.sdk.LDContext; +import com.launchdarkly.sdk.server.LDClient; +import org.springframework.stereotype.Component; +import org.springframework.stereotype.Service; + +import javax.inject.Inject; + +@Component +public class FeatureFlagStoreLaunchDarklyImpl implements FeatureFlagStore { + + private final LDClient client; + + @Inject + public FeatureFlagStoreLaunchDarklyImpl(AtlasFeatureFlagClient client) { + this.client = client.getClient(); + } + + @Override + public boolean evaluate(FeatureFlag flag, String key, boolean value) { + boolean ret; + try { + ret = client.boolVariation(flag.getKey(), getContext(key, value), flag.getDefaultValue()); + } catch (Exception e) { + return flag.getDefaultValue(); + } + + return ret; + } + + @Override + public boolean evaluate(FeatureFlag flag, String key, String value) { + boolean ret; + try { + ret = client.boolVariation(flag.getKey(), getContext(key, value), flag.getDefaultValue()); + } catch (Exception e) { + return flag.getDefaultValue(); + } + + return ret; + } + + private LDContext getContext(String key, String value) { + LDContext ldContext = LDContext.builder(AtlasFeatureFlagClient.UNQ_CONTEXT_KEY) + .name(AtlasFeatureFlagClient.CONTEXT_NAME) + .set(key, value) + .build(); + + return ldContext; + } + + private LDContext getContext(String key, boolean value) { + LDContext ldContext = LDContext.builder(AtlasFeatureFlagClient.UNQ_CONTEXT_KEY) + .name(AtlasFeatureFlagClient.CONTEXT_NAME) + .set(key, value) + .build(); + + return ldContext; + } +} \ No newline at end of file diff --git a/intg/src/main/java/org/apache/atlas/model/audit/AuditSearchParams.java b/intg/src/main/java/org/apache/atlas/model/audit/AuditSearchParams.java index a0538a0f09d..bccf84fccfc 100644 --- a/intg/src/main/java/org/apache/atlas/model/audit/AuditSearchParams.java +++ b/intg/src/main/java/org/apache/atlas/model/audit/AuditSearchParams.java @@ -3,8 +3,9 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.databind.annotation.JsonSerialize; import org.apache.atlas.type.AtlasType; +import org.apache.commons.collections.CollectionUtils; -import java.util.Map; +import java.util.*; import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE; import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ONLY; @@ -14,7 +15,20 @@ public class AuditSearchParams { + private static Map defaultSort = new HashMap<>(); + private Map dsl; + private Set attributes; + + private boolean suppressLogs; + + public AuditSearchParams() { + Map order = new HashMap<>(); + order.put("order", "desc"); + + defaultSort.put("created", order); + suppressLogs = true; + } public void setDsl(Map dsl) { this.dsl = dsl; @@ -24,6 +38,14 @@ public Map getDsl() { return this.dsl; } + public boolean getSuppressLogs() { + return suppressLogs; + } + + public void setSuppressLogs(boolean suppressLogs) { + this.suppressLogs = suppressLogs; + } + public String getQueryStringForGuid(String guid) { String queryWithEntityFilter; if (dsl.get("query") == null || ((Map) dsl.get("query")).isEmpty()) { @@ -40,6 +62,20 @@ public String getQueryStringForGuid(String guid) { } public String getQueryString() { - return dsl != null ? AtlasType.toJson(dsl) : ""; + if (this.dsl != null) { + if (!this.dsl.containsKey("sort")) { + dsl.put("sort", Collections.singleton(defaultSort)); + } + return AtlasType.toJson(dsl); + } + return ""; + } + + public Set getAttributes() { + return attributes; + } + + public void setAttributes(Set attributes) { + this.attributes = attributes; } } diff --git a/intg/src/main/java/org/apache/atlas/model/audit/EntityAuditEventV2.java b/intg/src/main/java/org/apache/atlas/model/audit/EntityAuditEventV2.java index d287ab1bcf4..9a4b03df734 100644 --- a/intg/src/main/java/org/apache/atlas/model/audit/EntityAuditEventV2.java +++ b/intg/src/main/java/org/apache/atlas/model/audit/EntityAuditEventV2.java @@ -25,6 +25,7 @@ import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasEntityHeader; import org.apache.atlas.type.AtlasType; +import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.StringUtils; import javax.xml.bind.annotation.XmlAccessType; @@ -53,6 +54,14 @@ public class EntityAuditEventV2 implements Serializable, Clearable { public static final String SORT_COLUMN_ACTION = "action"; public static final String SORT_COLUMN_TIMESTAMP = "timestamp"; + public AtlasEntityHeader getEntityDetail() { + return entityDetail; + } + + public void setEntityDetail(AtlasEntityHeader entityDetail) { + this.entityDetail = entityDetail; + } + public enum EntityAuditType { ENTITY_AUDIT_V1, ENTITY_AUDIT_V2 } public enum EntityAuditActionV2 { @@ -112,6 +121,8 @@ public static EntityAuditActionV2 fromString(String strValue) { } } + private String entityQualifiedName; + private String typeName; private String entityId; private long timestamp; private long created; @@ -122,6 +133,8 @@ public static EntityAuditActionV2 fromString(String strValue) { private AtlasEntity entity; private EntityAuditType type; private Map detail; + private AtlasEntityHeader entityDetail; + private Map headers; public EntityAuditEventV2() { } @@ -221,6 +234,22 @@ public void setDetail(Map detail) { this.detail = detail; } + public String getTypeName() { + return typeName; + } + + public void setTypeName(String typeName) { + this.typeName = typeName; + } + + public Map getHeaders() { + return headers; + } + + public void setHeaders(Map headers) { + this.headers = headers; + } + @JsonIgnore public String getEntityDefinitionString() { if (entity != null) { @@ -235,6 +264,14 @@ public void setEntityDefinition(String entityDefinition) { this.entity = AtlasType.fromJson(entityDefinition, AtlasEntity.class); } + public String getEntityQualifiedName() { + return entityQualifiedName; + } + + public void setEntityQualifiedName(String entityQualifiedName) { + this.entityQualifiedName = entityQualifiedName; + } + @Override public boolean equals(Object o) { if (this == o) { return true; } @@ -250,12 +287,15 @@ public boolean equals(Object o) { Objects.equals(entity, that.entity) && Objects.equals(type, that.type) && Objects.equals(detail, that.detail) && - Objects.equals(created, that.created); + Objects.equals(created, that.created) && + Objects.equals(typeName, that.typeName) && + Objects.equals(entityQualifiedName, that.entityQualifiedName) && + Objects.equals(headers, that.headers); } @Override public int hashCode() { - return Objects.hash(entityId, timestamp, user, action, details, eventKey, entity, type, detail, created); + return Objects.hash(entityId, timestamp, user, action, details, eventKey, entity, type, detail, created, entityQualifiedName, typeName, headers); } @Override @@ -263,6 +303,8 @@ public String toString() { final StringBuilder sb = new StringBuilder("EntityAuditEventV2{"); sb.append("entityId='").append(entityId).append('\''); + sb.append("typeName='").append(typeName).append('\''); + sb.append("entityQualifiedName='").append(entityQualifiedName).append('\''); sb.append(", timestamp=").append(timestamp); sb.append(", user='").append(user).append('\''); sb.append(", action=").append(action); @@ -272,6 +314,7 @@ public String toString() { sb.append(", type=").append(type); sb.append(", detail=").append(detail); sb.append(", created=").append(created); + sb.append(", headers=").append(headers); sb.append('}'); return sb.toString(); @@ -291,6 +334,8 @@ public AtlasEntityHeader getEntityHeader() { @Override public void clear() { entityId = null; + typeName = null; + entityQualifiedName = null; timestamp = 0L; user = null; action = null; @@ -300,6 +345,7 @@ public void clear() { type = null; detail = null; created = 0L; + headers = null; } private String getJsonPartFromDetails() { @@ -309,6 +355,8 @@ private String getJsonPartFromDetails() { if(bracketStartPosition != -1) { ret = details.substring(bracketStartPosition); } + } else if(MapUtils.isNotEmpty(detail)) { + ret = AtlasType.toJson(detail); } return ret; diff --git a/intg/src/main/java/org/apache/atlas/model/discovery/IndexSearchParams.java b/intg/src/main/java/org/apache/atlas/model/discovery/IndexSearchParams.java index d6cb2ab9a09..c89a9c1c8d1 100644 --- a/intg/src/main/java/org/apache/atlas/model/discovery/IndexSearchParams.java +++ b/intg/src/main/java/org/apache/atlas/model/discovery/IndexSearchParams.java @@ -1,31 +1,27 @@ package org.apache.atlas.model.discovery; -import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.fasterxml.jackson.annotation.JsonInclude; import org.apache.atlas.type.AtlasType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.HashMap; -import java.util.HashSet; import java.util.Map; import java.util.Set; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE; -import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ONLY; - -@JsonAutoDetect(getterVisibility = PUBLIC_ONLY, setterVisibility = PUBLIC_ONLY, fieldVisibility = NONE) -@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) +@JsonInclude(JsonInclude.Include.NON_EMPTY) +@JsonIgnoreProperties(ignoreUnknown=true) public class IndexSearchParams extends SearchParams { private static final Logger LOG = LoggerFactory.getLogger(IndexSearchParams.class); +<<<<<<< HEAD private static final Pattern pattern = Pattern.compile("(?<=\").+?(?=\")"); +======= +>>>>>>> 7f3c811fb15361ba82bfb4edd7a8393798863ff0 private Map dsl; + private String purpose; + private String persona; private String queryString; /* @@ -42,6 +38,10 @@ public String getQuery() { return queryString; } + public Map getDsl() { + return dsl; + } + public void setDsl(Map dsl) { this.dsl = dsl; queryString = AtlasType.toJson(dsl); @@ -55,6 +55,25 @@ public void setAllowDeletedRelations(boolean allowDeletedRelations) { this.allowDeletedRelations = allowDeletedRelations; } +<<<<<<< HEAD +======= + public String getPurpose() { + return purpose; + } + + public void setPurpose(String purpose) { + this.purpose = purpose; + } + + public String getPersona() { + return persona; + } + + public void setPersona(String persona) { + this.persona = persona; + } + +>>>>>>> 7f3c811fb15361ba82bfb4edd7a8393798863ff0 public void setRelationAttributes(Set relationAttributes) { this.relationAttributes = relationAttributes; } @@ -62,10 +81,11 @@ public void setRelationAttributes(Set relationAttributes) { @Override public String toString() { return "IndexSearchParams{" + - "dsl='" + dsl + '\'' + + "dsl=" + dsl + + ", purpose='" + purpose + '\'' + + ", persona='" + persona + '\'' + ", queryString='" + queryString + '\'' + - ", attributes=" + attributes + - ", relationAttributes=" + relationAttributes + + ", allowDeletedRelations=" + allowDeletedRelations + '}'; } } diff --git a/intg/src/main/java/org/apache/atlas/model/discovery/SearchParams.java b/intg/src/main/java/org/apache/atlas/model/discovery/SearchParams.java index f0583a0d23f..dc5e84c3849 100644 --- a/intg/src/main/java/org/apache/atlas/model/discovery/SearchParams.java +++ b/intg/src/main/java/org/apache/atlas/model/discovery/SearchParams.java @@ -1,14 +1,29 @@ package org.apache.atlas.model.discovery; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; + import java.util.Set; -public abstract class SearchParams { + +@JsonInclude(JsonInclude.Include.NON_EMPTY) +@JsonIgnoreProperties(ignoreUnknown=true) +public class SearchParams { Set attributes; Set relationAttributes; + Set collapseAttributes; + Set collapseRelationAttributes; boolean showSearchScore; + boolean suppressLogs; + boolean excludeMeanings; + boolean excludeClassifications; + + RequestMetadata requestMetadata = new RequestMetadata(); - public abstract String getQuery(); + public String getQuery() { + return getQuery(); + } public Set getAttributes() { return attributes; @@ -26,6 +41,30 @@ public void setRelationAttributes(Set relationAttributes) { this.relationAttributes = relationAttributes; } + public Set getCollapseAttributes() { + return collapseAttributes; + } + + public void setCollapseAttributes(Set collapseAttributes) { + this.collapseAttributes = collapseAttributes; + } + + public Set getCollapseRelationAttributes() { + return collapseRelationAttributes; + } + + public void setCollapseRelationAttributes(Set collapseRelationAttributes) { + this.collapseRelationAttributes = collapseRelationAttributes; + } + + public Set getUtmTags() { + return requestMetadata.utmTags; + } + + public void setUtmTags(Set utmTags) { + this.requestMetadata.utmTags = utmTags; + } + public boolean getShowSearchScore() { return showSearchScore; } @@ -33,4 +72,78 @@ public boolean getShowSearchScore() { public void setShowSearchScore(boolean showSearchScore) { this.showSearchScore = showSearchScore; } + + public boolean getSuppressLogs() { + return suppressLogs; + } + + public void setSuppressLogs(boolean suppressLogs) { + this.suppressLogs = suppressLogs; + } + + public boolean isExcludeClassifications() { + return excludeClassifications; + } + + public void setExcludeClassifications(boolean excludeClassifications) { + this.excludeClassifications = excludeClassifications; + } + + public boolean isExcludeMeanings() { + return excludeMeanings; + } + + public void setExcludeMeanings(boolean excludeMeanings) { + this.excludeMeanings = excludeMeanings; + } + + public boolean isSaveSearchLog() { + return requestMetadata.saveSearchLog; + } + + public void setSaveSearchLog(boolean saveSearchLog) { + this.requestMetadata.saveSearchLog = saveSearchLog; + } + + public RequestMetadata getRequestMetadata() { + return requestMetadata; + } + + public void setRequestMetadata(RequestMetadata requestMetadata) { + this.requestMetadata = requestMetadata; + } + + public String getSearchInput() { + return this.requestMetadata.getSearchInput(); + } + + static class RequestMetadata { + private String searchInput; + private Set utmTags; + private boolean saveSearchLog; + + public String getSearchInput() { + return searchInput; + } + + public Set getUtmTags() { + return utmTags; + } + + public boolean isSaveSearchLog() { + return saveSearchLog; + } + + public void setSearchInput(String searchInput) { + this.searchInput = searchInput; + } + + public void setUtmTags(Set utmTags) { + this.utmTags = utmTags; + } + + public void setSaveSearchLog(boolean saveSearchLog) { + this.saveSearchLog = saveSearchLog; + } + } } diff --git a/intg/src/main/java/org/apache/atlas/model/instance/AtlasClassification.java b/intg/src/main/java/org/apache/atlas/model/instance/AtlasClassification.java index 9be6e8b4725..cf16ef90570 100644 --- a/intg/src/main/java/org/apache/atlas/model/instance/AtlasClassification.java +++ b/intg/src/main/java/org/apache/atlas/model/instance/AtlasClassification.java @@ -60,6 +60,7 @@ public class AtlasClassification extends AtlasStruct implements Serializable { private Boolean propagate = null; private List validityPeriods = null; private Boolean removePropagationsOnEntityDelete = null; + private Boolean restrictPropagationThroughLineage = null; public AtlasClassification() { this(null, null); @@ -91,6 +92,7 @@ public AtlasClassification(AtlasClassification other) { setValidityPeriods(other.getValidityPeriods()); setDisplayName(other.getDisplayName()); setRemovePropagationsOnEntityDelete(other.getRemovePropagationsOnEntityDelete()); + setRestrictPropagationThroughLineage(other.getRestrictPropagationThroughLineage()); } } @@ -145,6 +147,13 @@ public Boolean getRemovePropagationsOnEntityDelete() { public void setRemovePropagationsOnEntityDelete(Boolean removePropagationsOnEntityDelete) { this.removePropagationsOnEntityDelete = removePropagationsOnEntityDelete; } + public void setRestrictPropagationThroughLineage(Boolean restrictPropagationThroughLineage){ + this.restrictPropagationThroughLineage = restrictPropagationThroughLineage; + } + + public Boolean getRestrictPropagationThroughLineage(){ + return restrictPropagationThroughLineage; + } @JsonIgnore public void addValityPeriod(TimeBoundary validityPeriod) { @@ -169,12 +178,25 @@ public boolean equals(Object o) { Objects.equals(removePropagationsOnEntityDelete, that.removePropagationsOnEntityDelete) && Objects.equals(entityGuid, that.entityGuid) && entityStatus == that.entityStatus && - Objects.equals(validityPeriods, that.validityPeriods); + Objects.equals(validityPeriods, that.validityPeriods) && Objects.equals(restrictPropagationThroughLineage, that.restrictPropagationThroughLineage); + } + + public boolean checkForUpdate(Object o) { + if (this == o) { return true; } + if (o == null || getClass() != o.getClass()) { return false; } + if (!super.equals(o)) { return false; } + AtlasClassification that = (AtlasClassification) o; + return Objects.equals(entityGuid, that.entityGuid) && + entityStatus == that.entityStatus && + Objects.equals(validityPeriods, that.validityPeriods) && + (Objects.equals(propagate, that.propagate) || (propagate == null)) && + (Objects.equals(removePropagationsOnEntityDelete, that.removePropagationsOnEntityDelete) || (removePropagationsOnEntityDelete == null)) && + (Objects.equals(restrictPropagationThroughLineage, that.restrictPropagationThroughLineage) || (restrictPropagationThroughLineage == null)); } @Override public int hashCode() { - return Objects.hash(super.hashCode(), entityGuid, entityStatus, propagate, removePropagationsOnEntityDelete); + return Objects.hash(super.hashCode(), entityGuid, entityStatus, propagate, removePropagationsOnEntityDelete, restrictPropagationThroughLineage); } @Override @@ -187,10 +209,12 @@ public String toString() { sb.append(", removePropagationsOnEntityDelete=").append(removePropagationsOnEntityDelete); sb.append(", displayName=").append(displayName); sb.append(", validityPeriods=").append(validityPeriods); + sb.append(", restrictPropagationThroughLineage=").append(restrictPropagationThroughLineage); sb.append('}'); return sb.toString(); } + /** * REST serialization friendly list. */ diff --git a/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntity.java b/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntity.java index a880370e883..e9c923653ec 100644 --- a/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntity.java +++ b/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntity.java @@ -74,6 +74,7 @@ public class AtlasEntity extends AtlasStruct implements Serializable { /** * Status of the entity - can be active or deleted. Deleted entities are not removed from Atlas store. + * Purged status also used as Relationship status when entity is Hard Deleted/Purged */ public enum Status { ACTIVE, DELETED, PURGED } @@ -88,6 +89,7 @@ public enum Status { ACTIVE, DELETED, PURGED } private Date createTime = null; private Date updateTime = null; private Long version = 0L; + private Boolean starred = null; private Map relationshipAttributes; private List classifications; @@ -279,6 +281,14 @@ public void setStatus(Status status) { this.status = status; } + public Boolean getStarred() { + return starred; + } + + public void setStarred(Boolean starred) { + this.starred = starred; + } + public String getCreatedBy() { return createdBy; } diff --git a/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntityHeader.java b/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntityHeader.java index 52226180f0f..ed97c30b0c9 100644 --- a/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntityHeader.java +++ b/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntityHeader.java @@ -23,6 +23,7 @@ import com.fasterxml.jackson.databind.annotation.JsonSerialize; import org.apache.atlas.model.PList; import org.apache.atlas.model.SearchFilter.SortType; +import org.apache.atlas.model.discovery.AtlasSearchResult; import org.apache.atlas.model.glossary.relations.AtlasTermAssignmentHeader; import org.apache.atlas.model.typedef.AtlasBaseTypeDef; import org.apache.atlas.model.typedef.AtlasEntityDef; @@ -70,6 +71,10 @@ public class AtlasEntityHeader extends AtlasStruct implements Serializable { private Date createTime = null; private Date updateTime = null; private String deleteHandler = null; +<<<<<<< HEAD +======= + private Map collapse = null; +>>>>>>> 7f3c811fb15361ba82bfb4edd7a8393798863ff0 public AtlasEntityHeader() { this(null, null); @@ -250,6 +255,17 @@ public void setDeleteHandler(String deleteHandler) { this.deleteHandler = deleteHandler; } +<<<<<<< HEAD +======= + public Map getCollapse() { + return collapse; + } + + public void setCollapse(Map collapse) { + this.collapse = collapse; + } + +>>>>>>> 7f3c811fb15361ba82bfb4edd7a8393798863ff0 @Override public StringBuilder toString(StringBuilder sb) { if (sb == null) { diff --git a/intg/src/main/java/org/apache/atlas/model/instance/AtlasEvaluatePolicyRequest.java b/intg/src/main/java/org/apache/atlas/model/instance/AtlasEvaluatePolicyRequest.java index d69467f0d8d..b2fe6738eb1 100644 --- a/intg/src/main/java/org/apache/atlas/model/instance/AtlasEvaluatePolicyRequest.java +++ b/intg/src/main/java/org/apache/atlas/model/instance/AtlasEvaluatePolicyRequest.java @@ -60,6 +60,7 @@ public class AtlasEvaluatePolicyRequest implements Serializable { private String classification; + private String businessMetadata; public AtlasEvaluatePolicyRequest() { @@ -163,6 +164,14 @@ public void setClassification(String classification) { this.classification = classification; } + public String getBusinessMetadata() { + return businessMetadata; + } + + public void setBusinessMetadata(String businessMetadata) { + this.businessMetadata = businessMetadata; + } + public StringBuilder toString(StringBuilder sb) { if (sb == null) { sb = new StringBuilder(); diff --git a/intg/src/main/java/org/apache/atlas/model/instance/AtlasEvaluatePolicyResponse.java b/intg/src/main/java/org/apache/atlas/model/instance/AtlasEvaluatePolicyResponse.java index 7bcfe4236fc..28fd3239cd3 100644 --- a/intg/src/main/java/org/apache/atlas/model/instance/AtlasEvaluatePolicyResponse.java +++ b/intg/src/main/java/org/apache/atlas/model/instance/AtlasEvaluatePolicyResponse.java @@ -17,7 +17,6 @@ */ package org.apache.atlas.model.instance; -import org.apache.atlas.model.typedef.AtlasBaseTypeDef; import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.databind.annotation.JsonSerialize; @@ -26,7 +25,6 @@ import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlRootElement; import java.io.Serializable; -import java.util.Set; import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE; import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ONLY; @@ -49,6 +47,7 @@ public class AtlasEvaluatePolicyResponse implements Serializable { private String entityId; private String classification; + private String businessMetadata; private String relationShipTypeName; private String entityTypeEnd1; @@ -66,13 +65,14 @@ public class AtlasEvaluatePolicyResponse implements Serializable { - public AtlasEvaluatePolicyResponse(String typeName, String entityGuid, String action, String entityId , Boolean allowed ,String errorCode) { + public AtlasEvaluatePolicyResponse(String typeName, String entityGuid, String action, String entityId , Boolean allowed ,String errorCode, String businessMetadata) { this.typeName = typeName; this.entityGuid = entityGuid; this.action = action; this.entityId = entityId; this.allowed = allowed; this.errorCode = errorCode; + this.businessMetadata = businessMetadata; } public AtlasEvaluatePolicyResponse(String typeName, String entityGuid, String action, String entityId , String classification, Boolean allowed, String errorCode) { @@ -209,6 +209,16 @@ public void setErrorCode(String errorCode) { this.errorCode = errorCode; } + + public String getBusinessMetadata() { + return businessMetadata; + } + + public void setBusinessMetadata(String businessMetadata) { + this.businessMetadata = businessMetadata; + } + + public StringBuilder toString(StringBuilder sb) { if (sb == null) { sb = new StringBuilder(); diff --git a/intg/src/main/java/org/apache/atlas/model/instance/AtlasHasLineageRequest.java b/intg/src/main/java/org/apache/atlas/model/instance/AtlasHasLineageRequest.java new file mode 100644 index 00000000000..42f97dba33a --- /dev/null +++ b/intg/src/main/java/org/apache/atlas/model/instance/AtlasHasLineageRequest.java @@ -0,0 +1,94 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.model.instance; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import java.io.Serializable; + +import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE; +import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ONLY; + +@JsonAutoDetect(getterVisibility=PUBLIC_ONLY, setterVisibility=PUBLIC_ONLY, fieldVisibility=NONE) +@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown=true) +@XmlRootElement +@XmlAccessorType(XmlAccessType.PROPERTY) +public class AtlasHasLineageRequest implements Serializable { + private static final long serialVersionUID = 1L; + + public AtlasHasLineageRequest() { + } + + private String assetGuid; + private String processGuid; + private String endGuid; + private String label; + + public String getProcessGuid() { + return processGuid; + } + + public void setProcessGuid(String processGuid) { + this.processGuid = processGuid; + } + + public String getEndGuid() { + return endGuid; + } + + public void setEndGuid(String endGuid) { + this.endGuid = endGuid; + } + + public String getLabel() { + return label; + } + + public void setLabel(String label) { + this.label = label; + } + + public String getAssetGuid() { + return assetGuid; + } + + public void setAssetGuid(String assetGuid) { + this.assetGuid = assetGuid; + } + + @Override + public String toString() { + return "AtlasHasLineageRequest{" + + "assetGuid='" + assetGuid + '\'' + + "processGuid='" + processGuid + '\'' + + ", endGuid='" + endGuid + '\'' + + ", label='" + label + '\'' + + '}'; + } + +} + + + + diff --git a/intg/src/main/java/org/apache/atlas/model/instance/AtlasHasLineageRequests.java b/intg/src/main/java/org/apache/atlas/model/instance/AtlasHasLineageRequests.java new file mode 100644 index 00000000000..77ce57c47a9 --- /dev/null +++ b/intg/src/main/java/org/apache/atlas/model/instance/AtlasHasLineageRequests.java @@ -0,0 +1,64 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.model.instance; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import java.io.Serializable; +import java.util.List; + +import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE; +import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ONLY; + +@JsonAutoDetect(getterVisibility=PUBLIC_ONLY, setterVisibility=PUBLIC_ONLY, fieldVisibility=NONE) +@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown=true) +@XmlRootElement +@XmlAccessorType(XmlAccessType.PROPERTY) +public class AtlasHasLineageRequests implements Serializable { + private static final long serialVersionUID = 1L; + private List request; + + public AtlasHasLineageRequests() { + } + + public List getRequest() { + return request; + } + + public void setRequest(List request) { + this.request = request; + } + + @Override + public String toString() { + return "AtlasHasLineageRequests{" + + "request=" + request + + '}'; + } + +} + + + + diff --git a/intg/src/main/java/org/apache/atlas/model/instance/AtlasRelationship.java b/intg/src/main/java/org/apache/atlas/model/instance/AtlasRelationship.java index 8681cfd9025..4ce72f3bb46 100644 --- a/intg/src/main/java/org/apache/atlas/model/instance/AtlasRelationship.java +++ b/intg/src/main/java/org/apache/atlas/model/instance/AtlasRelationship.java @@ -25,6 +25,7 @@ import org.apache.atlas.model.typedef.AtlasRelationshipDef; import org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; @@ -71,6 +72,10 @@ public class AtlasRelationship extends AtlasStruct implements Serializable { public static final String KEY_BLOCKED_PROPAGATED_CLASSIFICATIONS = "blockedPropagatedClassifications"; public static final String KEY_PROPAGATED_CLASSIFICATIONS = "propagatedClassifications"; + // RELATIONSHIP_DEF and RELATIONSHIP_END_TO_ES_DOC_ID_MAP used for Kafka consumers to handle ES relationship sync + public static final String RELATIONSHIP_DEF = "relationshipDef"; + public static final String RELATIONSHIP_END_TO_ES_DOC_ID_MAP = "relationshipEndToESDocIdMap"; + private String guid = null; private String homeId = null; private Integer provenanceType = null; @@ -85,11 +90,14 @@ public class AtlasRelationship extends AtlasStruct implements Serializable { private Date updateTime = null; private Long version = 0L; - public enum Status { ACTIVE, DELETED } + public enum Status { ACTIVE, DELETED, PURGED} private Set propagatedClassifications; private Set blockedPropagatedClassifications; + private Map relationshipDef; + private Map relationshipEndToESDocIdMap; + @JsonIgnore private static AtomicLong s_nextId = new AtomicLong(System.nanoTime()); @@ -247,6 +255,22 @@ public AtlasRelationship(AtlasRelationship other) { } } + public Map getRelationshipDef() { + return relationshipDef; + } + + public void setRelationshipDef(Map relationshipDef) { + this.relationshipDef = relationshipDef; + } + + public Map getRelationshipEndToESDocIdMap() { + return relationshipEndToESDocIdMap; + } + + public void setRelationshipEndToESDocIdMap(Map relationshipEndToESDocIdMap) { + this.relationshipEndToESDocIdMap = relationshipEndToESDocIdMap; + } + public String getGuid() { return guid; } @@ -404,6 +428,12 @@ public StringBuilder toString(StringBuilder sb) { sb.append(", blockedPropagatedClassifications=["); dumpObjects(blockedPropagatedClassifications, sb); sb.append("]"); + sb.append(", relationshipDef=["); + dumpObjects(relationshipDef, sb); + sb.append("]"); + sb.append(", relationshipEndToESDocIdMap=["); + dumpObjects(relationshipEndToESDocIdMap, sb); + sb.append("]"); sb.append('}'); return sb; diff --git a/intg/src/main/java/org/apache/atlas/model/instance/AtlasRelationshipHeader.java b/intg/src/main/java/org/apache/atlas/model/instance/AtlasRelationshipHeader.java index ad3b98ed4c0..a9ce33907cb 100644 --- a/intg/src/main/java/org/apache/atlas/model/instance/AtlasRelationshipHeader.java +++ b/intg/src/main/java/org/apache/atlas/model/instance/AtlasRelationshipHeader.java @@ -30,7 +30,9 @@ import javax.xml.bind.annotation.XmlRootElement; import javax.xml.bind.annotation.XmlSeeAlso; import java.io.Serializable; +import java.util.Date; import java.util.List; +import java.util.Map; import java.util.Objects; import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE; @@ -51,6 +53,14 @@ public class AtlasRelationshipHeader extends AtlasStruct implements Serializable private String label = null; private AtlasObjectId end1 = null; private AtlasObjectId end2 = null; + private Map relationshipDef = null; + private Map relationshipEndToESDocIdMap = null; + private int provenanceType = 0; + private String createdBy = null; + private String updatedBy = null; + private Date createTime = null; + private Date updateTime = null; + private long version = 0L; public AtlasRelationshipHeader() { @@ -61,17 +71,25 @@ public AtlasRelationshipHeader(String typeName, String guid) { setGuid(guid); } - public AtlasRelationshipHeader(String typeName, String guid, AtlasObjectId end1, AtlasObjectId end2, AtlasRelationshipDef.PropagateTags propagateTags) { + public AtlasRelationshipHeader(String typeName, String guid, AtlasObjectId end1, AtlasObjectId end2, AtlasRelationshipDef.PropagateTags propagateTags, Map relationshipDef, Map relationshipEndToESDocIdMap) { this(typeName, guid); this.propagateTags = propagateTags; setEnd1(end1); setEnd2(end2); + setRelationshipDef(relationshipDef); + setRelationshipEndToESDocIdMap(relationshipEndToESDocIdMap); } public AtlasRelationshipHeader(AtlasRelationship relationship) { - this(relationship.getTypeName(), relationship.getGuid(), relationship.getEnd1(), relationship.getEnd2(), relationship.getPropagateTags()); + this(relationship.getTypeName(), relationship.getGuid(), relationship.getEnd1(), relationship.getEnd2(), relationship.getPropagateTags(), relationship.getRelationshipDef(), relationship.getRelationshipEndToESDocIdMap()); setLabel(relationship.getLabel()); + setCreatedBy(relationship.getCreatedBy()); + setCreateTime(relationship.getCreateTime()); + setUpdatedBy(relationship.getUpdatedBy()); + setUpdateTime(relationship.getUpdateTime()); + setVersion(relationship.getVersion()); + switch (relationship.getStatus()) { case ACTIVE: setStatus(AtlasEntity.Status.ACTIVE); @@ -80,6 +98,10 @@ public AtlasRelationshipHeader(AtlasRelationship relationship) { case DELETED: setStatus(AtlasEntity.Status.DELETED); break; + + case PURGED: + setStatus(AtlasEntity.Status.PURGED); + break; } } @@ -132,6 +154,70 @@ public void setEnd2(AtlasObjectId end2) { this.end2 = end2; } + public Map getRelationshipDef() { + return relationshipDef; + } + + public void setRelationshipDef(Map relationshipDef) { + this.relationshipDef = relationshipDef; + } + + public Map getRelationshipEndToESDocIdMap() { + return relationshipEndToESDocIdMap; + } + + public void setRelationshipEndToESDocIdMap(Map relationshipEndToESDocIdMap) { + this.relationshipEndToESDocIdMap = relationshipEndToESDocIdMap; + } + + public int getProvenanceType() { + return provenanceType; + } + + public void setProvenanceType(int provenanceType) { + this.provenanceType = provenanceType; + } + + public String getCreatedBy() { + return createdBy; + } + + public void setCreatedBy(String createdBy) { + this.createdBy = createdBy; + } + + public String getUpdatedBy() { + return updatedBy; + } + + public void setUpdatedBy(String updatedBy) { + this.updatedBy = updatedBy; + } + + public Date getCreateTime() { + return createTime; + } + + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + public Date getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(Date updateTime) { + this.updateTime = updateTime; + } + + public long getVersion() { + return version; + } + + public void setVersion(long version) { + this.version = version; + } + public StringBuilder toString(StringBuilder sb) { if (sb == null) { sb = new StringBuilder(); @@ -144,6 +230,8 @@ public StringBuilder toString(StringBuilder sb) { sb.append(", propagateTags=").append(propagateTags); sb.append(", end1=").append(end1); sb.append(", end2=").append(end2); + sb.append(", relationshipDef=").append(relationshipDef); + sb.append(", relationshipEndToESDocIdMap=").append(relationshipEndToESDocIdMap); super.toString(sb); sb.append('}'); diff --git a/intg/src/main/java/org/apache/atlas/model/instance/AtlasStruct.java b/intg/src/main/java/org/apache/atlas/model/instance/AtlasStruct.java index 027b1607c5d..1dd720aa6e9 100644 --- a/intg/src/main/java/org/apache/atlas/model/instance/AtlasStruct.java +++ b/intg/src/main/java/org/apache/atlas/model/instance/AtlasStruct.java @@ -31,6 +31,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.ArrayList; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; @@ -39,7 +40,9 @@ import org.apache.atlas.model.PList; import org.apache.atlas.model.SearchFilter.SortType; +import org.apache.atlas.model.TypeCategory; import org.apache.atlas.model.typedef.AtlasBaseTypeDef; +import org.apache.atlas.type.AtlasStructType; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; @@ -65,6 +68,14 @@ public class AtlasStruct implements Serializable { @Deprecated public static final DateFormat DATE_FORMATTER = new SimpleDateFormat(SERIALIZED_DATE_FORMAT_STR); + private static final List ALLOWED_DATATYPES_FOR_DEFAULT_NULL = new ArrayList() { + { + add("int"); + add("long"); + add("float"); + } + }; + private String typeName; private Map attributes; @@ -99,6 +110,22 @@ public AtlasStruct(Map map) { } } + public AtlasStruct(Map map, Map attributeTypes, boolean setDefaultValues) { + if (map != null) { + Object typeName = map.get(KEY_TYPENAME); + Map attributes = (map.get(KEY_ATTRIBUTES) instanceof Map) ? (Map) map.get(KEY_ATTRIBUTES) : map; + + if (typeName != null) { + setTypeName(typeName.toString()); + } + if (setDefaultValues) { + setAttributes(new HashMap<>(attributes), attributeTypes); + } else { + setAttributes(new HashMap<>(attributes)); + } + } + } + public AtlasStruct(AtlasStruct other) { if (other != null) { setTypeName(other.getTypeName()); @@ -122,6 +149,35 @@ public void setAttributes(Map attributes) { this.attributes = attributes; } + public void setAttributes(Map attributes, + Map attributeTypes) { + if (MapUtils.isNotEmpty(attributeTypes) && MapUtils.isNotEmpty(attributes)) { + this.attributes = new HashMap<>(); + for (Map.Entry entry : attributeTypes.entrySet()) { + String attrName = entry.getKey(); + AtlasStructType.AtlasAttribute attrType = entry.getValue(); + Object attrValue = attributes.get(attrName); + + if (attrType.getAttributeType().getTypeCategory() == TypeCategory.PRIMITIVE) { + if (attrValue == null) { + if (attrType.getAttributeDef().getDefaultValue() != null) { + attrValue = attrType.getAttributeType().createDefaultValue(); + } else if (attrType.getAttributeDef().getIsDefaultValueNull() + && ALLOWED_DATATYPES_FOR_DEFAULT_NULL.contains(attrType.getAttributeType().getTypeName())) { + attrValue = null; + } else if (attrType.getAttributeDef().getIsOptional()) { + attrValue = attrType.getAttributeType().createOptionalDefaultValue(); + } else { + attrValue = attrType.getAttributeType().createDefaultValue(); + } + } + } + + this.attributes.put(attrName, attrValue); + } + } + } + public boolean hasAttribute(String name) { Map a = this.attributes; diff --git a/intg/src/main/java/org/apache/atlas/model/instance/RelationshipMutationContext.java b/intg/src/main/java/org/apache/atlas/model/instance/RelationshipMutationContext.java new file mode 100644 index 00000000000..8ab1ab6d215 --- /dev/null +++ b/intg/src/main/java/org/apache/atlas/model/instance/RelationshipMutationContext.java @@ -0,0 +1,36 @@ +package org.apache.atlas.model.instance; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.curator.shaded.com.google.common.collect.ImmutableList; + +import java.util.ArrayList; +import java.util.List; + +public final class RelationshipMutationContext { + + private final List createdRelationships; + private final List updatedRelationships; + private final List deletedRelationships; + + private RelationshipMutationContext(List createdRelationships, List updatedRelationships, List deletedRelationships) { + this.createdRelationships = CollectionUtils.isNotEmpty(createdRelationships) ? ImmutableList.copyOf(createdRelationships) : new ArrayList<>(); + this.updatedRelationships = CollectionUtils.isNotEmpty(updatedRelationships) ? ImmutableList.copyOf(updatedRelationships) : new ArrayList<>(); + this.deletedRelationships = CollectionUtils.isNotEmpty(deletedRelationships) ? ImmutableList.copyOf(deletedRelationships) : new ArrayList<>(); + } + + public static RelationshipMutationContext getInstance(List createdRelationships, List updatedRelationships, List deletedRelationships) { + return new RelationshipMutationContext(createdRelationships, updatedRelationships, deletedRelationships); + } + + public List getCreatedRelationships() { + return createdRelationships; + } + + public List getUpdatedRelationships() { + return updatedRelationships; + } + + public List getDeletedRelationships() { + return deletedRelationships; + } +} diff --git a/intg/src/main/java/org/apache/atlas/model/lineage/AtlasLineageInfo.java b/intg/src/main/java/org/apache/atlas/model/lineage/AtlasLineageInfo.java index 27186ca7c2a..4a51f513b93 100644 --- a/intg/src/main/java/org/apache/atlas/model/lineage/AtlasLineageInfo.java +++ b/intg/src/main/java/org/apache/atlas/model/lineage/AtlasLineageInfo.java @@ -21,13 +21,13 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.databind.annotation.JsonSerialize; - import org.apache.atlas.model.instance.AtlasEntityHeader; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlRootElement; import java.io.Serializable; +import java.util.HashMap; import java.util.Map; import java.util.Objects; import java.util.Set; @@ -37,36 +37,79 @@ @JsonAutoDetect(getterVisibility = PUBLIC_ONLY, setterVisibility = PUBLIC_ONLY, fieldVisibility = NONE) @JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) -@JsonIgnoreProperties(ignoreUnknown = true) @XmlRootElement @XmlAccessorType(XmlAccessType.PROPERTY) public class AtlasLineageInfo implements Serializable { - private String baseEntityGuid; - private LineageDirection lineageDirection; - private int lineageDepth; + private String baseEntityGuid; + private LineageDirection lineageDirection; + private int lineageDepth; + private int limit; + private int offset; + private Long remainingUpstreamVertexCount; + private Long remainingDownstreamVertexCount; + private boolean hasMoreUpstreamVertices = false; + private boolean hasMoreDownstreamVertices = false; private Map guidEntityMap; - private Set relations; + private Set relations; + private final Map> vertexChildrenInfo = new HashMap<>(); - public AtlasLineageInfo() {} + public AtlasLineageInfo() { + } - public enum LineageDirection { INPUT, OUTPUT, BOTH } + public enum LineageDirection {INPUT, OUTPUT, BOTH} /** * Captures lineage information for an entity instance like hive_table - - * @param baseEntityGuid guid of the lineage entity . + * + * @param baseEntityGuid guid of the lineage entity . + * @param guidEntityMap map of entity guid to AtlasEntityHeader (minimal entity info) + * @param relations list of lineage relations for the entity (fromEntityId -> toEntityId) * @param lineageDirection direction of lineage, can be INPUT, OUTPUT or INPUT_AND_OUTPUT - * @param lineageDepth lineage depth to be fetched. - * @param guidEntityMap map of entity guid to AtlasEntityHeader (minimal entity info) - * @param relations list of lineage relations for the entity (fromEntityId -> toEntityId) + * @param lineageDepth lineage depth to be fetched. + * @param offset */ public AtlasLineageInfo(String baseEntityGuid, Map guidEntityMap, - Set relations, LineageDirection lineageDirection, int lineageDepth) { - this.baseEntityGuid = baseEntityGuid; + Set relations, LineageDirection lineageDirection, + int lineageDepth, int limit, int offset) { + this.baseEntityGuid = baseEntityGuid; this.lineageDirection = lineageDirection; - this.lineageDepth = lineageDepth; - this.guidEntityMap = guidEntityMap; - this.relations = relations; + this.lineageDepth = lineageDepth; + this.limit = limit; + this.guidEntityMap = guidEntityMap; + this.relations = relations; + this.offset = offset; + } + + public void calculateRemainingUpstreamVertexCount(Long totalUpstreamVertexCount) { + remainingUpstreamVertexCount = Math.max(totalUpstreamVertexCount - offset - limit, 0); + } + + public void calculateRemainingDownstreamVertexCount(Long totalUpstreamVertexCount) { + remainingDownstreamVertexCount = Math.max(totalUpstreamVertexCount - offset - limit, 0); + } + + public void setRemainingUpstreamVertexCount(long remainingUpstreamVertexCount) { + this.remainingUpstreamVertexCount = remainingUpstreamVertexCount; + } + + public void setRemainingDownstreamVertexCount(long remainingDownstreamVertexCount) { + this.remainingDownstreamVertexCount = remainingDownstreamVertexCount; + } + + public boolean getHasMoreUpstreamVertices() { + return hasMoreUpstreamVertices; + } + + public void setHasMoreUpstreamVertices(boolean hasMoreUpstreamVertices) { + this.hasMoreUpstreamVertices = hasMoreUpstreamVertices; + } + + public boolean getHasMoreDownstreamVertices() { + return hasMoreDownstreamVertices; + } + + public void setHasMoreDownstreamVertices(boolean hasMoreDownstreamVertices) { + this.hasMoreDownstreamVertices = hasMoreDownstreamVertices; } public String getBaseEntityGuid() { @@ -109,6 +152,44 @@ public void setLineageDepth(int lineageDepth) { this.lineageDepth = lineageDepth; } + public int getLimit() { + return limit; + } + + public void setLimit(int limit) { + this.limit = limit; + } + + public int getOffset() { + return offset; + } + + public void setOffset(int offset) { + this.offset = offset; + } + + public Long getRemainingUpstreamVertexCount() { + return remainingUpstreamVertexCount; + } + + public Long getRemainingDownstreamVertexCount() { + return remainingDownstreamVertexCount; + } + + public Map> getVertexChildrenInfo() { + return vertexChildrenInfo; + } + + public void setHasChildrenForDirection(String guid, LineageChildrenInfo lineageChildrenInfo) { + Map entityChildrenMap = vertexChildrenInfo.get(guid); + if (entityChildrenMap == null) { + entityChildrenMap = new HashMap<>(); + } + entityChildrenMap.put(lineageChildrenInfo.getDirection(), lineageChildrenInfo.getHasMoreChildren()); + + vertexChildrenInfo.put(guid, entityChildrenMap); + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -118,12 +199,13 @@ public boolean equals(Object o) { Objects.equals(baseEntityGuid, that.baseEntityGuid) && lineageDirection == that.lineageDirection && Objects.equals(guidEntityMap, that.guidEntityMap) && - Objects.equals(relations, that.relations); + Objects.equals(relations, that.relations) && + Objects.equals(limit, that.limit); } @Override public int hashCode() { - return Objects.hash(baseEntityGuid, lineageDirection, lineageDepth, guidEntityMap, relations); + return Objects.hash(baseEntityGuid, lineageDirection, lineageDepth, guidEntityMap, relations, limit); } @Override @@ -134,6 +216,7 @@ public String toString() { ", relations=" + relations + ", lineageDirection=" + lineageDirection + ", lineageDepth=" + lineageDepth + + ", limit=" + limit + '}'; } @@ -146,15 +229,22 @@ public static class LineageRelation { private String fromEntityId; private String toEntityId; private String relationshipId; + private String processId; - public LineageRelation() { } + public LineageRelation() { + } public LineageRelation(String fromEntityId, String toEntityId, final String relationshipId) { this.fromEntityId = fromEntityId; - this.toEntityId = toEntityId; + this.toEntityId = toEntityId; this.relationshipId = relationshipId; } + public LineageRelation(String fromEntityId, String toEntityId, final String relationshipId, String processId) { + this(fromEntityId, toEntityId, relationshipId); + this.processId = processId; + } + public String getFromEntityId() { return fromEntityId; } @@ -179,6 +269,14 @@ public void setRelationshipId(final String relationshipId) { this.relationshipId = relationshipId; } + public String getProcessId() { + return processId; + } + + public void setProcessId(String processId) { + this.processId = processId; + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -186,12 +284,13 @@ public boolean equals(Object o) { LineageRelation that = (LineageRelation) o; return Objects.equals(fromEntityId, that.fromEntityId) && Objects.equals(toEntityId, that.toEntityId) && - Objects.equals(relationshipId, that.relationshipId); + Objects.equals(relationshipId, that.relationshipId) && + Objects.equals(processId, that.processId); } @Override public int hashCode() { - return Objects.hash(fromEntityId, toEntityId, relationshipId); + return Objects.hash(fromEntityId, toEntityId, relationshipId, processId); } @Override @@ -200,8 +299,9 @@ public String toString() { "fromEntityId='" + fromEntityId + '\'' + ", toEntityId='" + toEntityId + '\'' + ", relationshipId='" + relationshipId + '\'' + + ", processId='" + processId + '\'' + '}'; } } -} \ No newline at end of file +} diff --git a/intg/src/main/java/org/apache/atlas/model/lineage/AtlasLineageListInfo.java b/intg/src/main/java/org/apache/atlas/model/lineage/AtlasLineageListInfo.java new file mode 100644 index 00000000000..aaa0db95f15 --- /dev/null +++ b/intg/src/main/java/org/apache/atlas/model/lineage/AtlasLineageListInfo.java @@ -0,0 +1,95 @@ +package org.apache.atlas.model.lineage; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import org.apache.atlas.model.instance.AtlasEntityHeader; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import java.io.Serializable; +import java.util.*; + +import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE; +import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ONLY; + +@JsonAutoDetect(getterVisibility = PUBLIC_ONLY, setterVisibility = PUBLIC_ONLY, fieldVisibility = NONE) +@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown = true, value = {"visitedEdges", "skippedEdges", "traversalQueue"}) +@XmlRootElement +@XmlAccessorType(XmlAccessType.PROPERTY) +public class AtlasLineageListInfo implements Serializable { + private List entities; + private boolean hasMore; + private int entityCount; + private LineageListRequest searchParameters; + + public AtlasLineageListInfo() {} + + /** + * Captures lineage list information for an entity instance like hive_table + * + * @param entities list of entities + */ + public AtlasLineageListInfo(List entities) { + this.entities = entities; + } + + public List getEntities() { + return entities; + } + + public void setEntities(List entities) { + this.entities = entities; + } + + + public LineageListRequest getSearchParameters() { + return searchParameters; + } + + public void setSearchParameters(LineageListRequest searchParameters) { + this.searchParameters = searchParameters; + } + + public boolean isHasMore() { + return hasMore; + } + + public void setHasMore(boolean hasMore) { + this.hasMore = hasMore; + } + + public int getEntityCount() { + return entityCount; + } + + public void setEntityCount(int entityCount) { + this.entityCount = entityCount; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + AtlasLineageListInfo that = (AtlasLineageListInfo) o; + return hasMore == that.hasMore && entityCount == that.entityCount && Objects.equals(entities, that.entities) && Objects.equals(searchParameters, that.searchParameters); + } + + @Override + public int hashCode() { + return Objects.hash(entities, hasMore, entityCount, searchParameters); + } + + @Override + public String toString() { + return "AtlasLineageListInfo{" + + "entities=" + entities + + ", hasMore=" + hasMore + + ", relationsCount=" + entityCount + + ", searchParameters=" + searchParameters + + '}'; + } + +} diff --git a/intg/src/main/java/org/apache/atlas/model/lineage/AtlasLineageOnDemandInfo.java b/intg/src/main/java/org/apache/atlas/model/lineage/AtlasLineageOnDemandInfo.java new file mode 100644 index 00000000000..1c026341d7e --- /dev/null +++ b/intg/src/main/java/org/apache/atlas/model/lineage/AtlasLineageOnDemandInfo.java @@ -0,0 +1,396 @@ +package org.apache.atlas.model.lineage; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import org.apache.atlas.model.instance.AtlasEntityHeader; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import java.io.Serializable; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE; +import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ONLY; + +@JsonAutoDetect(getterVisibility = PUBLIC_ONLY, setterVisibility = PUBLIC_ONLY, fieldVisibility = NONE) +@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown = true, value = {"visitedEdges", "skippedEdges"}) +@XmlRootElement +@XmlAccessorType(XmlAccessType.PROPERTY) +public class AtlasLineageOnDemandInfo implements Serializable { + private String baseEntityGuid; + private Map guidEntityMap; + private Set relations; + private Set visitedEdges; + private Set skippedEdges; + private Map relationsOnDemand; + private LineageOnDemandRequest lineageOnDemandPayload; + private boolean upstreamEntityLimitReached; + private boolean downstreamEntityLimitReached; + + public AtlasLineageOnDemandInfo() { + } + + public enum LineageDirection {INPUT, OUTPUT, BOTH} + + /** + * Captures lineage information for an entity instance like hive_table + * + * @param baseEntityGuid guid of the lineage entity . + * @param guidEntityMap map of entity guid to AtlasEntityHeader (minimal entity info) + * @param relations list of lineage relations for the entity (fromEntityId -> toEntityId) + */ + public AtlasLineageOnDemandInfo(String baseEntityGuid, Map guidEntityMap, + Set relations) { + this.baseEntityGuid = baseEntityGuid; + this.guidEntityMap = guidEntityMap; + this.relations = relations; + } + + public AtlasLineageOnDemandInfo(String baseEntityGuid, Map guidEntityMap, + Set relations, Set visitedEdges, Set skippedEdges, + Map relationsOnDemand) { + this.baseEntityGuid = baseEntityGuid; + this.guidEntityMap = guidEntityMap; + this.relations = relations; + this.visitedEdges = visitedEdges; + this.skippedEdges = skippedEdges; + this.relationsOnDemand = relationsOnDemand; + } + + public String getBaseEntityGuid() { + return baseEntityGuid; + } + + public void setBaseEntityGuid(String baseEntityGuid) { + this.baseEntityGuid = baseEntityGuid; + } + + public Map getGuidEntityMap() { + return guidEntityMap; + } + + public void setGuidEntityMap(Map guidEntityMap) { + this.guidEntityMap = guidEntityMap; + } + + public Set getRelations() { + return relations; + } + + public void setRelations(Set relations) { + this.relations = relations; + } + + public Set getVisitedEdges() { + return visitedEdges; + } + + public void setVisitedEdges(Set visitedEdges) { + this.visitedEdges = visitedEdges; + } + + public Set getSkippedEdges() { + return skippedEdges; + } + + public void setSkippedEdges(Set skippedEdges) { + this.skippedEdges = skippedEdges; + } + + public Map getRelationsOnDemand() { + return relationsOnDemand; + } + + public void setRelationsOnDemand(Map relationsOnDemand) { + this.relationsOnDemand = relationsOnDemand; + } + public LineageOnDemandRequest getLineageOnDemandPayload() { + return lineageOnDemandPayload; + } + + public void setLineageOnDemandPayload(LineageOnDemandRequest lineageOnDemandRequest) { + this.lineageOnDemandPayload = lineageOnDemandRequest; + } + + public boolean isUpstreamEntityLimitReached() { + return upstreamEntityLimitReached; + } + + public void setUpstreamEntityLimitReached(boolean upstreamEntityLimitReached) { + this.upstreamEntityLimitReached = upstreamEntityLimitReached; + } + + public boolean isDownstreamEntityLimitReached() { + return downstreamEntityLimitReached; + } + + public void setDownstreamEntityLimitReached(boolean downstreamEntityLimitReached) { + this.downstreamEntityLimitReached = downstreamEntityLimitReached; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + AtlasLineageOnDemandInfo that = (AtlasLineageOnDemandInfo) o; + return Objects.equals(baseEntityGuid, that.baseEntityGuid) && + Objects.equals(guidEntityMap, that.guidEntityMap) && + Objects.equals(relations, that.relations); + } + + @Override + public int hashCode() { + return Objects.hash(baseEntityGuid, guidEntityMap, relations); + } + + @Override + public String toString() { + return "AtlasLineageInfo{" + + "baseEntityGuid=" + baseEntityGuid + + ", guidEntityMap=" + guidEntityMap + + ", relations=" + relations + + '}'; + } + + @JsonAutoDetect(getterVisibility = PUBLIC_ONLY, setterVisibility = PUBLIC_ONLY, fieldVisibility = NONE) + @JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) + @JsonIgnoreProperties(ignoreUnknown = true, value = {"inputRelationsReachedLimit", "outputRelationsReachedLimit", "fromCounter"}) + @XmlRootElement + @XmlAccessorType(XmlAccessType.PROPERTY) + public static class LineageInfoOnDemand { + @JsonProperty + boolean hasMoreInputs; + @JsonProperty + boolean hasMoreOutputs; + int inputRelationsCount; + int outputRelationsCount; + boolean isInputRelationsReachedLimit; + boolean isOutputRelationsReachedLimit; + @JsonProperty + boolean hasUpstream; + @JsonProperty + boolean hasDownstream; + LineageOnDemandConstraints onDemandConstraints; + int fromCounter; // Counter for relations to be skipped + + public LineageInfoOnDemand() { } + + public LineageInfoOnDemand(LineageOnDemandConstraints onDemandConstraints) { + this.onDemandConstraints = onDemandConstraints; + this.hasMoreInputs = false; + this.hasMoreOutputs = false; + this.inputRelationsCount = 0; + this.outputRelationsCount = 0; + this.isInputRelationsReachedLimit = false; + this.isOutputRelationsReachedLimit = false; + this.hasUpstream = false; + this.hasDownstream = false; + this.fromCounter = 0; + } + + public boolean isInputRelationsReachedLimit() { + return isInputRelationsReachedLimit; + } + + public void setInputRelationsReachedLimit(boolean isInputRelationsReachedLimit) { + this.isInputRelationsReachedLimit = isInputRelationsReachedLimit; + } + + public boolean isOutputRelationsReachedLimit() { + return isOutputRelationsReachedLimit; + } + + public void setOutputRelationsReachedLimit(boolean isOutputRelationsReachedLimit) { + this.isOutputRelationsReachedLimit = isOutputRelationsReachedLimit; + } + + public boolean hasMoreInputs() { + return hasMoreInputs; + } + + public void setHasMoreInputs(boolean hasMoreInputs) { + this.hasMoreInputs = hasMoreInputs; + } + + public boolean hasMoreOutputs() { + return hasMoreOutputs; + } + + public void setHasMoreOutputs(boolean hasMoreOutputs) { + this.hasMoreOutputs = hasMoreOutputs; + } + + public boolean hasUpstream() { + return hasUpstream; + } + + public void setHasUpstream(boolean hasUpstream) { + this.hasUpstream = hasUpstream; + } + + public boolean hasDownstream() { + return hasDownstream; + } + + public void setHasDownstream(boolean hasDownstream) { + this.hasDownstream = hasDownstream; + } + + public int getFromCounter() { + return fromCounter; + } + + public void incrementFromCounter() { + fromCounter++; + } + + public int getInputRelationsCount() { + return inputRelationsCount; + } + + public void incrementInputRelationsCount() { + if (hasMoreInputs) { + return; + } + + if (isInputRelationsReachedLimit) { + setHasMoreInputs(true); + return; + } + + this.inputRelationsCount++; + + if (inputRelationsCount == onDemandConstraints.getInputRelationsLimit()) { + this.setInputRelationsReachedLimit(true); + return; + } + } + + public int getOutputRelationsCount() { + return outputRelationsCount; + } + + public void incrementOutputRelationsCount() { + if (hasMoreOutputs) { + return; + } + + if (isOutputRelationsReachedLimit) { + setHasMoreOutputs(true); + return; + } + + this.outputRelationsCount++; + + if (outputRelationsCount == onDemandConstraints.getOutputRelationsLimit()) { + this.setOutputRelationsReachedLimit(true); + return; + } + } + + @Override + public String toString() { + return "LineageInfoOnDemand{" + + "hasMoreInputs='" + hasMoreInputs + '\'' + + ", hasMoreOutputs='" + hasMoreOutputs + '\'' + + ", inputRelationsCount='" + inputRelationsCount + '\'' + + ", outputRelationsCount='" + outputRelationsCount + '\'' + + ", hasUpstream='" + hasUpstream + '\'' + + ", hasDownstream='" + hasDownstream + '\'' + + '}'; + } + + } + + @JsonAutoDetect(getterVisibility = PUBLIC_ONLY, setterVisibility = PUBLIC_ONLY, fieldVisibility = NONE) + @JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) + @JsonIgnoreProperties(ignoreUnknown = true) + @XmlRootElement + @XmlAccessorType(XmlAccessType.PROPERTY) + public static class LineageRelation { + private String fromEntityId; + private String toEntityId; + private String relationshipId; + private String processId; + + public LineageRelation() { + } + + public LineageRelation(String fromEntityId, String toEntityId, final String relationshipId) { + this.fromEntityId = fromEntityId; + this.toEntityId = toEntityId; + this.relationshipId = relationshipId; + } + + public LineageRelation(String fromEntityId, String toEntityId, final String relationshipId, String processId) { + this(fromEntityId, toEntityId, relationshipId); + this.processId = processId; + } + + public String getFromEntityId() { + return fromEntityId; + } + + public void setFromEntityId(String fromEntityId) { + this.fromEntityId = fromEntityId; + } + + public String getToEntityId() { + return toEntityId; + } + + public void setToEntityId(String toEntityId) { + this.toEntityId = toEntityId; + } + + public String getRelationshipId() { + return relationshipId; + } + + public void setRelationshipId(final String relationshipId) { + this.relationshipId = relationshipId; + } + + public String getProcessId() { + return processId; + } + + public void setProcessId(String processId) { + this.processId = processId; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + LineageRelation that = (LineageRelation) o; + return Objects.equals(fromEntityId, that.fromEntityId) && + Objects.equals(toEntityId, that.toEntityId) && + Objects.equals(relationshipId, that.relationshipId) && + Objects.equals(processId, that.processId); + } + + @Override + public int hashCode() { + return Objects.hash(fromEntityId, toEntityId, relationshipId, processId); + } + + @Override + public String toString() { + return "LineageRelation{" + + "fromEntityId='" + fromEntityId + '\'' + + ", toEntityId='" + toEntityId + '\'' + + ", relationshipId='" + relationshipId + '\'' + + ", processId='" + processId + '\'' + + '}'; + } + } + +} diff --git a/intg/src/main/java/org/apache/atlas/model/lineage/AtlasLineageRequest.java b/intg/src/main/java/org/apache/atlas/model/lineage/AtlasLineageRequest.java new file mode 100644 index 00000000000..dd9791c36ed --- /dev/null +++ b/intg/src/main/java/org/apache/atlas/model/lineage/AtlasLineageRequest.java @@ -0,0 +1,177 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.model.lineage; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import org.apache.atlas.AtlasErrorCode; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.discovery.SearchParameters; +import org.apache.atlas.model.lineage.AtlasLineageInfo.LineageDirection; +import org.apache.commons.lang.StringUtils; + +import java.util.HashSet; +import java.util.Set; + +import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE; +import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ONLY; +import static org.apache.atlas.AtlasErrorCode.BAD_REQUEST; +import static org.apache.atlas.model.lineage.AtlasLineageInfo.LineageDirection.BOTH; + + +@JsonAutoDetect(getterVisibility = PUBLIC_ONLY, setterVisibility = PUBLIC_ONLY, fieldVisibility = NONE) +@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) +public class AtlasLineageRequest { + private String guid; + private int depth; + private int offset = -1; + private int limit = -1; + private boolean calculateRemainingVertexCounts; + private boolean hideProcess; + private boolean allowDeletedProcess; + private LineageDirection direction = BOTH; + private SearchParameters.FilterCriteria entityFilters; + + private Set attributes; + private Set ignoredProcesses; + private Set relationAttributes; + + public AtlasLineageRequest() { + } + + public AtlasLineageRequest(String guid, int depth, LineageDirection direction, boolean hideProcess, int offset, int limit, boolean calculateRemainingVertexCounts) throws AtlasBaseException { + this.guid = guid; + this.depth = depth; + this.direction = direction; + this.hideProcess = hideProcess; + this.offset = offset; + this.limit = limit; + this.calculateRemainingVertexCounts = calculateRemainingVertexCounts; + this.attributes = new HashSet<>(); + this.ignoredProcesses = new HashSet<>(); + performValidation(); + } + + public void performValidation() throws AtlasBaseException { + if (StringUtils.isEmpty(guid)) { + throw new AtlasBaseException(BAD_REQUEST, "guid is not specified"); + } else if ((offset != -1 && limit == -1) || (offset == -1 && limit != -1)) { + throw new AtlasBaseException(AtlasErrorCode.INVALID_PAGINATION_STATE); + } else if (depth != 1 && offset != -1) { + throw new AtlasBaseException(AtlasErrorCode.PAGINATION_CAN_ONLY_BE_USED_WITH_DEPTH_ONE); + } else if (offset == -1 && calculateRemainingVertexCounts) { + throw new AtlasBaseException(AtlasErrorCode.CANT_CALCULATE_VERTEX_COUNTS_WITHOUT_PAGINATION); + } + } + + public String getGuid() { + return guid; + } + + public void setGuid(String guid) { + this.guid = guid; + } + + public int getDepth() { + return depth; + } + + public void setDepth(int depth) { + this.depth = depth; + } + + public int getLimit() { + return limit; + } + + public void setLimit(int limit) { + this.limit = limit; + } + + public LineageDirection getDirection() { + return direction; + } + + public void setDirection(LineageDirection direction) { + this.direction = direction; + } + + public boolean isHideProcess() { + return hideProcess; + } + + public void setHideProcess(boolean hideProcess) { + this.hideProcess = hideProcess; + } + + public Set getAttributes() { + return attributes; + } + + public void setAttributes(Set attributes) { + this.attributes = attributes; + } + + public Set getIgnoredProcesses() { + return ignoredProcesses; + } + + public void setIgnoredProcesses(Set ignoredProcesses) { + this.ignoredProcesses = ignoredProcesses; + } + + public SearchParameters.FilterCriteria getEntityFilters() { + return entityFilters; + } + + public void setEntityFilters(SearchParameters.FilterCriteria entityFilters) { + this.entityFilters = entityFilters; + } + + public boolean isAllowDeletedProcess() { + return allowDeletedProcess; + } + + public void setAllowDeletedProcess(boolean allowDeletedProcess) { + this.allowDeletedProcess = allowDeletedProcess; + } + + public int getOffset() { + return offset; + } + + public void setOffset(int offset) { + this.offset = offset; + } + + public boolean getCalculateRemainingVertexCounts() { + return calculateRemainingVertexCounts; + } + + public void setCalculateRemainingVertexCounts(boolean calculateRemainingVertexCounts) { + this.calculateRemainingVertexCounts = calculateRemainingVertexCounts; + } + + public Set getRelationAttributes() { + return relationAttributes; + } + + public void setRelationAttributes(Set relationAttributes) { + this.relationAttributes = relationAttributes; + } +} diff --git a/intg/src/main/java/org/apache/atlas/model/lineage/LineageChildrenInfo.java b/intg/src/main/java/org/apache/atlas/model/lineage/LineageChildrenInfo.java new file mode 100644 index 00000000000..0efd939797b --- /dev/null +++ b/intg/src/main/java/org/apache/atlas/model/lineage/LineageChildrenInfo.java @@ -0,0 +1,20 @@ +package org.apache.atlas.model.lineage; + +public class LineageChildrenInfo { + + private final AtlasLineageInfo.LineageDirection direction; + private final boolean hasMoreChildren; + + public LineageChildrenInfo(AtlasLineageInfo.LineageDirection direction, boolean hasMoreChildren) { + this.direction = direction; + this.hasMoreChildren = hasMoreChildren; + } + + public AtlasLineageInfo.LineageDirection getDirection() { + return direction; + } + + public boolean getHasMoreChildren() { + return hasMoreChildren; + } +} diff --git a/intg/src/main/java/org/apache/atlas/model/lineage/LineageListRequest.java b/intg/src/main/java/org/apache/atlas/model/lineage/LineageListRequest.java new file mode 100644 index 00000000000..66ff5f71d7b --- /dev/null +++ b/intg/src/main/java/org/apache/atlas/model/lineage/LineageListRequest.java @@ -0,0 +1,137 @@ +package org.apache.atlas.model.lineage; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import org.apache.atlas.model.discovery.SearchParameters; + +import java.util.HashSet; +import java.util.Set; + +import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE; +import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ONLY; + +@JsonAutoDetect(getterVisibility = PUBLIC_ONLY, setterVisibility = PUBLIC_ONLY, fieldVisibility = NONE) +@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) +public class LineageListRequest { + private String guid; + private Integer size; + private Integer from; + private Integer depth; + private LineageDirection direction; + private SearchParameters.FilterCriteria entityFilters; + private SearchParameters.FilterCriteria entityTraversalFilters; + private SearchParameters.FilterCriteria relationshipTraversalFilters; + private Set attributes; + private Boolean excludeMeanings; + private Boolean excludeClassifications; + + public enum LineageDirection {INPUT, OUTPUT} + + public LineageListRequest() { + this.attributes = new HashSet<>(); + } + + public LineageListRequest(String guid, Integer size, Integer from, Integer depth, LineageDirection direction, SearchParameters.FilterCriteria entityFilters, + SearchParameters.FilterCriteria entityTraversalFilters, SearchParameters.FilterCriteria relationshipTraversalFilters, + Set attributes, boolean excludeMeanings, boolean excludeClassifications) { + this.guid = guid; + this.size = size; + this.from = from; + this.depth = depth; + this.direction = direction; + this.entityFilters = entityFilters; + this.entityTraversalFilters = entityTraversalFilters; + this.relationshipTraversalFilters = relationshipTraversalFilters; + this.attributes = attributes; + this.excludeMeanings = excludeMeanings; + this.excludeClassifications = excludeClassifications; + } + + public String getGuid() { + return guid; + } + + public void setGuid(String guid) { + this.guid = guid; + } + + public Integer getSize() { + return size; + } + + public void setSize(Integer size) { + this.size = size; + } + + public Integer getFrom() { + return from; + } + + public void setFrom(Integer from) { + this.from = from; + } + + public Integer getDepth() { + return depth; + } + + public void setDepth(Integer depth) { + this.depth = depth; + } + + public LineageDirection getDirection() { + return direction; + } + + public void setDirection(LineageDirection direction) { + this.direction = direction; + } + + public SearchParameters.FilterCriteria getEntityFilters() { + return entityFilters; + } + + public void setEntityFilters(SearchParameters.FilterCriteria entityFilters) { + this.entityFilters = entityFilters; + } + + public SearchParameters.FilterCriteria getEntityTraversalFilters() { + return entityTraversalFilters; + } + + public void setEntityTraversalFilters(SearchParameters.FilterCriteria entityTraversalFilters) { + this.entityTraversalFilters = entityTraversalFilters; + } + + public SearchParameters.FilterCriteria getRelationshipTraversalFilters() { + return relationshipTraversalFilters; + } + + public void setRelationshipTraversalFilters(SearchParameters.FilterCriteria relationshipTraversalFilters) { + this.relationshipTraversalFilters = relationshipTraversalFilters; + } + + public Set getAttributes() { + return attributes; + } + + public void setAttributes(Set attributes) { + this.attributes = attributes; + } + + public Boolean isExcludeMeanings() { + return excludeMeanings; + } + + public void setExcludeMeanings(Boolean excludeMeanings) { + this.excludeMeanings = excludeMeanings; + } + + public Boolean isExcludeClassifications() { + return excludeClassifications; + } + + public void setExcludeClassifications(Boolean excludeClassifications) { + this.excludeClassifications = excludeClassifications; + } +} diff --git a/intg/src/main/java/org/apache/atlas/model/lineage/LineageOnDemandBaseParams.java b/intg/src/main/java/org/apache/atlas/model/lineage/LineageOnDemandBaseParams.java new file mode 100644 index 00000000000..524d76309f8 --- /dev/null +++ b/intg/src/main/java/org/apache/atlas/model/lineage/LineageOnDemandBaseParams.java @@ -0,0 +1,45 @@ +package org.apache.atlas.model.lineage; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import org.apache.atlas.AtlasConfiguration; + +import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE; +import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ONLY; + +@JsonAutoDetect(getterVisibility = PUBLIC_ONLY, setterVisibility = PUBLIC_ONLY, fieldVisibility = NONE) +@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown = true) +public class LineageOnDemandBaseParams { + private int inputRelationsLimit; + private int outputRelationsLimit; + + public static final int LINEAGE_ON_DEMAND_DEFAULT_NODE_COUNT = AtlasConfiguration.LINEAGE_ON_DEMAND_DEFAULT_NODE_COUNT.getInt(); + + public LineageOnDemandBaseParams() { + this.inputRelationsLimit = LINEAGE_ON_DEMAND_DEFAULT_NODE_COUNT; + this.outputRelationsLimit = LINEAGE_ON_DEMAND_DEFAULT_NODE_COUNT; + } + + public LineageOnDemandBaseParams(int inputRelationsLimit, int outputRelationsLimit) { + this.inputRelationsLimit = inputRelationsLimit; + this.outputRelationsLimit = outputRelationsLimit; + } + + public int getInputRelationsLimit() { + return inputRelationsLimit; + } + + public void setInputRelationsLimit(int inputRelationsLimit) { + this.inputRelationsLimit = inputRelationsLimit; + } + + public int getOutputRelationsLimit() { + return outputRelationsLimit; + } + + public void setOutputRelationsLimit(int outputRelationsLimit) { + this.outputRelationsLimit = outputRelationsLimit; + } +} \ No newline at end of file diff --git a/intg/src/main/java/org/apache/atlas/model/lineage/LineageOnDemandConstraints.java b/intg/src/main/java/org/apache/atlas/model/lineage/LineageOnDemandConstraints.java new file mode 100644 index 00000000000..4fe87613820 --- /dev/null +++ b/intg/src/main/java/org/apache/atlas/model/lineage/LineageOnDemandConstraints.java @@ -0,0 +1,95 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.model.lineage; + + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import org.apache.atlas.AtlasConfiguration; + +import java.io.Serializable; + +import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE; +import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ONLY; + +@JsonAutoDetect(getterVisibility = PUBLIC_ONLY, setterVisibility = PUBLIC_ONLY, fieldVisibility = NONE) +@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown = true) +/** + * This is the root class representing the input for lineage search on-demand. + */ +public class LineageOnDemandConstraints extends LineageOnDemandBaseParams implements Serializable { + private static final long serialVersionUID = 1L; + + private AtlasLineageOnDemandInfo.LineageDirection direction; + private int depth; + private int from; + + private static final int LINEAGE_ON_DEMAND_DEFAULT_NODE_COUNT = AtlasConfiguration.LINEAGE_ON_DEMAND_DEFAULT_NODE_COUNT.getInt(); + private static final int LINEAGE_ON_DEMAND_DEFAULT_DEPTH = 3; + + public LineageOnDemandConstraints() { + this(AtlasLineageOnDemandInfo.LineageDirection.BOTH, -1, -1, LINEAGE_ON_DEMAND_DEFAULT_DEPTH); + } + + public LineageOnDemandConstraints(LineageOnDemandBaseParams baseParams) { + this(AtlasLineageOnDemandInfo.LineageDirection.BOTH, baseParams.getInputRelationsLimit(), baseParams.getOutputRelationsLimit(), LINEAGE_ON_DEMAND_DEFAULT_DEPTH); + } + + public LineageOnDemandConstraints(AtlasLineageOnDemandInfo.LineageDirection direction, LineageOnDemandBaseParams baseParams, int depth) { + this(direction, baseParams.getInputRelationsLimit(), baseParams.getOutputRelationsLimit(), depth); + } + + public LineageOnDemandConstraints(AtlasLineageOnDemandInfo.LineageDirection direction, int inputRelationsLimit, int outputRelationsLimit, int depth) { + super(inputRelationsLimit, outputRelationsLimit); + this.direction = direction; + this.depth = depth; + } + + public LineageOnDemandConstraints(AtlasLineageOnDemandInfo.LineageDirection direction, int inputRelationsLimit, int outputRelationsLimit, int depth, int from) { + super(inputRelationsLimit, outputRelationsLimit); + this.direction = direction; + this.depth = depth; + this.from = from; + } + + public AtlasLineageOnDemandInfo.LineageDirection getDirection() { + return direction; + } + + public void setDirection(AtlasLineageOnDemandInfo.LineageDirection direction) { + this.direction = direction; + } + + public int getDepth() { + return depth; + } + + public void setDepth(int depth) { + this.depth = depth; + } + + public int getFrom() { + return from; + } + + public void setFrom(int from) { + this.from = from; + } +} \ No newline at end of file diff --git a/intg/src/main/java/org/apache/atlas/model/lineage/LineageOnDemandRequest.java b/intg/src/main/java/org/apache/atlas/model/lineage/LineageOnDemandRequest.java new file mode 100644 index 00000000000..7fa953373a6 --- /dev/null +++ b/intg/src/main/java/org/apache/atlas/model/lineage/LineageOnDemandRequest.java @@ -0,0 +1,90 @@ +package org.apache.atlas.model.lineage; + +import com.fasterxml.jackson.annotation.JsonAlias; +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import org.apache.atlas.model.discovery.SearchParameters; + +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE; +import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ONLY; + +@JsonAutoDetect(getterVisibility = PUBLIC_ONLY, setterVisibility = PUBLIC_ONLY, fieldVisibility = NONE) +@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) +public class LineageOnDemandRequest { + private Map constraints; + @JsonAlias({"traversalFilters", "entityTraversalFilters"}) // TODO: Will be deprecated after FE changes. + private SearchParameters.FilterCriteria entityTraversalFilters; + private SearchParameters.FilterCriteria relationshipTraversalFilters; + private Set attributes; + private Set relationAttributes; + private LineageOnDemandBaseParams defaultParams; + + public LineageOnDemandRequest() { + this.attributes = new HashSet<>(); + this.relationAttributes = new HashSet<>(); + this.defaultParams = new LineageOnDemandBaseParams(); + } + + public LineageOnDemandRequest(Map constraints, SearchParameters.FilterCriteria entityTraversalFilters, + SearchParameters.FilterCriteria relationshipTraversalFilters, Set attributes, + Set relationAttributes, LineageOnDemandBaseParams defaultParams) { + this.constraints = constraints; + this.entityTraversalFilters = entityTraversalFilters; + this.relationshipTraversalFilters = relationshipTraversalFilters; + this.attributes = attributes; + this.relationAttributes = relationAttributes; + this.defaultParams = defaultParams; + } + + public Map getConstraints() { + return constraints; + } + + public void setConstraints(Map constraints) { + this.constraints = constraints; + } + + public SearchParameters.FilterCriteria getEntityTraversalFilters() { + return entityTraversalFilters; + } + + public void setEntityTraversalFilters(SearchParameters.FilterCriteria entityTraversalFilters) { + this.entityTraversalFilters = entityTraversalFilters; + } + + public SearchParameters.FilterCriteria getRelationshipTraversalFilters() { + return relationshipTraversalFilters; + } + + public void setRelationshipTraversalFilters(SearchParameters.FilterCriteria relationshipTraversalFilters) { + this.relationshipTraversalFilters = relationshipTraversalFilters; + } + + public Set getAttributes() { + return attributes; + } + + public void setAttributes(Set attributes) { + this.attributes = attributes; + } + + public Set getRelationAttributes() { + return relationAttributes; + } + + public void setRelationAttributes(Set relationAttributes) { + this.relationAttributes = relationAttributes; + } + + public LineageOnDemandBaseParams getDefaultParams() { + return defaultParams; + } + + public void setDefaultParams(LineageOnDemandBaseParams defaultParams) { + this.defaultParams = defaultParams; + } +} diff --git a/intg/src/main/java/org/apache/atlas/model/notification/EntityNotification.java b/intg/src/main/java/org/apache/atlas/model/notification/EntityNotification.java index 90e131ac684..a6c111f7531 100644 --- a/intg/src/main/java/org/apache/atlas/model/notification/EntityNotification.java +++ b/intg/src/main/java/org/apache/atlas/model/notification/EntityNotification.java @@ -28,7 +28,11 @@ import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlRootElement; import java.io.Serializable; +<<<<<<< HEAD import java.util.List; +======= +import java.util.Map; +>>>>>>> 7f3c811fb15361ba82bfb4edd7a8393798863ff0 import java.util.Objects; import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE; @@ -102,15 +106,17 @@ public static class EntityNotificationV2 extends EntityNotification implements S private static final long serialVersionUID = 1L; public enum OperationType { - ENTITY_CREATE, ENTITY_UPDATE, ENTITY_DELETE, - CLASSIFICATION_ADD, CLASSIFICATION_DELETE, CLASSIFICATION_UPDATE, - RELATIONSHIP_CREATE, RELATIONSHIP_UPDATE, RELATIONSHIP_DELETE + ENTITY_CREATE, ENTITY_UPDATE, ENTITY_DELETE, CLASSIFICATION_ADD, + CLASSIFICATION_DELETE, CLASSIFICATION_UPDATE, RELATIONSHIP_CREATE, + RELATIONSHIP_UPDATE, RELATIONSHIP_DELETE, BUSINESS_ATTRIBUTE_UPDATE } - private AtlasEntityHeader entity; + private AtlasEntityHeader entity; private AtlasRelationshipHeader relationship; - private OperationType operationType; + private OperationType operationType; private long eventTime; + private Object mutatedDetails; + private Map headers; private List classifications; @@ -134,22 +140,37 @@ public EntityNotificationV2(AtlasEntityHeader entity, OperationType operationTyp setEventTime(eventTime); } +<<<<<<< HEAD public EntityNotificationV2(AtlasEntityHeader entity,List classifications, OperationType operationType, long eventTime) { +======= + public EntityNotificationV2(AtlasEntityHeader entity, Object mutatedDetails, OperationType operationType, + long eventTime, Map requestContextHeaders) { +>>>>>>> 7f3c811fb15361ba82bfb4edd7a8393798863ff0 super(ENTITY_NOTIFICATION_V2); setEntity(entity); setOperationType(operationType); setEventTime(eventTime); +<<<<<<< HEAD setClassifications(classifications); } public EntityNotificationV2(AtlasRelationshipHeader relationship, OperationType operationType, long eventTime) { +======= + setMutatedDetails(mutatedDetails); + setHeaders(requestContextHeaders); + } + + public EntityNotificationV2(AtlasRelationshipHeader relationship, OperationType operationType, + long eventTime, Map requestContextHeaders) { +>>>>>>> 7f3c811fb15361ba82bfb4edd7a8393798863ff0 super(ENTITY_NOTIFICATION_V2); setRelationship(relationship); setOperationType(operationType); setEventTime(eventTime); + setHeaders(requestContextHeaders); } public AtlasEntityHeader getEntity() { @@ -160,6 +181,15 @@ public void setEntity(AtlasEntityHeader entity) { this.entity = entity; } + public void setMutatedDetails(Object mutatedDetails){ + this.mutatedDetails = mutatedDetails; + } + + public Object getMutatedDetails(){ + return mutatedDetails; + } + + public AtlasRelationshipHeader getRelationship() { return relationship; } @@ -184,12 +214,21 @@ public void setEventTime(long eventTime) { this.eventTime = eventTime; } +<<<<<<< HEAD public List getClassifications() { return classifications; } public void setClassifications(List classifications) { this.classifications = classifications; +======= + public Map getHeaders() { + return headers; + } + + public void setHeaders(Map headers) { + this.headers = headers; +>>>>>>> 7f3c811fb15361ba82bfb4edd7a8393798863ff0 } @Override @@ -199,12 +238,13 @@ public boolean equals(Object o) { EntityNotificationV2 that = (EntityNotificationV2) o; return Objects.equals(type, that.type) && Objects.equals(entity, that.entity) && - operationType == that.operationType; + operationType == that.operationType && + headers == that.headers; } @Override public int hashCode() { - return Objects.hash(type, entity, operationType); + return Objects.hash(type, entity, operationType, headers); } @Override @@ -223,6 +263,7 @@ public StringBuilder toString(StringBuilder sb) { } sb.append(", operationType=").append(operationType); sb.append(", eventTime=").append(eventTime); + sb.append(", headers=").append(headers); sb.append("}"); return sb; diff --git a/intg/src/main/java/org/apache/atlas/model/searchlog/SearchLogSearchParams.java b/intg/src/main/java/org/apache/atlas/model/searchlog/SearchLogSearchParams.java new file mode 100644 index 00000000000..285352b5037 --- /dev/null +++ b/intg/src/main/java/org/apache/atlas/model/searchlog/SearchLogSearchParams.java @@ -0,0 +1,61 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.model.searchlog; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import org.apache.atlas.type.AtlasType; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + + +@JsonInclude(JsonInclude.Include.NON_EMPTY) +@JsonIgnoreProperties(ignoreUnknown=true) +public class SearchLogSearchParams { + + private static Map defaultSort = new HashMap<>(); + static { + Map order = new HashMap<>(); + order.put("order", "desc"); + + defaultSort.put("created", order); + } + + private Map dsl; + + public void setDsl(Map dsl) { + this.dsl = dsl; + } + + public Map getDsl() { + return this.dsl; + } + + public String getQueryString() { + if (this.dsl != null) { + if (!this.dsl.containsKey("sort")) { + dsl.put("sort", Collections.singleton(defaultSort)); + } + return AtlasType.toJson(dsl); + } + return ""; + } +} \ No newline at end of file diff --git a/intg/src/main/java/org/apache/atlas/model/searchlog/SearchLogSearchResult.java b/intg/src/main/java/org/apache/atlas/model/searchlog/SearchLogSearchResult.java new file mode 100644 index 00000000000..d91768fa232 --- /dev/null +++ b/intg/src/main/java/org/apache/atlas/model/searchlog/SearchLogSearchResult.java @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.atlas.model.searchlog; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; + +import java.util.List; +import java.util.Map; + + +@JsonInclude(JsonInclude.Include.NON_EMPTY) +@JsonIgnoreProperties(ignoreUnknown=true) +public class SearchLogSearchResult { + + private SearchLogSearchParams searchParameters; + private List> logs; + private Map aggregations; + private long approximateCount; + + public SearchLogSearchParams getSearchParameters() { + return searchParameters; + } + + public void setSearchParameters(SearchLogSearchParams searchParameters) { + this.searchParameters = searchParameters; + } + + public List> getLogs() { + return logs; + } + + public void setLogs(List> logs) { + this.logs = logs; + } + + public Map getAggregations() { + return aggregations; + } + + public void setAggregations(Map aggregations) { + this.aggregations = aggregations; + } + + public long getApproximateCount() { + return approximateCount; + } + + public void setApproximateCount(long approximateCount) { + this.approximateCount = approximateCount; + } +} \ No newline at end of file diff --git a/intg/src/main/java/org/apache/atlas/model/searchlog/SearchRequestLogData.java b/intg/src/main/java/org/apache/atlas/model/searchlog/SearchRequestLogData.java new file mode 100644 index 00000000000..068b093bae2 --- /dev/null +++ b/intg/src/main/java/org/apache/atlas/model/searchlog/SearchRequestLogData.java @@ -0,0 +1,409 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.model.searchlog; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.atlas.type.AtlasType; + +import java.util.Map; +import java.util.Set; + +@JsonInclude(JsonInclude.Include.NON_EMPTY) +public class SearchRequestLogData { + + private Map dsl; + private String dslText; + private Set attributes; + private Set relationAttributes; + + private String persona; + private String purpose; + + private String userAgent; + private String host; + private String ipAddress; + private String userName; + private String errorDetails; + private String errorCode; + + private Set entityGuidsAll; + private Set entityQFNamesAll; + private Set entityGuidsAllowed; + private Set entityQFNamesAllowed; + private Set entityGuidsDenied; + private Set entityQFNamesDenied; + private Set entityTypeNamesAll; + private Set entityTypeNamesAllowed; + private Set entityTypeNamesDenied; + private Set utmTags; + private String searchInput; + + private boolean hasResult; + private boolean isFailed; + + private long resultsCount; + private long responseTime; + private long createdAt; + private long timestamp; + + public SearchRequestLogData(Map dsl, Set attributes, Set relationAttributes, + String searchInput, String persona, String purpose, + String userAgent, String host, String ipAddress, String userName, + String errorDetails, String errorCode, + Set entityGuidsAll, Set entityQFNamesAll, Set entityGuidsAllowed, + Set entityQFNamesAllowed, Set entityGuidsDenied, Set entityQFNamesDenied, + Set entityTypeNamesAll, Set entityTypeNamesAllowed, Set entityTypeNamesDenied, + Set utmTags, boolean hasResult, boolean isFailed, + long resultsCount, long responseTime, long timestamp) { + this.dsl = dsl; + this.dslText = AtlasType.toJson(dsl); + this.attributes = attributes; + this.relationAttributes = relationAttributes; + this.persona = persona; + this.purpose = purpose; + this.userAgent = userAgent; + this.host = host; + this.ipAddress = ipAddress; + this.userName = userName; + this.errorDetails = errorDetails; + this.errorCode = errorCode; + this.entityGuidsAll = entityGuidsAll; + this.entityQFNamesAll = entityQFNamesAll; + this.entityGuidsAllowed = entityGuidsAllowed; + this.entityQFNamesAllowed = entityQFNamesAllowed; + this.entityGuidsDenied = entityGuidsDenied; + this.entityQFNamesDenied = entityQFNamesDenied; + this.entityTypeNamesAll = entityTypeNamesAll; + this.entityTypeNamesAllowed = entityTypeNamesAllowed; + this.entityTypeNamesDenied = entityTypeNamesDenied; + this.utmTags = utmTags; + this.searchInput = searchInput; + this.hasResult = hasResult; + this.isFailed = isFailed; + this.resultsCount = resultsCount; + this.responseTime = responseTime; + this.timestamp = timestamp; + } + + @JsonProperty("request.dsl") + public Map getDsl() { + return dsl; + } + + @JsonProperty("request.dslText") + public String getDslText() { + return dslText; + } + + @JsonProperty("request.attributes") + public Set getAttributes() { + return attributes; + } + + @JsonProperty("request.relationAttributes") + public Set getRelationAttributes() { + return relationAttributes; + } + + public String getPersona() { + return persona; + } + + public String getPurpose() { + return purpose; + } + + public String getUserAgent() { + return userAgent; + } + + public String getHost() { + return host; + } + + public String getIpAddress() { + return ipAddress; + } + + public String getUserName() { + return userName; + } + + public String getErrorDetails() { + return errorDetails; + } + + public String getErrorCode() { + return errorCode; + } + + public Set getEntityGuidsAll() { + return entityGuidsAll; + } + + public Set getEntityQFNamesAll() { + return entityQFNamesAll; + } + + public Set getEntityGuidsAllowed() { + return entityGuidsAllowed; + } + + public Set getEntityQFNamesAllowed() { + return entityQFNamesAllowed; + } + + public Set getEntityGuidsDenied() { + return entityGuidsDenied; + } + + public Set getEntityQFNamesDenied() { + return entityQFNamesDenied; + } + + public Set getEntityTypeNamesAll() { + return entityTypeNamesAll; + } + + public Set getEntityTypeNamesAllowed() { + return entityTypeNamesAllowed; + } + + public Set getEntityTypeNamesDenied() { + return entityTypeNamesDenied; + } + + public Set getUtmTags() { + return utmTags; + } + + public boolean isHasResult() { + return hasResult; + } + + public boolean isFailed() { + return isFailed; + } + + public long getResultsCount() { + return resultsCount; + } + + public long getResponseTime() { + return responseTime; + } + + public long getTimestamp() { + return timestamp; + } + + public long getCreatedAt() { + return createdAt; + } + + public void setCreatedAt(long createdAt) { + this.createdAt = createdAt; + } + + public String getSearchInput() { + return searchInput; + } + + public static class SearchRequestLogDataBuilder { + private Map dsl; + private Set attributes; + private Set relationAttributes; + + private String persona; + private String purpose; + + private String userAgent; + private String host; + private String ipAddress; + private String userName; + private String errorDetails; + private String errorCode; + + private Set entityGuidsAll; + private Set entityQFNamesAll; + private Set entityGuidsAllowed; + private Set entityQFNamesAllowed; + private Set entityGuidsDenied; + private Set entityQFNamesDenied; + private Set entityTypeNamesAll; + private Set entityTypeNamesAllowed; + private Set entityTypeNamesDenied; + private Set utmTags; + private String searchInput; + + private boolean hasResult; + private boolean isFailed; + + private long resultsCount; + private long responseTime; + private long timestamp; + + public SearchRequestLogDataBuilder(){} + + public SearchRequestLogDataBuilder setDsl(Map dsl) { + this.dsl = dsl; + return this; + } + + public SearchRequestLogDataBuilder setSearchInput(String searchInput) { + this.searchInput = searchInput; + return this; + } + + public SearchRequestLogDataBuilder setPersona(String persona) { + this.persona = persona; + return this; + } + + public SearchRequestLogDataBuilder setPurpose(String purpose) { + this.purpose = purpose; + return this; + } + + public SearchRequestLogDataBuilder setAttributes(Set attributes) { + this.attributes = attributes; + return this; + } + + public SearchRequestLogDataBuilder setRelationAttributes(Set relationAttributes) { + this.relationAttributes = relationAttributes; + return this; + } + + public SearchRequestLogDataBuilder setUserAgent(String userAgent) { + this.userAgent = userAgent; + return this; + } + + public SearchRequestLogDataBuilder setHost(String host) { + this.host = host; + return this; + } + + public SearchRequestLogDataBuilder setIpAddress(String ipAddress) { + this.ipAddress = ipAddress; + return this; + } + + public SearchRequestLogDataBuilder setUserName(String userName) { + this.userName = userName; + return this; + } + + public SearchRequestLogDataBuilder setErrorDetails(String errorDetails) { + this.errorDetails = errorDetails; + return this; + } + + public SearchRequestLogDataBuilder setErrorCode(String errorCode) { + this.errorCode = errorCode; + return this; + } + + public SearchRequestLogDataBuilder setEntityGuidsAll(Set entityGuidsAll) { + this.entityGuidsAll = entityGuidsAll; + return this; + } + + public SearchRequestLogDataBuilder setEntityQFNamesAll(Set entityQFNamesAll) { + this.entityQFNamesAll = entityQFNamesAll; + return this; + } + + public SearchRequestLogDataBuilder setEntityGuidsAllowed(Set entityGuidsAllowed) { + this.entityGuidsAllowed = entityGuidsAllowed; + return this; + } + + public SearchRequestLogDataBuilder setEntityQFNamesAllowed(Set entityQFNamesAllowed) { + this.entityQFNamesAllowed = entityQFNamesAllowed; + return this; + } + + public SearchRequestLogDataBuilder setEntityGuidsDenied(Set entityGuidsDenied) { + this.entityGuidsDenied = entityGuidsDenied; + return this; + } + + public SearchRequestLogDataBuilder setEntityQFNamesDenied(Set entityQFNamesDenied) { + this.entityQFNamesDenied = entityQFNamesDenied; + return this; + } + + public SearchRequestLogDataBuilder setEntityTypeNamesAll(Set entityTypeNamesAll) { + this.entityTypeNamesAll = entityTypeNamesAll; + return this; + } + + public SearchRequestLogDataBuilder setEntityTypeNamesAllowed(Set entityTypeNamesAllowed) { + this.entityTypeNamesAllowed = entityTypeNamesAllowed; + return this; + } + + public SearchRequestLogDataBuilder setEntityTypeNamesDenied(Set entityTypeNamesDenied) { + this.entityTypeNamesDenied = entityTypeNamesDenied; + return this; + } + + public SearchRequestLogDataBuilder setUtmTags(Set utmTags) { + this.utmTags = utmTags; + return this; + } + + public SearchRequestLogDataBuilder setHasResult(boolean hasResult) { + this.hasResult = hasResult; + return this; + } + + public SearchRequestLogDataBuilder setFailed(boolean failed) { + isFailed = failed; + return this; + } + + public SearchRequestLogDataBuilder setResultsCount(long resultsCount) { + this.resultsCount = resultsCount; + return this; + } + + public SearchRequestLogDataBuilder setResponseTime(long responseTime) { + this.responseTime = responseTime; + return this; + } + + public SearchRequestLogDataBuilder setTimestamp(long timestamp) { + this.timestamp = timestamp; + return this; + } + + public SearchRequestLogData build() { + return new SearchRequestLogData(dsl, attributes, relationAttributes, searchInput, persona, purpose, + userAgent, host, ipAddress, userName, + errorDetails, errorCode, entityGuidsAll, entityQFNamesAll, entityGuidsAllowed, + entityQFNamesAllowed, entityGuidsDenied, entityQFNamesDenied, + entityTypeNamesAll, entityTypeNamesAllowed, entityTypeNamesDenied, utmTags, + hasResult, isFailed, resultsCount, responseTime, timestamp); + } + } +} diff --git a/intg/src/main/java/org/apache/atlas/model/tasks/AtlasTask.java b/intg/src/main/java/org/apache/atlas/model/tasks/AtlasTask.java index 7918f0b770e..46f1f7d14eb 100644 --- a/intg/src/main/java/org/apache/atlas/model/tasks/AtlasTask.java +++ b/intg/src/main/java/org/apache/atlas/model/tasks/AtlasTask.java @@ -38,11 +38,46 @@ public class AtlasTask { @JsonIgnore public static final int MAX_ATTEMPT_COUNT = 3; + public String getClassificationName() { + return classificationName; + } + + public void setClassificationName(String classificationName) { + this.classificationName = classificationName; + } + public enum Status { PENDING, IN_PROGRESS, COMPLETE, - FAILED; + FAILED, + DELETED; + + public static Status from(String s) { + if(StringUtils.isEmpty(s)) { + return PENDING; + } + + switch (s.toLowerCase()) { + case "pending": + return PENDING; + + case "in_progress": + return IN_PROGRESS; + + case "complete": + return COMPLETE; + + case "failed": + return FAILED; + + case "deleted": + return DELETED; + + default: + return PENDING; + } + } } private String type; @@ -52,23 +87,30 @@ public enum Status { private Date updatedTime; private Date startTime; private Date endTime; + private Long timeTakenInSeconds; private Map parameters; private int attemptCount; private String errorMessage; private Status status; + private String classificationId; + private String entityGuid; + private String classificationName; public AtlasTask() { } - public AtlasTask(String type, String createdBy, Map parameters) { - this.guid = UUID.randomUUID().toString(); - this.type = type; - this.createdBy = createdBy; - this.createdTime = new Date(); - this.updatedTime = this.createdTime; - this.parameters = parameters; - this.status = Status.PENDING; - this.attemptCount = 0; + public AtlasTask(String type, String createdBy, Map parameters, String classificationId, + String entityGuid) { + this.guid = UUID.randomUUID().toString(); + this.type = type; + this.createdBy = createdBy; + this.createdTime = new Date(); + this.updatedTime = this.createdTime; + this.parameters = parameters; + this.status = Status.PENDING; + this.attemptCount = 0; + this.classificationId = classificationId; + this.entityGuid = entityGuid; } public String getGuid() { @@ -173,6 +215,30 @@ public void setEndTime(Date endTime) { this.endTime = endTime; } + public Long getTimeTakenInSeconds() { + return timeTakenInSeconds; + } + + public void setTimeTakenInSeconds(Long timeTakenInSeconds) { + this.timeTakenInSeconds = timeTakenInSeconds; + } + + public void setClassificationId(String classificationId) { + this.classificationId = classificationId; + } + + public String getClassificationId() { + return classificationId; + } + + public void setEntityGuid(String entityGuid) { + this.entityGuid = entityGuid; + } + + public String getEntityGuid() { + return entityGuid; + } + @JsonIgnore public void start() { this.setStatus(Status.IN_PROGRESS); @@ -181,7 +247,6 @@ public void start() { @JsonIgnore public void end() { - this.status = Status.COMPLETE; this.setEndTime(new Date()); } @@ -189,4 +254,22 @@ public void end() { public void updateStatusFromAttemptCount() { setStatus((attemptCount < MAX_ATTEMPT_COUNT) ? AtlasTask.Status.PENDING : AtlasTask.Status.FAILED); } + + @Override + public String toString() { + return "AtlasTask{" + + "type='" + type + '\'' + + ", guid='" + guid + '\'' + + ", createdBy='" + createdBy + '\'' + + ", createdTime=" + createdTime + + ", updatedTime=" + updatedTime + + ", startTime=" + startTime + + ", endTime=" + endTime + + ", timeTakenInSeconds=" + timeTakenInSeconds + + ", parameters=" + parameters + + ", attemptCount=" + attemptCount + + ", errorMessage='" + errorMessage + '\'' + + ", status=" + status + + '}'; + } } \ No newline at end of file diff --git a/intg/src/main/java/org/apache/atlas/model/tasks/TaskSearchParams.java b/intg/src/main/java/org/apache/atlas/model/tasks/TaskSearchParams.java new file mode 100644 index 00000000000..1e8253bff38 --- /dev/null +++ b/intg/src/main/java/org/apache/atlas/model/tasks/TaskSearchParams.java @@ -0,0 +1,70 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.model.tasks; + +import org.apache.atlas.model.discovery.SearchParams; +import org.apache.atlas.type.AtlasType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +public class TaskSearchParams extends SearchParams { + private static final Logger LOG = LoggerFactory.getLogger(TaskSearchParams.class); + + private Map dsl; + + public void setDsl(Map dsl) { + this.dsl = dsl; + } + + public Map getDsl() { + return this.dsl; + } + + @Override + public String getQuery() { + if (dsl != null) { + if (!dsl.containsKey("sort")) { + dsl.put("sort", Collections.singleton(mapOf("__task_timestamp", mapOf("order", "desc")))); + } + + try { + Map query = (Map) dsl.get("query"); + if (query.containsKey("match_all")) { + dsl.put("query", mapOf("exists", mapOf("field", "__task_v_type"))); + } + } catch (Exception e) { + LOG.warn("Failed to verify match_all clause in query"); + } + + return AtlasType.toJson(dsl); + } + return ""; + } + + private Map mapOf(String key, Object value) { + Map ret = new HashMap(); + ret.put(key, value); + + return ret; + } +} \ No newline at end of file diff --git a/intg/src/main/java/org/apache/atlas/model/tasks/TaskSearchResult.java b/intg/src/main/java/org/apache/atlas/model/tasks/TaskSearchResult.java new file mode 100644 index 00000000000..2a6b79332f3 --- /dev/null +++ b/intg/src/main/java/org/apache/atlas/model/tasks/TaskSearchResult.java @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.model.tasks; + +import java.util.List; +import java.util.Map; + +public class TaskSearchResult { + + public TaskSearchResult() { + } + + private List tasks; + private Map aggregations; + private long approximateCount; + + public List getTasks() { + return tasks; + } + + public void setTasks(List tasks) { + this.tasks = tasks; + } + + public Map getAggregations() { + return aggregations; + } + + public void setAggregations(Map aggregations) { + this.aggregations = aggregations; + } + + public long getApproximateCount() { + return approximateCount; + } + + public void setApproximateCount(long approximateCount) { + this.approximateCount = approximateCount; + } +} \ No newline at end of file diff --git a/intg/src/main/java/org/apache/atlas/model/typedef/AtlasClassificationDef.java b/intg/src/main/java/org/apache/atlas/model/typedef/AtlasClassificationDef.java index 1c60a7f25eb..de89bcdbfdc 100644 --- a/intg/src/main/java/org/apache/atlas/model/typedef/AtlasClassificationDef.java +++ b/intg/src/main/java/org/apache/atlas/model/typedef/AtlasClassificationDef.java @@ -52,7 +52,6 @@ public class AtlasClassificationDef extends AtlasStructDef implements AtlasNamed private Set entityTypes; private String displayName; - // subTypes field below is derived from 'superTypes' specified in all AtlasClassificationDef // this value is ignored during create & update operations private Set subTypes; @@ -101,7 +100,6 @@ public AtlasClassificationDef(String name, String displayName, String descriptio Set entityTypes, Map options) { super(TypeCategory.CLASSIFICATION, name, description, typeVersion, attributeDefs, options); this.setDisplayName(displayName); - setSuperTypes(superTypes); setEntityTypes(entityTypes); } diff --git a/intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java b/intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java index 56cde4b1ac6..286efc4cd09 100644 --- a/intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java +++ b/intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java @@ -307,6 +307,7 @@ public enum IndexType { DEFAULT, STRING} private List constraints; private Map options; private String displayName; + private boolean isDefaultValueNull; HashMap indexTypeESConfig; HashMap> indexTypeESFields; HashMap autoUpdateAttributes; @@ -356,6 +357,14 @@ private AtlasAttributeDef(String name, String typeName, boolean isOptional, Card public AtlasAttributeDef(String name, String typeName, boolean isOptional, Cardinality cardinality, int valuesMinCount, int valuesMaxCount, boolean isUnique, boolean isIndexable, boolean includeInNotification, String defaultValue, List constraints, Map options, String description, int searchWeight, IndexType indexType, boolean skipScrubbing) { + + this(name, typeName, isOptional, cardinality, valuesMinCount, valuesMaxCount, isUnique, isIndexable, includeInNotification, defaultValue, + constraints, options, description, searchWeight, indexType, skipScrubbing, false); + } + + public AtlasAttributeDef(String name, String typeName, boolean isOptional, Cardinality cardinality, + int valuesMinCount, int valuesMaxCount, boolean isUnique, boolean isIndexable, boolean includeInNotification, String defaultValue, + List constraints, Map options, String description, int searchWeight, IndexType indexType, boolean skipScrubbing, boolean isDefaultValueNull) { setName(name); setTypeName(typeName); setIsOptional(isOptional); @@ -372,6 +381,7 @@ public AtlasAttributeDef(String name, String typeName, boolean isOptional, Cardi setSearchWeight(searchWeight); setIndexType(indexType); setSkipScrubbing(skipScrubbing); + setIsDefaultValueNull(isDefaultValueNull); } public AtlasAttributeDef(AtlasAttributeDef other) { @@ -396,6 +406,7 @@ public AtlasAttributeDef(AtlasAttributeDef other) { setIndexTypeESFields(other.getIndexTypeESFields()); setAutoUpdateAttributes(other.getAutoUpdateAttributes()); setSkipScrubbing(other.getSkipScrubbing()); + setIsDefaultValueNull(other.getIsDefaultValueNull()); } } @@ -431,6 +442,14 @@ public void setName(String name) { this.name = name; } + public boolean getIsDefaultValueNull() { + return isDefaultValueNull; + } + + public void setIsDefaultValueNull(boolean isDefaultValueNull) { + this.isDefaultValueNull = isDefaultValueNull; + } + public String getTypeName() { return typeName; } @@ -623,6 +642,7 @@ public StringBuilder toString(StringBuilder sb) { sb.append(", indexTypeESFields='").append(indexTypeESFields).append('\''); sb.append(", autoUpdateAttributes='").append(autoUpdateAttributes).append('\''); sb.append(", skipScrubbing='").append(skipScrubbing).append('\''); + sb.append(", isDefaultValueNull='").append(isDefaultValueNull).append('\''); sb.append(", constraints=["); if (CollectionUtils.isNotEmpty(constraints)) { int i = 0; @@ -655,6 +675,7 @@ public boolean equals(Object o) { Objects.equals(typeName, that.typeName) && cardinality == that.cardinality && Objects.equals(defaultValue, that.defaultValue) && + Objects.equals(isDefaultValueNull, that.isDefaultValueNull) && Objects.equals(description, that.description) && Objects.equals(constraints, that.constraints) && Objects.equals(options, that.options) && @@ -669,7 +690,7 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hash(name, typeName, isOptional, cardinality, valuesMinCount, valuesMaxCount, isUnique, isIndexable, includeInNotification, defaultValue, constraints, options, description, searchWeight, indexType, displayName, indexTypeESConfig, indexTypeESFields, autoUpdateAttributes, skipScrubbing); + return Objects.hash(name, typeName, isOptional, cardinality, valuesMinCount, valuesMaxCount, isUnique, isIndexable, includeInNotification, defaultValue, constraints, options, description, searchWeight, indexType, displayName, indexTypeESConfig, indexTypeESFields, autoUpdateAttributes, skipScrubbing, isDefaultValueNull); } @Override diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java b/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java index b98ce7e725b..143cc8257f0 100644 --- a/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java +++ b/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java @@ -711,7 +711,12 @@ private AtlasStruct getStructFromValue(Object val) { if (val instanceof AtlasStruct) { ret = (AtlasStruct) val; } else if (val instanceof Map) { - ret = new AtlasStruct((Map) val); + ret = new AtlasStruct((Map) val, this.allAttributes, true); + + if (StringUtils.isEmpty(ret.getTypeName()) && !StringUtils.isEmpty(this.getTypeName())) { + ret.setTypeName(this.getTypeName()); + } + } else if (val instanceof String) { Map map = AtlasType.fromJson(val.toString(), Map.class); @@ -766,7 +771,7 @@ public AtlasAttribute(AtlasStructType definedInType, AtlasAttributeDef attrDef, this.vertexPropertyName = generateVertexPropertyName(attributeDef); this.vertexUniquePropertyName = attrDef.getIsUnique() ? encodePropertyKey(UNIQUE_ATTRIBUTE_SHADE_PROPERTY_PREFIX + attributeDef.getName()) : null; this.relationshipName = relationshipName; - this.relationshipEdgeLabel = getRelationshipEdgeLabel(relationshipLabel); + this.relationshipEdgeLabel = getRelationshipEdgeLabel(definedInType.getStructDef(), relationshipLabel); boolean isOwnedRef = false; String inverseRefAttribute = null; @@ -1142,7 +1147,19 @@ private static boolean shouldQuoteIndexQueryForChar(char c) { return false; } - private String getRelationshipEdgeLabel(String relationshipLabel) { + private static boolean isRootType(AtlasStructDef structDef) { + return StringUtils.equals(structDef.getName(), AtlasEntityType.ENTITY_ROOT.getTypeName()) || + StringUtils.equals(structDef.getName(), AtlasClassificationType.CLASSIFICATION_ROOT.getTypeName()); + } + + private String getRelationshipEdgeLabel(AtlasStructDef structDef, String relationshipLabel) { + String qualifiedName = ""; + if (isRootType(structDef)) { + qualifiedName = this.qualifiedName; + } else { + qualifiedName = String.format("%s.%s", structDef.getName(), this.qualifiedName); + } + return (relationshipLabel == null) ? getEdgeLabel(qualifiedName) : relationshipLabel; } diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasType.java b/intg/src/main/java/org/apache/atlas/type/AtlasType.java index c573877e596..084343c954f 100644 --- a/intg/src/main/java/org/apache/atlas/type/AtlasType.java +++ b/intg/src/main/java/org/apache/atlas/type/AtlasType.java @@ -26,6 +26,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.io.InputStream; import java.util.List; import java.util.Map; import java.util.Objects; @@ -143,9 +145,14 @@ public static T fromJson(String jsonStr, Class type) { return AtlasJson.fromJson(jsonStr, type); } + public static T fromJson(InputStream reader, Class type) throws IOException { + return AtlasJson.fromJson(reader, type); + } + public static T fromLinkedHashMap(Object obj, Class type) { return AtlasJson.fromLinkedHashMap(obj, type); } + public static String toV1Json(Object obj) { return AtlasJson.toV1Json(obj); } diff --git a/intg/src/main/java/org/apache/atlas/type/Constants.java b/intg/src/main/java/org/apache/atlas/type/Constants.java index cc0e9298203..01550ae9c00 100644 --- a/intg/src/main/java/org/apache/atlas/type/Constants.java +++ b/intg/src/main/java/org/apache/atlas/type/Constants.java @@ -55,7 +55,9 @@ public final class Constants { public static final String CATEGORIES_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "categories"); public static final String CATEGORIES_PARENT_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "parentCategory"); public static final String MEANINGS_TEXT_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "meaningsText"); + public static final String MEANING_NAMES_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "meaningNames"); public static final String HAS_LINEAGE = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "hasLineage"); + public static final String HAS_LINEAGE_VALID = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "hasLineageValid"); //Classification-Only System Attributes public static final String CLASSIFICATION_ENTITY_STATUS_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "entityStatus"); diff --git a/notification/pom.xml b/notification/pom.xml index aaf11c7a4b6..4d7c810708b 100644 --- a/notification/pom.xml +++ b/notification/pom.xml @@ -89,6 +89,13 @@ test + + ch.qos.reload4j + reload4j + ${reload4j.version} + compile + + @@ -160,9 +167,9 @@ ${slf4j.version} - log4j - log4j - ${log4j.version} + ch.qos.reload4j + reload4j + ${reload4j.version} org.apache.kafka diff --git a/notification/src/main/java/org/apache/atlas/kafka/KafkaNotification.java b/notification/src/main/java/org/apache/atlas/kafka/KafkaNotification.java index 32f5183a00a..676a917f967 100644 --- a/notification/src/main/java/org/apache/atlas/kafka/KafkaNotification.java +++ b/notification/src/main/java/org/apache/atlas/kafka/KafkaNotification.java @@ -60,10 +60,12 @@ public class KafkaNotification extends AbstractNotification implements Service { public static final String PROPERTY_PREFIX = "atlas.kafka"; public static final String ATLAS_HOOK_TOPIC = AtlasConfiguration.NOTIFICATION_HOOK_TOPIC_NAME.getString(); public static final String ATLAS_ENTITIES_TOPIC = AtlasConfiguration.NOTIFICATION_ENTITIES_TOPIC_NAME.getString(); + public static final String ATLAS_RELATIONSHIPS_TOPIC = AtlasConfiguration.NOTIFICATION_RELATIONSHIPS_TOPIC_NAME.getString(); protected static final String CONSUMER_GROUP_ID_PROPERTY = "group.id"; private static final String[] ATLAS_HOOK_CONSUMER_TOPICS = AtlasConfiguration.NOTIFICATION_HOOK_CONSUMER_TOPIC_NAMES.getStringArray(ATLAS_HOOK_TOPIC); private static final String[] ATLAS_ENTITIES_CONSUMER_TOPICS = AtlasConfiguration.NOTIFICATION_ENTITIES_CONSUMER_TOPIC_NAMES.getStringArray(ATLAS_ENTITIES_TOPIC); + private static final String[] ATLAS_RELATIONSHIPS_CONSUMER_TOPICS = AtlasConfiguration.NOTIFICATION_RELATIONSHIPS_CONSUMER_TOPIC_NAMES.getStringArray(ATLAS_RELATIONSHIPS_TOPIC); private static final String DEFAULT_CONSUMER_CLOSED_ERROR_MESSAGE = "This consumer has already been closed."; @@ -71,6 +73,7 @@ public class KafkaNotification extends AbstractNotification implements Service { { put(NotificationType.HOOK, ATLAS_HOOK_TOPIC); put(NotificationType.ENTITIES, ATLAS_ENTITIES_TOPIC); + put(NotificationType.RELATIONSHIPS, ATLAS_RELATIONSHIPS_TOPIC); } }; @@ -78,6 +81,7 @@ public class KafkaNotification extends AbstractNotification implements Service { { put(NotificationType.HOOK, trimAndPurge(ATLAS_HOOK_CONSUMER_TOPICS)); put(NotificationType.ENTITIES, trimAndPurge(ATLAS_ENTITIES_CONSUMER_TOPICS)); + put(NotificationType.RELATIONSHIPS, trimAndPurge(ATLAS_RELATIONSHIPS_CONSUMER_TOPICS)); } }; diff --git a/notification/src/main/java/org/apache/atlas/notification/NotificationInterface.java b/notification/src/main/java/org/apache/atlas/notification/NotificationInterface.java index a9cd4a6bbbf..3fc6f532a19 100644 --- a/notification/src/main/java/org/apache/atlas/notification/NotificationInterface.java +++ b/notification/src/main/java/org/apache/atlas/notification/NotificationInterface.java @@ -47,7 +47,9 @@ enum NotificationType { HOOK(new HookMessageDeserializer()), // Notifications to entity change consumers. - ENTITIES(new EntityMessageDeserializer()); + ENTITIES(new EntityMessageDeserializer()), + + RELATIONSHIPS(new EntityMessageDeserializer()); private final AtlasNotificationMessageDeserializer deserializer; diff --git a/pom.xml b/pom.xml index ae0ab2584f2..7d0bb1abec9 100644 --- a/pom.xml +++ b/pom.xml @@ -697,7 +697,7 @@ 4.3.0 1.8 3.2.2 - 7.6.2 + 7.16.2 org.apache.atlas.repository.audit.InMemoryEntityAuditRepository 2.13.2 2.18.1 @@ -706,7 +706,7 @@ solr berkeleyje 2.5 - 25.1-jre + 29.0-jre 4.1.0 ${hadoop.version} 3.3.0 @@ -730,11 +730,15 @@ 3.2.11 1.1 4.13.1 + 4.10.0 + 2.9.0 2.12 2.8.1 15.0.2.1 - 1.2.17 - 2.15.0 + 20220608.1 + 6.0.5 + 1.2.19 + 2.17.1 8.6.3 3.7 512m @@ -757,12 +761,13 @@ true false false + false 1.7.30 8.6.3 8.6.3 1.3.1 5.5.1 - 5.3.8 + 5.3.18 1.4.6.2.3.99.0-195 2.3.0 2C @@ -771,6 +776,8 @@ 3.5.1 5.0.3 3.4.6 + 3.22.1 + 1.11.1 @@ -782,11 +789,19 @@ server-api notification client + client-keycloak graphdb + repository authorization dashboardv2 dashboardv3 + + auth-agents-cred + auth-agents-common + auth-audits + auth-plugin-atlas + webapp docs @@ -798,8 +813,6 @@ addons/falcon-bridge addons/sqoop-bridge-shim addons/sqoop-bridge - addons/storm-bridge-shim - addons/storm-bridge addons/hbase-bridge-shim addons/hbase-bridge addons/hbase-testing-util @@ -861,6 +874,13 @@ Typesafe Repository https://repo.typesafe.com/typesafe/releases/ + + restlet + https://maven.restlet.talend.com/ + + false + + @@ -871,6 +891,13 @@ ${antlr4.version} + + + com.googlecode.owasp-java-html-sanitizer + owasp-java-html-sanitizer + ${owasp-html-sanitizer.version} + + com.google.guava guava @@ -888,49 +915,43 @@ org.slf4j slf4j-api ${slf4j.version} + + + log4j + log4j + + org.slf4j slf4j-log4j12 ${slf4j.version} + + + log4j + log4j + + org.slf4j jul-to-slf4j ${slf4j.version} - - - - log4j - log4j - ${log4j.version} - compile - com.sun.jdmk - jmxtools - - - com.sun.jmx - jmxri - - - javax.mail - mail - - - javax.jms - jmx - - - javax.jms - jms + log4j + log4j + + ch.qos.reload4j + reload4j + ${reload4j.version} + org.apache.hadoop @@ -973,7 +994,12 @@ junit junit + + log4j + log4j + + @@ -1652,7 +1678,7 @@ org.yaml snakeyaml - 1.26 + 1.33 diff --git a/repository/pom.xml b/repository/pom.xml index bfc2fe60b09..afe4708f62e 100755 --- a/repository/pom.xml +++ b/repository/pom.xml @@ -113,8 +113,16 @@ io.netty netty-handler + + org.slf4j + log4j-over-slf4j + + + io.dropwizard.metrics + metrics-core + - 2.1.8 + 3.11.12 @@ -234,6 +242,13 @@ + + + io.dropwizard.metrics + metrics-core + 3.2.6 + + org.cassandraunit cassandra-unit @@ -294,6 +309,18 @@ java-statsd-client 3.1.0 + + org.apache.atlas + atlas-graphdb-janus + ${project.version} + compile + + + + org.apache.atlas + client-keycloak + 3.0.0-SNAPSHOT + diff --git a/repository/src/main/java/org/apache/atlas/GraphTransactionInterceptor.java b/repository/src/main/java/org/apache/atlas/GraphTransactionInterceptor.java index c8b7ff8d918..2d072d95fca 100644 --- a/repository/src/main/java/org/apache/atlas/GraphTransactionInterceptor.java +++ b/repository/src/main/java/org/apache/atlas/GraphTransactionInterceptor.java @@ -26,9 +26,7 @@ import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasVertex; -import org.apache.atlas.tasks.TaskManagement; import org.apache.atlas.utils.AtlasPerfMetrics.MetricRecorder; -import org.apache.commons.collections.CollectionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; @@ -56,7 +54,6 @@ public class GraphTransactionInterceptor implements MethodInterceptor { private static final ThreadLocal> guidVertexCache = ThreadLocal.withInitial(() -> new HashMap<>()); private final AtlasGraph graph; - private final TaskManagement taskManagement; private static final ThreadLocal> vertexGuidCache = new ThreadLocal>() { @@ -83,9 +80,8 @@ public Map initialValue() { }; @Inject - public GraphTransactionInterceptor(AtlasGraph graph, TaskManagement taskManagement) { + public GraphTransactionInterceptor(AtlasGraph graph) { this.graph = graph; - this.taskManagement = taskManagement; } @Override @@ -132,7 +128,11 @@ public Object invoke(MethodInvocation invocation) throws Throwable { } else { doRollback(logRollback, t); } - throw t; + if (checkForBatchTooLargeError(t)) { + throw new AtlasBaseException(AtlasErrorCode.BATCH_SIZE_TOO_LARGE, t); + } else { + throw t; + } } } finally { RequestContext.get().endMetricRecord(metric); @@ -168,13 +168,25 @@ public Object invoke(MethodInvocation invocation) throws Throwable { } OBJECT_UPDATE_SYNCHRONIZER.releaseLockedObjects(); + } + } - if (isSuccess) { - submitTasks(); + public boolean checkForBatchTooLargeError(Throwable t) { + Throwable currentCause = t; + while (currentCause != null) { + String message = currentCause.getMessage(); + if (message != null && + message.contains("Batch too large") && + currentCause.getClass().getSimpleName().equals("InvalidQueryException")) { + return true; } + currentCause = currentCause.getCause(); } + return false; } + + private void doCommitOrRollback(final String invokingClass, final String invokedMethodName) { if (innerFailure.get()) { if (LOG.isDebugEnabled()) { @@ -239,14 +251,6 @@ public static void clearCache() { edgeStateCache.get().clear(); } - private void submitTasks() { - if (CollectionUtils.isEmpty(RequestContext.get().getQueuedTasks()) || taskManagement == null) { - return; - } - - taskManagement.addAll(RequestContext.get().getQueuedTasks()); - } - boolean logException(Throwable t) { if (t instanceof AtlasBaseException) { Response.Status httpCode = ((AtlasBaseException) t).getAtlasErrorCode().getHttpCode(); diff --git a/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java index 8e9c42be215..dc567be4fe3 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java @@ -27,6 +27,8 @@ import org.apache.atlas.model.discovery.SearchParameters; import org.apache.atlas.model.discovery.SearchParams; import org.apache.atlas.model.profile.AtlasUserSavedSearch; +import org.apache.atlas.model.searchlog.SearchLogSearchParams; +import org.apache.atlas.model.searchlog.SearchLogSearchResult; import java.util.List; @@ -156,6 +158,14 @@ AtlasSearchResult searchUsingBasicQuery(String query, String type, String classi */ AtlasSearchResult directIndexSearch(SearchParams searchParams) throws AtlasBaseException; + /** + * Search for direct ES query on search logs index + * @param searchParams Search criteria + * @return Matching search logs + * @throws AtlasBaseException + */ + SearchLogSearchResult searchLogs(SearchLogSearchParams searchParams) throws AtlasBaseException; + /** * Should return top 5 suggestion strings for the given prefix. * @param prefixString the prefix string diff --git a/repository/src/main/java/org/apache/atlas/discovery/AtlasLineageContext.java b/repository/src/main/java/org/apache/atlas/discovery/AtlasLineageContext.java new file mode 100644 index 00000000000..377c3b360b8 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/discovery/AtlasLineageContext.java @@ -0,0 +1,199 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.discovery; + +import com.google.common.annotations.VisibleForTesting; +import org.apache.atlas.model.discovery.SearchParameters; +import org.apache.atlas.model.lineage.AtlasLineageInfo; +import org.apache.atlas.model.lineage.AtlasLineageRequest; +import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.commons.collections.Predicate; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Set; + +import static org.apache.atlas.model.lineage.AtlasLineageInfo.LineageDirection.BOTH; + +public class AtlasLineageContext { + private static final Logger LOG = LoggerFactory.getLogger(AtlasLineageContext.class); + + private int depth; + private int offset; + private int limit; + private String guid; + private boolean hideProcess; + private boolean allowDeletedProcess; + private boolean calculateRemainingVertexCounts; + private AtlasLineageInfo.LineageDirection direction = BOTH; + + private boolean isDataset; + private boolean isProcess; + + private Set attributes; + private Set ignoredProcesses; + private Predicate predicate; + + private AtlasVertex startDatasetVertex = null; + + public AtlasLineageContext(AtlasLineageRequest lineageRequest, AtlasTypeRegistry typeRegistry) { + this.guid = lineageRequest.getGuid(); + this.limit = lineageRequest.getLimit(); + this.depth = lineageRequest.getDepth(); + this.direction = lineageRequest.getDirection(); + this.hideProcess = lineageRequest.isHideProcess(); + this.allowDeletedProcess = lineageRequest.isAllowDeletedProcess(); + this.attributes = lineageRequest.getAttributes(); + this.ignoredProcesses = lineageRequest.getIgnoredProcesses(); + this.offset = lineageRequest.getOffset(); + this.calculateRemainingVertexCounts = lineageRequest.getCalculateRemainingVertexCounts(); + + predicate = constructInMemoryPredicate(typeRegistry, lineageRequest.getEntityFilters()); + } + + @VisibleForTesting + AtlasLineageContext() { + } + + public int getDepth() { + return depth; + } + + public void setDepth(int depth) { + this.depth = depth; + } + + public int getLimit() { + return limit; + } + + public void setLimit(int limit) { + this.limit = limit; + } + + public String getGuid() { + return guid; + } + + public void setGuid(String guid) { + this.guid = guid; + } + + public boolean isDataset() { + return isDataset; + } + + public void setDataset(boolean dataset) { + isDataset = dataset; + } + + public boolean isProcess() { + return isProcess; + } + + public void setProcess(boolean process) { + isProcess = process; + } + + public AtlasLineageInfo.LineageDirection getDirection() { + return direction; + } + + public void setDirection(AtlasLineageInfo.LineageDirection direction) { + this.direction = direction; + } + + public boolean isHideProcess() { + return hideProcess; + } + + public void setHideProcess(boolean hideProcess) { + this.hideProcess = hideProcess; + } + + public Set getAttributes() { + return attributes; + } + + public void setAttributes(Set attributes) { + this.attributes = attributes; + } + + public Set getIgnoredProcesses() { + return ignoredProcesses; + } + + public void setIgnoredProcesses(Set ignoredProcesses) { + this.ignoredProcesses = ignoredProcesses; + } + + public AtlasVertex getStartDatasetVertex() { + return startDatasetVertex; + } + + public void setStartDatasetVertex(AtlasVertex startDatasetVertex) { + this.startDatasetVertex = startDatasetVertex; + } + + public boolean isAllowDeletedProcess() { + return allowDeletedProcess; + } + + public void setAllowDeletedProcess(boolean allowDeletedProcess) { + this.allowDeletedProcess = allowDeletedProcess; + } + + public int getOffset() { + return offset; + } + + protected Predicate constructInMemoryPredicate(AtlasTypeRegistry typeRegistry, SearchParameters.FilterCriteria filterCriteria) { + LineageSearchProcessor lineageSearchProcessor = new LineageSearchProcessor(); + return lineageSearchProcessor.constructInMemoryPredicate(typeRegistry, filterCriteria); + } + + protected boolean evaluate(AtlasVertex vertex) { + if (predicate != null) { + return predicate.evaluate(vertex); + } + return true; + } + + public boolean shouldApplyPagination() { + return offset > -1; + } + + public boolean isCalculateRemainingVertexCounts() { + return calculateRemainingVertexCounts; + } + + @Override + public String toString() { + return "LineageRequestContext{" + + "depth=" + depth + + ", guid='" + guid + '\'' + + ", isDataset=" + isDataset + + ", isProcess=" + isProcess + + ", allowDeletedProcess=" + allowDeletedProcess + + ", direction=" + direction + + ", attributes=" + attributes + + ", ignoredProcesses=" + ignoredProcesses + + '}'; + } +} diff --git a/repository/src/main/java/org/apache/atlas/discovery/AtlasLineageListContext.java b/repository/src/main/java/org/apache/atlas/discovery/AtlasLineageListContext.java new file mode 100644 index 00000000000..b0ac3bd3ccc --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/discovery/AtlasLineageListContext.java @@ -0,0 +1,183 @@ +package org.apache.atlas.discovery; + +import org.apache.atlas.model.discovery.SearchParameters; +import org.apache.atlas.model.lineage.LineageListRequest; +import org.apache.atlas.repository.graphdb.AtlasEdge; +import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.commons.collections.Predicate; +import java.util.Set; + +public final class AtlasLineageListContext { + private String guid; + private int size; + private int from; + private int depth; + private LineageListRequest.LineageDirection direction; + private Predicate vertexPredicate; + private Predicate vertexTraversalPredicate; + private Predicate edgeTraversalPredicate; + private Set attributes; + private int currentFromCounter; + private int currentEntityCounter; + private boolean depthLimitReached; + private boolean hasMoreUpdated; + + public AtlasLineageListContext(LineageListRequest lineageListRequest, AtlasTypeRegistry typeRegistry) { + this.guid = lineageListRequest.getGuid(); + this.size = lineageListRequest.getSize(); + this.from = lineageListRequest.getFrom(); + this.depth = lineageListRequest.getDepth(); + this.direction = lineageListRequest.getDirection(); + this.vertexPredicate = constructInMemoryPredicate(typeRegistry, lineageListRequest.getEntityFilters()); + this.vertexTraversalPredicate = constructInMemoryPredicate(typeRegistry, lineageListRequest.getEntityTraversalFilters()); + this.edgeTraversalPredicate = constructInMemoryPredicate(typeRegistry, lineageListRequest.getRelationshipTraversalFilters()); + this.attributes = lineageListRequest.getAttributes(); + } + + public String getGuid() { + return guid; + } + + public void setGuid(String guid) { + this.guid = guid; + } + + public int getSize() { + return size; + } + + public void setSize(int size) { + this.size = size; + } + + public int getFrom() { + return from; + } + + public void setFrom(int from) { + this.from = from; + } + + /* + * Clients assume depth limit at node level + * eg. Atlas depth 1 would return processes (BFS algo) and depth 2 would return processes + nodes, whereas client needs processes + nodes for depth 1 + */ + public int getDepth() { + return 2*depth; + } + + public void setDepth(int depth) { + this.depth = depth; + } + + public LineageListRequest.LineageDirection getDirection() { + return direction; + } + + public void setDirection(LineageListRequest.LineageDirection direction) { + this.direction = direction; + } + + public Predicate getVertexPredicate() { + return vertexPredicate; + } + + public void setVertexPredicate(Predicate vertexPredicate) { + this.vertexPredicate = vertexPredicate; + } + + public Predicate getVertexTraversalPredicate() { + return vertexTraversalPredicate; + } + + public void setVertexTraversalPredicate(Predicate vertexTraversalPredicate) { + this.vertexTraversalPredicate = vertexTraversalPredicate; + } + + public Predicate getEdgeTraversalPredicate() { + return edgeTraversalPredicate; + } + + public void setEdgeTraversalPredicate(Predicate edgeTraversalPredicate) { + this.edgeTraversalPredicate = edgeTraversalPredicate; + } + + public Set getAttributes() { + return attributes; + } + + public void setAttributes(Set attributes) { + this.attributes = attributes; + } + + public int getCurrentFromCounter() { + return currentFromCounter; + } + + public void setCurrentFromCounter(int currentFromCounter) { + this.currentFromCounter = currentFromCounter; + } + + public int getCurrentEntityCounter() { + return currentEntityCounter; + } + + public void setCurrentEntityCounter(int currentEntityCounter) { + this.currentEntityCounter = currentEntityCounter; + } + + protected Predicate constructInMemoryPredicate(AtlasTypeRegistry typeRegistry, SearchParameters.FilterCriteria filterCriteria) { + LineageSearchProcessor lineageSearchProcessor = new LineageSearchProcessor(); + return lineageSearchProcessor.constructInMemoryPredicate(typeRegistry, filterCriteria); + } + + protected boolean evaluateVertexFilter(AtlasVertex vertex) { + if (vertexPredicate != null) { + return vertexPredicate.evaluate(vertex); + } + return true; + } + + protected boolean evaluateTraversalFilter(AtlasVertex vertex) { + if (vertexTraversalPredicate != null) { + return vertexTraversalPredicate.evaluate(vertex); + } + return true; + } + + protected boolean evaluateTraversalFilter(AtlasEdge edge) { + if (edgeTraversalPredicate != null) { + return edgeTraversalPredicate.evaluate(edge); + } + return true; + } + + public void incrementCurrentFromCounter() { + this.currentFromCounter++; + } + + public boolean isEntityLimitReached() { + return this.currentEntityCounter == this.size; + } + + public void incrementEntityCount() { + this.currentEntityCounter++; + } + + public boolean isDepthLimitReached() { + return depthLimitReached; + } + + public void setDepthLimitReached(boolean depthLimitReached) { + this.depthLimitReached = depthLimitReached; + } + + public boolean isHasMoreUpdated() { + return hasMoreUpdated; + } + + public void setHasMoreUpdated(boolean hasMoreUpdated) { + this.hasMoreUpdated = hasMoreUpdated; + } +} diff --git a/repository/src/main/java/org/apache/atlas/discovery/AtlasLineageOnDemandContext.java b/repository/src/main/java/org/apache/atlas/discovery/AtlasLineageOnDemandContext.java new file mode 100644 index 00000000000..55096848550 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/discovery/AtlasLineageOnDemandContext.java @@ -0,0 +1,102 @@ +package org.apache.atlas.discovery; + +import org.apache.atlas.model.discovery.SearchParameters; +import org.apache.atlas.model.lineage.LineageOnDemandBaseParams; +import org.apache.atlas.model.lineage.LineageOnDemandConstraints; +import org.apache.atlas.model.lineage.LineageOnDemandRequest; +import org.apache.atlas.repository.graphdb.AtlasEdge; +import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.commons.collections.Predicate; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; +import java.util.Set; + +public class AtlasLineageOnDemandContext { + private static final Logger LOG = LoggerFactory.getLogger(AtlasLineageContext.class); + + private Map constraints; + private Predicate vertexPredicate; + private Predicate edgePredicate; + private Set attributes; + private Set relationAttributes; + private LineageOnDemandBaseParams defaultParams; + + public AtlasLineageOnDemandContext(LineageOnDemandRequest lineageOnDemandRequest, AtlasTypeRegistry typeRegistry) { + this.constraints = lineageOnDemandRequest.getConstraints(); + this.attributes = lineageOnDemandRequest.getAttributes(); + this.relationAttributes = lineageOnDemandRequest.getRelationAttributes(); + this.defaultParams = lineageOnDemandRequest.getDefaultParams(); + this.vertexPredicate = constructInMemoryPredicate(typeRegistry, lineageOnDemandRequest.getEntityTraversalFilters()); + this.edgePredicate = constructInMemoryPredicate(typeRegistry, lineageOnDemandRequest.getRelationshipTraversalFilters()); + } + + public Map getConstraints() { + return constraints; + } + + public void setConstraints(Map constraints) { + this.constraints = constraints; + } + + public Predicate getVertexPredicate() { + return vertexPredicate; + } + + public void setVertexPredicate(Predicate vertexPredicate) { + this.vertexPredicate = vertexPredicate; + } + + public Predicate getEdgePredicate() { + return edgePredicate; + } + + public void setEdgePredicate(Predicate edgePredicate) { + this.edgePredicate = edgePredicate; + } + + public Set getAttributes() { + return attributes; + } + + public void setAttributes(Set attributes) { + this.attributes = attributes; + } + + public Set getRelationAttributes() { + return relationAttributes; + } + + public void setRelationAttributes(Set relationAttributes) { + this.relationAttributes = relationAttributes; + } + + public LineageOnDemandBaseParams getDefaultParams() { + return defaultParams; + } + + public void setDefaultParams(LineageOnDemandBaseParams defaultParams) { + this.defaultParams = defaultParams; + } + + protected Predicate constructInMemoryPredicate(AtlasTypeRegistry typeRegistry, SearchParameters.FilterCriteria filterCriteria) { + LineageSearchProcessor lineageSearchProcessor = new LineageSearchProcessor(); + return lineageSearchProcessor.constructInMemoryPredicate(typeRegistry, filterCriteria); + } + + protected boolean evaluate(AtlasVertex vertex) { + if (vertexPredicate != null) { + return vertexPredicate.evaluate(vertex); + } + return true; + } + + protected boolean evaluate(AtlasEdge edge) { + if (edgePredicate != null) { + return edgePredicate.evaluate(edge); + } + return true; + } +} diff --git a/repository/src/main/java/org/apache/atlas/discovery/AtlasLineageService.java b/repository/src/main/java/org/apache/atlas/discovery/AtlasLineageService.java index b35f7f17179..69c3e387bca 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/AtlasLineageService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/AtlasLineageService.java @@ -20,10 +20,11 @@ import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.model.lineage.AtlasLineageInfo; +import org.apache.atlas.model.lineage.*; import org.apache.atlas.model.lineage.AtlasLineageInfo.LineageDirection; import org.apache.atlas.v1.model.lineage.SchemaResponse.SchemaDetails; + public interface AtlasLineageService { /** * @param entityGuid unique ID of the entity @@ -34,12 +35,20 @@ public interface AtlasLineageService { AtlasLineageInfo getAtlasLineageInfo(String entityGuid, LineageDirection direction, int depth) throws AtlasBaseException; /** - * @param entityGuid unique ID of the entity - * @param direction direction of lineage - INPUT, OUTPUT or BOTH - * @param depth number of hops in lineage + * @param entityGuid unique ID of the entity + * @param direction direction of lineage - INPUT, OUTPUT or BOTH + * @param depth number of hops in lineage + * @param page + * @param recordPerPage * @return AtlasLineageInfo */ - AtlasLineageInfo getAtlasLineageInfo(String entityGuid, LineageDirection direction, int depth, boolean hideProcess) throws AtlasBaseException; + AtlasLineageInfo getAtlasLineageInfo(String guid, LineageDirection direction, int depth, boolean hideProcess, int offset, int limit, boolean calculateRemainingVertexCounts) throws AtlasBaseException; + + /** + * @param lineageRequest AtlasLineageRequest + * @return AtlasLineageInfo + */ + AtlasLineageInfo getAtlasLineageInfo(AtlasLineageRequest lineageRequest) throws AtlasBaseException; /** * Return the schema for the given datasetName. @@ -56,4 +65,19 @@ public interface AtlasLineageService { * @return Schema as JSON */ SchemaDetails getSchemaForHiveTableByGuid(String guid) throws AtlasBaseException; + + /** + * @param entityGuid unique ID of the entity + * @param lineageOnDemandRequest lineage on demand request object + * @return AtlasLineageInfo + */ + AtlasLineageOnDemandInfo getAtlasLineageInfo(String entityGuid, LineageOnDemandRequest lineageOnDemandRequest) throws AtlasBaseException; + + /** + * @param entityGuid unique ID of the entity + * @param lineageListRequest lineage list request object + * @return AtlasLineageListInfo + */ + AtlasLineageListInfo getLineageListInfoOnDemand(String entityGuid, LineageListRequest lineageListRequest) throws AtlasBaseException; + } diff --git a/repository/src/main/java/org/apache/atlas/discovery/CachedVertexEdgesKey.java b/repository/src/main/java/org/apache/atlas/discovery/CachedVertexEdgesKey.java new file mode 100644 index 00000000000..ba424820aa1 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/discovery/CachedVertexEdgesKey.java @@ -0,0 +1,31 @@ +package org.apache.atlas.discovery; + +import org.apache.atlas.repository.graphdb.AtlasEdgeDirection; + +import java.util.Objects; + +public class CachedVertexEdgesKey { + + private final Object vertexId; + private final AtlasEdgeDirection direction; + private final String edgeLabel; + + public CachedVertexEdgesKey(Object vertexId, AtlasEdgeDirection direction, String edgeLabel) { + this.vertexId = vertexId; + this.direction = direction; + this.edgeLabel = edgeLabel; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + CachedVertexEdgesKey that = (CachedVertexEdgesKey) o; + return vertexId.equals(that.vertexId) && direction == that.direction && edgeLabel.equals(that.edgeLabel); + } + + @Override + public int hashCode() { + return Objects.hash(vertexId, direction, edgeLabel); + } +} diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java index c43b644cc36..581689a4dbe 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java @@ -18,30 +18,28 @@ package org.apache.atlas.discovery; import com.google.common.annotations.VisibleForTesting; -import org.apache.atlas.ApplicationProperties; -import org.apache.atlas.AtlasConfiguration; -import org.apache.atlas.AtlasErrorCode; -import org.apache.atlas.AtlasException; -import org.apache.atlas.RequestContext; -import org.apache.atlas.SortOrder; +import org.apache.atlas.*; import org.apache.atlas.annotation.GraphTransaction; import org.apache.atlas.authorize.AtlasAuthorizationUtils; import org.apache.atlas.authorize.AtlasSearchResultScrubRequest; import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.model.discovery.AtlasAggregationEntry; -import org.apache.atlas.model.discovery.AtlasQuickSearchResult; -import org.apache.atlas.model.discovery.AtlasSearchResult; +import org.apache.atlas.model.discovery.*; import org.apache.atlas.model.discovery.AtlasSearchResult.AtlasFullTextResult; import org.apache.atlas.model.discovery.AtlasSearchResult.AtlasQueryType; +<<<<<<< HEAD import org.apache.atlas.model.discovery.AtlasSuggestionsResult; import org.apache.atlas.model.discovery.IndexSearchParams; import org.apache.atlas.model.discovery.SearchParams; import org.apache.atlas.model.discovery.SearchParameters; import org.apache.atlas.model.discovery.QuickSearchParameters; +======= +>>>>>>> 7f3c811fb15361ba82bfb4edd7a8393798863ff0 import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasEntityHeader; import org.apache.atlas.model.instance.AtlasObjectId; import org.apache.atlas.model.profile.AtlasUserSavedSearch; +import org.apache.atlas.model.searchlog.SearchLogSearchParams; +import org.apache.atlas.model.searchlog.SearchLogSearchResult; import org.apache.atlas.query.QueryParams; import org.apache.atlas.query.executors.DSLQueryExecutor; import org.apache.atlas.query.executors.ScriptEngineBasedExecutor; @@ -49,29 +47,21 @@ import org.apache.atlas.repository.Constants; import org.apache.atlas.repository.graph.GraphBackedSearchIndexer; import org.apache.atlas.repository.graph.GraphHelper; -import org.apache.atlas.repository.graphdb.AtlasEdge; -import org.apache.atlas.repository.graphdb.AtlasEdgeDirection; -import org.apache.atlas.repository.graphdb.AtlasGraph; -import org.apache.atlas.repository.graphdb.AtlasIndexQuery; +import org.apache.atlas.repository.graphdb.*; import org.apache.atlas.repository.graphdb.AtlasIndexQuery.Result; -import org.apache.atlas.repository.graphdb.AtlasVertex; -import org.apache.atlas.repository.graphdb.DirectIndexQueryResult; import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2; import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; import org.apache.atlas.repository.userprofile.UserProfileService; +import org.apache.atlas.searchlog.ESSearchLogger; import org.apache.atlas.stats.StatsClient; -import org.apache.atlas.type.AtlasArrayType; +import org.apache.atlas.type.*; import org.apache.atlas.type.AtlasBuiltInTypes.AtlasObjectIdType; -import org.apache.atlas.type.AtlasClassificationType; -import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasStructType.AtlasAttribute; -import org.apache.atlas.type.AtlasType; -import org.apache.atlas.type.AtlasTypeRegistry; -import org.apache.atlas.type.AtlasStructType; import org.apache.atlas.util.AtlasGremlinQueryProvider; import org.apache.atlas.util.AtlasGremlinQueryProvider.AtlasGremlinQuery; import org.apache.atlas.util.SearchPredicateUtil; import org.apache.atlas.util.SearchTracker; +import org.apache.atlas.utils.AtlasPerfMetrics; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.Predicate; import org.apache.commons.collections4.IteratorUtils; @@ -85,17 +75,9 @@ import javax.inject.Inject; import javax.script.ScriptEngine; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; import javax.script.ScriptException; +import java.util.*; +import java.util.stream.Collectors; import static org.apache.atlas.AtlasErrorCode.*; import static org.apache.atlas.SortOrder.ASCENDING; @@ -103,6 +85,7 @@ import static org.apache.atlas.model.instance.AtlasEntity.Status.DELETED; import static org.apache.atlas.repository.Constants.ASSET_ENTITY_TYPE; import static org.apache.atlas.repository.Constants.OWNER_ATTRIBUTE; +import static org.apache.atlas.repository.Constants.VERTEX_INDEX_NAME; import static org.apache.atlas.util.AtlasGremlinQueryProvider.AtlasGremlinQuery.BASIC_SEARCH_STATE_FILTER; import static org.apache.atlas.util.AtlasGremlinQueryProvider.AtlasGremlinQuery.TO_RANGE_LIST; @@ -127,7 +110,7 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { private final StatsClient statsClient; @Inject - EntityDiscoveryService(AtlasTypeRegistry typeRegistry, + public EntityDiscoveryService(AtlasTypeRegistry typeRegistry, AtlasGraph graph, GraphBackedSearchIndexer indexer, SearchTracker searchTracker, @@ -149,7 +132,6 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { this.dslQueryExecutor = AtlasConfiguration.DSL_EXECUTOR_TRAVERSAL.getBoolean() ? new TraversalBasedExecutor(typeRegistry, graph, entityRetriever) : new ScriptEngineBasedExecutor(typeRegistry, graph, entityRetriever); - LOG.info("DSL Executor: {}", this.dslQueryExecutor.getClass().getSimpleName()); } @Override @@ -420,7 +402,7 @@ public AtlasQuickSearchResult quickSearch(QuickSearchParameters quickSearchParam query = query + "*"; } quickSearchParameters.setQuery(query); - + SearchContext searchContext = new SearchContext(createSearchParameters(quickSearchParameters), typeRegistry, graph, @@ -956,7 +938,13 @@ private void checkSavedSearchOwnership(String claimedOwner, AtlasUserSavedSearch } private void scrubSearchResults(AtlasSearchResult result) throws AtlasBaseException { - AtlasAuthorizationUtils.scrubSearchResults(new AtlasSearchResultScrubRequest(typeRegistry, result)); + scrubSearchResults(result, false); + } + + private void scrubSearchResults(AtlasSearchResult result, boolean suppressLogs) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder scrubSearchResultsMetrics = RequestContext.get().startMetricRecord("scrubSearchResults"); + AtlasAuthorizationUtils.scrubSearchResults(new AtlasSearchResultScrubRequest(typeRegistry, result), suppressLogs); + RequestContext.get().endMetricRecord(scrubSearchResultsMetrics); } private Set getAggregationFields() { @@ -1009,10 +997,59 @@ public AtlasSearchResult directIndexSearch(SearchParams searchParams) throws Atl } try { - indexQuery = graph.elasticsearchQuery(Constants.VERTEX_INDEX, searchParams); + if(LOG.isDebugEnabled()){ + LOG.debug("Performing ES search for the params ({})", searchParams); + } + String indexName = getIndexName(params); + + indexQuery = graph.elasticsearchQuery(indexName); + AtlasPerfMetrics.MetricRecorder elasticSearchQueryMetric = RequestContext.get().startMetricRecord("elasticSearchQuery"); DirectIndexQueryResult indexQueryResult = indexQuery.vertices(searchParams); + RequestContext.get().endMetricRecord(elasticSearchQueryMetric); + prepareSearchResult(ret, indexQueryResult, resultAttributes, true); + + ret.setAggregations(indexQueryResult.getAggregationMap()); + ret.setApproximateCount(indexQuery.vertexTotals()); + } catch (Exception e) { + LOG.error("Error while performing direct search for the params ({}), {}", searchParams, e.getMessage()); + throw e; + } + return ret; + } + @Override + public SearchLogSearchResult searchLogs(SearchLogSearchParams searchParams) throws AtlasBaseException { + SearchLogSearchResult ret = new SearchLogSearchResult(); + ret.setSearchParameters(searchParams); + AtlasIndexQuery indexQuery = null; + + try { + indexQuery = graph.elasticsearchQuery(ESSearchLogger.INDEX_NAME); + Map result = indexQuery.directIndexQuery(searchParams.getQueryString()); + + if (result.get("total") != null) + ret.setApproximateCount( ((Integer) result.get("total")).longValue()); + + List hits = (List) result.get("data"); + + List> logs = hits.stream().map(x -> (HashMap) x.get("_source")).collect(Collectors.toList()); + + ret.setLogs(logs); + ret.setAggregations((Map) result.get("aggregations")); + + return ret; + } catch (AtlasBaseException be) { + throw be; + } + } + + private void prepareSearchResult(AtlasSearchResult ret, DirectIndexQueryResult indexQueryResult, Set resultAttributes, boolean fetchCollapsedResults) throws AtlasBaseException { + SearchParams searchParams = ret.getSearchParameters(); + try { + if(LOG.isDebugEnabled()){ + LOG.debug("Preparing search results for ({})", ret.getSearchParameters()); + } Iterator iterator = indexQueryResult.getIterator(); boolean showSearchScore = searchParams.getShowSearchScore(); @@ -1020,21 +1057,99 @@ public AtlasSearchResult directIndexSearch(SearchParams searchParams) throws Atl Result result = iterator.next(); AtlasVertex vertex = result.getVertex(); + if (vertex == null) { + LOG.warn("vertex in null"); + continue; + } + AtlasEntityHeader header = entityRetriever.toAtlasEntityHeader(vertex, resultAttributes); - header.setClassifications(entityRetriever.getAllClassifications(vertex)); + if(RequestContext.get().includeClassifications()){ + header.setClassifications(entityRetriever.getAllClassifications(vertex)); + } if (showSearchScore) { ret.addEntityScore(header.getGuid(), result.getScore()); } + if (fetchCollapsedResults) { + Map collapse = new HashMap<>(); + + Set collapseKeys = result.getCollapseKeys(); + for (String collapseKey : collapseKeys) { + AtlasSearchResult collapseRet = new AtlasSearchResult(); + collapseRet.setSearchParameters(ret.getSearchParameters()); + + Set collapseResultAttributes = new HashSet<>(); + if (searchParams.getCollapseAttributes() != null) { + collapseResultAttributes.addAll(searchParams.getCollapseAttributes()); + } else { + collapseResultAttributes = resultAttributes; + } + + if (searchParams.getCollapseRelationAttributes() != null) { + RequestContext.get().getRelationAttrsForSearch().clear(); + RequestContext.get().setRelationAttrsForSearch(searchParams.getCollapseRelationAttributes()); + } + + DirectIndexQueryResult indexQueryCollapsedResult = result.getCollapseVertices(collapseKey); + collapseRet.setApproximateCount(indexQueryCollapsedResult.getApproximateCount()); + prepareSearchResult(collapseRet, indexQueryCollapsedResult, collapseResultAttributes, false); + + collapseRet.setSearchParameters(null); + collapse.put(collapseKey, collapseRet); + } + if (!collapse.isEmpty()) { + header.setCollapse(collapse); + } + } + ret.addEntity(header); } - - ret.setAggregations(indexQueryResult.getAggregationMap()); - ret.setApproximateCount(indexQuery.vertexTotals()); } catch (Exception e) { - throw e; + throw e; } + scrubSearchResults(ret, searchParams.getSuppressLogs()); + } - scrubSearchResults(ret); - return ret; + private Map getMap(String key, Object value) { + Map map = new HashMap<>(); + map.put(key, value); + return map; + } + + public List searchUsingTermQualifiedName(int from, int size, String termQName, + Set attributes, SetrelationAttributes) throws AtlasBaseException { + IndexSearchParams indexSearchParams = new IndexSearchParams(); + Map dsl = getMap("from", from); + dsl.put("size", size); + dsl.put("query", getMap("term", getMap("__meanings", getMap("value",termQName)))); + + indexSearchParams.setDsl(dsl); + indexSearchParams.setAttributes(attributes); + indexSearchParams.setRelationAttributes(relationAttributes); + AtlasSearchResult searchResult = null; + searchResult = directIndexSearch(indexSearchParams); + List entityHeaders = searchResult.getEntities(); + return entityHeaders; + } + + private String getIndexName(IndexSearchParams params) throws AtlasBaseException { + if (StringUtils.isEmpty(params.getPersona()) && StringUtils.isEmpty(params.getPurpose())) { + return VERTEX_INDEX_NAME; + } + + String qualifiedName = ""; + if (StringUtils.isNotEmpty(params.getPersona())) { + qualifiedName = params.getPersona(); + } else { + qualifiedName = params.getPurpose(); + } + + String[] parts = qualifiedName.split("/"); + String aliasName = parts[parts.length - 1]; + + if (StringUtils.isNotEmpty(aliasName)) { + return aliasName; + } else { + throw new AtlasBaseException("ES alias not found for purpose/persona " + params.getPurpose()); + } } } diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java index 7dc4bd01393..084a8908e15 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java @@ -19,20 +19,27 @@ package org.apache.atlas.discovery; +import com.google.common.annotations.VisibleForTesting; import org.apache.atlas.AtlasConfiguration; import org.apache.atlas.AtlasErrorCode; +import org.apache.atlas.GraphTransactionInterceptor; +import org.apache.atlas.RequestContext; import org.apache.atlas.annotation.GraphTransaction; import org.apache.atlas.authorize.AtlasAuthorizationUtils; import org.apache.atlas.authorize.AtlasEntityAccessRequest; import org.apache.atlas.authorize.AtlasPrivilege; +import org.apache.atlas.authorize.AtlasSearchResultScrubRequest; import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.discovery.AtlasSearchResult; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo; import org.apache.atlas.model.instance.AtlasEntityHeader; import org.apache.atlas.model.instance.AtlasObjectId; -import org.apache.atlas.model.lineage.AtlasLineageInfo; +import org.apache.atlas.model.lineage.*; import org.apache.atlas.model.lineage.AtlasLineageInfo.LineageDirection; import org.apache.atlas.model.lineage.AtlasLineageInfo.LineageRelation; +import org.apache.atlas.model.lineage.AtlasLineageOnDemandInfo.LineageInfoOnDemand; +import org.apache.atlas.repository.Constants; import org.apache.atlas.repository.graphdb.AtlasEdge; import org.apache.atlas.repository.graphdb.AtlasEdgeDirection; import org.apache.atlas.repository.graphdb.AtlasGraph; @@ -43,10 +50,12 @@ import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.type.AtlasTypeUtil; import org.apache.atlas.util.AtlasGremlinQueryProvider; +import org.apache.atlas.utils.AtlasPerfMetrics; import org.apache.atlas.v1.model.lineage.SchemaResponse.SchemaDetails; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.tuple.Pair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; @@ -54,90 +63,573 @@ import javax.inject.Inject; import javax.script.ScriptEngine; import javax.script.ScriptException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; +import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import static org.apache.atlas.AtlasClient.DATA_SET_SUPER_TYPE; import static org.apache.atlas.AtlasClient.PROCESS_SUPER_TYPE; import static org.apache.atlas.AtlasErrorCode.INSTANCE_LINEAGE_QUERY_FAILED; -import static org.apache.atlas.model.lineage.AtlasLineageInfo.LineageDirection.BOTH; -import static org.apache.atlas.model.lineage.AtlasLineageInfo.LineageDirection.INPUT; -import static org.apache.atlas.model.lineage.AtlasLineageInfo.LineageDirection.OUTPUT; +import static org.apache.atlas.model.instance.AtlasEntity.Status.DELETED; +import static org.apache.atlas.model.lineage.AtlasLineageInfo.LineageDirection.*; +import static org.apache.atlas.repository.Constants.ACTIVE_STATE_VALUE; import static org.apache.atlas.repository.Constants.RELATIONSHIP_GUID_PROPERTY_KEY; +import static org.apache.atlas.repository.graph.GraphHelper.*; import static org.apache.atlas.repository.graphdb.AtlasEdgeDirection.IN; import static org.apache.atlas.repository.graphdb.AtlasEdgeDirection.OUT; -import static org.apache.atlas.util.AtlasGremlinQueryProvider.AtlasGremlinQuery.FULL_LINEAGE_DATASET; -import static org.apache.atlas.util.AtlasGremlinQueryProvider.AtlasGremlinQuery.FULL_LINEAGE_PROCESS; -import static org.apache.atlas.util.AtlasGremlinQueryProvider.AtlasGremlinQuery.PARTIAL_LINEAGE_DATASET; -import static org.apache.atlas.util.AtlasGremlinQueryProvider.AtlasGremlinQuery.PARTIAL_LINEAGE_PROCESS; +import static org.apache.atlas.util.AtlasGremlinQueryProvider.AtlasGremlinQuery.*; @Service public class EntityLineageService implements AtlasLineageService { private static final Logger LOG = LoggerFactory.getLogger(EntityLineageService.class); - private static final String PROCESS_INPUTS_EDGE = "__Process.inputs"; - private static final String PROCESS_OUTPUTS_EDGE = "__Process.outputs"; - private static final String COLUMNS = "columns"; + private static final String PROCESS_INPUTS_EDGE = "__Process.inputs"; + private static final String PROCESS_OUTPUTS_EDGE = "__Process.outputs"; + private static final String COLUMNS = "columns"; private static final boolean LINEAGE_USING_GREMLIN = AtlasConfiguration.LINEAGE_USING_GREMLIN.getBoolean(); + private static final Integer DEFAULT_LINEAGE_MAX_NODE_COUNT = 9000; + private static final int LINEAGE_ON_DEMAND_DEFAULT_DEPTH = 3; + private static final String SEPARATOR = "->"; - private final AtlasGraph graph; + private final AtlasGraph graph; private final AtlasGremlinQueryProvider gremlinQueryProvider; - private final EntityGraphRetriever entityRetriever; - private final AtlasTypeRegistry atlasTypeRegistry; + private final EntityGraphRetriever entityRetriever; + private final AtlasTypeRegistry atlasTypeRegistry; + private final VertexEdgeCache vertexEdgeCache; @Inject - EntityLineageService(AtlasTypeRegistry typeRegistry, AtlasGraph atlasGraph) { + EntityLineageService(AtlasTypeRegistry typeRegistry, AtlasGraph atlasGraph, VertexEdgeCache vertexEdgeCache) { this.graph = atlasGraph; this.gremlinQueryProvider = AtlasGremlinQueryProvider.INSTANCE; this.entityRetriever = new EntityGraphRetriever(atlasGraph, typeRegistry); this.atlasTypeRegistry = typeRegistry; + this.vertexEdgeCache = vertexEdgeCache; + } + + @VisibleForTesting + EntityLineageService() { + this.graph = null; + this.gremlinQueryProvider = null; + this.entityRetriever = null; + this.atlasTypeRegistry = null; + this.vertexEdgeCache = null; + } + + @Override + public AtlasLineageInfo getAtlasLineageInfo(String guid, LineageDirection direction, int depth, boolean hideProcess, int offset, int limit, boolean calculateRemainingVertexCounts) throws AtlasBaseException { + return getAtlasLineageInfo(new AtlasLineageRequest(guid, depth, direction, hideProcess, offset, limit, calculateRemainingVertexCounts)); } @Override @GraphTransaction - public AtlasLineageInfo getAtlasLineageInfo(String guid, LineageDirection direction, int depth, boolean hideProcess) throws AtlasBaseException { + public AtlasLineageInfo getAtlasLineageInfo(AtlasLineageRequest lineageRequest) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("getAtlasLineageInfo"); + AtlasLineageInfo ret; + String guid = lineageRequest.getGuid(); + AtlasLineageContext lineageRequestContext = new AtlasLineageContext(lineageRequest, atlasTypeRegistry); + RequestContext.get().setRelationAttrsForSearch(lineageRequest.getRelationAttributes()); AtlasEntityHeader entity = entityRetriever.toAtlasEntityHeaderWithClassifications(guid); - AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(atlasTypeRegistry, AtlasPrivilege.ENTITY_READ, entity), "read entity lineage: guid=", guid); - AtlasEntityType entityType = atlasTypeRegistry.getEntityTypeByName(entity.getTypeName()); if (entityType == null) { throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_NOT_FOUND, entity.getTypeName()); } - boolean isDataSet = entityType.getTypeAndAllSuperTypes().contains(DATA_SET_SUPER_TYPE); - - if (!isDataSet) { - boolean isProcess = entityType.getTypeAndAllSuperTypes().contains(PROCESS_SUPER_TYPE); - - if (!isProcess) { - throw new AtlasBaseException(AtlasErrorCode.INVALID_LINEAGE_ENTITY_TYPE, guid, entity.getTypeName()); - } else if (hideProcess) { + boolean isProcess = entityType.getTypeAndAllSuperTypes().contains(PROCESS_SUPER_TYPE); + if (isProcess) { + if (lineageRequest.isHideProcess()) { throw new AtlasBaseException(AtlasErrorCode.INVALID_LINEAGE_ENTITY_TYPE_HIDE_PROCESS, guid, entity.getTypeName()); } + lineageRequestContext.setProcess(true); + }else { + boolean isDataSet = entityType.getTypeAndAllSuperTypes().contains(DATA_SET_SUPER_TYPE); + if (!isDataSet) { + throw new AtlasBaseException(AtlasErrorCode.INVALID_LINEAGE_ENTITY_TYPE, guid, entity.getTypeName()); + } + lineageRequestContext.setDataset(true); } if (LINEAGE_USING_GREMLIN) { - ret = getLineageInfoV1(guid, direction, depth, isDataSet); + ret = getLineageInfoV1(lineageRequestContext); } else { - ret = getLineageInfoV2(guid, direction, depth, isDataSet, hideProcess); + ret = getLineageInfoV2(lineageRequestContext); } + scrubLineageEntities(ret.getGuidEntityMap().values()); + RequestContext.get().endMetricRecord(metric); return ret; } @Override @GraphTransaction public AtlasLineageInfo getAtlasLineageInfo(String guid, LineageDirection direction, int depth) throws AtlasBaseException { - return getAtlasLineageInfo(guid, direction, depth, false); + return getAtlasLineageInfo(guid, direction, depth, false, -1, -1, false); + } + + @Override + @GraphTransaction + public AtlasLineageOnDemandInfo getAtlasLineageInfo(String guid, LineageOnDemandRequest lineageOnDemandRequest) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("getAtlasLineageInfo"); + + RequestContext.get().setRelationAttrsForSearch(lineageOnDemandRequest.getRelationAttributes()); + AtlasLineageOnDemandContext atlasLineageOnDemandContext = new AtlasLineageOnDemandContext(lineageOnDemandRequest, atlasTypeRegistry); + boolean isDataSet = validateEntityTypeAndCheckIfDataSet(guid); + AtlasLineageOnDemandInfo ret = getLineageInfoOnDemand(guid, atlasLineageOnDemandContext, isDataSet); + appendLineageOnDemandPayload(ret, lineageOnDemandRequest); + // filtering out on-demand relations which has input & output nodes within the limit + cleanupRelationsOnDemand(ret); + scrubLineageEntities(ret.getGuidEntityMap().values()); + RequestContext.get().endMetricRecord(metricRecorder); + + return ret; + } + + @Override + @GraphTransaction + public AtlasLineageListInfo getLineageListInfoOnDemand(String guid, LineageListRequest lineageListRequest) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("getLineageListInfoOnDemand"); + + AtlasLineageListInfo ret = new AtlasLineageListInfo(new ArrayList<>()); + traverseEdgesUsingBFS(guid, new AtlasLineageListContext(lineageListRequest, atlasTypeRegistry), ret); + ret.setSearchParameters(lineageListRequest); + + RequestContext.get().endMetricRecord(metricRecorder); + return ret; + } + + private boolean validateEntityTypeAndCheckIfDataSet(String guid) throws AtlasBaseException { + String typeName = entityRetriever.getEntityVertex(guid).getProperty(Constants.TYPE_NAME_PROPERTY_KEY, String.class); + AtlasEntityType entityType = atlasTypeRegistry.getEntityTypeByName(typeName); + if (entityType == null) { + throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_NOT_FOUND, typeName); + } + boolean isProcess = entityType.getTypeAndAllSuperTypes().contains(PROCESS_SUPER_TYPE); + if (!isProcess) { + boolean isDataSet = entityType.getTypeAndAllSuperTypes().contains(DATA_SET_SUPER_TYPE); + if (!isDataSet) { + throw new AtlasBaseException(AtlasErrorCode.INVALID_LINEAGE_ENTITY_TYPE, guid, typeName); + } + } + return !isProcess; + } + + private LineageOnDemandConstraints getLineageConstraints(String guid, LineageOnDemandBaseParams defaultParams) { + if (LOG.isDebugEnabled()) { + LOG.debug("No lineage on-demand constraints provided for guid: {}, configuring with default values direction: {}, inputRelationsLimit: {}, outputRelationsLimit: {}, depth: {}", + guid, BOTH, defaultParams.getInputRelationsLimit(), defaultParams.getOutputRelationsLimit(), LINEAGE_ON_DEMAND_DEFAULT_DEPTH); + } + + return new LineageOnDemandConstraints(defaultParams); + } + + private LineageOnDemandConstraints getAndValidateLineageConstraintsByGuid(String guid, AtlasLineageOnDemandContext context) { + Map lineageConstraintsMap = context.getConstraints(); + LineageOnDemandBaseParams defaultParams = context.getDefaultParams(); + + if (lineageConstraintsMap == null || !lineageConstraintsMap.containsKey(guid)) { + return getLineageConstraints(guid, defaultParams); + } + + LineageOnDemandConstraints lineageConstraintsByGuid = lineageConstraintsMap.get(guid); + if (lineageConstraintsByGuid == null) { + return getLineageConstraints(guid, defaultParams); + } + + if (Objects.isNull(lineageConstraintsByGuid.getDirection())) { + LOG.info("No lineage on-demand direction provided for guid: {}, configuring with default value {}", guid, LineageDirection.BOTH); + lineageConstraintsByGuid.setDirection(AtlasLineageOnDemandInfo.LineageDirection.BOTH); + } + + if (lineageConstraintsByGuid.getInputRelationsLimit() < 0) { + LOG.info("No lineage on-demand constraint inputRelationsLimit provided for guid: {}, configuring with default value {}", guid, context.getDefaultParams().getInputRelationsLimit()); + lineageConstraintsByGuid.setInputRelationsLimit(context.getDefaultParams().getInputRelationsLimit()); + } + + if (lineageConstraintsByGuid.getOutputRelationsLimit() < 0) { + LOG.info("No lineage on-demand constraint outputRelationsLimit provided for guid: {}, configuring with default value {}", guid, context.getDefaultParams().getOutputRelationsLimit()); + lineageConstraintsByGuid.setOutputRelationsLimit(context.getDefaultParams().getOutputRelationsLimit()); + } + + if (lineageConstraintsByGuid.getDepth() == 0) { + LOG.info("No lineage on-demand depth provided for guid: {}, configuring with default value {}", guid, LINEAGE_ON_DEMAND_DEFAULT_DEPTH); + lineageConstraintsByGuid.setDepth(LINEAGE_ON_DEMAND_DEFAULT_DEPTH); + } + + return lineageConstraintsByGuid; + + } + + private void appendLineageOnDemandPayload(AtlasLineageOnDemandInfo lineageInfo, LineageOnDemandRequest lineageOnDemandRequest) { + if (lineageInfo == null) { + return; + } + lineageInfo.setLineageOnDemandPayload(lineageOnDemandRequest); + } + + //Consider only relationsOnDemand which has either more inputs or more outputs than given limit + private void cleanupRelationsOnDemand(AtlasLineageOnDemandInfo lineageInfo) { + if (lineageInfo != null && MapUtils.isNotEmpty(lineageInfo.getRelationsOnDemand())) { + lineageInfo.getRelationsOnDemand().entrySet().removeIf(x -> + !(x.getValue().hasMoreInputs() || x.getValue().hasMoreOutputs() + || x.getValue().hasUpstream() || x.getValue().hasDownstream())); + } + } + + private AtlasLineageOnDemandInfo getLineageInfoOnDemand(String guid, AtlasLineageOnDemandContext atlasLineageOnDemandContext, boolean isDataSet) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("getLineageInfoOnDemand"); + + LineageOnDemandConstraints lineageConstraintsByGuid = getAndValidateLineageConstraintsByGuid(guid, atlasLineageOnDemandContext); + AtlasLineageOnDemandInfo.LineageDirection direction = lineageConstraintsByGuid.getDirection(); + int depth = lineageConstraintsByGuid.getDepth(); + AtlasLineageOnDemandInfo ret = initializeLineageOnDemandInfo(guid); + + if (depth == 0) + depth = -1; + if (!ret.getRelationsOnDemand().containsKey(guid)) + ret.getRelationsOnDemand().put(guid, new LineageInfoOnDemand(lineageConstraintsByGuid)); + + AtomicInteger inputEntitiesTraversed = new AtomicInteger(0); + AtomicInteger outputEntitiesTraversed = new AtomicInteger(0); + if (isDataSet) { + AtlasVertex datasetVertex = AtlasGraphUtilsV2.findByGuid(this.graph, guid); + if (direction == AtlasLineageOnDemandInfo.LineageDirection.INPUT || direction == AtlasLineageOnDemandInfo.LineageDirection.BOTH) + traverseEdgesOnDemand(datasetVertex, true, depth, new HashSet<>(), atlasLineageOnDemandContext, ret, guid, inputEntitiesTraversed); + if (direction == AtlasLineageOnDemandInfo.LineageDirection.OUTPUT || direction == AtlasLineageOnDemandInfo.LineageDirection.BOTH) + traverseEdgesOnDemand(datasetVertex, false, depth, new HashSet<>(), atlasLineageOnDemandContext, ret, guid, outputEntitiesTraversed); + AtlasEntityHeader baseEntityHeader = entityRetriever.toAtlasEntityHeader(datasetVertex, atlasLineageOnDemandContext.getAttributes()); + ret.getGuidEntityMap().put(guid, baseEntityHeader); + } else { + AtlasVertex processVertex = AtlasGraphUtilsV2.findByGuid(this.graph, guid); + // make one hop to the next dataset vertices from process vertex and traverse with 'depth = depth - 1' + if (direction == AtlasLineageOnDemandInfo.LineageDirection.INPUT || direction == AtlasLineageOnDemandInfo.LineageDirection.BOTH) { + Iterator processEdges = processVertex.getEdges(AtlasEdgeDirection.OUT, PROCESS_INPUTS_EDGE).iterator(); + traverseEdgesOnDemand(processEdges, true, depth, atlasLineageOnDemandContext, ret, processVertex, guid, inputEntitiesTraversed); + } + if (direction == AtlasLineageOnDemandInfo.LineageDirection.OUTPUT || direction == AtlasLineageOnDemandInfo.LineageDirection.BOTH) { + Iterator processEdges = processVertex.getEdges(AtlasEdgeDirection.OUT, PROCESS_OUTPUTS_EDGE).iterator(); + traverseEdgesOnDemand(processEdges, false, depth, atlasLineageOnDemandContext, ret, processVertex, guid, outputEntitiesTraversed); + } + } + RequestContext.get().endMetricRecord(metricRecorder); + return ret; + } + + + private void traverseEdgesOnDemand(Iterator processEdges, boolean isInput, int depth, AtlasLineageOnDemandContext atlasLineageOnDemandContext, AtlasLineageOnDemandInfo ret, AtlasVertex processVertex, String baseGuid, AtomicInteger entitiesTraversed) throws AtlasBaseException { + AtlasLineageOnDemandInfo.LineageDirection direction = isInput ? AtlasLineageOnDemandInfo.LineageDirection.INPUT : AtlasLineageOnDemandInfo.LineageDirection.OUTPUT; + while (processEdges.hasNext()) { + AtlasEdge processEdge = processEdges.next(); + AtlasVertex datasetVertex = processEdge.getInVertex(); + + if (!vertexMatchesEvaluation(datasetVertex, atlasLineageOnDemandContext) || !edgeMatchesEvaluation(processEdge, atlasLineageOnDemandContext)) { + continue; + } + + if (checkForOffset(processEdge, processVertex, atlasLineageOnDemandContext, ret)) { + continue; + } + + boolean isInputEdge = processEdge.getLabel().equalsIgnoreCase(PROCESS_INPUTS_EDGE); + if (incrementAndCheckIfRelationsLimitReached(processEdge, isInputEdge, atlasLineageOnDemandContext, ret, depth, baseGuid, direction, entitiesTraversed)) { + break; + } else { + addEdgeToResult(processEdge, ret, atlasLineageOnDemandContext); + } + + String inGuid = AtlasGraphUtilsV2.getIdFromVertex(datasetVertex); + LineageOnDemandConstraints inGuidLineageConstrains = getAndValidateLineageConstraintsByGuid(inGuid, atlasLineageOnDemandContext); + + if (!ret.getRelationsOnDemand().containsKey(inGuid)) { + ret.getRelationsOnDemand().put(inGuid, new LineageInfoOnDemand(inGuidLineageConstrains)); + } + + traverseEdgesOnDemand(datasetVertex, isInput, depth - 1, new HashSet<>(), atlasLineageOnDemandContext, ret, baseGuid, entitiesTraversed); + } + } + + private void traverseEdgesOnDemand(AtlasVertex datasetVertex, boolean isInput, int depth, Set visitedVertices, AtlasLineageOnDemandContext atlasLineageOnDemandContext, AtlasLineageOnDemandInfo ret, String baseGuid, AtomicInteger entitiesTraversed) throws AtlasBaseException { + if (isEntityTraversalLimitReached(entitiesTraversed)) + return; + if (depth != 0) { // base condition of recursion for depth + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("traverseEdgesOnDemand"); + + AtlasLineageOnDemandInfo.LineageDirection direction = isInput ? AtlasLineageOnDemandInfo.LineageDirection.INPUT : AtlasLineageOnDemandInfo.LineageDirection.OUTPUT; + + // keep track of visited vertices to avoid circular loop + visitedVertices.add(getId(datasetVertex)); + + AtlasPerfMetrics.MetricRecorder traverseEdgesOnDemandGetEdgesIn = RequestContext.get().startMetricRecord("traverseEdgesOnDemandGetEdgesIn"); + Iterator incomingEdges = datasetVertex.getEdges(IN, isInput ? PROCESS_OUTPUTS_EDGE : PROCESS_INPUTS_EDGE).iterator(); + RequestContext.get().endMetricRecord(traverseEdgesOnDemandGetEdgesIn); + + while (incomingEdges.hasNext()) { + AtlasEdge incomingEdge = incomingEdges.next(); + AtlasVertex processVertex = incomingEdge.getOutVertex(); + + if (!vertexMatchesEvaluation(processVertex, atlasLineageOnDemandContext) || !edgeMatchesEvaluation(incomingEdge, atlasLineageOnDemandContext)) { + continue; + } + + if (checkForOffset(incomingEdge, datasetVertex, atlasLineageOnDemandContext, ret)) { + continue; + } + + if (incrementAndCheckIfRelationsLimitReached(incomingEdge, !isInput, atlasLineageOnDemandContext, ret, depth, baseGuid, direction, entitiesTraversed)) { + break; + } else { + addEdgeToResult(incomingEdge, ret, atlasLineageOnDemandContext); + } + + AtlasPerfMetrics.MetricRecorder traverseEdgesOnDemandGetEdgesOut = RequestContext.get().startMetricRecord("traverseEdgesOnDemandGetEdgesOut"); + Iterator outgoingEdges = processVertex.getEdges(OUT, isInput ? PROCESS_INPUTS_EDGE : PROCESS_OUTPUTS_EDGE).iterator(); + RequestContext.get().endMetricRecord(traverseEdgesOnDemandGetEdgesOut); + + while (outgoingEdges.hasNext()) { + AtlasEdge outgoingEdge = outgoingEdges.next(); + AtlasVertex entityVertex = outgoingEdge.getInVertex(); + + if (!vertexMatchesEvaluation(entityVertex, atlasLineageOnDemandContext) || !edgeMatchesEvaluation(outgoingEdge, atlasLineageOnDemandContext)) { + continue; + } + + if (checkForOffset(outgoingEdge, processVertex, atlasLineageOnDemandContext, ret)) { + continue; + } + if (incrementAndCheckIfRelationsLimitReached(outgoingEdge, isInput, atlasLineageOnDemandContext, ret, depth, baseGuid, direction, entitiesTraversed)) { + break; + } else { + addEdgeToResult(outgoingEdge, ret, atlasLineageOnDemandContext); + entitiesTraversed.incrementAndGet(); + if (isEntityTraversalLimitReached(entitiesTraversed)) + setEntityLimitReachedFlag(isInput, ret); + } + if (entityVertex != null && !visitedVertices.contains(getId(entityVertex))) { + traverseEdgesOnDemand(entityVertex, isInput, depth - 1, visitedVertices, atlasLineageOnDemandContext, ret, baseGuid, entitiesTraversed); // execute inner depth + } + } + } + + RequestContext.get().endMetricRecord(metricRecorder); + } + } + + private static void setEntityLimitReachedFlag(boolean isInput, AtlasLineageOnDemandInfo ret) { + if (isInput) + ret.setUpstreamEntityLimitReached(true); + else + ret.setDownstreamEntityLimitReached(true); + } + + private void traverseEdgesUsingBFS(String baseGuid, AtlasLineageListContext lineageListContext, AtlasLineageListInfo ret) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("traverseEdgesUsingBFS"); + + Set visitedVertices = new HashSet<>(); + visitedVertices.add(baseGuid); + Set skippedVertices = new HashSet<>(); + Queue traversalQueue = new LinkedList<>(); + + AtlasVertex baseVertex = AtlasGraphUtilsV2.findByGuid(this.graph, baseGuid); + enqueueNeighbours(baseVertex, validateEntityTypeAndCheckIfDataSet(baseGuid), lineageListContext, traversalQueue, visitedVertices, skippedVertices); + int currentDepth = 0; + + while (!traversalQueue.isEmpty() && !lineageListContext.isEntityLimitReached() && currentDepth < lineageListContext.getDepth()) { + currentDepth++; + int entitiesInCurrentDepth = traversalQueue.size(); + for (int i = 0; i < entitiesInCurrentDepth; i++) { + if (lineageListContext.isEntityLimitReached()) + break; + + String currentGUID = traversalQueue.poll(); + AtlasVertex currentVertex = AtlasGraphUtilsV2.findByGuid(this.graph, currentGUID); + if (Objects.isNull(currentVertex)) + throw new AtlasBaseException("Found null vertex during lineage graph traversal for guid: " + currentGUID); + + boolean isDataset = validateEntityTypeAndCheckIfDataSet(currentGUID); + if (!lineageListContext.evaluateVertexFilter(currentVertex)) { + enqueueNeighbours(currentVertex, isDataset, lineageListContext, traversalQueue, visitedVertices, skippedVertices); + continue; + } + if (checkOffsetAndSkipEntity(lineageListContext, ret)) { + skippedVertices.add(currentGUID); + enqueueNeighbours(currentVertex, isDataset, lineageListContext, traversalQueue, visitedVertices, skippedVertices); + continue; + } + + lineageListContext.incrementEntityCount(); + appendToResult(currentVertex, lineageListContext, ret); + enqueueNeighbours(currentVertex, isDataset, lineageListContext, traversalQueue, visitedVertices, skippedVertices); + if (isLastEntityInLastDepth(lineageListContext.getDepth(), currentDepth, entitiesInCurrentDepth, i)) { + ret.setHasMore(false); + lineageListContext.setHasMoreUpdated(true); + } + } + } + if (currentDepth > lineageListContext.getDepth()) + lineageListContext.setDepthLimitReached(true); + + setPageMetadata(lineageListContext, ret, traversalQueue); + RequestContext.get().endMetricRecord(metricRecorder); + } + + private void enqueueNeighbours(AtlasVertex currentVertex, boolean isDataset, AtlasLineageListContext lineageListContext, + Queue traversalQueue, Set visitedVertices, Set skippedVertices) { + AtlasPerfMetrics.MetricRecorder traverseEdgesOnDemandGetEdges = RequestContext.get().startMetricRecord("traverseEdgesOnDemandGetEdges"); + Iterator edges; + if (isDataset) + edges = currentVertex.getEdges(IN, isInputDirection(lineageListContext) ? PROCESS_OUTPUTS_EDGE : PROCESS_INPUTS_EDGE).iterator(); + else + edges = currentVertex.getEdges(OUT, isInputDirection(lineageListContext) ? PROCESS_INPUTS_EDGE : PROCESS_OUTPUTS_EDGE).iterator(); + RequestContext.get().endMetricRecord(traverseEdgesOnDemandGetEdges); + + while (edges.hasNext()) { + AtlasEdge currentEdge = edges.next(); + if (!lineageListContext.evaluateTraversalFilter(currentEdge)) + continue; + AtlasVertex neighbourVertex; + if (isDataset) + neighbourVertex = currentEdge.getOutVertex(); + else + neighbourVertex = currentEdge.getInVertex(); + + String vertexGuid = getGuid(neighbourVertex); + if (StringUtils.isEmpty(vertexGuid) || !lineageListContext.evaluateTraversalFilter(neighbourVertex)) + continue; + + if (!skippedVertices.contains(vertexGuid) && !visitedVertices.contains(vertexGuid)) { + visitedVertices.add(vertexGuid); + traversalQueue.add(vertexGuid); + addEntitiesToCache(neighbourVertex); + } + } + } + + private void appendToResult(AtlasVertex currentVertex, AtlasLineageListContext lineageListContext, AtlasLineageListInfo ret) throws AtlasBaseException { + ret.getEntities().add(entityRetriever.toAtlasEntityHeader(currentVertex, lineageListContext.getAttributes())); + } + + private static void addEntitiesToCache(AtlasVertex vertex) { + GraphTransactionInterceptor.addToVertexCache(getGuid(vertex), vertex); + } + + private static void setPageMetadata(AtlasLineageListContext lineageListContext, AtlasLineageListInfo ret, Queue traversalQueue) { + if (!lineageListContext.isHasMoreUpdated()) + updateHasMore(lineageListContext, ret, traversalQueue); + ret.setEntityCount(lineageListContext.getCurrentEntityCounter()); + } + + private static void updateHasMore(AtlasLineageListContext lineageListContext, AtlasLineageListInfo ret, Queue traversalQueue) { + if (!traversalQueue.isEmpty()) + ret.setHasMore(true); + if (lineageListContext.isDepthLimitReached()) + ret.setHasMore(false); + } + + private static boolean isLastEntityInLastDepth(int lastDepth, int currentDepth, int entitiesInCurrentDepth, int entityIndexInCurrentDepth) { + return entityIndexInCurrentDepth == entitiesInCurrentDepth - 1 && currentDepth == lastDepth; + } + + private static boolean isInputDirection(AtlasLineageListContext lineageListContext) { + return LineageListRequest.LineageDirection.INPUT.equals(lineageListContext.getDirection()); + } + + private boolean checkForOffset(AtlasEdge atlasEdge, AtlasVertex entityVertex, AtlasLineageOnDemandContext atlasLineageOnDemandContext, AtlasLineageOnDemandInfo ret) { + String entityGuid = getGuid(entityVertex); + LineageOnDemandConstraints entityConstraints = getAndValidateLineageConstraintsByGuid(entityGuid, atlasLineageOnDemandContext); + LineageInfoOnDemand entityLineageInfo = ret.getRelationsOnDemand().containsKey(entityGuid) ? ret.getRelationsOnDemand().get(entityGuid) : new LineageInfoOnDemand(entityConstraints); + + if (entityConstraints.getFrom() != 0 && entityLineageInfo.getFromCounter() < entityConstraints.getFrom()) { + if (! lineageContainsSkippedEdgeV2(ret, atlasEdge)) { + addEdgeToSkippedEdges(ret, atlasEdge); + entityLineageInfo.incrementFromCounter(); + } + return true; + } + return false; + } + + private boolean checkOffsetAndSkipEntity(AtlasLineageListContext atlasLineageListContext, AtlasLineageListInfo ret) { + if (atlasLineageListContext.getFrom() != 0 && atlasLineageListContext.getCurrentFromCounter() < atlasLineageListContext.getFrom()) { + atlasLineageListContext.incrementCurrentFromCounter(); + return true; + } + return false; + } + + private static String getId(AtlasVertex vertex) { + return vertex.getIdForDisplay(); + } + + private boolean incrementAndCheckIfRelationsLimitReached(AtlasEdge atlasEdge, boolean isInput, AtlasLineageOnDemandContext atlasLineageOnDemandContext, AtlasLineageOnDemandInfo ret, int depth, String baseGuid, AtlasLineageOnDemandInfo.LineageDirection direction, AtomicInteger entitiesTraversed) { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("incrementAndCheckIfRelationsLimitReached"); + + boolean hasRelationsLimitReached = false; + + AtlasVertex inVertex = isInput ? atlasEdge.getOutVertex() : atlasEdge.getInVertex(); + String inGuid = AtlasGraphUtilsV2.getIdFromVertex(inVertex); + LineageOnDemandConstraints inGuidLineageConstraints = getAndValidateLineageConstraintsByGuid(inGuid, atlasLineageOnDemandContext); + + AtlasVertex outVertex = isInput ? atlasEdge.getInVertex() : atlasEdge.getOutVertex(); + String outGuid = AtlasGraphUtilsV2.getIdFromVertex(outVertex); + LineageOnDemandConstraints outGuidLineageConstraints = getAndValidateLineageConstraintsByGuid(outGuid, atlasLineageOnDemandContext); + + boolean selfCyclic = AtlasLineageOnDemandInfo.LineageDirection.OUTPUT.equals(direction) && baseGuid.equals(outGuid) && isSelfCyclic(ret, inGuid, outGuid); + boolean skipIncrement = AtlasLineageOnDemandInfo.LineageDirection.INPUT.equals(direction) && baseGuid.equals(outGuid); + + if (lineageContainsVisitedEdgeV2(ret, atlasEdge) && !selfCyclic) { + return false; + } + + // Keep track of already visited vertices for horizontal pagination to not process it again + boolean isOutVertexVisited = ret.getRelationsOnDemand().containsKey(outGuid); + boolean isInVertexVisited = ret.getRelationsOnDemand().containsKey(inGuid); + + LineageInfoOnDemand inLineageInfo = ret.getRelationsOnDemand().containsKey(inGuid) ? ret.getRelationsOnDemand().get(inGuid) : new LineageInfoOnDemand(inGuidLineageConstraints); + LineageInfoOnDemand outLineageInfo = ret.getRelationsOnDemand().containsKey(outGuid) ? ret.getRelationsOnDemand().get(outGuid) : new LineageInfoOnDemand(outGuidLineageConstraints); + + if (inLineageInfo.isInputRelationsReachedLimit() || isEntityTraversalLimitReached(entitiesTraversed)) { + inLineageInfo.setHasMoreInputs(true); + hasRelationsLimitReached = true; + }else { + inLineageInfo.incrementInputRelationsCount(); + } + + if (outLineageInfo.isOutputRelationsReachedLimit() || isEntityTraversalLimitReached(entitiesTraversed)) { + outLineageInfo.setHasMoreOutputs(true); + hasRelationsLimitReached = true; + } else if (! skipIncrement) { + outLineageInfo.incrementOutputRelationsCount(); + } + + // Handle horizontal pagination + if (depth == 1 || entitiesTraversed.get() == getLineageMaxNodeAllowedCount()-1) { // is the vertex a leaf? + if (isInput && ! isOutVertexVisited) { + outLineageInfo.setHasUpstream(outVertex.getEdges(IN, PROCESS_OUTPUTS_EDGE).iterator().hasNext()); + } else if (! isInput && ! isInVertexVisited) { + inLineageInfo.setHasDownstream(inVertex.getEdges(IN, PROCESS_INPUTS_EDGE).iterator().hasNext()); + } + } + + if (!hasRelationsLimitReached) { + ret.getRelationsOnDemand().put(inGuid, inLineageInfo); + ret.getRelationsOnDemand().put(outGuid, outLineageInfo); + } + RequestContext.get().endMetricRecord(metricRecorder); + + return hasRelationsLimitReached; + } + + private boolean isEntityTraversalLimitReached(AtomicInteger entitiesTraversed) { + return entitiesTraversed.get() == getLineageMaxNodeAllowedCount(); + } + + private boolean isSelfCyclic(AtlasLineageOnDemandInfo ret, String inGuid, String outGuid) { + return ret.getRelations().stream().anyMatch(r -> r.getFromEntityId().equals(inGuid)) && + ret.getRelations().stream().anyMatch(r -> r.getToEntityId().equals(outGuid)) && + ret.getRelations().stream().anyMatch(r -> r.getFromEntityId().equals(outGuid)) && + ret.getRelations().stream().anyMatch(r -> r.getToEntityId().equals(inGuid)); } @Override @@ -169,29 +661,40 @@ public SchemaDetails getSchemaForHiveTableByGuid(final String guid) throws Atlas ret.setDataType(AtlasTypeUtil.toClassTypeDefinition(hive_column)); AtlasEntityWithExtInfo entityWithExtInfo = entityRetriever.toAtlasEntityWithExtInfo(guid); - AtlasEntity entity = entityWithExtInfo.getEntity(); + AtlasEntity entity = entityWithExtInfo.getEntity(); AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(atlasTypeRegistry, AtlasPrivilege.ENTITY_READ, new AtlasEntityHeader(entity)), - "read entity schema: guid=", guid); + "read entity schema: guid=", guid); Map referredEntities = entityWithExtInfo.getReferredEntities(); - List columnIds = getColumnIds(entity); + List columnIds = getColumnIds(entity); if (MapUtils.isNotEmpty(referredEntities)) { List> rows = referredEntities.entrySet() - .stream() - .filter(e -> isColumn(columnIds, e)) - .map(e -> AtlasTypeUtil.toMap(e.getValue())) - .collect(Collectors.toList()); + .stream() + .filter(e -> isColumn(columnIds, e)) + .map(e -> AtlasTypeUtil.toMap(e.getValue())) + .collect(Collectors.toList()); ret.setRows(rows); } return ret; } + private void scrubLineageEntities(Collection entityHeaders) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("scrubLineageEntities"); + + AtlasSearchResult searchResult = new AtlasSearchResult(); + searchResult.setEntities(new ArrayList<>(entityHeaders)); + AtlasSearchResultScrubRequest request = new AtlasSearchResultScrubRequest(atlasTypeRegistry, searchResult); + + AtlasAuthorizationUtils.scrubSearchResults(request, true); + RequestContext.get().endMetricRecord(metricRecorder); + } + private List getColumnIds(AtlasEntity entity) { - List ret = new ArrayList<>(); - Object columnObjs = entity.getAttribute(COLUMNS); + List ret = new ArrayList<>(); + Object columnObjs = entity.getAttribute(COLUMNS); if (columnObjs instanceof List) { for (Object pkObj : (List) columnObjs) { @@ -208,162 +711,500 @@ private boolean isColumn(List columnIds, Map.Entry return columnIds.contains(e.getValue().getGuid()); } - private AtlasLineageInfo getLineageInfoV1(String guid, LineageDirection direction, int depth, boolean isDataSet) throws AtlasBaseException { + private AtlasLineageInfo getLineageInfoV1(AtlasLineageContext lineageContext) throws AtlasBaseException { AtlasLineageInfo ret; + LineageDirection direction = lineageContext.getDirection(); if (direction.equals(INPUT)) { - ret = getLineageInfo(guid, INPUT, depth, isDataSet); + ret = getLineageInfo(lineageContext, INPUT); } else if (direction.equals(OUTPUT)) { - ret = getLineageInfo(guid, OUTPUT, depth, isDataSet); + ret = getLineageInfo(lineageContext, OUTPUT); } else { - ret = getBothLineageInfoV1(guid, depth, isDataSet); + ret = getBothLineageInfoV1(lineageContext); } return ret; } - private AtlasLineageInfo getLineageInfo(String guid, LineageDirection direction, int depth, boolean isDataSet) throws AtlasBaseException { - final Map bindings = new HashMap<>(); - String lineageQuery = getLineageQuery(guid, direction, depth, isDataSet, bindings); - List results = executeGremlinScript(bindings, lineageQuery); - Map entities = new HashMap<>(); - Set relations = new HashSet<>(); + private AtlasLineageInfo getLineageInfo(AtlasLineageContext lineageContext, LineageDirection direction) throws AtlasBaseException { + int depth = lineageContext.getDepth(); + String guid = lineageContext.getGuid(); + boolean isDataSet = lineageContext.isDataset(); + + final Map bindings = new HashMap<>(); + String lineageQuery = getLineageQuery(guid, direction, depth, isDataSet, bindings); + List results = executeGremlinScript(bindings, lineageQuery); + Map entities = new HashMap<>(); + Set relations = new HashSet<>(); if (CollectionUtils.isNotEmpty(results)) { for (Object result : results) { if (result instanceof Map) { for (final Object o : ((Map) result).entrySet()) { final Map.Entry entry = (Map.Entry) o; - Object value = entry.getValue(); + Object value = entry.getValue(); if (value instanceof List) { for (Object elem : (List) value) { if (elem instanceof AtlasEdge) { - processEdge((AtlasEdge) elem, entities, relations); + processEdge((AtlasEdge) elem, entities, relations, lineageContext); } else { LOG.warn("Invalid value of type {} found, ignoring", (elem != null ? elem.getClass().getSimpleName() : "null")); } } } else if (value instanceof AtlasEdge) { - processEdge((AtlasEdge) value, entities, relations); + processEdge((AtlasEdge) value, entities, relations, lineageContext); } else { LOG.warn("Invalid value of type {} found, ignoring", (value != null ? value.getClass().getSimpleName() : "null")); } } } else if (result instanceof AtlasEdge) { - processEdge((AtlasEdge) result, entities, relations); + processEdge((AtlasEdge) result, entities, relations, lineageContext); } } } - return new AtlasLineageInfo(guid, entities, relations, direction, depth); + return new AtlasLineageInfo(guid, entities, relations, direction, depth, -1, -1); } - private AtlasLineageInfo getLineageInfoV2(String guid, LineageDirection direction, int depth, boolean isDataSet, boolean hideProcess) throws AtlasBaseException { - AtlasLineageInfo ret = initializeLineageInfo(guid, direction, depth); + private AtlasLineageInfo getLineageInfoV2(AtlasLineageContext lineageContext) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("getLineageInfoV2"); + + int depth = lineageContext.getDepth(); + String guid = lineageContext.getGuid(); + LineageDirection direction = lineageContext.getDirection(); + + AtlasLineageInfo ret = initializeLineageInfo(guid, direction, depth, lineageContext.getLimit(), lineageContext.getOffset()); if (depth == 0) { depth = -1; } - if (isDataSet) { + if (lineageContext.isDataset()) { AtlasVertex datasetVertex = AtlasGraphUtilsV2.findByGuid(this.graph, guid); + lineageContext.setStartDatasetVertex(datasetVertex); if (direction == INPUT || direction == BOTH) { - traverseEdges(datasetVertex, true, depth, ret, hideProcess); + traverseEdges(datasetVertex, true, depth, new HashSet<>(), ret, lineageContext); } if (direction == OUTPUT || direction == BOTH) { - traverseEdges(datasetVertex, false, depth, ret, hideProcess); + traverseEdges(datasetVertex, false, depth, new HashSet<>(), ret, lineageContext); } - } else { + } else { AtlasVertex processVertex = AtlasGraphUtilsV2.findByGuid(this.graph, guid); // make one hop to the next dataset vertices from process vertex and traverse with 'depth = depth - 1' if (direction == INPUT || direction == BOTH) { - Iterable processEdges = processVertex.getEdges(AtlasEdgeDirection.OUT, PROCESS_INPUTS_EDGE); + Iterator processEdges = vertexEdgeCache.getEdges(processVertex, OUT, PROCESS_INPUTS_EDGE).iterator(); - for (AtlasEdge processEdge : processEdges) { - addEdgeToResult(processEdge, ret); + List qualifyingEdges = getQualifyingProcessEdges(processEdges, lineageContext); + ret.setHasChildrenForDirection(getGuid(processVertex), new LineageChildrenInfo(INPUT, hasMoreChildren(qualifyingEdges))); + + for (AtlasEdge processEdge : qualifyingEdges) { + addEdgeToResult(processEdge, ret, lineageContext); AtlasVertex datasetVertex = processEdge.getInVertex(); - traverseEdges(datasetVertex, true, depth - 1, ret); + traverseEdges(datasetVertex, true, depth - 1, new HashSet<>(), ret, lineageContext); } } if (direction == OUTPUT || direction == BOTH) { - Iterable processEdges = processVertex.getEdges(AtlasEdgeDirection.OUT, PROCESS_OUTPUTS_EDGE); + Iterator processEdges = vertexEdgeCache.getEdges(processVertex, OUT, PROCESS_OUTPUTS_EDGE).iterator(); + + List qualifyingEdges = getQualifyingProcessEdges(processEdges, lineageContext); + ret.setHasChildrenForDirection(getGuid(processVertex), new LineageChildrenInfo(OUTPUT, hasMoreChildren(qualifyingEdges))); - for (AtlasEdge processEdge : processEdges) { - addEdgeToResult(processEdge, ret); + for (AtlasEdge processEdge : qualifyingEdges) { + addEdgeToResult(processEdge, ret, lineageContext); AtlasVertex datasetVertex = processEdge.getInVertex(); - traverseEdges(datasetVertex, false, depth - 1, ret); + traverseEdges(datasetVertex, false, depth - 1, new HashSet<>(), ret, lineageContext); } } } + RequestContext.get().endMetricRecord(metric); return ret; } - private void traverseEdges(AtlasVertex datasetVertex, boolean isInput, int depth, AtlasLineageInfo ret, boolean hideProcess) throws AtlasBaseException { - traverseEdges(datasetVertex, isInput, depth, new HashSet<>(), ret, hideProcess); + private List getQualifyingProcessEdges(Iterator processEdges, AtlasLineageContext lineageContext) { + AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("getQualifyingProcessEdges"); + List qualifyingEdges = new ArrayList<>(); + while (processEdges.hasNext()) { + AtlasEdge processEdge = processEdges.next(); + if (shouldProcessEdge(lineageContext, processEdge) && lineageContext.evaluate(processEdge.getInVertex())) { + qualifyingEdges.add(processEdge); + } + } + RequestContext.get().endMetricRecord(metric); + return qualifyingEdges; + } + + private void addEdgeToResult(AtlasEdge edge, AtlasLineageInfo lineageInfo, + AtlasLineageContext requestContext) throws AtlasBaseException { + if (!lineageContainsEdge(lineageInfo, edge)) { + processEdge(edge, lineageInfo, requestContext); + } } - private void traverseEdges(AtlasVertex datasetVertex, boolean isInput, int depth, AtlasLineageInfo ret) throws AtlasBaseException { - traverseEdges(datasetVertex, isInput, depth, new HashSet<>(), ret, false); + private void addEdgeToResult(AtlasEdge edge, AtlasLineageOnDemandInfo lineageInfo, AtlasLineageOnDemandContext atlasLineageOnDemandContext) throws AtlasBaseException { + if (!lineageContainsVisitedEdgeV2(lineageInfo, edge)) { + processEdge(edge, lineageInfo, atlasLineageOnDemandContext); + } } - private void traverseEdges(AtlasVertex datasetVertex, boolean isInput, int depth, Set visitedVertices, AtlasLineageInfo ret, boolean hideProcess) throws AtlasBaseException { + private int getLineageMaxNodeAllowedCount() { + return AtlasConfiguration.LINEAGE_MAX_NODE_COUNT.getInt(); + } + + private String getEdgeLabel(AtlasEdge edge) { + AtlasVertex inVertex = edge.getInVertex(); + AtlasVertex outVertex = edge.getOutVertex(); + String inGuid = AtlasGraphUtilsV2.getIdFromVertex(inVertex); + String outGuid = AtlasGraphUtilsV2.getIdFromVertex(outVertex); + String relationGuid = AtlasGraphUtilsV2.getEncodedProperty(edge, RELATIONSHIP_GUID_PROPERTY_KEY, String.class); + boolean isInputEdge = edge.getLabel().equalsIgnoreCase(PROCESS_INPUTS_EDGE); + + if (isLineageOnDemandEnabled()) { + return getEdgeLabelFromGuids(isInputEdge, inGuid, outGuid); + } + return relationGuid; + } + + private String getEdgeLabelFromGuids(boolean isInputEdge, String inGuid, String outGuid) { + return isInputEdge ? inGuid + SEPARATOR + outGuid : outGuid + SEPARATOR + inGuid; + } + + private boolean hasMoreChildren(List edges) { + return edges.stream().anyMatch(edge -> getStatus(edge) == AtlasEntity.Status.ACTIVE); + } + + private void traverseEdges(AtlasVertex currentVertex, boolean isInput, int depth, Set visitedVertices, AtlasLineageInfo ret, + AtlasLineageContext lineageContext) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("traverseEdges"); if (depth != 0) { - // keep track of visited vertices to avoid circular loop - visitedVertices.add(getId(datasetVertex)); + processIntermediateLevel(currentVertex, isInput, depth, visitedVertices, ret, lineageContext); + } else { + processLastLevel(currentVertex, isInput, ret, lineageContext); + } + RequestContext.get().endMetricRecord(metric); + } - Iterable incomingEdges = datasetVertex.getEdges(IN, isInput ? PROCESS_OUTPUTS_EDGE : PROCESS_INPUTS_EDGE); + private void processIntermediateLevel(AtlasVertex currentVertex, + boolean isInput, + int depth, + Set visitedVertices, + AtlasLineageInfo ret, + AtlasLineageContext lineageContext) throws AtlasBaseException { + // keep track of visited vertices to avoid circular loop + visitedVertices.add(currentVertex.getIdForDisplay()); + + if (!vertexMatchesEvaluation(currentVertex, lineageContext)) { + return; + } + List currentVertexEdges = getEdgesOfCurrentVertex(currentVertex, isInput, lineageContext); + if (lineageContext.shouldApplyPagination()) { + if (lineageContext.isCalculateRemainingVertexCounts()) { + calculateRemainingVertexCounts(currentVertex, isInput, ret); + } + addPaginatedVerticesToResult(isInput, depth, visitedVertices, ret, lineageContext, currentVertexEdges, currentVertex); + } else { + addLimitlessVerticesToResult(isInput, depth, visitedVertices, ret, lineageContext, currentVertexEdges); + } + } - for (AtlasEdge incomingEdge : incomingEdges) { - AtlasVertex processVertex = incomingEdge.getOutVertex(); - Iterable outgoingEdges = processVertex.getEdges(OUT, isInput ? PROCESS_INPUTS_EDGE : PROCESS_OUTPUTS_EDGE); + private void calculateRemainingVertexCounts(AtlasVertex currentVertex, boolean isInput, AtlasLineageInfo ret) { + if (isInput) { + Long totalUpstreamVertexCount = getTotalUpstreamVertexCount(getGuid(currentVertex)); + ret.calculateRemainingUpstreamVertexCount(totalUpstreamVertexCount); + } else { + Long totalDownstreamVertexCount = getTotalDownstreamVertexCount(getGuid(currentVertex)); + ret.calculateRemainingDownstreamVertexCount(totalDownstreamVertexCount); + } + } - for (AtlasEdge outgoingEdge : outgoingEdges) { - AtlasVertex entityVertex = outgoingEdge.getInVertex(); + private Long getTotalDownstreamVertexCount(String guid) { + return (Long) graph + .V() + .has("__guid", guid) + .inE("__Process.inputs").has("__state", "ACTIVE") + .outV().has("__state", "ACTIVE") + .outE("__Process.outputs").has("__state", "ACTIVE") + .inV() + .count() + .next(); - if (entityVertex != null) { - if (hideProcess) { - addVirtualEdgeToResult(incomingEdge, outgoingEdge, ret); - } else { - addEdgeToResult(incomingEdge, ret); - addEdgeToResult(outgoingEdge, ret); - } + } - if (!visitedVertices.contains(getId(entityVertex))) { - traverseEdges(entityVertex, isInput, depth - 1, visitedVertices, ret, hideProcess); - } + private Long getTotalUpstreamVertexCount(String guid) { + return (Long) graph + .V() + .has("__guid", guid) + .outE("__Process.outputs").has("__state", "ACTIVE") + .inV().has("__state", "ACTIVE") + .inE("__Process.inputs").has("__state", "ACTIVE") + .inV() + .count() + .next(); + } + + private void addPaginatedVerticesToResult(boolean isInput, + int depth, + Set visitedVertices, + AtlasLineageInfo ret, + AtlasLineageContext lineageContext, + List currentVertexEdges, + AtlasVertex currentVertex) throws AtlasBaseException { + Set> paginationCalculatedProcessOutputPair = new HashSet<>(); + long inputVertexCount = !isInput ? nonProcessEntityCount(ret) : 0; + int currentOffset = lineageContext.getOffset(); + boolean isFirstValidProcessReached = false; + for (int i = 0; i < currentVertexEdges.size(); i++) { + AtlasEdge edge = currentVertexEdges.get(i); + AtlasVertex processVertex = edge.getOutVertex(); + paginationCalculatedProcessOutputPair.add(Pair.of(getGuid(processVertex), getGuid(currentVertex))); + if (!shouldProcessDeletedProcess(lineageContext, processVertex) || getStatus(edge) == DELETED) { + continue; + } + + List edgesOfProcess = getEdgesOfProcess(isInput, lineageContext, paginationCalculatedProcessOutputPair, processVertex, currentOffset); + + if (isFirstValidProcessReached) + currentOffset = 0; + if (edgesOfProcess.size() > currentOffset) { + isFirstValidProcessReached = true; + ret.setHasChildrenForDirection(getGuid(processVertex), new LineageChildrenInfo(isInput ? INPUT : OUTPUT, hasMoreChildren(edgesOfProcess))); + boolean isLimitReached = executeCurrentProcessVertex(isInput, depth, visitedVertices, ret, lineageContext, currentVertexEdges, inputVertexCount, currentOffset, i, edge, edgesOfProcess); + if (isLimitReached) + return; + } else + currentOffset -= edgesOfProcess.size(); + } + } + + private List getEdgesOfProcess(boolean isInput, AtlasLineageContext lineageContext, Set> paginationCalculatedProcessOutputPair, AtlasVertex processVertex, int currentOffset) { + List> processEdgeOutputVertexIdPairs = getUnvisitedProcessEdgesWithOutputVertexIds(isInput, lineageContext, paginationCalculatedProcessOutputPair, processVertex, currentOffset); + processEdgeOutputVertexIdPairs.forEach(pair -> paginationCalculatedProcessOutputPair.add(Pair.of(getGuid(processVertex), pair.getRight()))); + return processEdgeOutputVertexIdPairs + .stream() + .map(Pair::getLeft) + .collect(Collectors.toList()); + } + + private boolean executeCurrentProcessVertex(boolean isInput, + int depth, + Set visitedVertices, + AtlasLineageInfo ret, + AtlasLineageContext lineageContext, + List currentVertexEdges, + long inputVertexCount, int currentOffset, int vertexEdgeIndex, + AtlasEdge edge, + List edgesOfProcess) throws AtlasBaseException { + for (int j = currentOffset; j < edgesOfProcess.size(); j++) { + AtlasEdge edgeOfProcess = edgesOfProcess.get(j); + AtlasVertex entityVertex = edgeOfProcess.getInVertex(); + if (entityVertex == null) { + continue; + } + if (shouldTerminate(isInput, ret, lineageContext, currentVertexEdges, inputVertexCount, vertexEdgeIndex, edgesOfProcess, j)) { + return true; + } + if (!visitedVertices.contains(entityVertex.getIdForDisplay())) { + traverseEdges(entityVertex, isInput, depth - 1, visitedVertices, ret, lineageContext); + } + if (lineageContext.isHideProcess()) { + processVirtualEdge(edge, edgeOfProcess, ret, lineageContext); + } else { + processEdges(edge, edgeOfProcess, ret, lineageContext); + } + } + return false; + } + + private boolean shouldProcessDeletedProcess(AtlasLineageContext lineageContext, AtlasVertex processVertex) { + return isVertexActive(processVertex) || lineageContext.isAllowDeletedProcess(); + } + + private boolean isVertexActive(AtlasVertex vertex) { + return getStatus(vertex) == AtlasEntity.Status.ACTIVE; + } + + private List> getUnvisitedProcessEdgesWithOutputVertexIds(boolean isInput, AtlasLineageContext lineageContext, Set> paginationCalculatedProcessOutputPair, AtlasVertex processVertex, int currentOffset) { + if (lineageContext.getIgnoredProcesses() != null && + lineageContext.getIgnoredProcesses().contains(processVertex.getProperty(Constants.ENTITY_TYPE_PROPERTY_KEY, String.class))) { + return Collections.emptyList(); + } + + List> unvisitedProcessEdgesWithOutputVertexIds = new ArrayList<>(); + + Iterable outgoingEdges = vertexEdgeCache.getEdges(processVertex, OUT, isInput ? PROCESS_INPUTS_EDGE : PROCESS_OUTPUTS_EDGE); + + for (AtlasEdge outgoingEdge : outgoingEdges) { + AtlasVertex outputVertex = outgoingEdge.getInVertex(); + if (outputVertex != null && + shouldProcessEdge(lineageContext, outgoingEdge) && + vertexMatchesEvaluation(outputVertex, lineageContext) && + !paginationCalculatedProcessOutputPair.contains(Pair.of(getGuid(processVertex), outputVertex.getIdForDisplay()))) { + unvisitedProcessEdgesWithOutputVertexIds.add(Pair.of(outgoingEdge, outputVertex.getIdForDisplay())); + if (unvisitedProcessEdgesWithOutputVertexIds.size() == lineageContext.getLimit() + currentOffset + 1) { // +1 is required for downstream check while trying to terminate the loop before it ends + break; + } + } + } + + return unvisitedProcessEdgesWithOutputVertexIds; + } + + @VisibleForTesting + boolean shouldTerminate(boolean isInput, + AtlasLineageInfo ret, + AtlasLineageContext lineageContext, + List currentVertexEdges, + long inputVertexCount, + int currentVertexEdgeIndex, + List edgesOfProcess, + int processEdgeIndex) { + if (lineageContext.getDirection() == BOTH) { + if (isInput && nonProcessEntityCount(ret) == lineageContext.getLimit()) { + ret.setHasMoreUpstreamVertices(true); + return true; + } else if (!isInput && nonProcessEntityCount(ret) - inputVertexCount == lineageContext.getLimit()) { + ret.setHasMoreDownstreamVertices(true); + return true; + } + } else if (nonProcessEntityCount(ret) == lineageContext.getLimit()) { + setVertexCountsForOneDirection(isInput, ret, currentVertexEdges, currentVertexEdgeIndex, edgesOfProcess, processEdgeIndex); + return true; + } + return false; + } + + private void setVertexCountsForOneDirection(boolean isInput, AtlasLineageInfo ret, List currentVertexEdges, int currentVertexEdgeIndex, List edgesOfProcess, int processEdgeIndex) { + if (isInput) { + ret.setHasMoreUpstreamVertices(true); + } else { + ret.setHasMoreDownstreamVertices(true); + } + } + + private long nonProcessEntityCount(AtlasLineageInfo ret) { + long nonProcessVertexCount = ret.getGuidEntityMap() + .values() + .stream() + .filter(vertex -> !vertex.getTypeName().contains("Process")) + .count(); + + //We subtract 1 because the base entity is added to the result as well. We want 'limit' number of child + //vertices, excluding the base entity. + return Math.max(nonProcessVertexCount - 1, 0); + } + + private void addLimitlessVerticesToResult(boolean isInput, int depth, Set visitedVertices, AtlasLineageInfo ret, AtlasLineageContext lineageContext, List currentVertexEdges) throws AtlasBaseException { + for (AtlasEdge edge : currentVertexEdges) { + AtlasVertex processVertex = edge.getOutVertex(); + List outputEdgesOfProcess = getEdgesOfProcess(isInput, lineageContext, processVertex); + + ret.setHasChildrenForDirection(getGuid(processVertex), new LineageChildrenInfo(isInput ? INPUT : OUTPUT, hasMoreChildren(outputEdgesOfProcess))); + for (AtlasEdge outgoingEdge : outputEdgesOfProcess) { + AtlasVertex entityVertex = outgoingEdge.getInVertex(); + + if (entityVertex != null) { + if (lineageContext.isHideProcess()) { + processVirtualEdge(edge, outgoingEdge, ret, lineageContext); + } else { + processEdges(edge, outgoingEdge, ret, lineageContext); + } + + if (!visitedVertices.contains(entityVertex.getIdForDisplay())) { + traverseEdges(entityVertex, isInput, depth - 1, visitedVertices, ret, lineageContext); } } } } } - private void addEdgeToResult(AtlasEdge edge, AtlasLineageInfo lineageInfo) throws AtlasBaseException { - if (!lineageContainsEdge(lineageInfo, edge)) { - processEdge(edge, lineageInfo); + private void processLastLevel(AtlasVertex currentVertex, boolean isInput, AtlasLineageInfo ret, AtlasLineageContext lineageContext) { + List processEdges = vertexEdgeCache.getEdges(currentVertex, IN, isInput ? PROCESS_OUTPUTS_EDGE : PROCESS_INPUTS_EDGE); + + // Filter lineages based on ignored process types + processEdges = CollectionUtils.isNotEmpty(lineageContext.getIgnoredProcesses()) ? + processEdges.stream() + .filter(processEdge -> processEdge.getOutVertex() != null) + .filter(processEdge -> !lineageContext.getIgnoredProcesses().contains(processEdge.getOutVertex().getProperty(Constants.ENTITY_TYPE_PROPERTY_KEY, String.class))) + .collect(Collectors.toList()) + : processEdges; + + // Filter lineages if child has only self-cyclic relation + processEdges = processEdges.stream() + .filter(processEdge -> processEdge.getOutVertex() != null) + .filter(processEdge -> !childHasOnlySelfCycle(processEdge.getOutVertex(), currentVertex, isInput)) + .collect(Collectors.toList()); + + ret.setHasChildrenForDirection(getGuid(currentVertex), new LineageChildrenInfo(isInput ? INPUT : OUTPUT, hasMoreChildren(processEdges))); + } + + private boolean childHasOnlySelfCycle(AtlasVertex processVertex, AtlasVertex currentVertex, boolean isInput) { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("childHasSelfCycle"); + Iterator processEdgeIterator; + processEdgeIterator = processVertex.getEdges(OUT, isInput ? PROCESS_INPUTS_EDGE : PROCESS_OUTPUTS_EDGE).iterator(); + Set processOutputEdges = new HashSet<>(); + while (processEdgeIterator.hasNext()) { + processOutputEdges.add(processEdgeIterator.next()); } + + Set linkedVertices = processOutputEdges.stream().map(x -> x.getInVertex()).collect(Collectors.toSet()); + RequestContext.get().endMetricRecord(metricRecorder); + return linkedVertices.size() == 1 && linkedVertices.contains(currentVertex); + } + + private List getEdgesOfProcess(boolean isInput, AtlasLineageContext lineageContext, AtlasVertex processVertex) { + if (lineageContext.getIgnoredProcesses() != null && + lineageContext.getIgnoredProcesses().contains(processVertex.getProperty(Constants.ENTITY_TYPE_PROPERTY_KEY, String.class))) { + return Collections.emptyList(); + } + + return vertexEdgeCache.getEdges(processVertex, OUT, isInput ? PROCESS_INPUTS_EDGE : PROCESS_OUTPUTS_EDGE) + .stream() + .filter(edge -> shouldProcessEdge(lineageContext, edge) && vertexMatchesEvaluation(edge.getInVertex(), lineageContext)) + .sorted(Comparator.comparing(edge -> edge.getProperty("_r__guid", String.class))) + .collect(Collectors.toList()); + } + + private boolean vertexMatchesEvaluation(AtlasVertex currentVertex, AtlasLineageContext lineageContext) { + return currentVertex.equals(lineageContext.getStartDatasetVertex()) || lineageContext.evaluate(currentVertex); + } + + private boolean vertexMatchesEvaluation(AtlasVertex currentVertex, AtlasLineageOnDemandContext atlasLineageOnDemandContext) { + return atlasLineageOnDemandContext.evaluate(currentVertex); + } + + private boolean edgeMatchesEvaluation(AtlasEdge currentEdge, AtlasLineageOnDemandContext atlasLineageOnDemandContext) { + return atlasLineageOnDemandContext.evaluate(currentEdge); + } + + private boolean shouldProcessEdge(AtlasLineageContext lineageContext, AtlasEdge edge) { + return lineageContext.isAllowDeletedProcess() || + (getStatus(edge.getOutVertex()) == AtlasEntity.Status.ACTIVE && getStateAsString(edge).equals(ACTIVE_STATE_VALUE)); } - private void addVirtualEdgeToResult(AtlasEdge incomingEdge, AtlasEdge outgoingEdge, AtlasLineageInfo lineageInfo) throws AtlasBaseException { - processVirtualEdge(incomingEdge, outgoingEdge, lineageInfo.getGuidEntityMap(), lineageInfo.getRelations()); + private List getEdgesOfCurrentVertex(AtlasVertex currentVertex, boolean isInput, AtlasLineageContext lineageContext) { + return vertexEdgeCache + .getEdges(currentVertex, IN, isInput ? PROCESS_OUTPUTS_EDGE : PROCESS_INPUTS_EDGE) + .stream() + .sorted(Comparator.comparing(edge -> edge.getProperty("_r__guid", String.class))) + .filter(edge -> shouldProcessEdge(lineageContext, edge)) + .collect(Collectors.toList()); } private boolean lineageContainsEdge(AtlasLineageInfo lineageInfo, AtlasEdge edge) { boolean ret = false; if (lineageInfo != null && CollectionUtils.isNotEmpty(lineageInfo.getRelations()) && edge != null) { - String relationGuid = AtlasGraphUtilsV2.getEncodedProperty(edge, RELATIONSHIP_GUID_PROPERTY_KEY, String.class); - Set relations = lineageInfo.getRelations(); - + String relationGuid = AtlasGraphUtilsV2.getEncodedProperty(edge, RELATIONSHIP_GUID_PROPERTY_KEY, String.class); + Set relations = lineageInfo.getRelations(); for (LineageRelation relation : relations) { if (relation.getRelationshipId().equals(relationGuid)) { ret = true; @@ -375,20 +1216,54 @@ private boolean lineageContainsEdge(AtlasLineageInfo lineageInfo, AtlasEdge edge return ret; } - private void processEdge(final AtlasEdge edge, final AtlasLineageInfo lineageInfo) throws AtlasBaseException { - processEdge(edge, lineageInfo.getGuidEntityMap(), lineageInfo.getRelations()); + private boolean lineageContainsVisitedEdgeV2(AtlasLineageOnDemandInfo lineageInfo, AtlasEdge edge) { + AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("lineageContainsVisitedEdgeV2"); + + boolean ret = false; + + if (edge != null && lineageInfo != null && CollectionUtils.isNotEmpty(lineageInfo.getVisitedEdges())) { + if (lineageInfo.getVisitedEdges().contains(getEdgeLabel(edge))) { + ret = true; + } + } + + RequestContext.get().endMetricRecord(metric); + + return ret; } - private AtlasLineageInfo initializeLineageInfo(String guid, LineageDirection direction, int depth) { - return new AtlasLineageInfo(guid, new HashMap<>(), new HashSet<>(), direction, depth); + private boolean lineageContainsSkippedEdgeV2(AtlasLineageOnDemandInfo lineageInfo, AtlasEdge edge) { + AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("lineageContainsSkippedEdgeV2"); + + boolean ret = false; + + if (edge != null && lineageInfo != null && CollectionUtils.isNotEmpty(lineageInfo.getSkippedEdges())) { + if (lineageInfo.getSkippedEdges().contains(getEdgeLabel(edge))) { + ret = true; + } + } + + RequestContext.get().endMetricRecord(metric); + + return ret; } - private static String getId(AtlasVertex vertex) { - return vertex.getIdForDisplay(); + private void addEdgeToSkippedEdges(AtlasLineageOnDemandInfo lineageInfo, AtlasEdge edge) { + if (lineageInfo.getSkippedEdges() != null) { + lineageInfo.getSkippedEdges().add(getEdgeLabel(edge)); + } + } + + private AtlasLineageInfo initializeLineageInfo(String guid, LineageDirection direction, int depth, int limit, int offset) { + return new AtlasLineageInfo(guid, new HashMap<>(), new HashSet<>(), direction, depth, limit, offset); + } + + private AtlasLineageOnDemandInfo initializeLineageOnDemandInfo(String guid) { + return new AtlasLineageOnDemandInfo(guid, new HashMap<>(), new HashSet<>(), new HashSet<>(), new HashSet<>(), new HashMap<>()); } private List executeGremlinScript(Map bindings, String lineageQuery) throws AtlasBaseException { - List ret; + List ret; ScriptEngine engine = graph.getGremlinScriptEngine(); try { @@ -402,21 +1277,113 @@ private List executeGremlinScript(Map bindings, String lineageQu return ret; } - private void processVirtualEdge(final AtlasEdge incomingEdge, final AtlasEdge outgoingEdge, final Map entities, final Set relations) throws AtlasBaseException { - AtlasVertex inVertex = incomingEdge.getInVertex(); - AtlasVertex outVertex = outgoingEdge.getInVertex(); - String inGuid = AtlasGraphUtilsV2.getIdFromVertex(inVertex); - String outGuid = AtlasGraphUtilsV2.getIdFromVertex(outVertex); - String relationGuid = null; - boolean isInputEdge = incomingEdge.getLabel().equalsIgnoreCase(PROCESS_INPUTS_EDGE); + private boolean processVirtualEdge(final AtlasEdge incomingEdge, final AtlasEdge outgoingEdge, AtlasLineageInfo lineageInfo, + AtlasLineageContext lineageContext) throws AtlasBaseException { + final Map entities = lineageInfo.getGuidEntityMap(); + final Set relations = lineageInfo.getRelations(); + + AtlasVertex inVertex = incomingEdge.getInVertex(); + AtlasVertex outVertex = outgoingEdge.getInVertex(); + AtlasVertex processVertex = outgoingEdge.getOutVertex(); + String inGuid = AtlasGraphUtilsV2.getIdFromVertex(inVertex); + String outGuid = AtlasGraphUtilsV2.getIdFromVertex(outVertex); + String processGuid = AtlasGraphUtilsV2.getIdFromVertex(processVertex); + String relationGuid = null; + boolean isInputEdge = incomingEdge.getLabel().equalsIgnoreCase(PROCESS_INPUTS_EDGE); + + if (!entities.containsKey(inGuid)) { + AtlasEntityHeader entityHeader = entityRetriever.toAtlasEntityHeader(inVertex, lineageContext.getAttributes()); + GraphTransactionInterceptor.addToVertexGuidCache(inVertex.getId(), entityHeader.getGuid()); + entities.put(inGuid, entityHeader); + } + + if (!entities.containsKey(outGuid)) { + AtlasEntityHeader entityHeader = entityRetriever.toAtlasEntityHeader(outVertex, lineageContext.getAttributes()); + GraphTransactionInterceptor.addToVertexGuidCache(outVertex.getId(), entityHeader.getGuid()); + entities.put(outGuid, entityHeader); + } + + if (!entities.containsKey(processGuid)) { + AtlasEntityHeader entityHeader = entityRetriever.toAtlasEntityHeader(processVertex, lineageContext.getAttributes()); + GraphTransactionInterceptor.addToVertexGuidCache(processVertex.getId(), entityHeader.getGuid()); + entities.put(processGuid, entityHeader); + } + + if (isInputEdge) { + relations.add(new LineageRelation(inGuid, outGuid, relationGuid, getGuid(processVertex))); + } else { + relations.add(new LineageRelation(outGuid, inGuid, relationGuid, getGuid(processVertex))); + } + return false; + } + + private void processEdges(final AtlasEdge incomingEdge, AtlasEdge outgoingEdge, AtlasLineageInfo lineageInfo, + AtlasLineageContext lineageContext) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("processEdges"); + + final Map entities = lineageInfo.getGuidEntityMap(); + final Set relations = lineageInfo.getRelations(); + + AtlasVertex leftVertex = incomingEdge.getInVertex(); + AtlasVertex processVertex = incomingEdge.getOutVertex(); + AtlasVertex rightVertex = outgoingEdge.getInVertex(); + + String leftGuid = AtlasGraphUtilsV2.getIdFromVertex(leftVertex); + String rightGuid = AtlasGraphUtilsV2.getIdFromVertex(rightVertex); + String processGuid = AtlasGraphUtilsV2.getIdFromVertex(processVertex); + + if (!entities.containsKey(leftGuid)) { + AtlasEntityHeader entityHeader = entityRetriever.toAtlasEntityHeaderWithClassifications(leftVertex, lineageContext.getAttributes()); + entities.put(leftGuid, entityHeader); + } + + if (!entities.containsKey(processGuid)) { + AtlasEntityHeader entityHeader = entityRetriever.toAtlasEntityHeaderWithClassifications(processVertex, lineageContext.getAttributes()); + entities.put(processGuid, entityHeader); + } + + if (!entities.containsKey(rightGuid)) { + AtlasEntityHeader entityHeader = entityRetriever.toAtlasEntityHeaderWithClassifications(rightVertex, lineageContext.getAttributes()); + entities.put(rightGuid, entityHeader); + } + + String relationGuid = AtlasGraphUtilsV2.getEncodedProperty(incomingEdge, RELATIONSHIP_GUID_PROPERTY_KEY, String.class); + if (incomingEdge.getLabel().equalsIgnoreCase(PROCESS_INPUTS_EDGE)) { + relations.add(new LineageRelation(leftGuid, processGuid, relationGuid)); + } else { + relations.add(new LineageRelation(processGuid, leftGuid, relationGuid)); + } + + relationGuid = AtlasGraphUtilsV2.getEncodedProperty(outgoingEdge, RELATIONSHIP_GUID_PROPERTY_KEY, String.class); + if (outgoingEdge.getLabel().equalsIgnoreCase(PROCESS_INPUTS_EDGE)) { + relations.add(new LineageRelation(rightGuid, processGuid, relationGuid)); + } else { + relations.add(new LineageRelation(processGuid, rightGuid, relationGuid)); + } + RequestContext.get().endMetricRecord(metric); + } + + private void processEdge(final AtlasEdge edge, AtlasLineageInfo lineageInfo, + AtlasLineageContext lineageContext) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("processEdge"); + + final Map entities = lineageInfo.getGuidEntityMap(); + final Set relations = lineageInfo.getRelations(); + + AtlasVertex inVertex = edge.getInVertex(); + AtlasVertex outVertex = edge.getOutVertex(); + String inGuid = AtlasGraphUtilsV2.getIdFromVertex(inVertex); + String outGuid = AtlasGraphUtilsV2.getIdFromVertex(outVertex); + String relationGuid = AtlasGraphUtilsV2.getEncodedProperty(edge, RELATIONSHIP_GUID_PROPERTY_KEY, String.class); + boolean isInputEdge = edge.getLabel().equalsIgnoreCase(PROCESS_INPUTS_EDGE); if (!entities.containsKey(inGuid)) { - AtlasEntityHeader entityHeader = entityRetriever.toAtlasEntityHeader(inVertex); + AtlasEntityHeader entityHeader = entityRetriever.toAtlasEntityHeaderWithClassifications(inVertex, lineageContext.getAttributes()); entities.put(inGuid, entityHeader); } if (!entities.containsKey(outGuid)) { - AtlasEntityHeader entityHeader = entityRetriever.toAtlasEntityHeader(outVertex); + AtlasEntityHeader entityHeader = entityRetriever.toAtlasEntityHeaderWithClassifications(outVertex, lineageContext.getAttributes()); entities.put(outGuid, entityHeader); } @@ -425,9 +1392,42 @@ private void processVirtualEdge(final AtlasEdge incomingEdge, final AtlasEdge ou } else { relations.add(new LineageRelation(outGuid, inGuid, relationGuid)); } + RequestContext.get().endMetricRecord(metric); } - private void processEdge(final AtlasEdge edge, final Map entities, final Set relations) throws AtlasBaseException { + private void processEdge(final AtlasEdge edge, final Map entities, + final Set relations, AtlasLineageContext lineageContext) throws AtlasBaseException { + //Backward compatibility method + AtlasVertex inVertex = edge.getInVertex(); + AtlasVertex outVertex = edge.getOutVertex(); + String inGuid = AtlasGraphUtilsV2.getIdFromVertex(inVertex); + String outGuid = AtlasGraphUtilsV2.getIdFromVertex(outVertex); + String relationGuid = AtlasGraphUtilsV2.getEncodedProperty(edge, RELATIONSHIP_GUID_PROPERTY_KEY, String.class); + boolean isInputEdge = edge.getLabel().equalsIgnoreCase(PROCESS_INPUTS_EDGE); + + if (!entities.containsKey(inGuid)) { + AtlasEntityHeader entityHeader = entityRetriever.toAtlasEntityHeaderWithClassifications(inVertex, lineageContext.getAttributes()); + entities.put(inGuid, entityHeader); + } + + if (!entities.containsKey(outGuid)) { + AtlasEntityHeader entityHeader = entityRetriever.toAtlasEntityHeaderWithClassifications(outVertex, lineageContext.getAttributes()); + entities.put(outGuid, entityHeader); + } + + if (isInputEdge) { + relations.add(new LineageRelation(inGuid, outGuid, relationGuid)); + } else { + relations.add(new LineageRelation(outGuid, inGuid, relationGuid)); + } + } + + private void processEdge(final AtlasEdge edge, final AtlasLineageOnDemandInfo lineageInfo, AtlasLineageOnDemandContext atlasLineageOnDemandContext) throws AtlasBaseException { + processEdge(edge, lineageInfo.getGuidEntityMap(), lineageInfo.getRelations(), lineageInfo.getVisitedEdges(), atlasLineageOnDemandContext.getAttributes()); + } + + private void processEdge(final AtlasEdge edge, final Map entities, final Set relations, final Set visitedEdges, final Set attributes) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processEdge"); AtlasVertex inVertex = edge.getInVertex(); AtlasVertex outVertex = edge.getOutVertex(); String inGuid = AtlasGraphUtilsV2.getIdFromVertex(inVertex); @@ -435,27 +1435,32 @@ private void processEdge(final AtlasEdge edge, final Map bindings) { String incomingFrom = null; - String outgoingTo = null; + String outgoingTo = null; String ret; if (direction.equals(INPUT)) { incomingFrom = PROCESS_OUTPUTS_EDGE; - outgoingTo = PROCESS_INPUTS_EDGE; + outgoingTo = PROCESS_INPUTS_EDGE; } else if (direction.equals(OUTPUT)) { incomingFrom = PROCESS_INPUTS_EDGE; - outgoingTo = PROCESS_OUTPUTS_EDGE; + outgoingTo = PROCESS_OUTPUTS_EDGE; } bindings.put("guid", entityGuid); @@ -485,12 +1490,17 @@ private String getLineageQuery(String entityGuid, LineageDirection direction, in if (depth < 1) { ret = isDataSet ? gremlinQueryProvider.getQuery(FULL_LINEAGE_DATASET) : - gremlinQueryProvider.getQuery(FULL_LINEAGE_PROCESS); + gremlinQueryProvider.getQuery(FULL_LINEAGE_PROCESS); } else { ret = isDataSet ? gremlinQueryProvider.getQuery(PARTIAL_LINEAGE_DATASET) : - gremlinQueryProvider.getQuery(PARTIAL_LINEAGE_PROCESS); + gremlinQueryProvider.getQuery(PARTIAL_LINEAGE_PROCESS); } return ret; } + + public boolean isLineageOnDemandEnabled() { + return AtlasConfiguration.LINEAGE_ON_DEMAND_ENABLED.getBoolean(); + } + } \ No newline at end of file diff --git a/repository/src/main/java/org/apache/atlas/discovery/LineageSearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/LineageSearchProcessor.java new file mode 100644 index 00000000000..eee9164fb70 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/discovery/LineageSearchProcessor.java @@ -0,0 +1,255 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.discovery; + +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.discovery.SearchParameters; +import org.apache.atlas.model.typedef.AtlasBaseTypeDef; +import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.type.*; +import org.apache.atlas.util.SearchPredicateUtil; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.Predicate; +import org.apache.commons.collections.PredicateUtils; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.*; + +import static org.apache.atlas.repository.Constants.ATTRIBUTE_VALUE_DELIMITER; +import static org.apache.atlas.util.SearchPredicateUtil.*; +import static org.apache.atlas.util.SearchPredicateUtil.getInRangePredicateGenerator; + +public class LineageSearchProcessor { + private static final Logger LOG = LoggerFactory.getLogger(LineageSearchProcessor.class); + + private static final Map OPERATOR_PREDICATE_MAP = new HashMap<>(); + + static + { + OPERATOR_PREDICATE_MAP.put(SearchParameters.Operator.LT, getLTPredicateGenerator()); + OPERATOR_PREDICATE_MAP.put(SearchParameters.Operator.GT, getGTPredicateGenerator()); + OPERATOR_PREDICATE_MAP.put(SearchParameters.Operator.LTE, getLTEPredicateGenerator()); + OPERATOR_PREDICATE_MAP.put(SearchParameters.Operator.GTE, getGTEPredicateGenerator()); + OPERATOR_PREDICATE_MAP.put(SearchParameters.Operator.EQ, getEQPredicateGenerator()); + OPERATOR_PREDICATE_MAP.put(SearchParameters.Operator.NEQ, getNEQPredicateGenerator()); + OPERATOR_PREDICATE_MAP.put(SearchParameters.Operator.IN, getINPredicateGenerator()); // this should be a list of quoted strings + OPERATOR_PREDICATE_MAP.put(SearchParameters.Operator.LIKE, getLIKEPredicateGenerator()); // this should be regex pattern + OPERATOR_PREDICATE_MAP.put(SearchParameters.Operator.STARTS_WITH, getStartsWithPredicateGenerator()); + OPERATOR_PREDICATE_MAP.put(SearchParameters.Operator.ENDS_WITH, getEndsWithPredicateGenerator()); + OPERATOR_PREDICATE_MAP.put(SearchParameters.Operator.CONTAINS, getContainsPredicateGenerator()); + OPERATOR_PREDICATE_MAP.put(SearchParameters.Operator.NOT_CONTAINS, getNotContainsPredicateGenerator()); + OPERATOR_PREDICATE_MAP.put(SearchParameters.Operator.IS_NULL, getIsNullPredicateGenerator()); + OPERATOR_PREDICATE_MAP.put(SearchParameters.Operator.NOT_NULL, getNotNullPredicateGenerator()); + OPERATOR_PREDICATE_MAP.put(SearchParameters.Operator.NOT_EMPTY, getNotEmptyPredicateGenerator()); + OPERATOR_PREDICATE_MAP.put(SearchParameters.Operator.TIME_RANGE, getInRangePredicateGenerator()); + } + + protected Predicate constructInMemoryPredicate(AtlasTypeRegistry typeRegistry, SearchParameters.FilterCriteria filterCriteria) { + Predicate ret = null; + if (filterCriteria != null) { + if (LOG.isDebugEnabled()) { + LOG.debug("Processing Filters"); + } + final List entityTypes = new ArrayList<>(typeRegistry.getAllEntityTypes()); + Set allAttributes = new HashSet<>(); + getAllAttributes(filterCriteria, allAttributes); + + ret = toInMemoryPredicate(entityTypes, filterCriteria, allAttributes); + } + return ret; + } + + private Predicate toInMemoryPredicate(List structTypes, SearchParameters.FilterCriteria criteria, + Set allAttributes) { + + Set filterAttributes = new HashSet<>(); + filterAttributes.addAll(allAttributes); + + if (criteria.getCondition() != null && CollectionUtils.isNotEmpty(criteria.getCriterion())) { + List predicates = new ArrayList<>(); + + for (SearchParameters.FilterCriteria filterCriteria : criteria.getCriterion()) { + Predicate predicate = toInMemoryPredicate(structTypes, filterCriteria, filterAttributes); + + if (predicate != null) { + predicates.add(predicate); + } + } + + if (CollectionUtils.isNotEmpty(predicates)) { + if (criteria.getCondition() == SearchParameters.FilterCriteria.Condition.AND) { + return PredicateUtils.allPredicate(predicates); + } else { + return PredicateUtils.anyPredicate(predicates); + } + } + } else if (StringUtils.isNotEmpty(criteria.getAttributeName())) { + try { + ArrayList predicates = new ArrayList<>(); + + for (AtlasStructType structType : structTypes) { + AtlasStructType.AtlasAttribute attribute = structType.getAttribute(criteria.getAttributeName()); + if (attribute == null) { + continue; + } + String name = structType.getVertexPropertyName(criteria.getAttributeName()); + + if (filterAttributes.contains(name)) { + String attrName = criteria.getAttributeName(); + String attrValue = criteria.getAttributeValue(); + SearchParameters.Operator operator = criteria.getOperator(); + + predicates.add(toInMemoryPredicate(structType, attrName, operator, attrValue)); + filterAttributes.remove(name); + break; + } + + } + + if (CollectionUtils.isNotEmpty(predicates)) { + if (predicates.size() > 1) { + return PredicateUtils.anyPredicate(predicates); + } else { + return predicates.iterator().next(); + } + } + } catch (AtlasBaseException e) { + LOG.warn(e.getMessage()); + } + } + return null; + } + + private Predicate toInMemoryPredicate(AtlasStructType type, String attrName, SearchParameters.Operator op, String attrVal) { + Predicate ret = null; + AtlasStructType.AtlasAttribute attribute = type.getAttribute(attrName); + SearchPredicateUtil.ElementAttributePredicateGenerator predicate = OPERATOR_PREDICATE_MAP.get(op); + + if (attribute != null && predicate != null) { + final AtlasType attrType = attribute.getAttributeType(); + final Class attrClass; + final Object attrValue; + Object attrValue2 = null; + + // Some operators support null comparison, thus the parsing has to be conditional + switch (attrType.getTypeName()) { + case AtlasBaseTypeDef.ATLAS_TYPE_STRING: + attrClass = String.class; + attrValue = attrVal; + break; + case AtlasBaseTypeDef.ATLAS_TYPE_SHORT: + attrClass = Short.class; + attrValue = StringUtils.isEmpty(attrVal) ? null : Short.parseShort(attrVal); + break; + case AtlasBaseTypeDef.ATLAS_TYPE_INT: + attrClass = Integer.class; + attrValue = StringUtils.isEmpty(attrVal) ? null : Integer.parseInt(attrVal); + break; + case AtlasBaseTypeDef.ATLAS_TYPE_BIGINTEGER: + attrClass = BigInteger.class; + attrValue = StringUtils.isEmpty(attrVal) ? null : new BigInteger(attrVal); + break; + case AtlasBaseTypeDef.ATLAS_TYPE_BOOLEAN: + attrClass = Boolean.class; + attrValue = StringUtils.isEmpty(attrVal) ? null : Boolean.parseBoolean(attrVal); + break; + case AtlasBaseTypeDef.ATLAS_TYPE_BYTE: + attrClass = Byte.class; + attrValue = StringUtils.isEmpty(attrVal) ? null : Byte.parseByte(attrVal); + break; + case AtlasBaseTypeDef.ATLAS_TYPE_LONG: + case AtlasBaseTypeDef.ATLAS_TYPE_DATE: + attrClass = Long.class; + String rangeStart = ""; + String rangeEnd = ""; + if (op == SearchParameters.Operator.TIME_RANGE) { + String[] parts = attrVal.split(ATTRIBUTE_VALUE_DELIMITER); + if (parts.length == 2) { + rangeStart = parts[0]; + rangeEnd = parts[1]; + } + } + if (StringUtils.isNotEmpty(rangeStart) && StringUtils.isNotEmpty(rangeEnd)) { + attrValue = Long.parseLong(rangeStart); + attrValue2 = Long.parseLong(rangeEnd); + } else { + attrValue = StringUtils.isEmpty(attrVal) ? null : Long.parseLong(attrVal); + } + break; + case AtlasBaseTypeDef.ATLAS_TYPE_FLOAT: + attrClass = Float.class; + attrValue = StringUtils.isEmpty(attrVal) ? null : Float.parseFloat(attrVal); + break; + case AtlasBaseTypeDef.ATLAS_TYPE_DOUBLE: + attrClass = Double.class; + attrValue = StringUtils.isEmpty(attrVal) ? null : Double.parseDouble(attrVal); + break; + case AtlasBaseTypeDef.ATLAS_TYPE_BIGDECIMAL: + attrClass = BigDecimal.class; + attrValue = StringUtils.isEmpty(attrVal) ? null : new BigDecimal(attrVal); + break; + default: + if (attrType instanceof AtlasEnumType) { + attrClass = String.class; + } else if (attrType instanceof AtlasArrayType) { + attrClass = List.class; + } else { + attrClass = Object.class; + } + + attrValue = attrVal; + break; + } + + String vertexPropertyName = attribute.getVertexPropertyName(); + if (attrValue != null && attrValue2 != null) { + ret = predicate.generatePredicate(StringUtils.isEmpty(vertexPropertyName) ? attribute.getQualifiedName() : vertexPropertyName, + attrValue, attrValue2, attrClass); + } else { + ret = predicate.generatePredicate( + StringUtils.isEmpty(vertexPropertyName) ? attribute.getQualifiedName() : vertexPropertyName, + attrValue, attrClass); + } + } + + return ret; + } + + private Set getAllAttributes(SearchParameters.FilterCriteria filterCriteria, Set allAttributes) { + if (filterCriteria == null) { + return null; + } + + SearchParameters.FilterCriteria.Condition filterCondition = filterCriteria.getCondition(); + List criterion = filterCriteria.getCriterion(); + + if (filterCondition != null && CollectionUtils.isNotEmpty(criterion)) { + for (SearchParameters.FilterCriteria criteria : criterion) { + getAllAttributes(criteria, allAttributes); + } + } else if (StringUtils.isNotEmpty(filterCriteria.getAttributeName())) { + allAttributes.add(filterCriteria.getAttributeName()); + } + return allAttributes; + } +} diff --git a/repository/src/main/java/org/apache/atlas/discovery/VertexEdgeCache.java b/repository/src/main/java/org/apache/atlas/discovery/VertexEdgeCache.java new file mode 100644 index 00000000000..72c96fdffd0 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/discovery/VertexEdgeCache.java @@ -0,0 +1,30 @@ +package org.apache.atlas.discovery; + +import org.apache.atlas.repository.graphdb.AtlasEdge; +import org.apache.atlas.repository.graphdb.AtlasEdgeDirection; +import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.springframework.stereotype.Component; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static com.google.common.collect.Lists.newArrayList; + +@Component +public class VertexEdgeCache { + + private final ThreadLocal>> edgeCache = ThreadLocal.withInitial(HashMap::new); + + public List getEdges(AtlasVertex vertex, AtlasEdgeDirection direction, String edgeLabel) { + CachedVertexEdgesKey key = new CachedVertexEdgesKey(vertex.getId(), direction, edgeLabel); + Map> cache = edgeCache.get(); + if (cache.containsKey(key)) { + return cache.get(key); + } else { + List edges = newArrayList(vertex.getEdges(direction, edgeLabel)); + cache.put(key, edges); + return edges; + } + } +} diff --git a/repository/src/main/java/org/apache/atlas/glossary/GlossaryTermUtils.java b/repository/src/main/java/org/apache/atlas/glossary/GlossaryTermUtils.java index e5bb431aa1c..72c6b8dbd32 100644 --- a/repository/src/main/java/org/apache/atlas/glossary/GlossaryTermUtils.java +++ b/repository/src/main/java/org/apache/atlas/glossary/GlossaryTermUtils.java @@ -32,7 +32,6 @@ import org.apache.atlas.model.instance.AtlasStruct; import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.ogm.DataAccess; -import org.apache.atlas.repository.store.graph.AtlasEntityStore; import org.apache.atlas.repository.store.graph.AtlasRelationshipStore; import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2; import org.apache.atlas.type.AtlasRelationshipType; @@ -63,6 +62,7 @@ import static org.apache.atlas.type.Constants.GLOSSARY_PROPERTY_KEY; import static org.apache.atlas.type.Constants.MEANINGS_PROPERTY_KEY; import static org.apache.atlas.type.Constants.MEANINGS_TEXT_PROPERTY_KEY; +import static org.apache.atlas.type.Constants.MEANING_NAMES_PROPERTY_KEY; public class GlossaryTermUtils extends GlossaryUtils { private static final Logger LOG = LoggerFactory.getLogger(GlossaryTermUtils.class); @@ -105,8 +105,8 @@ public void processTermAssignments(AtlasGlossaryTerm glossaryTerm, Collection entities, List classifications) throws AtlasBaseException; + void onClassificationsAdded(List entities, List classifications, boolean forceInline) throws AtlasBaseException; /** * This is upon updating classifications to an entity. diff --git a/repository/src/main/java/org/apache/atlas/repository/audit/AbstractStorageBasedAuditRepository.java b/repository/src/main/java/org/apache/atlas/repository/audit/AbstractStorageBasedAuditRepository.java index 1aac37577bc..538f150830a 100644 --- a/repository/src/main/java/org/apache/atlas/repository/audit/AbstractStorageBasedAuditRepository.java +++ b/repository/src/main/java/org/apache/atlas/repository/audit/AbstractStorageBasedAuditRepository.java @@ -24,6 +24,7 @@ import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.listener.ActiveStateChangeHandler; import org.apache.atlas.model.audit.EntityAuditEventV2; +import org.apache.atlas.model.audit.EntityAuditSearchResult; import org.apache.atlas.service.Service; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.configuration.Configuration; @@ -31,10 +32,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; /** * This abstract base class should be used when adding support for an audit storage backend. @@ -162,4 +160,5 @@ protected byte[] getKey(String id, Long ts, int index) { return Bytes.toBytes(keyStr); } + public abstract EntityAuditSearchResult searchEvents(String queryString) throws AtlasBaseException; } diff --git a/repository/src/main/java/org/apache/atlas/repository/audit/CassandraBasedAuditRepository.java b/repository/src/main/java/org/apache/atlas/repository/audit/CassandraBasedAuditRepository.java index d21e62d473a..2fdb205bada 100644 --- a/repository/src/main/java/org/apache/atlas/repository/audit/CassandraBasedAuditRepository.java +++ b/repository/src/main/java/org/apache/atlas/repository/audit/CassandraBasedAuditRepository.java @@ -25,7 +25,6 @@ import com.datastax.driver.core.Row; import com.datastax.driver.core.Session; import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.Lists; import org.apache.atlas.AtlasException; import org.apache.atlas.EntityAuditEvent; import org.apache.atlas.annotation.ConditionalOnAtlasProperty; @@ -118,17 +117,14 @@ public void putEventsV1(List events) throws AtlasException { @Override public void putEventsV2(List events) throws AtlasBaseException { - List> chunkedEventsList = Lists.partition(events, AUDITS_INSERT_BATCH_SIZE); - for (List chunkedEvents: chunkedEventsList) { - BatchStatement batch = new BatchStatement(); - for (EntityAuditEventV2 event : chunkedEvents) { - BoundStatement stmt = new BoundStatement(insertStatement); - batch.add(stmt.bind(event.getEntityId(), event.getTimestamp(), - event.getAction().toString(), event.getUser(), event.getDetails(), - (persistEntityDefinition ? event.getEntityDefinitionString() : null))); - } - cassSession.execute(batch); + BatchStatement batch = new BatchStatement(); + for (EntityAuditEventV2 event : events) { + BoundStatement stmt = new BoundStatement(insertStatement); + batch.add(stmt.bind(event.getEntityId(), event.getTimestamp(), + event.getAction().toString(), event.getUser(), event.getDetails(), + (persistEntityDefinition ? event.getEntityDefinitionString() : null))); } + cassSession.execute(batch); } private BoundStatement getSelectStatement(String entityId, String startKey) { @@ -263,5 +259,4 @@ void createSession() throws AtlasException { public void stop() throws AtlasException { cassSession.close(); } - } diff --git a/repository/src/main/java/org/apache/atlas/repository/audit/ESBasedAuditRepository.java b/repository/src/main/java/org/apache/atlas/repository/audit/ESBasedAuditRepository.java index 92cb9e221e8..4464d8f9d81 100644 --- a/repository/src/main/java/org/apache/atlas/repository/audit/ESBasedAuditRepository.java +++ b/repository/src/main/java/org/apache/atlas/repository/audit/ESBasedAuditRepository.java @@ -17,15 +17,20 @@ */ package org.apache.atlas.repository.audit; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.annotations.VisibleForTesting; import org.apache.atlas.ApplicationProperties; +import org.apache.atlas.AtlasConfiguration; import org.apache.atlas.AtlasException; import org.apache.atlas.EntityAuditEvent; +import org.apache.atlas.RequestContext; import org.apache.atlas.annotation.ConditionalOnAtlasProperty; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.audit.EntityAuditEventV2; import org.apache.atlas.model.audit.EntityAuditSearchResult; import org.apache.atlas.type.AtlasType; +import org.apache.commons.collections.MapUtils; import org.apache.commons.configuration.Configuration; import org.apache.commons.lang.NotImplementedException; import org.apache.commons.lang.StringUtils; @@ -40,36 +45,65 @@ import org.elasticsearch.client.RestClientBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.core.annotation.Order; import org.springframework.stereotype.Component; +import javax.inject.Inject; +import javax.inject.Singleton; import java.io.File; import java.io.IOException; +import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.nio.file.Files; +import java.nio.file.Paths; import java.text.MessageFormat; import java.util.*; -import javax.inject.Singleton; +import static java.nio.charset.Charset.defaultCharset; +import static org.springframework.util.StreamUtils.copyToString; /** * This class provides cassandra support as the backend for audit storage support. */ @Singleton @Component +@Order(8) @ConditionalOnAtlasProperty(property = "atlas.EntityAuditRepositorySearch.impl") public class ESBasedAuditRepository extends AbstractStorageBasedAuditRepository { private static final Logger LOG = LoggerFactory.getLogger(ESBasedAuditRepository.class); public static final String INDEX_BACKEND_CONF = "atlas.graph.index.search.hostname"; + private static final String TOTAL_FIELD_LIMIT = "atlas.index.audit.elasticsearch.total_field_limit"; public static final String INDEX_NAME = "entity_audits"; private static final String ENTITYID = "entityId"; +<<<<<<< HEAD + private static final String CREATED = "created"; +======= + private static final String TYPE_NAME = "typeName"; + private static final String ENTITY_QUALIFIED_NAME = "entityQualifiedName"; private static final String CREATED = "created"; + private static final String TIMESTAMP = "timestamp"; +>>>>>>> 7f3c811fb15361ba82bfb4edd7a8393798863ff0 private static final String EVENT_KEY = "eventKey"; private static final String ACTION = "action"; private static final String USER = "user"; private static final String DETAIL = "detail"; private static final String ENTITY = "entity"; + private static final String bulkMetadata = String.format("{ \"index\" : { \"_index\" : \"%s\" } }%n", INDEX_NAME); + + /* + * created → event creation time + timestamp → entity modified timestamp + eventKey → entityId:timestamp + * */ private RestClient lowLevelClient; + private final Configuration configuration; + + @Inject + public ESBasedAuditRepository(Configuration configuration) { + this.configuration = configuration; + } + @Override public void putEventsV1(List events) throws AtlasException { @@ -85,6 +119,7 @@ public List listEventsV1(String entityId, String startKey, sho public void putEventsV2(List events) throws AtlasBaseException { try { if (events != null && events.size() > 0) { +<<<<<<< HEAD String entityPayloadTemplate = "'{'\"entityId\":\"{0}\",\"created\":{1},\"action\":\"{2}\",\"detail\":{3},\"user\":\"{4}\", \"eventKey\":\"{5}\"'}'"; String bulkMetadata = String.format("{ \"index\" : { \"_index\" : \"%s\" } }%n", INDEX_NAME); StringBuilder bulkRequestBody = new StringBuilder(); @@ -99,6 +134,29 @@ public void putEventsV2(List events) throws AtlasBaseExcepti } String bulkItem = MessageFormat.format(entityPayloadTemplate, event.getEntityId(), created, event.getAction(), details, event.getUser(), event.getEntityId() + ":" + created); +======= + + Map requestContextHeaders = RequestContext.get().getRequestContextHeaders(); + String entityPayloadTemplate = getQueryTemplate(requestContextHeaders); + + StringBuilder bulkRequestBody = new StringBuilder(); + for (EntityAuditEventV2 event : events) { + String created = String.format("%s", event.getTimestamp()); + String auditDetailPrefix = EntityAuditListenerV2.getV2AuditPrefix(event.getAction()); + String details = event.getDetails().substring(auditDetailPrefix.length()); + + String bulkItem = MessageFormat.format(entityPayloadTemplate, + event.getEntityId(), + event.getAction(), + details, + event.getUser(), + event.getEntityId() + ":" + event.getEntity().getUpdateTime().getTime(), + event.getEntityQualifiedName(), + event.getEntity().getTypeName(), + created, + "" + event.getEntity().getUpdateTime().getTime()); + +>>>>>>> 7f3c811fb15361ba82bfb4edd7a8393798863ff0 bulkRequestBody.append(bulkMetadata); bulkRequestBody.append(bulkItem); bulkRequestBody.append("\n"); @@ -108,24 +166,58 @@ public void putEventsV2(List events) throws AtlasBaseExcepti Request request = new Request("POST", endpoint); request.setEntity(entity); Response response = lowLevelClient.performRequest(request); - int statusCode = response.getStatusLine().getStatusCode();; + int statusCode = response.getStatusLine().getStatusCode(); if (statusCode != 200) { throw new AtlasException("Unable to push entity audits to ES"); } String responseString = EntityUtils.toString(response.getEntity()); Map responseMap = AtlasType.fromJson(responseString, Map.class); if ((boolean) responseMap.get("errors")) { - throw new AtlasException("Unable to push entity audits to ES (errors: true returned by es)"); + List errors = new ArrayList<>(); + List> resultItems = (List>) responseMap.get("items"); + for (Map resultItem : resultItems) { + if (resultItem.get("index") != null) { + Map resultIndex = (Map) resultItem.get("index"); + if (resultIndex.get("error") != null) { + errors.add(resultIndex.get("error").toString()); + } + } + } + throw new AtlasException(errors.toString()); } } } catch (Exception e) { - throw new AtlasBaseException(e); + throw new AtlasBaseException("Unable to push entity audits to ES", e); + } + } + + private String getQueryTemplate(Map requestContextHeaders) { + StringBuilder template = new StringBuilder(); + + template.append("'{'\"entityId\":\"{0}\",\"action\":\"{1}\",\"detail\":{2},\"user\":\"{3}\", \"eventKey\":\"{4}\", " + + "\"entityQualifiedName\": {5}, \"typeName\": \"{6}\",\"created\":{7}, \"timestamp\":{8}"); + + if (MapUtils.isNotEmpty(requestContextHeaders)) { + template.append(",") + .append("\"").append("headers").append("\"") + .append(":") + .append(AtlasType.toJson(requestContextHeaders).replaceAll("\\{", "'{").replaceAll("\\}", "'}")); + } + + template.append("'}'"); + + return template.toString(); } @Override public List listEventsV2(String entityId, EntityAuditEventV2.EntityAuditActionV2 auditAction, String startKey, short maxResultCount) throws AtlasBaseException { - return null; + List ret; + String queryTemplate = "{\"query\":{\"bool\":{\"must\":[{\"term\":{\"entityId\":\"%s\"}},{\"term\":{\"action\":\"%s\"}}]}}}"; + String queryWithEntityFilter = String.format(queryTemplate, entityId, auditAction); + ret = searchEvents(queryWithEntityFilter).getEntityAudits(); + + return ret; } @Override @@ -144,27 +236,44 @@ public EntityAuditSearchResult searchEvents(String queryString) throws AtlasBase } } - private EntityAuditSearchResult getResultFromResponse(String responseString) { + private EntityAuditSearchResult getResultFromResponse(String responseString) throws AtlasBaseException { List entityAudits = new ArrayList<>(); EntityAuditSearchResult searchResult = new EntityAuditSearchResult(); Map responseMap = AtlasType.fromJson(responseString, Map.class); Map hits_0 = (Map) responseMap.get("hits"); List hits_1 = (List) hits_0.get("hits"); - for (LinkedHashMap hit: hits_1) { + for (LinkedHashMap hit : hits_1) { Map source = (Map) hit.get("_source"); + String entityGuid = (String) source.get(ENTITYID); EntityAuditEventV2 event = new EntityAuditEventV2(); - event.setEntityId((String) source.get(ENTITYID)); + event.setEntityId(entityGuid); event.setAction(EntityAuditEventV2.EntityAuditActionV2.fromString((String) source.get(ACTION))); event.setDetail((Map) source.get(DETAIL)); event.setUser((String) source.get(USER)); event.setCreated((long) source.get(CREATED)); +<<<<<<< HEAD event.setTimestamp((long) source.get(CREATED)); +======= + if (source.get(TIMESTAMP) != null) { + event.setTimestamp((long) source.get(TIMESTAMP)); + } + if (source.get(TYPE_NAME) != null) { + event.setTypeName((String) source.get(TYPE_NAME)); + } + + event.setEntityQualifiedName((String) source.get(ENTITY_QUALIFIED_NAME)); +>>>>>>> 7f3c811fb15361ba82bfb4edd7a8393798863ff0 String eventKey = (String) source.get(EVENT_KEY); if (StringUtils.isEmpty(eventKey)) { eventKey = event.getEntityId() + ":" + event.getTimestamp(); } +<<<<<<< HEAD +======= + event.setHeaders((Map) source.get("headers")); + +>>>>>>> 7f3c811fb15361ba82bfb4edd7a8393798863ff0 event.setEventKey(eventKey); entityAudits.add(event); } @@ -213,6 +322,11 @@ void createSession() throws AtlasException { LOG.info("Create ES index for entity audits in ES Based Audit Repo"); createAuditIndex(); } + if (shouldUpdateFieldLimitSetting()) { + LOG.info("Updating ES total field limit"); + updateFieldLimit(); + } + updateMappingsIfChanged(); } catch (IOException e) { LOG.error("error", e); throw new AtlasException(e); @@ -220,6 +334,18 @@ void createSession() throws AtlasException { } + private boolean checkIfIndexExists() throws IOException { + Request request = new Request("HEAD", INDEX_NAME); + Response response = lowLevelClient.performRequest(request); + int statusCode = response.getStatusLine().getStatusCode(); + if (statusCode == 200) { + LOG.info("Entity audits index exists!"); + return true; + } + LOG.info("Entity audits index does not exist!"); + return false; + } + private boolean createAuditIndex() throws IOException { LOG.info("ESBasedAuditRepo - createAuditIndex!"); String esMappingsString = getAuditIndexMappings(); @@ -227,29 +353,88 @@ private boolean createAuditIndex() throws IOException { Request request = new Request("PUT", INDEX_NAME); request.setEntity(entity); Response response = lowLevelClient.performRequest(request); - int statusCode = response.getStatusLine().getStatusCode();; - return statusCode == 200 ? true: false; + return isSuccess(response); } - private String getAuditIndexMappings() throws IOException { - String atlasHomeDir = System.getProperty("atlas.home"); - String elasticsearchSettingsFilePath = (org.apache.commons.lang3.StringUtils.isEmpty(atlasHomeDir) ? "." : atlasHomeDir) + File.separator + "elasticsearch" + File.separator + "es-audit-mappings.json"; - File elasticsearchSettingsFile = new File(elasticsearchSettingsFilePath); - String jsonString = new String(Files.readAllBytes(elasticsearchSettingsFile.toPath()), StandardCharsets.UTF_8); - return jsonString; + private boolean shouldUpdateFieldLimitSetting() { + JsonNode currentFieldLimit; + try { + currentFieldLimit = getIndexFieldLimit(); + } catch (IOException e) { + LOG.error("Problem while retrieving the index field limit!", e); + return false; + } + Integer fieldLimitFromConfigurationFile = configuration.getInt(TOTAL_FIELD_LIMIT, 0); + return currentFieldLimit == null || fieldLimitFromConfigurationFile > currentFieldLimit.asInt(); } - private boolean checkIfIndexExists() throws IOException { - Request request = new Request("HEAD", INDEX_NAME); - Response response = lowLevelClient.performRequest(request); - int statusCode = response.getStatusLine().getStatusCode();; - if (statusCode == 200) { - LOG.info("Entity audits index exists!"); - return true; - } - LOG.info("Entity audits index does not exist!"); - return false; - } + private JsonNode getIndexFieldLimit() throws IOException { + Request request = new Request("GET", INDEX_NAME + "/_settings"); + Response response = lowLevelClient.performRequest(request); + ObjectMapper objectMapper = new ObjectMapper(); + String fieldPath = String.format("/%s/settings/index/mapping/total_fields/limit", INDEX_NAME); + + return objectMapper.readTree(copyToString(response.getEntity().getContent(), Charset.defaultCharset())).at(fieldPath); + } + + private void updateFieldLimit() { + Request request = new Request("PUT", INDEX_NAME + "/_settings"); + String requestBody = String.format("{\"index.mapping.total_fields.limit\": %d}", configuration.getInt(TOTAL_FIELD_LIMIT)); + HttpEntity entity = new NStringEntity(requestBody, ContentType.APPLICATION_JSON); + request.setEntity(entity); + Response response; + try { + response = lowLevelClient.performRequest(request); + if (response.getStatusLine().getStatusCode() != 200) { + LOG.error("Error while updating the Elasticsearch total field limits! Error: " + copyToString(response.getEntity().getContent(), defaultCharset())); + } else { + LOG.info("ES total field limit has been updated"); + } + } catch (IOException e) { + LOG.error("Error while updating the field limit", e); + } + } + + private void updateMappingsIfChanged() throws IOException, AtlasException { + LOG.info("ESBasedAuditRepo - updateMappings!"); + ObjectMapper mapper = new ObjectMapper(); + JsonNode activeIndexInformation = getActiveIndexInfoAsJson(mapper); + JsonNode indexInformationFromConfigurationFile = mapper.readTree(getAuditIndexMappings()); + if (!areConfigurationsSame(activeIndexInformation, indexInformationFromConfigurationFile)) { + Response response = updateMappings(indexInformationFromConfigurationFile); + if (isSuccess(response)) { + LOG.info("ESBasedAuditRepo - Elasticsearch mappings have been updated!"); + } else { + LOG.error("Error while updating the Elasticsearch indexes!"); + throw new AtlasException(copyToString(response.getEntity().getContent(), Charset.defaultCharset())); + } + } + } + + private JsonNode getActiveIndexInfoAsJson(ObjectMapper mapper) throws IOException { + Request request = new Request("GET", INDEX_NAME); + Response response = lowLevelClient.performRequest(request); + String responseString = copyToString(response.getEntity().getContent(), Charset.defaultCharset()); + return mapper.readTree(responseString); + } + + private boolean areConfigurationsSame(JsonNode activeIndexInformation, JsonNode indexInformationFromConfigurationFile) { + return indexInformationFromConfigurationFile.get("mappings").equals(activeIndexInformation.get("entity_audits").get("mappings")); + } + + private Response updateMappings(JsonNode indexInformationFromConfigurationFile) throws IOException { + Request request = new Request("PUT", INDEX_NAME + "/_mapping"); + HttpEntity entity = new NStringEntity(indexInformationFromConfigurationFile.get("mappings").toString(), ContentType.APPLICATION_JSON); + request.setEntity(entity); + return lowLevelClient.performRequest(request); + } + + private String getAuditIndexMappings() throws IOException { + String atlasHomeDir = System.getProperty("atlas.home"); + String atlasHome = StringUtils.isEmpty(atlasHomeDir) ? "." : atlasHomeDir; + File elasticsearchSettingsFile = Paths.get(atlasHome, "elasticsearch", "es-audit-mappings.json").toFile(); + return new String(Files.readAllBytes(elasticsearchSettingsFile.toPath()), StandardCharsets.UTF_8); + } @Override public void stop() throws AtlasException { @@ -273,8 +458,8 @@ private void setLowLevelClient() throws AtlasException { RestClientBuilder builder = RestClient.builder(httpHosts.get(0)); builder.setRequestConfigCallback(requestConfigBuilder -> requestConfigBuilder - .setConnectTimeout(900000) - .setSocketTimeout(900000)); + .setConnectTimeout(AtlasConfiguration.INDEX_CLIENT_CONNECTION_TIMEOUT.getInt()) + .setSocketTimeout(AtlasConfiguration.INDEX_CLIENT_SOCKET_TIMEOUT.getInt())); lowLevelClient = builder.build(); } catch (AtlasException e) { @@ -289,7 +474,7 @@ public static List getHttpHosts() throws AtlasException { Configuration configuration = ApplicationProperties.get(); String indexConf = configuration.getString(INDEX_BACKEND_CONF); String[] hosts = indexConf.split(","); - for (String host: hosts) { + for (String host : hosts) { host = host.trim(); String[] hostAndPort = host.split(":"); if (hostAndPort.length == 1) { @@ -303,4 +488,7 @@ public static List getHttpHosts() throws AtlasException { return httpHosts; } + private boolean isSuccess(Response response) { + return response.getStatusLine().getStatusCode() == 200; + } } diff --git a/repository/src/main/java/org/apache/atlas/repository/audit/EntityAuditListener.java b/repository/src/main/java/org/apache/atlas/repository/audit/EntityAuditListener.java index b693341aae6..6c484c130b8 100644 --- a/repository/src/main/java/org/apache/atlas/repository/audit/EntityAuditListener.java +++ b/repository/src/main/java/org/apache/atlas/repository/audit/EntityAuditListener.java @@ -44,6 +44,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import static org.apache.atlas.EntityAuditEvent.EntityAuditAction.TERM_ADD; import static org.apache.atlas.EntityAuditEvent.EntityAuditAction.TERM_DELETE; @@ -55,12 +56,12 @@ public class EntityAuditListener implements EntityChangeListener { private static final Logger LOG = LoggerFactory.getLogger(EntityAuditListener.class); - private final EntityAuditRepository auditRepository; + private final Set auditRepositories; private final AtlasTypeRegistry typeRegistry; @Inject - public EntityAuditListener(CassandraBasedAuditRepository auditRepository, AtlasTypeRegistry typeRegistry) { - this.auditRepository = auditRepository; + public EntityAuditListener(Set auditRepositories, AtlasTypeRegistry typeRegistry) { + this.auditRepositories = auditRepositories; this.typeRegistry = typeRegistry; } @@ -68,13 +69,15 @@ public EntityAuditListener(CassandraBasedAuditRepository auditRepository, AtlasT public void onEntitiesAdded(Collection entities, boolean isImport) throws AtlasException { MetricRecorder metric = RequestContext.get().startMetricRecord("entityAudit"); - List events = new ArrayList<>(); - for (Referenceable entity : entities) { - EntityAuditEvent event = createEvent(entity, isImport ? EntityAuditAction.ENTITY_IMPORT_CREATE : EntityAuditAction.ENTITY_CREATE); - events.add(event); - } + for (EntityAuditRepository auditRepository: auditRepositories) { + List events = new ArrayList<>(); + for (Referenceable entity : entities) { + EntityAuditEvent event = createEvent(entity, isImport ? EntityAuditAction.ENTITY_IMPORT_CREATE : EntityAuditAction.ENTITY_CREATE); + events.add(event); + } - auditRepository.putEventsV1(events); + auditRepository.putEventsV1(events); + } RequestContext.get().endMetricRecord(metric); } @@ -89,7 +92,9 @@ public void onEntitiesUpdated(Collection entities, boolean isImpo events.add(event); } - auditRepository.putEventsV1(events); + for (EntityAuditRepository auditRepository: auditRepositories) { + auditRepository.putEventsV1(events); + } RequestContext.get().endMetricRecord(metric); } @@ -103,7 +108,9 @@ public void onTraitsAdded(Referenceable entity, Collection tra EntityAuditEvent event = createEvent(entity, EntityAuditAction.TAG_ADD, "Added trait: " + AtlasType.toV1Json(trait)); - auditRepository.putEventsV1(event); + for (EntityAuditRepository auditRepository: auditRepositories) { + auditRepository.putEventsV1(event); + } } RequestContext.get().endMetricRecord(metric); @@ -118,7 +125,9 @@ public void onTraitsDeleted(Referenceable entity, Collection t for (Struct trait : traits) { EntityAuditEvent event = createEvent(entity, EntityAuditAction.TAG_DELETE, "Deleted trait: " + trait.getTypeName()); - auditRepository.putEventsV1(event); + for (EntityAuditRepository auditRepository: auditRepositories) { + auditRepository.putEventsV1(event); + } } RequestContext.get().endMetricRecord(metric); @@ -134,7 +143,9 @@ public void onTraitsUpdated(Referenceable entity, Collection t EntityAuditEvent event = createEvent(entity, EntityAuditAction.TAG_UPDATE, "Updated trait: " + AtlasType.toV1Json(trait)); - auditRepository.putEventsV1(event); + for (EntityAuditRepository auditRepository: auditRepositories) { + auditRepository.putEventsV1(event); + } } RequestContext.get().endMetricRecord(metric); @@ -151,7 +162,9 @@ public void onEntitiesDeleted(Collection entities, boolean isImpo events.add(event); } - auditRepository.putEventsV1(events); + for (EntityAuditRepository auditRepository: auditRepositories) { + auditRepository.putEventsV1(events); + } RequestContext.get().endMetricRecord(metric); } @@ -166,7 +179,9 @@ public void onTermAdded(Collection entities, AtlasGlossaryTerm te events.add(createEvent(entity, TERM_ADD, "Added term: " + term.toAuditString())); } - auditRepository.putEventsV1(events); + for (EntityAuditRepository auditRepository: auditRepositories) { + auditRepository.putEventsV1(events); + } RequestContext.get().endMetricRecord(metric); } @@ -181,15 +196,13 @@ public void onTermDeleted(Collection entities, AtlasGlossaryTerm events.add(createEvent(entity, TERM_DELETE, "Deleted term: " + term.toAuditString())); } - auditRepository.putEventsV1(events); + for (EntityAuditRepository auditRepository: auditRepositories) { + auditRepository.putEventsV1(events); + } RequestContext.get().endMetricRecord(metric); } - public List getAuditEvents(String guid) throws AtlasException{ - return auditRepository.listEventsV1(guid, null, (short) 10); - } - private EntityAuditEvent createEvent(Referenceable entity, EntityAuditAction action) throws AtlasException { String detail = getAuditEventDetail(entity, action); @@ -209,7 +222,7 @@ private String getAuditEventDetail(Referenceable entity, EntityAuditAction actio String auditString = auditPrefix + AtlasType.toV1Json(entity); byte[] auditBytes = auditString.getBytes(StandardCharsets.UTF_8); long auditSize = auditBytes != null ? auditBytes.length : 0; - long auditMaxSize = auditRepository.repositoryMaxSize(); + long auditMaxSize = 1024 * 1024; if (auditMaxSize >= 0 && auditSize > auditMaxSize) { // don't store attributes in audit LOG.warn("audit record too long: entityType={}, guid={}, size={}; maxSize={}. entity attribute values not stored in audit", @@ -243,7 +256,7 @@ private String getAuditEventDetail(Referenceable entity, EntityAuditAction actio private Map pruneEntityAttributesForAudit(Referenceable entity) throws AtlasException { Map ret = null; Map entityAttributes = entity.getValuesMap(); - List excludeAttributes = auditRepository.getAuditExcludeAttributes(entity.getTypeName()); + List excludeAttributes = null; AtlasEntityType entityType = typeRegistry.getEntityTypeByName(entity.getTypeName()); if (CollectionUtils.isNotEmpty(excludeAttributes) && MapUtils.isNotEmpty(entityAttributes) && entityType != null) { diff --git a/repository/src/main/java/org/apache/atlas/repository/audit/EntityAuditListenerV2.java b/repository/src/main/java/org/apache/atlas/repository/audit/EntityAuditListenerV2.java index d47b54d2618..e83690774fc 100644 --- a/repository/src/main/java/org/apache/atlas/repository/audit/EntityAuditListenerV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/audit/EntityAuditListenerV2.java @@ -22,16 +22,13 @@ import org.apache.atlas.AtlasException; import org.apache.atlas.EntityAuditEvent.EntityAuditAction; import org.apache.atlas.RequestContext; +import org.apache.atlas.annotation.EnableConditional; import org.apache.atlas.model.audit.EntityAuditEventV2; import org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditActionV2; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.listener.EntityChangeListenerV2; import org.apache.atlas.model.glossary.AtlasGlossaryTerm; -import org.apache.atlas.model.instance.AtlasClassification; -import org.apache.atlas.model.instance.AtlasEntity; -import org.apache.atlas.model.instance.AtlasRelatedObjectId; -import org.apache.atlas.model.instance.AtlasRelationship; -import org.apache.atlas.model.instance.AtlasStruct; +import org.apache.atlas.model.instance.*; import org.apache.atlas.repository.converters.AtlasInstanceConverter; import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasStructType.AtlasAttribute; @@ -51,6 +48,8 @@ import javax.inject.Inject; import java.nio.charset.StandardCharsets; import java.util.*; +import java.util.function.Function; +import java.util.stream.Collectors; import static org.apache.atlas.AtlasConfiguration.STORE_DIFFERENTIAL_AUDITS; import static org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditActionV2.BUSINESS_ATTRIBUTE_UPDATE; @@ -74,6 +73,7 @@ import static org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditActionV2.TERM_DELETE; @Component +@EnableConditional(property = "atlas.enable.entity.audits") public class EntityAuditListenerV2 implements EntityChangeListenerV2 { private static final Logger LOG = LoggerFactory.getLogger(EntityAuditListenerV2.class); private static final ThreadLocal> AUDIT_EVENTS_BUFFER = @@ -81,6 +81,8 @@ public class EntityAuditListenerV2 implements EntityChangeListenerV2 { AtlasConfiguration.NOTIFICATION_FIXED_BUFFER_ITEMS_INCREMENT_COUNT.getInt())); private static final long AUDIT_REPOSITORY_MAX_SIZE_DEFAULT = 1024 * 1024; + private static final long CASSANDRA_AUDIT_REPOSITORY_MAX_SIZE_DEFAULT = 50 * 1024; + private static final String QUALIFIED_NAME = "qualifiedName"; private final Set auditRepositories; private final AtlasTypeRegistry typeRegistry; private final AtlasInstanceConverter instanceConverter; @@ -88,23 +90,34 @@ public class EntityAuditListenerV2 implements EntityChangeListenerV2 { protected Map> auditExcludedAttributesCache = new HashMap<>(); private static final String AUDIT_EXCLUDE_ATTRIBUTE_PROPERTY = "atlas.audit.hbase.entity"; + private static boolean DIFFERENTIAL_AUDITS = false; + @Inject public EntityAuditListenerV2(Set auditRepositories, AtlasTypeRegistry typeRegistry, AtlasInstanceConverter instanceConverter) { this.auditRepositories = auditRepositories; this.typeRegistry = typeRegistry; this.instanceConverter = instanceConverter; + + DIFFERENTIAL_AUDITS = STORE_DIFFERENTIAL_AUDITS.getBoolean(); + } + + private long getAuditMaxSize(EntityAuditRepository auditRepository, int entityCount) { + boolean isCassandraRepository = auditRepository.getClass().equals(CassandraBasedAuditRepository.class); + // Subtracting 150 for other details in the Insert statement. + long auditMaxSize = isCassandraRepository ? ((CASSANDRA_AUDIT_REPOSITORY_MAX_SIZE_DEFAULT / entityCount) - 150): AUDIT_REPOSITORY_MAX_SIZE_DEFAULT; + return auditMaxSize; } @Override public void onEntitiesAdded(List entities, boolean isImport) throws AtlasBaseException { MetricRecorder metric = RequestContext.get().startMetricRecord("entityAudit"); - FixedBufferList entitiesAdded = getAuditEventsList(); - for (AtlasEntity entity : entities) { - createEvent(entitiesAdded.next(), entity, isImport ? ENTITY_IMPORT_CREATE : ENTITY_CREATE); - } - for (EntityAuditRepository auditRepository: auditRepositories) { + FixedBufferList entitiesAdded = getAuditEventsList(); + for (AtlasEntity entity : entities) { + long auditMaxSize = getAuditMaxSize(auditRepository, entities.size()); + createEvent(entitiesAdded.next(), entity, isImport ? ENTITY_IMPORT_CREATE : ENTITY_CREATE, auditMaxSize); + } auditRepository.putEventsV2(entitiesAdded.toList()); } @@ -115,32 +128,38 @@ public void onEntitiesAdded(List entities, boolean isImport) throws public void onEntitiesUpdated(List entities, boolean isImport) throws AtlasBaseException { RequestContext reqContext = RequestContext.get(); MetricRecorder metric = reqContext.startMetricRecord("entityAudit"); - FixedBufferList updatedEvents = getAuditEventsList(); Collection updatedEntites; - if (STORE_DIFFERENTIAL_AUDITS.getBoolean()) { + Map entitiesMap = entities.stream().collect(Collectors.toMap(AtlasEntity::getGuid, Function.identity())); + + if (DIFFERENTIAL_AUDITS) { updatedEntites = reqContext.getDifferentialEntities(); } else { updatedEntites = entities; } - for (AtlasEntity entity : updatedEntites) { - final EntityAuditActionV2 action; - - if (isImport) { - action = ENTITY_IMPORT_UPDATE; - } else if (reqContext.checkIfEntityIsForCustomAttributeUpdate(entity.getGuid())) { - action = CUSTOM_ATTRIBUTE_UPDATE; - } else if (reqContext.checkIfEntityIsForBusinessAttributeUpdate(entity.getGuid())) { - action = BUSINESS_ATTRIBUTE_UPDATE; - } else { - action = ENTITY_UPDATE; - } - - createEvent(updatedEvents.next(), entity, action); - } - for (EntityAuditRepository auditRepository: auditRepositories) { + FixedBufferList updatedEvents = getAuditEventsList(); + for (AtlasEntity entity : updatedEntites) { + final EntityAuditActionV2 action; + + if (isImport) { + action = ENTITY_IMPORT_UPDATE; + } else if (reqContext.checkIfEntityIsForCustomAttributeUpdate(entity.getGuid())) { + action = CUSTOM_ATTRIBUTE_UPDATE; + } else if (reqContext.checkIfEntityIsForBusinessAttributeUpdate(entity.getGuid())) { + action = BUSINESS_ATTRIBUTE_UPDATE; + } else { + action = ENTITY_UPDATE; + } + + long auditMaxSize = getAuditMaxSize(auditRepository, entities.size()); + if (DIFFERENTIAL_AUDITS) { + createEvent(updatedEvents.next(), entity, entitiesMap.get(entity.getGuid()), action, auditMaxSize); + } else { + createEvent(updatedEvents.next(), entity, action, auditMaxSize); + } + } auditRepository.putEventsV2(updatedEvents.toList()); } @@ -151,12 +170,12 @@ public void onEntitiesUpdated(List entities, boolean isImport) thro public void onEntitiesDeleted(List entities, boolean isImport) throws AtlasBaseException { MetricRecorder metric = RequestContext.get().startMetricRecord("entityAudit"); - FixedBufferList deletedEntities = getAuditEventsList(); - for (AtlasEntity entity : entities) { - createEvent(deletedEntities.next(), entity, isImport ? ENTITY_IMPORT_DELETE : ENTITY_DELETE, "Deleted entity"); - } - for (EntityAuditRepository auditRepository: auditRepositories) { + FixedBufferList deletedEntities = getAuditEventsList(); + for (AtlasEntity entity : entities) { + long auditMaxSize = getAuditMaxSize(auditRepository, entities.size()); + createEvent(deletedEntities.next(), entity, isImport ? ENTITY_IMPORT_DELETE : ENTITY_DELETE, auditMaxSize); + } auditRepository.putEventsV2(deletedEntities.toList()); } @@ -167,12 +186,12 @@ public void onEntitiesDeleted(List entities, boolean isImport) thro public void onEntitiesPurged(List entities) throws AtlasBaseException { MetricRecorder metric = RequestContext.get().startMetricRecord("entityAudit"); - FixedBufferList eventsPurged = getAuditEventsList(); - for (AtlasEntity entity : entities) { - createEvent(eventsPurged.next(), entity, ENTITY_PURGE); - } - for (EntityAuditRepository auditRepository: auditRepositories) { + FixedBufferList eventsPurged = getAuditEventsList(); + for (AtlasEntity entity : entities) { + long auditMaxSize = getAuditMaxSize(auditRepository, entities.size()); + createEvent(eventsPurged.next(), entity, ENTITY_PURGE, auditMaxSize); + } auditRepository.putEventsV2(eventsPurged.toList()); } @@ -202,7 +221,7 @@ public void onClassificationsAdded(AtlasEntity entity, List } @Override - public void onClassificationsAdded(List entities, List classifications) throws AtlasBaseException { + public void onClassificationsAdded(List entities, List classifications, boolean forceInline) throws AtlasBaseException { if (CollectionUtils.isNotEmpty(classifications)) { MetricRecorder metric = RequestContext.get().startMetricRecord("entityAudit"); FixedBufferList events = getAuditEventsList(); @@ -446,32 +465,49 @@ public void onBusinessAttributesUpdated(AtlasEntity entity, Map prunedAttributes = pruneEntityAttributesForAudit(entity); String auditPrefix = getV2AuditPrefix(action); - String auditString = auditPrefix + AtlasType.toJson(entity); + String auditString = auditPrefix + getAuditString(entity, action); byte[] auditBytes = auditString.getBytes(StandardCharsets.UTF_8); long auditSize = auditBytes != null ? auditBytes.length : 0; - long auditMaxSize = AUDIT_REPOSITORY_MAX_SIZE_DEFAULT; if (auditMaxSize >= 0 && auditSize > auditMaxSize) { // don't store attributes in audit LOG.warn("audit record too long: entityType={}, guid={}, size={}; maxSize={}. entity attribute values not stored in audit", @@ -483,7 +519,7 @@ private String getAuditEventDetail(AtlasEntity entity, EntityAuditActionV2 actio entity.setAttributes(null); entity.setRelationshipAttributes(null); - auditString = auditPrefix + AtlasType.toJson(entity); + auditString = auditPrefix + getAuditString(entity, attrValues, action); auditBytes = auditString.getBytes(StandardCharsets.UTF_8); // recheck auditString size auditSize = auditBytes != null ? auditBytes.length : 0; @@ -502,6 +538,9 @@ private String getAuditEventDetail(AtlasEntity entity, EntityAuditActionV2 actio shallowEntity.setStatus(entity.getStatus()); shallowEntity.setVersion(entity.getVersion()); + //entity.attributes will only have uniqueAttributes + shallowEntity.setAttribute(QUALIFIED_NAME, entity.getAttribute(QUALIFIED_NAME)); + auditString = auditPrefix + AtlasType.toJson(shallowEntity); } @@ -545,6 +584,43 @@ private boolean hasPropagatedEntry(Map> propag return ret; } + private String getAuditString(AtlasEntity entity, Map attrValues, EntityAuditActionV2 action) { + AtlasEntityType entityType = typeRegistry.getEntityTypeByName(entity.getTypeName()); + StringBuilder sb = new StringBuilder(); + + if (action == ENTITY_DELETE || action == ENTITY_IMPORT_DELETE || action == ENTITY_PURGE) { + entity.setAttributes(null); + for (AtlasAttribute attribute : entityType.getUniqAttributes().values()) { + + String attrName = attribute.getName(); + Object attrValue = attrValues.get(attrName); + + entity.setAttribute(attrName, attrValue); + } + } + sb.append(AtlasType.toJson(entity)); + return sb.toString(); + } + + private String getAuditString(AtlasEntity entity, EntityAuditActionV2 action) { + AtlasEntityType entityType = typeRegistry.getEntityTypeByName(entity.getTypeName()); + StringBuilder sb = new StringBuilder(); + + if (action == ENTITY_DELETE || action == ENTITY_IMPORT_DELETE || action == ENTITY_PURGE) { + Map attrValues = new HashMap<>(entity.getAttributes()); + entity.setAttributes(null); + for (AtlasAttribute attribute : entityType.getUniqAttributes().values()) { + + String attrName = attribute.getName(); + Object attrValue = attrValues.get(attrName); + + entity.setAttribute(attrName, attrValue); + } + } + sb.append(AtlasType.toJson(entity)); + return sb.toString(); + } + private Map pruneEntityAttributesForAudit(AtlasEntity entity) { Map ret = null; Map entityAttributes = entity.getAttributes(); diff --git a/repository/src/main/java/org/apache/atlas/repository/audit/HBaseBasedAuditRepository.java b/repository/src/main/java/org/apache/atlas/repository/audit/HBaseBasedAuditRepository.java index a9163e1502c..556cf8a7403 100644 --- a/repository/src/main/java/org/apache/atlas/repository/audit/HBaseBasedAuditRepository.java +++ b/repository/src/main/java/org/apache/atlas/repository/audit/HBaseBasedAuditRepository.java @@ -582,6 +582,7 @@ public List getAuditExcludeAttributes(String entityType) { return ret; } + private String getResultString(Result result, byte[] columnName) { byte[] rawValue = result.getValue(COLUMN_FAMILY, columnName); if ( rawValue != null) { diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java index a26ad563a19..467e72c9432 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java @@ -69,9 +69,9 @@ import static org.apache.atlas.type.Constants.CATEGORIES_PROPERTY_KEY; import static org.apache.atlas.type.Constants.CATEGORIES_PARENT_PROPERTY_KEY; import static org.apache.atlas.type.Constants.GLOSSARY_PROPERTY_KEY; -import static org.apache.atlas.type.Constants.HAS_LINEAGE; import static org.apache.atlas.type.Constants.MEANINGS_PROPERTY_KEY; import static org.apache.atlas.type.Constants.MEANINGS_TEXT_PROPERTY_KEY; +import static org.apache.atlas.type.Constants.MEANING_NAMES_PROPERTY_KEY; import static org.apache.atlas.type.Constants.PENDING_TASKS_PROPERTY_KEY; @@ -332,6 +332,18 @@ private void initialize(AtlasGraph graph) throws RepositoryException, IndexExcep HashMap> KEYWORD_MULTIFIELD = new HashMap<>(); KEYWORD_MULTIFIELD.put("keyword", ES_KEYWORD_FIELD); + HashMap ES_GLOSSARY_ANALYZER_TEXT_FIELD = new HashMap<>(); + ES_GLOSSARY_ANALYZER_TEXT_FIELD.put("type", "text"); + ES_GLOSSARY_ANALYZER_TEXT_FIELD.put("analyzer", "atlan_glossary_analyzer"); + HashMap> ES_GLOSSARY_ANALYZER_MULTIFIELD = new HashMap<>(); + ES_GLOSSARY_ANALYZER_MULTIFIELD.put("text", ES_GLOSSARY_ANALYZER_TEXT_FIELD); + + HashMap ES_SPACE_ANALYZER_TEXT_FIELD = new HashMap<>(); + ES_SPACE_ANALYZER_TEXT_FIELD.put("type", "text"); + ES_SPACE_ANALYZER_TEXT_FIELD.put("analyzer", "whitespace"); + HashMap> ES_SPACE_ANALYZER_MULTIFIELD = new HashMap<>(); + ES_SPACE_ANALYZER_MULTIFIELD.put("text", ES_SPACE_ANALYZER_TEXT_FIELD); + HashMap ES_ATLAN_TEXT_ANALYZER_CONFIG = new HashMap<>(); ES_ATLAN_TEXT_ANALYZER_CONFIG.put("analyzer", "atlan_text_analyzer"); @@ -353,7 +365,7 @@ private void initialize(AtlasGraph graph) throws RepositoryException, IndexExcep createCommonVertexIndex(management, MODIFICATION_TIMESTAMP_PROPERTY_KEY, UniqueKind.NONE, Long.class, SINGLE, false, false, false, new HashMap<>(), TIMESTAMP_MULTIFIELDS); createCommonVertexIndex(management, STATE_PROPERTY_KEY, UniqueKind.NONE, String.class, SINGLE, false, false, true); createCommonVertexIndex(management, CREATED_BY_KEY, UniqueKind.NONE, String.class, SINGLE, false, false, true); - createCommonVertexIndex(management, CLASSIFICATION_TEXT_KEY, UniqueKind.NONE, String.class, SINGLE, false, false); + createCommonVertexIndex(management, CLASSIFICATION_TEXT_KEY, UniqueKind.NONE, String.class, SINGLE, false, false, false, new HashMap<>(), ES_SPACE_ANALYZER_MULTIFIELD); createCommonVertexIndex(management, MODIFIED_BY_KEY, UniqueKind.NONE, String.class, SINGLE, false, false, true); createCommonVertexIndex(management, CLASSIFICATION_NAMES_KEY, UniqueKind.NONE, String.class, SINGLE, true, false); createCommonVertexIndex(management, PROPAGATED_CLASSIFICATION_NAMES_KEY, UniqueKind.NONE, String.class, SINGLE, true, false); @@ -372,10 +384,13 @@ private void initialize(AtlasGraph graph) throws RepositoryException, IndexExcep createCommonVertexIndex(management, PATCH_STATE_PROPERTY_KEY, UniqueKind.NONE, String.class, SINGLE, true, false); createCommonVertexIndex(management, MEANINGS_PROPERTY_KEY, UniqueKind.NONE, String.class, SET, true, false, true); createCommonVertexIndex(management, MEANINGS_TEXT_PROPERTY_KEY, UniqueKind.NONE, String.class, SINGLE, true, false, false, ES_ATLAN_TEXT_COMMA_ANALYZER_CONFIG, new HashMap<>()); +<<<<<<< HEAD +======= + createCommonVertexIndex(management, MEANING_NAMES_PROPERTY_KEY, UniqueKind.NONE, String.class, LIST, true, false, true, new HashMap<>(), ES_GLOSSARY_ANALYZER_MULTIFIELD); +>>>>>>> 7f3c811fb15361ba82bfb4edd7a8393798863ff0 createCommonVertexIndex(management, GLOSSARY_PROPERTY_KEY, UniqueKind.NONE, String.class, SINGLE, true, false, true); createCommonVertexIndex(management, CATEGORIES_PROPERTY_KEY, UniqueKind.NONE, String.class, SET, true, false, true); createCommonVertexIndex(management, CATEGORIES_PARENT_PROPERTY_KEY, UniqueKind.NONE, String.class, SINGLE, true, false, true); - createCommonVertexIndex(management, HAS_LINEAGE, UniqueKind.NONE, Boolean.class, SINGLE, true, false); // tasks @@ -384,6 +399,18 @@ private void initialize(AtlasGraph graph) throws RepositoryException, IndexExcep createCommonVertexIndex(management, TASK_CREATED_TIME, UniqueKind.NONE, Long.class, SINGLE, true, false); createCommonVertexIndex(management, TASK_STATUS, UniqueKind.NONE, String.class, SINGLE, true, false); + createCommonVertexIndex(management, TASK_TYPE, UniqueKind.NONE, String.class, SINGLE, true, false, true); + createCommonVertexIndex(management, TASK_CREATED_BY, UniqueKind.NONE, String.class, SINGLE, false, false, true); + createCommonVertexIndex(management, TASK_CLASSIFICATION_ID, UniqueKind.NONE, String.class, SINGLE, false, false, true); + createCommonVertexIndex(management, TASK_ENTITY_GUID, UniqueKind.NONE, String.class, SINGLE, false, false, true); + createCommonVertexIndex(management, TASK_ERROR_MESSAGE, UniqueKind.NONE, String.class, SINGLE, false, false); + createCommonVertexIndex(management, TASK_ATTEMPT_COUNT, UniqueKind.NONE, Integer.class, SINGLE, false, false); + + createCommonVertexIndex(management, TASK_UPDATED_TIME, UniqueKind.NONE, Long.class, SINGLE, false, false); + createCommonVertexIndex(management, TASK_TIME_TAKEN_IN_SECONDS, UniqueKind.NONE, Long.class, SINGLE, false, false); + createCommonVertexIndex(management, TASK_START_TIME, UniqueKind.NONE, Long.class, SINGLE, false, false); + createCommonVertexIndex(management, TASK_END_TIME, UniqueKind.NONE, Long.class, SINGLE, false, false); + // index recovery createCommonVertexIndex(management, PROPERTY_KEY_INDEX_RECOVERY_NAME, UniqueKind.GLOBAL_UNIQUE, String.class, SINGLE, true, false); @@ -395,6 +422,7 @@ private void initialize(AtlasGraph graph) throws RepositoryException, IndexExcep // create edge indexes createEdgeIndex(management, RELATIONSHIP_GUID_PROPERTY_KEY, String.class, SINGLE, true); createEdgeIndex(management, EDGE_ID_IN_IMPORT_KEY, String.class, SINGLE, true); + createEdgeIndex(management, ATTRIBUTE_INDEX_PROPERTY_KEY, Integer.class, SINGLE, true); // create fulltext indexes createFullTextIndex(management, ENTITY_TEXT_PROPERTY_KEY, String.class, SINGLE); @@ -631,10 +659,17 @@ private void createIndexForAttribute(AtlasGraphManagement management, AtlasStruc if (isArrayOfEnum) { primitiveClassType = String.class; } else { +<<<<<<< HEAD primitiveClassType = isArrayOfPrimitiveType ? getPrimitiveClass(arrayElementType.getTypeName()) : getPrimitiveClass(attribTypeName); } if (primitiveClassType == String.class) { +======= + primitiveClassType = isArrayOfPrimitiveType ? getPrimitiveClass(arrayElementType.getTypeName()): getPrimitiveClass(attribTypeName); + } + + if(primitiveClassType == String.class) { +>>>>>>> 7f3c811fb15361ba82bfb4edd7a8393798863ff0 isStringField = AtlasAttributeDef.IndexType.STRING.equals(indexType); } @@ -643,6 +678,7 @@ private void createIndexForAttribute(AtlasGraphManagement management, AtlasStruc if (uniqPropName != null) { createVertexIndex(management, uniqPropName, UniqueKind.PER_TYPE_UNIQUE, primitiveClassType, cardinality, isIndexable, true, isStringField); } + } } else if (isEnumType(attributeType)) { if (isRelationshipType(atlasType)) { @@ -800,15 +836,15 @@ public String createVertexIndex(AtlasGraphManagement management, String property if (propertyKey == null) { propertyKey = management.makePropertyKey(propertyName, propertyClass, cardinality); + } - if (isIndexApplicable(propertyClass, cardinality)) { - if (LOG.isDebugEnabled()) { - LOG.debug("Creating backing index for vertex property {} of type {} ", propertyName, propertyClass.getName()); - } - - indexFieldName = management.addMixedIndex(VERTEX_INDEX, propertyKey, isStringField, indexTypeESConfig, indexTypeESFields); - LOG.info("Created backing index for vertex property {} of type {} ", propertyName, propertyClass.getName()); + if (isIndexApplicable(propertyClass, cardinality) && !management.getGraphIndex(VERTEX_INDEX).getFieldKeys().contains(propertyKey)) { + if (LOG.isDebugEnabled()) { + LOG.debug("Creating backing index for vertex property {} of type {} ", propertyName, propertyClass.getName()); } + + indexFieldName = management.addMixedIndex(VERTEX_INDEX, propertyKey, isStringField, indexTypeESConfig, indexTypeESFields); + LOG.info("Created backing index for vertex property {} of type {} ", propertyName, propertyClass.getName()); } if(indexFieldName == null && isIndexApplicable(propertyClass, cardinality)) { diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java index c605a4104c8..e4ccfc0a286 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java @@ -33,7 +33,6 @@ import org.apache.atlas.model.instance.AtlasEntityHeader; import org.apache.atlas.model.instance.AtlasObjectId; import org.apache.atlas.model.instance.AtlasRelationship; -import org.apache.atlas.repository.graphdb.AtlasVertexQuery; import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2; import org.apache.atlas.type.AtlasArrayType; import org.apache.atlas.type.AtlasMapType; @@ -51,6 +50,7 @@ import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasGraphQuery; import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.repository.graphdb.AtlasVertexQuery; import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasType; import org.apache.atlas.exception.EntityNotFoundException; @@ -80,9 +80,12 @@ import static org.apache.atlas.repository.Constants.*; import static org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.isReference; +import static org.apache.atlas.repository.store.graph.v2.tasks.ClassificationPropagateTaskFactory.CLASSIFICATION_ONLY_PROPAGATION_DELETE; import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.BOTH; import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.IN; import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.OUT; +import static org.apache.atlas.type.Constants.HAS_LINEAGE; +import static org.apache.atlas.type.Constants.HAS_LINEAGE_VALID; /** * Utility class for graph operations. @@ -100,14 +103,14 @@ public final class GraphHelper { private int maxRetries = 3; private long retrySleepTimeMillis = 1000; - private boolean removePropagations = false; + private boolean removePropagations = true; public GraphHelper(AtlasGraph graph) { this.graph = graph; try { maxRetries = ApplicationProperties.get().getInt(RETRY_COUNT, 3); retrySleepTimeMillis = ApplicationProperties.get().getLong(RETRY_DELAY, 1000); - removePropagations = ApplicationProperties.get().getBoolean(DEFAULT_REMOVE_PROPAGATIONS_ON_ENTITY_DELETE, false); + removePropagations = ApplicationProperties.get().getBoolean(DEFAULT_REMOVE_PROPAGATIONS_ON_ENTITY_DELETE, true); } catch (AtlasException e) { LOG.error("Could not load configuration. Setting to default value for " + RETRY_COUNT, e); } @@ -136,7 +139,7 @@ public AtlasEdge addEdge(AtlasVertex fromVertex, AtlasVertex toVertex, String ed } String fromGuid = getGuid(fromVertex); - if (fromGuid.equals(getGuid(toVertex))) { + if (fromGuid != null && fromGuid.equals(getGuid(toVertex))) { LOG.error("Attempting to create a relationship between same vertex with guid {}", fromGuid); throw new AtlasBaseException(RELATIONSHIP_CREATE_INVALID_PARAMS, fromGuid); } @@ -158,6 +161,18 @@ public AtlasEdge addEdge(AtlasVertex fromVertex, AtlasVertex toVertex, String ed return ret; } + public AtlasEdge getEdge(AtlasVertex outVertex, AtlasVertex inVertex, String edgeLabel) throws RepositoryException, AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("getEdge"); + + if (inVertex.hasEdges(AtlasEdgeDirection.IN, edgeLabel) && outVertex.hasEdges(AtlasEdgeDirection.OUT, edgeLabel)) { + AtlasEdge edge = graph.getEdgeBetweenVertices(outVertex, inVertex, edgeLabel); + return edge; + } + + RequestContext.get().endMetricRecord(metric); + return null; + } + public AtlasEdge getOrCreateEdge(AtlasVertex outVertex, AtlasVertex inVertex, String edgeLabel) throws RepositoryException, AtlasBaseException { AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("getOrCreateEdge"); boolean skipRetry = false; @@ -326,6 +341,18 @@ public static boolean getRemovePropagations(AtlasVertex classificationVertex) { return ret; } + public static boolean getRestrictPropagationThroughLineage(AtlasVertex classificationVertex) { + boolean ret = false; + + if (classificationVertex != null) { + Boolean restrictPropagationThroughLineage = AtlasGraphUtilsV2.getEncodedProperty(classificationVertex, CLASSIFICATION_VERTEX_RESTRICT_PROPAGATE_THROUGH_LINEAGE, Boolean.class); + + ret = (restrictPropagationThroughLineage == null) ? false : restrictPropagationThroughLineage; + } + + return ret; + } + public static AtlasVertex getClassificationVertex(AtlasVertex entityVertex, String classificationName) { AtlasVertex ret = null; Iterable edges = entityVertex.query().direction(AtlasEdgeDirection.OUT).label(CLASSIFICATION_LABEL) @@ -362,6 +389,34 @@ public static AtlasEdge getClassificationEdge(AtlasVertex entityVertex, AtlasVer return ret; } + public static boolean isClassificationAttached(AtlasVertex entityVertex, AtlasVertex classificationVertex) { + AtlasPerfMetrics.MetricRecorder isClassificationAttachedMetricRecorder = RequestContext.get().startMetricRecord("isClassificationAttached"); + String classificationId = classificationVertex.getIdForDisplay(); + try { + Iterator vertices = entityVertex.query() + .direction(AtlasEdgeDirection.OUT) + .label(CLASSIFICATION_LABEL) + .has(CLASSIFICATION_EDGE_NAME_PROPERTY_KEY, getTypeName(classificationVertex)) + .vertices().iterator(); + + if (vertices != null) { + while (vertices.hasNext()) { + AtlasVertex vertex = vertices.next(); + if (vertex != null) { + if (vertex.getIdForDisplay().equals(classificationId)) { + return true; + } + } + } + } + } catch (Exception err) { + throw err; + } finally { + RequestContext.get().endMetricRecord(isClassificationAttachedMetricRecorder); + } + return false; + } + public static AtlasEdge getPropagatedClassificationEdge(AtlasVertex entityVertex, String classificationName, String associatedEntityGuid) { AtlasEdge ret = null; Iterable edges = entityVertex.query().direction(AtlasEdgeDirection.OUT).label(CLASSIFICATION_LABEL) @@ -428,6 +483,25 @@ public static List getPropagatedEdges(AtlasVertex classificationVerte return ret; } + public static List getPropagatedVerticesIds (AtlasVertex classificationVertex) { + List ret = new ArrayList<>(); + Iterator vertices = classificationVertex.query().direction(AtlasEdgeDirection.IN).label(CLASSIFICATION_LABEL) + .has(CLASSIFICATION_EDGE_IS_PROPAGATED_PROPERTY_KEY, true) + .has(CLASSIFICATION_EDGE_NAME_PROPERTY_KEY, getTypeName(classificationVertex)) + .vertices().iterator(); + + if (vertices != null) { + while (vertices.hasNext()) { + AtlasVertex vertex = vertices.next(); + if (vertex != null) { + ret.add(vertex.getIdForDisplay()); + } + } + } + + return ret; + } + public static boolean hasEntityReferences(AtlasVertex classificationVertex) { return classificationVertex.hasEdges(AtlasEdgeDirection.IN, CLASSIFICATION_LABEL); } @@ -737,7 +811,10 @@ public static List getTraitNames(AtlasVertex entityVertex, Boolean propa public static List getPropagatableClassifications(AtlasEdge edge) { List ret = new ArrayList<>(); - if (edge != null && getStatus(edge) != DELETED) { + RequestContext requestContext = RequestContext.get(); + + if ((edge != null && getStatus(edge) != DELETED) || + (requestContext.getCurrentTask() != null && CLASSIFICATION_ONLY_PROPAGATION_DELETE.equals(requestContext.getCurrentTask().getType()))) { PropagateTags propagateTags = getPropagateTags(edge); AtlasVertex outVertex = edge.getOutVertex(); AtlasVertex inVertex = edge.getInVertex(); @@ -753,8 +830,27 @@ public static List getPropagatableClassifications(AtlasEdge edge) { return ret; } + //Returns the vertex from which the tag is being propagated + public static AtlasVertex getPropagatingVertex(AtlasEdge edge) { + if(edge != null) { + PropagateTags propagateTags = getPropagateTags(edge); + AtlasVertex outVertex = edge.getOutVertex(); + AtlasVertex inVertex = edge.getInVertex(); + + if (propagateTags == PropagateTags.ONE_TO_TWO || propagateTags == PropagateTags.BOTH) { + return outVertex; + } + + if (propagateTags == PropagateTags.TWO_TO_ONE || propagateTags == PropagateTags.BOTH) { + return inVertex; + } + + } + return null; + } public static List getPropagationEnabledClassificationVertices(AtlasVertex entityVertex) { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("getPropagationEnabledClassificationVertices"); List ret = new ArrayList<>(); if (entityVertex.hasEdges(AtlasEdgeDirection.OUT, CLASSIFICATION_LABEL)) { Iterable edges = entityVertex.query().direction(AtlasEdgeDirection.OUT).label(CLASSIFICATION_LABEL).edges(); @@ -776,9 +872,25 @@ public static List getPropagationEnabledClassificationVertices(Atla } } + RequestContext.get().endMetricRecord(metricRecorder); return ret; } + public static boolean propagatedClassificationAttachedToVertex(AtlasVertex classificationVertex, AtlasVertex entityVertex) { + Iterator classificationVertices = entityVertex.query().direction(AtlasEdgeDirection.OUT) + .label(CLASSIFICATION_LABEL) + .has(CLASSIFICATION_EDGE_NAME_PROPERTY_KEY, getTypeName(classificationVertex)) + .vertices().iterator(); + + while (classificationVertices.hasNext()) { + String _classificationVertexId = classificationVertices.next().getIdForDisplay(); + if (_classificationVertexId.equals(classificationVertex.getIdForDisplay())) { + return true; + } + } + return false; + } + public static List getClassificationEdges(AtlasVertex entityVertex) { return getClassificationEdges(entityVertex, false); } @@ -875,6 +987,22 @@ public static Boolean isEntityIncomplete(AtlasElement element) { return ret; } + public static Boolean getEntityHasLineage(AtlasElement element) { + if (element.getPropertyKeys().contains(HAS_LINEAGE)) { + return element.getProperty(HAS_LINEAGE, Boolean.class); + } else { + return false; + } + } + + public static Boolean getEntityHasLineageValid(AtlasElement element) { + if (element.getPropertyKeys().contains(HAS_LINEAGE_VALID)) { + return element.getProperty(HAS_LINEAGE_VALID, Boolean.class); + } else { + return false; + } + } + public static Map getCustomAttributes(AtlasElement element) { Map ret = null; String customAttrsString = element.getProperty(CUSTOM_ATTRIBUTES_PROPERTY_KEY, String.class); @@ -1017,6 +1145,10 @@ public static String getModifiedByAsString(AtlasElement element){ return element.getProperty(MODIFIED_BY_KEY, String.class); } + public static void setModifiedByAsString(AtlasElement element, String modifiedBy){ + element.setProperty(MODIFIED_BY_KEY, modifiedBy); + } + public static long getCreatedTime(AtlasElement element){ return element.getProperty(TIMESTAMP_PROPERTY_KEY, Long.class); } @@ -1025,6 +1157,10 @@ public static long getModifiedTime(AtlasElement element){ return element.getProperty(MODIFICATION_TIMESTAMP_PROPERTY_KEY, Long.class); } + public static void setModifiedTime(AtlasElement element, Long modifiedTime) { + element.setProperty(MODIFICATION_TIMESTAMP_PROPERTY_KEY, modifiedTime); + } + public static boolean isActive(AtlasEntity entity) { return entity != null ? entity.getStatus() == ACTIVE : false; } @@ -1381,7 +1517,19 @@ public static List getArrayElementsProperty(AtlasType elementType, Atlas boolean isArrayOfEnum = elementType.getTypeCategory().equals(TypeCategory.ENUM); if (isReference(elementType)) { +<<<<<<< HEAD return (List) getCollectionElementsUsingRelationship(instanceVertex, attribute); +======= + boolean isStruct = TypeCategory.STRUCT == attribute.getDefinedInType().getTypeCategory() || + TypeCategory.STRUCT == elementType.getTypeCategory(); + + if (isStruct) { + String edgeLabel = AtlasGraphUtilsV2.getEdgeLabel(attribute.getName()); + return (List) getCollectionElementsUsingRelationship(instanceVertex, attribute, edgeLabel); + } else { + return (List) getCollectionElementsUsingRelationship(instanceVertex, attribute); + } +>>>>>>> 7f3c811fb15361ba82bfb4edd7a8393798863ff0 } else if (isArrayOfPrimitiveType || isArrayOfEnum) { return (List) instanceVertex.getMultiValuedProperty(propertyName, elementType.getClass()); } else { @@ -1431,8 +1579,19 @@ public static Map getPrimitiveMap(AtlasVertex instanceVertex, St } public static List getCollectionElementsUsingRelationship(AtlasVertex vertex, AtlasAttribute attribute) { + String edgeLabel = attribute.getRelationshipEdgeLabel(); + return getCollectionElementsUsingRelationship(vertex, attribute, edgeLabel); + } + + public static List getCollectionElementsUsingRelationship(AtlasVertex vertex, AtlasAttribute attribute, + boolean isStructType) { + String edgeLabel = isStructType ? AtlasGraphUtilsV2.getEdgeLabel(attribute.getName()) : attribute.getRelationshipEdgeLabel(); + return getCollectionElementsUsingRelationship(vertex, attribute, edgeLabel); + } + + + public static List getCollectionElementsUsingRelationship(AtlasVertex vertex, AtlasAttribute attribute, String edgeLabel) { List ret; - String edgeLabel = attribute.getRelationshipEdgeLabel(); AtlasRelationshipEdgeDirection edgeDirection = attribute.getRelationshipEdgeDirection(); Iterator edgesForLabel = getEdgesForLabel(vertex, edgeLabel, edgeDirection); @@ -1714,6 +1873,45 @@ public static String getDelimitedClassificationNames(Collection classifi return ret; } + /** + * Get all the active parents + * @param vertex entity vertex + * @param parentEdgeLabel Edge label of parent + * @return Iterator of children vertices + */ + public static Iterator getActiveParentVertices(AtlasVertex vertex, String parentEdgeLabel) throws AtlasBaseException { + return getActiveVertices(vertex, parentEdgeLabel, AtlasEdgeDirection.IN); + } + + /** + * Get all the active children of category + * @param vertex entity vertex + * @param childrenEdgeLabel Edge label of children + * @return Iterator of children vertices + */ + public static Iterator getActiveChildrenVertices(AtlasVertex vertex, String childrenEdgeLabel) throws AtlasBaseException { + return getActiveVertices(vertex, childrenEdgeLabel, AtlasEdgeDirection.OUT); + } + + public static Iterator getActiveVertices(AtlasVertex vertex, String childrenEdgeLabel, AtlasEdgeDirection direction) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("CategoryPreProcessor.getEdges"); + + try { + return vertex.query() + .direction(direction) + .label(childrenEdgeLabel) + .has(STATE_PROPERTY_KEY, ACTIVE_STATE_VALUE) + .vertices() + .iterator(); + } catch (Exception e) { + LOG.error("Error while getting active children of category for edge label " + childrenEdgeLabel, e); + throw new AtlasBaseException(AtlasErrorCode.INTERNAL_ERROR, e); + } + finally { + RequestContext.get().endMetricRecord(metricRecorder); + } + } + private static Set parseLabelsString(String labels) { Set ret = new HashSet<>(); @@ -1723,4 +1921,4 @@ private static Set parseLabelsString(String labels) { return ret; } -} \ No newline at end of file +} diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/IndexRecoveryService.java b/repository/src/main/java/org/apache/atlas/repository/graph/IndexRecoveryService.java index 92341468c23..f6154727ac9 100644 --- a/repository/src/main/java/org/apache/atlas/repository/graph/IndexRecoveryService.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/IndexRecoveryService.java @@ -19,6 +19,7 @@ import com.google.common.annotations.VisibleForTesting; import org.apache.atlas.ApplicationProperties; +import org.apache.atlas.AtlasConstants; import org.apache.atlas.AtlasException; import org.apache.atlas.ha.HAConfiguration; import org.apache.atlas.listener.ActiveStateChangeHandler; @@ -26,7 +27,7 @@ import org.apache.atlas.repository.graphdb.AtlasGraphQuery; import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.service.Service; -import org.apache.atlas.util.NanoIdUtils; +import org.apache.atlas.service.redis.RedisService; import org.apache.commons.configuration.Configuration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -41,9 +42,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import static org.apache.atlas.ApplicationProperties.DEFAULT_INDEX_RECOVERY; -import static org.apache.atlas.repository.Constants.PROPERTY_KEY_INDEX_RECOVERY_NAME; -import static org.apache.atlas.repository.Constants.PROPERTY_KEY_INDEX_RECOVERY_PREV_TIME; -import static org.apache.atlas.repository.Constants.PROPERTY_KEY_INDEX_RECOVERY_START_TIME; +import static org.apache.atlas.repository.Constants.*; import static org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.setEncodedProperty; @Component @@ -55,6 +54,7 @@ public class IndexRecoveryService implements Service, ActiveStateChangeHandler { private static final String SOLR_STATUS_CHECK_RETRY_INTERVAL = "atlas.graph.index.status.check.frequency"; private static final String SOLR_INDEX_RECOVERY_CONFIGURED_START_TIME = "atlas.graph.index.recovery.start.time"; private static final long SOLR_STATUS_RETRY_DEFAULT_MS = 30000; // 30 secs default + private static final String ATLAS_INDEX_RECOVERY_LOCK = "atlas:index-recovery:lock"; private final Thread indexHealthMonitor; private final RecoveryInfoManagement recoveryInfoManagement; @@ -63,14 +63,14 @@ public class IndexRecoveryService implements Service, ActiveStateChangeHandler { private RecoveryThread recoveryThread; @Inject - public IndexRecoveryService(Configuration config, AtlasGraph graph) { + public IndexRecoveryService(Configuration config, AtlasGraph graph, RedisService redisService) { this.configuration = config; this.isIndexRecoveryEnabled = config.getBoolean(ApplicationProperties.INDEX_RECOVERY_CONF, DEFAULT_INDEX_RECOVERY); long recoveryStartTimeFromConfig = getRecoveryStartTimeFromConfig(config); long healthCheckFrequencyMillis = config.getLong(SOLR_STATUS_CHECK_RETRY_INTERVAL, SOLR_STATUS_RETRY_DEFAULT_MS); this.recoveryInfoManagement = new RecoveryInfoManagement(graph); - this.recoveryThread = new RecoveryThread(recoveryInfoManagement, graph, recoveryStartTimeFromConfig, healthCheckFrequencyMillis); + this.recoveryThread = new RecoveryThread(recoveryInfoManagement, graph, recoveryStartTimeFromConfig, healthCheckFrequencyMillis, redisService); this.indexHealthMonitor = new Thread(recoveryThread, INDEX_HEALTH_MONITOR_THREAD_NAME); } @@ -106,7 +106,6 @@ public void start() throws AtlasException { public void stop() throws AtlasException { try { recoveryThread.shutdown(); - indexHealthMonitor.join(); } catch (InterruptedException e) { LOG.error("indexHealthMonitor: Interrupted", e); @@ -153,13 +152,16 @@ private static class RecoveryThread implements Runnable { private final RecoveryInfoManagement recoveryInfoManagement; private long indexStatusCheckRetryMillis; private Object txRecoveryObject; + private final RedisService redisService; private final AtomicBoolean shouldRun = new AtomicBoolean(false); - private RecoveryThread(RecoveryInfoManagement recoveryInfoManagement, AtlasGraph graph, long startTimeFromConfig, long healthCheckFrequencyMillis) { + private RecoveryThread(RecoveryInfoManagement recoveryInfoManagement, AtlasGraph graph, long startTimeFromConfig, long healthCheckFrequencyMillis, + RedisService redisService) { this.graph = graph; this.recoveryInfoManagement = recoveryInfoManagement; this.indexStatusCheckRetryMillis = healthCheckFrequencyMillis; + this.redisService = redisService; if (startTimeFromConfig > 0) { this.recoveryInfoManagement.updateStartTime(startTimeFromConfig); @@ -174,6 +176,10 @@ public void run() { while (true) { if (shouldRun.get()) { try { + if(!redisService.acquireDistributedLock(ATLAS_INDEX_RECOVERY_LOCK)){ + Thread.sleep(AtlasConstants.TASK_WAIT_TIME_MS); + continue; + } boolean indexHealthy = isIndexHealthy(); if (this.txRecoveryObject == null && indexHealthy) { @@ -186,6 +192,9 @@ public void run() { } catch (Exception e) { LOG.error("Error: Index recovery monitoring!", e); } + finally { + redisService.releaseDistributedLock(ATLAS_INDEX_RECOVERY_LOCK); + } } } } @@ -201,6 +210,7 @@ public void shutdown() { } shouldRun.set(false); + this.redisService.releaseDistributedLock(ATLAS_INDEX_RECOVERY_LOCK); } finally { LOG.info("Index Health Monitor: Shutdown: Done!"); } diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/SolrIndexHelper.java b/repository/src/main/java/org/apache/atlas/repository/graph/SolrIndexHelper.java index 401bc024a53..0d349382e99 100644 --- a/repository/src/main/java/org/apache/atlas/repository/graph/SolrIndexHelper.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/SolrIndexHelper.java @@ -78,6 +78,8 @@ public void onChange(ChangedTypeDefs changedTypeDefs) { return; } + LOG.info("SolrIndexHelper:initializationCompleted: {}", initializationCompleted); + if(initializationCompleted) { try { AtlasGraph graph = AtlasGraphProvider.getGraphInstance(); diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/TypeCacheRefresher.java b/repository/src/main/java/org/apache/atlas/repository/graph/TypeCacheRefresher.java new file mode 100644 index 00000000000..ac80b4b6ca7 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/graph/TypeCacheRefresher.java @@ -0,0 +1,190 @@ +package org.apache.atlas.repository.graph; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.atlas.ApplicationProperties; +import org.apache.atlas.AtlasException; +import org.apache.atlas.RequestContext; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.ha.HAConfiguration; +import org.apache.atlas.repository.RepositoryException; +import org.apache.commons.configuration.Configuration; +import org.apache.commons.lang.StringUtils; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.utils.URIBuilder; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.util.EntityUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import javax.inject.Inject; +import java.io.IOException; +import java.net.URISyntaxException; +import java.util.*; + +import static org.apache.atlas.AtlasErrorCode.CINV_UNHEALTHY; +import static org.apache.atlas.repository.Constants.VERTEX_INDEX; + +@Component +public class TypeCacheRefresher { + private static final Logger LOG = LoggerFactory.getLogger(TypeCacheRefresher.class); + private String cacheRefresherEndpoint; + private String cacheRefresherHealthEndpoint; + private final IAtlasGraphProvider provider; + private boolean isActiveActiveHAEnabled; + + @Inject + public TypeCacheRefresher(final IAtlasGraphProvider provider) { + this.provider = provider; + } + + @PostConstruct + public void init() throws AtlasException { + Configuration configuration = ApplicationProperties.get(); + this.cacheRefresherEndpoint = configuration.getString("atlas.server.type.cache-refresher"); + this.cacheRefresherHealthEndpoint = configuration.getString("atlas.server.type.cache-refresher-health"); + this.isActiveActiveHAEnabled = HAConfiguration.isActiveActiveHAEnabled(configuration); + LOG.info("Found {} as cache-refresher endpoint", cacheRefresherEndpoint); + LOG.info("Found {} as cache-refresher-health endpoint", cacheRefresherHealthEndpoint); + } + + public void verifyCacheRefresherHealth() throws AtlasBaseException, IOException { + if (StringUtils.isBlank(cacheRefresherHealthEndpoint) || !isActiveActiveHAEnabled) { + LOG.info("Skipping type-def cache refresher health checking as URL is {} and isActiveActiveHAEnabled is {}", cacheRefresherHealthEndpoint, isActiveActiveHAEnabled); + return; + } + final String healthResponseBody; + try (CloseableHttpClient client = HttpClients.createDefault()) { + final HttpGet healthRequest = new HttpGet(cacheRefresherHealthEndpoint); + healthResponseBody = executeGet(client, healthRequest); + } + LOG.debug("Response Body from cache-refresh-health = {}", healthResponseBody); + final ObjectMapper mapper = new ObjectMapper(); + final CacheRefresherHealthResponse jsonResponse = mapper.readValue(healthResponseBody, CacheRefresherHealthResponse.class); + if (!"Healthy".equalsIgnoreCase(jsonResponse.getMessage())) { + throw new AtlasBaseException(CINV_UNHEALTHY); + } + } + + public void refreshAllHostCache() throws IOException, URISyntaxException, RepositoryException { + final String traceId = RequestContext.get().getTraceId(); + if(StringUtils.isBlank(cacheRefresherEndpoint) || !isActiveActiveHAEnabled) { + LOG.info("Skipping type-def cache refresh :: traceId {}", traceId); + return; + } + + int totalFieldKeys = provider.get().getManagementSystem().getGraphIndex(VERTEX_INDEX).getFieldKeys().size(); + LOG.info("Found {} totalFieldKeys to be expected in other nodes :: traceId {}", totalFieldKeys, traceId); + refreshCache(totalFieldKeys, traceId); + } + + private void refreshCache(final int totalFieldKeys, final String traceId) throws IOException, URISyntaxException { + URIBuilder builder = new URIBuilder(cacheRefresherEndpoint); + builder.setParameter("expectedFieldKeys", String.valueOf(totalFieldKeys)); + builder.setParameter("traceId", traceId); + final HttpPost httpPost = new HttpPost(builder.build()); + LOG.info("Invoking cache refresh endpoint {} :: traceId {}", cacheRefresherEndpoint, traceId); + + String responseBody; + try (CloseableHttpClient client = HttpClients.createDefault()) { + responseBody = executePost(traceId, client, httpPost); + } + LOG.info("Response Body from cache-refresh = {} :: traceId {}", responseBody, traceId); + CacheRefreshResponseEnvelope cacheRefreshResponseEnvelope = convertStringToObject(responseBody); + + for (CacheRefreshResponse responseOfEachNode : cacheRefreshResponseEnvelope.getResponse()) { + if (responseOfEachNode.getStatus() != 204) { + //Do not throw exception in this case as node must have been in passive state now + LOG.error("Error while performing cache refresh on host {} . HTTP code = {} :: traceId {}", responseOfEachNode.getHost(), + responseOfEachNode.getStatus(), traceId); + } else { + LOG.info("Host {} returns response code {} :: traceId {}", responseOfEachNode.getHost(), responseOfEachNode.getStatus(), traceId); + } + } + LOG.info("Refreshed cache successfully on all hosts :: traceId {}", traceId); + } + + private String executePost(String traceId, CloseableHttpClient client, HttpPost httpPost) throws IOException { + try (CloseableHttpResponse response = client.execute(httpPost)) { + LOG.info("Received HTTP response code {} from cache refresh endpoint :: traceId {}", response.getStatusLine().getStatusCode(), traceId); + if (response.getStatusLine().getStatusCode() != 200) { + throw new RuntimeException("Error while calling cache-refresher on host " + cacheRefresherEndpoint + ". HTTP code = " + response.getStatusLine().getStatusCode() + " :: traceId " + traceId); + } + return EntityUtils.toString(response.getEntity()); + } + } + + private String executeGet(CloseableHttpClient client, HttpGet getRequest) throws IOException, AtlasBaseException { + try (CloseableHttpResponse closeableHttpResponse = client.execute(getRequest)) { + LOG.info("Received HTTP response code {} from cache refresh health endpoint", closeableHttpResponse.getStatusLine().getStatusCode()); + if (closeableHttpResponse.getStatusLine().getStatusCode() != 200) { + throw new AtlasBaseException(CINV_UNHEALTHY); + } + return EntityUtils.toString(closeableHttpResponse.getEntity()); + } + } + + private CacheRefreshResponseEnvelope convertStringToObject(final String responseBody) throws JsonProcessingException { + final ObjectMapper mapper = new ObjectMapper(); + return mapper.readValue(responseBody, CacheRefreshResponseEnvelope.class); + } +} + +class CacheRefreshResponseEnvelope { + private List response; + + public List getResponse() { + return response; + } + + public void setResponse(List response) { + this.response = response; + } +} + +class CacheRefreshResponse { + private String host; + private int status; + private Map headers; + + public String getHost() { + return host; + } + + public void setHost(String host) { + this.host = host; + } + + public int getStatus() { + return status; + } + + public void setStatus(int status) { + this.status = status; + } + + public Map getHeaders() { + return headers; + } + + public void setHeaders(Map headers) { + this.headers = headers; + } +} + +class CacheRefresherHealthResponse { + private String message; + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } +} \ No newline at end of file diff --git a/repository/src/main/java/org/apache/atlas/repository/impexp/ImportService.java b/repository/src/main/java/org/apache/atlas/repository/impexp/ImportService.java index 1d29bf8332a..344e44ca064 100644 --- a/repository/src/main/java/org/apache/atlas/repository/impexp/ImportService.java +++ b/repository/src/main/java/org/apache/atlas/repository/impexp/ImportService.java @@ -113,12 +113,12 @@ AtlasImportResult run(EntityImportStream source, AtlasImportRequest request, Str setEntityTransformerHandlers(source, transformers); startTimestamp = System.currentTimeMillis(); - processTypes(source.getTypesDef(), result); + //processTypes(source.getTypesDef(), result); setStartPosition(request, source); processEntities(userName, source, result); - processReplicationDeletion(source.getExportResult().getRequest(), request); + //processReplicationDeletion(source.getExportResult().getRequest(), request); } catch (AtlasBaseException excp) { LOG.error("import(user={}, from={}): failed", userName, requestingIP, excp); @@ -239,7 +239,7 @@ private void processEntities(String userName, EntityImportStream importSource, A return; } - auditsWriter.write(userName, result, startTimestamp, endTimestamp, importSource.getCreationOrder()); + //auditsWriter.write(userName, result, startTimestamp, endTimestamp, importSource.getCreationOrder()); } private void processReplicationDeletion(AtlasExportRequest exportRequest, AtlasImportRequest importRequest) throws AtlasBaseException { diff --git a/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSourceWithBackingDirectory.java b/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSourceWithBackingDirectory.java index 79638009d99..448ed53d1c6 100644 --- a/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSourceWithBackingDirectory.java +++ b/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSourceWithBackingDirectory.java @@ -45,11 +45,15 @@ import java.util.zip.ZipInputStream; import static org.apache.atlas.AtlasErrorCode.IMPORT_ATTEMPTING_EMPTY_ZIP; +import static org.apache.atlas.AtlasErrorCode.IMPORT_INVALID_ZIP_ENTRY; public class ZipSourceWithBackingDirectory implements EntityImportStream { private static final Logger LOG = LoggerFactory.getLogger(ZipSourceWithBackingDirectory.class); private static final String TEMPORARY_DIRECTORY_PREFIX = "atlas-import-temp-"; private static final String EXT_JSON = ".json"; + private static final String RELATIVE_PARENT_PATH = ".."; + private static final String RELATIVE_PARENT_PATH_WITH_SEP_PREFIX = File.separator + RELATIVE_PARENT_PATH; + private static final String RELATIVE_PARENT_PATH_WITH_SEP_SUFFIX = RELATIVE_PARENT_PATH + File.separator; private Path tempDirectory; @@ -174,7 +178,11 @@ public AtlasEntity getByGuid(String guid) { @Override public void onImportComplete(String guid) { - getFileFromTemporaryDirectory(guid + EXT_JSON).delete(); + try { + getFileFromTemporaryDirectory(guid + EXT_JSON).delete(); + } catch (AtlasBaseException excp) { + LOG.error("onImportComplete(guid={}): failed", guid, excp); + } } @Override @@ -277,14 +285,39 @@ private void unzipToTempDirectory(InputStream inputStream) throws IOException { } private void writeJsonToFile(String entryName, byte[] jsonPayload) throws IOException { - File f = getFileFromTemporaryDirectory(entryName); - Files.write(f.toPath(), jsonPayload); + try { + File f = getFileFromTemporaryDirectory(entryName); + Files.write(f.toPath(), jsonPayload); + } catch (AtlasBaseException excp) { + LOG.error("writeJsonToFile(entryName={}): failed", entryName, excp); + + throw new IOException(excp); + } } - private File getFileFromTemporaryDirectory(String entryName) { + private File getFileFromTemporaryDirectory(String entryName) throws AtlasBaseException { + if (hasRelativeParentPath(entryName)) { + LOG.error("failed to initialize import: found zipEntry having relative parent path - {}", entryName); + + throw new AtlasBaseException(IMPORT_INVALID_ZIP_ENTRY, entryName, "has relative parent path"); + } return new File(tempDirectory.toFile(), entryName); } + private boolean hasRelativeParentPath(String path) { + final boolean ret; + + if (path == null || !path.contains(RELATIVE_PARENT_PATH)) { + ret = false; + } else { + ret = path.startsWith(RELATIVE_PARENT_PATH) || + path.contains(RELATIVE_PARENT_PATH_WITH_SEP_PREFIX) || + path.contains(RELATIVE_PARENT_PATH_WITH_SEP_SUFFIX); + } + + return ret; + } + private void setupIterator() { try { creationOrder = getJsonFromEntry(ZipExportFileNames.ATLAS_EXPORT_ORDER_NAME.toString(), ArrayList.class); diff --git a/repository/src/main/java/org/apache/atlas/repository/migration/DataMigrationStatusService.java b/repository/src/main/java/org/apache/atlas/repository/migration/DataMigrationStatusService.java index 5b22f9cd54a..3d357ddcfad 100644 --- a/repository/src/main/java/org/apache/atlas/repository/migration/DataMigrationStatusService.java +++ b/repository/src/main/java/org/apache/atlas/repository/migration/DataMigrationStatusService.java @@ -59,7 +59,7 @@ public void init(String fileToImport) { LOG.error("Not able to create Migration status", e); } - if (!this.migrationStatusVertexManagement.exists(fileToImport)) { + if (!this.migrationStatusVertexManagement.exists(status.getFileHash())) { return; } diff --git a/repository/src/main/java/org/apache/atlas/repository/migration/ZipFileMigrationImporter.java b/repository/src/main/java/org/apache/atlas/repository/migration/ZipFileMigrationImporter.java index 2b3f1792c3f..021aaf02d95 100644 --- a/repository/src/main/java/org/apache/atlas/repository/migration/ZipFileMigrationImporter.java +++ b/repository/src/main/java/org/apache/atlas/repository/migration/ZipFileMigrationImporter.java @@ -251,6 +251,7 @@ private void performImport(String fileToImport, int streamSize, String startPosi LOG.info("Migration Import: {}: Starting at: {}...", fileToImport, startPosition); InputStream fs = new FileInputStream(fileToImport); RequestContext.get().setUser(getUserNameFromEnvironment(), null); + RequestContext.get().setImportInProgress(true); importService.run(fs, getImportRequest(fileToImport, streamSize, startPosition), getUserNameFromEnvironment(), @@ -266,7 +267,7 @@ private void performImport(String fileToImport, int streamSize, String startPosi } private String getUserNameFromEnvironment() { - return System.getProperty(ENV_USER_NAME); + return RequestContext.get().getUser(); } private AtlasImportRequest getImportRequest(String fileToImport, int streamSize, String position) throws AtlasException { diff --git a/repository/src/main/java/org/apache/atlas/repository/patches/AtlasPatchManager.java b/repository/src/main/java/org/apache/atlas/repository/patches/AtlasPatchManager.java index 44cd8efff13..d9201bf82a9 100644 --- a/repository/src/main/java/org/apache/atlas/repository/patches/AtlasPatchManager.java +++ b/repository/src/main/java/org/apache/atlas/repository/patches/AtlasPatchManager.java @@ -88,13 +88,6 @@ private void init() { this.context = new PatchContext(atlasGraph, typeRegistry, indexer, entityGraphMapper); // register all java patches here - handlers.add(new UniqueAttributePatch(context)); - handlers.add(new ClassificationTextPatch(context)); - handlers.add(new FreeTextRequestHandlerPatch(context)); - handlers.add(new SuggestionsRequestHandlerPatch(context)); - handlers.add(new IndexConsistencyPatch(context)); - handlers.add(new ReIndexPatch(context)); - handlers.add(new ProcessNamePatch(context)); handlers.add(new UpdateCompositeIndexStatusPatch(context)); LOG.info("<== AtlasPatchManager.init()"); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/aliasstore/ESAliasStore.java b/repository/src/main/java/org/apache/atlas/repository/store/aliasstore/ESAliasStore.java new file mode 100644 index 00000000000..b6211041c82 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/store/aliasstore/ESAliasStore.java @@ -0,0 +1,234 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.repository.store.aliasstore; + +import org.apache.atlas.AtlasConfiguration; +import org.apache.atlas.AtlasErrorCode; +import org.apache.atlas.ESAliasRequestBuilder; +import org.apache.atlas.ESAliasRequestBuilder.AliasAction; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.repository.graphdb.AtlasGraph; +import org.apache.atlas.repository.graphdb.janus.AtlasElasticsearchDatabase; +import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; +import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest; +import org.elasticsearch.client.GetAliasesResponse; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.cluster.metadata.AliasMetadata; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import javax.inject.Inject; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.apache.atlas.ESAliasRequestBuilder.ESAliasAction.ADD; +import static org.apache.atlas.repository.Constants.PERSONA_ENTITY_TYPE; +import static org.apache.atlas.repository.Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY; +import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; +import static org.apache.atlas.repository.Constants.TRAIT_NAMES_PROPERTY_KEY; +import static org.apache.atlas.repository.Constants.VERTEX_INDEX_NAME; +import static org.apache.atlas.repository.util.AccessControlUtils.ACCESS_READ_PERSONA_METADATA; +import static org.apache.atlas.repository.util.AccessControlUtils.ACCESS_READ_PERSONA_GLOSSARY; +import static org.apache.atlas.repository.util.AccessControlUtils.getConnectionQualifiedNameFromPolicyAssets; +import static org.apache.atlas.repository.util.AccessControlUtils.getESAliasName; +import static org.apache.atlas.repository.util.AccessControlUtils.getIsAllowPolicy; +import static org.apache.atlas.repository.util.AccessControlUtils.getPolicies; +import static org.apache.atlas.repository.util.AccessControlUtils.getPolicyActions; +import static org.apache.atlas.repository.util.AccessControlUtils.getPolicyAssets; +import static org.apache.atlas.repository.util.AccessControlUtils.getPolicyConnectionQN; +import static org.apache.atlas.repository.util.AccessControlUtils.getPurposeTags; +import static org.apache.atlas.repository.util.AtlasEntityUtils.mapOf; + + +@Component +public class ESAliasStore implements IndexAliasStore { + private static final Logger LOG = LoggerFactory.getLogger(ESAliasStore.class); + + private final AtlasGraph graph; + private final EntityGraphRetriever entityRetriever; + + private final int assetsMaxLimit = AtlasConfiguration.PERSONA_POLICY_ASSET_MAX_LIMIT.getInt(); + + @Inject + public ESAliasStore(AtlasGraph graph, + EntityGraphRetriever entityRetriever) { + this.graph = graph; + this.entityRetriever = entityRetriever; + } + + @Override + public boolean createAlias(AtlasEntity entity) throws AtlasBaseException { + String aliasName = getAliasName(entity); + + ESAliasRequestBuilder requestBuilder = new ESAliasRequestBuilder(); + + if (PERSONA_ENTITY_TYPE.equals(entity.getTypeName())) { + requestBuilder.addAction(ADD, new AliasAction(getIndexNameFromAliasIfExists(VERTEX_INDEX_NAME), aliasName)); + } else { + requestBuilder.addAction(ADD, new AliasAction(getIndexNameFromAliasIfExists(VERTEX_INDEX_NAME), aliasName, getFilterForPurpose(entity))); + } + + graph.createOrUpdateESAlias(requestBuilder); + return true; + } + + private String getIndexNameFromAliasIfExists(final String aliasIndexName) throws AtlasBaseException { + try { + RestHighLevelClient esClient = AtlasElasticsearchDatabase.getClient(); + GetAliasesRequest aliasesRequest = new GetAliasesRequest(aliasIndexName); + GetAliasesResponse aliasesResponse = esClient.indices().getAlias(aliasesRequest, RequestOptions.DEFAULT); + Map> aliases = aliasesResponse.getAliases(); + for (Map.Entry> entry : aliases.entrySet()) { + String indexName = entry.getKey(); + Set aliasMetadataList = entry.getValue(); + for (AliasMetadata aliasMetadata : aliasMetadataList) { + if (aliasIndexName.equals(aliasMetadata.alias())) { + return indexName; + } + } + } + } catch (Exception e) { + LOG.error("Error while fetching index for alias index {}", aliasIndexName, e); + throw new AtlasBaseException(e); + } + return aliasIndexName; + } + + @Override + public boolean updateAlias(AtlasEntity.AtlasEntityWithExtInfo accessControl, AtlasEntity policy) throws AtlasBaseException { + String aliasName = getAliasName(accessControl.getEntity()); + + Map filter; + + if (PERSONA_ENTITY_TYPE.equals(accessControl.getEntity().getTypeName())) { + filter = getFilterForPersona(accessControl, policy); + } else { + filter = getFilterForPurpose(accessControl.getEntity()); + } + + ESAliasRequestBuilder requestBuilder = new ESAliasRequestBuilder(); + requestBuilder.addAction(ADD, new AliasAction(getIndexNameFromAliasIfExists(VERTEX_INDEX_NAME), aliasName, filter)); + + graph.createOrUpdateESAlias(requestBuilder); + + return true; + } + + @Override + public boolean deleteAlias(String aliasName) throws AtlasBaseException { + graph.deleteESAlias(getIndexNameFromAliasIfExists(VERTEX_INDEX_NAME), aliasName); + return true; + } + + private Map getFilterForPersona(AtlasEntity.AtlasEntityWithExtInfo persona, AtlasEntity policy) throws AtlasBaseException { + List> allowClauseList = new ArrayList<>(); + + List policies = getPolicies(persona); + if (policy != null) { + policies.add(policy); + } + if (CollectionUtils.isNotEmpty(policies)) { + personaPolicyToESDslClauses(policies, allowClauseList); + } + + return esClausesToFilter(allowClauseList); + } + + private Map getFilterForPurpose(AtlasEntity purpose) throws AtlasBaseException { + + List> allowClauseList = new ArrayList<>(); + + List tags = getPurposeTags(purpose); + addPurposeMetadataFilterClauses(tags, allowClauseList); + + return esClausesToFilter(allowClauseList); + } + + private void personaPolicyToESDslClauses(List policies, + List> allowClauseList) throws AtlasBaseException { + List terms = new ArrayList<>(); + + for (AtlasEntity policy: policies) { + + if (policy.getStatus() == null || AtlasEntity.Status.ACTIVE.equals(policy.getStatus())) { + List assets = getPolicyAssets(policy); + + if (!getIsAllowPolicy(policy)) { + continue; + } + + if (getPolicyActions(policy).contains(ACCESS_READ_PERSONA_METADATA)) { + + String connectionQName = getPolicyConnectionQN(policy); + if (StringUtils.isEmpty(connectionQName)) { + connectionQName = getConnectionQualifiedNameFromPolicyAssets(entityRetriever, assets); + } + + for (String asset : assets) { + terms.add(asset); + allowClauseList.add(mapOf("wildcard", mapOf(QUALIFIED_NAME, asset + "/*"))); + } + + terms.add(connectionQName); + + } else if (getPolicyActions(policy).contains(ACCESS_READ_PERSONA_GLOSSARY)) { + + for (String glossaryQName : assets) { + terms.add(glossaryQName); + allowClauseList.add(mapOf("wildcard", mapOf(QUALIFIED_NAME, "*@" + glossaryQName))); + } + } + } + + if (terms.size() > assetsMaxLimit) { + throw new AtlasBaseException(AtlasErrorCode.PERSONA_POLICY_ASSETS_LIMIT_EXCEEDED, String.valueOf(assetsMaxLimit), String.valueOf(terms.size())); + } + } + + allowClauseList.add(mapOf("terms", mapOf(QUALIFIED_NAME, terms))); + } + + private Map esClausesToFilter(List> allowClauseList) { + if (CollectionUtils.isNotEmpty(allowClauseList)) { + return mapOf("bool", mapOf("should", allowClauseList)); + } + return null; + } + + private Map getEmptyFilter() { + return mapOf("match_none", new HashMap<>()); + } + + private String getAliasName(AtlasEntity entity) { + return getESAliasName(entity); + } + + private void addPurposeMetadataFilterClauses(List tags, List> clauseList) { + clauseList.add(mapOf("terms", mapOf(TRAIT_NAMES_PROPERTY_KEY, tags))); + clauseList.add(mapOf("terms", mapOf(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, tags))); + } +} diff --git a/repository/src/main/java/org/apache/atlas/repository/store/aliasstore/IndexAliasStore.java b/repository/src/main/java/org/apache/atlas/repository/store/aliasstore/IndexAliasStore.java new file mode 100644 index 00000000000..b551668754f --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/store/aliasstore/IndexAliasStore.java @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.repository.store.aliasstore; + +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.instance.AtlasEntity; +import org.springframework.stereotype.Component; + +@Component +public interface IndexAliasStore { + + public boolean createAlias(AtlasEntity entity) throws AtlasBaseException; + + public boolean updateAlias(AtlasEntity.AtlasEntityWithExtInfo accessControl, AtlasEntity policy) throws AtlasBaseException; + + public boolean deleteAlias(String aliasName) throws AtlasBaseException; +} diff --git a/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java b/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java index 89e9422cd01..2a867452b67 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java @@ -26,6 +26,7 @@ import org.apache.atlas.RequestContext; import org.apache.atlas.authorize.AtlasAuthorizerFactory; import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.featureflag.FeatureFlagStore; import org.apache.atlas.ha.HAConfiguration; import org.apache.atlas.listener.ActiveStateChangeHandler; import org.apache.atlas.model.TypeCategory; @@ -103,7 +104,7 @@ public class AtlasTypeDefStoreInitializer implements ActiveStateChangeHandler { @Inject public AtlasTypeDefStoreInitializer(AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry, - AtlasGraph graph, Configuration conf, AtlasPatchManager patchManager) { + AtlasGraph graph, Configuration conf, AtlasPatchManager patchManager) throws AtlasBaseException { this.typeDefStore = typeDefStore; this.typeRegistry = typeRegistry; this.conf = conf; @@ -383,7 +384,7 @@ private void startInternal() { loadBootstrapTypeDefs(); typeDefStore.notifyLoadCompletion(); try { - AtlasAuthorizerFactory.getAtlasAuthorizer(); + AtlasAuthorizerFactory.getAtlasAuthorizer(typeRegistry); } catch (Throwable t) { LOG.error("AtlasTypeDefStoreInitializer.instanceIsActive(): Unable to obtain AtlasAuthorizer", t); } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AuthPoliciesBootstrapper.java b/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AuthPoliciesBootstrapper.java new file mode 100644 index 00000000000..9b1327ba1fc --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AuthPoliciesBootstrapper.java @@ -0,0 +1,171 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.repository.store.bootstrap; + +import org.apache.atlas.ApplicationProperties; +import org.apache.atlas.AtlasException; +import org.apache.atlas.RequestContext; +import org.apache.atlas.listener.ActiveStateChangeHandler; +import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo; +import org.apache.atlas.repository.graphdb.AtlasGraph; +import org.apache.atlas.repository.store.graph.AtlasEntityStore; +import org.apache.atlas.repository.store.graph.v2.AtlasEntityStream; +import org.apache.atlas.repository.store.graph.v2.EntityStream; +import org.apache.atlas.service.Service; +import org.apache.atlas.type.AtlasType; +import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.ArrayUtils; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.core.annotation.Order; +import org.springframework.stereotype.Component; + +import javax.inject.Inject; +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.Arrays; + +@Component +@Order(9) +public class AuthPoliciesBootstrapper implements ActiveStateChangeHandler, Service { + public static final Logger LOG = LoggerFactory.getLogger(AuthPoliciesBootstrapper.class); + + private final AtlasGraph graph; + private final AtlasTypeRegistry typeRegistry; + private final AtlasEntityStore entityStore; + + @Inject + public AuthPoliciesBootstrapper(AtlasGraph graph, + AtlasEntityStore entityStore, + AtlasTypeRegistry typeRegistry) { + this.graph = graph; + this.entityStore = entityStore; + this.typeRegistry = typeRegistry; + } + + private void startInternal() { + try { + String authorizer = ApplicationProperties.get().getString("atlas.authorizer.impl", ""); + + if ("atlas".equalsIgnoreCase(authorizer)) { + loadBootstrapAuthPolicies(); + } else { + LOG.info("AuthPoliciesBootstrapper: startInternal: Skipping as not needed"); + } + } catch (Exception e) { + LOG.error("Failed to init after becoming active", e); + } finally { + RequestContext.clear(); + } + } + + private void loadBootstrapAuthPolicies() { + LOG.info("==> AuthPoliciesBootstrapper.loadBootstrapAuthPolicies()"); + + RequestContext.get().setSkipAuthorizationCheck(true); + + try { + String atlasHomeDir = System.getProperty("atlas.home"); + String policiesDirName = (StringUtils.isEmpty(atlasHomeDir) ? "." : atlasHomeDir) + File.separator + "policies"; + + File topPoliciesDir = new File(policiesDirName); + loadPoliciesInFolder(topPoliciesDir); + } finally { + RequestContext.get().setSkipAuthorizationCheck(false); + } + + LOG.info("<== AuthPoliciesBootstrapper.loadBootstrapAuthPolicies()"); + } + + private void loadPoliciesInFolder (File folder) { + LOG.info("==> AuthPoliciesBootstrapper.loadPoliciesInFolder({})", folder); + + String policiesDirName = folder.getName(); + File[] policyFiles = folder.exists() ? folder.listFiles() : null; + + if (ArrayUtils.isNotEmpty(policyFiles)) { + Arrays.sort(policyFiles); + + for (File item : policyFiles) { + if (!item.isFile()) { + loadPoliciesInFolder(item); + } else { + loadPoliciesInFile(item); + } + } + } else { + LOG.warn("No policies for Bootstrapping in directory {}..", policiesDirName); + } + + LOG.info("<== AuthPoliciesBootstrapper.loadPoliciesInFolder({})", folder); + } + + private void loadPoliciesInFile (File policyFile) { + LOG.info("==> AuthPoliciesBootstrapper.loadPoliciesInFile({})", policyFile); + + try { + String jsonStr = new String(Files.readAllBytes(policyFile.toPath()), StandardCharsets.UTF_8); + AtlasEntitiesWithExtInfo policies = AtlasType.fromJson(jsonStr, AtlasEntitiesWithExtInfo.class); + + if (policies == null || CollectionUtils.isEmpty(policies.getEntities())) { + LOG.info("No policy in file {}", policyFile.getAbsolutePath()); + + return; + } + + EntityStream entityStream = new AtlasEntityStream(policies); + + entityStore.createOrUpdate(entityStream, false); + + } catch (Throwable t) { + LOG.error("error while registering policies in file {}", policyFile.getAbsolutePath(), t); + } + + + LOG.info("<== AuthPoliciesBootstrapper.loadPoliciesInFile({})", policyFile); + } + + @Override + public void instanceIsActive() throws AtlasException { + startInternal(); + } + + @Override + public void instanceIsPassive() { + + } + + @Override + public int getHandlerOrder() { + return HandlerOrder.AUTH_POLICIES_INITIALIZER.getOrder(); + } + + @Override + public void start() throws AtlasException { + startInternal(); + } + + @Override + public void stop() throws AtlasException { + + } +} diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java index ac28145aa24..c5e58e8fec9 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java @@ -18,15 +18,16 @@ */ package org.apache.atlas.repository.store.graph; +import org.apache.atlas.authorize.AtlasAccessorRequest; +import org.apache.atlas.authorize.AtlasAccessorResponse; import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.model.instance.AtlasCheckStateRequest; -import org.apache.atlas.model.instance.AtlasCheckStateResult; -import org.apache.atlas.model.instance.AtlasClassification; +import org.apache.atlas.model.instance.*; import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo; import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo; import org.apache.atlas.model.instance.AtlasEntityHeader; import org.apache.atlas.model.instance.AtlasEntityHeaders; import org.apache.atlas.model.instance.AtlasObjectId; +import org.apache.atlas.model.instance.AtlasHasLineageRequests; import org.apache.atlas.model.instance.EntityMutationResponse; import org.apache.atlas.repository.store.graph.v2.EntityStream; import org.apache.atlas.type.AtlasEntityType; @@ -88,6 +89,9 @@ public interface AtlasEntityStore { public AtlasEntityHeader getEntityHeaderByUniqueAttributes(AtlasEntityType entityType, Map uniqAttributes) throws AtlasBaseException; + + public AtlasEntityHeader getAtlasEntityHeaderWithoutAuthorization(String guid, String qualifiedName, String typeName) throws AtlasBaseException ; + /** * Batch GET to retrieve entities by their ID * @param guid @@ -168,7 +172,10 @@ AtlasEntityWithExtInfo getByUniqueAttributes(AtlasEntityType entityType, Map objectIds) */ BulkImportResponse bulkCreateOrUpdateBusinessAttributes(InputStream inputStream, String fileName) throws AtlasBaseException; + + List getAccessors(List request) throws AtlasBaseException; + void repairIndex() throws AtlasBaseException; + + void repairHasLineage(AtlasHasLineageRequests requests) throws AtlasBaseException; + + void repairMeaningAttributeForTerms(List termGuids) throws AtlasBaseException; + } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasRelationshipStore.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasRelationshipStore.java index 1abe10d7396..69380d6bd6c 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasRelationshipStore.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasRelationshipStore.java @@ -24,6 +24,8 @@ import org.apache.atlas.repository.graphdb.AtlasVertex; import java.util.List; +import java.util.Map; +import java.util.Set; /** @@ -68,9 +70,6 @@ public interface AtlasRelationshipStore { AtlasEdge getOrCreate(AtlasVertex end1Vertex, AtlasVertex end2Vertex, AtlasRelationship relationship) throws AtlasBaseException; - AtlasEdge getRelationship(AtlasVertex end1Vertex, AtlasVertex end2Vertex, AtlasRelationship relationship) throws AtlasBaseException; - - AtlasEdge createRelationship(AtlasVertex end1Vertex, AtlasVertex end2Vertex, AtlasRelationship relationship) throws AtlasBaseException; /** * Retrieve a relationship if it exists or creates a new relationship instance. @@ -97,4 +96,6 @@ public interface AtlasRelationshipStore { * @param forceDelete force delete the relationship edge */ void deleteById(String guid, boolean forceDelete) throws AtlasBaseException; + + void onRelationshipsMutated(Map> relationshipsMutationMap) throws AtlasBaseException; } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java index 8447ae2b432..a4f818980f8 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java @@ -48,9 +48,11 @@ import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.Arrays; import static org.apache.atlas.model.discovery.SearchParameters.ALL_ENTITY_TYPES; import static org.apache.atlas.model.discovery.SearchParameters.ALL_CLASSIFICATION_TYPES; +import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_BUILTIN_TYPES; import static org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer.getTypesToCreate; import static org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer.getTypesToUpdate; @@ -484,6 +486,11 @@ public AtlasTypesDef createUpdateTypesDef(AtlasTypesDef typesToCreate, AtlasType return ret; } + @Override + public boolean hasBuiltInTypeName(AtlasBaseTypeDef typeDef){ + return Arrays.stream(ATLAS_BUILTIN_TYPES).anyMatch(builtinInName -> builtinInName.equals(typeDef.getName())); + } + @Override @GraphTransaction public AtlasTypesDef updateTypesDef(AtlasTypesDef typesDef) throws AtlasBaseException { diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerDelegate.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerDelegate.java index 0c5ece05d1d..f1a98fd5265 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerDelegate.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerDelegate.java @@ -67,6 +67,9 @@ public DeleteHandlerV1 getHandler(DeleteType deleteType) { case HARD: return hardDeleteHandler; + case PURGE: + return hardDeleteHandler; + default: return defaultHandler; } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java index e1e47412d58..d4437922b6e 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java @@ -21,6 +21,9 @@ import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.AtlasException; import org.apache.atlas.RequestContext; +import org.apache.atlas.authorize.AtlasAuthorizationUtils; +import org.apache.atlas.authorize.AtlasPrivilege; +import org.apache.atlas.authorize.AtlasRelationshipAccessRequest; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.TypeCategory; import org.apache.atlas.model.instance.AtlasClassification; @@ -29,6 +32,8 @@ import org.apache.atlas.model.instance.AtlasObjectId; import org.apache.atlas.model.instance.AtlasRelationship; import org.apache.atlas.model.tasks.AtlasTask; +import org.apache.atlas.model.tasks.TaskSearchResult; +import org.apache.atlas.model.typedef.AtlasRelationshipDef; import org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags; import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef; import org.apache.atlas.repository.graph.GraphHelper; @@ -37,49 +42,60 @@ import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2; +import org.apache.atlas.repository.store.graph.v2.AtlasRelationshipStoreV2; import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; import org.apache.atlas.DeleteType; import org.apache.atlas.repository.store.graph.v2.tasks.ClassificationTask; +import org.apache.atlas.repository.store.graph.v2.tasks.TaskUtil; import org.apache.atlas.tasks.TaskManagement; import org.apache.atlas.type.*; import org.apache.atlas.type.AtlasStructType.AtlasAttribute; import org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection; import org.apache.atlas.utils.AtlasEntityUtil; +import org.apache.atlas.utils.AtlasPerfMetrics; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.atlas.repository.graph.GraphHelper.getTypeName; + import java.util.*; import java.util.stream.Collectors; +import static org.apache.atlas.AtlasClient.DATA_SET_SUPER_TYPE; +import static org.apache.atlas.AtlasClient.PROCESS_SUPER_TYPE; import static org.apache.atlas.model.TypeCategory.*; import static org.apache.atlas.model.instance.AtlasEntity.Status.ACTIVE; import static org.apache.atlas.model.instance.AtlasEntity.Status.DELETED; import static org.apache.atlas.model.instance.AtlasEntity.Status.PURGED; import static org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags.ONE_TO_TWO; import static org.apache.atlas.repository.Constants.*; -import static org.apache.atlas.repository.graph.GraphHelper.getTypeName; import static org.apache.atlas.repository.graph.GraphHelper.*; import static org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.getState; import static org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.*; import static org.apache.atlas.repository.store.graph.v2.tasks.ClassificationPropagateTaskFactory.CLASSIFICATION_PROPAGATION_ADD; import static org.apache.atlas.repository.store.graph.v2.tasks.ClassificationPropagateTaskFactory.CLASSIFICATION_PROPAGATION_DELETE; +import static org.apache.atlas.repository.store.graph.v2.tasks.ClassificationPropagateTaskFactory.CLASSIFICATION_REFRESH_PROPAGATION; import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.OUT; +import static org.apache.atlas.type.Constants.HAS_LINEAGE; import static org.apache.atlas.type.Constants.PENDING_TASKS_PROPERTY_KEY; public abstract class DeleteHandlerV1 { - public static final Logger LOG = LoggerFactory.getLogger(DeleteHandlerV1.class); + public static final Logger LOG = LoggerFactory.getLogger(DeleteHandlerV1.class); - private static final boolean DEFERRED_ACTION_ENABLED = AtlasConfiguration.TASKS_USE_ENABLED.getBoolean(); + static final boolean DEFERRED_ACTION_ENABLED = AtlasConfiguration.TASKS_USE_ENABLED.getBoolean(); + static final int PENDING_TASK_QUERY_SIZE_LIMIT = 20; protected final GraphHelper graphHelper; private final AtlasTypeRegistry typeRegistry; - private final EntityGraphRetriever entityRetriever; + protected final EntityGraphRetriever entityRetriever; private final boolean shouldUpdateInverseReferences; private final boolean softDelete; private final TaskManagement taskManagement; + private final AtlasGraph graph; + private final TaskUtil taskUtil; public DeleteHandlerV1(AtlasGraph graph, AtlasTypeRegistry typeRegistry, boolean shouldUpdateInverseReference, boolean softDelete, TaskManagement taskManagement) { @@ -89,6 +105,8 @@ public DeleteHandlerV1(AtlasGraph graph, AtlasTypeRegistry typeRegistry, boolean this.shouldUpdateInverseReferences = shouldUpdateInverseReference; this.softDelete = softDelete; this.taskManagement = taskManagement; + this.graph = graph; + this.taskUtil = new TaskUtil(graph); } /** @@ -102,18 +120,13 @@ public DeleteHandlerV1(AtlasGraph graph, AtlasTypeRegistry typeRegistry, boolean public void deleteEntities(Collection instanceVertices) throws AtlasBaseException { final RequestContext requestContext = RequestContext.get(); final Set deletionCandidateVertices = new HashSet<>(); - final boolean isPurgeRequested = requestContext.isPurgeRequested(); for (AtlasVertex instanceVertex : instanceVertices) { final String guid = AtlasGraphUtilsV2.getIdFromVertex(instanceVertex); if (skipVertexForDelete(instanceVertex)) { if (LOG.isDebugEnabled()) { - if (isPurgeRequested) { - LOG.debug("Skipping purging of entity={} as it is active or already purged", guid); - } else { LOG.debug("Skipping deletion of entity={} as it is already deleted", guid); - } } continue; } @@ -134,8 +147,18 @@ public void deleteEntities(Collection instanceVertices) throws Atla // Delete traits and vertices. for (AtlasVertex deletionCandidateVertex : deletionCandidateVertices) { + RequestContext.get().getDeletedEdgesIds().clear(); + deleteAllClassifications(deletionCandidateVertex); deleteTypeVertex(deletionCandidateVertex, isInternalType(deletionCandidateVertex)); + + if (DEFERRED_ACTION_ENABLED) { + Set deletedEdgeIds = RequestContext.get().getDeletedEdgesIds(); + for (String deletedEdgeId : deletedEdgeIds) { + AtlasEdge edge = graph.getEdge(deletedEdgeId); + createAndQueueClassificationRefreshPropagationTask(edge); + } + } } } @@ -161,20 +184,14 @@ public void deleteRelationships(Collection edges, final boolean force for (AtlasEdge edge : edges) { boolean isInternal = isInternalType(edge.getInVertex()) && isInternalType(edge.getOutVertex()); - boolean needToSkip = !isInternal && (getState(edge) == (isPurgeRequested ? ACTIVE : DELETED)); + boolean needToSkip = !isInternal && (!isPurgeRequested && DELETED.equals(getState(edge))); if (needToSkip) { if (LOG.isDebugEnabled()) { - if(isPurgeRequested) { - LOG.debug("Skipping purging of edge={} as it is active or already purged", getIdFromEdge(edge)); - } else{ - LOG.debug("Skipping deletion of edge={} as it is already deleted", getIdFromEdge(edge)); - } + LOG.debug("Skipping deletion of edge={} as it is already deleted", getIdFromEdge(edge)); } - continue; } - deleteEdge(edge, isInternal || forceDelete); } } @@ -198,9 +215,8 @@ public Collection getOwnedVertices(AtlasVertex entityVer AtlasVertex vertex = vertices.pop(); AtlasEntity.Status state = getState(vertex); - //In case of purge If the reference vertex is active then skip it or else - //If the vertex marked for deletion, skip it - if (state == (isPurgeRequested ? ACTIVE : DELETED)) { + //If the vertex marked for deletion, if we are not purging, skip it + if (!isPurgeRequested && DELETED.equals(state)) { continue; } @@ -235,14 +251,16 @@ public Collection getOwnedVertices(AtlasVertex entityVer String softRefVal = vertex.getProperty(attributeInfo.getVertexPropertyName(), String.class); AtlasObjectId refObjId = AtlasEntityUtil.parseSoftRefValue(softRefVal); AtlasVertex refVertex = refObjId != null ? AtlasGraphUtilsV2.findByGuid(this.graphHelper.getGraph(), refObjId.getGuid()) : null; - + if (refObjId.getGuid() == null) { + LOG.warn("OBJECT_ID_TYPE type category - null guid passed in findByGuid!"); + } if (refVertex != null) { vertices.push(refVertex); } } else { AtlasEdge edge = graphHelper.getEdgeForLabel(vertex, edgeLabel); - if (edge == null || (getState(edge) == (isPurgeRequested ? ACTIVE : DELETED))) { + if (edge == null || (!isPurgeRequested && DELETED.equals(getState(edge)))) { continue; } @@ -269,7 +287,9 @@ public Collection getOwnedVertices(AtlasVertex entityVer if (CollectionUtils.isNotEmpty(refObjIds)) { for (AtlasObjectId refObjId : refObjIds) { AtlasVertex refVertex = AtlasGraphUtilsV2.findByGuid(this.graphHelper.getGraph(), refObjId.getGuid()); - + if (refObjId.getGuid() == null) { + LOG.warn("ARRAY type category - null guid passed in findByGuid!"); + } if (refVertex != null) { vertices.push(refVertex); } @@ -282,6 +302,9 @@ public Collection getOwnedVertices(AtlasVertex entityVer if (MapUtils.isNotEmpty(refObjIds)) { for (AtlasObjectId refObjId : refObjIds.values()) { AtlasVertex refVertex = AtlasGraphUtilsV2.findByGuid(this.graphHelper.getGraph(), refObjId.getGuid()); + if (refObjId.getGuid() == null) { + LOG.warn("MAP type category - null guid passed in findByGuid!"); + } if (refVertex != null) { vertices.push(refVertex); @@ -295,7 +318,7 @@ public Collection getOwnedVertices(AtlasVertex entityVer if (CollectionUtils.isNotEmpty(edges)) { for (AtlasEdge edge : edges) { - if (edge == null || (getState(edge) == (isPurgeRequested ? ACTIVE : DELETED))) { + if (edge == null || (!isPurgeRequested && DELETED.equals(getState(edge)))) { continue; } @@ -359,7 +382,6 @@ public boolean deleteEdgeReference(AtlasEdge edge, TypeCategory typeCategory, bo // only delete the reference relationship edge if (GraphHelper.isRelationshipEdge(edge)) { deleteEdge(edge, isInternalType); - AtlasVertex referencedVertex = entityRetriever.getReferencedEntityVertex(edge, relationshipDirection, entityVertex); if (referencedVertex != null) { @@ -424,7 +446,7 @@ private void addTagPropagation(AtlasVertex fromVertex, AtlasVertex toVertex, Atl } public List addTagPropagation(AtlasVertex classificationVertex, List propagatedEntityVertices) throws AtlasBaseException { - List ret = null; + List ret = new ArrayList<>(); if (CollectionUtils.isNotEmpty(propagatedEntityVertices) && classificationVertex != null) { String classificationName = getTypeName(classificationVertex); @@ -439,7 +461,8 @@ public List addTagPropagation(AtlasVertex classificationVertex, Lis } continue; - } else if (getPropagatedClassificationEdge(propagatedEntityVertex, classificationVertex) != null) { + } + if (getPropagatedClassificationEdge(propagatedEntityVertex, classificationVertex) != null) { if (LOG.isDebugEnabled()) { LOG.debug(" --> Propagated classification edge already exists from [{}] --> [{}][{}] using edge label: [{}]", getTypeName(propagatedEntityVertex), getTypeName(classificationVertex), getTypeName(associatedEntityVertex), CLASSIFICATION_LABEL); @@ -447,6 +470,7 @@ public List addTagPropagation(AtlasVertex classificationVertex, Lis continue; } + AtlasPerfMetrics.MetricRecorder countMetricRecorder = RequestContext.get().startMetricRecord("countPropagations"); String entityTypeName = getTypeName(propagatedEntityVertex); AtlasEntityType entityType = typeRegistry.getEntityTypeByName(entityTypeName); @@ -461,21 +485,11 @@ public List addTagPropagation(AtlasVertex classificationVertex, Lis continue; } - AtlasEdge existingEdge = getPropagatedClassificationEdge(propagatedEntityVertex, classificationVertex); - - if (existingEdge != null) { - continue; - } - if (LOG.isDebugEnabled()) { LOG.debug(" --> Adding propagated classification: [{}] to {} ({}) using edge label: [{}]", classificationName, getTypeName(propagatedEntityVertex), GraphHelper.getGuid(propagatedEntityVertex), CLASSIFICATION_LABEL); } - if (ret == null) { - ret = new ArrayList<>(); - } - ret.add(propagatedEntityVertex); graphHelper.addClassificationEdge(propagatedEntityVertex, classificationVertex, true); @@ -487,34 +501,36 @@ public List addTagPropagation(AtlasVertex classificationVertex, Lis AtlasClassification classification = entityRetriever.toAtlasClassification(classificationVertex); context.recordAddedPropagation(entityGuid, classification); + RequestContext.get().endMetricRecord(countMetricRecorder); } } - return ret; } - public void removeTagPropagation(AtlasEdge edge) throws AtlasBaseException { - if (edge == null || !isRelationshipEdge(edge)) { + public void authorizeRemoveRelation(AtlasEdge edge) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("authoriseRemoveRelation"); + AtlasEntityHeader end1Entity, end2Entity; + String relationShipType = getTypeName(edge); + AtlasRelationshipDef relationshipDef = typeRegistry.getRelationshipDefByName(relationShipType); + if (relationshipDef == null) { return; } - List currentClassificationVertices = getPropagatableClassifications(edge); - Map> currentClassificationsMap = entityRetriever.getClassificationPropagatedEntitiesMapping(currentClassificationVertices); - Map> updatedClassificationsMap = entityRetriever.getClassificationPropagatedEntitiesMapping(currentClassificationVertices, getRelationshipGuid(edge)); - Map> removePropagationsMap = new HashMap<>(); + end1Entity = entityRetriever.toAtlasEntityHeaderWithClassifications(edge.getOutVertex()); + end2Entity = entityRetriever.toAtlasEntityHeaderWithClassifications(edge.getInVertex()); - if (MapUtils.isNotEmpty(currentClassificationsMap) && MapUtils.isEmpty(updatedClassificationsMap)) { - removePropagationsMap.putAll(currentClassificationsMap); - } else { - for (AtlasVertex classificationVertex : updatedClassificationsMap.keySet()) { - List currentPropagatingEntities = currentClassificationsMap.containsKey(classificationVertex) ? currentClassificationsMap.get(classificationVertex) : Collections.emptyList(); - List updatedPropagatingEntities = updatedClassificationsMap.containsKey(classificationVertex) ? updatedClassificationsMap.get(classificationVertex) : Collections.emptyList(); - List entitiesRemoved = (List) CollectionUtils.subtract(currentPropagatingEntities, updatedPropagatingEntities); + AtlasAuthorizationUtils.verifyAccess(new AtlasRelationshipAccessRequest(typeRegistry, AtlasPrivilege.RELATIONSHIP_REMOVE, relationShipType, end1Entity, end2Entity )); - if (CollectionUtils.isNotEmpty(entitiesRemoved)) { - removePropagationsMap.put(classificationVertex, entitiesRemoved); - } - } + RequestContext.get().endMetricRecord(metric); + } + + public Map> removeTagPropagation(AtlasEdge edge) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("removeTagPropagationEdge"); + + Map> removePropagationsMap = getRemovePropagationMap(edge); + + if (removePropagationsMap == null) { + return null; } boolean isTermEntityEdge = isTermEntityEdge(edge); @@ -526,6 +542,38 @@ public void removeTagPropagation(AtlasEdge edge) throws AtlasBaseException { removeTagPropagation(classificationVertex, removePropagationsMap.get(classificationVertex)); } } + + RequestContext.get().endMetricRecord(metric); + return removePropagationsMap; + } + + public Map> getRemovePropagationMap(AtlasEdge edge) throws AtlasBaseException{ + + if (edge == null || !isRelationshipEdge(edge)) { + return null; + } + + List currentClassificationVertices = getPropagatableClassifications(edge); + + Map> classificationMapWithEdge = entityRetriever.getClassificationPropagatedEntitiesMapping(currentClassificationVertices); + Map> classificationMapWithOutEdge = entityRetriever.getClassificationPropagatedEntitiesMapping(currentClassificationVertices, getRelationshipGuid(edge)); + Map> removePropagationsMap = new HashMap<>(); + + if (MapUtils.isNotEmpty(classificationMapWithEdge) && MapUtils.isEmpty(classificationMapWithOutEdge)) { + removePropagationsMap.putAll(classificationMapWithEdge); + } else { + for (AtlasVertex classificationVertex : classificationMapWithOutEdge.keySet()) { + + List currentPropagatingEntities = classificationMapWithEdge.getOrDefault(classificationVertex, Collections.emptyList()); + List updatedPropagatingEntities = classificationMapWithOutEdge.getOrDefault(classificationVertex, Collections.emptyList()); + List entitiesRemoved = (List) CollectionUtils.subtract(currentPropagatingEntities, updatedPropagatingEntities); + + if (CollectionUtils.isNotEmpty(entitiesRemoved)) { + removePropagationsMap.put(classificationVertex, entitiesRemoved); + } + } + } + return removePropagationsMap; } public boolean isRelationshipEdge(AtlasEdge edge) { @@ -542,7 +590,28 @@ public boolean isRelationshipEdge(AtlasEdge edge) { return ret; } + public List removeTagPropagation(AtlasClassification classification, List propagatedEdges) throws AtlasBaseException { + List ret = new ArrayList<>(); + + for (AtlasEdge propagatedEdge : propagatedEdges) { + AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("removeTagPropagationEdges"); + AtlasVertex entityVertex = propagatedEdge.getOutVertex(); + + ret.add(entityVertex); + + // record remove propagation details to send notifications inline + RequestContext.get().recordRemovedPropagation(getGuid(entityVertex), classification); + + deletePropagatedEdge(propagatedEdge); + + RequestContext.get().endMetricRecord(metric); + } + + return ret; + } + public List removeTagPropagation(AtlasVertex classificationVertex) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("removeTagPropagationVertex"); List ret = new ArrayList<>(); if (classificationVertex != null) { @@ -563,31 +632,41 @@ public List removeTagPropagation(AtlasVertex classificationVertex) } } } - + RequestContext.get().endMetricRecord(metric); return ret; } - public void removeTagPropagation(AtlasVertex classificationVertex, List entityVertices) throws AtlasBaseException { + public List removeTagPropagation(AtlasVertex classificationVertex, List entityVertices) throws AtlasBaseException { + List ret = new ArrayList<>(); if (classificationVertex != null && CollectionUtils.isNotEmpty(entityVertices)) { + AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("removeTagPropagationVertices"); String classificationName = getClassificationName(classificationVertex); AtlasClassification classification = entityRetriever.toAtlasClassification(classificationVertex); String entityGuid = getClassificationEntityGuid(classificationVertex); RequestContext context = RequestContext.get(); for (AtlasVertex entityVertex : entityVertices) { + if(!entityVertex.exists()) { + continue; + } AtlasEdge propagatedEdge = getPropagatedClassificationEdge(entityVertex, classificationName, entityGuid); if (propagatedEdge != null) { deletePropagatedEdge(propagatedEdge); + ret.add(entityVertex); + // record remove propagation details to send notifications at the end context.recordRemovedPropagation(getGuid(entityVertex), classification); } } + RequestContext.get().endMetricRecord(metric); } + return ret; } public void deletePropagatedClassification(AtlasVertex entityVertex, String classificationName, String associatedEntityGuid) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("deletePropagatedClassification"); AtlasEdge propagatedEdge = getPropagatedClassificationEdge(entityVertex, classificationName, associatedEntityGuid); if (propagatedEdge == null) { @@ -616,6 +695,8 @@ public void deletePropagatedClassification(AtlasVertex entityVertex, String clas // record remove propagation details to send notifications at the end RequestContext.get().recordRemovedPropagation(getGuid(entityVertex), classification); + + RequestContext.get().endMetricRecord(metricRecorder); } public void deletePropagatedEdge(AtlasEdge edge) throws AtlasBaseException { @@ -916,8 +997,16 @@ protected void deleteVertex(AtlasVertex instanceVertex, boolean force) throws At // Delete external references to this vertex - incoming edges from lineage or glossary term edges final Iterable incomingEdges = instanceVertex.getEdges(AtlasEdgeDirection.IN); + final Iterable outgoingEdges = instanceVertex.getEdges(AtlasEdgeDirection.OUT); final boolean isPurgeRequested = RequestContext.get().isPurgeRequested(); + if (RequestContext.get().getDeleteType().equals(DeleteType.HARD) || RequestContext.get().getDeleteType().equals(DeleteType.PURGE)) { + for (AtlasEdge edge : outgoingEdges) { + if (isRelationshipEdge(edge)) + AtlasRelationshipStoreV2.recordRelationshipMutation(AtlasRelationshipStoreV2.RelationshipMutation.RELATIONSHIP_HARD_DELETE, edge, entityRetriever); + } + } + for (AtlasEdge edge : incomingEdges) { AtlasEntity.Status edgeStatus = getStatus(edge); boolean isProceed = edgeStatus == (isPurgeRequested ? DELETED : ACTIVE); @@ -1017,6 +1106,10 @@ private String getDelimitedPropagatedClassificationNames(AtlasVertex entityVerte * @throws AtlasException */ private void deleteAllClassifications(AtlasVertex instanceVertex) throws AtlasBaseException { + // If instance is deleted no need to operate classification deleted + if (!ACTIVE.equals(getState(instanceVertex))) + return; + List classificationEdges = getAllClassificationEdges(instanceVertex); for (AtlasEdge edge : classificationEdges) { @@ -1047,7 +1140,7 @@ private boolean skipVertexForDelete(AtlasVertex vertex) { if(guid != null && !reqContext.isDeletedEntity(guid)) { final AtlasEntity.Status vertexState = getState(vertex); if (reqContext.isPurgeRequested()) { - ret = vertexState == ACTIVE; // skip purging ACTIVE vertices + ret = false; // Delete all ACTIVE or DELETED assets in PURGING } else { ret = vertexState == DELETED; // skip deleting DELETED vertices } @@ -1199,12 +1292,31 @@ private void setBlockedClassificationIds(AtlasEdge edge, List classifica } } } - - public void createAndQueueTask(String taskType, AtlasVertex entityVertex, String classificationVertexId, String relationshipGuid) { + public void createAndQueueTaskWithoutCheck(String taskType, AtlasVertex entityVertex, String classificationVertexId, String relationshipGuid) throws AtlasBaseException { String currentUser = RequestContext.getCurrentUser(); String entityGuid = GraphHelper.getGuid(entityVertex); Map taskParams = ClassificationTask.toParameters(entityGuid, classificationVertexId, relationshipGuid); - AtlasTask task = taskManagement.createTask(taskType, currentUser, taskParams); + AtlasTask task = taskManagement.createTask(taskType, currentUser, taskParams, classificationVertexId, entityGuid); + + AtlasGraphUtilsV2.addEncodedProperty(entityVertex, PENDING_TASKS_PROPERTY_KEY, task.getGuid()); + + RequestContext.get().queueTask(task); + } + + public void createAndQueueTask(String taskType, AtlasVertex entityVertex, String classificationVertexId, String relationshipGuid) throws AtlasBaseException { + if (!CLASSIFICATION_PROPAGATION_DELETE.equals(taskType) && skipClassificationTaskCreation(classificationVertexId)) { + LOG.info("Task is already scheduled for classification id {}, no need to schedule task for vertex {}", classificationVertexId, entityVertex.getIdForDisplay()); + return; + } + + createAndQueueTaskWithoutCheck(taskType, entityVertex, classificationVertexId, relationshipGuid); + } + + public void createAndQueueTaskWithoutCheck(String taskType, AtlasVertex entityVertex, String classificationVertexId, String relationshipGuid, Boolean currentRestrictPropagationThroughLineage) throws AtlasBaseException { + String currentUser = RequestContext.getCurrentUser(); + String entityGuid = GraphHelper.getGuid(entityVertex); + Map taskParams = ClassificationTask.toParameters(entityGuid, classificationVertexId, relationshipGuid, currentRestrictPropagationThroughLineage); + AtlasTask task = taskManagement.createTask(taskType, currentUser, taskParams, classificationVertexId, entityGuid); AtlasGraphUtilsV2.addEncodedProperty(entityVertex, PENDING_TASKS_PROPERTY_KEY, task.getGuid()); @@ -1215,10 +1327,233 @@ public void createAndQueueTask(String taskType, AtlasEdge relationshipEdge, Atla String currentUser = RequestContext.getCurrentUser(); String relationshipEdgeId = relationshipEdge.getIdForDisplay(); Map taskParams = ClassificationTask.toParameters(relationshipEdgeId, relationship); + AtlasTask task = taskManagement.createTask(taskType, currentUser, taskParams); AtlasGraphUtilsV2.addItemToListProperty(relationshipEdge, EDGE_PENDING_TASKS_PROPERTY_KEY, task.getGuid()); RequestContext.get().queueTask(task); } + + public void createAndQueueClassificationRefreshPropagationTask(AtlasEdge edge) throws AtlasBaseException{ + + if (taskManagement==null) { + LOG.warn("Task management is null, can't schedule task now"); + return; + } + + String currentUser = RequestContext.getCurrentUser(); + boolean isRelationshipEdge = isRelationshipEdge(edge); + boolean isTermEntityEdge = GraphHelper.isTermEntityEdge(edge); + + if (edge == null || !isRelationshipEdge) { + LOG.warn("Edge is null or it is not relationship edge, can't schedule task now"); + return; + } + + + AtlasVertex referenceVertex = GraphHelper.getPropagatingVertex(edge); + if(referenceVertex == null) { + return; + } + + List currentClassificationVertices = GraphHelper.getPropagatableClassifications(edge); + for (AtlasVertex currentClassificationVertex : currentClassificationVertices) { + String currentClassificationId = currentClassificationVertex.getIdForDisplay(); + boolean removePropagationOnEntityDelete = GraphHelper.getRemovePropagations(currentClassificationVertex); + + if (!(isTermEntityEdge || removePropagationOnEntityDelete)) { + LOG.debug("This edge is not term edge or remove propagation isn't enabled"); + continue; + } + + if(skipClassificationTaskCreation(currentClassificationId)) { + LOG.info("Task is already scheduled for classification id {}, no need to schedule task for edge {}", currentClassificationId, edge.getIdForDisplay()); + continue; + } + + Map taskParams = ClassificationTask.toParameters(currentClassificationVertex.getIdForDisplay()); + AtlasTask task = taskManagement.createTask(CLASSIFICATION_REFRESH_PROPAGATION, currentUser, taskParams, currentClassificationId, GraphHelper.getGuid(referenceVertex)); + + RequestContext.get().queueTask(task); + } + + } + + private boolean skipClassificationTaskCreation(String classificationId) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("skipClassificationTaskCreation"); + /* + If any of, + 1. CLASSIFICATION_PROPAGATION_DELETE + 2. CLASSIFICATION_REFRESH_PROPAGATION task scheduled already + skip classification task creation + */ + try { + + List taskTypes = Arrays.asList(CLASSIFICATION_REFRESH_PROPAGATION, CLASSIFICATION_PROPAGATION_DELETE); + List tasksInRequestContext = RequestContext.get().getQueuedTasks(); + if ( + tasksInRequestContext != null && + tasksInRequestContext.stream().filter(Objects::nonNull) + .anyMatch(task -> task.getClassificationId().equals(classificationId) + && taskTypes.contains(task.getType()) && task.getStatus().equals(AtlasTask.Status.PENDING)) + ) { + return true; + } + + TaskSearchResult taskSearchResult = taskUtil.findPendingTasksByClassificationId(0, PENDING_TASK_QUERY_SIZE_LIMIT, + classificationId, taskTypes , new ArrayList<>()); + + List pendingTasks = taskSearchResult.getTasks(); + if(CollectionUtils.isEmpty(pendingTasks)) { + return false; + } + + List pendingRefreshPropagationTasks = pendingTasks.stream() + .filter(task -> CLASSIFICATION_REFRESH_PROPAGATION.equals(task.getType())) + .collect(Collectors.toList()); + + // Ideally there should be only refresh propagation task + if (pendingRefreshPropagationTasks.size() > 1) { + LOG.warn("More than one {} task found for classification id {}", CLASSIFICATION_REFRESH_PROPAGATION, classificationId); + } + + // if any task have status as PENDING, then skip task creation + if ( + pendingTasks.stream() + .filter(Objects::nonNull) + .anyMatch(task -> task.getClassificationId().equals(classificationId) + && taskTypes.contains(task.getType()) && task.getStatus().equals(AtlasTask.Status.PENDING)) + ) { + return true; + } else { + LOG.warn("There is inconsistency " + + "in task queue, there are no pending tasks for classification id {} but there are tasks in queue", classificationId); + } + } catch (AtlasBaseException e) { + LOG.error("Error while checking if classification task creation is required for classification id {}", classificationId, e); + throw e; + } finally { + RequestContext.get().endMetricRecord(metric); + } + + return false; + } + + + public void removeHasLineageOnDelete(Collection vertices) { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("removeHasLineageOnDelete"); + + for (AtlasVertex vertexToBeDeleted : vertices) { + if (ACTIVE.equals(getStatus(vertexToBeDeleted))) { + AtlasEntityType entityType = typeRegistry.getEntityTypeByName(getTypeName(vertexToBeDeleted)); + boolean isProcess = entityType.getTypeAndAllSuperTypes().contains(PROCESS_SUPER_TYPE); + boolean isCatalog = entityType.getTypeAndAllSuperTypes().contains(DATA_SET_SUPER_TYPE); + + if (isCatalog || isProcess) { + + Iterator edgeIterator = vertexToBeDeleted.getEdges(AtlasEdgeDirection.BOTH, PROCESS_EDGE_LABELS).iterator(); + + Set edgesToBeDeleted = new HashSet<>(); + + while (edgeIterator.hasNext()) { + AtlasEdge edge = edgeIterator.next(); + if (ACTIVE.equals(getStatus(edge))) { + edgesToBeDeleted.add(edge); + } + } + + resetHasLineageOnInputOutputDelete(edgesToBeDeleted, vertexToBeDeleted); + } + } + } + RequestContext.get().endMetricRecord(metricRecorder); + } + + + public void resetHasLineageOnInputOutputDelete(Collection removedEdges, AtlasVertex deletedVertex) { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("resetHasLineageOnInputOutputDelete"); + + for (AtlasEdge atlasEdge : removedEdges) { + + boolean isOutputEdge = PROCESS_OUTPUTS.equals(atlasEdge.getLabel()); + + AtlasVertex processVertex = atlasEdge.getOutVertex(); + AtlasVertex assetVertex = atlasEdge.getInVertex(); + + if (getStatus(assetVertex) == ACTIVE && !assetVertex.equals(deletedVertex)) { + updateAssetHasLineageStatus(assetVertex, atlasEdge, removedEdges); + } + + if (getStatus(processVertex) == ACTIVE && !processVertex.equals(deletedVertex)) { + String edgeLabel = isOutputEdge ? PROCESS_OUTPUTS : PROCESS_INPUTS; + + Iterator edgeIterator = processVertex.getEdges(AtlasEdgeDirection.BOTH, edgeLabel).iterator(); + boolean activeEdgeFound = false; + + while (edgeIterator.hasNext()) { + AtlasEdge edge = edgeIterator.next(); + if (getStatus(edge) == ACTIVE && !removedEdges.contains(edge)) { + AtlasVertex relatedAssetVertex = edge.getInVertex(); + + if (getStatus(relatedAssetVertex) == ACTIVE) { + activeEdgeFound = true; + break; + } + } + } + + if (!activeEdgeFound) { + AtlasGraphUtilsV2.setEncodedProperty(processVertex, HAS_LINEAGE, false); + + String oppositeEdgeLabel = isOutputEdge ? PROCESS_INPUTS : PROCESS_OUTPUTS; + + Iterator processEdgeIterator = processVertex.getEdges(AtlasEdgeDirection.BOTH, oppositeEdgeLabel).iterator(); + + while (processEdgeIterator.hasNext()) { + AtlasEdge edge = processEdgeIterator.next(); + + if (!removedEdges.contains(edge)) { + AtlasVertex relatedAssetVertex = edge.getInVertex(); + updateAssetHasLineageStatus(relatedAssetVertex, edge, removedEdges); + } + } + } + } + } + RequestContext.get().endMetricRecord(metricRecorder); + } + + private void updateAssetHasLineageStatus(AtlasVertex assetVertex, AtlasEdge currentEdge, Collection removedEdges) { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("updateAssetHasLineageStatus"); + + removedEdges.forEach(edge -> RequestContext.get().addToDeletedEdgesIdsForResetHasLineage(edge.getIdForDisplay())); + + Iterator edgeIterator = assetVertex.query() + .direction(AtlasEdgeDirection.BOTH) + .label(PROCESS_EDGE_LABELS) + .has(STATE_PROPERTY_KEY, ACTIVE.name()) + .edges() + .iterator(); + + int processHasLineageCount = 0; + while (edgeIterator.hasNext()) { + AtlasEdge edge = edgeIterator.next(); + if (!RequestContext.get().getDeletedEdgesIdsForResetHasLineage().contains(edge.getIdForDisplay()) && !currentEdge.equals(edge)) { + AtlasVertex relatedProcessVertex = edge.getOutVertex(); + boolean processHasLineage = getEntityHasLineage(relatedProcessVertex); + if (processHasLineage) { + processHasLineageCount++; + break; + } + } + } + + if (processHasLineageCount == 0) { + AtlasGraphUtilsV2.setEncodedProperty(assetVertex, HAS_LINEAGE, false); + } + + RequestContext.get().endMetricRecord(metricRecorder); + } + } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/HardDeleteHandlerV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/HardDeleteHandlerV1.java index eebb40a796b..b3942d05821 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/HardDeleteHandlerV1.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/HardDeleteHandlerV1.java @@ -24,6 +24,7 @@ import org.apache.atlas.repository.graphdb.AtlasEdge; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.repository.store.graph.v2.AtlasRelationshipStoreV2; import org.apache.atlas.tasks.TaskManagement; import org.apache.atlas.type.AtlasTypeRegistry; import org.springframework.stereotype.Component; @@ -53,9 +54,18 @@ protected void deleteEdge(AtlasEdge edge, boolean force) throws AtlasBaseExcepti if (LOG.isDebugEnabled()) { LOG.debug("==> HardDeleteHandlerV1.deleteEdge({}, {})", GraphHelper.string(edge), force); } + boolean isRelationshipEdge = isRelationshipEdge(edge); - removeTagPropagation(edge); + authorizeRemoveRelation(edge); + + if (DEFERRED_ACTION_ENABLED) { + createAndQueueClassificationRefreshPropagationTask(edge); + } else { + removeTagPropagation(edge); + } graphHelper.removeEdge(edge); + if (isRelationshipEdge) + AtlasRelationshipStoreV2.recordRelationshipMutation(AtlasRelationshipStoreV2.RelationshipMutation.RELATIONSHIP_HARD_DELETE, edge, entityRetriever); } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/RestoreHandlerV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/RestoreHandlerV1.java index 6feb79d0d78..f865ec86cca 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/RestoreHandlerV1.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/RestoreHandlerV1.java @@ -24,7 +24,6 @@ import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasEntityHeader; import org.apache.atlas.model.instance.AtlasObjectId; -import org.apache.atlas.model.typedef.AtlasRelationshipDef; import org.apache.atlas.model.typedef.AtlasStructDef; import org.apache.atlas.repository.graph.GraphHelper; import org.apache.atlas.repository.graphdb.AtlasEdge; @@ -32,6 +31,7 @@ import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2; +import org.apache.atlas.repository.store.graph.v2.AtlasRelationshipStoreV2; import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; import org.apache.atlas.type.*; import org.apache.atlas.utils.AtlasEntityUtil; @@ -84,6 +84,9 @@ private void restoreEdge(AtlasEdge edge) throws AtlasBaseException { AtlasGraphUtilsV2.setEncodedProperty(edge, MODIFICATION_TIMESTAMP_PROPERTY_KEY, RequestContext.get().getRequestTime()); AtlasGraphUtilsV2.setEncodedProperty(edge, MODIFIED_BY_KEY, RequestContext.get().getUser()); } + + if (isRelationshipEdge(edge)) + AtlasRelationshipStoreV2.recordRelationshipMutation(AtlasRelationshipStoreV2.RelationshipMutation.RELATIONSHIP_RESTORE, edge, entityRetriever); } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/SoftDeleteHandlerV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/SoftDeleteHandlerV1.java index f51b4855aae..d408a2cd051 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/SoftDeleteHandlerV1.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/SoftDeleteHandlerV1.java @@ -26,8 +26,10 @@ import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2; +import org.apache.atlas.repository.store.graph.v2.AtlasRelationshipStoreV2; import org.apache.atlas.tasks.TaskManagement; import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.commons.collections.CollectionUtils; import javax.inject.Inject; @@ -35,6 +37,7 @@ import static org.apache.atlas.repository.Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY; import static org.apache.atlas.repository.Constants.MODIFIED_BY_KEY; import static org.apache.atlas.repository.Constants.STATE_PROPERTY_KEY; +import static org.apache.atlas.repository.graph.GraphHelper.getPropagatableClassifications; public class SoftDeleteHandlerV1 extends DeleteHandlerV1 { @@ -64,22 +67,38 @@ protected void _deleteVertex(AtlasVertex instanceVertex, boolean force) { @Override protected void deleteEdge(AtlasEdge edge, boolean force) throws AtlasBaseException { - if (LOG.isDebugEnabled()) { - LOG.debug("==> SoftDeleteHandlerV1.deleteEdge({}, {})",GraphHelper.string(edge), force); - } + try { + if (LOG.isDebugEnabled()) { + LOG.debug("==> SoftDeleteHandlerV1.deleteEdge({}, {})", GraphHelper.string(edge), force); + } + boolean isRelationshipEdge = isRelationshipEdge(edge); - removeTagPropagation(edge); + authorizeRemoveRelation(edge); - if (force) { - graphHelper.removeEdge(edge); - } else { - Status state = AtlasGraphUtilsV2.getState(edge); + if (DEFERRED_ACTION_ENABLED && RequestContext.get().getCurrentTask() == null) { + if (CollectionUtils.isNotEmpty(getPropagatableClassifications(edge))) { + RequestContext.get().addToDeletedEdgesIds(edge.getIdForDisplay()); + } + } else { + removeTagPropagation(edge); + } - if (state != DELETED) { - AtlasGraphUtilsV2.setEncodedProperty(edge, STATE_PROPERTY_KEY, DELETED.name()); - AtlasGraphUtilsV2.setEncodedProperty(edge, MODIFICATION_TIMESTAMP_PROPERTY_KEY, RequestContext.get().getRequestTime()); - AtlasGraphUtilsV2.setEncodedProperty(edge, MODIFIED_BY_KEY, RequestContext.get().getUser()); + if (force) { + graphHelper.removeEdge(edge); + } else { + Status state = AtlasGraphUtilsV2.getState(edge); + + if (state != DELETED) { + AtlasGraphUtilsV2.setEncodedProperty(edge, STATE_PROPERTY_KEY, DELETED.name()); + AtlasGraphUtilsV2.setEncodedProperty(edge, MODIFICATION_TIMESTAMP_PROPERTY_KEY, RequestContext.get().getRequestTime()); + AtlasGraphUtilsV2.setEncodedProperty(edge, MODIFIED_BY_KEY, RequestContext.get().getUser()); + } } + if (isRelationshipEdge) + AtlasRelationshipStoreV2.recordRelationshipMutation(AtlasRelationshipStoreV2.RelationshipMutation.RELATIONSHIP_SOFT_DELETE, edge, entityRetriever); + } catch (Exception e) { + LOG.error("Error while deleting edge {}", GraphHelper.string(edge), e); + throw new AtlasBaseException(e); } } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasClassificationDefStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasClassificationDefStoreV2.java index 362f158946f..284e4aa868a 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasClassificationDefStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasClassificationDefStoreV2.java @@ -19,6 +19,7 @@ import org.apache.atlas.AtlasErrorCode; +import org.apache.atlas.RequestContext; import org.apache.atlas.authorize.AtlasPrivilege; import org.apache.atlas.authorize.AtlasAuthorizationUtils; import org.apache.atlas.authorize.AtlasTypeAccessRequest; @@ -80,13 +81,14 @@ public AtlasVertex preCreate(AtlasClassificationDef classificationDef) throws At if (ret != null) { throw new AtlasBaseException(AtlasErrorCode.TYPE_ALREADY_EXISTS, classificationDef.getName()); } - ret = typeDefStore.findTypeVertexByDisplayName( - classificationDef.getDisplayName(), TypeCategory.TRAIT); - if (ret != null) { - throw new AtlasBaseException(AtlasErrorCode.TYPE_WITH_DISPLAY_NAME_ALREADY_EXISTS, classificationDef.getDisplayName()); + if(!RequestContext.get().getAllowDuplicateDisplayName()) { + ret = typeDefStore.findTypeVertexByDisplayName( + classificationDef.getDisplayName(), TypeCategory.TRAIT); + if (ret != null) { + throw new AtlasBaseException(AtlasErrorCode.TYPE_WITH_DISPLAY_NAME_ALREADY_EXISTS, classificationDef.getDisplayName()); + } } - - ret = typeDefStore.createTypeVertex(classificationDef); + ret = typeDefStore.createTypeVertex(classificationDef); updateVertexPreCreate(classificationDef, (AtlasClassificationType)type, ret); @@ -189,12 +191,12 @@ public void validateType(AtlasBaseTypeDef typeDef) throws AtlasBaseException { throw new AtlasBaseException(AtlasErrorCode.MISSING_CLASSIFICATION_DISPLAY_NAME); } - //validate uniqueness of display name - AtlasVertex ret = typeDefStore.findTypeVertexByDisplayName( - classifiDef.getDisplayName(), DataTypes.TypeCategory.TRAIT); - if (ret != null && ( - classifiDef.getGuid() == null || !classifiDef.getGuid().equals(ret.getProperty(Constants.GUID_PROPERTY_KEY, String.class)))) { - throw new AtlasBaseException(AtlasErrorCode.TYPE_WITH_DISPLAY_NAME_ALREADY_EXISTS, classifiDef.getDisplayName()); + if(!RequestContext.get().getAllowDuplicateDisplayName()){ + AtlasVertex ret = typeDefStore.findTypeVertexByDisplayName( + classifiDef.getDisplayName(), DataTypes.TypeCategory.TRAIT); + if (ret != null && (classifiDef.getGuid() == null || !classifiDef.getGuid().equals(ret.getProperty(Constants.GUID_PROPERTY_KEY, String.class)))){ + throw new AtlasBaseException(AtlasErrorCode.TYPE_WITH_DISPLAY_NAME_ALREADY_EXISTS, classifiDef.getDisplayName()); + } } } @@ -305,7 +307,11 @@ public AtlasVertex preDeleteByName(String name) throws AtlasBaseException { AtlasVertex ret = typeDefStore.findTypeVertexByNameAndCategory(name, TypeCategory.TRAIT); if (AtlasGraphUtilsV2.typeHasInstanceVertex(name)) { - throw new AtlasBaseException(AtlasErrorCode.TYPE_HAS_REFERENCES, name); + String displayName = ""; + if (existingDef != null) { + displayName = existingDef.getDisplayName() != null ? existingDef.getDisplayName() : ""; + } + throw new AtlasBaseException(AtlasErrorCode.CLASSIFICATION_TYPE_HAS_REFERENCES, displayName, name); } if (ret == null) { @@ -336,7 +342,11 @@ public AtlasVertex preDeleteByGuid(String guid) throws AtlasBaseException { String typeName = AtlasGraphUtilsV2.getEncodedProperty(ret, TYPENAME_PROPERTY_KEY, String.class); if (AtlasGraphUtilsV2.typeHasInstanceVertex(typeName)) { - throw new AtlasBaseException(AtlasErrorCode.TYPE_HAS_REFERENCES, typeName); + String displayName = ""; + if (existingDef != null) { + displayName = existingDef.getDisplayName() != null ? existingDef.getDisplayName() : ""; + } + throw new AtlasBaseException(AtlasErrorCode.CLASSIFICATION_TYPE_HAS_REFERENCES, displayName, typeName); } if (ret == null) { diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityChangeNotifier.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityChangeNotifier.java index bb0df111bf4..7fc95c3981b 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityChangeNotifier.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityChangeNotifier.java @@ -18,6 +18,7 @@ package org.apache.atlas.repository.store.graph.v2; +import org.apache.atlas.AtlasConfiguration; import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.AtlasException; import org.apache.atlas.RequestContext; @@ -54,12 +55,7 @@ import org.springframework.stereotype.Component; import javax.inject.Inject; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.ListIterator; -import java.util.Map; -import java.util.Set; +import java.util.*; import java.util.function.Predicate; import java.util.stream.Collectors; @@ -78,7 +74,7 @@ public class AtlasEntityChangeNotifier implements IAtlasEntityChangeNotifier { private final FullTextMapperV2 fullTextMapperV2; private final AtlasTypeRegistry atlasTypeRegistry; private final boolean isV2EntityNotificationEnabled; - + private static final List ALLOWED_RELATIONSHIP_TYPES = Arrays.asList(AtlasConfiguration.SUPPORTED_RELATIONSHIP_EVENTS.getStringArray()); @Inject public AtlasEntityChangeNotifier(Set entityChangeListeners, @@ -129,6 +125,9 @@ public void notifyRelationshipMutation(List relationships, En return; } + relationships = relationships.stream().filter(r -> ALLOWED_RELATIONSHIP_TYPES.contains(r.getTypeName())).collect(Collectors.toList()); + if (CollectionUtils.isEmpty(relationships)) + return; switch (operationType) { case RELATIONSHIP_CREATE: notifyRelationshipListeners(relationships, EntityOperation.CREATE, false); @@ -173,12 +172,12 @@ public void onClassificationAddedToEntity(AtlasEntity entity, List entities, List addedClassifications) throws AtlasBaseException { + public void onClassificationsAddedToEntities(List entities, List addedClassifications, boolean forceInline) throws AtlasBaseException { if (isV2EntityNotificationEnabled) { doFullTextMappingHelper(entities); for (EntityChangeListenerV2 listener : entityChangeListenersV2) { - listener.onClassificationsAdded(entities, addedClassifications); + listener.onClassificationsAdded(entities, addedClassifications, forceInline); } } else { updateFullTextMapping(entities, addedClassifications); @@ -603,8 +602,10 @@ private List toAtlasEntities(List entityHeaders, entity = new AtlasEntity(entityHeader); } else { String entityGuid = entityHeader.getGuid(); - entity = fullTextMapperV2.getAndCacheEntity(entityGuid); + if (operation == EntityOperation.UPDATE || entityHeader.getAttributes() != null) { + entity.setAttributes(entityHeader.getAttributes()); + } } if (entity != null) { diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityComparator.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityComparator.java index d962385c2ca..ce38f4898e1 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityComparator.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityComparator.java @@ -19,6 +19,7 @@ package org.apache.atlas.repository.store.graph.v2; import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.TypeCategory; import org.apache.atlas.model.instance.AtlasClassification; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.repository.graphdb.AtlasVertex; @@ -28,6 +29,7 @@ import org.apache.atlas.utils.AtlasEntityUtil; import org.apache.commons.collections.MapUtils; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Objects; @@ -81,7 +83,28 @@ private AtlasEntityDiffResult getDiffResult(AtlasEntity updatedEntity, AtlasEnti continue; } - Object newVal = entry.getValue(); + TypeCategory category = attribute.getAttributeType().getTypeCategory(); + boolean isDefaultValueNotNull = !attribute.getAttributeDef().getIsDefaultValueNull(); + Object newVal; + + if (entry.getValue() == null && isDefaultValueNotNull) { + switch (category) { + case PRIMITIVE: + newVal = attribute.getAttributeType().createDefaultValue(); + break; + case ARRAY: + newVal = new ArrayList<>(); + break; + case MAP: + newVal = MapUtils.EMPTY_MAP; + break; + default: + newVal = entry.getValue(); + } + } else { + newVal = entry.getValue(); + } + Object currVal = (storedEntity != null) ? storedEntity.getAttribute(attrName) : entityRetriever.getEntityAttribute(storedVertex, attribute); if (!attribute.getAttributeType().areEqualValues(currVal, newVal, guidRefMap)) { diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index ef9d337a48d..529a9420d2f 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -24,37 +24,53 @@ import org.apache.atlas.DeleteType; import org.apache.atlas.GraphTransactionInterceptor; import org.apache.atlas.RequestContext; +import org.apache.atlas.AtlasException; +import org.apache.atlas.AtlasConfiguration; import org.apache.atlas.annotation.GraphTransaction; -import org.apache.atlas.authorize.AtlasAdminAccessRequest; -import org.apache.atlas.authorize.AtlasAuthorizationUtils; -import org.apache.atlas.authorize.AtlasEntityAccessRequest; +import org.apache.atlas.authorize.*; import org.apache.atlas.authorize.AtlasEntityAccessRequest.AtlasEntityAccessRequestBuilder; import org.apache.atlas.authorize.AtlasPrivilege; +import org.apache.atlas.discovery.EntityDiscoveryService; import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.featureflag.FeatureFlagStore; import org.apache.atlas.model.TypeCategory; -import org.apache.atlas.model.instance.AtlasCheckStateRequest; -import org.apache.atlas.model.instance.AtlasCheckStateResult; -import org.apache.atlas.model.instance.AtlasClassification; -import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.model.instance.*; import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo; import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo; import org.apache.atlas.model.instance.AtlasEntity.Status; -import org.apache.atlas.model.instance.AtlasEntityHeader; -import org.apache.atlas.model.instance.AtlasEntityHeaders; -import org.apache.atlas.model.instance.AtlasObjectId; -import org.apache.atlas.model.instance.EntityMutationResponse; +import org.apache.atlas.model.tasks.AtlasTask; import org.apache.atlas.model.typedef.AtlasBaseTypeDef; +import org.apache.atlas.repository.Constants; +import org.apache.atlas.repository.RepositoryException; import org.apache.atlas.repository.graph.GraphHelper; +import org.apache.atlas.repository.graphdb.AtlasEdge; +import org.apache.atlas.repository.graphdb.AtlasEdgeDirection; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.patches.PatchContext; import org.apache.atlas.repository.patches.ReIndexPatch; import org.apache.atlas.repository.store.graph.AtlasEntityStore; +import org.apache.atlas.repository.store.graph.AtlasRelationshipStore; import org.apache.atlas.repository.store.graph.EntityGraphDiscovery; import org.apache.atlas.repository.store.graph.EntityGraphDiscoveryContext; import org.apache.atlas.repository.store.graph.v1.DeleteHandlerDelegate; import org.apache.atlas.repository.store.graph.v2.AtlasEntityComparator.AtlasEntityDiffResult; import org.apache.atlas.repository.store.graph.v1.RestoreHandlerV1; +import org.apache.atlas.repository.store.graph.v2.preprocessor.AuthPolicyPreProcessor; +import org.apache.atlas.repository.store.graph.v2.preprocessor.ConnectionPreProcessor; +import org.apache.atlas.repository.store.graph.v2.preprocessor.resource.LinkPreProcessor; +import org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessor; +import org.apache.atlas.repository.store.graph.v2.preprocessor.accesscontrol.PersonaPreProcessor; +import org.apache.atlas.repository.store.graph.v2.preprocessor.accesscontrol.PurposePreProcessor; +import org.apache.atlas.repository.store.graph.v2.preprocessor.glossary.CategoryPreProcessor; +import org.apache.atlas.repository.store.graph.v2.preprocessor.glossary.GlossaryPreProcessor; +import org.apache.atlas.repository.store.graph.v2.preprocessor.glossary.TermPreProcessor; +import org.apache.atlas.repository.store.graph.v2.preprocessor.resource.ReadmePreProcessor; +import org.apache.atlas.repository.store.graph.v2.preprocessor.sql.QueryCollectionPreProcessor; +import org.apache.atlas.repository.store.graph.v2.preprocessor.sql.QueryFolderPreProcessor; +import org.apache.atlas.repository.store.graph.v2.preprocessor.sql.QueryPreProcessor; +import org.apache.atlas.repository.store.graph.v2.tasks.MeaningsTask; +import org.apache.atlas.tasks.TaskManagement; import org.apache.atlas.type.AtlasArrayType; import org.apache.atlas.type.AtlasBusinessMetadataType; import org.apache.atlas.type.AtlasBusinessMetadataType.AtlasBusinessAttribute; @@ -69,6 +85,7 @@ import org.apache.atlas.bulkimport.BulkImportResponse.ImportInfo; import org.apache.atlas.util.FileUtils; import org.apache.atlas.utils.AtlasEntityUtil; +import org.apache.atlas.utils.AtlasPerfMetrics; import org.apache.atlas.utils.AtlasPerfMetrics.MetricRecorder; import org.apache.atlas.utils.AtlasPerfTracer; import org.apache.commons.collections.CollectionUtils; @@ -81,15 +98,30 @@ import javax.inject.Inject; import java.io.InputStream; import java.util.*; +import java.util.stream.Collectors; import static java.lang.Boolean.FALSE; import static org.apache.atlas.AtlasConfiguration.STORE_DIFFERENTIAL_AUDITS; import static org.apache.atlas.bulkimport.BulkImportResponse.ImportStatus.FAILED; +import static org.apache.atlas.model.instance.AtlasEntity.Status.ACTIVE; import static org.apache.atlas.model.instance.EntityMutations.EntityOperation.*; import static org.apache.atlas.repository.Constants.*; +<<<<<<< HEAD import static org.apache.atlas.repository.graph.GraphHelper.getTypeName; import static org.apache.atlas.repository.graph.GraphHelper.isEntityIncomplete; +======= +import static org.apache.atlas.repository.graph.GraphHelper.*; +import static org.apache.atlas.repository.graph.GraphHelper.getStatus; +>>>>>>> 7f3c811fb15361ba82bfb4edd7a8393798863ff0 import static org.apache.atlas.repository.store.graph.v2.EntityGraphMapper.validateLabels; +import static org.apache.atlas.repository.store.graph.v2.tasks.MeaningsTaskFactory.*; +import static org.apache.atlas.type.Constants.HAS_LINEAGE; +import static org.apache.atlas.type.Constants.HAS_LINEAGE_VALID; +import static org.apache.atlas.type.Constants.MEANINGS_TEXT_PROPERTY_KEY; +import static org.apache.atlas.type.Constants.MEANINGS_PROPERTY_KEY; +import static org.apache.atlas.type.Constants.MEANING_NAMES_PROPERTY_KEY; +import static org.apache.atlas.type.Constants.PENDING_TASKS_PROPERTY_KEY; + @Component @@ -97,6 +129,10 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore { private static final Logger LOG = LoggerFactory.getLogger(AtlasEntityStoreV2.class); private static final Logger PERF_LOG = AtlasPerfTracer.getPerfLogger("store.EntityStore"); + static final boolean DEFERRED_ACTION_ENABLED = AtlasConfiguration.TASKS_USE_ENABLED.getBoolean(); + + private static final String ATTR_MEANINGS = "meanings"; + private final AtlasGraph graph; private final DeleteHandlerDelegate deleteDelegate; private final RestoreHandlerV1 restoreHandlerV1; @@ -105,10 +141,16 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore { private final EntityGraphMapper entityGraphMapper; private final EntityGraphRetriever entityRetriever; private boolean storeDifferentialAudits; + private final GraphHelper graphHelper; + private final TaskManagement taskManagement; + private EntityDiscoveryService discovery; + private final AtlasRelationshipStore atlasRelationshipStore; + private final FeatureFlagStore featureFlagStore; @Inject public AtlasEntityStoreV2(AtlasGraph graph, DeleteHandlerDelegate deleteDelegate, RestoreHandlerV1 restoreHandlerV1, AtlasTypeRegistry typeRegistry, - IAtlasEntityChangeNotifier entityChangeNotifier, EntityGraphMapper entityGraphMapper) { + IAtlasEntityChangeNotifier entityChangeNotifier, EntityGraphMapper entityGraphMapper, TaskManagement taskManagement, + AtlasRelationshipStore atlasRelationshipStore, FeatureFlagStore featureFlagStore) { this.graph = graph; this.deleteDelegate = deleteDelegate; this.restoreHandlerV1 = restoreHandlerV1; @@ -117,6 +159,17 @@ public AtlasEntityStoreV2(AtlasGraph graph, DeleteHandlerDelegate deleteDelegate this.entityGraphMapper = entityGraphMapper; this.entityRetriever = new EntityGraphRetriever(graph, typeRegistry); this.storeDifferentialAudits = STORE_DIFFERENTIAL_AUDITS.getBoolean(); + this.graphHelper = new GraphHelper(graph); + this.taskManagement = taskManagement; + this.atlasRelationshipStore = atlasRelationshipStore; + this.featureFlagStore = featureFlagStore; + + try { + this.discovery = new EntityDiscoveryService(typeRegistry, graph, null, null, null, null); + } catch (AtlasException e) { + e.printStackTrace(); + } + } @VisibleForTesting @@ -326,6 +379,12 @@ public AtlasEntityWithExtInfo getByUniqueAttributes(AtlasEntityType entityType, return ret; } + @Override + @GraphTransaction + public AtlasEntityHeader getAtlasEntityHeaderWithoutAuthorization(String guid, String qualifiedName, String typeName) throws AtlasBaseException { + return extractEntityHeader( guid, qualifiedName, typeName); + } + @Override @GraphTransaction public AtlasEntityHeader getEntityHeaderByUniqueAttributes(AtlasEntityType entityType, Map uniqAttributes) throws AtlasBaseException { @@ -385,8 +444,9 @@ public EntityMutationResponse createOrUpdate(EntityStream entityStream, boolean @Override @GraphTransaction - public EntityMutationResponse createOrUpdate(EntityStream entityStream, boolean replaceClassifications, boolean replaceBusinessAttributes) throws AtlasBaseException { - return createOrUpdate(entityStream, false, replaceClassifications, replaceBusinessAttributes); + public EntityMutationResponse createOrUpdate(EntityStream entityStream, boolean replaceClassifications, + boolean replaceBusinessAttributes, boolean isOverwriteBusinessAttributes) throws AtlasBaseException { + return createOrUpdate(entityStream, false, replaceClassifications, replaceBusinessAttributes, isOverwriteBusinessAttributes); } @Override @@ -398,12 +458,12 @@ public EntityMutationResponse createOrUpdateGlossary(EntityStream entityStream, @Override @GraphTransaction(logRollback = false) public EntityMutationResponse createOrUpdateForImport(EntityStream entityStream) throws AtlasBaseException { - return createOrUpdate(entityStream, false, true, true); + return createOrUpdate(entityStream, false, true, true, false); } @Override public EntityMutationResponse createOrUpdateForImportNoCommit(EntityStream entityStream) throws AtlasBaseException { - return createOrUpdate(entityStream, false, true, true); + return createOrUpdate(entityStream, false, true, true, false); } @Override @@ -435,7 +495,7 @@ public EntityMutationResponse updateEntity(AtlasObjectId objectId, AtlasEntityWi entity.setGuid(guid); - return createOrUpdate(new AtlasEntityStream(updatedEntityInfo), isPartialUpdate, false, false); + return createOrUpdate(new AtlasEntityStream(updatedEntityInfo), isPartialUpdate, false, false, false); } @Override @@ -455,9 +515,9 @@ public EntityMutationResponse updateByUniqueAttributes(AtlasEntityType entityTyp entity.setGuid(guid); - AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_UPDATE, new AtlasEntityHeader(entity)), "update entity ByUniqueAttributes"); + AtlasAuthorizationUtils.verifyUpdateEntityAccess(typeRegistry, new AtlasEntityHeader(entity), "update entity ByUniqueAttributes"); - return createOrUpdate(new AtlasEntityStream(updatedEntityInfo), true, false, false); + return createOrUpdate(new AtlasEntityStream(updatedEntityInfo), true, false, false, false); } @Override @@ -472,7 +532,7 @@ public EntityMutationResponse updateEntityAttributeByGuid(String guid, String at AtlasEntityType entityType = (AtlasEntityType) typeRegistry.getType(entity.getTypeName()); AtlasAttribute attr = entityType.getAttribute(attrName); - AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_UPDATE, entity), "update entity ByUniqueAttributes : guid=", guid ); + AtlasAuthorizationUtils.verifyUpdateEntityAccess(typeRegistry, entity, "update entity ByUniqueAttributes : guid=" + guid); if (attr == null) { attr = entityType.getRelationshipAttribute(attrName, AtlasEntityUtil.getRelationshipType(attrValue)); @@ -511,7 +571,7 @@ public EntityMutationResponse updateEntityAttributeByGuid(String guid, String at throw new AtlasBaseException(AtlasErrorCode.ATTRIBUTE_UPDATE_NOT_SUPPORTED, attrName, attrType.getTypeName()); } - return createOrUpdate(new AtlasEntityStream(updateEntity), true, false, false); + return createOrUpdate(new AtlasEntityStream(updateEntity), true, false, false, false); } @Override @@ -527,7 +587,7 @@ public EntityMutationResponse deleteById(final String guid) throws AtlasBaseExce if (vertex != null) { AtlasEntityHeader entityHeader = entityRetriever.toAtlasEntityHeaderWithClassifications(vertex); - AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_DELETE, entityHeader), "delete entity: guid=", guid); + AtlasAuthorizationUtils.verifyDeleteEntityAccess(typeRegistry, entityHeader, "delete entity: guid=" + guid); deletionCandidates.add(vertex); } else { @@ -540,9 +600,12 @@ public EntityMutationResponse deleteById(final String guid) throws AtlasBaseExce EntityMutationResponse ret = deleteVertices(deletionCandidates); + if(ret.getDeletedEntities()!=null) + processTermEntityDeletion(ret.getDeletedEntities()); + // Notify the change listeners entityChangeNotifier.onEntitiesMutated(ret, false); - + atlasRelationshipStore.onRelationshipsMutated(RequestContext.get().getRelationshipMutationMap()); return ret; } @@ -570,7 +633,7 @@ public EntityMutationResponse deleteByIds(final List guids) throws Atlas AtlasEntityHeader entityHeader = entityRetriever.toAtlasEntityHeaderWithClassifications(vertex); - AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_DELETE, entityHeader), "delete entity: guid=", guid); + AtlasAuthorizationUtils.verifyDeleteEntityAccess(typeRegistry, entityHeader, "delete entity: guid=" + guid); deletionCandidates.add(vertex); } @@ -581,12 +644,16 @@ public EntityMutationResponse deleteByIds(final List guids) throws Atlas EntityMutationResponse ret = deleteVertices(deletionCandidates); + if(ret.getDeletedEntities() != null) + processTermEntityDeletion(ret.getDeletedEntities()); + // Notify the change listeners entityChangeNotifier.onEntitiesMutated(ret, false); - + atlasRelationshipStore.onRelationshipsMutated(RequestContext.get().getRelationshipMutationMap()); return ret; } + @Override @GraphTransaction public EntityMutationResponse restoreByIds(final List guids) throws AtlasBaseException { @@ -609,7 +676,7 @@ public EntityMutationResponse restoreByIds(final List guids) throws Atla AtlasEntityHeader entityHeader = entityRetriever.toAtlasEntityHeaderWithClassifications(vertex); - AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_DELETE, entityHeader), "delete entity: guid=", guid); + AtlasAuthorizationUtils.verifyDeleteEntityAccess(typeRegistry, entityHeader, "delete entity: guid=" + guid); restoreCandidates.add(vertex); } @@ -622,7 +689,7 @@ public EntityMutationResponse restoreByIds(final List guids) throws Atla // Notify the change listeners entityChangeNotifier.onEntitiesMutated(ret, false); - + atlasRelationshipStore.onRelationshipsMutated(RequestContext.get().getRelationshipMutationMap()); return ret; } @@ -646,7 +713,7 @@ public EntityMutationResponse purgeByIds(Set guids) throws AtlasBaseExce continue; } - + this.recordRelationshipsToBePurged(vertex); purgeCandidates.add(vertex); } @@ -658,7 +725,7 @@ public EntityMutationResponse purgeByIds(Set guids) throws AtlasBaseExce // Notify the change listeners entityChangeNotifier.onEntitiesMutated(ret, false); - + atlasRelationshipStore.onRelationshipsMutated(RequestContext.get().getRelationshipMutationMap()); return ret; } @@ -675,7 +742,8 @@ public EntityMutationResponse deleteByUniqueAttributes(AtlasEntityType entityTyp if (vertex != null) { AtlasEntityHeader entityHeader = entityRetriever.toAtlasEntityHeaderWithClassifications(vertex); - AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_DELETE, entityHeader), "delete entity: typeName=", entityType.getTypeName(), ", uniqueAttributes=", uniqAttributes); + AtlasAuthorizationUtils.verifyDeleteEntityAccess(typeRegistry, entityHeader, + "delete entity: typeName=" + entityType.getTypeName() + ", uniqueAttributes=" + uniqAttributes); deletionCandidates.add(vertex); } else { @@ -688,9 +756,12 @@ public EntityMutationResponse deleteByUniqueAttributes(AtlasEntityType entityTyp EntityMutationResponse ret = deleteVertices(deletionCandidates); + if(ret.getDeletedEntities()!=null) + processTermEntityDeletion(ret.getDeletedEntities()); + // Notify the change listeners entityChangeNotifier.onEntitiesMutated(ret, false); - + atlasRelationshipStore.onRelationshipsMutated(RequestContext.get().getRelationshipMutationMap()); return ret; } @@ -703,6 +774,7 @@ public EntityMutationResponse deleteByUniqueAttributes(List objec EntityMutationResponse ret = new EntityMutationResponse(); Collection deletionCandidates = new ArrayList<>(); +<<<<<<< HEAD for (AtlasObjectId objectId: objectIds) { if (StringUtils.isEmpty(objectId.getTypeName())) { @@ -743,6 +815,146 @@ public EntityMutationResponse deleteByUniqueAttributes(List objec return ret; } +======= + try { + for (AtlasObjectId objectId : objectIds) { + if (StringUtils.isEmpty(objectId.getTypeName())) { + throw new AtlasBaseException(AtlasErrorCode.INVALID_PARAMETERS, "typeName not specified"); + } + + if (MapUtils.isEmpty(objectId.getUniqueAttributes())) { + throw new AtlasBaseException(AtlasErrorCode.INVALID_PARAMETERS, "uniqueAttributes not specified"); + } + + AtlasEntityType entityType = typeRegistry.getEntityTypeByName(objectId.getTypeName()); + + if (entityType == null) { + throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_INVALID, TypeCategory.ENTITY.name(), objectId.getTypeName()); + } + + AtlasVertex vertex = AtlasGraphUtilsV2.findByUniqueAttributes(graph, entityType, objectId.getUniqueAttributes()); + + if (vertex != null) { + AtlasEntityHeader entityHeader = entityRetriever.toAtlasEntityHeaderWithClassifications(vertex); + + AtlasAuthorizationUtils.verifyDeleteEntityAccess(typeRegistry, entityHeader, + "delete entity: typeName=" + entityType.getTypeName() + ", uniqueAttributes=" + objectId.getUniqueAttributes()); + + deletionCandidates.add(vertex); + } else { + if (LOG.isDebugEnabled()) { + // Entity does not exist - treat as non-error, since the caller + // wanted to delete the entity and it's already gone. + LOG.debug("Deletion request ignored for non-existent entity with uniqueAttributes " + objectId.getUniqueAttributes()); + } + } + } + + ret = deleteVertices(deletionCandidates); + + if (ret.getDeletedEntities() != null) + processTermEntityDeletion(ret.getDeletedEntities()); + // Notify the change listeners + entityChangeNotifier.onEntitiesMutated(ret, false); + atlasRelationshipStore.onRelationshipsMutated(RequestContext.get().getRelationshipMutationMap()); + + } catch (Exception e) { + LOG.error("Failed to delete objects:{}", objectIds.stream().map(AtlasObjectId::getUniqueAttributes).collect(Collectors.toList()), e); + throw new AtlasBaseException(e); + } + return ret; + } + + private void processTermEntityDeletion(List deletedEntities) throws AtlasBaseException{ + for(AtlasEntityHeader entity:deletedEntities){ + if(ATLAS_GLOSSARY_TERM_ENTITY_TYPE.equals(entity.getTypeName())){ + + String termQualifiedName = entity.getAttribute(QUALIFIED_NAME).toString(); + String termName = entity.getAttribute(NAME).toString(); + String guid = entity.getGuid(); + Boolean isHardDelete = DeleteType.HARD.name().equals(entity.getDeleteHandler()); + + if(checkEntityTermAssociation(termQualifiedName)){ + if(DEFERRED_ACTION_ENABLED && taskManagement!=null){ + createAndQueueTask(termName, termQualifiedName, guid, isHardDelete); + }else{ + updateMeaningsNamesInEntitiesOnTermDelete(termName, termQualifiedName, guid); + } + } + } + } + } + + private boolean checkEntityTermAssociation(String termQName) throws AtlasBaseException{ + List entityHeader; + + try { + entityHeader = discovery.searchUsingTermQualifiedName(0, 1, termQName,null, null); + } catch (AtlasBaseException e) { + throw e; + } + Boolean hasEntityAssociation = entityHeader != null ? true : false; + + return hasEntityAssociation; + } + + public void updateMeaningsNamesInEntitiesOnTermDelete(String termName, String termQName, String termGuid) throws AtlasBaseException { + int from = 0; + + Set attributes = new HashSet(){{ + add(ATTR_MEANINGS); + }}; + Set relationAttributes = new HashSet(){{ + add(STATE_PROPERTY_KEY); + add(NAME); + }}; + + while (true) { + List entityHeaders = discovery.searchUsingTermQualifiedName(from, ELASTICSEARCH_PAGINATION_SIZE, + termQName, attributes, relationAttributes); + + if (entityHeaders == null) + break; + + for (AtlasEntityHeader entityHeader : entityHeaders) { + List meanings = (List) entityHeader.getAttribute(ATTR_MEANINGS); + + String updatedMeaningsText = meanings.stream() + .filter(x -> !termGuid.equals(x.getGuid())) + .filter(x -> ACTIVE.name().equals(x.getAttributes().get(STATE_PROPERTY_KEY))) + .map(x -> x.getAttributes().get(NAME).toString()) + .collect(Collectors.joining(",")); + + + AtlasVertex entityVertex = AtlasGraphUtilsV2.findByGuid(entityHeader.getGuid()); + AtlasGraphUtilsV2.removeItemFromListPropertyValue(entityVertex, MEANINGS_PROPERTY_KEY, termQName); + AtlasGraphUtilsV2.setEncodedProperty(entityVertex, MEANINGS_TEXT_PROPERTY_KEY, updatedMeaningsText); + AtlasGraphUtilsV2.removeItemFromListPropertyValue(entityVertex, MEANING_NAMES_PROPERTY_KEY, termName); + } + from += ELASTICSEARCH_PAGINATION_SIZE; + + if (entityHeaders.size() < ELASTICSEARCH_PAGINATION_SIZE) + break; + } + + } + + public void createAndQueueTask(String termName, String termQName, String termGuid, Boolean isHardDelete){ + String taskType = isHardDelete ? UPDATE_ENTITY_MEANINGS_ON_TERM_HARD_DELETE : UPDATE_ENTITY_MEANINGS_ON_TERM_SOFT_DELETE; + String currentUser = RequestContext.getCurrentUser(); + Map taskParams = MeaningsTask.toParameters(termName, termQName, termGuid); + AtlasTask task = taskManagement.createTask(taskType, currentUser, taskParams); + + if(!isHardDelete){ + AtlasVertex termVertex = AtlasGraphUtilsV2.findByGuid(termGuid); + AtlasGraphUtilsV2.addEncodedProperty(termVertex, PENDING_TASKS_PROPERTY_KEY, task.getGuid()); + } + + RequestContext.get().queueTask(task); + } + + +>>>>>>> 7f3c811fb15361ba82bfb4edd7a8393798863ff0 @Override @GraphTransaction public String getGuidByUniqueAttributes(AtlasEntityType entityType, Map uniqAttributes) throws AtlasBaseException{ @@ -1059,56 +1271,7 @@ public void addOrUpdateBusinessAttributes(String guid, Map addOrUpdateBusinessAttributes(guid={}, businessAttributes={}, isOverwrite={})", guid, businessAttrbutes, isOverwrite); } - if (StringUtils.isEmpty(guid)) { - throw new AtlasBaseException(AtlasErrorCode.INVALID_PARAMETERS, "guid is null/empty"); - } - - if (MapUtils.isEmpty(businessAttrbutes)) { - throw new AtlasBaseException(AtlasErrorCode.INVALID_PARAMETERS, "businessAttributes is null/empty"); - } - - AtlasVertex entityVertex = AtlasGraphUtilsV2.findByGuid(graph, guid); - - if (entityVertex == null) { - throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND, guid); - } - - String typeName = getTypeName(entityVertex); - AtlasEntityType entityType = typeRegistry.getEntityTypeByName(typeName); - AtlasEntityHeader entityHeader = entityRetriever.toAtlasEntityHeaderWithClassifications(entityVertex); - Map> currEntityBusinessAttributes = entityRetriever.getBusinessMetadata(entityVertex); - Set updatedBusinessMetadataNames = new HashSet<>(); - - for (String bmName : entityType.getBusinessAttributes().keySet()) { - Map bmAttrs = businessAttrbutes.get(bmName); - Map currBmAttrs = currEntityBusinessAttributes != null ? currEntityBusinessAttributes.get(bmName) : null; - - if (bmAttrs == null && !isOverwrite) { - continue; - } else if (MapUtils.isEmpty(bmAttrs) && MapUtils.isEmpty(currBmAttrs)) { // no change - continue; - } else if (Objects.equals(bmAttrs, currBmAttrs)) { // no change - continue; - } - - updatedBusinessMetadataNames.add(bmName); - } - - AtlasEntityAccessRequestBuilder requestBuilder = new AtlasEntityAccessRequestBuilder(typeRegistry, AtlasPrivilege.ENTITY_UPDATE_BUSINESS_METADATA, entityHeader); - - for (String bmName : updatedBusinessMetadataNames) { - requestBuilder.setBusinessMetadata(bmName); - - AtlasAuthorizationUtils.verifyAccess(requestBuilder.build(), "add/update business-metadata: guid=", guid, ", business-metadata-name=", bmName); - } - - validateBusinessAttributes(entityVertex, entityType, businessAttrbutes, isOverwrite); - - if (isOverwrite) { - entityGraphMapper.setBusinessAttributes(entityVertex, entityType, businessAttrbutes); - } else { - entityGraphMapper.addOrUpdateBusinessAttributes(entityVertex, entityType, businessAttrbutes); - } + entityGraphMapper.addOrUpdateBusinessAttributes(guid, businessAttrbutes, isOverwrite); if (LOG.isDebugEnabled()) { LOG.debug("<== addOrUpdateBusinessAttributes(guid={}, businessAttributes={}, isOverwrite={})", guid, businessAttrbutes, isOverwrite); @@ -1138,14 +1301,6 @@ public void removeBusinessAttributes(String guid, Map createOrUpdate()"); } @@ -1312,10 +1467,12 @@ private EntityMutationResponse createOrUpdate(EntityStream entityStream, boolean final EntityMutationContext context = preCreateOrUpdate(entityStream, entityGraphMapper, isPartialUpdate); // Check if authorized to create entities - if (!RequestContext.get().isImportInProgress()) { + if (!RequestContext.get().isImportInProgress() && !RequestContext.get().isSkipAuthorizationCheck()) { for (AtlasEntity entity : context.getCreatedEntities()) { - AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_CREATE, new AtlasEntityHeader(entity)), - "create entity: type=", entity.getTypeName()); + if (!PreProcessor.skipInitialAuthCheckTypes.contains(entity.getTypeName())) { + AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_CREATE, new AtlasEntityHeader(entity)), + "create entity: type=", entity.getTypeName()); + } } } @@ -1366,22 +1523,45 @@ private EntityMutationResponse createOrUpdate(EntityStream entityStream, boolean // Check if authorized to update entities if (!reqContext.isImportInProgress()) { for (AtlasEntity entity : context.getUpdatedEntities()) { +<<<<<<< HEAD AtlasEntityHeader entityHeader = entityRetriever.toAtlasEntityHeaderWithClassifications(entity.getGuid()); AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_UPDATE, entityHeader), "update entity: type=", entity.getTypeName()); +======= + AtlasEntityHeader entityHeaderWithClassifications = entityRetriever.toAtlasEntityHeaderWithClassifications(entity.getGuid()); + AtlasEntityHeader entityHeader = new AtlasEntityHeader(entity); + + if(CollectionUtils.isNotEmpty(entityHeaderWithClassifications.getClassifications())) { + entityHeader.setClassifications(entityHeaderWithClassifications.getClassifications()); + } + + AtlasEntity diffEntity = reqContext.getDifferentialEntity(entity.getGuid()); + boolean skipAuthBaseConditions = diffEntity != null && MapUtils.isEmpty(diffEntity.getCustomAttributes()) && MapUtils.isEmpty(diffEntity.getBusinessAttributes()) && CollectionUtils.isEmpty(diffEntity.getClassifications()) && CollectionUtils.isEmpty(diffEntity.getLabels()); + boolean skipAuthMeaningsUpdate = diffEntity != null && MapUtils.isNotEmpty(diffEntity.getRelationshipAttributes()) && diffEntity.getRelationshipAttributes().containsKey("meanings") && diffEntity.getRelationshipAttributes().size() == 1 && MapUtils.isEmpty(diffEntity.getAttributes()); + boolean skipAuthStarredDetailsUpdate = diffEntity != null && MapUtils.isEmpty(diffEntity.getRelationshipAttributes()) && MapUtils.isNotEmpty(diffEntity.getAttributes()) && diffEntity.getAttributes().size() == 3 && diffEntity.getAttributes().containsKey(ATTR_STARRED_BY) && diffEntity.getAttributes().containsKey(ATTR_STARRED_COUNT) && diffEntity.getAttributes().containsKey(ATTR_STARRED_DETAILS_LIST); + if (skipAuthBaseConditions && (skipAuthMeaningsUpdate || skipAuthStarredDetailsUpdate)) { + //do nothing, only diff is relationshipAttributes.meanings or starred, allow update + } else { + AtlasAuthorizationUtils.verifyUpdateEntityAccess(typeRegistry, entityHeader,"update entity: type=" + entity.getTypeName()); + } +>>>>>>> 7f3c811fb15361ba82bfb4edd7a8393798863ff0 } } reqContext.endMetricRecord(checkForUnchangedEntities); } - EntityMutationResponse ret = entityGraphMapper.mapAttributesAndClassifications(context, isPartialUpdate, replaceClassifications, replaceBusinessAttributes); + executePreProcessor(context); + + EntityMutationResponse ret = entityGraphMapper.mapAttributesAndClassifications(context, isPartialUpdate, + replaceClassifications, replaceBusinessAttributes, isOverwriteBusinessAttribute); ret.setGuidAssignments(context.getGuidAssignments()); + // Notify the change listeners entityChangeNotifier.onEntitiesMutated(ret, RequestContext.get().isImportInProgress()); - + atlasRelationshipStore.onRelationshipsMutated(RequestContext.get().getRelationshipMutationMap()); if (LOG.isDebugEnabled()) { LOG.debug("<== createOrUpdate()"); } @@ -1394,6 +1574,33 @@ private EntityMutationResponse createOrUpdate(EntityStream entityStream, boolean } } + private void executePreProcessor(EntityMutationContext context) throws AtlasBaseException { + AtlasEntityType entityType; + PreProcessor preProcessor; + + List copyOfCreated = new ArrayList<>(context.getCreatedEntities()); + for (int i = 0; i < copyOfCreated.size() ; i++) { + AtlasEntity entity = ((List) context.getCreatedEntities()).get(i); + entityType = context.getType(entity.getGuid()); + preProcessor = getPreProcessor(entityType.getTypeName()); + + if (preProcessor != null) { + preProcessor.processAttributes(entity, context, CREATE); + } + } + + List copyOfUpdated = new ArrayList<>(context.getUpdatedEntities()); + for (int i = 0; i < copyOfUpdated.size() ; i++) { + AtlasEntity entity = ((List) context.getUpdatedEntities()).get(i); + entityType = context.getType(entity.getGuid()); + preProcessor = getPreProcessor(entityType.getTypeName()); + + if (preProcessor != null) { + preProcessor.processAttributes(entity, context, UPDATE); + } + } + } + private EntityMutationContext preCreateOrUpdate(EntityStream entityStream, EntityGraphMapper entityGraphMapper, boolean isPartialUpdate) throws AtlasBaseException { MetricRecorder metric = RequestContext.get().startMetricRecord("preCreateOrUpdate"); @@ -1415,9 +1622,12 @@ private EntityMutationContext preCreateOrUpdate(EntityStream entityStream, Entit } compactAttributes(entity, entityType); + flushAutoUpdateAttributes(entity, entityType); AtlasVertex vertex = getResolvedEntityVertex(discoveryContext, entity); + autoUpdateStarredDetailsAttributes(entity, vertex); + try { if (vertex != null) { if (!isPartialUpdate) { @@ -1442,6 +1652,7 @@ private EntityMutationContext preCreateOrUpdate(EntityStream entityStream, Entit } context.addUpdated(guid, entity, entityType, vertex); + } else { graphDiscoverer.validateAndNormalize(entity); @@ -1513,6 +1724,158 @@ private EntityMutationContext preCreateOrUpdate(EntityStream entityStream, Entit return context; } + private void autoUpdateStarredDetailsAttributes(AtlasEntity entity, AtlasVertex vertex) { + + MetricRecorder metric = RequestContext.get().startMetricRecord("autoUpdateStarredDetailsAttributes"); + + Boolean starEntityForUser = entity.getStarred(); + + if (starEntityForUser != null) { + + long requestTime = RequestContext.get().getRequestTime(); + String requestUser = RequestContext.get().getUser(); + + Set starredBy = new HashSet<>(); + Set starredDetailsList = new HashSet<>(); + int starredCount = 0; + + if (vertex != null) { + Set vertexStarredBy = vertex.getMultiValuedSetProperty(ATTR_STARRED_BY, String.class); + if (vertexStarredBy != null) { + starredBy = vertexStarredBy; + } + + Iterable starredDetailsEdges = vertex.getEdges(AtlasEdgeDirection.OUT, "__" + ATTR_STARRED_DETAILS_LIST); + for (AtlasEdge starredDetailsEdge : starredDetailsEdges) { + AtlasVertex starredDetailsVertex = starredDetailsEdge.getInVertex(); + String assetStarredBy = starredDetailsVertex.getProperty(ATTR_ASSET_STARRED_BY, String.class); + Long assetStarredAt = starredDetailsVertex.getProperty(ATTR_ASSET_STARRED_AT, Long.class); + AtlasStruct starredDetails = getStarredDetailsStruct(assetStarredBy, assetStarredAt); + starredDetailsList.add(starredDetails); + } + + starredCount = starredBy.size(); + } + + if (starEntityForUser) { + addUserToStarredAttributes(requestUser, requestTime, starredBy, starredDetailsList); + } else { + removeUserFromStarredAttributes(requestUser, starredBy, starredDetailsList); + } + + // Update entity attributes + if (starredBy.size() != starredCount) { + entity.setAttribute(ATTR_STARRED_BY, starredBy); + entity.setAttribute(ATTR_STARRED_DETAILS_LIST, starredDetailsList); + entity.setAttribute(ATTR_STARRED_COUNT, starredBy.size()); + } + + } + + RequestContext.get().endMetricRecord(metric); + } + + private void addUserToStarredAttributes(String requestUser, long requestTime, Set starredBy, Set starredDetailsList) { + //Check and update starredBy Attribute + if (!starredBy.contains(requestUser)){ + starredBy.add(requestUser); + } + + //Check and update starredDetailsList Attribute + boolean isStarredDetailsListUpdated = false; + for (AtlasStruct starredDetails : starredDetailsList) { + String assetStarredBy = (String) starredDetails.getAttribute(ATTR_ASSET_STARRED_BY); + if (assetStarredBy.equals(requestUser)) { + starredDetails.setAttribute(ATTR_ASSET_STARRED_AT, requestTime); + isStarredDetailsListUpdated = true; + break; + } + } + if (!isStarredDetailsListUpdated) { + AtlasStruct starredDetails = getStarredDetailsStruct(requestUser, requestTime); + starredDetailsList.add(starredDetails); + } + } + + private void removeUserFromStarredAttributes(String requestUser, Set starredBy, Set starredDetailsList) { + //Check and update starredBy Attribute + if (starredBy.contains(requestUser)){ + starredBy.remove(requestUser); + } + + for (AtlasStruct starredDetails : starredDetailsList) { + String assetStarredBy = (String) starredDetails.getAttribute(ATTR_ASSET_STARRED_BY); + if (assetStarredBy.equals(requestUser)) { + starredDetailsList.remove(starredDetails); + break; + } + } + } + + private AtlasStruct getStarredDetailsStruct(String assetStarredBy, long assetStarredAt) { + AtlasStruct starredDetails = new AtlasStruct(); + starredDetails.setTypeName(STRUCT_STARRED_DETAILS); + starredDetails.setAttribute(ATTR_ASSET_STARRED_BY, assetStarredBy); + starredDetails.setAttribute(ATTR_ASSET_STARRED_AT, assetStarredAt); + return starredDetails; + } + + public PreProcessor getPreProcessor(String typeName) { + PreProcessor preProcessor = null; + + switch (typeName) { + case ATLAS_GLOSSARY_ENTITY_TYPE: + preProcessor = new GlossaryPreProcessor(typeRegistry, entityRetriever); + break; + + case ATLAS_GLOSSARY_TERM_ENTITY_TYPE: + preProcessor = new TermPreProcessor(typeRegistry, entityRetriever, graph, taskManagement); + break; + + case ATLAS_GLOSSARY_CATEGORY_ENTITY_TYPE: + preProcessor = new CategoryPreProcessor(typeRegistry, entityRetriever, graph, taskManagement, entityGraphMapper); + break; + + case QUERY_ENTITY_TYPE: + preProcessor = new QueryPreProcessor(typeRegistry, entityRetriever); + break; + + case QUERY_FOLDER_ENTITY_TYPE: + preProcessor = new QueryFolderPreProcessor(typeRegistry, entityRetriever); + break; + + case QUERY_COLLECTION_ENTITY_TYPE: + preProcessor = new QueryCollectionPreProcessor(typeRegistry, discovery, entityRetriever, featureFlagStore, this); + break; + + case PERSONA_ENTITY_TYPE: + preProcessor = new PersonaPreProcessor(graph, typeRegistry, entityRetriever, this); + break; + + case PURPOSE_ENTITY_TYPE: + preProcessor = new PurposePreProcessor(graph, typeRegistry, entityRetriever, this); + break; + + case POLICY_ENTITY_TYPE: + preProcessor = new AuthPolicyPreProcessor(graph, typeRegistry, entityRetriever, featureFlagStore); + break; + + case CONNECTION_ENTITY_TYPE: + preProcessor = new ConnectionPreProcessor(graph, discovery, entityRetriever, featureFlagStore, deleteDelegate, this); + break; + + case LINK_ENTITY_TYPE: + preProcessor = new LinkPreProcessor(typeRegistry, entityRetriever); + break; + + case README_ENTITY_TYPE: + preProcessor = new ReadmePreProcessor(typeRegistry, entityRetriever); + break; + } + + return preProcessor; + } + private AtlasVertex getResolvedEntityVertex(EntityGraphDiscoveryContext context, AtlasEntity entity) throws AtlasBaseException { AtlasObjectId objectId = getAtlasObjectId(entity); AtlasVertex ret = context.getResolvedEntityVertex(entity.getGuid()); @@ -1546,8 +1909,10 @@ private AtlasObjectId getAtlasObjectId(AtlasEntity entity) { private EntityMutationResponse deleteVertices(Collection deletionCandidates) throws AtlasBaseException { EntityMutationResponse response = new EntityMutationResponse(); - RequestContext req = RequestContext.get(); + try { + RequestContext req = RequestContext.get(); +<<<<<<< HEAD Collection categories = new ArrayList<>(); Collection other = new ArrayList<>(); @@ -1558,19 +1923,36 @@ private EntityMutationResponse deleteVertices(Collection deletionCa categories.add(vertex); } else { other.add(vertex); +======= + Collection categories = new ArrayList<>(); + Collection others = new ArrayList<>(); + + MetricRecorder metric = RequestContext.get().startMetricRecord("filterCategoryVertices"); + for (AtlasVertex vertex : deletionCandidates) { + String typeName = getTypeName(vertex); + + PreProcessor preProcessor = getPreProcessor(typeName); + if (preProcessor != null) { + preProcessor.processDelete(vertex); + } + + if (ATLAS_GLOSSARY_CATEGORY_ENTITY_TYPE.equals(typeName)) { + categories.add(vertex); + } else { + others.add(vertex); + } +>>>>>>> 7f3c811fb15361ba82bfb4edd7a8393798863ff0 } - } - RequestContext.get().endMetricRecord(metric); + RequestContext.get().endMetricRecord(metric); - if (CollectionUtils.isNotEmpty(categories)) { - entityGraphMapper.removeAttrForCategoryDelete(categories); - deleteDelegate.getHandler(DeleteType.HARD).deleteEntities(categories); - } + if (CollectionUtils.isNotEmpty(categories)) { + entityGraphMapper.removeAttrForCategoryDelete(categories); + deleteDelegate.getHandler(DeleteType.HARD).deleteEntities(categories); + } - if (CollectionUtils.isNotEmpty(other)) { - deleteDelegate.getHandler().deleteEntities(other); - } + if (CollectionUtils.isNotEmpty(others)) { +<<<<<<< HEAD for (AtlasEntityHeader entity : req.getDeletedEntities()) { String handler; if (ATLAS_GLOSSARY_CATEGORY_ENTITY_TYPE.contains(entity.getTypeName())) { @@ -1582,9 +1964,31 @@ private EntityMutationResponse deleteVertices(Collection deletionCa entity.setStatus(Status.DELETED); response.addEntity(DELETE, entity); } +======= + deleteDelegate.getHandler().removeHasLineageOnDelete(others); + deleteDelegate.getHandler().deleteEntities(others); + } +>>>>>>> 7f3c811fb15361ba82bfb4edd7a8393798863ff0 - for (AtlasEntityHeader entity : req.getUpdatedEntities()) { - response.addEntity(UPDATE, entity); + for (AtlasEntityHeader entity : req.getDeletedEntities()) { + String handler; + if (ATLAS_GLOSSARY_CATEGORY_ENTITY_TYPE.equals(entity.getTypeName())) { + handler = req.getDeleteType().equals(DeleteType.PURGE) ? + DeleteType.PURGE.name() : DeleteType.HARD.name(); + } else { + handler = RequestContext.get().getDeleteType().name(); + } + entity.setDeleteHandler(handler); + entity.setStatus(Status.DELETED); + response.addEntity(DELETE, entity); + } + + for (AtlasEntityHeader entity : req.getUpdatedEntities()) { + response.addEntity(UPDATE, entity); + } + } catch (Exception e) { + LOG.error("Delete vertices request failed", e); + throw new AtlasBaseException(e); } return response; @@ -1719,54 +2123,28 @@ private void compactAttributes(AtlasEntity entity, AtlasEntityType entityType) { } } - private void validateBusinessAttributes(AtlasVertex entityVertex, AtlasEntityType entityType, Map> businessAttributes, boolean isOverwrite) throws AtlasBaseException { - List messages = new ArrayList<>(); - - Map> entityTypeBusinessMetadata = entityType.getBusinessAttributes(); - - for (String bmName : businessAttributes.keySet()) { - if (!entityTypeBusinessMetadata.containsKey(bmName)) { - messages.add(bmName + ": invalid business-metadata for entity type " + entityType.getTypeName()); - - continue; - } - - Map entityTypeBusinessAttributes = entityTypeBusinessMetadata.get(bmName); - Map entityBusinessAttributes = businessAttributes.get(bmName); - - for (AtlasBusinessAttribute bmAttribute : entityTypeBusinessAttributes.values()) { - AtlasType attrType = bmAttribute.getAttributeType(); - String attrName = bmAttribute.getName(); - Object attrValue = entityBusinessAttributes.get(attrName); - String fieldName = entityType.getTypeName() + "." + bmName + "." + attrName; - - if (attrValue != null) { - attrType.validateValue(attrValue, fieldName, messages); - boolean isValidLength = bmAttribute.isValidLength(attrValue); - if (!isValidLength) { - messages.add(fieldName + ": Business attribute-value exceeds maximum length limit"); - } - - } else if (!bmAttribute.getAttributeDef().getIsOptional()) { - final boolean isAttrValuePresent; - - if (isOverwrite) { - isAttrValuePresent = false; - } else { - Object existingValue = AtlasGraphUtilsV2.getEncodedProperty(entityVertex, bmAttribute.getVertexPropertyName(), Object.class); - - isAttrValuePresent = existingValue != null; - } - - if (!isAttrValuePresent) { - messages.add(fieldName + ": mandatory business-metadata attribute value missing in type " + entityType.getTypeName()); - } + private void flushAutoUpdateAttributes(AtlasEntity entity, AtlasEntityType entityType) { + if (entityType.getAllAttributes() != null) { + Set flushAttributes = new HashSet<>(); + + for (String attrName : entityType.getAllAttributes().keySet()) { + AtlasAttribute atlasAttribute = entityType.getAttribute(attrName); + HashMap autoUpdateAttributes = atlasAttribute.getAttributeDef().getAutoUpdateAttributes(); + if (MapUtils.isNotEmpty(autoUpdateAttributes)) { + autoUpdateAttributes.values() + .stream() + .flatMap(List::stream) + .forEach(flushAttributes::add); } } - } - if (!messages.isEmpty()) { - throw new AtlasBaseException(AtlasErrorCode.INSTANCE_CRUD_INVALID_PARAMS, messages); +// for (String attrName : entityType.getAllAttributes().keySet()) { +// if (ATTR_STARRED_BY.equals(attrName) || ATTR_STARRED_COUNT.equals(attrName) || ATTR_STARRED_DETAILS_LIST.equals(attrName)) { +// flushAttributes.add(attrName); +// } +// } + + flushAttributes.forEach(entity::removeAttribute); } } @@ -1800,6 +2178,130 @@ public BulkImportResponse bulkCreateOrUpdateBusinessAttributes(InputStream input return ret; } + @Override + public List getAccessors(List atlasAccessorRequestList) throws AtlasBaseException { + List ret = new ArrayList<>(); + + for (AtlasAccessorRequest accessorRequest : atlasAccessorRequestList) { + try { + AtlasAccessorResponse result = null; + AtlasPrivilege action = AtlasPrivilege.valueOf(accessorRequest.getAction());; + + switch (action) { + case ENTITY_READ: + case ENTITY_CREATE: + case ENTITY_UPDATE: + case ENTITY_DELETE: + AtlasEntityAccessRequestBuilder entityAccessRequestBuilder = getEntityAccessRequest(accessorRequest, action); + result = AtlasAuthorizationUtils.getAccessors(entityAccessRequestBuilder.build()); + break; + + case ENTITY_READ_CLASSIFICATION: + case ENTITY_ADD_CLASSIFICATION: + case ENTITY_UPDATE_CLASSIFICATION: + case ENTITY_REMOVE_CLASSIFICATION: + entityAccessRequestBuilder = getEntityAccessRequest(accessorRequest, action); + entityAccessRequestBuilder.setClassification(new AtlasClassification(accessorRequest.getClassification())); + result = AtlasAuthorizationUtils.getAccessors(entityAccessRequestBuilder.build()); + break; + + case ENTITY_ADD_LABEL: + case ENTITY_REMOVE_LABEL: + entityAccessRequestBuilder = getEntityAccessRequest(accessorRequest, action); + entityAccessRequestBuilder.setLabel(accessorRequest.getLabel()); + result = AtlasAuthorizationUtils.getAccessors(entityAccessRequestBuilder.build()); + break; + + case ENTITY_UPDATE_BUSINESS_METADATA: + entityAccessRequestBuilder = getEntityAccessRequest(accessorRequest, action); + entityAccessRequestBuilder.setBusinessMetadata(accessorRequest.getBusinessMetadata()); + result = AtlasAuthorizationUtils.getAccessors(entityAccessRequestBuilder.build()); + break; + + + case RELATIONSHIP_ADD: + case RELATIONSHIP_UPDATE: + case RELATIONSHIP_REMOVE: + AtlasEntityHeader end1EntityHeader = extractEntityHeader(accessorRequest.getEntityGuidEnd1(), accessorRequest.getEntityQualifiedNameEnd1(), accessorRequest.getEntityTypeEnd1()); + AtlasEntityHeader end2EntityHeader = extractEntityHeader(accessorRequest.getEntityGuidEnd2(), accessorRequest.getEntityQualifiedNameEnd2(), accessorRequest.getEntityTypeEnd2()); + + AtlasRelationshipAccessRequest relAccessRequest = new AtlasRelationshipAccessRequest(typeRegistry, + action, accessorRequest.getRelationshipTypeName(), end1EntityHeader, end2EntityHeader); + + result = AtlasAuthorizationUtils.getAccessors(relAccessRequest); + break; + + + case TYPE_READ: + case TYPE_CREATE: + case TYPE_UPDATE: + case TYPE_DELETE: + AtlasBaseTypeDef typeDef = typeRegistry.getTypeDefByName(accessorRequest.getTypeName()); + AtlasTypeAccessRequest typeAccessRequest = new AtlasTypeAccessRequest(action, typeDef); + + result = AtlasAuthorizationUtils.getAccessors(typeAccessRequest); + break; + + + default: + LOG.error("No implementation found for action: {}", accessorRequest.getAction()); + } + + if (result == null) { + throw new AtlasBaseException(); + } + result.populateRequestDetails(accessorRequest); + ret.add(result); + + } catch (AtlasBaseException e) { + e.getErrorDetailsMap().put("accessorRequest", AtlasType.toJson(accessorRequest)); + throw e; + } + } + + return ret; + } + + private AtlasEntityAccessRequestBuilder getEntityAccessRequest(AtlasAccessorRequest element, AtlasPrivilege action) throws AtlasBaseException { + AtlasEntityHeader entityHeader = extractEntityHeader(element.getGuid(), element.getQualifiedName(), element.getTypeName()); + + return new AtlasEntityAccessRequestBuilder(typeRegistry, action, entityHeader); + } + + private AtlasEntityHeader extractEntityHeader(String guid, String qualifiedName, String typeName) throws AtlasBaseException { + AtlasEntityHeader entityHeader = null; + + if (StringUtils.isNotEmpty(guid)) { + entityHeader = entityRetriever.toAtlasEntityHeaderWithClassifications(guid); + + } else { + AtlasEntityType entityType = typeRegistry.getEntityTypeByName(typeName); + if (entityType != null) { + try { + Map uniqueAttrs = new HashMap<>(); + uniqueAttrs.put(QUALIFIED_NAME, qualifiedName); + + AtlasVertex vertex = AtlasGraphUtilsV2.getVertexByUniqueAttributes(this.graph, entityType, uniqueAttrs); + entityHeader = entityRetriever.toAtlasEntityHeaderWithClassifications(vertex); + + } catch (AtlasBaseException abe) { + if (abe.getAtlasErrorCode() != AtlasErrorCode.INSTANCE_BY_UNIQUE_ATTRIBUTE_NOT_FOUND) { + throw abe; + } + + Map attributes = new HashMap<>(); + attributes.put(QUALIFIED_NAME, qualifiedName); + entityHeader = new AtlasEntityHeader(entityType.getTypeName(), attributes); + } + } else { + Map attributes = new HashMap<>(); + attributes.put(QUALIFIED_NAME, qualifiedName); + entityHeader = new AtlasEntityHeader(typeName, attributes); + } + } + return entityHeader; + } + private Map getBusinessMetadataDefList(List fileData, BulkImportResponse bulkImportResponse) throws AtlasBaseException { Map ret = new HashMap<>(); Map vertexCache = new HashMap<>(); @@ -1994,4 +2496,228 @@ public void repairIndex() throws AtlasBaseException { throw new AtlasBaseException(AtlasErrorCode.REPAIR_INDEX_FAILED, exception.toString()); } } + + + @Override + @GraphTransaction + public void repairHasLineage(AtlasHasLineageRequests requests) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("repairHasLineage"); + + Set inputOutputEdges = new HashSet<>(); + + for (AtlasHasLineageRequest request : requests.getRequest()) { + if (StringUtils.isNotEmpty(request.getAssetGuid())) { + //only supports repairing scenario mentioned here - https://atlanhq.atlassian.net/browse/DG-128?focusedCommentId=20652 + repairHasLineageForAsset(request); + + } else { + AtlasVertex processVertex = AtlasGraphUtilsV2.findByGuid(this.graph, request.getProcessGuid()); + AtlasVertex assetVertex = AtlasGraphUtilsV2.findByGuid(this.graph, request.getEndGuid()); + AtlasEdge edge = null; + try { + if (processVertex != null && assetVertex != null) { + edge = graphHelper.getEdge(processVertex, assetVertex, request.getLabel()); + } else { + LOG.warn("Skipping since vertex is null for processGuid {} and asset Guid {}" + ,request.getProcessGuid(),request.getEndGuid() ); + } + } catch (RepositoryException re) { + throw new AtlasBaseException(AtlasErrorCode.HAS_LINEAGE_GET_EDGE_FAILED, re); + } + + if (edge != null) { + inputOutputEdges.add(edge); + } + } + } + + if (CollectionUtils.isNotEmpty(inputOutputEdges)) { + repairHasLineageWithAtlasEdges(inputOutputEdges); + } + + RequestContext.get().endMetricRecord(metricRecorder); + } + + private void repairHasLineageForAsset(AtlasHasLineageRequest request) { + //only supports repairing scenario mentioned here - https://atlanhq.atlassian.net/browse/DG-128?focusedCommentId=20652 + + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("repairHasLineageForAssetGetById"); + AtlasVertex assetVertex = AtlasGraphUtilsV2.findByGuid(this.graph, request.getAssetGuid()); + RequestContext.get().endMetricRecord(metricRecorder); + + if (getEntityHasLineage(assetVertex)) { + metricRecorder = RequestContext.get().startMetricRecord("repairHasLineageForAssetGetRelations"); + Iterator lineageEdges = assetVertex.getEdges(AtlasEdgeDirection.BOTH, PROCESS_EDGE_LABELS).iterator(); + RequestContext.get().endMetricRecord(metricRecorder); + boolean foundActiveRel = false; + + while (lineageEdges.hasNext()) { + AtlasEdge edge = lineageEdges.next(); + if (getStatus(edge) == ACTIVE) { + foundActiveRel = true; + break; + } + } + + if (!foundActiveRel) { + metricRecorder = RequestContext.get().startMetricRecord("repairHasLineageForRequiredAsset"); + AtlasGraphUtilsV2.setEncodedProperty(assetVertex, HAS_LINEAGE, false); + LOG.info("repairHasLineage: repairHasLineageForAsset: Repaired {}", request.getAssetGuid()); + RequestContext.get().endMetricRecord(metricRecorder); + } + } + } + + public void repairHasLineageWithAtlasEdges(Set inputOutputEdges) { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("repairHasLineageWithAtlasEdges"); + + for (AtlasEdge atlasEdge : inputOutputEdges) { + + if (getStatus(atlasEdge) != ACTIVE) { + LOG.warn("Edge id {} is not Active, so skipping " , getRelationshipGuid(atlasEdge)); + continue; + } + + boolean isOutputEdge = PROCESS_OUTPUTS.equals(atlasEdge.getLabel()); + + AtlasVertex processVertex = atlasEdge.getOutVertex(); + AtlasVertex assetVertex = atlasEdge.getInVertex(); + + if (getEntityHasLineageValid(processVertex) && getEntityHasLineage(processVertex)) { + AtlasGraphUtilsV2.setEncodedProperty(assetVertex, HAS_LINEAGE, true); + AtlasGraphUtilsV2.setEncodedProperty(assetVertex, HAS_LINEAGE_VALID, true); + continue; + } + + String oppositeEdgeLabel = isOutputEdge ? PROCESS_INPUTS : PROCESS_OUTPUTS; + + Iterator oppositeEdges = processVertex.getEdges(AtlasEdgeDirection.BOTH, oppositeEdgeLabel).iterator(); + boolean isHasLineageSet = false; + while (oppositeEdges.hasNext()) { + AtlasEdge oppositeEdge = oppositeEdges.next(); + AtlasVertex oppositeEdgeAssetVertex = oppositeEdge.getInVertex(); + + if (getStatus(oppositeEdge) == ACTIVE && getStatus(oppositeEdgeAssetVertex) == ACTIVE) { + if (!isHasLineageSet) { + AtlasGraphUtilsV2.setEncodedProperty(assetVertex, HAS_LINEAGE, true); + AtlasGraphUtilsV2.setEncodedProperty(processVertex, HAS_LINEAGE, true); + + AtlasGraphUtilsV2.setEncodedProperty(assetVertex, HAS_LINEAGE_VALID, true); + AtlasGraphUtilsV2.setEncodedProperty(processVertex, HAS_LINEAGE_VALID, true); + + isHasLineageSet = true; + } + break; + } + } + + if (!isHasLineageSet) { + AtlasGraphUtilsV2.setEncodedProperty(assetVertex, HAS_LINEAGE, false); + AtlasGraphUtilsV2.setEncodedProperty(processVertex, HAS_LINEAGE, false); + AtlasGraphUtilsV2.setEncodedProperty(assetVertex, HAS_LINEAGE_VALID, true); + AtlasGraphUtilsV2.setEncodedProperty(processVertex, HAS_LINEAGE_VALID, true); + } + + } + RequestContext.get().endMetricRecord(metricRecorder); + } + + private void recordRelationshipsToBePurged(AtlasVertex instanceVertex) throws AtlasBaseException { + Iterable incomingEdges = instanceVertex.getEdges(AtlasEdgeDirection.IN); + Iterable outgoingEdges = instanceVertex.getEdges(AtlasEdgeDirection.OUT); + + recordInComingEdgesToBeDeleted(incomingEdges); + recordOutGoingEdgesToBeDeleted(outgoingEdges); + } + + private void recordInComingEdgesToBeDeleted(Iterable incomingEdges) throws AtlasBaseException { + for (AtlasEdge edge : incomingEdges) { + if (isRelationshipEdge(edge)) + AtlasRelationshipStoreV2.recordRelationshipMutation(AtlasRelationshipStoreV2.RelationshipMutation.RELATIONSHIP_HARD_DELETE, edge, entityRetriever); + } + } + + private void recordOutGoingEdgesToBeDeleted(Iterable outgoingEdges) throws AtlasBaseException { + for (AtlasEdge edge : outgoingEdges) { + if (isRelationshipEdge(edge)) + AtlasRelationshipStoreV2.recordRelationshipMutation(AtlasRelationshipStoreV2.RelationshipMutation.RELATIONSHIP_HARD_DELETE, edge, entityRetriever); + } + } + + @Override + @GraphTransaction + public void repairMeaningAttributeForTerms(List termGuid) { + + for (String guid : termGuid) { + LOG.info(" term guid " + guid); + + AtlasVertex termVertex = AtlasGraphUtilsV2.findByGuid(this.graph, guid); + + if(termVertex!= null && ATLAS_GLOSSARY_TERM_ENTITY_TYPE.equals(getTypeName(termVertex)) && + GraphHelper.getStatus(termVertex) == AtlasEntity.Status.ACTIVE) { + Iterable edges = termVertex.getEdges(AtlasEdgeDirection.OUT, Constants.TERM_ASSIGNMENT_LABEL); + // Get entity to tagged with term. + if (edges != null) { + for (Iterator iter = edges.iterator(); iter.hasNext(); ) { + AtlasEdge edge = iter.next(); + if (GraphHelper.getStatus(edge) == AtlasEntity.Status.ACTIVE) { + AtlasVertex entityVertex = edge.getInVertex(); + if (entityVertex != null & getStatus(entityVertex) == AtlasEntity.Status.ACTIVE) { + if(!RequestContext.get().getProcessGuidIds().contains(getGuid(entityVertex))) { + repairMeanings(entityVertex); + } + } + } + } + } + } + } + } + + private void repairMeanings(AtlasVertex assetVertex) { + + Iterable edges = assetVertex.getEdges(AtlasEdgeDirection.IN, Constants.TERM_ASSIGNMENT_LABEL); + List termQNList = new ArrayList<>(); + List termNameList = new ArrayList<>(); + if (edges != null) { + for (Iterator iter = edges.iterator(); iter.hasNext(); ) { + AtlasEdge edge = iter.next(); + if (GraphHelper.getStatus(edge) == AtlasEntity.Status.ACTIVE) { + AtlasVertex termVertex = edge.getOutVertex(); + if (termVertex != null & getStatus(termVertex) == AtlasEntity.Status.ACTIVE) { + String termQN = termVertex.getProperty(QUALIFIED_NAME, String.class); + String termName = termVertex.getProperty(NAME, String.class); + termQNList.add(termQN); + termNameList.add(termName); + } + } + } + } + + if (termQNList.size() > 0) { + + assetVertex.removeProperty(MEANINGS_PROPERTY_KEY); + assetVertex.removeProperty(MEANINGS_TEXT_PROPERTY_KEY); + assetVertex.removeProperty(MEANING_NAMES_PROPERTY_KEY); + + if (CollectionUtils.isNotEmpty(termQNList)) { + termQNList.forEach(q -> AtlasGraphUtilsV2.addEncodedProperty(assetVertex, MEANINGS_PROPERTY_KEY, q)); + } + + if (CollectionUtils.isNotEmpty(termNameList)) { + AtlasGraphUtilsV2.setEncodedProperty(assetVertex, MEANINGS_TEXT_PROPERTY_KEY, StringUtils.join(termNameList, ",")); + } + + if (CollectionUtils.isNotEmpty(termNameList)) { + termNameList.forEach(q -> AtlasGraphUtilsV2.addListProperty(assetVertex, MEANING_NAMES_PROPERTY_KEY, q, true)); + } + + RequestContext.get().addProcessGuidIds(getGuid(assetVertex)); + + LOG.info("Updated asset {} with term {} ", getGuid(assetVertex) , StringUtils.join(termNameList, ",")); + } + + } } + + diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasGraphUtilsV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasGraphUtilsV2.java index 93764df2ce6..9ba794c44e7 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasGraphUtilsV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasGraphUtilsV2.java @@ -48,15 +48,7 @@ import org.slf4j.LoggerFactory; import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import static org.apache.atlas.repository.Constants.ATLAS_GLOSSARY_ENTITY_TYPE; import static org.apache.atlas.repository.Constants.CLASSIFICATION_NAMES_KEY; @@ -236,6 +228,19 @@ public static AtlasVertex addListProperty(AtlasVertex vertex, String propertyNam return vertex; } + public static AtlasVertex deleteProperty(AtlasVertex vertex, String encodedPropertyName, String propertyValue) { + Collection existingValues = vertex.getPropertyValues(encodedPropertyName, Object.class); + + if (! existingValues.isEmpty() && existingValues.contains(propertyValue)) { + if (LOG.isDebugEnabled()) { + LOG.debug("Removing value {} from property {} of entity {}", propertyValue, encodedPropertyName, toString(vertex)); + } + vertex.removePropertyValue(encodedPropertyName, propertyValue); + } + + return vertex; + } + public static void setProperty(T element, String propertyName, Object value) { setProperty(element, propertyName, value, false); } @@ -390,6 +395,11 @@ public static AtlasVertex findByGuid(String guid) { public static AtlasVertex findByGuid(AtlasGraph graph, String guid) { AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("findByGuid"); + if (guid == null) { + LOG.warn("findByGuid -> called for null guid!"); + return null; + } + AtlasVertex ret = GraphTransactionInterceptor.getVertexFromCache(guid); if (ret == null) { diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java index 9b896f71356..61877fb89e3 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java @@ -31,6 +31,7 @@ import org.apache.atlas.model.instance.AtlasObjectId; import org.apache.atlas.model.instance.AtlasRelationship; import org.apache.atlas.model.instance.AtlasRelationship.AtlasRelationshipWithExtInfo; +import org.apache.atlas.model.instance.RelationshipMutationContext; import org.apache.atlas.model.notification.EntityNotification.EntityNotificationV2.OperationType; import org.apache.atlas.model.typedef.AtlasRelationshipDef; import org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags; @@ -42,15 +43,16 @@ import org.apache.atlas.repository.graphdb.AtlasEdgeDirection; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.repository.graphdb.janus.JanusUtils; import org.apache.atlas.repository.store.graph.AtlasRelationshipStore; import org.apache.atlas.repository.store.graph.v1.DeleteHandlerDelegate; +import org.apache.atlas.repository.store.graph.v1.SoftDeleteHandlerV1; import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasRelationshipType; import org.apache.atlas.type.AtlasStructType.AtlasAttribute; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.type.AtlasTypeUtil; import org.apache.atlas.utils.AtlasPerfMetrics; -import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; @@ -58,15 +60,7 @@ import org.springframework.stereotype.Component; import javax.inject.Inject; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.UUID; +import java.util.*; import static org.apache.atlas.AtlasConfiguration.NOTIFICATION_RELATIONSHIPS_ENABLED; import static org.apache.atlas.model.instance.AtlasEntity.Status.DELETED; @@ -80,8 +74,7 @@ import static org.apache.atlas.repository.Constants.RELATIONSHIPTYPE_TAG_PROPAGATION_KEY; import static org.apache.atlas.repository.Constants.RELATIONSHIP_GUID_PROPERTY_KEY; import static org.apache.atlas.repository.Constants.VERSION_PROPERTY_KEY; -import static org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.getState; -import static org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.getTypeName; +import static org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.*; import static org.apache.atlas.repository.store.graph.v2.tasks.ClassificationPropagateTaskFactory.CLASSIFICATION_PROPAGATION_RELATIONSHIP_UPDATE; @Component @@ -99,6 +92,26 @@ public class AtlasRelationshipStoreV2 implements AtlasRelationshipStore { private final GraphHelper graphHelper; private final IAtlasEntityChangeNotifier entityChangeNotifier; + private static final String RELATIONSHIP_DEF_MAP_KEY = "relationshipDef"; + private static final String END_1_CARDINALITY_KEY = "end1Cardinality"; + private static final String END_1_NAME_KEY = "end1Name"; + private static final String IS_END_1_CONTAINER_KEY = "IsEnd1Container"; + private static final String END_2_CARDINALITY_KEY = "end2Cardinality"; + private static final String END_2_NAME_KEY = "end2Name"; + private static final String IS_END_2_CONTAINER_KEY = "IsEnd2Container"; + + private static final String END_1_DOC_ID_KEY = "end1DocId"; + private static final String END_2_DOC_ID_KEY = "end2DocId"; + private static final String ES_DOC_ID_MAP_KEY = "esDocIdMap"; + + public enum RelationshipMutation { + RELATIONSHIP_CREATE, + RELATIONSHIP_UPDATE, + RELATIONSHIP_RESTORE, + RELATIONSHIP_SOFT_DELETE, + RELATIONSHIP_HARD_DELETE + } + @Inject public AtlasRelationshipStoreV2(AtlasGraph graph, AtlasTypeRegistry typeRegistry, DeleteHandlerDelegate deleteDelegate, IAtlasEntityChangeNotifier entityChangeNotifier) { this.graph = graph; @@ -122,11 +135,11 @@ public AtlasRelationship create(AtlasRelationship relationship) throws AtlasBase AtlasEdge edge = createRelationship(end1Vertex, end2Vertex, relationship); AtlasRelationship ret = edge != null ? entityRetriever.mapEdgeToAtlasRelationship(edge) : null; - + recordRelationshipMutation(RelationshipMutation.RELATIONSHIP_CREATE, edge, entityRetriever); + onRelationshipsMutated(RequestContext.get().getRelationshipMutationMap()); if (LOG.isDebugEnabled()) { LOG.debug("<== create({}): {}", relationship, ret); } - return ret; } @@ -203,12 +216,11 @@ public AtlasRelationship update(AtlasRelationship relationship) throws AtlasBase } AtlasRelationship ret = updateRelationship(edge, relationship); - sendNotifications(ret, OperationType.RELATIONSHIP_UPDATE); - + recordRelationshipMutation(RelationshipMutation.RELATIONSHIP_UPDATE, edge, entityRetriever); + onRelationshipsMutated(RequestContext.get().getRelationshipMutationMap()); if (LOG.isDebugEnabled()) { LOG.debug("<== update({}): {}", relationship, ret); } - return ret; } @@ -283,7 +295,7 @@ public void deleteByIds(List guids) throws AtlasBaseException { LOG.debug("==> deleteByIds({}})", guids.size()); } - List deletedRelationships = new ArrayList<>(); + Set deletedRelationships = new HashSet<>(); List edgesToDelete = new ArrayList<>(); for (String guid : guids) { @@ -297,19 +309,23 @@ public void deleteByIds(List guids) throws AtlasBaseException { throw new AtlasBaseException(AtlasErrorCode.RELATIONSHIP_ALREADY_DELETED, guid); } - String relationShipType = graphHelper.getTypeName(edge); - AtlasEntityHeader end1Entity = entityRetriever.toAtlasEntityHeaderWithClassifications(edge.getOutVertex()); - AtlasEntityHeader end2Entity = entityRetriever.toAtlasEntityHeaderWithClassifications(edge.getInVertex()); - - AtlasAuthorizationUtils.verifyAccess(new AtlasRelationshipAccessRequest(typeRegistry,AtlasPrivilege.RELATIONSHIP_REMOVE, relationShipType, end1Entity, end2Entity )); - edgesToDelete.add(edge); - deletedRelationships.add(entityRetriever.mapEdgeToAtlasRelationship(edge)); + AtlasRelationship relationshipToDelete = entityRetriever.mapEdgeToAtlasRelationship(edge); + deletedRelationships.add(relationshipToDelete); } + deleteDelegate.getHandler().resetHasLineageOnInputOutputDelete(edgesToDelete,null); deleteDelegate.getHandler().deleteRelationships(edgesToDelete, false); - sendNotifications(deletedRelationships, OperationType.RELATIONSHIP_DELETE); + if (DEFERRED_ACTION_ENABLED) { + Set deletedEdgeIds = RequestContext.get().getDeletedEdgesIds(); + for (String deletedEdgeId : deletedEdgeIds) { + AtlasEdge deletedEdge = graph.getEdge(deletedEdgeId); + deleteDelegate.getHandler().createAndQueueClassificationRefreshPropagationTask(deletedEdge); + } + } + + onRelationshipsMutated(RequestContext.get().getRelationshipMutationMap()); if (LOG.isDebugEnabled()) { LOG.debug("<== deleteByIds({}):", guids.size()); @@ -334,8 +350,6 @@ public void deleteById(String guid, boolean forceDelete) throws AtlasBaseExcepti } AtlasEdge edge = graphHelper.getEdgeForGUID(guid); - - if (edge == null) { throw new AtlasBaseException(AtlasErrorCode.RELATIONSHIP_GUID_NOT_FOUND, guid); } @@ -343,38 +357,36 @@ public void deleteById(String guid, boolean forceDelete) throws AtlasBaseExcepti if (getState(edge) == DELETED) { throw new AtlasBaseException(AtlasErrorCode.RELATIONSHIP_ALREADY_DELETED, guid); } - - String relationShipType = graphHelper.getTypeName(edge); - AtlasEntityHeader end1Entity = entityRetriever.toAtlasEntityHeaderWithClassifications(edge.getOutVertex()); - AtlasEntityHeader end2Entity = entityRetriever.toAtlasEntityHeaderWithClassifications(edge.getInVertex()); - - AtlasAuthorizationUtils.verifyAccess(new AtlasRelationshipAccessRequest(typeRegistry,AtlasPrivilege.RELATIONSHIP_REMOVE, relationShipType, end1Entity, end2Entity )); - - + deleteDelegate.getHandler().resetHasLineageOnInputOutputDelete(Collections.singleton(edge), null); deleteDelegate.getHandler().deleteRelationships(Collections.singleton(edge), forceDelete); - sendNotifications(entityRetriever.mapEdgeToAtlasRelationship(edge), OperationType.RELATIONSHIP_DELETE); + if (DEFERRED_ACTION_ENABLED) { + Set deletedEdgeIds = RequestContext.get().getDeletedEdgesIds(); + for (String deletedEdgeId : deletedEdgeIds) { + AtlasEdge deletedEdge = graph.getEdge(deletedEdgeId); + deleteDelegate.getHandler().createAndQueueClassificationRefreshPropagationTask(deletedEdge); + } + } + onRelationshipsMutated(RequestContext.get().getRelationshipMutationMap()); if (LOG.isDebugEnabled()) { LOG.debug("<== deleteById({}): {}", guid); } } - - @Override public AtlasEdge getOrCreate(AtlasVertex end1Vertex, AtlasVertex end2Vertex, AtlasRelationship relationship) throws AtlasBaseException { AtlasEdge ret = getRelationship(end1Vertex, end2Vertex, relationship); if (ret == null) { ret = createRelationship(end1Vertex, end2Vertex, relationship, false); + recordRelationshipMutation(RelationshipMutation.RELATIONSHIP_CREATE, ret, entityRetriever); } return ret; } - @Override - public AtlasEdge getRelationship(AtlasVertex fromVertex, AtlasVertex toVertex, AtlasRelationship relationship) throws AtlasBaseException { + private AtlasEdge getRelationship(AtlasVertex fromVertex, AtlasVertex toVertex, AtlasRelationship relationship) throws AtlasBaseException { String relationshipLabel = getRelationshipEdgeLabel(fromVertex, toVertex, relationship.getTypeName()); return getRelationshipEdge(fromVertex, toVertex, relationshipLabel); @@ -392,28 +404,32 @@ public AtlasRelationship getOrCreate(AtlasRelationship relationship) throws Atla // check if relationship exists AtlasEdge relationshipEdge = getRelationship(end1Vertex, end2Vertex, relationship); - + boolean isRelationshipCreated = false; if (relationshipEdge == null) { relationshipEdge = createRelationship(end1Vertex, end2Vertex, relationship, false); + isRelationshipCreated = true; } if (relationshipEdge != null){ ret = entityRetriever.mapEdgeToAtlasRelationship(relationshipEdge); } + if (isRelationshipCreated) { + recordRelationshipMutation(RelationshipMutation.RELATIONSHIP_CREATE, relationshipEdge, entityRetriever); + onRelationshipsMutated(RequestContext.get().getRelationshipMutationMap()); + } + if (LOG.isDebugEnabled()) { LOG.debug("<== getOrCreate({}): {}", relationship, ret); } - return ret; } - @Override - public AtlasEdge createRelationship(AtlasVertex end1Vertex, AtlasVertex end2Vertex, AtlasRelationship relationship) throws AtlasBaseException { + private AtlasEdge createRelationship(AtlasVertex end1Vertex, AtlasVertex end2Vertex, AtlasRelationship relationship) throws AtlasBaseException { return createRelationship(end1Vertex, end2Vertex, relationship, true); } - public AtlasEdge createRelationship(AtlasVertex end1Vertex, AtlasVertex end2Vertex, AtlasRelationship relationship, boolean existingRelationshipCheck) throws AtlasBaseException { + private AtlasEdge createRelationship(AtlasVertex end1Vertex, AtlasVertex end2Vertex, AtlasRelationship relationship, boolean existingRelationshipCheck) throws AtlasBaseException { AtlasEdge ret; try { @@ -432,20 +448,18 @@ public AtlasEdge createRelationship(AtlasVertex end1Vertex, AtlasVertex end2Vert AtlasRelationshipType relationType = typeRegistry.getRelationshipTypeByName(relationship.getTypeName()); - if (!relationType.hasLegacyAttributeEnd()) { // skip authorization for legacy attributes, as these would be covered as entity-update - AtlasEntityHeader end1Entity = entityRetriever.toAtlasEntityHeaderWithClassifications(end1Vertex); - AtlasEntityHeader end2Entity = entityRetriever.toAtlasEntityHeaderWithClassifications(end2Vertex); - AtlasAuthorizationUtils.verifyAccess(new AtlasRelationshipAccessRequest(typeRegistry, AtlasPrivilege.RELATIONSHIP_ADD, + AtlasEntityHeader end1Entity = entityRetriever.toAtlasEntityHeaderWithClassifications(end1Vertex); + AtlasEntityHeader end2Entity = entityRetriever.toAtlasEntityHeaderWithClassifications(end2Vertex); + + AtlasAuthorizationUtils.verifyAccess(new AtlasRelationshipAccessRequest(typeRegistry, AtlasPrivilege.RELATIONSHIP_ADD, relationship.getTypeName(), end1Entity, end2Entity)); - } + if (existingRelationshipCheck) { ret = graphHelper.getOrCreateEdge(end1Vertex, end2Vertex, relationshipLabel); - } else { ret = graphHelper.addEdge(end1Vertex, end2Vertex, relationshipLabel); - if (LOG.isDebugEnabled()) { LOG.debug("Created relationship edge from [{}] --> [{}] using edge label: [{}]", getTypeName(end1Vertex), getTypeName(end2Vertex), relationshipLabel); } @@ -484,8 +498,6 @@ public AtlasEdge createRelationship(AtlasVertex end1Vertex, AtlasVertex end2Vert } catch (RepositoryException e) { throw new AtlasBaseException(AtlasErrorCode.INTERNAL_ERROR, e); } - - sendNotifications(entityRetriever.mapEdgeToAtlasRelationship(ret), OperationType.RELATIONSHIP_CREATE); return ret; } @@ -733,7 +745,7 @@ private Long getRelationshipVersion(AtlasRelationship relationship) { return (ret != null) ? ret : DEFAULT_RELATIONSHIP_VERSION; } - private AtlasVertex getVertexFromEndPoint(AtlasObjectId endPoint) { + public AtlasVertex getVertexFromEndPoint(AtlasObjectId endPoint) { AtlasVertex ret = null; if (StringUtils.isNotEmpty(endPoint.getGuid())) { @@ -864,18 +876,115 @@ private String getRelationshipEdgeLabel(String typeName, String relationshipType return (attribute != null) ? attribute.getRelationshipEdgeLabel() : null; } - private void sendNotifications(AtlasRelationship ret, OperationType relationshipUpdate) throws AtlasBaseException { - sendNotifications(Collections.singletonList(ret), relationshipUpdate); + public void onRelationshipsMutated(Map> relationshipsMutationMap) throws AtlasBaseException { + entityChangeNotifier.notifyPropagatedEntities(); + RelationshipMutationContext relationshipMutationContext = getRelationshipMutationContext(relationshipsMutationMap); + if (notificationsEnabled) { + entityChangeNotifier.notifyRelationshipMutation(relationshipMutationContext.getCreatedRelationships(), OperationType.RELATIONSHIP_CREATE); + entityChangeNotifier.notifyRelationshipMutation(relationshipMutationContext.getUpdatedRelationships(), OperationType.RELATIONSHIP_UPDATE); + entityChangeNotifier.notifyRelationshipMutation(relationshipMutationContext.getDeletedRelationships(), OperationType.RELATIONSHIP_DELETE); + } + } + + private RelationshipMutationContext getRelationshipMutationContext(Map> relationshipsMutationMap) { + final List createdRelationships = new ArrayList<>(); + final List deletedRelationships = new ArrayList<>(); + final List updatedRelationships = new ArrayList<>(); + + relationshipsMutationMap.keySet().forEach((relationshipMutation) -> { + final Set relationships = relationshipsMutationMap.getOrDefault(relationshipMutation, new HashSet<>()); + this.addRelationshipMetadataForNotificationEvent(relationships); + if (RelationshipMutation.RELATIONSHIP_CREATE.name().equals(relationshipMutation)) { + createdRelationships.addAll(relationships); + } else if (RelationshipMutation.RELATIONSHIP_SOFT_DELETE.name().equals(relationshipMutation) || RelationshipMutation.RELATIONSHIP_HARD_DELETE.name().equals(relationshipMutation)) { + deletedRelationships.addAll(relationships); + } else if (RelationshipMutation.RELATIONSHIP_UPDATE.name().equals(relationshipMutation) || RelationshipMutation.RELATIONSHIP_RESTORE.name().equals(relationshipMutation)) { + updatedRelationships.addAll(relationships); + } + }); + return RelationshipMutationContext.getInstance(createdRelationships, updatedRelationships, deletedRelationships); } - private void sendNotifications(List ret, OperationType relationshipUpdate) throws AtlasBaseException { - entityChangeNotifier.notifyPropagatedEntities(); - if (notificationsEnabled){ - entityChangeNotifier.notifyRelationshipMutation(ret, relationshipUpdate); + public static void recordRelationshipMutation(RelationshipMutation relationshipMutation, AtlasEdge edge, EntityGraphRetriever entityRetriever) throws AtlasBaseException { + if (Objects.isNull(edge)) + throw new IllegalStateException("edge cannot be null"); + final AtlasRelationship relationship = entityRetriever.mapEdgeToAtlasRelationship(edge); + if (relationshipMutation.equals(RelationshipMutation.RELATIONSHIP_HARD_DELETE)) + relationship.setStatus(AtlasRelationship.Status.PURGED); + AtlasRelationshipStoreV2.setEdgeVertexIdsInContext(edge); + RequestContext.get().saveRelationshipsMutationContext(relationshipMutation.name(), relationship); + } + + private void addRelationshipMetadataForNotificationEvent(Set relationships) { + for (AtlasRelationship r : relationships) { + final Map relationshipDef = buildRelationshipDefMap(r); + r.setRelationshipDef(relationshipDef); + final Map relationshipEndToESDocIdMap = builsESDocIdMapping(r); + r.setRelationshipEndToESDocIdMap(relationshipEndToESDocIdMap); } } + private static Map builsESDocIdMapping(AtlasRelationship r) { + + final Map esDocIdMapping = new HashMap<>(); + + if(r == null || r.getEnd1() == null || r.getEnd2() == null) { + LOG.warn(" AtlasRelationship is null {} " , r); + return esDocIdMapping; + } + + final Map relationshipEndToVertexIdMap = RequestContext.get().getRelationshipEndToVertexIdMap(); + String end1DocId = ""; + String end2DocId = ""; + + for (AtlasObjectId atlasObjectId : relationshipEndToVertexIdMap.keySet()) { + if(atlasObjectId == null) { + LOG.warn(" atlasObjectId is null."); + return esDocIdMapping; + } + + final String docId = JanusUtils.toLongEncoding(relationshipEndToVertexIdMap.get(atlasObjectId)); + String guid = atlasObjectId.getGuid(); + AtlasObjectId end1 = r.getEnd1(); + AtlasObjectId end2 = r.getEnd2(); + + if(guid == null) { + LOG.warn(" atlasObjectId.getGuid() is null. atlasObjectId {}" , atlasObjectId); + return esDocIdMapping; + } + + if (guid.equals(end1.getGuid())) { + end1DocId = docId; + } else if (guid.equals(end2.getGuid())) { + end2DocId = docId; + } + + } + + esDocIdMapping.put(END_1_DOC_ID_KEY, end1DocId); + esDocIdMapping.put(END_2_DOC_ID_KEY, end2DocId); + return esDocIdMapping; + } + + private Map buildRelationshipDefMap(AtlasRelationship relationship) { + final AtlasRelationshipDef relationshipDef = typeRegistry.getRelationshipDefByName(relationship.getTypeName()); + Map relationshipDefMap = new HashMap<>(); + relationshipDefMap.put(END_1_CARDINALITY_KEY, relationshipDef.getEndDef1().getCardinality().toString()); + relationshipDefMap.put(END_1_NAME_KEY, relationshipDef.getEndDef1().getName()); + relationshipDefMap.put(IS_END_1_CONTAINER_KEY, relationshipDef.getEndDef1().getIsContainer()); + + relationshipDefMap.put(END_2_CARDINALITY_KEY, relationshipDef.getEndDef2().getCardinality().toString()); + relationshipDefMap.put(END_2_NAME_KEY, relationshipDef.getEndDef2().getName()); + relationshipDefMap.put(IS_END_2_CONTAINER_KEY, relationshipDef.getEndDef2().getIsContainer()); + return relationshipDefMap; + } + private void createAndQueueTask(String taskType, AtlasEdge relationshipEdge, AtlasRelationship relationship) { deleteDelegate.getHandler().createAndQueueTask(taskType, relationshipEdge, relationship); } -} + + private static void setEdgeVertexIdsInContext(AtlasEdge edge) { + RequestContext.get().addRelationshipEndToVertexIdMapping(GraphHelper.getAtlasObjectIdForOutVertex(edge), edge.getOutVertex().getId()); + RequestContext.get().addRelationshipEndToVertexIdMapping(GraphHelper.getAtlasObjectIdForInVertex(edge), edge.getInVertex().getId()); + } +} \ No newline at end of file diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasStructDefStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasStructDefStoreV2.java index ab9230e7bf1..3a0f6422466 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasStructDefStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasStructDefStoreV2.java @@ -556,7 +556,7 @@ public static String toJsonFromAttribute(AtlasAttribute attribute) { attribInfo.put("indexTypeESFields", attributeDef.getIndexTypeESFields()); attribInfo.put("autoUpdateAttributes", attributeDef.getAutoUpdateAttributes()); attribInfo.put("skipScrubbing", attributeDef.getSkipScrubbing()); - + attribInfo.put("isDefaultValueNull", attributeDef.getIsDefaultValueNull()); if(attributeDef.getOptions() != null) { attribInfo.put("options", AtlasType.toJson(attributeDef.getOptions())); } @@ -603,7 +603,7 @@ public static AtlasAttributeDef toAttributeDefFromJson(AtlasStructDef ret.setIncludeInNotification((Boolean) attribInfo.get("includeInNotification")); ret.setDefaultValue((String) attribInfo.get("defaultValue")); ret.setDescription((String) attribInfo.get("description")); - + if(attribInfo.get("isDefaultValueNull")!=null) ret.setIsDefaultValueNull((Boolean) attribInfo.get("isDefaultValueNull")); if(attribInfo.get("skipScrubbing")!=null) { ret.setSkipScrubbing((Boolean) attribInfo.get("skipScrubbing")); } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasTypeDefGraphStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasTypeDefGraphStoreV2.java index 1711edf81f9..810e2a07570 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasTypeDefGraphStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasTypeDefGraphStoreV2.java @@ -241,7 +241,7 @@ void updateTypeVertex(AtlasBaseTypeDef typeDef, AtlasVertex vertex) { * updateVertexProperty(vertex, Constants.TYPENAME_PROPERTY_KEY, typeDef.getName()); */ - updateVertexProperty(vertex, Constants.TYPEDESCRIPTION_PROPERTY_KEY, typeDef.getDescription()); + updateVertexPropertyAllowNull(vertex, Constants.TYPEDESCRIPTION_PROPERTY_KEY, typeDef.getDescription()); updateVertexProperty(vertex, Constants.TYPEVERSION_PROPERTY_KEY, typeDef.getTypeVersion()); updateVertexProperty(vertex, Constants.TYPEOPTIONS_PROPERTY_KEY, AtlasType.toJson(typeDef.getOptions())); if (typeDef instanceof AtlasNamedTypeDef) { @@ -549,6 +549,17 @@ private void updateVertexProperty(AtlasVertex vertex, String propertyName, Strin } } + /* + * update the given vertex property, if the new value is also null + */ + private void updateVertexPropertyAllowNull(AtlasVertex vertex, String propertyName, String newValue) { + String currValue = vertex.getProperty(propertyName, String.class); + + if (!StringUtils.equals(currValue, newValue)) { + vertex.setProperty(propertyName, newValue); + } + } + /* * update the given vertex property, if the new value is not-null */ diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/BulkImporterImpl.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/BulkImporterImpl.java index 8e17fd41086..9adbd7d1dce 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/BulkImporterImpl.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/BulkImporterImpl.java @@ -53,13 +53,16 @@ public class BulkImporterImpl implements BulkImporter { private final AtlasEntityStore entityStore; private final AtlasGraph atlasGraph; - private AtlasTypeRegistry typeRegistry; + private final AtlasTypeRegistry typeRegistry; + private final AtlasEntityChangeNotifier entityChangeNotifier; @Inject - public BulkImporterImpl(AtlasGraph atlasGraph, AtlasEntityStore entityStore, AtlasTypeRegistry typeRegistry) { + public BulkImporterImpl(AtlasGraph atlasGraph, AtlasEntityStore entityStore, + AtlasTypeRegistry typeRegistry, AtlasEntityChangeNotifier entityChangeNotifier) { this.atlasGraph = atlasGraph; this.entityStore = entityStore; this.typeRegistry = typeRegistry; + this.entityChangeNotifier = entityChangeNotifier; } @Override @@ -68,7 +71,7 @@ public EntityMutationResponse bulkImport(EntityImportStream entityStream, AtlasI if (importResult.getRequest().getOptions() != null && importResult.getRequest().getOptions().containsKey(AtlasImportRequest.OPTION_KEY_MIGRATION)) { - importStrategy = new MigrationImport(this.atlasGraph, new AtlasGraphProvider(), this.typeRegistry); + importStrategy = new MigrationImport(this.atlasGraph, new AtlasGraphProvider(), this.typeRegistry, entityChangeNotifier); } else { importStrategy = new RegularImport(this.atlasGraph, this.entityStore, this.typeRegistry); } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/ClassificationAssociator.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/ClassificationAssociator.java index 5c98454beda..a5cb554e5cc 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/ClassificationAssociator.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/ClassificationAssociator.java @@ -319,12 +319,12 @@ public List intersect(List lhs, List rhs) { List result = new ArrayList<>(); for (V c : rhs) { - V found = findFrom(lhs, c); - if (found != null) { - result.add(found); + V foundSame = findObjectFrom(lhs, c); + V foundSameTypeName = findFrom(lhs, c); + if ((foundSameTypeName != null) && (foundSame == null)) { + result.add(foundSameTypeName); } } - return result; } @@ -344,6 +344,11 @@ public List subtract(List lhs, List rhs) { return result; } + private V findObjectFrom(List reference, V check) { + return (V) CollectionUtils.find(reference, ox -> + ((V) ox).checkForUpdate(check)); + } + private V findFrom(List reference, V check) { return (V) CollectionUtils.find(reference, ox -> ((V) ox).getTypeName().equals(check.getTypeName())); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java index e21fe6beb91..4b0220a8c4b 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java @@ -19,10 +19,7 @@ import com.google.common.annotations.VisibleForTesting; -import org.apache.atlas.AtlasConfiguration; -import org.apache.atlas.AtlasErrorCode; -import org.apache.atlas.GraphTransactionInterceptor; -import org.apache.atlas.RequestContext; +import org.apache.atlas.*; import org.apache.atlas.annotation.GraphTransaction; import org.apache.atlas.authorize.AtlasAuthorizationUtils; import org.apache.atlas.authorize.AtlasEntityAccessRequest; @@ -39,7 +36,9 @@ import org.apache.atlas.model.instance.AtlasRelationship; import org.apache.atlas.model.instance.AtlasStruct; import org.apache.atlas.model.instance.EntityMutationResponse; +import org.apache.atlas.model.instance.EntityMutations; import org.apache.atlas.model.instance.EntityMutations.EntityOperation; +import org.apache.atlas.model.tasks.AtlasTask; import org.apache.atlas.model.typedef.AtlasEntityDef; import org.apache.atlas.model.typedef.AtlasEntityDef.AtlasRelationshipAttributeDef; import org.apache.atlas.model.typedef.AtlasRelationshipDef; @@ -57,13 +56,7 @@ import org.apache.atlas.repository.store.graph.AtlasRelationshipStore; import org.apache.atlas.repository.store.graph.EntityGraphDiscoveryContext; import org.apache.atlas.repository.store.graph.v1.DeleteHandlerDelegate; -import org.apache.atlas.repository.store.graph.v2.preprocessor.glossary.CategoryPreProcessor; -import org.apache.atlas.repository.store.graph.v2.preprocessor.glossary.GlossaryPreProcessor; -import org.apache.atlas.repository.store.graph.v2.preprocessor.glossary.TermPreProcessor; -import org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessor; -import org.apache.atlas.repository.store.graph.v2.preprocessor.sql.QueryCollectionPreProcessor; -import org.apache.atlas.repository.store.graph.v2.preprocessor.sql.QueryFolderPreProcessor; -import org.apache.atlas.repository.store.graph.v2.preprocessor.sql.QueryPreProcessor; +import org.apache.atlas.repository.store.graph.v2.tasks.ClassificationTask; import org.apache.atlas.tasks.TaskManagement; import org.apache.atlas.type.AtlasArrayType; import org.apache.atlas.repository.store.graph.v1.RestoreHandlerV1; @@ -80,6 +73,7 @@ import org.apache.atlas.type.AtlasTypeUtil; import org.apache.atlas.utils.AtlasEntityUtil; import org.apache.atlas.utils.AtlasJson; +import org.apache.atlas.utils.AtlasPerfMetrics; import org.apache.atlas.utils.AtlasPerfMetrics.MetricRecorder; import org.apache.atlas.utils.AtlasPerfTracer; import org.apache.commons.codec.digest.DigestUtils; @@ -108,8 +102,8 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; -import static org.apache.atlas.AtlasClient.PROCESS_SUPER_TYPE; import static org.apache.atlas.AtlasConfiguration.LABEL_MAX_LENGTH; +import static org.apache.atlas.AtlasConfiguration.STORE_DIFFERENTIAL_AUDITS; import static org.apache.atlas.model.TypeCategory.ARRAY; import static org.apache.atlas.model.TypeCategory.CLASSIFICATION; import static org.apache.atlas.model.instance.AtlasEntity.Status.ACTIVE; @@ -137,8 +131,13 @@ import static org.apache.atlas.repository.graph.GraphHelper.isRelationshipEdge; import static org.apache.atlas.repository.graph.GraphHelper.string; import static org.apache.atlas.repository.graph.GraphHelper.updateModificationMetadata; -import static org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.getIdFromVertex; -import static org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.isReference; +import static org.apache.atlas.repository.graph.GraphHelper.getEntityHasLineage; +import static org.apache.atlas.repository.graph.GraphHelper.isTermEntityEdge; +import static org.apache.atlas.repository.graph.GraphHelper.getRemovePropagations; +import static org.apache.atlas.repository.graph.GraphHelper.getPropagatedEdges; +import static org.apache.atlas.repository.graph.GraphHelper.getPropagatableClassifications; +import static org.apache.atlas.repository.graph.GraphHelper.getClassificationEntityGuid; +import static org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.*; import static org.apache.atlas.repository.store.graph.v2.tasks.ClassificationPropagateTaskFactory.CLASSIFICATION_PROPAGATION_ADD; import static org.apache.atlas.repository.store.graph.v2.tasks.ClassificationPropagateTaskFactory.CLASSIFICATION_PROPAGATION_DELETE; import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.IN; @@ -150,6 +149,7 @@ import static org.apache.atlas.type.Constants.HAS_LINEAGE; import static org.apache.atlas.type.Constants.MEANINGS_PROPERTY_KEY; import static org.apache.atlas.type.Constants.MEANINGS_TEXT_PROPERTY_KEY; +import static org.apache.atlas.type.Constants.MEANING_NAMES_PROPERTY_KEY; @Component @@ -175,11 +175,22 @@ public class EntityGraphMapper { private static final String ATTR_MEANINGS = "meanings"; private static final String ATTR_ANCHOR = "anchor"; private static final String ATTR_CATEGORIES = "categories"; - + private static final List ALLOWED_DATATYPES_FOR_DEFAULT_NULL = new ArrayList() { + { + add("int"); + add("long"); + add("float"); + } + }; private static final boolean ENTITY_CHANGE_NOTIFY_IGNORE_RELATIONSHIP_ATTRIBUTES = AtlasConfiguration.ENTITY_CHANGE_NOTIFY_IGNORE_RELATIONSHIP_ATTRIBUTES.getBoolean(); private static final boolean CLASSIFICATION_PROPAGATION_DEFAULT = AtlasConfiguration.CLASSIFICATION_PROPAGATION_DEFAULT.getBoolean(); + private static final boolean RESTRICT_PROPAGATION_THROUGH_LINEAGE_DEFAULT = false; private boolean DEFERRED_ACTION_ENABLED = AtlasConfiguration.TASKS_USE_ENABLED.getBoolean(); + private boolean DIFFERENTIAL_AUDITS = STORE_DIFFERENTIAL_AUDITS.getBoolean(); + + private static final int MAX_NUMBER_OF_RETRIES = AtlasConfiguration.MAX_NUMBER_OF_RETRIES.getInt(); + private static final int CHUNK_SIZE = AtlasConfiguration.TASKS_GRAPH_COMMIT_CHUNK_SIZE.getInt(); private final GraphHelper graphHelper; private final AtlasGraph graph; @@ -191,13 +202,14 @@ public class EntityGraphMapper { private final AtlasInstanceConverter instanceConverter; private final EntityGraphRetriever entityRetriever; private final IFullTextMapper fullTextMapperV2; - private final TaskManagement taskManagement; + private final TaskManagement taskManagement; + private final TransactionInterceptHelper transactionInterceptHelper; @Inject public EntityGraphMapper(DeleteHandlerDelegate deleteDelegate, RestoreHandlerV1 restoreHandlerV1, AtlasTypeRegistry typeRegistry, AtlasGraph graph, AtlasRelationshipStore relationshipStore, IAtlasEntityChangeNotifier entityChangeNotifier, AtlasInstanceConverter instanceConverter, IFullTextMapper fullTextMapperV2, - TaskManagement taskManagement) { + TaskManagement taskManagement, TransactionInterceptHelper transactionInterceptHelper) { this.restoreHandlerV1 = restoreHandlerV1; this.graphHelper = new GraphHelper(graph); this.deleteDelegate = deleteDelegate; @@ -209,6 +221,7 @@ public EntityGraphMapper(DeleteHandlerDelegate deleteDelegate, RestoreHandlerV1 this.entityRetriever = new EntityGraphRetriever(graph, typeRegistry); this.fullTextMapperV2 = fullTextMapperV2; this.taskManagement = taskManagement; + this.transactionInterceptHelper = transactionInterceptHelper; } @VisibleForTesting @@ -274,7 +287,9 @@ public AtlasVertex createVertexWithGuid(AtlasEntity entity, String guid) throws setCustomAttributes(ret, entity); - setLabels(ret, entity.getLabels()); + if (CollectionUtils.isNotEmpty(entity.getLabels())) { + setLabels(ret, entity.getLabels()); + } GraphTransactionInterceptor.addToVertexCache(guid, ret); @@ -323,7 +338,12 @@ public void updateSystemAttributes(AtlasVertex vertex, AtlasEntity entity) throw } } - public EntityMutationResponse mapAttributesAndClassifications(EntityMutationContext context, final boolean isPartialUpdate, final boolean replaceClassifications, boolean replaceBusinessAttributes) throws AtlasBaseException { + public EntityMutationResponse mapAttributesAndClassifications(EntityMutationContext context, + final boolean isPartialUpdate, + final boolean replaceClassifications, + boolean replaceBusinessAttributes, + boolean isOverwriteBusinessAttribute) throws AtlasBaseException { + MetricRecorder metric = RequestContext.get().startMetricRecord("mapAttributesAndClassifications"); EntityMutationResponse resp = new EntityMutationResponse(); @@ -339,13 +359,18 @@ public EntityMutationResponse mapAttributesAndClassifications(EntityMutationCont if (CollectionUtils.isNotEmpty(createdEntities)) { for (AtlasEntity createdEntity : createdEntities) { try { + reqContext.getDeletedEdgesIds().clear(); + String guid = createdEntity.getGuid(); AtlasVertex vertex = context.getVertex(guid); AtlasEntityType entityType = context.getType(guid); +<<<<<<< HEAD PreProcessor preProcessor = getPreProcessor(entityType.getTypeName()); if (preProcessor != null) { preProcessor.processAttributes(createdEntity, context, CREATE); } +======= +>>>>>>> 7f3c811fb15361ba82bfb4edd7a8393798863ff0 mapAttributes(createdEntity, entityType, vertex, CREATE, context); mapRelationshipAttributes(createdEntity, entityType, vertex, CREATE, context); @@ -355,9 +380,35 @@ public EntityMutationResponse mapAttributesAndClassifications(EntityMutationCont resp.addEntity(CREATE, constructHeader(createdEntity, vertex, entityType.getAllAttributes())); addClassifications(context, guid, createdEntity.getClassifications()); - addOrUpdateBusinessAttributes(vertex, entityType, createdEntity.getBusinessAttributes()); + if (MapUtils.isNotEmpty(createdEntity.getBusinessAttributes())) { + addOrUpdateBusinessAttributes(vertex, entityType, createdEntity.getBusinessAttributes()); + } + + Set inOutEdges = getNewCreatedInputOutputEdges(guid); + + if (inOutEdges != null && inOutEdges.size() > 0) { + boolean isRestoreEntity = false; + if (CollectionUtils.isNotEmpty(context.getEntitiesToRestore())) { + isRestoreEntity = context.getEntitiesToRestore().contains(vertex); + } + addHasLineage(inOutEdges, isRestoreEntity); + } + + Set removedEdges = getRemovedInputOutputEdges(guid); + + if (removedEdges != null && removedEdges.size() > 0) { + deleteDelegate.getHandler().resetHasLineageOnInputOutputDelete(removedEdges, null); + } reqContext.cache(createdEntity); + + if (DEFERRED_ACTION_ENABLED) { + Set deletedEdgeIds = reqContext.getDeletedEdgesIds(); + for (String deletedEdgeId : deletedEdgeIds) { + AtlasEdge edge = graph.getEdge(deletedEdgeId); + deleteDelegate.getHandler().createAndQueueClassificationRefreshPropagationTask(edge); + } + } } catch (AtlasBaseException baseException) { setEntityGuidToException(createdEntity, baseException, context); throw baseException; @@ -370,15 +421,12 @@ public EntityMutationResponse mapAttributesAndClassifications(EntityMutationCont if (CollectionUtils.isNotEmpty(updatedEntities)) { for (AtlasEntity updatedEntity : updatedEntities) { try { + reqContext.getDeletedEdgesIds().clear(); + String guid = updatedEntity.getGuid(); AtlasVertex vertex = context.getVertex(guid); AtlasEntityType entityType = context.getType(guid); - PreProcessor preProcessor = getPreProcessor(entityType.getTypeName()); - if (preProcessor != null) { - preProcessor.processAttributes(updatedEntity, context, UPDATE); - } - mapAttributes(updatedEntity, entityType, vertex, updateType, context); mapRelationshipAttributes(updatedEntity, entityType, vertex, UPDATE, context); @@ -390,12 +438,47 @@ public EntityMutationResponse mapAttributesAndClassifications(EntityMutationCont } if (replaceBusinessAttributes) { - setBusinessAttributes(vertex, entityType, updatedEntity.getBusinessAttributes()); + if (MapUtils.isEmpty(updatedEntity.getBusinessAttributes()) && isOverwriteBusinessAttribute) { + Map> businessMetadata = entityRetriever.getBusinessMetadata(vertex); + if (MapUtils.isNotEmpty(businessMetadata)){ + removeBusinessAttributes(vertex, entityType, businessMetadata); + } + } else { + addOrUpdateBusinessAttributes(guid, updatedEntity.getBusinessAttributes(), isOverwriteBusinessAttribute); + } } + setSystemAttributesToEntity(vertex,updatedEntity); resp.addEntity(updateType, constructHeader(updatedEntity, vertex, entityType.getAllAttributes())); + + // Add hasLineage for newly created edges + Set newlyCreatedEdges = getNewCreatedInputOutputEdges(guid); + if (newlyCreatedEdges.size() > 0) { + addHasLineage(newlyCreatedEdges, false); + } + + // Add hasLineage for restored edges + if (CollectionUtils.isNotEmpty(context.getEntitiesToRestore()) && context.getEntitiesToRestore().contains(vertex)) { + Set restoredInputOutputEdges = getRestoredInputOutputEdges(vertex); + addHasLineage(restoredInputOutputEdges, true); + } + + Set removedEdges = getRemovedInputOutputEdges(guid); + + if (removedEdges != null && removedEdges.size() > 0) { + deleteDelegate.getHandler().resetHasLineageOnInputOutputDelete(removedEdges, null); + } + reqContext.cache(updatedEntity); + if (DEFERRED_ACTION_ENABLED) { + Set deletedEdgeIds = reqContext.getDeletedEdgesIds(); + for (String deletedEdgeId : deletedEdgeIds) { + AtlasEdge edge = graph.getEdge(deletedEdgeId); + deleteDelegate.getHandler().createAndQueueClassificationRefreshPropagationTask(edge); + } + } + } catch (AtlasBaseException baseException) { setEntityGuidToException(updatedEntity, baseException, context); throw baseException; @@ -436,6 +519,15 @@ private void setSystemAttributesToEntity(AtlasVertex entityVertex, AtlasEntity c createdEntity.setUpdatedBy(GraphHelper.getModifiedByAsString(entityVertex)); createdEntity.setCreateTime(new Date(GraphHelper.getCreatedTime(entityVertex))); createdEntity.setUpdateTime(new Date(GraphHelper.getModifiedTime(entityVertex))); + + + if (DIFFERENTIAL_AUDITS) { + AtlasEntity diffEntity = RequestContext.get().getDifferentialEntity(createdEntity.getGuid()); + if (diffEntity != null) { + diffEntity.setUpdateTime(createdEntity.getUpdateTime()); + diffEntity.setUpdatedBy(createdEntity.getUpdatedBy()); + } + } } @@ -450,39 +542,6 @@ private void setEntityGuidToException(AtlasEntity entity, AtlasBaseException exc exception.setEntityGuid(guid); } - private PreProcessor getPreProcessor(String typeName) throws AtlasBaseException { - PreProcessor preProcessor = null; - - switch (typeName) { - case ATLAS_GLOSSARY_ENTITY_TYPE: - preProcessor = new GlossaryPreProcessor(typeRegistry, entityRetriever); - break; - - case ATLAS_GLOSSARY_TERM_ENTITY_TYPE: - preProcessor = new TermPreProcessor(typeRegistry, entityRetriever); - break; - - case ATLAS_GLOSSARY_CATEGORY_ENTITY_TYPE: - preProcessor = new CategoryPreProcessor(typeRegistry, entityRetriever); - break; - - case QUERY_ENTITY_TYPE: - preProcessor = new QueryPreProcessor(typeRegistry, entityRetriever); - break; - - case QUERY_FOLDER_ENTITY_TYPE: - preProcessor = new QueryFolderPreProcessor(typeRegistry, entityRetriever); - break; - - case QUERY_COLLECTION_ENTITY_TYPE: - preProcessor = new QueryCollectionPreProcessor(typeRegistry, entityRetriever); - break; - - } - - return preProcessor; - } - public void setCustomAttributes(AtlasVertex vertex, AtlasEntity entity) { String customAttributesString = getCustomAttributesString(entity); @@ -491,6 +550,12 @@ public void setCustomAttributes(AtlasVertex vertex, AtlasEntity entity) { } } + public void mapGlossaryRelationshipAttribute(AtlasAttribute attribute, AtlasObjectId glossaryObjectId, + AtlasVertex entityVertex, EntityMutationContext context) throws AtlasBaseException { + + mapAttribute(attribute, glossaryObjectId, entityVertex, EntityMutations.EntityOperation.UPDATE, context); + } + public void setLabels(AtlasVertex vertex, Set labels) throws AtlasBaseException { final Set currentLabels = getLabels(vertex); final Set addedLabels; @@ -549,6 +614,55 @@ public void removeLabels(AtlasVertex vertex, Set labels) throws AtlasBas } } + public void addOrUpdateBusinessAttributes(String guid, Map> businessAttrbutes, boolean isOverwrite) throws AtlasBaseException { + if (StringUtils.isEmpty(guid)) { + throw new AtlasBaseException(AtlasErrorCode.INVALID_PARAMETERS, "guid is null/empty"); + } + + if (MapUtils.isEmpty(businessAttrbutes)) { + return; + } + + AtlasVertex entityVertex = AtlasGraphUtilsV2.findByGuid(graph, guid); + + if (entityVertex == null) { + throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND, guid); + } + + String typeName = getTypeName(entityVertex); + AtlasEntityType entityType = typeRegistry.getEntityTypeByName(typeName); + AtlasEntityHeader entityHeader = entityRetriever.toAtlasEntityHeaderWithClassifications(entityVertex); + Map> currEntityBusinessAttributes = entityRetriever.getBusinessMetadata(entityVertex); + Set updatedBusinessMetadataNames = new HashSet<>(); + + for (String bmName : entityType.getBusinessAttributes().keySet()) { + Map bmAttrs = businessAttrbutes.get(bmName); + Map currBmAttrs = currEntityBusinessAttributes != null ? currEntityBusinessAttributes.get(bmName) : null; + + if (MapUtils.isEmpty(bmAttrs) && MapUtils.isEmpty(currBmAttrs)) { // no change + continue; + } else if (Objects.equals(bmAttrs, currBmAttrs)) { // no change + continue; + } + + updatedBusinessMetadataNames.add(bmName); + } + + AtlasEntityAccessRequest.AtlasEntityAccessRequestBuilder requestBuilder = new AtlasEntityAccessRequest.AtlasEntityAccessRequestBuilder(typeRegistry, AtlasPrivilege.ENTITY_UPDATE_BUSINESS_METADATA, entityHeader); + + for (String bmName : updatedBusinessMetadataNames) { + requestBuilder.setBusinessMetadata(bmName); + + AtlasAuthorizationUtils.verifyAccess(requestBuilder.build(), "add/update business-metadata: guid=", guid, ", business-metadata-name=", bmName); + } + + if (isOverwrite) { + setBusinessAttributes(entityVertex, entityType, businessAttrbutes); + } else { + addOrUpdateBusinessAttributes(entityVertex, entityType, businessAttrbutes); + } + } + /* * reset/overwrite business attributes of the entity with given values */ @@ -557,6 +671,8 @@ public void setBusinessAttributes(AtlasVertex entityVertex, AtlasEntityType enti LOG.debug("==> setBusinessAttributes(entityVertex={}, entityType={}, businessAttributes={}", entityVertex, entityType.getTypeName(), businessAttributes); } + validateBusinessAttributes(entityVertex, entityType, businessAttributes, true); + Map> entityTypeBusinessAttributes = entityType.getBusinessAttributes(); Map> updatedBusinessAttributes = new HashMap<>(); @@ -634,6 +750,8 @@ public void addOrUpdateBusinessAttributes(AtlasVertex entityVertex, AtlasEntityT LOG.debug("==> addOrUpdateBusinessAttributes(entityVertex={}, entityType={}, businessAttributes={}", entityVertex, entityType.getTypeName(), businessAttributes); } + validateBusinessAttributes(entityVertex, entityType, businessAttributes, true); + Map> entityTypeBusinessAttributes = entityType.getBusinessAttributes(); Map> updatedBusinessAttributes = new HashMap<>(); @@ -643,14 +761,20 @@ public void addOrUpdateBusinessAttributes(AtlasVertex entityVertex, AtlasEntityT Map bmAttributes = entry.getValue(); Map entityBmAttributes = businessAttributes.get(bmName); - if (MapUtils.isEmpty(entityBmAttributes)) { + if (MapUtils.isEmpty(entityBmAttributes) && !businessAttributes.containsKey(bmName)) { continue; } for (AtlasBusinessAttribute bmAttribute : bmAttributes.values()) { String bmAttrName = bmAttribute.getName(); - if (!entityBmAttributes.containsKey(bmAttrName)) { + if (MapUtils.isEmpty(entityBmAttributes)) { + entityVertex.removeProperty(bmAttribute.getVertexPropertyName()); + addToUpdatedBusinessAttributes(updatedBusinessAttributes, bmAttribute, null); + continue; + + } else if (!entityBmAttributes.containsKey(bmAttrName)) { + //since overwriteBusinessAttributes is false, ignore in case BM attr is not passed at all continue; } @@ -678,9 +802,15 @@ public void addOrUpdateBusinessAttributes(AtlasVertex entityVertex, AtlasEntityT } } else { if (!Objects.equals(existingValue, bmAttrValue)) { - mapAttribute(bmAttribute, bmAttrValue, entityVertex, UPDATE, new EntityMutationContext()); - addToUpdatedBusinessAttributes(updatedBusinessAttributes, bmAttribute, bmAttrValue); + if( bmAttrValue != null) { + mapAttribute(bmAttribute, bmAttrValue, entityVertex, UPDATE, new EntityMutationContext()); + + addToUpdatedBusinessAttributes(updatedBusinessAttributes, bmAttribute, bmAttrValue); + } else { + entityVertex.removeProperty(bmAttribute.getVertexPropertyName()); + addToUpdatedBusinessAttributes(updatedBusinessAttributes, bmAttribute, null); + } } } } @@ -704,6 +834,15 @@ public void removeBusinessAttributes(AtlasVertex entityVertex, AtlasEntityType e LOG.debug("==> removeBusinessAttributes(entityVertex={}, entityType={}, businessAttributes={}", entityVertex, entityType.getTypeName(), businessAttributes); } + AtlasEntityHeader entityHeader = entityRetriever.toAtlasEntityHeaderWithClassifications(entityVertex); + AtlasEntityAccessRequest.AtlasEntityAccessRequestBuilder requestBuilder = new AtlasEntityAccessRequest.AtlasEntityAccessRequestBuilder(typeRegistry, AtlasPrivilege.ENTITY_UPDATE_BUSINESS_METADATA, entityHeader); + + for (String bmName : businessAttributes.keySet()) { + requestBuilder.setBusinessMetadata(bmName); + + AtlasAuthorizationUtils.verifyAccess(requestBuilder.build(), "remove business-metadata: guid=", entityHeader.getGuid(), ", business-metadata=", bmName); + } + Map> entityTypeBusinessAttributes = entityType.getBusinessAttributes(); Map> updatedBusinessAttributes = new HashMap<>(); @@ -832,17 +971,32 @@ private void mapAttributes(AtlasStruct struct, AtlasStructType structType, Atlas Object attrOldValue = null; boolean isArrayOfPrimitiveType = false; boolean isArrayOfEnum = false; +<<<<<<< HEAD +======= + + boolean isStruct = (TypeCategory.STRUCT == attribute.getDefinedInType().getTypeCategory() + || TypeCategory.STRUCT == attribute.getAttributeType().getTypeCategory()); + +>>>>>>> 7f3c811fb15361ba82bfb4edd7a8393798863ff0 if (attribute.getAttributeType().getTypeCategory().equals(ARRAY)) { AtlasArrayType attributeType = (AtlasArrayType) attribute.getAttributeType(); AtlasType elementType = attributeType.getElementType(); isArrayOfPrimitiveType = elementType.getTypeCategory().equals(TypeCategory.PRIMITIVE); isArrayOfEnum = elementType.getTypeCategory().equals(TypeCategory.ENUM); } +<<<<<<< HEAD +======= + +>>>>>>> 7f3c811fb15361ba82bfb4edd7a8393798863ff0 if (isArrayOfPrimitiveType || isArrayOfEnum) { attrOldValue = vertex.getPropertyValues(attribute.getVertexPropertyName(),attribute.getClass()); + } else if (isStruct) { + String edgeLabel = AtlasGraphUtilsV2.getEdgeLabel(attribute.getName()); + attrOldValue = getCollectionElementsUsingRelationship(vertex, attribute, edgeLabel); } else { attrOldValue = vertex.getProperty(attribute.getVertexPropertyName(),attribute.getClass()); } + if (attrValue != null && !attrValue.equals(attrOldValue)) { addValuesToAutoUpdateAttributesList(attribute, userAutoUpdateAttributes, timestampAutoUpdateAttributes); } @@ -930,6 +1084,8 @@ private void mapAttribute(AtlasAttribute attribute, Object attrValue, AtlasVerte if (attrType.getTypeCategory() == TypeCategory.PRIMITIVE) { if (attributeDef.getDefaultValue() != null) { attrValue = attrType.createDefaultValue(attributeDef.getDefaultValue()); + } else if (attributeDef.getIsDefaultValueNull() && ALLOWED_DATATYPES_FOR_DEFAULT_NULL.contains(attribute.getTypeName())) { + attrValue = null; } else { if (attribute.getAttributeDef().getIsOptional()) { attrValue = attrType.createOptionalDefaultValue(); @@ -1311,9 +1467,11 @@ private AtlasEdge mapStructValue(AttributeMutationContext ctx, EntityMutationCon if (structVal != null) { updateVertex(structVal, ctx.getCurrentEdge().getInVertex(), context); + ret = ctx.getCurrentEdge(); + } else { + ret = null; } - ret = ctx.getCurrentEdge(); } else if (ctx.getValue() != null) { String edgeLabel = AtlasGraphUtilsV2.getEdgeLabel(ctx.getVertexProperty()); @@ -1594,7 +1752,8 @@ public List mapArrayValue(AttributeMutationContext ctx, EntityMutationContext co List newElements = (List) ctx.getValue(); AtlasArrayType arrType = (AtlasArrayType) attribute.getAttributeType(); AtlasType elementType = arrType.getElementType(); - boolean isStructType = (elementType.getTypeCategory() == TypeCategory.STRUCT); + boolean isStructType = (TypeCategory.STRUCT == elementType.getTypeCategory()) || + (TypeCategory.STRUCT == attribute.getDefinedInType().getTypeCategory()); boolean isReference = isReference(elementType); boolean isSoftReference = ctx.getAttribute().getAttributeDef().isSoftReferenced(); AtlasAttribute inverseRefAttribute = attribute.getInverseRefAttribute(); @@ -1613,7 +1772,7 @@ public List mapArrayValue(AttributeMutationContext ctx, EntityMutationContext co } if (isReference && !isSoftReference) { - currentElements = (List) getCollectionElementsUsingRelationship(ctx.getReferringVertex(), attribute); + currentElements = (List) getCollectionElementsUsingRelationship(ctx.getReferringVertex(), attribute, isStructType); } else { currentElements = (List) getArrayElementsProperty(elementType, isSoftReference, ctx.getReferringVertex(), ctx.getVertexProperty()); } @@ -1670,7 +1829,7 @@ public List mapArrayValue(AttributeMutationContext ctx, EntityMutationContext co if (isAppendOnPartialUpdate) { allArrayElements = unionCurrentAndNewElements(attribute, (List) currentElements, (List) newElementsCreated); } else { - removedElements = removeUnusedArrayEntries(attribute, (List) currentElements, (List) newElementsCreated, ctx.getReferringVertex()); + removedElements = removeUnusedArrayEntries(attribute, (List) currentElements, (List) newElementsCreated, ctx); allArrayElements = unionCurrentAndNewElements(attribute, removedElements, (List) newElementsCreated); } @@ -1694,7 +1853,7 @@ public List mapArrayValue(AttributeMutationContext ctx, EntityMutationContext co } switch (ctx.getAttribute().getRelationshipEdgeLabel()) { - case TERM_ASSIGNMENT_LABEL: addMeaningsToEntity(ctx, newElementsCreated); + case TERM_ASSIGNMENT_LABEL: addMeaningsToEntity(ctx, newElementsCreated, removedElements); break; case CATEGORY_TERMS_EDGE_LABEL: addCategoriesToTermEntity(ctx, newElementsCreated, removedElements); @@ -1704,7 +1863,7 @@ public List mapArrayValue(AttributeMutationContext ctx, EntityMutationContext co break; case PROCESS_INPUTS: - case PROCESS_OUTPUTS: addHasLineage(ctx, context, newElementsCreated, removedElements); + case PROCESS_OUTPUTS: addEdgesToContext(GraphHelper.getGuid(ctx.referringVertex), newElementsCreated, removedElements); break; } @@ -1715,6 +1874,47 @@ public List mapArrayValue(AttributeMutationContext ctx, EntityMutationContext co return allArrayElements; } +<<<<<<< HEAD +======= + private void addEdgesToContext(String guid, List newElementsCreated, List removedElements) { + + if (newElementsCreated.size() > 0) { + List elements = (RequestContext.get().getNewElementsCreatedMap()).get(guid); + if (elements == null) { + ArrayList newElements = new ArrayList<>(); + newElements.addAll(newElementsCreated); + (RequestContext.get().getNewElementsCreatedMap()).put(guid, newElements); + } else { + elements.addAll(newElementsCreated); + RequestContext.get().getNewElementsCreatedMap().put(guid, elements); + } + } + + if (removedElements.size() > 0) { + List removedElement = (RequestContext.get().getRemovedElementsMap()).get(guid); + + if (removedElement == null) { + removedElement = new ArrayList<>(); + removedElement.addAll(removedElements); + (RequestContext.get().getRemovedElementsMap()).put(guid, removedElement); + } else { + removedElement.addAll(removedElements); + (RequestContext.get().getRemovedElementsMap()).put(guid, removedElement); + } + } + } + + private boolean shouldDeleteExistingRelations(AttributeMutationContext ctx, AtlasAttribute attribute) { + boolean ret = false; + + AtlasEntityType entityType = typeRegistry.getEntityTypeByName(AtlasGraphUtilsV2.getTypeName(ctx.getReferringVertex())); + if (entityType !=null && entityType.hasRelationshipAttribute(attribute.getName())) { + AtlasRelationshipDef relationshipDef = typeRegistry.getRelationshipDefByName(ctx.getAttribute().getRelationshipName()); + ret = !(relationshipDef.getEndDef1().getCardinality() == SET && relationshipDef.getEndDef2().getCardinality() == SET); + } + return ret; + } +>>>>>>> 7f3c811fb15361ba82bfb4edd7a8393798863ff0 /* * Before creating new edges between referring vertex & new vertex coming from array, @@ -1733,13 +1933,29 @@ private void removeExistingRelationWithOtherVertex(AttributeMutationContext arrC EntityMutationContext context) throws AtlasBaseException { MetricRecorder metric = RequestContext.get().startMetricRecord("removeExistingRelationWithOtherVertex"); - AtlasVertex referredVertex = context.getVertex(((AtlasObjectId) arrCtx.getValue()).getGuid()); + AtlasObjectId entityObject = (AtlasObjectId) arrCtx.getValue(); + String entityGuid = entityObject.getGuid(); + + AtlasVertex referredVertex = null; + + if (StringUtils.isNotEmpty(entityGuid)) { + referredVertex = context.getVertex(entityGuid); + } + if (referredVertex == null) { try { - referredVertex = entityRetriever.getEntityVertex(((AtlasObjectId) arrCtx.getValue()).getGuid()); - } catch (AtlasBaseException ebe) { + if (StringUtils.isNotEmpty(entityGuid)) { + referredVertex = entityRetriever.getEntityVertex(((AtlasObjectId) arrCtx.getValue()).getGuid()); + } else { + AtlasEntityType entityType = typeRegistry.getEntityTypeByName(entityObject.getTypeName()); + if (entityType != null && MapUtils.isNotEmpty(entityObject.getUniqueAttributes())) { + referredVertex = AtlasGraphUtilsV2.findByUniqueAttributes(this.graph, entityType, entityObject.getUniqueAttributes()); + } + } + } catch (AtlasBaseException e) { //in case if importing zip, referredVertex might not have been create yet //e.g. importing zip with db & its tables, while processing db edges, tables vertices are not yet created + LOG.warn("removeExistingRelationWithOtherVertex - vertex not found!", e); } } @@ -1762,6 +1978,7 @@ private void removeExistingRelationWithOtherVertex(AttributeMutationContext arrC if (LOG.isDebugEnabled()) { LOG.debug("Delete existing relation"); } + deleteDelegate.getHandler().deleteEdgeReference(existingEdgeToReferredVertex, ctx.getAttrType().getTypeCategory(), ctx.getAttribute().isOwnedRef(), true, ctx.getAttribute().getRelationshipEdgeDirection(), ctx.getReferringVertex()); } @@ -1817,7 +2034,10 @@ public void removeAttrForCategoryDelete(Collection categories) { Iterator edgeIterator = vertex.getEdges(AtlasEdgeDirection.OUT, CATEGORY_PARENT_EDGE_LABEL).iterator(); while (edgeIterator.hasNext()) { AtlasEdge childEdge = edgeIterator.next(); - childEdge.getInVertex().removeProperty(CATEGORIES_PARENT_PROPERTY_KEY); + AtlasEntity.Status edgeStatus = getStatus(childEdge); + if (ACTIVE.equals(edgeStatus)) { + childEdge.getInVertex().removeProperty(CATEGORIES_PARENT_PROPERTY_KEY); + } } String catQualifiedName = vertex.getProperty(QUALIFIED_NAME, String.class); @@ -1849,77 +2069,6 @@ private void addCatParentAttr(AttributeMutationContext ctx, List newElem RequestContext.get().endMetricRecord(metricRecorder); } - private void addHasLineage(AttributeMutationContext ctx, EntityMutationContext context, - List newElementsCreated, List removedElements){ - MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("addHasLineage"); - - AtlasVertex toVertex = ctx.getReferringVertex(); - AtlasEntityType entityType = typeRegistry.getEntityTypeByName(getTypeName(toVertex)); - boolean isProcess = entityType.getTypeAndAllSuperTypes().contains(PROCESS_SUPER_TYPE); - - if (CollectionUtils.isNotEmpty(newElementsCreated)) { - AtlasGraphUtilsV2.setEncodedProperty(toVertex, HAS_LINEAGE, true); - - List vertices; - if (isProcess) { - vertices = newElementsCreated.stream().map(x -> ((AtlasEdge) x).getInVertex()).collect(Collectors.toList()); - } else { - vertices = newElementsCreated.stream().map(x -> ((AtlasEdge) x).getOutVertex()).collect(Collectors.toList()); - } - - vertices.stream().forEach(v -> AtlasGraphUtilsV2.setEncodedProperty(v, HAS_LINEAGE, true)); - - } else if (CollectionUtils.isNotEmpty(removedElements)) { - Set removedGuids = removedElements.stream().map(x -> GraphHelper.getRelationshipGuid(x)).collect(Collectors.toSet()); - context.addRemovedLineageRelations(removedGuids); - - boolean removeAttr = true; - Iterator edgeIterator; - - if (ctx.getAttribute().getRelationshipEdgeLabel().equals(PROCESS_INPUTS)) { - edgeIterator = toVertex.getEdges(AtlasEdgeDirection.OUT, PROCESS_OUTPUTS).iterator(); - } else { - edgeIterator = toVertex.getEdges(AtlasEdgeDirection.OUT, PROCESS_INPUTS).iterator(); - } - - while (edgeIterator.hasNext()) { - AtlasEdge edg = edgeIterator.next(); - if (ACTIVE.equals(getStatus(edg)) && !context.getRemovedLineageRelations().contains(GraphHelper.getRelationshipGuid(edg))) { - removeAttr = false; break; - } - } - - if (removeAttr) { - toVertex.removeProperty(HAS_LINEAGE); - } - - String[] edgeLabels = {PROCESS_OUTPUTS, PROCESS_INPUTS}; - - List vertices; - if (isProcess) { - vertices = removedElements.stream().map(x -> ((AtlasEdge) x).getInVertex()).collect(Collectors.toList()); - } else { - vertices = removedElements.stream().map(x -> ((AtlasEdge) x).getOutVertex()).collect(Collectors.toList()); - } - - for (AtlasVertex vertex : vertices) { - removeAttr = true; - edgeIterator = vertex.getEdges(AtlasEdgeDirection.BOTH, edgeLabels).iterator(); - - while (edgeIterator.hasNext()) { - AtlasEdge edg = edgeIterator.next(); - if (ACTIVE.equals(getStatus(edg)) && !context.getRemovedLineageRelations().contains(GraphHelper.getRelationshipGuid(edg))) { - removeAttr = false; break; - } - } - - if (removeAttr) { - vertex.removeProperty(HAS_LINEAGE); - } - } - } - RequestContext.get().endMetricRecord(metricRecorder); - } private void addCategoriesToTermEntity(AttributeMutationContext ctx, List newElementsCreated, List removedElements) { MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("addCategoriesToTermEntity"); @@ -1949,14 +2098,27 @@ private void addCategoriesToTermEntity(AttributeMutationContext ctx, List newElementsCreated) { + private void addMeaningsToEntity(AttributeMutationContext ctx, List createdElements, List deletedElements) { MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("addMeaningsToEntity"); // handle __terms attribute of entity - List meanings = newElementsCreated.stream().map(x -> ((AtlasEdge) x).getOutVertex()).collect(Collectors.toList()); + List meanings = createdElements.stream() + .map(x -> ((AtlasEdge) x).getOutVertex()) + .filter(x -> ACTIVE.name().equals(x.getProperty(STATE_PROPERTY_KEY, String.class))) + .collect(Collectors.toList()); + List currentMeaningsQNames = ctx.getReferringVertex().getMultiValuedProperty(MEANINGS_PROPERTY_KEY,String.class); Set qNames = meanings.stream().map(x -> x.getProperty(QUALIFIED_NAME, String.class)).collect(Collectors.toSet()); List names = meanings.stream().map(x -> x.getProperty(NAME, String.class)).collect(Collectors.toList()); + List deletedMeaningsNames = deletedElements.stream().map(x -> x.getOutVertex()) + . map(x -> x.getProperty(NAME,String.class)) + .collect(Collectors.toList()); + + List newMeaningsNames = meanings.stream() + .filter(x -> !currentMeaningsQNames.contains(x.getProperty(QUALIFIED_NAME,String.class))) + .map(x -> x.getProperty(NAME, String.class)) + .collect(Collectors.toList()); + ctx.getReferringVertex().removeProperty(MEANINGS_PROPERTY_KEY); ctx.getReferringVertex().removeProperty(MEANINGS_TEXT_PROPERTY_KEY); @@ -1967,6 +2129,19 @@ private void addMeaningsToEntity(AttributeMutationContext ctx, List newE if (CollectionUtils.isNotEmpty(names)) { AtlasGraphUtilsV2.setEncodedProperty(ctx.referringVertex, MEANINGS_TEXT_PROPERTY_KEY, StringUtils.join(names, ",")); } + + if (CollectionUtils.isNotEmpty(newMeaningsNames)) { + newMeaningsNames.forEach(q -> AtlasGraphUtilsV2.addListProperty(ctx.getReferringVertex(), MEANING_NAMES_PROPERTY_KEY, q, true)); + } + + if(createdElements.isEmpty()){ + ctx.getReferringVertex().removeProperty(MEANING_NAMES_PROPERTY_KEY); + + } else if (CollectionUtils.isNotEmpty(deletedMeaningsNames)) { + deletedMeaningsNames.forEach(q -> AtlasGraphUtilsV2.removeItemFromListPropertyValue(ctx.getReferringVertex(), MEANING_NAMES_PROPERTY_KEY, q)); + + } + RequestContext.get().endMetricRecord(metricRecorder); } @@ -2380,9 +2555,10 @@ private List unionCurrentAndNewElements(AtlasAttribute attribute, Lis //Removes unused edges from the old collection, compared to the new collection - private List removeUnusedArrayEntries(AtlasAttribute attribute, List currentEntries, List newEntries, AtlasVertex entityVertex) throws AtlasBaseException { + private List removeUnusedArrayEntries(AtlasAttribute attribute, List currentEntries, List newEntries, AttributeMutationContext ctx) throws AtlasBaseException { if (CollectionUtils.isNotEmpty(currentEntries)) { AtlasType entryType = ((AtlasArrayType) attribute.getAttributeType()).getElementType(); + AtlasVertex entityVertex = ctx.getReferringVertex(); if (isReference(entryType)) { Collection edgesToRemove = CollectionUtils.subtract(currentEntries, newEntries); @@ -2391,6 +2567,10 @@ private List removeUnusedArrayEntries(AtlasAttribute attribute, List< List additionalElements = new ArrayList<>(); for (AtlasEdge edge : edgesToRemove) { + if (getStatus(edge) == DELETED ) { + continue; + } + boolean deleted = deleteDelegate.getHandler().deleteEdgeReference(edge, entryType.getTypeCategory(), attribute.isOwnedRef(), true, attribute.getRelationshipEdgeDirection(), entityVertex); @@ -2424,6 +2604,42 @@ private void setArrayElementsProperty(AtlasType elementType, boolean isSoftRefer } } + + private Set getNewCreatedInputOutputEdges(String guid) { + List newElementsCreated = RequestContext.get().getNewElementsCreatedMap().get(guid); + + Set newEdge = new HashSet<>(); + if (newElementsCreated != null && newElementsCreated.size() > 0) { + newEdge = newElementsCreated.stream().map(x -> (AtlasEdge) x).collect(Collectors.toSet()); + } + + return newEdge; + } + + private Set getRestoredInputOutputEdges(AtlasVertex vertex) { + Set activatedEdges = new HashSet<>(); + Iterator iterator = vertex.getEdges(AtlasEdgeDirection.BOTH, new String[]{PROCESS_INPUTS, PROCESS_OUTPUTS}).iterator(); + while (iterator.hasNext()) { + AtlasEdge edge = iterator.next(); + if (edge.getProperty(STATE_PROPERTY_KEY, String.class).equalsIgnoreCase(ACTIVE_STATE_VALUE)) { + activatedEdges.add(edge); + } + } + return activatedEdges; + } + + private Set getRemovedInputOutputEdges(String guid) { + List removedElements = RequestContext.get().getRemovedElementsMap().get(guid); + Set removedEdges = null; + + if (removedElements != null) { + removedEdges = removedElements.stream().map(x -> (AtlasEdge) x).collect(Collectors.toSet()); + } + + return removedEdges; + } + + private AtlasEntityHeader constructHeader(AtlasEntity entity, AtlasVertex vertex, Map attributeMap ) throws AtlasBaseException { AtlasEntityHeader header = entityRetriever.toAtlasEntityHeaderWithClassifications(vertex, attributeMap.keySet()); if (entity.getClassifications() == null) { @@ -2461,6 +2677,7 @@ public void addClassifications(final EntityMutationContext context, String guid, String classificationName = classification.getTypeName(); Boolean propagateTags = classification.isPropagate(); Boolean removePropagations = classification.getRemovePropagationsOnEntityDelete(); + Boolean restrictPropagationThroughLineage = classification.getRestrictPropagationThroughLineage(); if (propagateTags != null && propagateTags && classification.getEntityGuid() != null && @@ -2491,6 +2708,10 @@ public void addClassifications(final EntityMutationContext context, String guid, classification.setRemovePropagationsOnEntityDelete(removePropagations); } + if (restrictPropagationThroughLineage == null) { + classification.setRestrictPropagationThroughLineage(RESTRICT_PROPAGATION_THROUGH_LINEAGE_DEFAULT); + } + // set associated entity id to classification if (classification.getEntityGuid() == null) { classification.setEntityGuid(guid); @@ -2534,7 +2755,11 @@ public void addClassifications(final EntityMutationContext context, String guid, if (propagateTags) { // compute propagatedEntityVertices only once if (entitiesToPropagateTo == null) { - entitiesToPropagateTo = entityRetriever.getImpactedVerticesV2(entityVertex); + String propagationMode = CLASSIFICATION_PROPAGATION_MODE_DEFAULT; + if (classification.getRestrictPropagationThroughLineage() != null && classification.getRestrictPropagationThroughLineage()) { + propagationMode = CLASSIFICATION_PROPAGATION_MODE_RESTRICT_LINEAGE; + } + entitiesToPropagateTo = entityRetriever.getImpactedVerticesV2(entityVertex, CLASSIFICATION_PROPAGATION_EXCLUSION_MAP.get(propagationMode)); } if (CollectionUtils.isNotEmpty(entitiesToPropagateTo)) { @@ -2572,58 +2797,64 @@ public void addClassifications(final EntityMutationContext context, String guid, Set vertices = addedClassifications.get(classification); List propagatedEntities = updateClassificationText(classification, vertices); - entityChangeNotifier.onClassificationsAddedToEntities(propagatedEntities, Collections.singletonList(classification)); + entityChangeNotifier.onClassificationsAddedToEntities(propagatedEntities, Collections.singletonList(classification), false); } RequestContext.get().endMetricRecord(metric); } } - @GraphTransaction - public List propagateClassification(String entityGuid, String classificationVertexId, String relationshipGuid) throws AtlasBaseException { + public List propagateClassification(String entityGuid, String classificationVertexId, String relationshipGuid, Boolean previousRestrictPropagationThroughLineage) throws AtlasBaseException { try { if (StringUtils.isEmpty(entityGuid) || StringUtils.isEmpty(classificationVertexId)) { - LOG.warn("propagateClassification(entityGuid={}, classificationVertexId={}): entityGuid and/or classification vertex id is empty", entityGuid, classificationVertexId); + LOG.error("propagateClassification(entityGuid={}, classificationVertexId={}): entityGuid and/or classification vertex id is empty", entityGuid, classificationVertexId); - return null; + throw new AtlasBaseException(String.format("propagateClassification(entityGuid=%s, classificationVertexId=%s): entityGuid and/or classification vertex id is empty", entityGuid, classificationVertexId)); } AtlasVertex entityVertex = graphHelper.getVertexForGUID(entityGuid); if (entityVertex == null) { - LOG.warn("propagateClassification(entityGuid={}, classificationVertexId={}): entity vertex not found", entityGuid, classificationVertexId); + LOG.error("propagateClassification(entityGuid={}, classificationVertexId={}): entity vertex not found", entityGuid, classificationVertexId); - return null; + throw new AtlasBaseException(String.format("propagateClassification(entityGuid=%s, classificationVertexId=%s): entity vertex not found", entityGuid, classificationVertexId)); } AtlasVertex classificationVertex = graph.getVertex(classificationVertexId); if (classificationVertex == null) { - LOG.warn("propagateClassification(entityGuid={}, classificationVertexId={}): classification vertex not found", entityGuid, classificationVertexId); + LOG.error("propagateClassification(entityGuid={}, classificationVertexId={}): classification vertex not found", entityGuid, classificationVertexId); - return null; + throw new AtlasBaseException(String.format("propagateClassification(entityGuid=%s, classificationVertexId=%s): classification vertex not found", entityGuid, classificationVertexId)); } - List impactedVertices = entityRetriever.getIncludedImpactedVerticesV2(entityVertex, relationshipGuid, classificationVertexId); - if (CollectionUtils.isEmpty(impactedVertices)) { - LOG.debug("propagateClassification(entityGuid={}, classificationVertexId={}): found no entities to propagate the classification", entityGuid, classificationVertexId); + /* + If restrictPropagateThroughLineage was false at past + then updated to true we need to delete the propagated + classifications and then put the classifications as intended + */ - return null; - } + Boolean currentRestrictPropagationThroughLineage = AtlasGraphUtilsV2.getProperty(classificationVertex, CLASSIFICATION_VERTEX_RESTRICT_PROPAGATE_THROUGH_LINEAGE, Boolean.class); - List impactedVerticesGuidsToLock = impactedVertices.stream().map(x -> GraphHelper.getGuid(x)).collect(Collectors.toList()); - GraphTransactionInterceptor.lockObjectAndReleasePostCommit(impactedVerticesGuidsToLock); + if (previousRestrictPropagationThroughLineage != null && currentRestrictPropagationThroughLineage != null && !previousRestrictPropagationThroughLineage && currentRestrictPropagationThroughLineage) { + deleteDelegate.getHandler().removeTagPropagation(classificationVertex); + } - AtlasClassification classification = entityRetriever.toAtlasClassification(classificationVertex); - List entitiesPropagatedTo = deleteDelegate.getHandler().addTagPropagation(classificationVertex, impactedVertices); + String propagationMode = CLASSIFICATION_PROPAGATION_MODE_DEFAULT; - if (CollectionUtils.isEmpty(entitiesPropagatedTo)) { - return null; + if (currentRestrictPropagationThroughLineage != null && currentRestrictPropagationThroughLineage) { + propagationMode = CLASSIFICATION_PROPAGATION_MODE_RESTRICT_LINEAGE; } - List propagatedEntities = updateClassificationText(classification, entitiesPropagatedTo); + List edgeLabelsToExclude = CLASSIFICATION_PROPAGATION_EXCLUSION_MAP.get(propagationMode); - entityChangeNotifier.onClassificationsAddedToEntities(propagatedEntities, Collections.singletonList(classification)); + List impactedVertices = entityRetriever.getIncludedImpactedVerticesV2(entityVertex, relationshipGuid, classificationVertexId, edgeLabelsToExclude); - return propagatedEntities.stream().map(x -> x.getGuid()).collect(Collectors.toList()); + if (CollectionUtils.isEmpty(impactedVertices)) { + LOG.debug("propagateClassification(entityGuid={}, classificationVertexId={}): found no entities to propagate the classification", entityGuid, classificationVertexId); + + return null; + } + + return processClassificationPropagationAddition(impactedVertices, classificationVertex); } catch (Exception e) { LOG.error("propagateClassification(entityGuid={}, classificationVertexId={}): error while propagating classification", entityGuid, classificationVertexId, e); @@ -2631,6 +2862,53 @@ public List propagateClassification(String entityGuid, String classifica } } + public List processClassificationPropagationAddition(List verticesToPropagate, AtlasVertex classificationVertex) throws AtlasBaseException{ + AtlasPerfMetrics.MetricRecorder classificationPropagationMetricRecorder = RequestContext.get().startMetricRecord("processClassificationPropagationAddition"); + List propagatedEntitiesGuids = new ArrayList<>(); + int impactedVerticesSize = verticesToPropagate.size(); + int offset = 0; + int toIndex; + LOG.info(String.format("Total number of vertices to propagate: %d", impactedVerticesSize)); + + try { + do { + toIndex = ((offset + CHUNK_SIZE > impactedVerticesSize) ? impactedVerticesSize : (offset + CHUNK_SIZE)); + List chunkedVerticesToPropagate = verticesToPropagate.subList(offset, toIndex); + + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("lockObjectsAfterTraverse"); + List impactedVerticesGuidsToLock = chunkedVerticesToPropagate.stream().map(x -> GraphHelper.getGuid(x)).collect(Collectors.toList()); + GraphTransactionInterceptor.lockObjectAndReleasePostCommit(impactedVerticesGuidsToLock); + RequestContext.get().endMetricRecord(metricRecorder); + + AtlasClassification classification = entityRetriever.toAtlasClassification(classificationVertex); + List entitiesPropagatedTo = deleteDelegate.getHandler().addTagPropagation(classificationVertex, chunkedVerticesToPropagate); + + if (CollectionUtils.isEmpty(entitiesPropagatedTo)) { + return null; + } + + List propagatedEntitiesChunked = updateClassificationText(classification, entitiesPropagatedTo); + List chunkedPropagatedEntitiesGuids = propagatedEntitiesChunked.stream().map(x -> x.getGuid()).collect(Collectors.toList()); + entityChangeNotifier.onClassificationsAddedToEntities(propagatedEntitiesChunked, Collections.singletonList(classification), false); + + propagatedEntitiesGuids.addAll(chunkedPropagatedEntitiesGuids); + + offset += CHUNK_SIZE; + + transactionInterceptHelper.intercept(); + + } while (offset < impactedVerticesSize); + } catch (AtlasBaseException exception) { + LOG.error("Error occurred while adding classification propagation for classification with propagation id {}", classificationVertex.getIdForDisplay()); + throw exception; + } finally { + RequestContext.get().endMetricRecord(classificationPropagationMetricRecorder); + } + + return propagatedEntitiesGuids; + + } + public void deleteClassification(String entityGuid, String classificationName, String associatedEntityGuid) throws AtlasBaseException { if (StringUtils.isEmpty(associatedEntityGuid) || associatedEntityGuid.equals(entityGuid)) { deleteClassification(entityGuid, classificationName); @@ -2689,6 +2967,15 @@ entityHeader, new AtlasClassification(classificationName)), validateClassificationExists(traitNames, classificationName); AtlasVertex classificationVertex = getClassificationVertex(entityVertex, classificationName); + + // Get in progress task to see if there already is a propagation for this particular vertex + List inProgressTasks = taskManagement.getInProgressTasks(); + for (AtlasTask task : inProgressTasks) { + if (isTaskMatchingWithVertexIdAndEntityGuid(task, classificationVertex.getIdForDisplay(), entityGuid)) { + throw new AtlasBaseException(AtlasErrorCode.CLASSIFICATION_CURRENTLY_BEING_PROPAGATED, classificationName); + } + } + AtlasClassification classification = entityRetriever.toAtlasClassification(classificationVertex); if (classification == null) { @@ -2700,7 +2987,37 @@ entityHeader, new AtlasClassification(classificationName)), if (isPropagationEnabled(classificationVertex)) { if (taskManagement != null && DEFERRED_ACTION_ENABLED) { - createAndQueueTask(CLASSIFICATION_PROPAGATION_DELETE, entityVertex, classificationVertex.getIdForDisplay()); + boolean propagateDelete = true; + String classificationVertexId = classificationVertex.getIdForDisplay(); + + List entityTaskGuids = (List) entityVertex.getPropertyValues(PENDING_TASKS_PROPERTY_KEY, String.class); + + if (CollectionUtils.isNotEmpty(entityTaskGuids)) { + List entityPendingTasks = taskManagement.getByGuidsES(entityTaskGuids); + + boolean pendingTaskExists = entityPendingTasks.stream() + .anyMatch(x -> isTaskMatchingWithVertexIdAndEntityGuid(x, classificationVertexId, entityGuid)); + + if (pendingTaskExists) { + List entityClassificationPendingTasks = entityPendingTasks.stream() + .filter(t -> t.getParameters().containsKey("entityGuid") + && t.getParameters().containsKey("classificationVertexId")) + .filter(t -> t.getParameters().get("entityGuid").equals(entityGuid) + && t.getParameters().get("classificationVertexId").equals(classificationVertexId) + && t.getType().equals(CLASSIFICATION_PROPAGATION_ADD)) + .collect(Collectors.toList()); + for (AtlasTask entityClassificationPendingTask: entityClassificationPendingTasks) { + String taskGuid = entityClassificationPendingTask.getGuid(); + taskManagement.deleteByGuid(taskGuid, TaskManagement.DeleteType.SOFT); + AtlasGraphUtilsV2.deleteProperty(entityVertex, PENDING_TASKS_PROPERTY_KEY, taskGuid); +// propagateDelete = false; TODO: Uncomment when all unnecessary ADD tasks are resolved + } + } + } + + if (propagateDelete) { + createAndQueueTask(CLASSIFICATION_PROPAGATION_DELETE, entityVertex, classificationVertex.getIdForDisplay()); + } entityVertices = new ArrayList<>(); } else { @@ -2750,6 +3067,18 @@ entityHeader, new AtlasClassification(classificationName)), AtlasPerfTracer.log(perf); } + private boolean isTaskMatchingWithVertexIdAndEntityGuid(AtlasTask task, String classificationVertexId, String entityGuid) { + try { + if (CLASSIFICATION_PROPAGATION_ADD.equals(task.getType())) { + return task.getParameters().get(ClassificationTask.PARAM_CLASSIFICATION_VERTEX_ID).equals(classificationVertexId) + && task.getParameters().get(ClassificationTask.PARAM_ENTITY_GUID).equals(entityGuid); + } + } catch (NullPointerException npe) { + LOG.warn("Task classificationVertexId or entityGuid is null"); + } + return false; + } + private AtlasEntity updateClassificationText(AtlasVertex vertex) throws AtlasBaseException { String guid = graphHelper.getGuid(vertex); AtlasEntity entity = instanceConverter.getAndCacheEntity(guid, ENTITY_CHANGE_NOTIFY_IGNORE_RELATIONSHIP_ATTRIBUTES); @@ -2887,14 +3216,18 @@ public void updateClassifications(EntityMutationContext context, String guid, Li isClassificationUpdated = true; } + boolean removePropagation = false; // check for removePropagationsOnEntityDelete update Boolean currentRemovePropagations = currentClassification.getRemovePropagationsOnEntityDelete(); Boolean updatedRemovePropagations = classification.getRemovePropagationsOnEntityDelete(); - - if (updatedRemovePropagations != null && (updatedRemovePropagations != currentRemovePropagations)) { + if (updatedRemovePropagations != null && !updatedRemovePropagations.equals(currentRemovePropagations)) { AtlasGraphUtilsV2.setEncodedProperty(classificationVertex, CLASSIFICATION_VERTEX_REMOVE_PROPAGATIONS_KEY, updatedRemovePropagations); - isClassificationUpdated = true; + + boolean isEntityDeleted = DELETED.toString().equals(entityVertex.getProperty(STATE_PROPERTY_KEY, String.class)); + if (isEntityDeleted && updatedRemovePropagations) { + removePropagation = true; + } } if (isClassificationUpdated) { @@ -2910,10 +3243,6 @@ public void updateClassifications(EntityMutationContext context, String guid, Li mapClassification(EntityOperation.UPDATE, context, classification, entityType, entityVertex, classificationVertex); updateModificationMetadata(entityVertex); - // handle update of 'propagate' flag - Boolean currentTagPropagation = currentClassification.isPropagate(); - Boolean updatedTagPropagation = classification.isPropagate(); - /* ----------------------------- | Current Tag | Updated Tag | | Propagation | Propagation | @@ -2927,19 +3256,37 @@ public void updateClassifications(EntityMutationContext context, String guid, Li | true | false | => Remove Tag Propagation (send REMOVE classification notifications) |-------------|-------------| */ - if (taskManagement != null && DEFERRED_ACTION_ENABLED) { - String propagationType = updatedTagPropagation ? CLASSIFICATION_PROPAGATION_ADD : CLASSIFICATION_PROPAGATION_DELETE; - - createAndQueueTask(propagationType, entityVertex, classificationVertex.getIdForDisplay()); - - updatedTagPropagation = null; + Boolean currentTagPropagation = currentClassification.isPropagate(); + Boolean updatedTagPropagation = classification.isPropagate(); + Boolean currentRestrictPropagationThroughLineage = currentClassification.getRestrictPropagationThroughLineage(); + Boolean updatedRestrictPropagationThroughLineage = classification.getRestrictPropagationThroughLineage(); + + if ((!Objects.equals(updatedRemovePropagations, currentRemovePropagations) || + !Objects.equals(currentTagPropagation, updatedTagPropagation) || + !Objects.equals(currentRestrictPropagationThroughLineage, updatedRestrictPropagationThroughLineage)) && + taskManagement != null && DEFERRED_ACTION_ENABLED) { + + String propagationType = CLASSIFICATION_PROPAGATION_ADD; + if (removePropagation || !updatedTagPropagation) + { + propagationType = CLASSIFICATION_PROPAGATION_DELETE; + } + createAndQueueTask(propagationType, entityVertex, classificationVertex.getIdForDisplay(), currentRestrictPropagationThroughLineage); } // compute propagatedEntityVertices once and use it for subsequent iterations and notifications - if (updatedTagPropagation != null && currentTagPropagation != updatedTagPropagation) { + if (updatedTagPropagation != null && (currentTagPropagation != updatedTagPropagation || currentRestrictPropagationThroughLineage != updatedRestrictPropagationThroughLineage)) { if (updatedTagPropagation) { + if (updatedRestrictPropagationThroughLineage != null && !currentRestrictPropagationThroughLineage && updatedRestrictPropagationThroughLineage) { + deleteDelegate.getHandler().removeTagPropagation(classificationVertex); + + } if (CollectionUtils.isEmpty(entitiesToPropagateTo)) { - entitiesToPropagateTo = entityRetriever.getImpactedVerticesV2(entityVertex, null, classificationVertex.getIdForDisplay()); + String propagationMode = CLASSIFICATION_PROPAGATION_MODE_DEFAULT; + if (updatedRemovePropagations !=null && updatedRestrictPropagationThroughLineage) { + propagationMode = CLASSIFICATION_PROPAGATION_MODE_RESTRICT_LINEAGE; + } + entitiesToPropagateTo = entityRetriever.getImpactedVerticesV2(entityVertex, null, classificationVertex.getIdForDisplay(), CLASSIFICATION_PROPAGATION_EXCLUSION_MAP.get(propagationMode)); } if (CollectionUtils.isNotEmpty(entitiesToPropagateTo)) { @@ -3027,6 +3374,10 @@ private AtlasEdge mapClassification(EntityOperation operation, final EntityMuta AtlasGraphUtilsV2.setEncodedProperty(traitInstanceVertex, CLASSIFICATION_VERTEX_REMOVE_PROPAGATIONS_KEY, classification.getRemovePropagationsOnEntityDelete()); } + if(classification.getRestrictPropagationThroughLineage() != null){ + AtlasGraphUtilsV2.setEncodedProperty(traitInstanceVertex, CLASSIFICATION_VERTEX_RESTRICT_PROPAGATE_THROUGH_LINEAGE, classification.getRestrictPropagationThroughLineage()); + } + // map all the attributes to this newly created AtlasVertex mapAttributes(classification, traitInstanceVertex, operation, context); @@ -3055,7 +3406,6 @@ public void deleteClassifications(String guid) throws AtlasBaseException { } } - @GraphTransaction public List deleteClassificationPropagation(String entityGuid, String classificationVertexId) throws AtlasBaseException { try { if (StringUtils.isEmpty(classificationVertexId)) { @@ -3072,25 +3422,297 @@ public List deleteClassificationPropagation(String entityGuid, String cl } AtlasClassification classification = entityRetriever.toAtlasClassification(classificationVertex); - List entityVertices = deleteDelegate.getHandler().removeTagPropagation(classificationVertex); - deleteDelegate.getHandler().deleteClassificationVertex(classificationVertex, true); - if (CollectionUtils.isEmpty(entityVertices)) { + + List propagatedEdges = getPropagatedEdges(classificationVertex); + if (propagatedEdges.isEmpty()) { + LOG.warn("deleteClassificationPropagation(classificationVertexId={}): classification edges empty", classificationVertexId); + return null; } - List impactedGuids = entityVertices.stream().map(x -> GraphHelper.getGuid(x)).collect(Collectors.toList()); - GraphTransactionInterceptor.lockObjectAndReleasePostCommit(impactedGuids); + int propagatedEdgesSize = propagatedEdges.size(); - List propagatedEntities = updateClassificationText(classification, entityVertices); + LOG.info(String.format("Number of edges to be deleted : %s for classification vertex with id : %s", propagatedEdgesSize, classificationVertexId)); - entityChangeNotifier.onClassificationsDeletedFromEntities(propagatedEntities, Collections.singletonList(classification)); + List deletedPropagationsGuid = processClassificationEdgeDeletionInChunk(classification, propagatedEdges); + + deleteDelegate.getHandler().deleteClassificationVertex(classificationVertex, true); + + transactionInterceptHelper.intercept(); - return propagatedEntities.stream().map(x -> x.getGuid()).collect(Collectors.toList()); + return deletedPropagationsGuid; } catch (Exception e) { + LOG.error("Error while removing classification id {} with error {} ", classificationVertexId, e.getMessage()); throw new AtlasBaseException(e); } } + public void deleteClassificationOnlyPropagation(Set deletedEdgeIds) throws AtlasBaseException { + RequestContext.get().getDeletedEdgesIds().clear(); + RequestContext.get().getDeletedEdgesIds().addAll(deletedEdgeIds); + + for (AtlasEdge edge : deletedEdgeIds.stream().map(x -> graph.getEdge(x)).collect(Collectors.toList())) { + + boolean isRelationshipEdge = deleteDelegate.getHandler().isRelationshipEdge(edge); + String relationshipGuid = GraphHelper.getRelationshipGuid(edge); + + if (edge == null || !isRelationshipEdge) { + continue; + } + + List currentClassificationVertices = getPropagatableClassifications(edge); + + for (AtlasVertex currentClassificationVertex : currentClassificationVertices) { + LOG.info("Starting Classification {} Removal for deletion of edge {}",currentClassificationVertex.getIdForDisplay(), edge.getIdForDisplay()); + boolean isTermEntityEdge = isTermEntityEdge(edge); + boolean removePropagationOnEntityDelete = getRemovePropagations(currentClassificationVertex); + + if (!(isTermEntityEdge || removePropagationOnEntityDelete)) { + LOG.debug("This edge is not term edge or remove propagation isn't enabled"); + continue; + } + + processClassificationDeleteOnlyPropagation(currentClassificationVertex, relationshipGuid); + LOG.info("Finished Classification {} Removal for deletion of edge {}",currentClassificationVertex.getIdForDisplay(), edge.getIdForDisplay()); + } + } + } + + public void deleteClassificationOnlyPropagation(String deletedEdgeId, String classificationVertexId) throws AtlasBaseException { + RequestContext.get().getDeletedEdgesIds().clear(); + RequestContext.get().getDeletedEdgesIds().add(deletedEdgeId); + + AtlasEdge edge = graph.getEdge(deletedEdgeId); + + boolean isRelationshipEdge = deleteDelegate.getHandler().isRelationshipEdge(edge); + String relationshipGuid = GraphHelper.getRelationshipGuid(edge); + + if (edge == null || !isRelationshipEdge) { + return; + } + + AtlasVertex currentClassificationVertex = graph.getVertex(classificationVertexId); + if (currentClassificationVertex == null) { + LOG.warn("Classification Vertex with ID {} is not present or Deleted", classificationVertexId); + return; + } + + List currentClassificationVertices = getPropagatableClassifications(edge); + if (! currentClassificationVertices.contains(currentClassificationVertex)) { + return; + } + + boolean isTermEntityEdge = isTermEntityEdge(edge); + boolean removePropagationOnEntityDelete = getRemovePropagations(currentClassificationVertex); + + if (!(isTermEntityEdge || removePropagationOnEntityDelete)) { + LOG.debug("This edge is not term edge or remove propagation isn't enabled"); + return; + } + + processClassificationDeleteOnlyPropagation(currentClassificationVertex, relationshipGuid); + + LOG.info("Finished Classification {} Removal for deletion of edge {}",currentClassificationVertex.getIdForDisplay(), edge.getIdForDisplay()); + } + + public void deleteClassificationOnlyPropagation(String classificationId, String referenceVertexId, boolean isTermEntityEdge) throws AtlasBaseException { + AtlasVertex classificationVertex = graph.getVertex(classificationId); + AtlasVertex referenceVertex = graph.getVertex(referenceVertexId); + + if (classificationVertex == null) { + LOG.warn("Classification Vertex with ID {} is not present or Deleted", classificationId); + return; + } + /* + If reference vertex is deleted, we can consider that as this connected vertex was deleted + some other task was created before it to remove propagations. No need to execute this task. + */ + if (referenceVertex == null) { + LOG.warn("Reference Vertex {} is deleted", referenceVertexId); + return; + } + + if (!GraphHelper.propagatedClassificationAttachedToVertex(classificationVertex, referenceVertex)) { + LOG.warn("No Classification is attached to the reference vertex {} for classification {}", referenceVertexId, classificationId); + return; + } + + boolean removePropagationOnEntityDelete = getRemovePropagations(classificationVertex); + + if (!(isTermEntityEdge || removePropagationOnEntityDelete)) { + LOG.debug("This edge is not term edge or remove propagation isn't enabled"); + return; + } + + processClassificationDeleteOnlyPropagation(classificationVertex, null); + + LOG.info("Completed propagation removal via edge for classification {}", classificationId); + } + + public void classificationRefreshPropagation(String classificationId) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder classificationRefreshPropagationMetricRecorder = RequestContext.get().startMetricRecord("classificationRefreshPropagation"); + + AtlasVertex currentClassificationVertex = graph.getVertex(classificationId); + if (currentClassificationVertex == null) { + LOG.warn("Classification vertex with ID {} is deleted", classificationId); + return; + } + + String sourceEntityId = getClassificationEntityGuid(currentClassificationVertex); + AtlasVertex sourceEntityVertex = AtlasGraphUtilsV2.findByGuid(this.graph, sourceEntityId); + AtlasClassification classification = entityRetriever.toAtlasClassification(currentClassificationVertex); + + String propagationMode = CLASSIFICATION_PROPAGATION_MODE_DEFAULT; + + Boolean restrictPropagationThroughLineage = AtlasGraphUtilsV2.getProperty(currentClassificationVertex, CLASSIFICATION_VERTEX_RESTRICT_PROPAGATE_THROUGH_LINEAGE, Boolean.class); + + if (restrictPropagationThroughLineage != null && restrictPropagationThroughLineage) { + propagationMode = CLASSIFICATION_PROPAGATION_MODE_RESTRICT_LINEAGE; + } + + List propagatedVerticesIds = GraphHelper.getPropagatedVerticesIds(currentClassificationVertex); + LOG.info("{} entity vertices have classification with id {} attached", propagatedVerticesIds.size(), classificationId); + + List verticesIdsToAddClassification = new ArrayList<>(); + List propagatedVerticesIdWithoutEdge = entityRetriever.getImpactedVerticesIdsClassificationAttached(sourceEntityVertex , classificationId, + CLASSIFICATION_PROPAGATION_EXCLUSION_MAP.get(propagationMode), verticesIdsToAddClassification); + + LOG.info("To add classification with id {} to {} vertices", classificationId, verticesIdsToAddClassification.size()); + + List verticesIdsToRemove = (List)CollectionUtils.subtract(propagatedVerticesIds, propagatedVerticesIdWithoutEdge); + + List verticesToRemove = verticesIdsToRemove.stream() + .map(x -> graph.getVertex(x)) + .filter(vertex -> vertex != null) + .collect(Collectors.toList()); + + List verticesToAddClassification = verticesIdsToAddClassification.stream() + .map(x -> graph.getVertex(x)) + .filter(vertex -> vertex != null) + .collect(Collectors.toList()); + + //Remove classifications from unreachable vertices + processPropagatedClassificationDeletionFromVertices(verticesToRemove, currentClassificationVertex, classification); + + //Add classification to the reachable vertices + if (CollectionUtils.isEmpty(verticesToAddClassification)) { + LOG.debug("propagateClassification(entityGuid={}, classificationVertexId={}): found no entities to propagate the classification", sourceEntityId, classificationId); + return; + } + processClassificationPropagationAddition(verticesToAddClassification, currentClassificationVertex); + + LOG.info("Completed refreshing propagation for classification with vertex id {} with classification name {} and source entity {}",classificationId, + classification.getTypeName(), classification.getEntityGuid()); + + RequestContext.get().endMetricRecord(classificationRefreshPropagationMetricRecorder); + } + + private void processClassificationDeleteOnlyPropagation(AtlasVertex currentClassificationVertex, String relationshipGuid) throws AtlasBaseException { + String classificationId = currentClassificationVertex.getIdForDisplay(); + String sourceEntityId = getClassificationEntityGuid(currentClassificationVertex); + AtlasVertex sourceEntityVertex = AtlasGraphUtilsV2.findByGuid(this.graph, sourceEntityId); + AtlasClassification classification = entityRetriever.toAtlasClassification(currentClassificationVertex); + + String propagationMode = CLASSIFICATION_PROPAGATION_MODE_DEFAULT; + + Boolean restrictPropagationThroughLineage = AtlasGraphUtilsV2.getProperty(currentClassificationVertex, CLASSIFICATION_VERTEX_RESTRICT_PROPAGATE_THROUGH_LINEAGE, Boolean.class); + + if (restrictPropagationThroughLineage != null && restrictPropagationThroughLineage) { + propagationMode = CLASSIFICATION_PROPAGATION_MODE_RESTRICT_LINEAGE; + } + + List propagatedVerticesIds = GraphHelper.getPropagatedVerticesIds(currentClassificationVertex); + LOG.info("Traversed {} vertices including edge with relationship GUID {} for classification vertex {}", propagatedVerticesIds.size(), relationshipGuid, classificationId); + + List propagatedVerticesIdWithoutEdge = entityRetriever.getImpactedVerticesIds(sourceEntityVertex, relationshipGuid , classificationId, + CLASSIFICATION_PROPAGATION_EXCLUSION_MAP.get(propagationMode)); + + LOG.info("Traversed {} vertices except edge with relationship GUID {} for classification vertex {}", propagatedVerticesIdWithoutEdge.size(), relationshipGuid, classificationId); + + List verticesIdsToRemove = (List)CollectionUtils.subtract(propagatedVerticesIds, propagatedVerticesIdWithoutEdge); + + List verticesToRemove = verticesIdsToRemove.stream() + .map(x -> graph.getVertex(x)) + .filter(vertex -> vertex != null) + .collect(Collectors.toList()); + + propagatedVerticesIdWithoutEdge.clear(); + propagatedVerticesIds.clear(); + + LOG.info("To delete classification from {} vertices for deletion of edge with relationship GUID {} and classification {}", verticesToRemove.size(), relationshipGuid, classificationId); + + processPropagatedClassificationDeletionFromVertices(verticesToRemove, currentClassificationVertex, classification); + + LOG.info("Completed remove propagation for edge with relationship GUID {} and classification vertex {} with classification name {} and source entity {}", relationshipGuid, + classificationId, classification.getTypeName(), classification.getEntityGuid()); + } + + private void processPropagatedClassificationDeletionFromVertices(List VerticesToRemoveTag, AtlasVertex classificationVertex, AtlasClassification classification) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder propagatedClassificationDeletionMetricRecorder = RequestContext.get().startMetricRecord("processPropagatedClassificationDeletionFromVertices"); + + int propagatedVerticesSize = VerticesToRemoveTag.size(); + int toIndex; + int offset = 0; + + LOG.info("To delete classification of vertex id {} from {} entity vertices", classificationVertex.getIdForDisplay(), propagatedVerticesSize); + + try { + do { + toIndex = ((offset + CHUNK_SIZE > propagatedVerticesSize) ? propagatedVerticesSize : (offset + CHUNK_SIZE)); + List verticesChunkToRemoveTag = VerticesToRemoveTag.subList(offset, toIndex); + + List impactedGuids = verticesChunkToRemoveTag.stream() + .map(entityVertex -> GraphHelper.getGuid(entityVertex)) + .collect(Collectors.toList()); + GraphTransactionInterceptor.lockObjectAndReleasePostCommit(impactedGuids); + + List updatedVertices = deleteDelegate.getHandler().removeTagPropagation(classificationVertex, verticesChunkToRemoveTag); + List updatedEntities = updateClassificationText(classification, updatedVertices); + entityChangeNotifier.onClassificationsDeletedFromEntities(updatedEntities, Collections.singletonList(classification)); + + offset += CHUNK_SIZE; + + transactionInterceptHelper.intercept(); + + } while (offset < propagatedVerticesSize); + } catch (AtlasBaseException exception) { + LOG.error("Error while removing classification from vertices with classification vertex id {}", classificationVertex.getIdForDisplay()); + throw exception; + } finally { + RequestContext.get().endMetricRecord(propagatedClassificationDeletionMetricRecorder); + } + } + + List processClassificationEdgeDeletionInChunk(AtlasClassification classification, List propagatedEdges) throws AtlasBaseException { + List deletedPropagationsGuid = new ArrayList<>(); + int propagatedEdgesSize = propagatedEdges.size(); + int toIndex; + int offset = 0; + + do { + toIndex = ((offset + CHUNK_SIZE > propagatedEdgesSize) ? propagatedEdgesSize : (offset + CHUNK_SIZE)); + + List entityVertices = deleteDelegate.getHandler().removeTagPropagation(classification, propagatedEdges.subList(offset, toIndex)); + List impactedGuids = entityVertices.stream().map(x -> GraphHelper.getGuid(x)).collect(Collectors.toList()); + + GraphTransactionInterceptor.lockObjectAndReleasePostCommit(impactedGuids); + + List propagatedEntities = updateClassificationText(classification, entityVertices); + + entityChangeNotifier.onClassificationsDeletedFromEntities(propagatedEntities, Collections.singletonList(classification)); + if(! propagatedEntities.isEmpty()) { + deletedPropagationsGuid.addAll(propagatedEntities.stream().map(x -> x.getGuid()).collect(Collectors.toList())); + } + + offset += CHUNK_SIZE; + + transactionInterceptHelper.intercept(); + + } while (offset < propagatedEdgesSize); + + return deletedPropagationsGuid; + } + @GraphTransaction public void updateTagPropagations(String relationshipEdgeId, AtlasRelationship relationship) throws AtlasBaseException { AtlasEdge relationshipEdge = graph.getEdge(relationshipEdgeId); @@ -3229,6 +3851,57 @@ private Set getRelatedEntitiesGuids(AtlasEntity entity) { return relGuidsSet; } + private void validateBusinessAttributes(AtlasVertex entityVertex, AtlasEntityType entityType, Map> businessAttributes, boolean isOverwrite) throws AtlasBaseException { + List messages = new ArrayList<>(); + + Map> entityTypeBusinessMetadata = entityType.getBusinessAttributes(); + + for (String bmName : businessAttributes.keySet()) { + if (!entityTypeBusinessMetadata.containsKey(bmName)) { + messages.add(bmName + ": invalid business-metadata for entity type " + entityType.getTypeName()); + + continue; + } + + Map entityTypeBusinessAttributes = entityTypeBusinessMetadata.get(bmName); + Map entityBusinessAttributes = businessAttributes.get(bmName); + + for (AtlasBusinessAttribute bmAttribute : entityTypeBusinessAttributes.values()) { + AtlasType attrType = bmAttribute.getAttributeType(); + String attrName = bmAttribute.getName(); + Object attrValue = entityBusinessAttributes == null ? null : entityBusinessAttributes.get(attrName); + String fieldName = entityType.getTypeName() + "." + bmName + "." + attrName; + + if (attrValue != null) { + attrType.validateValue(attrValue, fieldName, messages); + boolean isValidLength = bmAttribute.isValidLength(attrValue); + if (!isValidLength) { + messages.add(fieldName + ": Business attribute-value exceeds maximum length limit"); + } + + } else if (!bmAttribute.getAttributeDef().getIsOptional()) { + final boolean isAttrValuePresent; + + if (isOverwrite) { + isAttrValuePresent = false; + } else { + Object existingValue = AtlasGraphUtilsV2.getEncodedProperty(entityVertex, bmAttribute.getVertexPropertyName(), Object.class); + + isAttrValuePresent = existingValue != null; + } + + if (!isAttrValuePresent) { + messages.add(fieldName + ": mandatory business-metadata attribute value missing in type " + entityType.getTypeName()); + } + } + } + } + + if (!messages.isEmpty()) { + throw new AtlasBaseException(AtlasErrorCode.INSTANCE_CRUD_INVALID_PARAMS, messages); + } + } + public static void validateCustomAttributes(AtlasEntity entity) throws AtlasBaseException { Map customAttributes = entity.getCustomAttributes(); @@ -3276,21 +3949,37 @@ public static void validateLabels(Set labels) throws AtlasBaseException private List updateClassificationText(AtlasClassification classification, Collection propagatedVertices) throws AtlasBaseException { List propagatedEntities = new ArrayList<>(); + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("updateClassificationText"); if(CollectionUtils.isNotEmpty(propagatedVertices)) { for(AtlasVertex vertex : propagatedVertices) { - AtlasEntity entity = instanceConverter.getAndCacheEntity(graphHelper.getGuid(vertex), ENTITY_CHANGE_NOTIFY_IGNORE_RELATIONSHIP_ATTRIBUTES); + AtlasEntity entity = null; + for (int i = 1; i <= MAX_NUMBER_OF_RETRIES; i++) { + try { + entity = instanceConverter.getAndCacheEntity(graphHelper.getGuid(vertex), ENTITY_CHANGE_NOTIFY_IGNORE_RELATIONSHIP_ATTRIBUTES); + break; //do not retry on success + } catch (AtlasBaseException ex) { + if (i == MAX_NUMBER_OF_RETRIES) { + LOG.error(String.format("Maximum retries reached for fetching vertex with id %s from graph. Retried %s times. Skipping...", vertex.getId(), i)); + continue; + } + LOG.warn(String.format("Vertex with id %s could not be fetched from graph. Retrying for %s time", vertex.getId(), i)); + } + } if (isActive(entity)) { String classificationTextForEntity = fullTextMapperV2.getClassificationTextForEntity(entity); vertex.setProperty(CLASSIFICATION_TEXT_KEY, classificationTextForEntity); propagatedEntities.add(entity); - LOG.info("updateClassificationText: {}: {}", classification.getTypeName(), classificationTextForEntity); + if (LOG.isDebugEnabled()) { + LOG.debug("updateClassificationText: {}: {}", classification.getTypeName(), classificationTextForEntity); + } } } } + RequestContext.get().endMetricRecord(metricRecorder); return propagatedEntities; } @@ -3325,8 +4014,13 @@ private void addToUpdatedBusinessAttributes(Map> upd attributes.put(bmAttribute.getName(), attrValue); } - private void createAndQueueTask(String taskType, AtlasVertex entityVertex, String classificationVertexId) { - deleteDelegate.getHandler().createAndQueueTask(taskType, entityVertex, classificationVertexId, null); + private void createAndQueueTask(String taskType, AtlasVertex entityVertex, String classificationVertexId, Boolean currentPropagateThroughLineage) throws AtlasBaseException{ + + deleteDelegate.getHandler().createAndQueueTaskWithoutCheck(taskType, entityVertex, classificationVertexId, null, currentPropagateThroughLineage); + } + + private void createAndQueueTask(String taskType, AtlasVertex entityVertex, String classificationVertexId) throws AtlasBaseException { + deleteDelegate.getHandler().createAndQueueTaskWithoutCheck(taskType, entityVertex, classificationVertexId, null); } public void removePendingTaskFromEntity(String entityGuid, String taskGuid) throws EntityNotFoundException { @@ -3360,4 +4054,47 @@ public void removePendingTaskFromEdge(String edgeId, String taskGuid) throws Atl AtlasGraphUtilsV2.removeItemFromListProperty(edge, EDGE_PENDING_TASKS_PROPERTY_KEY, taskGuid); } + + + public void addHasLineage(Set inputOutputEdges, boolean isRestoreEntity) { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("addHasLineage"); + + for (AtlasEdge atlasEdge : inputOutputEdges) { + + boolean isOutputEdge = PROCESS_OUTPUTS.equals(atlasEdge.getLabel()); + + AtlasVertex processVertex = atlasEdge.getOutVertex(); + AtlasVertex assetVertex = atlasEdge.getInVertex(); + + if (getEntityHasLineage(processVertex)) { + AtlasGraphUtilsV2.setEncodedProperty(assetVertex, HAS_LINEAGE, true); + continue; + } + + String oppositeEdgeLabel = isOutputEdge ? PROCESS_INPUTS : PROCESS_OUTPUTS; + + Iterator oppositeEdges = processVertex.getEdges(AtlasEdgeDirection.BOTH, oppositeEdgeLabel).iterator(); + boolean isHasLineageSet = false; + while (oppositeEdges.hasNext()) { + AtlasEdge oppositeEdge = oppositeEdges.next(); + AtlasVertex oppositeEdgeAssetVertex = oppositeEdge.getInVertex(); + + if (getStatus(oppositeEdge) == ACTIVE && getStatus(oppositeEdgeAssetVertex) == ACTIVE) { + if (!isHasLineageSet) { + AtlasGraphUtilsV2.setEncodedProperty(assetVertex, HAS_LINEAGE, true); + AtlasGraphUtilsV2.setEncodedProperty(processVertex, HAS_LINEAGE, true); + isHasLineageSet = true; + } + + if (isRestoreEntity) { + AtlasGraphUtilsV2.setEncodedProperty(oppositeEdgeAssetVertex, HAS_LINEAGE, true); + } else { + break; + } + } + } + } + RequestContext.get().endMetricRecord(metricRecorder); + } + } \ No newline at end of file diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index dd79b26f302..d279136a10f 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -18,6 +18,7 @@ package org.apache.atlas.repository.store.graph.v2; import com.fasterxml.jackson.core.type.TypeReference; +import com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.atlas.AtlasConfiguration; import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.RequestContext; @@ -61,6 +62,10 @@ import org.apache.atlas.type.AtlasTypeUtil; import org.apache.atlas.utils.AtlasEntityUtil; import org.apache.atlas.utils.AtlasJson; +<<<<<<< HEAD +======= +import org.apache.atlas.utils.AtlasPerfMetrics; +>>>>>>> 7f3c811fb15361ba82bfb4edd7a8393798863ff0 import org.apache.atlas.v1.model.instance.Id; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; @@ -86,6 +91,10 @@ import java.util.Objects; import java.util.Queue; import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadFactory; import java.util.stream.Collectors; import static org.apache.atlas.glossary.GlossaryUtils.TERM_ASSIGNMENT_ATTR_CONFIDENCE; @@ -111,13 +120,11 @@ import static org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags.NONE; import static org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags.ONE_TO_TWO; import static org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags.TWO_TO_ONE; -import static org.apache.atlas.repository.Constants.CLASSIFICATION_ENTITY_GUID; -import static org.apache.atlas.repository.Constants.CLASSIFICATION_LABEL; -import static org.apache.atlas.repository.Constants.CLASSIFICATION_VALIDITY_PERIODS_KEY; -import static org.apache.atlas.repository.Constants.TERM_ASSIGNMENT_LABEL; +import static org.apache.atlas.repository.Constants.*; import static org.apache.atlas.repository.graph.GraphHelper.*; import static org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.getIdFromVertex; import static org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.isReference; +import static org.apache.atlas.repository.store.graph.v2.tasks.ClassificationPropagateTaskFactory.CLASSIFICATION_ONLY_PROPAGATION_DELETE; import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection; import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.BOTH; import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.IN; @@ -376,6 +383,7 @@ public AtlasClassification toAtlasClassification(AtlasVertex classificationVerte ret.setEntityStatus(getClassificationEntityStatus(classificationVertex)); ret.setPropagate(isPropagationEnabled(classificationVertex)); ret.setRemovePropagationsOnEntityDelete(getRemovePropagations(classificationVertex)); + ret.setRestrictPropagationThroughLineage(getRestrictPropagationThroughLineage(classificationVertex)); String strValidityPeriods = AtlasGraphUtilsV2.getEncodedProperty(classificationVertex, CLASSIFICATION_VALIDITY_PERIODS_KEY, String.class); @@ -510,11 +518,12 @@ public void evaluateClassificationPropagation(AtlasVertex classificationVertex, } } - public Map> getClassificationPropagatedEntitiesMapping(List classificationVertices) { + public Map> getClassificationPropagatedEntitiesMapping(List classificationVertices) throws AtlasBaseException{ return getClassificationPropagatedEntitiesMapping(classificationVertices, null); } - public Map> getClassificationPropagatedEntitiesMapping(List classificationVertices, String relationshipGuidToExclude) { + public Map> getClassificationPropagatedEntitiesMapping(List classificationVertices, String relationshipGuidToExclude) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("getClassificationPropagatedEntitiesMapping"); Map> ret = new HashMap<>(); if (CollectionUtils.isNotEmpty(classificationVertices)) { @@ -522,23 +531,42 @@ public Map> getClassificationPropagatedEntitiesMa String classificationId = classificationVertex.getIdForDisplay(); String sourceEntityId = getClassificationEntityGuid(classificationVertex); AtlasVertex sourceEntityVertex = AtlasGraphUtilsV2.findByGuid(this.graph, sourceEntityId); - List entitiesPropagatingTo = getImpactedVerticesV2(sourceEntityVertex, relationshipGuidToExclude, classificationId); + String propagationMode = CLASSIFICATION_PROPAGATION_MODE_DEFAULT; + + Boolean restrictPropagationThroughLineage = AtlasGraphUtilsV2.getProperty(classificationVertex, CLASSIFICATION_VERTEX_RESTRICT_PROPAGATE_THROUGH_LINEAGE, Boolean.class); + + if (restrictPropagationThroughLineage != null && restrictPropagationThroughLineage) { + propagationMode = CLASSIFICATION_PROPAGATION_MODE_RESTRICT_LINEAGE; + } + + List entitiesPropagatingTo = getImpactedVerticesV2(sourceEntityVertex, relationshipGuidToExclude, + classificationId, CLASSIFICATION_PROPAGATION_EXCLUSION_MAP.get(propagationMode)); + + LOG.info("Traversed {} vertices for Classification vertex id {} excluding RelationShip GUID {}", entitiesPropagatingTo.size(), classificationId, relationshipGuidToExclude); ret.put(classificationVertex, entitiesPropagatingTo); } } + RequestContext.get().endMetricRecord(metricRecorder); return ret; } public List getImpactedVerticesV2(AtlasVertex entityVertex) { - return getImpactedVerticesV2(entityVertex, null); + return getImpactedVerticesV2(entityVertex, (List) null); + } + + public List getImpactedVerticesV2(AtlasVertex entityVertex, List edgeLabelsToExclude){ + List ret = new ArrayList<>(); + traverseImpactedVertices(entityVertex, null, null, ret, edgeLabelsToExclude); + + return ret; } public List getImpactedVerticesV2(AtlasVertex entityVertex, String relationshipGuidToExclude) { List ret = new ArrayList<>(); - traverseImpactedVertices(entityVertex, relationshipGuidToExclude, null, ret); + traverseImpactedVertices(entityVertex, relationshipGuidToExclude, null, ret, null); return ret; } @@ -550,7 +578,18 @@ public List getIncludedImpactedVerticesV2(AtlasVertex entityVertex, public List getIncludedImpactedVerticesV2(AtlasVertex entityVertex, String relationshipGuidToExclude, String classificationId) { List ret = new ArrayList<>(Arrays.asList(entityVertex)); - traverseImpactedVertices(entityVertex, relationshipGuidToExclude, classificationId, ret); + traverseImpactedVertices(entityVertex, relationshipGuidToExclude, classificationId, ret, null); + + return ret; + } + public List getIncludedImpactedVerticesV2(AtlasVertex entityVertex, String relationshipGuidToExclude, String classificationId, List edgeLabelsToExclude) { + List vertexIds = new ArrayList<>(); + traverseImpactedVerticesByLevel(entityVertex, relationshipGuidToExclude, classificationId, vertexIds, edgeLabelsToExclude, null); + + List ret = vertexIds.stream().map(x -> graph.getVertex(x)) + .filter(vertex -> vertex != null) + .collect(Collectors.toList()); + ret.add(entityVertex); return ret; } @@ -558,16 +597,54 @@ public List getIncludedImpactedVerticesV2(AtlasVertex entityVertex, public List getImpactedVerticesV2(AtlasVertex entityVertex, String relationshipGuidToExclude, String classificationId) { List ret = new ArrayList<>(); - traverseImpactedVertices(entityVertex, relationshipGuidToExclude, classificationId, ret); + traverseImpactedVertices(entityVertex, relationshipGuidToExclude, classificationId, ret, null); + + return ret; + } + + public List getImpactedVerticesV2(AtlasVertex entityVertex, String relationshipGuidToExclude, String classificationId, List edgeLabelsToExclude) { + List ret = new ArrayList<>(); + + traverseImpactedVertices(entityVertex, relationshipGuidToExclude, classificationId, ret, edgeLabelsToExclude); + + return ret; + } + + + public List getImpactedVerticesIds(AtlasVertex entityVertex, String relationshipGuidToExclude, String classificationId, List edgeLabelsToExclude) { + List ret = new ArrayList<>(); + + traverseImpactedVerticesByLevel(entityVertex, relationshipGuidToExclude, classificationId, ret, edgeLabelsToExclude, null); return ret; } + public List getImpactedVerticesIdsClassificationAttached(AtlasVertex entityVertex, String classificationId, List edgeLabelsToExclude, List verticesWithoutClassification) { + List ret = new ArrayList<>(); + + GraphHelper.getClassificationEdges(entityVertex).forEach(classificationEdge -> { + AtlasVertex classificationVertex = classificationEdge.getInVertex(); + if (classificationVertex != null && classificationId.equals(classificationVertex.getIdForDisplay())) { + traverseImpactedVerticesByLevel(entityVertex, null, classificationId, ret, edgeLabelsToExclude, verticesWithoutClassification); + } + }); + + return ret; + } + + + private void traverseImpactedVertices(final AtlasVertex entityVertexStart, final String relationshipGuidToExclude, - final String classificationId, final List result) { + final String classificationId, final List result, List edgeLabelsToExclude) { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("traverseImpactedVertices"); Set visitedVertices = new HashSet<>(); - Queue queue = new ArrayDeque() {{ add(entityVertexStart); }}; + Queue queue = new ArrayDeque<>(); Map resultsMap = new HashMap<>(); + RequestContext requestContext = RequestContext.get(); + + if (entityVertexStart != null) { + queue.add(entityVertexStart); + } while (!queue.isEmpty()) { AtlasVertex entityVertex = queue.poll(); @@ -588,10 +665,19 @@ private void traverseImpactedVertices(final AtlasVertex entityVertexStart, final continue; } - Iterable propagationEdges = entityVertex.getEdges(AtlasEdgeDirection.BOTH, tagPropagationEdges); + if (edgeLabelsToExclude != null && !edgeLabelsToExclude.isEmpty()) { + tagPropagationEdges = Arrays.stream(tagPropagationEdges) + .filter(x -> !edgeLabelsToExclude.contains(x)) + .collect(Collectors.toList()) + .toArray(new String[0]); + } + + Iterator propagationEdges = entityVertex.getEdges(AtlasEdgeDirection.BOTH, tagPropagationEdges).iterator(); + + while (propagationEdges.hasNext()) { + AtlasEdge propagationEdge = propagationEdges.next(); - for (AtlasEdge propagationEdge : propagationEdges) { - if (getEdgeStatus(propagationEdge) != ACTIVE) { + if (getEdgeStatus(propagationEdge) != ACTIVE && !(requestContext.getCurrentTask() != null && requestContext.getDeletedEdgesIds().contains(propagationEdge.getIdForDisplay())) ) { continue; } @@ -635,6 +721,146 @@ private void traverseImpactedVertices(final AtlasVertex entityVertexStart, final } result.addAll(resultsMap.values()); + RequestContext.get().endMetricRecord(metricRecorder); + } + + private void traverseImpactedVerticesByLevel(final AtlasVertex entityVertexStart, final String relationshipGuidToExclude, + final String classificationId, final List result, List edgeLabelsToExclude, List verticesWithoutClassification) { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("traverseImpactedVerticesByLevel"); + Set visitedVerticesIds = new HashSet<>(); + Set verticesAtCurrentLevel = new HashSet<>(); + Set traversedVerticesIds = new HashSet<>(); + Set verticesWithOutClassification = new HashSet<>(); + RequestContext requestContext = RequestContext.get(); + AtlasVertex classificationVertex = graph.getVertex(classificationId); + boolean storeVerticesWithoutClassification = verticesWithoutClassification == null ? false : true; + + final ThreadFactory threadFactory = new ThreadFactoryBuilder() + .setNameFormat("Tasks-BFS-%d") + .setDaemon(true) + .build(); + + ExecutorService executorService = Executors.newFixedThreadPool(AtlasConfiguration.GRAPH_TRAVERSAL_PARALLELISM.getInt(), threadFactory); + + //Add Source vertex to level 1 + if (entityVertexStart != null) { + verticesAtCurrentLevel.add(entityVertexStart.getIdForDisplay()); + } + /* + Steps in each level: + 1. Add vertices to Visited Vertices set + 2. Then fetch adjacent vertices of that vertex using getAdjacentVerticesIds + 3. Use future to fetch it later as it is a Blocking call,so we want to execute it asynchronously + 4. Then fetch the result from futures + 5. It will return the adjacent vertices of the current level + After processing current level: + 1. Add the adjacent vertices of current level to verticesToVisitNextLevel + 2. Clear the processed vertices of current level + 3. After that insert verticesToVisitNextLevel into current level + Continue the steps until all vertices are processed by checking if verticesAtCurrentLevel is empty + */ + try { + while (!verticesAtCurrentLevel.isEmpty()) { + Set verticesToVisitNextLevel = new HashSet<>(); + List>> futures = verticesAtCurrentLevel.stream() + .map(t -> { + AtlasVertex entityVertex = graph.getVertex(t); + visitedVerticesIds.add(entityVertex.getIdForDisplay()); + // If we want to store vertices without classification attached + // Check if vertices has classification attached or not using function isClassificationAttached + + if(storeVerticesWithoutClassification && !GraphHelper.isClassificationAttached(entityVertex, classificationVertex)) { + verticesWithOutClassification.add(entityVertex.getIdForDisplay()); + } + + return CompletableFuture.supplyAsync(() -> getAdjacentVerticesIds(entityVertex, classificationId, + relationshipGuidToExclude, edgeLabelsToExclude, visitedVerticesIds), executorService); + }).collect(Collectors.toList()); + + futures.stream().map(CompletableFuture::join).forEach(x -> { + verticesToVisitNextLevel.addAll(x); + traversedVerticesIds.addAll(x); + }); + + verticesAtCurrentLevel.clear(); + verticesAtCurrentLevel.addAll(verticesToVisitNextLevel); + } + } finally { + executorService.shutdown(); + } + result.addAll(traversedVerticesIds); + + if(storeVerticesWithoutClassification) + verticesWithoutClassification.addAll(verticesWithOutClassification); + + requestContext.endMetricRecord(metricRecorder); + } + + private Set getAdjacentVerticesIds(AtlasVertex entityVertex,final String classificationId, final String relationshipGuidToExclude + ,List edgeLabelsToExclude, Set visitedVerticesIds) { + + AtlasEntityType entityType = typeRegistry.getEntityTypeByName(getTypeName(entityVertex)); + String[] tagPropagationEdges = entityType != null ? entityType.getTagPropagationEdgesArray() : null; + Set ret = new HashSet<>(); + RequestContext requestContext = RequestContext.get(); + + if (tagPropagationEdges == null) { + return null; + } + + if (edgeLabelsToExclude != null && !edgeLabelsToExclude.isEmpty()) { + tagPropagationEdges = Arrays.stream(tagPropagationEdges) + .filter(x -> !edgeLabelsToExclude.contains(x)) + .collect(Collectors.toList()) + .toArray(new String[0]); + } + + Iterator propagationEdges = entityVertex.getEdges(AtlasEdgeDirection.BOTH, tagPropagationEdges).iterator(); + + while (propagationEdges.hasNext()) { + AtlasEdge propagationEdge = propagationEdges.next(); + + if (getEdgeStatus(propagationEdge) != ACTIVE && !(requestContext.getCurrentTask() != null && requestContext.getDeletedEdgesIds().contains(propagationEdge.getIdForDisplay())) ) { + continue; + } + + PropagateTags tagPropagation = getPropagateTags(propagationEdge); + + if (tagPropagation == null || tagPropagation == NONE) { + continue; + } else if (tagPropagation == TWO_TO_ONE) { + if (isOutVertex(entityVertex, propagationEdge)) { + continue; + } + } else if (tagPropagation == ONE_TO_TWO) { + if (!isOutVertex(entityVertex, propagationEdge)) { + continue; + } + } + + if (relationshipGuidToExclude != null) { + if (StringUtils.equals(getRelationshipGuid(propagationEdge), relationshipGuidToExclude)) { + continue; + } + } + + if (classificationId != null) { + List blockedClassificationIds = getBlockedClassificationIds(propagationEdge); + + if (CollectionUtils.isNotEmpty(blockedClassificationIds) && blockedClassificationIds.contains(classificationId)) { + continue; + } + } + + AtlasVertex adjacentVertex = getOtherVertex(propagationEdge, entityVertex); + String adjacentVertexIdForDisplay = adjacentVertex.getIdForDisplay(); + + if (!visitedVerticesIds.contains(adjacentVertexIdForDisplay)) { + ret.add(adjacentVertexIdForDisplay); + } + } + + return ret; } private boolean isOutVertex(AtlasVertex vertex, AtlasEdge edge) { @@ -648,7 +874,7 @@ private AtlasVertex getOtherVertex(AtlasEdge edge, AtlasVertex vertex) { return StringUtils.equals(outVertex.getIdForDisplay(), vertex.getIdForDisplay()) ? inVertex : outVertex; } - private AtlasVertex getEntityVertex(AtlasObjectId objId) throws AtlasBaseException { + public AtlasVertex getEntityVertex(AtlasObjectId objId) throws AtlasBaseException { AtlasVertex ret = null; if (! AtlasTypeUtil.isValid(objId)) { @@ -750,6 +976,7 @@ private AtlasEntityHeader mapVertexToAtlasEntityHeader(AtlasVertex entityVertex) } private AtlasEntityHeader mapVertexToAtlasEntityHeader(AtlasVertex entityVertex, Set attributes) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("mapVertexToAtlasEntityHeader"); AtlasEntityHeader ret = new AtlasEntityHeader(); String typeName = entityVertex.getProperty(Constants.TYPE_NAME_PROPERTY_KEY, String.class); @@ -759,7 +986,9 @@ private AtlasEntityHeader mapVertexToAtlasEntityHeader(AtlasVertex entityVertex, ret.setTypeName(typeName); ret.setGuid(guid); ret.setStatus(GraphHelper.getStatus(entityVertex)); - ret.setClassificationNames(getAllTraitNames(entityVertex)); + if(RequestContext.get().includeClassifications()){ + ret.setClassificationNames(getAllTraitNames(entityVertex)); + } ret.setIsIncomplete(isIncomplete); ret.setLabels(getLabels(entityVertex)); @@ -768,10 +997,13 @@ private AtlasEntityHeader mapVertexToAtlasEntityHeader(AtlasVertex entityVertex, ret.setCreateTime(new Date(GraphHelper.getCreatedTime(entityVertex))); ret.setUpdateTime(new Date(GraphHelper.getModifiedTime(entityVertex))); - List termAssignmentHeaders = mapAssignedTerms(entityVertex); - ret.setMeanings(termAssignmentHeaders); - ret.setMeaningNames(termAssignmentHeaders.stream().map(AtlasTermAssignmentHeader::getDisplayText).collect(Collectors.toList())); - + if(RequestContext.get().includeMeanings()) { + List termAssignmentHeaders = mapAssignedTerms(entityVertex); + ret.setMeanings(termAssignmentHeaders); + ret.setMeaningNames( + termAssignmentHeaders.stream().map(AtlasTermAssignmentHeader::getDisplayText) + .collect(Collectors.toList())); + } AtlasEntityType entityType = typeRegistry.getEntityTypeByName(typeName); if (entityType != null) { @@ -815,7 +1047,7 @@ private AtlasEntityHeader mapVertexToAtlasEntityHeader(AtlasVertex entityVertex, } } } - + RequestContext.get().endMetricRecord(metricRecorder); return ret; } @@ -830,7 +1062,9 @@ private String toNonQualifiedName(String attrName) { return ret; } - private AtlasEntity mapSystemAttributes(AtlasVertex entityVertex, AtlasEntity entity) { + public AtlasEntity mapSystemAttributes(AtlasVertex entityVertex, AtlasEntity entity) { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("mapSystemAttributes"); + if (LOG.isDebugEnabled()) { LOG.debug("Mapping system attributes for type {}", entity.getTypeName()); } @@ -862,6 +1096,7 @@ private AtlasEntity mapSystemAttributes(AtlasVertex entityVertex, AtlasEntity en LOG.warn("Got exception while mapping system attributes for type {} : ", entity.getTypeName(), t); } + RequestContext.get().endMetricRecord(metricRecorder); return entity; } @@ -874,6 +1109,7 @@ private void mapAttributes(AtlasVertex entityVertex, AtlasStruct struct, AtlasEn } private void mapAttributes(AtlasVertex entityVertex, AtlasStruct struct, AtlasEntityExtInfo entityExtInfo, boolean isMinExtInfo, boolean includeReferences) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("mapAttributes"); AtlasType objType = typeRegistry.getType(struct.getTypeName()); if (!(objType instanceof AtlasStructType)) { @@ -887,6 +1123,7 @@ private void mapAttributes(AtlasVertex entityVertex, AtlasStruct struct, AtlasEn struct.setAttribute(attribute.getName(), attrValue); } + RequestContext.get().endMetricRecord(metricRecorder); } private void mapBusinessAttributes(AtlasVertex entityVertex, AtlasEntity entity) throws AtlasBaseException { @@ -894,6 +1131,11 @@ private void mapBusinessAttributes(AtlasVertex entityVertex, AtlasEntity entity) } public List getAllClassifications(AtlasVertex entityVertex) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("getAllClassifications"); + + if(LOG.isDebugEnabled()){ + LOG.debug("Performing getAllClassifications"); + } List ret = new ArrayList<>(); Iterable edges = entityVertex.query().direction(AtlasEdgeDirection.OUT).label(CLASSIFICATION_LABEL).edges(); @@ -911,6 +1153,7 @@ public List getAllClassifications(AtlasVertex entityVertex) } } + RequestContext.get().endMetricRecord(metricRecorder); return ret; } @@ -989,7 +1232,8 @@ private AtlasTermAssignmentHeader toTermAssignmentHeader(final AtlasEdge edge) { return ret; } - private void mapClassifications(AtlasVertex entityVertex, AtlasEntity entity) throws AtlasBaseException { + public void mapClassifications(AtlasVertex entityVertex, AtlasEntity entity) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("mapClassifications"); List edges = getAllClassificationEdges(entityVertex); if (CollectionUtils.isNotEmpty(edges)) { @@ -1006,6 +1250,7 @@ private void mapClassifications(AtlasVertex entityVertex, AtlasEntity entity) th entity.setClassifications(allClassifications); } + RequestContext.get().endMetricRecord(metricRecorder); } private Object mapVertexToAttribute(AtlasVertex entityVertex, AtlasAttribute attribute, AtlasEntityExtInfo entityExtInfo, final boolean isMinExtInfo) throws AtlasBaseException { @@ -1035,10 +1280,15 @@ private Object mapVertexToAttribute(AtlasVertex entityVertex, AtlasAttribute att ret = AtlasGraphUtilsV2.getEncodedProperty(entityVertex, attribute.getVertexPropertyName(), Object.class); break; case STRUCT: + edgeLabel = AtlasGraphUtilsV2.getEdgeLabel(attribute.getName()); ret = mapVertexToStruct(entityVertex, edgeLabel, null, entityExtInfo, isMinExtInfo); break; case OBJECT_ID_TYPE: if (includeReferences) { + if (attribute.getDefinedInType().getTypeCategory() == TypeCategory.STRUCT) { + //Struct attribute having ObjectId as type + edgeLabel = AtlasGraphUtilsV2.getEdgeLabel(attribute.getName()); + } ret = attribute.getAttributeDef().isSoftReferenced() ? mapVertexToObjectIdForSoftRef(entityVertex, attribute, entityExtInfo, isMinExtInfo) : mapVertexToObjectId(entityVertex, edgeLabel, null, entityExtInfo, isOwnedAttribute, edgeDirection, isMinExtInfo); } else { @@ -1389,7 +1639,7 @@ private AtlasStruct mapVertexToStruct(AtlasVertex entityVertex, String edgeLabel return ret; } - private Object getVertexAttribute(AtlasVertex vertex, AtlasAttribute attribute) throws AtlasBaseException { + public Object getVertexAttribute(AtlasVertex vertex, AtlasAttribute attribute) throws AtlasBaseException { return vertex != null && attribute != null ? mapVertexToAttribute(vertex, attribute, null, false) : null; } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/IAtlasEntityChangeNotifier.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/IAtlasEntityChangeNotifier.java index c0c22396d01..d82f30bf35b 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/IAtlasEntityChangeNotifier.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/IAtlasEntityChangeNotifier.java @@ -37,7 +37,7 @@ public interface IAtlasEntityChangeNotifier { void onClassificationAddedToEntity(AtlasEntity entity, List addedClassifications) throws AtlasBaseException; - void onClassificationsAddedToEntities(List entities, List addedClassifications) throws AtlasBaseException; + void onClassificationsAddedToEntities(List entities, List addedClassifications, boolean forceInline) throws AtlasBaseException; void onClassificationDeletedFromEntity(AtlasEntity entity, List deletedClassifications) throws AtlasBaseException; diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/TransactionInterceptHelper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/TransactionInterceptHelper.java new file mode 100644 index 00000000000..eee481bd4ea --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/TransactionInterceptHelper.java @@ -0,0 +1,16 @@ +package org.apache.atlas.repository.store.graph.v2; + +import org.apache.atlas.annotation.GraphTransaction; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +@Component +public class TransactionInterceptHelper { + private static final Logger LOG = LoggerFactory.getLogger(TransactionInterceptHelper.class); + + public TransactionInterceptHelper(){} + + @GraphTransaction + public void intercept(){} +} diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/EntityChangeNotifierNop.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/EntityChangeNotifierNop.java index 0e548da9dc5..b8af91218e9 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/EntityChangeNotifierNop.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/EntityChangeNotifierNop.java @@ -48,7 +48,7 @@ public void onClassificationAddedToEntity(AtlasEntity entity, List entities, List addedClassifications) throws AtlasBaseException { + public void onClassificationsAddedToEntities(List entities, List addedClassifications, boolean forceInline) throws AtlasBaseException { } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/GuidMutationResponsePair.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/GuidMutationResponsePair.java new file mode 100644 index 00000000000..67975809948 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/GuidMutationResponsePair.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.repository.store.graph.v2.bulkimport; + +import org.apache.atlas.model.instance.EntityMutationResponse; +import org.apache.atlas.v1.typesystem.types.utils.TypesUtil; + +public class GuidMutationResponsePair extends TypesUtil.Pair { + + public GuidMutationResponsePair(String guid, EntityMutationResponse response) { + super(guid, response); + } + + public String getGuid() { + return left; + } + + public EntityMutationResponse getMutationResponse() { + return right; + } +} \ No newline at end of file diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/MigrationImport.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/MigrationImport.java index cbe70320b27..0cc7c4a318b 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/MigrationImport.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/MigrationImport.java @@ -31,12 +31,7 @@ import org.apache.atlas.repository.store.graph.AtlasRelationshipStore; import org.apache.atlas.repository.store.graph.v1.DeleteHandlerDelegate; import org.apache.atlas.repository.store.graph.v1.RestoreHandlerV1; -import org.apache.atlas.repository.store.graph.v2.AtlasEntityStoreV2; -import org.apache.atlas.repository.store.graph.v2.AtlasRelationshipStoreV2; -import org.apache.atlas.repository.store.graph.v2.EntityGraphMapper; -import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; -import org.apache.atlas.repository.store.graph.v2.EntityImportStream; -import org.apache.atlas.repository.store.graph.v2.IAtlasEntityChangeNotifier; +import org.apache.atlas.repository.store.graph.v2.*; import org.apache.atlas.repository.store.graph.v2.bulkimport.pc.EntityConsumerBuilder; import org.apache.atlas.repository.store.graph.v2.bulkimport.pc.EntityCreationManager; import org.apache.atlas.type.AtlasTypeRegistry; @@ -49,11 +44,14 @@ public class MigrationImport extends ImportStrategy { private final AtlasGraph graph; private final AtlasGraphProvider graphProvider; private final AtlasTypeRegistry typeRegistry; + private final AtlasEntityChangeNotifier entityChangeNotifier; - public MigrationImport(AtlasGraph graph, AtlasGraphProvider graphProvider, AtlasTypeRegistry typeRegistry) { + public MigrationImport(AtlasGraph graph, AtlasGraphProvider graphProvider, + AtlasTypeRegistry typeRegistry, AtlasEntityChangeNotifier entityChangeNotifier) { this.graph = graph; this.graphProvider = graphProvider; this.typeRegistry = typeRegistry; + this.entityChangeNotifier = entityChangeNotifier; LOG.info("MigrationImport: Using bulkLoading..."); } @@ -112,7 +110,7 @@ private EntityCreationManager createEntityCreationManager(AtlasImportResult impo entityStoreBulk, entityGraphRetrieverBulk, batchSize); LOG.info("MigrationImport: EntityCreationManager: Created!"); - return new EntityCreationManager(consumerBuilder, batchSize, numWorkers, importResult, dataMigrationStatusService); + return new EntityCreationManager(consumerBuilder, batchSize, numWorkers, importResult, entityChangeNotifier, dataMigrationStatusService); } private static int getNumWorkers(int numWorkersFromOptions) { @@ -123,16 +121,15 @@ private static int getNumWorkers(int numWorkersFromOptions) { private AtlasEntityStoreV2 createEntityStore(AtlasGraph graph, AtlasTypeRegistry typeRegistry) { FullTextMapperV2Nop fullTextMapperV2 = new FullTextMapperV2Nop(); - IAtlasEntityChangeNotifier entityChangeNotifier = new EntityChangeNotifierNop(); DeleteHandlerDelegate deleteDelegate = new DeleteHandlerDelegate(graph, typeRegistry, null); RestoreHandlerV1 restoreHandlerV1 = new RestoreHandlerV1(graph, typeRegistry); - AtlasFormatConverters formatConverters = new AtlasFormatConverters(typeRegistry); - AtlasInstanceConverter instanceConverter = new AtlasInstanceConverter(graph, typeRegistry, formatConverters); AtlasRelationshipStore relationshipStore = new AtlasRelationshipStoreV2(graph, typeRegistry, deleteDelegate, entityChangeNotifier); - EntityGraphMapper entityGraphMapper = new EntityGraphMapper(deleteDelegate, restoreHandlerV1, typeRegistry, graph, relationshipStore, entityChangeNotifier, instanceConverter, fullTextMapperV2, null); + EntityGraphMapper entityGraphMapper = new EntityGraphMapper(deleteDelegate, restoreHandlerV1, typeRegistry, + graph, relationshipStore, entityChangeNotifier, getInstanceConverter(graph), fullTextMapperV2, null, null); + AtlasRelationshipStoreV2 atlasRelationshipStoreV2 = new AtlasRelationshipStoreV2(graph, typeRegistry, deleteDelegate, entityChangeNotifier); - return new AtlasEntityStoreV2(graph, deleteDelegate, restoreHandlerV1, typeRegistry, entityChangeNotifier, entityGraphMapper); + return new AtlasEntityStoreV2(graph, deleteDelegate, restoreHandlerV1, typeRegistry, entityChangeNotifier, entityGraphMapper, null, atlasRelationshipStoreV2, null); } private void shutdownEntityCreationManager(EntityCreationManager creationManager) { @@ -142,4 +139,9 @@ private void shutdownEntityCreationManager(EntityCreationManager creationManager LOG.error("Migration Import: Shutdown: Interrupted!", e); } } + + private AtlasInstanceConverter getInstanceConverter(AtlasGraph graph) { + AtlasFormatConverters formatConverters = new AtlasFormatConverters(typeRegistry); + return new AtlasInstanceConverter(graph, typeRegistry, formatConverters); + } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/pc/EntityConsumer.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/pc/EntityConsumer.java index b73988fd738..a610aa5d288 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/pc/EntityConsumer.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/pc/EntityConsumer.java @@ -21,7 +21,6 @@ import org.apache.atlas.RequestContext; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.instance.AtlasEntity; -import org.apache.atlas.model.instance.AtlasEntityHeader; import org.apache.atlas.model.instance.EntityMutationResponse; import org.apache.atlas.pc.WorkItemConsumer; import org.apache.atlas.repository.graphdb.AtlasGraph; @@ -33,7 +32,6 @@ import org.apache.atlas.repository.store.graph.v2.EntityStream; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.utils.AtlasPerfMetrics; -import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/pc/EntityCreationManager.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/pc/EntityCreationManager.java index 734add6d7c4..b242b1a5b9c 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/pc/EntityCreationManager.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/pc/EntityCreationManager.java @@ -26,6 +26,7 @@ import org.apache.atlas.repository.migration.DataMigrationStatusService; import org.apache.atlas.repository.store.graph.v2.BulkImporterImpl; import org.apache.atlas.repository.store.graph.v2.EntityImportStream; +import org.apache.atlas.repository.store.graph.v2.IAtlasEntityChangeNotifier; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -41,11 +42,16 @@ public class EntityCreationManager extends WorkItemManag private String currentTypeName; private float currentPercent; private EntityImportStream entityImportStream; + private final IAtlasEntityChangeNotifier entityChangeNotifier; - public EntityCreationManager(WorkItemBuilder builder, int batchSize, int numWorkers, AtlasImportResult importResult, DataMigrationStatusService dataMigrationStatusService) { + public EntityCreationManager(WorkItemBuilder builder, int batchSize, int numWorkers, + AtlasImportResult importResult, + IAtlasEntityChangeNotifier entityChangeNotifier, + DataMigrationStatusService dataMigrationStatusService) { super(builder, WORKER_PREFIX, batchSize, numWorkers, true); this.importResult = importResult; this.dataMigrationStatusService = dataMigrationStatusService; + this.entityChangeNotifier = entityChangeNotifier; this.statusReporter = new StatusReporter<>(STATUS_REPORT_TIMEOUT_DURATION); } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AuthPolicyPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AuthPolicyPreProcessor.java new file mode 100644 index 00000000000..5149568f1a5 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AuthPolicyPreProcessor.java @@ -0,0 +1,315 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.repository.store.graph.v2.preprocessor; + + +import org.apache.atlas.AtlasErrorCode; +import org.apache.atlas.RequestContext; +import org.apache.atlas.authorize.AtlasAuthorizationUtils; +import org.apache.atlas.authorize.AtlasEntityAccessRequest; +import org.apache.atlas.authorize.AtlasPrivilege; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.featureflag.FeatureFlagStore; +import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo; +import org.apache.atlas.model.instance.AtlasEntityHeader; +import org.apache.atlas.model.instance.AtlasObjectId; +import org.apache.atlas.model.instance.AtlasStruct; +import org.apache.atlas.model.instance.EntityMutations.EntityOperation; +import org.apache.atlas.repository.graphdb.AtlasGraph; +import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.repository.store.aliasstore.ESAliasStore; +import org.apache.atlas.repository.store.aliasstore.IndexAliasStore; +import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; +import org.apache.atlas.repository.store.graph.v2.EntityMutationContext; +import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.atlas.utils.AtlasPerfMetrics; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.apache.atlas.AtlasErrorCode.BAD_REQUEST; +import static org.apache.atlas.AtlasErrorCode.INSTANCE_BY_UNIQUE_ATTRIBUTE_NOT_FOUND; +import static org.apache.atlas.AtlasErrorCode.INSTANCE_GUID_NOT_FOUND; +import static org.apache.atlas.AtlasErrorCode.RESOURCE_NOT_FOUND; +import static org.apache.atlas.AtlasErrorCode.UNAUTHORIZED_CONNECTION_ADMIN; +import static org.apache.atlas.authorize.AtlasAuthorizationUtils.getCurrentUserName; +import static org.apache.atlas.authorize.AtlasAuthorizationUtils.verifyAccess; +import static org.apache.atlas.model.instance.EntityMutations.EntityOperation.CREATE; +import static org.apache.atlas.model.instance.EntityMutations.EntityOperation.UPDATE; +import static org.apache.atlas.repository.Constants.ATTR_ADMIN_ROLES; +import static org.apache.atlas.repository.Constants.KEYCLOAK_ROLE_ADMIN; +import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; +import static org.apache.atlas.repository.util.AccessControlUtils.*; +import static org.apache.atlas.repository.util.AccessControlUtils.getPolicySubCategory; + +public class AuthPolicyPreProcessor implements PreProcessor { + private static final Logger LOG = LoggerFactory.getLogger(AuthPolicyPreProcessor.class); + + private final AtlasGraph graph; + private final AtlasTypeRegistry typeRegistry; + private final EntityGraphRetriever entityRetriever; + private final FeatureFlagStore featureFlagStore ; + private IndexAliasStore aliasStore; + + public AuthPolicyPreProcessor(AtlasGraph graph, + AtlasTypeRegistry typeRegistry, + EntityGraphRetriever entityRetriever, + FeatureFlagStore featureFlagStore) { + this.graph = graph; + this.typeRegistry = typeRegistry; + this.entityRetriever = entityRetriever; + this.featureFlagStore = featureFlagStore; + + aliasStore = new ESAliasStore(graph, entityRetriever); + } + + @Override + public void processAttributes(AtlasStruct entityStruct, EntityMutationContext context, + EntityOperation operation) throws AtlasBaseException { + if (LOG.isDebugEnabled()) { + LOG.debug("AuthPolicyPreProcessor.processAttributes: pre processing {}, {}", entityStruct.getAttribute(QUALIFIED_NAME), operation); + } + + AtlasEntity entity = (AtlasEntity) entityStruct; + + switch (operation) { + case CREATE: + processCreatePolicy(entity); + break; + case UPDATE: + processUpdatePolicy(entity, context.getVertex(entity.getGuid())); + break; + } + } + + private void processCreatePolicy(AtlasStruct entity) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processCreatePolicy"); + AtlasEntity policy = (AtlasEntity) entity; + + String policyCategory = getPolicyCategory(policy); + if (StringUtils.isEmpty(policyCategory)) { + throw new AtlasBaseException(BAD_REQUEST, "Please provide attribute " + ATTR_POLICY_CATEGORY); + } + + entity.setAttribute(ATTR_POLICY_IS_ENABLED, entity.getAttributes().getOrDefault(ATTR_POLICY_IS_ENABLED, true)); + + AuthPolicyValidator validator = new AuthPolicyValidator(entityRetriever); + if (POLICY_CATEGORY_PERSONA.equals(policyCategory)) { + AtlasEntityWithExtInfo parent = getAccessControlEntity(policy); + AtlasEntity parentEntity = parent.getEntity(); + + validator.validate(policy, null, parentEntity, CREATE); + validateConnectionAdmin(policy); + + policy.setAttribute(QUALIFIED_NAME, String.format("%s/%s", getEntityQualifiedName(parentEntity), getUUID())); + + //extract role + String roleName = getPersonaRoleName(parentEntity); + List roles = Arrays.asList(roleName); + policy.setAttribute(ATTR_POLICY_ROLES, roles); + + policy.setAttribute(ATTR_POLICY_USERS, new ArrayList<>()); + policy.setAttribute(ATTR_POLICY_GROUPS, new ArrayList<>()); + + + //create ES alias + aliasStore.updateAlias(parent, policy); + + } else if (POLICY_CATEGORY_PURPOSE.equals(policyCategory)) { + AtlasEntityWithExtInfo parent = getAccessControlEntity(policy); + AtlasEntity parentEntity = parent.getEntity(); + + policy.setAttribute(QUALIFIED_NAME, String.format("%s/%s", getEntityQualifiedName(parentEntity), getUUID())); + + validator.validate(policy, null, parentEntity, CREATE); + + //extract tags + List purposeTags = getPurposeTags(parentEntity); + + List policyResources = purposeTags.stream().map(x -> "tag:" + x).collect(Collectors.toList()); + + policy.setAttribute(ATTR_POLICY_RESOURCES, policyResources); + + //create ES alias + aliasStore.updateAlias(parent, policy); + + } else { + validator.validate(policy, null, null, CREATE); + } + + RequestContext.get().endMetricRecord(metricRecorder); + } + + private void processUpdatePolicy(AtlasStruct entity, AtlasVertex vertex) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processUpdatePolicy"); + AtlasEntity policy = (AtlasEntity) entity; + AtlasEntity existingPolicy = entityRetriever.toAtlasEntityWithExtInfo(vertex).getEntity(); + + String policyCategory = policy.hasAttribute(ATTR_POLICY_CATEGORY) ? getPolicyCategory(policy) : getPolicyCategory(existingPolicy); + + AuthPolicyValidator validator = new AuthPolicyValidator(entityRetriever); + if (POLICY_CATEGORY_PERSONA.equals(policyCategory)) { + AtlasEntityWithExtInfo parent = getAccessControlEntity(policy); + AtlasEntity parentEntity = parent.getEntity(); + + validator.validate(policy, existingPolicy, parentEntity, UPDATE); + validateConnectionAdmin(policy); + + String qName = getEntityQualifiedName(existingPolicy); + policy.setAttribute(QUALIFIED_NAME, qName); + + //extract role + String roleName = getPersonaRoleName(parentEntity); + List roles = Arrays.asList(roleName); + + policy.setAttribute(ATTR_POLICY_ROLES, roles); + + policy.setAttribute(ATTR_POLICY_USERS, new ArrayList<>()); + policy.setAttribute(ATTR_POLICY_GROUPS, new ArrayList<>()); + + + //create ES alias + parent.addReferredEntity(policy); + aliasStore.updateAlias(parent, null); + + } else if (POLICY_CATEGORY_PURPOSE.equals(policyCategory)) { + + AtlasEntityWithExtInfo parent = getAccessControlEntity(policy); + AtlasEntity parentEntity = parent.getEntity(); + + validator.validate(policy, existingPolicy, parentEntity, UPDATE); + + String qName = getEntityQualifiedName(existingPolicy); + policy.setAttribute(QUALIFIED_NAME, qName); + + //extract tags + List purposeTags = getPurposeTags(parentEntity); + + List policyResources = purposeTags.stream().map(x -> "tag:" + x).collect(Collectors.toList()); + + policy.setAttribute(ATTR_POLICY_RESOURCES, policyResources); + + //create ES alias + parent.addReferredEntity(policy); + } else { + validator.validate(policy, null, null, UPDATE); + } + + RequestContext.get().endMetricRecord(metricRecorder); + } + + @Override + public void processDelete(AtlasVertex vertex) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processDeletePolicy"); + + try { + AtlasEntity policy = entityRetriever.toAtlasEntity(vertex); + + authorizeDeleteAuthPolicy(policy); + + if(!policy.getStatus().equals(AtlasEntity.Status.ACTIVE)) { + LOG.info("Policy with guid {} is already deleted/purged", policy.getGuid()); + return; + } + + AtlasEntityWithExtInfo parent = getAccessControlEntity(policy); + if (parent != null) { + parent.getReferredEntity(policy.getGuid()).setStatus(AtlasEntity.Status.DELETED); + aliasStore.updateAlias(parent, null); + } + } finally { + RequestContext.get().endMetricRecord(metricRecorder); + } + } + + private void authorizeDeleteAuthPolicy(AtlasEntity policy) throws AtlasBaseException { + if (!RequestContext.get().isSkipAuthorizationCheck()) { + AtlasEntityAccessRequest request = new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_DELETE, new AtlasEntityHeader(policy)); + verifyAccess(request, "delete entity: guid=" + policy.getGuid()); + } + /* else, + * skip auth check + * */ + } + + private void validateConnectionAdmin(AtlasEntity policy) throws AtlasBaseException { + String subCategory = getPolicySubCategory(policy); + if (POLICY_SUB_CATEGORY_METADATA.equals(subCategory) || POLICY_SUB_CATEGORY_DATA.equals(subCategory)) { + //connectionAdmins check + + String connQn = getPolicyConnectionQN(policy); + AtlasEntity connection = getEntityByQualifiedName(entityRetriever, connQn); + if (connection == null) { + throw new AtlasBaseException(RESOURCE_NOT_FOUND, "Connection entity for policy"); + } + String connectionRoleName = String.format(CONN_NAME_PATTERN, connection.getGuid()); + + Set userRoles = AtlasAuthorizationUtils.getRolesForCurrentUser(); + + List connRoles = new ArrayList<>(0); + if (connection.hasAttribute(ATTR_ADMIN_ROLES)) { + connRoles = (List) connection.getAttribute(ATTR_ADMIN_ROLES); + } + + if (userRoles.contains(connectionRoleName) || (userRoles.contains(KEYCLOAK_ROLE_ADMIN) && connRoles.contains(KEYCLOAK_ROLE_ADMIN))) { + //valid connection admin + } else if (ARGO_SERVICE_USER_NAME.equals(RequestContext.getCurrentUser())) { + // Argo service user Valid Service user for connection admin while access control is disabled + //TODO: Remove this once we complete migration to Atlas AuthZ based access control + } else { + throw new AtlasBaseException(UNAUTHORIZED_CONNECTION_ADMIN, getCurrentUserName(), connection.getGuid()); + } + } + } + + private AtlasEntityWithExtInfo getAccessControlEntity(AtlasEntity entity) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("AuthPolicyPreProcessor.getAccessControl"); + AtlasEntityWithExtInfo ret = null; + + AtlasObjectId objectId = (AtlasObjectId) entity.getRelationshipAttribute(REL_ATTR_ACCESS_CONTROL); + if (objectId != null) { + try { + ret = entityRetriever.toAtlasEntityWithExtInfo(objectId); + } catch (AtlasBaseException abe) { + AtlasErrorCode code = abe.getAtlasErrorCode(); + + if (INSTANCE_BY_UNIQUE_ATTRIBUTE_NOT_FOUND != code && INSTANCE_GUID_NOT_FOUND != code) { + throw abe; + } + } + } + + if (ret != null) { + List policies = (List) ret.getEntity().getRelationshipAttribute(REL_ATTR_POLICIES); + + for (AtlasObjectId policy : policies) { + ret.addReferredEntity(entityRetriever.toAtlasEntity(policy)); + } + } + + RequestContext.get().endMetricRecord(metricRecorder); + return ret; + } +} diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AuthPolicyValidator.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AuthPolicyValidator.java new file mode 100644 index 00000000000..8a91b4af5fe --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AuthPolicyValidator.java @@ -0,0 +1,323 @@ +package org.apache.atlas.repository.store.graph.v2.preprocessor; + +import org.apache.atlas.RequestContext; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.model.instance.AtlasObjectId; +import org.apache.atlas.model.instance.EntityMutations; +import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.apache.atlas.AtlasErrorCode.BAD_REQUEST; +import static org.apache.atlas.AtlasErrorCode.OPERATION_NOT_SUPPORTED; +import static org.apache.atlas.authorize.AtlasPrivilege.ENTITY_ADD_CLASSIFICATION; +import static org.apache.atlas.authorize.AtlasPrivilege.ENTITY_CREATE; +import static org.apache.atlas.authorize.AtlasPrivilege.ENTITY_DELETE; +import static org.apache.atlas.authorize.AtlasPrivilege.ENTITY_READ; +import static org.apache.atlas.authorize.AtlasPrivilege.ENTITY_READ_CLASSIFICATION; +import static org.apache.atlas.authorize.AtlasPrivilege.ENTITY_REMOVE_CLASSIFICATION; +import static org.apache.atlas.authorize.AtlasPrivilege.ENTITY_UPDATE; +import static org.apache.atlas.authorize.AtlasPrivilege.ENTITY_UPDATE_BUSINESS_METADATA; +import static org.apache.atlas.authorize.AtlasPrivilege.ENTITY_UPDATE_CLASSIFICATION; +import static org.apache.atlas.model.instance.EntityMutations.EntityOperation.CREATE; +import static org.apache.atlas.repository.Constants.CONNECTION_ENTITY_TYPE; +import static org.apache.atlas.repository.Constants.PERSONA_ENTITY_TYPE; +import static org.apache.atlas.repository.Constants.PURPOSE_ENTITY_TYPE; +import static org.apache.atlas.repository.util.AccessControlUtils.*; + +public class AuthPolicyValidator { + private final EntityGraphRetriever entityRetriever; + + AuthPolicyValidator(EntityGraphRetriever entityRetriever) { + this.entityRetriever = entityRetriever; + } + + private static final Set PERSONA_POLICY_VALID_SUB_CATEGORIES = new HashSet(){{ + add(POLICY_SUB_CATEGORY_METADATA); + add(POLICY_SUB_CATEGORY_DATA); + add(POLICY_SUB_CATEGORY_GLOSSARY); + }}; + + private static final Set PURPOSE_POLICY_VALID_SUB_CATEGORIES = new HashSet(){{ + add(POLICY_SUB_CATEGORY_METADATA); + add(POLICY_SUB_CATEGORY_DATA); + }}; + + private static final Set PERSONA_METADATA_POLICY_ACTIONS = new HashSet(){{ + add("persona-asset-read"); + add("persona-asset-update"); + add("persona-api-create"); + add("persona-api-delete"); + add("persona-business-update-metadata"); + add("persona-entity-add-classification"); + add("persona-entity-update-classification"); + add("persona-entity-remove-classification"); + add("persona-add-terms"); + add("persona-remove-terms"); + }}; + + private static final Set DATA_POLICY_ACTIONS = new HashSet(){{ + add("select"); + }}; + + private static final Set PERSONA_GLOSSARY_POLICY_ACTIONS = new HashSet(){{ + add("persona-glossary-read"); + add("persona-glossary-update"); + add("persona-glossary-create"); + add("persona-glossary-delete"); + add("persona-glossary-update-custom-metadata"); + add("persona-glossary-add-classifications"); + add("persona-glossary-update-classifications"); + add("persona-glossary-delete-classifications"); + }}; + + private static final Map> PERSONA_POLICY_VALID_ACTIONS = new HashMap>(){{ + put(POLICY_SUB_CATEGORY_METADATA, PERSONA_METADATA_POLICY_ACTIONS); + put(POLICY_SUB_CATEGORY_DATA, DATA_POLICY_ACTIONS); + put(POLICY_SUB_CATEGORY_GLOSSARY, PERSONA_GLOSSARY_POLICY_ACTIONS); + }}; + + private static final Set PURPOSE_METADATA_POLICY_ACTIONS = new HashSet(){{ + add(ENTITY_READ.getType()); + add(ENTITY_CREATE.getType()); + add(ENTITY_UPDATE.getType()); + add(ENTITY_DELETE.getType()); + add(ENTITY_READ_CLASSIFICATION.getType()); + add(ENTITY_ADD_CLASSIFICATION.getType()); + add(ENTITY_UPDATE_CLASSIFICATION.getType()); + add(ENTITY_REMOVE_CLASSIFICATION.getType()); + add(ENTITY_UPDATE_BUSINESS_METADATA.getType()); + add("purpose-add-terms"); + add("purpose-remove-terms"); + }}; + + private static final Map> PURPOSE_POLICY_VALID_ACTIONS = new HashMap>(){{ + put(POLICY_SUB_CATEGORY_METADATA, PURPOSE_METADATA_POLICY_ACTIONS); + put(POLICY_SUB_CATEGORY_DATA, DATA_POLICY_ACTIONS); + }}; + + private static final Set PERSONA_POLICY_VALID_RESOURCE_KEYS = new HashSet() {{ + add("entity"); + add("entity-type"); + }}; + + public void validate(AtlasEntity policy, AtlasEntity existingPolicy, + AtlasEntity accessControl, EntityMutations.EntityOperation operation) throws AtlasBaseException { + String policyCategory = null; + + if (policy.hasAttribute(ATTR_POLICY_CATEGORY)) { + policyCategory = getPolicyCategory(policy); + } else if (existingPolicy != null) { + policyCategory = getPolicyCategory(existingPolicy); + } + + if (POLICY_CATEGORY_PERSONA.equals(policyCategory) || POLICY_CATEGORY_PURPOSE.equals(policyCategory)) { + + if (operation == CREATE) { + String policySubCategory = getPolicySubCategory(policy); + List policyActions = getPolicyActions(policy); + + validateParam(StringUtils.isEmpty(getPolicyServiceName(policy)), "Please provide attribute " + ATTR_POLICY_SERVICE_NAME); + + validateParam(StringUtils.isEmpty(getPolicyType(policy)), "Please provide attribute " + ATTR_POLICY_TYPE); + + validateParam(CollectionUtils.isEmpty(policyActions), "Please provide attribute " + ATTR_POLICY_ACTIONS); + + validateOperation (!AtlasEntity.Status.ACTIVE.equals(accessControl.getStatus()), accessControl.getTypeName() + " is not Active"); + + if (POLICY_CATEGORY_PERSONA.equals(policyCategory)) { + validateParam (!PERSONA_ENTITY_TYPE.equals(accessControl.getTypeName()), "Please provide Persona as accesscontrol"); + + validateParam (!PERSONA_POLICY_VALID_SUB_CATEGORIES.contains(policySubCategory), + "Please provide valid value for attribute " + ATTR_POLICY_SUB_CATEGORY + ":"+ PERSONA_POLICY_VALID_SUB_CATEGORIES); + + + List resources = getPolicyResources(policy); + validateParam (CollectionUtils.isEmpty(resources), "Please provide attribute " + ATTR_POLICY_RESOURCES); + + //validate persona policy resources keys + Set policyResourceKeys = resources.stream().map(x -> x.split(RESOURCES_SPLITTER)[0]).collect(Collectors.toSet()); + policyResourceKeys.removeAll(PERSONA_POLICY_VALID_RESOURCE_KEYS); + validateParam(CollectionUtils.isNotEmpty(policyResourceKeys), + "Please provide valid type of policy resources: " + PERSONA_POLICY_VALID_RESOURCE_KEYS); + + + if (POLICY_SUB_CATEGORY_DATA.equals(policySubCategory)) { + validateParam (!POLICY_RESOURCE_CATEGORY_PERSONA_ENTITY.equals(getPolicyResourceCategory(policy)), "Invalid resource category for Persona"); + validateConnection(getPolicyConnectionQN(policy), resources); + + } else if (POLICY_SUB_CATEGORY_METADATA.equals(policySubCategory)) { + validateParam(!POLICY_RESOURCE_CATEGORY_PERSONA_CUSTOM.equals(getPolicyResourceCategory(policy)), "Invalid resource category for Persona"); + validateConnection(getPolicyConnectionQN(policy), resources); + + } else if (POLICY_SUB_CATEGORY_GLOSSARY.equals(policySubCategory)) { + + validateParam(!POLICY_RESOURCE_CATEGORY_PERSONA_CUSTOM.equals(getPolicyResourceCategory(policy)), "Invalid resource category for Persona"); + } + + //validate persona policy actions + Set validActions = PERSONA_POLICY_VALID_ACTIONS.get(policySubCategory); + List copyOfActions = new ArrayList<>(policyActions); + copyOfActions.removeAll(validActions); + validateParam (CollectionUtils.isNotEmpty(copyOfActions), + "Please provide valid values for attribute " + ATTR_POLICY_ACTIONS + ": Invalid actions "+ copyOfActions); + } + + if (POLICY_CATEGORY_PURPOSE.equals(policyCategory)) { + validateParam (!PURPOSE_ENTITY_TYPE.equals(accessControl.getTypeName()), "Please provide Purpose as accesscontrol"); + + validateParam (!PURPOSE_POLICY_VALID_SUB_CATEGORIES.contains(policySubCategory), + "Please provide valid value for attribute " + ATTR_POLICY_SUB_CATEGORY + ":"+ PURPOSE_POLICY_VALID_SUB_CATEGORIES); + + validateParam (!POLICY_RESOURCE_CATEGORY_PURPOSE.equals(getPolicyResourceCategory(policy)), "Invalid resource category for Purpose"); + + //validateParam (CollectionUtils.isNotEmpty(getPolicyResources(policy)), "Provided unexpected attribute " + ATTR_POLICY_RESOURCES); + + //validate purpose policy actions + Set validActions = PURPOSE_POLICY_VALID_ACTIONS.get(policySubCategory); + List copyOfActions = new ArrayList<>(policyActions); + copyOfActions.removeAll(validActions); + validateParam (CollectionUtils.isNotEmpty(copyOfActions), + "Please provide valid values for attribute " + ATTR_POLICY_ACTIONS + ": Invalid actions "+ copyOfActions); + } + + } else { + + validateOperation (StringUtils.isNotEmpty(policyCategory) && !policyCategory.equals(getPolicyCategory(existingPolicy)), + ATTR_POLICY_CATEGORY + " change not Allowed"); + + validateParam(policy.hasAttribute(ATTR_POLICY_ACTIONS) && CollectionUtils.isEmpty(getPolicyActions(policy)), + "Please provide attribute " + ATTR_POLICY_ACTIONS); + + String newServiceName = getPolicyServiceName(policy); + validateOperation (StringUtils.isNotEmpty(newServiceName) && !newServiceName.equals(getPolicyServiceName(existingPolicy)), + ATTR_POLICY_SERVICE_NAME + " change not Allowed"); + + String newResourceCategory = getPolicyResourceCategory(policy); + validateOperation (StringUtils.isNotEmpty(newResourceCategory) && !newResourceCategory.equals(getPolicyResourceCategory(existingPolicy)), + ATTR_POLICY_RESOURCES_CATEGORY + " change not Allowed"); + + validateOperation (!AtlasEntity.Status.ACTIVE.equals(existingPolicy.getStatus()), "Policy is not Active"); + + if (policy.hasAttribute(ATTR_POLICY_SUB_CATEGORY)) { + String newSubCategory = getPolicySubCategory(policy); + validateOperation (!getPolicySubCategory(existingPolicy).equals(newSubCategory), "Policy sub category change not Allowed"); + } + + List policyActions = policy.hasAttribute(ATTR_POLICY_ACTIONS) ? getPolicyActions(policy) : getPolicyActions(existingPolicy); + String policySubCategory = policy.hasAttribute(ATTR_POLICY_SUB_CATEGORY) ? getPolicySubCategory(policy) : getPolicySubCategory(existingPolicy); + + if (POLICY_CATEGORY_PERSONA.equals(policyCategory)) { + List resources = getPolicyResources(policy); + validateParam (policy.hasAttribute(ATTR_POLICY_RESOURCES) && CollectionUtils.isEmpty(resources), + "Please provide attribute " + ATTR_POLICY_RESOURCES); + + //validate persona policy resources keys + Set policyResourceKeys = resources.stream().map(x -> x.split(RESOURCES_SPLITTER)[0]).collect(Collectors.toSet()); + policyResourceKeys.removeAll(PERSONA_POLICY_VALID_RESOURCE_KEYS); + validateParam(CollectionUtils.isNotEmpty(policyResourceKeys), + "Please provide valid policy resources" + PERSONA_POLICY_VALID_RESOURCE_KEYS); + + String newConnectionQn = getPolicyConnectionQN(policy); + if (POLICY_SUB_CATEGORY_METADATA.equals(policySubCategory) || POLICY_SUB_CATEGORY_DATA.equals(policySubCategory)) { + validateParam(StringUtils.isEmpty(newConnectionQn), "Please provide attribute " + ATTR_POLICY_CONNECTION_QN); + + String existingConnectionQn = getPolicyConnectionQN(existingPolicy); + validateOperation (StringUtils.isNotEmpty(existingConnectionQn) && !newConnectionQn.equals(existingConnectionQn), ATTR_POLICY_CONNECTION_QN + " change not Allowed"); + + validateEntityResources(newConnectionQn, resources); + } + + + //validate persona policy actions + Set validActions = PERSONA_POLICY_VALID_ACTIONS.get(policySubCategory); + List copyOfActions = new ArrayList<>(policyActions); + copyOfActions.removeAll(validActions); + validateParam (CollectionUtils.isNotEmpty(copyOfActions), + "Please provide valid values for attribute " + ATTR_POLICY_ACTIONS + ": Invalid actions "+ copyOfActions); + + validateParentUpdate(policy, existingPolicy); + } + + if (POLICY_CATEGORY_PURPOSE.equals(policyCategory)) { + //validateParam (policy.hasAttribute(ATTR_POLICY_RESOURCES) && CollectionUtils.isNotEmpty(getPolicyResources(policy)), + // "Provided unexpected attribute " + ATTR_POLICY_RESOURCES); + + //validate purpose policy actions + Set validActions = PURPOSE_POLICY_VALID_ACTIONS.get(policySubCategory); + List copyOfActions = new ArrayList<>(policyActions); + copyOfActions.removeAll(validActions); + validateParam (CollectionUtils.isNotEmpty(copyOfActions), + "Please provide valid values for attribute " + ATTR_POLICY_ACTIONS + ": Invalid actions "+ copyOfActions); + + validateParentUpdate(policy, existingPolicy); + } + } + + } else { + //only allow argo & backend + if (!RequestContext.get().isSkipAuthorizationCheck()) { + String userName = RequestContext.getCurrentUser(); + validateOperation (!ARGO_SERVICE_USER_NAME.equals(userName) && !BACKEND_SERVICE_USER_NAME.equals(userName), + "Create/Update AuthPolicy with policyCategory other than persona & purpose"); + } + } + } + + private void validateEntityResources(String connQn, List resources) throws AtlasBaseException { + List entityResources = getFilteredPolicyResources(resources, RESOURCES_ENTITY); + + for (String entity : entityResources) { + if (!entity.startsWith(connQn)) { + throw new AtlasBaseException(BAD_REQUEST, entity + " does not belong to connection " + connQn); + } + } + } + + private void validateConnection(String connQn, List resources) throws AtlasBaseException { + validateParam(StringUtils.isEmpty(connQn), "Please provide attribute " + ATTR_POLICY_CONNECTION_QN); + + validateEntityResources(connQn, resources); + + AtlasEntity connection = getEntityByQualifiedName(entityRetriever, connQn); + + validateParam(!CONNECTION_ENTITY_TYPE.equals(connection.getTypeName()), "Please provide valid connectionQualifiedName"); + } + + private void validateParentUpdate(AtlasEntity policy, AtlasEntity existingPolicy) throws AtlasBaseException { + + Object object = policy.getRelationshipAttribute(REL_ATTR_ACCESS_CONTROL); + if (object != null) { + AtlasObjectId atlasObjectId = (AtlasObjectId) object; + String newParentGuid = atlasObjectId.getGuid(); + + object = existingPolicy.getRelationshipAttribute(REL_ATTR_ACCESS_CONTROL); + if (object != null) { + atlasObjectId = (AtlasObjectId) object; + String existingParentGuid = atlasObjectId.getGuid(); + if (!newParentGuid.equals(existingParentGuid)) { + throw new AtlasBaseException(OPERATION_NOT_SUPPORTED, "Policy parent (accesscontrol) change is not Allowed"); + } + } + } + } + + private static void validateParam(boolean isInvalid, String errorMessage) throws AtlasBaseException { + if (isInvalid) + throw new AtlasBaseException(BAD_REQUEST, errorMessage); + } + + private static void validateOperation(boolean isInvalid, String errorMessage) throws AtlasBaseException { + if (isInvalid) + throw new AtlasBaseException(OPERATION_NOT_SUPPORTED, errorMessage); + } +} diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java new file mode 100644 index 00000000000..b994398c23b --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java @@ -0,0 +1,266 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.repository.store.graph.v2.preprocessor; + +import org.apache.atlas.DeleteType; +import org.apache.atlas.RequestContext; +import org.apache.atlas.discovery.EntityDiscoveryService; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.featureflag.FeatureFlagStore; +import org.apache.atlas.model.discovery.AtlasSearchResult; +import org.apache.atlas.model.discovery.IndexSearchParams; +import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo; +import org.apache.atlas.model.instance.AtlasEntityHeader; +import org.apache.atlas.model.instance.AtlasStruct; +import org.apache.atlas.model.instance.EntityMutations; +import org.apache.atlas.repository.graphdb.AtlasGraph; +import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.repository.store.graph.AtlasEntityStore; +import org.apache.atlas.repository.store.graph.v1.DeleteHandlerDelegate; +import org.apache.atlas.repository.store.graph.v1.SoftDeleteHandlerV1; +import org.apache.atlas.repository.store.graph.v2.AtlasEntityStream; +import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; +import org.apache.atlas.repository.store.graph.v2.EntityMutationContext; +import org.apache.atlas.repository.store.graph.v2.EntityStream; +import org.apache.atlas.repository.store.users.KeycloakStore; +import org.apache.atlas.transformer.PreProcessorPoliciesTransformer; +import org.apache.atlas.utils.AtlasPerfMetrics; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; +import org.keycloak.representations.idm.RoleRepresentation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.apache.atlas.authorize.AtlasAuthorizerFactory.ATLAS_AUTHORIZER_IMPL; +import static org.apache.atlas.authorize.AtlasAuthorizerFactory.CURRENT_AUTHORIZER_IMPL; +import static org.apache.atlas.keycloak.client.AtlasKeycloakClient.getKeycloakClient; +import static org.apache.atlas.repository.Constants.ATTR_ADMIN_GROUPS; +import static org.apache.atlas.repository.Constants.ATTR_ADMIN_ROLES; +import static org.apache.atlas.repository.Constants.ATTR_ADMIN_USERS; +import static org.apache.atlas.repository.Constants.CREATED_BY_KEY; +import static org.apache.atlas.repository.Constants.POLICY_ENTITY_TYPE; +import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; +import static org.apache.atlas.repository.util.AtlasEntityUtils.mapOf; + +public class ConnectionPreProcessor implements PreProcessor { + private static final Logger LOG = LoggerFactory.getLogger(ConnectionPreProcessor.class); + + public static final String CONN_NAME_PATTERN = "connection_admins_%s"; + + private final AtlasGraph graph; + private final EntityGraphRetriever entityRetriever; + private AtlasEntityStore entityStore; + private EntityDiscoveryService discovery; + private PreProcessorPoliciesTransformer transformer; + private FeatureFlagStore featureFlagStore; + private KeycloakStore keycloakStore; + private final DeleteHandlerDelegate deleteDelegate; + + public ConnectionPreProcessor(AtlasGraph graph, + EntityDiscoveryService discovery, + EntityGraphRetriever entityRetriever, + FeatureFlagStore featureFlagStore, + DeleteHandlerDelegate deleteDelegate, + AtlasEntityStore entityStore) { + this.graph = graph; + this.entityRetriever = entityRetriever; + this.entityStore = entityStore; + this.featureFlagStore = featureFlagStore; + this.discovery = discovery; + this.deleteDelegate = deleteDelegate; + transformer = new PreProcessorPoliciesTransformer(); + keycloakStore = new KeycloakStore(); + } + + @Override + public void processAttributes(AtlasStruct entityStruct, EntityMutationContext context, + EntityMutations.EntityOperation operation) throws AtlasBaseException { + if (LOG.isDebugEnabled()) { + LOG.debug("PurposePreProcessor.processAttributes: pre processing {}, {}", entityStruct.getAttribute(QUALIFIED_NAME), operation); + } + + AtlasEntity entity = (AtlasEntity) entityStruct; + + switch (operation) { + case CREATE: + processCreateConnection(entity); + break; + case UPDATE: + processUpdateConnection(context, entity); + break; + } + } + + private void processCreateConnection(AtlasStruct struct) throws AtlasBaseException { + if (ATLAS_AUTHORIZER_IMPL.equalsIgnoreCase(CURRENT_AUTHORIZER_IMPL)) { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processCreateConnection"); + + AtlasEntity connection = (AtlasEntity) struct; + + //create connection role + String roleName = String.format(CONN_NAME_PATTERN, connection.getGuid()); + + List adminUsers = (List) connection.getAttribute(ATTR_ADMIN_USERS); + List adminGroups = (List) connection.getAttribute(ATTR_ADMIN_GROUPS); + List adminRoles = (List) connection.getAttribute(ATTR_ADMIN_ROLES); + + if (adminUsers == null) { + adminUsers = new ArrayList<>(); + } + + String creatorUser = RequestContext.get().getUser(); + if (StringUtils.isNotEmpty(creatorUser) && !adminUsers.contains(creatorUser)) { + adminUsers.add(creatorUser); + } + connection.setAttribute(ATTR_ADMIN_USERS, adminUsers); + + RoleRepresentation role = keycloakStore.createRoleForConnection(roleName, true, adminUsers, adminGroups, adminRoles); + + //create connection bootstrap policies + AtlasEntitiesWithExtInfo policies = transformer.transform(connection); + + try { + RequestContext.get().setSkipAuthorizationCheck(true); + EntityStream entityStream = new AtlasEntityStream(policies); + entityStore.createOrUpdate(entityStream, false); + LOG.info("Created bootstrap policies for connection {}", connection.getAttribute(QUALIFIED_NAME)); + } finally { + RequestContext.get().setSkipAuthorizationCheck(false); + } + + RequestContext.get().endMetricRecord(metricRecorder); + } + } + + private void processUpdateConnection(EntityMutationContext context, + AtlasStruct entity) throws AtlasBaseException { + + AtlasEntity connection = (AtlasEntity) entity; + + if (ATLAS_AUTHORIZER_IMPL.equalsIgnoreCase(CURRENT_AUTHORIZER_IMPL)) { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processUpdateConnection"); + + AtlasVertex vertex = context.getVertex(connection.getGuid()); + AtlasEntity existingConnEntity = entityRetriever.toAtlasEntity(vertex); + + String roleName = String.format(CONN_NAME_PATTERN, connection.getGuid()); + + String vertexQName = vertex.getProperty(QUALIFIED_NAME, String.class); + entity.setAttribute(QUALIFIED_NAME, vertexQName); + + RoleRepresentation representation = getKeycloakClient().getRoleByName(roleName); + String creatorUser = vertex.getProperty(CREATED_BY_KEY, String.class); + + if (connection.hasAttribute(ATTR_ADMIN_USERS)) { + List newAdminUsers = (List) connection.getAttribute(ATTR_ADMIN_USERS); + List currentAdminUsers = (List) existingConnEntity.getAttribute(ATTR_ADMIN_USERS); + if (StringUtils.isNotEmpty(creatorUser) && !newAdminUsers.contains(creatorUser)) { + newAdminUsers.add(creatorUser); + } + + connection.setAttribute(ATTR_ADMIN_USERS, newAdminUsers); + if (CollectionUtils.isNotEmpty(newAdminUsers) || CollectionUtils.isNotEmpty(currentAdminUsers)) { + keycloakStore.updateRoleUsers(roleName, currentAdminUsers, newAdminUsers, representation); + } + } + + if (connection.hasAttribute(ATTR_ADMIN_GROUPS)) { + List newAdminGroups = (List) connection.getAttribute(ATTR_ADMIN_GROUPS); + List currentAdminGroups = (List) existingConnEntity.getAttribute(ATTR_ADMIN_GROUPS); + + if (CollectionUtils.isNotEmpty(newAdminGroups) || CollectionUtils.isNotEmpty(currentAdminGroups)) { + keycloakStore.updateRoleGroups(roleName, currentAdminGroups, newAdminGroups, representation); + } + } + + if (connection.hasAttribute(ATTR_ADMIN_ROLES)) { + List newAdminRoles = (List) connection.getAttribute(ATTR_ADMIN_ROLES); + List currentAdminRoles = (List) existingConnEntity.getAttribute(ATTR_ADMIN_ROLES); + + if (CollectionUtils.isNotEmpty(newAdminRoles) || CollectionUtils.isNotEmpty(currentAdminRoles)) { + keycloakStore.updateRoleRoles(roleName, currentAdminRoles, newAdminRoles, representation); + } + } + + RequestContext.get().endMetricRecord(metricRecorder); + } + } + + @Override + public void processDelete(AtlasVertex vertex) throws AtlasBaseException { + // Process Delete connection role and policies in case of hard delete or purge + if (isDeleteTypeSoft()) { + LOG.info("Skipping processDelete for connection as delete type is {}", RequestContext.get().getDeleteType()); + return; + } + + if (ATLAS_AUTHORIZER_IMPL.equalsIgnoreCase(CURRENT_AUTHORIZER_IMPL)) { + AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo = entityRetriever.toAtlasEntityWithExtInfo(vertex); + AtlasEntity connection = entityWithExtInfo.getEntity(); + String roleName = String.format(CONN_NAME_PATTERN, connection.getGuid()); + + boolean connectionIsDeleted = AtlasEntity.Status.DELETED.equals(connection.getStatus()); + + if (connectionIsDeleted && DeleteType.HARD.equals(RequestContext.get().getDeleteType())) { + LOG.warn("Connection {} is SOFT DELETED! Can't HARD DELETE connection role and policies", connection.getAttribute(QUALIFIED_NAME)); + return; + } + + //delete connection policies + List policies = getConnectionPolicies(connection.getGuid(), roleName); + entityStore.deleteByIds(policies.stream().map(x -> x.getGuid()).collect(Collectors.toList())); + + keycloakStore.removeRoleByName(roleName); + } + } + + private boolean isDeleteTypeSoft() { + return deleteDelegate.getHandler().getClass().equals(SoftDeleteHandlerV1.class); + } + + private List getConnectionPolicies(String guid, String roleName) throws AtlasBaseException { + List ret = new ArrayList<>(); + + IndexSearchParams indexSearchParams = new IndexSearchParams(); + Map dsl = new HashMap<>(); + + List mustClauseList = new ArrayList(); + mustClauseList.add(mapOf("term", mapOf("__typeName.keyword", POLICY_ENTITY_TYPE))); + mustClauseList.add(mapOf("wildcard", mapOf(QUALIFIED_NAME, guid + "/*"))); + mustClauseList.add(mapOf("term", mapOf("policyRoles", roleName))); + + dsl.put("query", mapOf("bool", mapOf("must", mustClauseList))); + + indexSearchParams.setDsl(dsl); + indexSearchParams.setSuppressLogs(true); + + AtlasSearchResult result = discovery.directIndexSearch(indexSearchParams); + if (result != null) { + ret = result.getEntities(); + } + + return ret; + } +} diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessor.java index 8ba44ae1506..0ed51910044 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessor.java @@ -3,10 +3,26 @@ import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.instance.AtlasStruct; import org.apache.atlas.model.instance.EntityMutations; +import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.store.graph.v2.EntityMutationContext; +import java.util.HashSet; +import java.util.Set; + +import static org.apache.atlas.repository.Constants.ATLAS_GLOSSARY_CATEGORY_ENTITY_TYPE; +import static org.apache.atlas.repository.Constants.ATLAS_GLOSSARY_TERM_ENTITY_TYPE; + public interface PreProcessor { + Set skipInitialAuthCheckTypes = new HashSet() {{ + add(ATLAS_GLOSSARY_TERM_ENTITY_TYPE); + add(ATLAS_GLOSSARY_CATEGORY_ENTITY_TYPE); + }}; + void processAttributes(AtlasStruct entity, EntityMutationContext context, EntityMutations.EntityOperation operation) throws AtlasBaseException; + + default void processDelete(AtlasVertex vertex) throws AtlasBaseException { + //override this method for implementation + } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java index e67ed2755ca..6c849004602 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java @@ -1,21 +1,49 @@ package org.apache.atlas.repository.store.graph.v2.preprocessor; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.model.instance.AtlasObjectId; +import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; +import org.apache.atlas.repository.store.graph.v2.EntityMutationContext; +import org.apache.atlas.type.AtlasEntityType; +import org.apache.atlas.type.AtlasStructType; +import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.util.NanoIdUtils; +import org.apache.atlas.utils.AtlasEntityUtil; import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.atlas.repository.Constants.QUERY_COLLECTION_ENTITY_TYPE; +import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; +import static org.apache.atlas.repository.Constants.ENTITY_TYPE_PROPERTY_KEY; public class PreProcessorUtils { + private static final Logger LOG = LoggerFactory.getLogger(PreProcessorUtils.class); private static final char[] invalidNameChars = {'@'}; //Glossary models constants public static final String ANCHOR = "anchor"; + public static final String CATEGORY_TERMS = "terms"; public static final String CATEGORY_PARENT = "parentCategory"; public static final String CATEGORY_CHILDREN = "childrenCategories"; + public static final String GLOSSARY_TERM_REL_TYPE = "AtlasGlossaryTermAnchor"; + public static final String GLOSSARY_CATEGORY_REL_TYPE = "AtlasGlossaryCategoryAnchor"; //Query models constants - public static final String PREFIX_QUERY_QN = "/default/collection/"; + public static final String PREFIX_QUERY_QN = "default/collection/"; public static final String COLLECTION_QUALIFIED_NAME = "collectionQualifiedName"; + public static final String PARENT_QUALIFIED_NAME = "parentQualifiedName"; + public static final String PARENT_ATTRIBUTE_NAME = "parent"; + + /** + * Folder,Collection, Query relations + */ + public static final String CHILDREN_QUERIES = "__Namespace.childrenQueries"; + public static final String CHILDREN_FOLDERS = "__Namespace.childrenFolders"; public static String getUUID(){ return NanoIdUtils.randomNanoId(); @@ -28,4 +56,55 @@ public static String getUserName(){ public static boolean isNameInvalid(String name) { return StringUtils.containsAny(name, invalidNameChars); } + + public static String getCollectionPropertyName(AtlasVertex parentVertex) { + return QUERY_COLLECTION_ENTITY_TYPE.equals(parentVertex.getProperty(ENTITY_TYPE_PROPERTY_KEY, String.class)) ? QUALIFIED_NAME : COLLECTION_QUALIFIED_NAME; + } + + public static String updateQueryResourceAttributes(AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever, + AtlasEntity entity, AtlasVertex vertex, EntityMutationContext context) throws AtlasBaseException { + AtlasEntityType entityType = typeRegistry.getEntityTypeByName(entity.getTypeName()); + AtlasObjectId newParentObjectId = (AtlasObjectId) entity.getRelationshipAttribute(PARENT_ATTRIBUTE_NAME); + String relationshipType = AtlasEntityUtil.getRelationshipType(newParentObjectId); + AtlasStructType.AtlasAttribute parentAttribute = entityType.getRelationshipAttribute(PARENT_ATTRIBUTE_NAME, relationshipType); + AtlasObjectId currentParentObjectId = (AtlasObjectId) entityRetriever.getEntityAttribute(vertex, parentAttribute); + //Qualified name of the folder/query will not be updated if parent attribute is not changed + String qualifiedName = vertex.getProperty(QUALIFIED_NAME, String.class); + entity.setAttribute(QUALIFIED_NAME, qualifiedName); + + //Check if parent attribute is changed + if (currentParentObjectId == null || parentAttribute.getAttributeType().areEqualValues(currentParentObjectId, newParentObjectId, context.getGuidAssignments())) { + return null; + } + + AtlasVertex currentParentVertex = entityRetriever.getEntityVertex(currentParentObjectId); + AtlasVertex newParentVertex = entityRetriever.getEntityVertex(newParentObjectId); + + if (currentParentVertex == null || newParentVertex == null) { + LOG.error("Current or New parent vertex is null"); + throw new AtlasBaseException("Current or New parent vertex is null"); + } + + String currentCollectionQualifiedName = currentParentVertex.getProperty(getCollectionPropertyName(currentParentVertex), String.class); + String newCollectionQualifiedName = newParentVertex.getProperty(getCollectionPropertyName(newParentVertex), String.class); + String updatedParentQualifiedName = newParentVertex.getProperty(QUALIFIED_NAME, String.class); + + if (StringUtils.isEmpty(newCollectionQualifiedName) || StringUtils.isEmpty(currentCollectionQualifiedName)) { + LOG.error("Collection qualified name in parent or current entity is empty or null"); + throw new AtlasBaseException("Collection qualified name in parent or current entity is empty or null"); + } + + entity.setAttribute(PARENT_QUALIFIED_NAME, updatedParentQualifiedName); + + if(currentCollectionQualifiedName.equals(newCollectionQualifiedName)) { + return null; + } + + String updatedQualifiedName = qualifiedName.replaceAll(currentCollectionQualifiedName, newCollectionQualifiedName); + //Update this values into AtlasEntity + entity.setAttribute(QUALIFIED_NAME, updatedQualifiedName); + entity.setAttribute(COLLECTION_QUALIFIED_NAME, newCollectionQualifiedName); + + return newCollectionQualifiedName; + } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/PersonaPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/PersonaPreProcessor.java new file mode 100644 index 00000000000..f579dc4ff7c --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/PersonaPreProcessor.java @@ -0,0 +1,259 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.repository.store.graph.v2.preprocessor.accesscontrol; + + +import org.apache.atlas.RequestContext; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.keycloak.client.AtlasKeycloakClient; +import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.model.instance.AtlasObjectId; +import org.apache.atlas.model.instance.AtlasStruct; +import org.apache.atlas.model.instance.EntityMutations; +import org.apache.atlas.repository.graphdb.AtlasGraph; +import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.repository.store.aliasstore.ESAliasStore; +import org.apache.atlas.repository.store.aliasstore.IndexAliasStore; +import org.apache.atlas.repository.store.graph.AtlasEntityStore; +import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; +import org.apache.atlas.repository.store.graph.v2.EntityMutationContext; +import org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessor; +import org.apache.atlas.repository.store.users.KeycloakStore; +import org.apache.atlas.type.AtlasEntityType; +import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.atlas.utils.AtlasPerfMetrics; +import org.apache.commons.collections.CollectionUtils; +import org.keycloak.representations.idm.RoleRepresentation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.apache.atlas.AtlasErrorCode.OPERATION_NOT_SUPPORTED; +import static org.apache.atlas.repository.Constants.POLICY_ENTITY_TYPE; +import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; +import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_ACCESS_CONTROL_ENABLED; +import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_PERSONA_GROUPS; +import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_PERSONA_ROLE_ID; +import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_PERSONA_USERS; +import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_IS_ENABLED; +import static org.apache.atlas.repository.util.AccessControlUtils.REL_ATTR_POLICIES; +import static org.apache.atlas.repository.util.AccessControlUtils.getESAliasName; +import static org.apache.atlas.repository.util.AccessControlUtils.getEntityName; +import static org.apache.atlas.repository.util.AccessControlUtils.getIsAccessControlEnabled; +import static org.apache.atlas.repository.util.AccessControlUtils.getPersonaGroups; +import static org.apache.atlas.repository.util.AccessControlUtils.getPersonaRoleId; +import static org.apache.atlas.repository.util.AccessControlUtils.getPersonaRoleName; +import static org.apache.atlas.repository.util.AccessControlUtils.getPersonaUsers; +import static org.apache.atlas.repository.util.AccessControlUtils.getTenantId; +import static org.apache.atlas.repository.util.AccessControlUtils.getUUID; +import static org.apache.atlas.repository.util.AccessControlUtils.validateNoPoliciesAttached; + +public class PersonaPreProcessor implements PreProcessor { + private static final Logger LOG = LoggerFactory.getLogger(PersonaPreProcessor.class); + + private final AtlasGraph graph; + private final AtlasTypeRegistry typeRegistry; + private final EntityGraphRetriever entityRetriever; + private IndexAliasStore aliasStore; + private AtlasEntityStore entityStore; + private KeycloakStore keycloakStore; + + public PersonaPreProcessor(AtlasGraph graph, + AtlasTypeRegistry typeRegistry, + EntityGraphRetriever entityRetriever, + AtlasEntityStore entityStore) { + this.graph = graph; + this.typeRegistry = typeRegistry; + this.entityRetriever = entityRetriever; + this.entityStore = entityStore; + + aliasStore = new ESAliasStore(graph, entityRetriever); + keycloakStore = new KeycloakStore(true, true); + } + + @Override + public void processAttributes(AtlasStruct entityStruct, EntityMutationContext context, + EntityMutations.EntityOperation operation) throws AtlasBaseException { + if (LOG.isDebugEnabled()) { + LOG.debug("PersonaPreProcessor.processAttributes: pre processing {}, {}", entityStruct.getAttribute(QUALIFIED_NAME), operation); + } + + AtlasEntity entity = (AtlasEntity) entityStruct; + + switch (operation) { + case CREATE: + processCreatePersona(entity); + break; + case UPDATE: + processUpdatePersona(context, entity); + break; + } + } + + @Override + public void processDelete(AtlasVertex vertex) throws AtlasBaseException { + AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo = entityRetriever.toAtlasEntityWithExtInfo(vertex); + AtlasEntity persona = entityWithExtInfo.getEntity(); + + if(!persona.getStatus().equals(AtlasEntity.Status.ACTIVE)) { + LOG.info("Persona with guid {} is already deleted/purged", persona.getGuid()); + return; + } + + //delete policies + List policies = (List) persona.getRelationshipAttribute(REL_ATTR_POLICIES); + if (CollectionUtils.isNotEmpty(policies)) { + for (AtlasObjectId policyObjectId : policies) { + //AtlasVertex policyVertex = entityRetriever.getEntityVertex(policyObjectId.getGuid()); + entityStore.deleteById(policyObjectId.getGuid()); + } + } + + //remove role + keycloakStore.removeRole(getPersonaRoleId(persona)); + + //delete ES alias + aliasStore.deleteAlias(getESAliasName(persona)); + } + + private void processCreatePersona(AtlasStruct entity) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processCreatePersona"); + + validateNoPoliciesAttached((AtlasEntity) entity); + + String tenantId = getTenantId(entity); + + entity.setAttribute(QUALIFIED_NAME, String.format("%s/%s", tenantId, getUUID())); + entity.setAttribute(ATTR_ACCESS_CONTROL_ENABLED, entity.getAttributes().getOrDefault(ATTR_ACCESS_CONTROL_ENABLED, true)); + + //create keycloak role + String roleId = createKeycloakRole((AtlasEntity) entity); + + entity.setAttribute(ATTR_PERSONA_ROLE_ID, roleId); + + //create ES alias + aliasStore.createAlias((AtlasEntity) entity); + + RequestContext.get().endMetricRecord(metricRecorder); + } + + private void processUpdatePersona(EntityMutationContext context, AtlasStruct entity) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processUpdatePersona"); + + AtlasEntity persona = (AtlasEntity) entity; + validateNoPoliciesAttached(persona); + AtlasVertex vertex = context.getVertex(persona.getGuid()); + + AtlasEntity existingPersonaEntity = entityRetriever.toAtlasEntityWithExtInfo(vertex).getEntity(); + + if (!AtlasEntity.Status.ACTIVE.equals(existingPersonaEntity.getStatus())) { + throw new AtlasBaseException(OPERATION_NOT_SUPPORTED, "Persona not Active"); + } + + String vertexQName = vertex.getProperty(QUALIFIED_NAME, String.class); + entity.setAttribute(QUALIFIED_NAME, vertexQName); + entity.setAttribute(ATTR_PERSONA_ROLE_ID, getPersonaRoleId(existingPersonaEntity)); + + boolean isEnabled = getIsAccessControlEnabled(persona); + if (getIsAccessControlEnabled(existingPersonaEntity) != isEnabled) { + updatePoliciesIsEnabledAttr(context, existingPersonaEntity, isEnabled); + } + + updateKeycloakRole(persona, existingPersonaEntity); + + RequestContext.get().endMetricRecord(metricRecorder); + } + + private void updatePoliciesIsEnabledAttr(EntityMutationContext context, AtlasEntity existingPersonaEntity, + boolean enable) throws AtlasBaseException { + + List policies = (List) existingPersonaEntity.getRelationshipAttribute(REL_ATTR_POLICIES); + + if (CollectionUtils.isNotEmpty(policies)) { + AtlasEntityType entityType = typeRegistry.getEntityTypeByName(POLICY_ENTITY_TYPE); + + for (AtlasObjectId policy : policies) { + AtlasVertex policyVertex = entityRetriever.getEntityVertex(policy.getGuid()); + + AtlasEntity policyToBeUpdated = entityRetriever.toAtlasEntity(policyVertex); + policyToBeUpdated.setAttribute(ATTR_POLICY_IS_ENABLED, enable); + + context.addUpdated(policyToBeUpdated.getGuid(), policyToBeUpdated, entityType, policyVertex); + } + } + } + + private String createKeycloakRole(AtlasEntity entity) throws AtlasBaseException { + String roleName = getPersonaRoleName(entity); + List users = getPersonaUsers(entity); + List groups = getPersonaGroups(entity); + + Map> attributes = new HashMap<>(); + attributes.put("name", Collections.singletonList(roleName)); + attributes.put("type", Collections.singletonList("persona")); + attributes.put("level", Collections.singletonList("workspace")); + attributes.put("createdAt", Collections.singletonList(String.valueOf(System.currentTimeMillis()))); + attributes.put("createdBy", Collections.singletonList(RequestContext.get().getUser())); + attributes.put("enabled", Collections.singletonList(String.valueOf(true))); + attributes.put("displayName", Collections.singletonList(getEntityName(entity))); + + RoleRepresentation role = keycloakStore.createRole(roleName, users, groups, null); + + return role.getId(); + } + + private void updateKeycloakRole(AtlasEntity newPersona, AtlasEntity existingPersona) throws AtlasBaseException { + String roleId = getPersonaRoleId(existingPersona); + String roleName = getPersonaRoleName(existingPersona); + + RoleRepresentation roleRepresentation = AtlasKeycloakClient.getKeycloakClient().getRoleById(roleId); + + boolean isUpdated = false; + if (newPersona.hasAttribute(ATTR_PERSONA_USERS)) { + List newUsers = getPersonaUsers(newPersona); + List existingUsers = getPersonaUsers(existingPersona); + + keycloakStore.updateRoleUsers(roleName, existingUsers, newUsers, roleRepresentation); + isUpdated = true; + } + + if (newPersona.hasAttribute(ATTR_PERSONA_GROUPS)) { + List newGroups = getPersonaGroups(newPersona); + List existingGroups = getPersonaGroups(existingPersona); + + keycloakStore.updateRoleGroups(roleName, existingGroups, newGroups, roleRepresentation); + isUpdated = true; + } + + if (isUpdated) { + Map> attributes = new HashMap<>(); + attributes.put("updatedAt", Collections.singletonList(String.valueOf(System.currentTimeMillis()))); + attributes.put("updatedBy", Collections.singletonList(RequestContext.get().getUser())); + attributes.put("enabled", Collections.singletonList(String.valueOf(true))); + attributes.put("displayName", Collections.singletonList(getEntityName(newPersona))); + + roleRepresentation.setAttributes(attributes); + AtlasKeycloakClient.getKeycloakClient().updateRole(roleId, roleRepresentation); + LOG.info("Updated keycloak role with name {}", roleName); + } + } +} diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/PurposePreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/PurposePreProcessor.java new file mode 100644 index 00000000000..ec88e3b134a --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/PurposePreProcessor.java @@ -0,0 +1,226 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.repository.store.graph.v2.preprocessor.accesscontrol; + + +import org.apache.atlas.RequestContext; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.model.instance.AtlasObjectId; +import org.apache.atlas.model.instance.AtlasStruct; +import org.apache.atlas.model.instance.EntityMutations; +import org.apache.atlas.repository.graphdb.AtlasGraph; +import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.repository.store.aliasstore.ESAliasStore; +import org.apache.atlas.repository.store.aliasstore.IndexAliasStore; +import org.apache.atlas.repository.store.graph.AtlasEntityStore; +import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; +import org.apache.atlas.repository.store.graph.v2.EntityMutationContext; +import org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessor; +import org.apache.atlas.type.AtlasEntityType; +import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.atlas.utils.AtlasPerfMetrics; +import org.apache.commons.collections.CollectionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; +import java.util.stream.Collectors; + +import static org.apache.atlas.AtlasErrorCode.BAD_REQUEST; +import static org.apache.atlas.AtlasErrorCode.OPERATION_NOT_SUPPORTED; +import static org.apache.atlas.repository.Constants.POLICY_ENTITY_TYPE; +import static org.apache.atlas.repository.Constants.PURPOSE_ENTITY_TYPE; +import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; +import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_ACCESS_CONTROL_ENABLED; +import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_IS_ENABLED; +import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_RESOURCES; +import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_PURPOSE_CLASSIFICATIONS; +import static org.apache.atlas.repository.util.AccessControlUtils.REL_ATTR_POLICIES; +import static org.apache.atlas.repository.util.AccessControlUtils.getESAliasName; +import static org.apache.atlas.repository.util.AccessControlUtils.getIsAccessControlEnabled; +import static org.apache.atlas.repository.util.AccessControlUtils.getPurposeTags; +import static org.apache.atlas.repository.util.AccessControlUtils.getTenantId; +import static org.apache.atlas.repository.util.AccessControlUtils.getUUID; +import static org.apache.atlas.repository.util.AccessControlUtils.validateUniquenessByTags; + +public class PurposePreProcessor implements PreProcessor { + private static final Logger LOG = LoggerFactory.getLogger(PurposePreProcessor.class); + + private final AtlasGraph graph; + private final AtlasTypeRegistry typeRegistry; + private final EntityGraphRetriever entityRetriever; + private IndexAliasStore aliasStore; + private AtlasEntityStore entityStore; + + public PurposePreProcessor(AtlasGraph graph, + AtlasTypeRegistry typeRegistry, + EntityGraphRetriever entityRetriever, + AtlasEntityStore entityStore) { + this.graph = graph; + this.typeRegistry = typeRegistry; + this.entityRetriever = entityRetriever; + this.entityStore = entityStore; + + aliasStore = new ESAliasStore(graph, entityRetriever); + } + + @Override + public void processAttributes(AtlasStruct entityStruct, EntityMutationContext context, + EntityMutations.EntityOperation operation) throws AtlasBaseException { + if (LOG.isDebugEnabled()) { + LOG.debug("PurposePreProcessor.processAttributes: pre processing {}, {}", entityStruct.getAttribute(QUALIFIED_NAME), operation); + } + + AtlasEntity entity = (AtlasEntity) entityStruct; + + switch (operation) { + case CREATE: + processCreatePurpose(entity); + break; + case UPDATE: + processUpdatePurpose(context, entity, context.getVertex(entity.getGuid())); + break; + } + } + + private void processCreatePurpose(AtlasStruct entity) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processCreatePurpose"); + + validatePurpose(graph, (AtlasEntity) entity); + + String tenantId = getTenantId(entity); + + entity.setAttribute(QUALIFIED_NAME, String.format("%s/%s", tenantId, getUUID())); + entity.setAttribute(ATTR_ACCESS_CONTROL_ENABLED, entity.getAttributes().getOrDefault(ATTR_ACCESS_CONTROL_ENABLED, true)); + + //create ES alias + aliasStore.createAlias((AtlasEntity) entity); + + RequestContext.get().endMetricRecord(metricRecorder); + } + + private void processUpdatePurpose(EntityMutationContext context, + AtlasStruct entity, + AtlasVertex vertex) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processUpdatePurpose"); + + AtlasEntity purpose = (AtlasEntity) entity; + AtlasEntity.AtlasEntityWithExtInfo existingPurposeExtInfo = entityRetriever.toAtlasEntityWithExtInfo(vertex); + AtlasEntity existingPurposeEntity = existingPurposeExtInfo.getEntity(); + + if (!AtlasEntity.Status.ACTIVE.equals(existingPurposeEntity.getStatus())) { + throw new AtlasBaseException(OPERATION_NOT_SUPPORTED, "Purpose not Active"); + } + + String vertexQName = vertex.getProperty(QUALIFIED_NAME, String.class); + purpose.setAttribute(QUALIFIED_NAME, vertexQName); + + boolean isEnabled = getIsAccessControlEnabled(purpose); + if (getIsAccessControlEnabled(existingPurposeEntity) != isEnabled) { + updatePoliciesIsEnabledAttr(context, existingPurposeEntity, isEnabled); + } + if (entity.hasAttribute(ATTR_PURPOSE_CLASSIFICATIONS)) { + List newTags = getPurposeTags(purpose); + + if (CollectionUtils.isEmpty(newTags)) { + throw new AtlasBaseException(BAD_REQUEST, "Please provide at least one classification for Purpose"); + } + + if (!CollectionUtils.isEmpty(newTags) && !CollectionUtils.isEqualCollection(newTags, getPurposeTags(existingPurposeEntity))) { + validateUniquenessByTags(graph, newTags, PURPOSE_ENTITY_TYPE); + + List policies = (List) existingPurposeEntity.getRelationshipAttribute(REL_ATTR_POLICIES); + + if (CollectionUtils.isNotEmpty(policies)) { + AtlasEntityType entityType = typeRegistry.getEntityTypeByName(POLICY_ENTITY_TYPE); + List newTagsResources = newTags.stream().map(x -> "tag:" + x).collect(Collectors.toList()); + + for (AtlasObjectId policy : policies) { + AtlasVertex policyVertex = entityRetriever.getEntityVertex(policy.getGuid()); + + policyVertex.removeProperty(ATTR_POLICY_RESOURCES); + + AtlasEntity policyToBeUpdated = entityRetriever.toAtlasEntity(policyVertex); + policyToBeUpdated.setAttribute(ATTR_POLICY_RESOURCES, newTagsResources); + + context.addUpdated(policyToBeUpdated.getGuid(), policyToBeUpdated, entityType, policyVertex); + + existingPurposeExtInfo.addReferredEntity(policyToBeUpdated); + } + } + + existingPurposeExtInfo.getEntity().setAttribute(ATTR_PURPOSE_CLASSIFICATIONS, newTags); + aliasStore.updateAlias(existingPurposeExtInfo, null); + } + } + + RequestContext.get().endMetricRecord(metricRecorder); + } + + private void updatePoliciesIsEnabledAttr(EntityMutationContext context, AtlasEntity existingPersonaEntity, + boolean enable) throws AtlasBaseException { + + List policies = (List) existingPersonaEntity.getRelationshipAttribute(REL_ATTR_POLICIES); + + if (CollectionUtils.isNotEmpty(policies)) { + AtlasEntityType entityType = typeRegistry.getEntityTypeByName(POLICY_ENTITY_TYPE); + + for (AtlasObjectId policy : policies) { + AtlasVertex policyVertex = entityRetriever.getEntityVertex(policy.getGuid()); + + AtlasEntity policyToBeUpdated = entityRetriever.toAtlasEntity(policyVertex); + policyToBeUpdated.setAttribute(ATTR_POLICY_IS_ENABLED, enable); + + context.addUpdated(policyToBeUpdated.getGuid(), policyToBeUpdated, entityType, policyVertex); + } + } + } + + private void validatePurpose(AtlasGraph graph, AtlasEntity purpose) throws AtlasBaseException { + List tags = getPurposeTags(purpose); + + if (CollectionUtils.isEmpty(tags)) { + throw new AtlasBaseException(BAD_REQUEST, "Please provide purposeClassifications for Purpose"); + } + + validateUniquenessByTags(graph, tags, PURPOSE_ENTITY_TYPE); + } + + @Override + public void processDelete(AtlasVertex vertex) throws AtlasBaseException { + + AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo = entityRetriever.toAtlasEntityWithExtInfo(vertex); + AtlasEntity purpose = entityWithExtInfo.getEntity(); + + if(!purpose.getStatus().equals(AtlasEntity.Status.ACTIVE)) { + LOG.info("Purpose with guid {} is already deleted/purged", purpose.getGuid()); + return; + } + + //delete policies + List policies = (List) purpose.getRelationshipAttribute(REL_ATTR_POLICIES); + + for (AtlasObjectId policyObjectId : policies) { + entityStore.deleteById(policyObjectId.getGuid()); + } + + //delete ES alias + aliasStore.deleteAlias(getESAliasName(purpose)); + } +} diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/AbstractGlossaryPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/AbstractGlossaryPreProcessor.java new file mode 100644 index 00000000000..91950f783c5 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/AbstractGlossaryPreProcessor.java @@ -0,0 +1,285 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.repository.store.graph.v2.preprocessor.glossary; + +import org.apache.atlas.AtlasConfiguration; +import org.apache.atlas.AtlasErrorCode; +import org.apache.atlas.AtlasException; +import org.apache.atlas.RequestContext; +import org.apache.atlas.authorize.AtlasAuthorizationUtils; +import org.apache.atlas.authorize.AtlasEntityAccessRequest; +import org.apache.atlas.authorize.AtlasPrivilege; +import org.apache.atlas.discovery.EntityDiscoveryService; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.discovery.IndexSearchParams; +import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.model.instance.AtlasEntityHeader; +import org.apache.atlas.model.instance.AtlasObjectId; +import org.apache.atlas.model.tasks.AtlasTask; +import org.apache.atlas.repository.graph.GraphHelper; +import org.apache.atlas.repository.graphdb.AtlasGraph; +import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2; +import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; +import org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessor; +import org.apache.atlas.repository.store.graph.v2.tasks.MeaningsTask; +import org.apache.atlas.tasks.TaskManagement; +import org.apache.atlas.type.AtlasEntityType; +import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.atlas.utils.AtlasPerfMetrics; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.apache.atlas.repository.Constants.ATLAS_GLOSSARY_TERM_ENTITY_TYPE; +import static org.apache.atlas.repository.Constants.ELASTICSEARCH_PAGINATION_SIZE; +import static org.apache.atlas.repository.Constants.NAME; +import static org.apache.atlas.repository.Constants.STATE_PROPERTY_KEY; +import static org.apache.atlas.repository.util.AtlasEntityUtils.mapOf; +import static org.apache.atlas.type.Constants.MEANINGS_PROPERTY_KEY; +import static org.apache.atlas.type.Constants.MEANINGS_TEXT_PROPERTY_KEY; +import static org.apache.atlas.type.Constants.MEANING_NAMES_PROPERTY_KEY; +import static org.apache.atlas.type.Constants.PENDING_TASKS_PROPERTY_KEY; + +public abstract class AbstractGlossaryPreProcessor implements PreProcessor { + private static final Logger LOG = LoggerFactory.getLogger(AbstractGlossaryPreProcessor.class); + + static final boolean DEFERRED_ACTION_ENABLED = AtlasConfiguration.TASKS_USE_ENABLED.getBoolean(); + + protected static final String ATTR_MEANINGS = "meanings"; + protected static final String ATTR_CATEGORIES = "categories"; + + protected final AtlasTypeRegistry typeRegistry; + protected final EntityGraphRetriever entityRetriever; + protected final TaskManagement taskManagement; + + protected EntityDiscoveryService discovery; + + AbstractGlossaryPreProcessor(AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever, AtlasGraph graph, TaskManagement taskManagement) { + this.entityRetriever = entityRetriever; + this.typeRegistry = typeRegistry; + this.taskManagement = taskManagement; + + try { + this.discovery = new EntityDiscoveryService(typeRegistry, graph, null, null, null, null); + } catch (AtlasException e) { + e.printStackTrace(); + } + } + + public void termExists(String termName, String glossaryQName) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("termExists"); + boolean ret = false; + + try { + List mustClauseList = new ArrayList(); + mustClauseList.add(mapOf("term", mapOf("__glossary", glossaryQName))); + mustClauseList.add(mapOf("term", mapOf("__typeName.keyword", ATLAS_GLOSSARY_TERM_ENTITY_TYPE))); + mustClauseList.add(mapOf("term", mapOf("__state", "ACTIVE"))); + mustClauseList.add(mapOf("term", mapOf("name.keyword", termName))); + + Map dsl = mapOf("query", mapOf("bool", mapOf("must", mustClauseList))); + + List terms = indexSearchPaginated(dsl); + + if (CollectionUtils.isNotEmpty(terms)) { + ret = terms.stream().map(term -> (String) term.getAttribute(NAME)).anyMatch(name -> termName.equals(name)); + } + + if (ret) { + throw new AtlasBaseException(AtlasErrorCode.GLOSSARY_TERM_ALREADY_EXISTS, termName); + } + } finally { + RequestContext.get().endMetricRecord(metricRecorder); + } + } + + public void createAndQueueTask(String taskType, + String currentTermName, String updatedTermName, + String termQName, String updatedTermQualifiedName, + AtlasVertex termVertex) { + String termGuid = GraphHelper.getGuid(termVertex); + String currentUser = RequestContext.getCurrentUser(); + Map taskParams = MeaningsTask.toParameters(currentTermName, updatedTermName, termQName, updatedTermQualifiedName, termGuid); + AtlasTask task = taskManagement.createTask(taskType, currentUser, taskParams); + + AtlasGraphUtilsV2.addEncodedProperty(termVertex, PENDING_TASKS_PROPERTY_KEY, task.getGuid()); + + RequestContext.get().queueTask(task); + } + + public boolean checkEntityTermAssociation(String termQName) throws AtlasBaseException { + List entityHeader; + entityHeader = discovery.searchUsingTermQualifiedName(0,1,termQName,null,null); + return entityHeader != null; + } + + public List indexSearchPaginated(Map dsl) throws AtlasBaseException { + IndexSearchParams searchParams = new IndexSearchParams(); + List ret = new ArrayList<>(); + + List sortList = new ArrayList<>(0); + sortList.add(mapOf("__timestamp", mapOf("order", "asc"))); + sortList.add(mapOf("__guid", mapOf("order", "asc"))); + dsl.put("sort", sortList); + + int from = 0; + int size = 100; + boolean hasMore = true; + do { + dsl.put("from", from); + dsl.put("size", size); + searchParams.setDsl(dsl); + + List headers = discovery.directIndexSearch(searchParams).getEntities(); + + if (CollectionUtils.isNotEmpty(headers)) { + ret.addAll(headers); + } else { + hasMore = false; + } + + from += size; + + } while (hasMore); + + return ret; + } + + public void updateMeaningsAttributesInEntitiesOnTermUpdate(String currentTermName, String updatedTermName, + String termQName, String updatedTermQName, + String termGuid) throws AtlasBaseException { + Set attributes = new HashSet(){{ + add(ATTR_MEANINGS); + }}; + + Set relationAttributes = new HashSet(){{ + add(STATE_PROPERTY_KEY); + add(NAME); + }}; + + int from = 0; + while (true) { + List entityHeaders = discovery.searchUsingTermQualifiedName(from, ELASTICSEARCH_PAGINATION_SIZE, + termQName, attributes, relationAttributes); + + if (entityHeaders == null) + break; + + for (AtlasEntityHeader entityHeader : entityHeaders) { + AtlasVertex entityVertex = AtlasGraphUtilsV2.findByGuid(entityHeader.getGuid()); + + if (!currentTermName.equals(updatedTermName)) { + List meanings = (List) entityHeader.getAttribute(ATTR_MEANINGS); + + String updatedMeaningsText = meanings + .stream() + .filter(x -> AtlasEntity.Status.ACTIVE.name().equals(x.getAttributes().get(STATE_PROPERTY_KEY))) + .map(x -> x.getGuid().equals(termGuid) ? updatedTermName : x.getAttributes().get(NAME).toString()) + .collect(Collectors.joining(",")); + + AtlasGraphUtilsV2.setEncodedProperty(entityVertex, MEANINGS_TEXT_PROPERTY_KEY, updatedMeaningsText); + List meaningsNames = entityVertex.getMultiValuedProperty(MEANING_NAMES_PROPERTY_KEY, String.class); + + if (meaningsNames.contains(currentTermName)) { + AtlasGraphUtilsV2.removeItemFromListPropertyValue(entityVertex, MEANING_NAMES_PROPERTY_KEY, currentTermName); + AtlasGraphUtilsV2.addListProperty(entityVertex, MEANING_NAMES_PROPERTY_KEY, updatedTermName, true); + } + } + + if (StringUtils.isNotEmpty(updatedTermQName) && !termQName.equals(updatedTermQName)) { + AtlasGraphUtilsV2.removeItemFromListPropertyValue(entityVertex, MEANINGS_PROPERTY_KEY, termQName); + AtlasGraphUtilsV2.addEncodedProperty(entityVertex, MEANINGS_PROPERTY_KEY, updatedTermQName); + } + } + + from += ELASTICSEARCH_PAGINATION_SIZE; + + if (entityHeaders.size() < ELASTICSEARCH_PAGINATION_SIZE) { + break; + } + } + } + + protected void isAuthorized(AtlasEntityHeader sourceGlossary, AtlasEntityHeader targetGlossary) throws AtlasBaseException { + + // source -> CREATE + UPDATE + DELETE + AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_CREATE, sourceGlossary), + "create on source Glossary: ", sourceGlossary.getAttribute(NAME)); + + AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_UPDATE, sourceGlossary), + "update on source Glossary: ", sourceGlossary.getAttribute(NAME)); + + AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_DELETE, sourceGlossary), + "delete on source Glossary: ", sourceGlossary.getAttribute(NAME)); + + + // target -> CREATE + UPDATE + DELETE + AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_CREATE, targetGlossary), + "create on source Glossary: ", targetGlossary.getAttribute(NAME)); + + AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_UPDATE, targetGlossary), + "update on source Glossary: ", targetGlossary.getAttribute(NAME)); + + AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_DELETE, targetGlossary), + "delete on source Glossary: ", targetGlossary.getAttribute(NAME)); + } + + /** + * Record the updated child entities, it will be used to send notification and store audit logs + * @param entityVertex Child entity vertex + * @param updatedAttributes Updated attributes while updating required attributes on updating collection + */ + protected void recordUpdatedChildEntities(AtlasVertex entityVertex, Map updatedAttributes) { + RequestContext requestContext = RequestContext.get(); + AtlasPerfMetrics.MetricRecorder metricRecorder = requestContext.startMetricRecord("recordUpdatedChildEntities"); + AtlasEntity entity = new AtlasEntity(); + entity = entityRetriever.mapSystemAttributes(entityVertex, entity); + entity.setAttributes(updatedAttributes); + requestContext.cacheDifferentialEntity(new AtlasEntity(entity)); + + AtlasEntityType entityType = typeRegistry.getEntityTypeByName(entity.getTypeName()); + + //Add the min info attributes to entity header to be sent as part of notification + if(entityType != null) { + AtlasEntity finalEntity = entity; + entityType.getMinInfoAttributes().values().stream().filter(attribute -> !updatedAttributes.containsKey(attribute.getName())).forEach(attribute -> { + Object attrValue = null; + try { + attrValue = entityRetriever.getVertexAttribute(entityVertex, attribute); + } catch (AtlasBaseException e) { + LOG.error("Error while getting vertex attribute", e); + } + if(attrValue != null) { + finalEntity.setAttribute(attribute.getName(), attrValue); + } + }); + requestContext.recordEntityUpdate(new AtlasEntityHeader(finalEntity)); + } + + requestContext.endMetricRecord(metricRecorder); + } +} diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java index 34a0a0eaf4d..76b6cfebcb9 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java @@ -20,6 +20,9 @@ import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.RequestContext; +import org.apache.atlas.authorize.AtlasAuthorizationUtils; +import org.apache.atlas.authorize.AtlasEntityAccessRequest; +import org.apache.atlas.authorize.AtlasPrivilege; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasEntityHeader; @@ -27,10 +30,15 @@ import org.apache.atlas.model.instance.AtlasRelatedObjectId; import org.apache.atlas.model.instance.AtlasStruct; import org.apache.atlas.model.instance.EntityMutations; +import org.apache.atlas.repository.graph.GraphHelper; +import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.repository.store.graph.v2.EntityGraphMapper; import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; import org.apache.atlas.repository.store.graph.v2.EntityMutationContext; -import org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessor; +import org.apache.atlas.tasks.TaskManagement; +import org.apache.atlas.type.AtlasEntityType; +import org.apache.atlas.type.AtlasStructType; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.utils.AtlasPerfMetrics; import org.apache.commons.collections.CollectionUtils; @@ -48,26 +56,44 @@ import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.getUUID; import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.stream.Collectors; +import static org.apache.atlas.AtlasErrorCode.BAD_REQUEST; +import static org.apache.atlas.repository.Constants.ATLAS_GLOSSARY_CATEGORY_ENTITY_TYPE; +import static org.apache.atlas.repository.Constants.ATLAS_GLOSSARY_ENTITY_TYPE; +import static org.apache.atlas.repository.Constants.CATEGORY_PARENT_EDGE_LABEL; +import static org.apache.atlas.repository.Constants.CATEGORY_TERMS_EDGE_LABEL; +import static org.apache.atlas.repository.Constants.GUID_PROPERTY_KEY; import static org.apache.atlas.repository.Constants.NAME; import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; +import static org.apache.atlas.repository.graph.GraphHelper.getActiveChildrenVertices; +import static org.apache.atlas.repository.graph.GraphHelper.getActiveParentVertices; +import static org.apache.atlas.repository.graph.GraphHelper.getTypeName; import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.*; +import static org.apache.atlas.repository.store.graph.v2.tasks.MeaningsTaskFactory.UPDATE_ENTITY_MEANINGS_ON_TERM_UPDATE; +import static org.apache.atlas.repository.util.AtlasEntityUtils.mapOf; +import static org.apache.atlas.type.Constants.CATEGORIES_PARENT_PROPERTY_KEY; +import static org.apache.atlas.type.Constants.CATEGORIES_PROPERTY_KEY; +import static org.apache.atlas.type.Constants.GLOSSARY_PROPERTY_KEY; -public class CategoryPreProcessor implements PreProcessor { +public class CategoryPreProcessor extends AbstractGlossaryPreProcessor { private static final Logger LOG = LoggerFactory.getLogger(CategoryPreProcessor.class); - private final AtlasTypeRegistry typeRegistry; - private final EntityGraphRetriever entityRetriever; - private AtlasEntityHeader anchor; private AtlasEntityHeader parentCategory; + private EntityGraphMapper entityGraphMapper; + private EntityMutationContext context; - public CategoryPreProcessor(AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever) { - this.entityRetriever = entityRetriever; - this.typeRegistry = typeRegistry; + public CategoryPreProcessor(AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever, + AtlasGraph graph, TaskManagement taskManagement, EntityGraphMapper entityGraphMapper) { + super(typeRegistry, entityRetriever, graph, taskManagement); + this.entityGraphMapper = entityGraphMapper; } @Override @@ -79,6 +105,8 @@ public void processAttributes(AtlasStruct entityStruct, EntityMutationContext co entityStruct.getAttribute(QUALIFIED_NAME), operation); } + this.context = context; + AtlasEntity entity = (AtlasEntity) entityStruct; AtlasVertex vertex = context.getVertex(entity.getGuid()); @@ -102,18 +130,16 @@ private void processCreateCategory(AtlasEntity entity, AtlasVertex vertex) throw throw new AtlasBaseException(AtlasErrorCode.INVALID_DISPLAY_NAME); } - if (parentCategory != null) { - AtlasEntity newParent = entityRetriever.toAtlasEntity(parentCategory.getGuid()); - AtlasRelatedObjectId newAnchor = (AtlasRelatedObjectId) newParent.getRelationshipAttribute(ANCHOR); + String glossaryQualifiedName = (String) anchor.getAttribute(QUALIFIED_NAME); + categoryExists(catName, glossaryQualifiedName); + validateParent(glossaryQualifiedName); - if (newAnchor != null && !newAnchor.getGuid().equals(anchor.getGuid())){ - throw new AtlasBaseException(AtlasErrorCode.CATEGORY_PARENT_FROM_OTHER_GLOSSARY); - } - } + entity.setAttribute(QUALIFIED_NAME, createQualifiedName(vertex)); + AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_CREATE, new AtlasEntityHeader(entity)), + "create entity: type=", entity.getTypeName()); validateChildren(entity, null); - entity.setAttribute(QUALIFIED_NAME, createQualifiedName(vertex)); RequestContext.get().endMetricRecord(metricRecorder); } @@ -126,50 +152,284 @@ private void processUpdateCategory(AtlasEntity entity, AtlasVertex vertex) throw throw new AtlasBaseException(AtlasErrorCode.INVALID_DISPLAY_NAME); } - AtlasEntity storeObject = entityRetriever.toAtlasEntity(vertex); - AtlasRelatedObjectId existingAnchor = (AtlasRelatedObjectId) storeObject.getRelationshipAttribute(ANCHOR); - if (existingAnchor != null && !existingAnchor.getGuid().equals(anchor.getGuid())){ - throw new AtlasBaseException(AtlasErrorCode.ACHOR_UPDATION_NOT_SUPPORTED); + AtlasEntity storedCategory = entityRetriever.toAtlasEntity(vertex); + AtlasRelatedObjectId currentGlossary = (AtlasRelatedObjectId) storedCategory.getRelationshipAttribute(ANCHOR); + AtlasEntityHeader currentGlossaryHeader = entityRetriever.toAtlasEntityHeader(currentGlossary.getGuid()); + String currentGlossaryQualifiedName = (String) currentGlossaryHeader.getAttribute(QUALIFIED_NAME); + + String newGlossaryQualifiedName = (String) anchor.getAttribute(QUALIFIED_NAME); + + if (!currentGlossaryQualifiedName.equals(newGlossaryQualifiedName)){ + //Auth check + isAuthorized(currentGlossaryHeader, anchor); + + processMoveCategoryToAnotherGlossary(entity, vertex, currentGlossaryQualifiedName, newGlossaryQualifiedName, vertexQnName); + + } else { + String vertexName = vertex.getProperty(NAME, String.class); + if (!vertexName.equals(catName)) { + categoryExists(catName, newGlossaryQualifiedName); + } + validateChildren(entity, storedCategory); + validateParent(newGlossaryQualifiedName); + + entity.setAttribute(QUALIFIED_NAME, vertexQnName); } - if (parentCategory != null) { - AtlasEntity newParent = entityRetriever.toAtlasEntity(parentCategory.getGuid()); - AtlasRelatedObjectId newAnchor = (AtlasRelatedObjectId) newParent.getRelationshipAttribute(ANCHOR); + RequestContext.get().endMetricRecord(metricRecorder); + } - if (newAnchor != null && !newAnchor.getGuid().equals(existingAnchor.getGuid())){ - throw new AtlasBaseException(AtlasErrorCode.CATEGORY_PARENT_FROM_OTHER_GLOSSARY); + private void processMoveCategoryToAnotherGlossary(AtlasEntity category, + AtlasVertex categoryVertex, + String sourceGlossaryQualifiedName, + String targetGlossaryQualifiedName, + String currentCategoryQualifiedName) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("processMoveCategoryToAnotherGlossary"); + + try { + if (category.hasRelationshipAttribute(CATEGORY_CHILDREN) || category.hasRelationshipAttribute(CATEGORY_TERMS)) { + throw new AtlasBaseException(BAD_REQUEST, String.format("Please do not pass relationship attributes [%s, %s] while moving Category to different Glossary", + CATEGORY_CHILDREN, CATEGORY_TERMS)); } + + String categoryName = (String) category.getAttribute(NAME); + + LOG.info("Moving category {} to Glossary {}", categoryName, targetGlossaryQualifiedName); + + categoryExists(categoryName , targetGlossaryQualifiedName); + validateParentForGlossaryChange(category, categoryVertex, targetGlossaryQualifiedName); + + String updatedQualifiedName = currentCategoryQualifiedName.replace(sourceGlossaryQualifiedName, targetGlossaryQualifiedName); + + category.setAttribute(QUALIFIED_NAME, updatedQualifiedName); + + moveChildrenToAnotherGlossary(categoryVertex, null, sourceGlossaryQualifiedName, targetGlossaryQualifiedName); + + LOG.info("Moved category {} to Glossary {}", categoryName, targetGlossaryQualifiedName); + + } finally { + RequestContext.get().endMetricRecord(recorder); } + } - validateChildren(entity, storeObject); + private void moveChildrenToAnotherGlossary(AtlasVertex childCategoryVertex, + String parentCategoryQualifiedName, + String sourceGlossaryQualifiedName, + String targetGlossaryQualifiedName) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("moveChildrenToAnotherGlossary"); - entity.setAttribute(QUALIFIED_NAME, vertexQnName); - RequestContext.get().endMetricRecord(metricRecorder); + + try { + LOG.info("Moving child category {} to Glossary {}", childCategoryVertex.getProperty(NAME, String.class), targetGlossaryQualifiedName); + Map updatedAttributes = new HashMap<>(); + + String currentCategoryQualifiedName = childCategoryVertex.getProperty(QUALIFIED_NAME, String.class); + String updatedQualifiedName = currentCategoryQualifiedName.replace(sourceGlossaryQualifiedName, targetGlossaryQualifiedName); + + // Change category qualifiedName + childCategoryVertex.setProperty(QUALIFIED_NAME, updatedQualifiedName); + updatedAttributes.put(QUALIFIED_NAME, updatedQualifiedName); + + //change __glossary, __parentCategory + childCategoryVertex.setProperty(GLOSSARY_PROPERTY_KEY, targetGlossaryQualifiedName); + childCategoryVertex.setProperty(CATEGORIES_PARENT_PROPERTY_KEY, parentCategoryQualifiedName); + + // update glossary relationship + updateGlossaryRelationship(childCategoryVertex, GLOSSARY_CATEGORY_REL_TYPE); + + //update system properties + GraphHelper.setModifiedByAsString(childCategoryVertex, RequestContext.get().getUser()); + GraphHelper.setModifiedTime(childCategoryVertex, System.currentTimeMillis()); + + // move terms to target Glossary + Iterator terms = getActiveChildrenVertices(childCategoryVertex, CATEGORY_TERMS_EDGE_LABEL); + + while (terms.hasNext()) { + AtlasVertex termVertex = terms.next(); + moveChildTermToAnotherGlossary(termVertex, updatedQualifiedName, sourceGlossaryQualifiedName, targetGlossaryQualifiedName); + } + + // Get all children categories of current category + Iterator childCategories = getActiveChildrenVertices(childCategoryVertex, CATEGORY_PARENT_EDGE_LABEL); + + while (childCategories.hasNext()) { + AtlasVertex childVertex = childCategories.next(); + moveChildrenToAnotherGlossary(childVertex, updatedQualifiedName, sourceGlossaryQualifiedName, targetGlossaryQualifiedName); + } + + recordUpdatedChildEntities(childCategoryVertex, updatedAttributes); + + LOG.info("Moved child category {} to Glossary {}", childCategoryVertex.getProperty(NAME, String.class), targetGlossaryQualifiedName); + } finally { + RequestContext.get().endMetricRecord(recorder); + } } - private void validateChildren(AtlasEntity entity, AtlasEntity storeObject) throws AtlasBaseException { - AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("CategoryPreProcessor.validateChildren"); - List existingChildren = new ArrayList<>(); - if (storeObject != null) { - existingChildren = (List) storeObject.getRelationshipAttribute(CATEGORY_CHILDREN); + public void moveChildTermToAnotherGlossary(AtlasVertex termVertex, + String parentCategoryQualifiedName, + String sourceGlossaryQualifiedName, + String targetGlossaryQualifiedName) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("moveChildTermToAnotherGlossary"); + + try { + Map updatedAttributes = new HashMap<>(); + + String termName = termVertex.getProperty(NAME, String.class); + String termGuid = termVertex.getProperty(GUID_PROPERTY_KEY, String.class); + + LOG.info("Moving child term {} to Glossary {}", termName, targetGlossaryQualifiedName); + + //check duplicate term name + termExists(termName, targetGlossaryQualifiedName); + + String currentTermQualifiedName = termVertex.getProperty(QUALIFIED_NAME, String.class); + String updatedTermQualifiedName = currentTermQualifiedName.replace(sourceGlossaryQualifiedName, targetGlossaryQualifiedName); + + //qualifiedName + termVertex.setProperty(QUALIFIED_NAME, updatedTermQualifiedName); + updatedAttributes.put(QUALIFIED_NAME, updatedTermQualifiedName); + + // __glossary, __categories + termVertex.setProperty(GLOSSARY_PROPERTY_KEY, targetGlossaryQualifiedName); + termVertex.removeProperty(CATEGORIES_PROPERTY_KEY); + termVertex.setProperty(CATEGORIES_PROPERTY_KEY, parentCategoryQualifiedName); + + // update glossary relationship + updateGlossaryRelationship(termVertex, GLOSSARY_TERM_REL_TYPE); + + //update system properties + GraphHelper.setModifiedByAsString(termVertex, RequestContext.get().getUser()); + GraphHelper.setModifiedTime(termVertex, System.currentTimeMillis()); + + if (checkEntityTermAssociation(currentTermQualifiedName)) { + if (taskManagement != null && DEFERRED_ACTION_ENABLED) { + createAndQueueTask(UPDATE_ENTITY_MEANINGS_ON_TERM_UPDATE, termName, termName, currentTermQualifiedName, updatedTermQualifiedName, termVertex); + } else { + updateMeaningsAttributesInEntitiesOnTermUpdate(termName, termName, currentTermQualifiedName, updatedTermQualifiedName, termGuid); + } + } + + recordUpdatedChildEntities(termVertex, updatedAttributes); + + LOG.info("Moved child term {} to Glossary {}", termName, targetGlossaryQualifiedName); + } finally { + RequestContext.get().endMetricRecord(recorder); } - Set existingChildrenGuids = existingChildren.stream().map(x -> x.getGuid()).collect(Collectors.toSet()); + } - List children = (List) entity.getRelationshipAttribute(CATEGORY_CHILDREN); + private void validateParentForGlossaryChange(AtlasEntity category, + AtlasVertex categoryVertex, + String targetGlossaryQualifiedName) throws AtlasBaseException { - if (CollectionUtils.isNotEmpty(children)) { - for (AtlasObjectId child : children) { - if (!existingChildrenGuids.contains(child.getGuid())) { - AtlasEntity newChild = entityRetriever.toAtlasEntity(child.getGuid()); - AtlasRelatedObjectId newAnchor = (AtlasRelatedObjectId) newChild.getRelationshipAttribute(ANCHOR); + if (!category.hasRelationshipAttribute(CATEGORY_PARENT)) { + // parentCategory not present in payload, check in store + Iterator parentItr = getActiveParentVertices(categoryVertex, CATEGORY_PARENT_EDGE_LABEL); - if (newAnchor != null && !newAnchor.getGuid().equals(anchor.getGuid())){ - throw new AtlasBaseException(AtlasErrorCode.CATEGORY_PARENT_FROM_OTHER_GLOSSARY); + if (parentItr.hasNext()) { + AtlasVertex parentCategory = parentItr.next(); + String parentCategoryQualifiedName = parentCategory.getProperty(QUALIFIED_NAME, String.class); + + if (!parentCategoryQualifiedName.endsWith(targetGlossaryQualifiedName)){ + throw new AtlasBaseException(AtlasErrorCode.CATEGORY_PARENT_FROM_OTHER_GLOSSARY); + } + } + } else { + validateParent(targetGlossaryQualifiedName); + } + } + + private void categoryExists(String categoryName, String glossaryQualifiedName) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("categoryExists"); + + boolean exists = false; + try { + List mustClauseList = new ArrayList(); + mustClauseList.add(mapOf("term", mapOf("__glossary", glossaryQualifiedName))); + mustClauseList.add(mapOf("term", mapOf("__typeName.keyword", ATLAS_GLOSSARY_CATEGORY_ENTITY_TYPE))); + mustClauseList.add(mapOf("term", mapOf("__state", "ACTIVE"))); + mustClauseList.add(mapOf("term", mapOf("name.keyword", categoryName))); + + + Map bool = new HashMap<>(); + if (parentCategory != null) { + String parentQname = (String) parentCategory.getAttribute(QUALIFIED_NAME); + mustClauseList.add(mapOf("term", mapOf("__parentCategory", parentQname))); + } else { + List mustNotClauseList = new ArrayList(); + mustNotClauseList.add(mapOf("exists", mapOf("field", "__parentCategory"))); + bool.put("must_not", mustNotClauseList); + } + + bool.put("must", mustClauseList); + + Map dsl = mapOf("query", mapOf("bool", bool)); + + List categories = indexSearchPaginated(dsl); + + if (CollectionUtils.isNotEmpty(categories)) { + for (AtlasEntityHeader category : categories) { + String name = (String) category.getAttribute(NAME); + if (categoryName.equals(name)) { + exists = true; + break; } } } + } finally { + RequestContext.get().endMetricRecord(metricRecorder); + } + + if (exists) { + throw new AtlasBaseException(AtlasErrorCode.GLOSSARY_CATEGORY_ALREADY_EXISTS, categoryName); + } + } + + private void validateParent(String glossaryQualifiedName) throws AtlasBaseException { + // in case parent category is present, ensure it belongs to same Glossary + + if (parentCategory != null) { + String newParentGlossaryQualifiedName = (String) parentCategory.getAttribute(QUALIFIED_NAME); + + if (!newParentGlossaryQualifiedName.endsWith(glossaryQualifiedName)){ + throw new AtlasBaseException(AtlasErrorCode.CATEGORY_PARENT_FROM_OTHER_GLOSSARY); + } + } + } + + private void validateChildren(AtlasEntity entity, AtlasEntity storedCategory) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("CategoryPreProcessor.validateChildren"); + // in case new child is being added, ensure it belongs to same Glossary + + try { + if (entity.hasRelationshipAttribute(CATEGORY_CHILDREN) && entity.getRelationshipAttribute(CATEGORY_CHILDREN) != null) { + List children = (List) entity.getRelationshipAttribute(CATEGORY_CHILDREN); + + if (CollectionUtils.isNotEmpty(children)) { + Set existingChildrenGuids = new HashSet<>(); + + if (storedCategory != null && + storedCategory.hasRelationshipAttribute(CATEGORY_CHILDREN) && + storedCategory.getRelationshipAttribute(CATEGORY_CHILDREN) != null) { + List existingChildren = (List) storedCategory.getRelationshipAttribute(CATEGORY_CHILDREN); + + existingChildrenGuids = existingChildren.stream().map(x -> x.getGuid()).collect(Collectors.toSet()); + } + + for (AtlasObjectId child : children) { + if (!existingChildrenGuids.contains(child.getGuid())) { + AtlasEntity newChild = entityRetriever.toAtlasEntity(child.getGuid()); + AtlasRelatedObjectId newAnchor = (AtlasRelatedObjectId) newChild.getRelationshipAttribute(ANCHOR); + + if (newAnchor != null && !newAnchor.getGuid().equals(anchor.getGuid())){ + throw new AtlasBaseException(AtlasErrorCode.CATEGORY_PARENT_FROM_OTHER_GLOSSARY); + } + } + } + } + } + } finally { + RequestContext.get().endMetricRecord(metricRecorder); } - RequestContext.get().endMetricRecord(metricRecorder); } private void setAnchorAndParent(AtlasEntity entity, EntityMutationContext context) throws AtlasBaseException { @@ -216,6 +476,16 @@ private void setAnchorAndParent(AtlasEntity entity, EntityMutationContext contex RequestContext.get().endMetricRecord(metricRecorder); } + private void updateGlossaryRelationship(AtlasVertex entityVertex, String relationshipType) throws AtlasBaseException { + AtlasObjectId glossaryObjectId = new AtlasObjectId(anchor.getGuid(), ATLAS_GLOSSARY_ENTITY_TYPE); + + String typeName = getTypeName(entityVertex); + AtlasEntityType entityType = typeRegistry.getEntityTypeByName(typeName); + AtlasStructType.AtlasAttribute attribute = entityType.getRelationshipAttribute(ANCHOR, relationshipType); + + entityGraphMapper.mapGlossaryRelationshipAttribute(attribute, glossaryObjectId, entityVertex, context); + } + private String createQualifiedName(AtlasVertex vertex) { if (vertex != null) { diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java index 7b9fbf38877..67eae1778de 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java @@ -20,6 +20,9 @@ import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.RequestContext; +import org.apache.atlas.authorize.AtlasAuthorizationUtils; +import org.apache.atlas.authorize.AtlasEntityAccessRequest; +import org.apache.atlas.authorize.AtlasPrivilege; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasEntityHeader; @@ -27,36 +30,43 @@ import org.apache.atlas.model.instance.AtlasRelatedObjectId; import org.apache.atlas.model.instance.AtlasStruct; import org.apache.atlas.model.instance.EntityMutations; +import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasVertex; -import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2; import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; import org.apache.atlas.repository.store.graph.v2.EntityMutationContext; -import org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessor; +import org.apache.atlas.tasks.TaskManagement; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.utils.AtlasPerfMetrics; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; +import java.util.Iterator; +import java.util.List; +<<<<<<< HEAD import static org.apache.atlas.repository.Constants.NAME; import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.ANCHOR; import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.isNameInvalid; import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.getUUID; +======= +import static org.apache.atlas.repository.Constants.*; +import static org.apache.atlas.repository.graph.GraphHelper.getActiveParentVertices; +import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.*; +import static org.apache.atlas.repository.store.graph.v2.tasks.MeaningsTaskFactory.UPDATE_ENTITY_MEANINGS_ON_TERM_UPDATE; +>>>>>>> 7f3c811fb15361ba82bfb4edd7a8393798863ff0 -public class TermPreProcessor implements PreProcessor { +@Component +public class TermPreProcessor extends AbstractGlossaryPreProcessor { private static final Logger LOG = LoggerFactory.getLogger(TermPreProcessor.class); - private final AtlasTypeRegistry typeRegistry; - private final EntityGraphRetriever entityRetriever; - private AtlasEntityHeader anchor; - - public TermPreProcessor(AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever) { - this.entityRetriever = entityRetriever; - this.typeRegistry = typeRegistry; + public TermPreProcessor( AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever, AtlasGraph graph, TaskManagement taskManagement) { + super(typeRegistry, entityRetriever, graph, taskManagement); } @Override @@ -90,11 +100,16 @@ private void processCreateTerm(AtlasEntity entity, AtlasVertex vertex) throws At throw new AtlasBaseException(AtlasErrorCode.INVALID_DISPLAY_NAME); } - if (termExists(termName)) { - throw new AtlasBaseException(AtlasErrorCode.GLOSSARY_TERM_ALREADY_EXISTS, termName); - } + String glossaryQName = (String) anchor.getAttribute(QUALIFIED_NAME); + + termExists(termName, glossaryQName); + + validateCategory(entity); entity.setAttribute(QUALIFIED_NAME, createQualifiedName()); + AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_CREATE, new AtlasEntityHeader(entity)), + "create entity: type=", entity.getTypeName()); + RequestContext.get().endMetricRecord(metricRecorder); } @@ -102,40 +117,118 @@ private void processUpdateTerm(AtlasEntity entity, AtlasVertex vertex) throws At AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processUpdateTerm"); String termName = (String) entity.getAttribute(NAME); String vertexName = vertex.getProperty(NAME, String.class); - - if (!vertexName.equals(termName) && termExists(termName)) { - throw new AtlasBaseException(AtlasErrorCode.GLOSSARY_TERM_ALREADY_EXISTS, termName); - } + String termGuid = entity.getGuid(); if (StringUtils.isEmpty(termName) || isNameInvalid(termName)) { throw new AtlasBaseException(AtlasErrorCode.INVALID_DISPLAY_NAME); } - AtlasEntity storeObject = entityRetriever.toAtlasEntity(vertex); - AtlasRelatedObjectId existingAnchor = (AtlasRelatedObjectId) storeObject.getRelationshipAttribute(ANCHOR); - if (existingAnchor != null && !existingAnchor.getGuid().equals(anchor.getGuid())){ - throw new AtlasBaseException(AtlasErrorCode.ACHOR_UPDATION_NOT_SUPPORTED); - } + validateCategory(entity); + + AtlasEntity storedTerm = entityRetriever.toAtlasEntity(vertex); + AtlasRelatedObjectId currentGlossary = (AtlasRelatedObjectId) storedTerm.getRelationshipAttribute(ANCHOR); + AtlasEntityHeader currentGlossaryHeader = entityRetriever.toAtlasEntityHeader(currentGlossary.getGuid()); + String currentGlossaryQualifiedName = (String) currentGlossaryHeader.getAttribute(QUALIFIED_NAME); + + String termQualifiedName = vertex.getProperty(QUALIFIED_NAME, String.class); + + String newGlossaryQualifiedName = (String) anchor.getAttribute(QUALIFIED_NAME); + + if (!currentGlossaryQualifiedName.equals(newGlossaryQualifiedName)){ + //Auth check + isAuthorized(currentGlossaryHeader, anchor); - String vertexQnName = vertex.getProperty(QUALIFIED_NAME, String.class); + String updatedTermQualifiedName = moveTermToAnotherGlossary(entity, vertex, currentGlossaryQualifiedName, newGlossaryQualifiedName, termQualifiedName); + + if (checkEntityTermAssociation(termQualifiedName)) { + if (taskManagement != null && DEFERRED_ACTION_ENABLED) { + createAndQueueTask(UPDATE_ENTITY_MEANINGS_ON_TERM_UPDATE, vertexName, termName, termQualifiedName, updatedTermQualifiedName, vertex); + } else { + updateMeaningsAttributesInEntitiesOnTermUpdate(vertexName, termName, termQualifiedName, updatedTermQualifiedName, termGuid); + } + } + + } else { + + if (!vertexName.equals(termName)) { + termExists(termName, newGlossaryQualifiedName); + } + + entity.setAttribute(QUALIFIED_NAME, termQualifiedName); + + if (!termName.equals(vertexName) && checkEntityTermAssociation(termQualifiedName)) { + if (taskManagement != null && DEFERRED_ACTION_ENABLED) { + createAndQueueTask(UPDATE_ENTITY_MEANINGS_ON_TERM_UPDATE, vertexName, termName, termQualifiedName, null, vertex); + } else { + updateMeaningsAttributesInEntitiesOnTermUpdate(vertexName, termName, termQualifiedName, null, termGuid); + } + } + } - entity.setAttribute(QUALIFIED_NAME, vertexQnName); RequestContext.get().endMetricRecord(metricRecorder); } - private String createQualifiedName() { - return getUUID() + "@" + anchor.getAttribute(QUALIFIED_NAME); + private void validateCategory(AtlasEntity entity) throws AtlasBaseException { + String glossaryQualifiedName = (String) anchor.getAttribute(QUALIFIED_NAME); + + if (entity.hasRelationshipAttribute(ATTR_CATEGORIES) && entity.getRelationshipAttribute(ATTR_CATEGORIES) != null) { + List categories = (List) entity.getRelationshipAttribute(ATTR_CATEGORIES); + + if (CollectionUtils.isNotEmpty(categories)) { + AtlasObjectId category = categories.get(0); + String categoryQualifiedName; + + if (category.getUniqueAttributes() != null && category.getUniqueAttributes().containsKey(QUALIFIED_NAME)) { + categoryQualifiedName = (String) category.getUniqueAttributes().get(QUALIFIED_NAME); + } else { + AtlasVertex categoryVertex = entityRetriever.getEntityVertex(category.getGuid()); + categoryQualifiedName = categoryVertex.getProperty(QUALIFIED_NAME, String.class); + } + + if (!categoryQualifiedName.endsWith(glossaryQualifiedName)) { + throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Passed category doesn't belongs to Passed Glossary"); + } + } + } } - private boolean termExists(String termName) { - AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("termExists"); - boolean ret = false; - String glossaryQName = (String) anchor.getAttribute(QUALIFIED_NAME); + public String moveTermToAnotherGlossary(AtlasEntity entity, AtlasVertex vertex, + String sourceGlossaryQualifiedName, + String targetGlossaryQualifiedName, + String currentTermQualifiedName) throws AtlasBaseException { - ret = AtlasGraphUtilsV2.termExists(termName, glossaryQName); + //check duplicate term name + termExists((String) entity.getAttribute(NAME), targetGlossaryQualifiedName); - RequestContext.get().endMetricRecord(metricRecorder); - return ret; + + String updatedQualifiedName = currentTermQualifiedName.replace(sourceGlossaryQualifiedName, targetGlossaryQualifiedName); + + //qualifiedName + entity.setAttribute(QUALIFIED_NAME, updatedQualifiedName); + + // __categories + /* if category is not passed in relationshipAttributes, check + whether category belongs to target glossary, if not throw an exception + */ + if (!entity.hasRelationshipAttribute(ATTR_CATEGORIES)) { + Iterator categoriesItr = getActiveParentVertices(vertex, CATEGORY_TERMS_EDGE_LABEL); + + if (categoriesItr.hasNext()) { + AtlasVertex categoryVertex = categoriesItr.next(); + + String categoryQualifiedName = categoryVertex.getProperty(QUALIFIED_NAME, String.class); + + if (!categoryQualifiedName.endsWith(targetGlossaryQualifiedName)) { + throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Passed category doesn't belongs to Passed Glossary"); + } + } + } + + return updatedQualifiedName; + } + + private String createQualifiedName() { + return getUUID() + "@" + anchor.getAttribute(QUALIFIED_NAME); } private void setAnchor(AtlasEntity entity, EntityMutationContext context) throws AtlasBaseException { diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/resource/AbstractResourcePreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/resource/AbstractResourcePreProcessor.java new file mode 100644 index 00000000000..5bd89e45970 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/resource/AbstractResourcePreProcessor.java @@ -0,0 +1,140 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.repository.store.graph.v2.preprocessor.resource; + +import org.apache.atlas.RequestContext; +import org.apache.atlas.authorize.AtlasAuthorizationUtils; +import org.apache.atlas.authorize.AtlasEntityAccessRequest; +import org.apache.atlas.authorize.AtlasPrivilege; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.model.instance.AtlasEntityHeader; +import org.apache.atlas.model.instance.AtlasObjectId; +import org.apache.atlas.repository.graphdb.AtlasEdgeDirection; +import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; +import org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessor; +import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.atlas.utils.AtlasPerfMetrics; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Iterator; + +import static org.apache.atlas.repository.Constants.ACTIVE_STATE_VALUE; +import static org.apache.atlas.repository.Constants.ASSET_RELATION_ATTR; +import static org.apache.atlas.repository.Constants.STATE_PROPERTY_KEY; + +public abstract class AbstractResourcePreProcessor implements PreProcessor { + private static final Logger LOG = LoggerFactory.getLogger(AbstractResourcePreProcessor.class); + + private final AtlasTypeRegistry typeRegistry; + private final EntityGraphRetriever entityRetriever; + + AbstractResourcePreProcessor(AtlasTypeRegistry typeRegistry, + EntityGraphRetriever entityRetriever) { + this.typeRegistry = typeRegistry; + this.entityRetriever = entityRetriever; + } + + void authorizeResourceUpdate(AtlasEntity resourceEntity, AtlasVertex ResourceVertex, String edgeLabel) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("authorizeResourceUpdate"); + + try { + AtlasEntityHeader assetEntity = null; + + AtlasObjectId asset = getAssetRelationAttr(resourceEntity); + if (asset != null) { + //Found linked asset in payload + AtlasVertex assetVertex = entityRetriever.getEntityVertex(asset); + assetEntity = entityRetriever.toAtlasEntityHeaderWithClassifications(assetVertex); + + } else { + //Check for linked asset in store + Iterator atlasVertexIterator = ResourceVertex.query() + .direction(AtlasEdgeDirection.IN) + .label(edgeLabel) + .has(STATE_PROPERTY_KEY, ACTIVE_STATE_VALUE) + .vertices() + .iterator(); + + if (atlasVertexIterator.hasNext()) { + //Found linked asset in store + AtlasVertex assetVertex = (AtlasVertex) atlasVertexIterator.next(); + assetEntity = entityRetriever.toAtlasEntityHeaderWithClassifications(assetVertex); + } + } + + if (assetEntity != null) { + //First authorize entity update access + verifyAssetAccess(assetEntity, AtlasPrivilege.ENTITY_UPDATE, resourceEntity, AtlasPrivilege.ENTITY_UPDATE); + } else { + //No linked asset to the Resource, check for resource update permission + verifyAccess(resourceEntity, AtlasPrivilege.ENTITY_UPDATE); + } + + } finally { + RequestContext.get().endMetricRecord(metricRecorder); + } + } + + void authorizeResourceDelete(AtlasVertex resourceVertex) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("authorizeResourceDelete"); + + try { + AtlasEntity resourceEntity = entityRetriever.toAtlasEntity(resourceVertex); + + AtlasObjectId asset = getAssetRelationAttr(resourceEntity); + if (asset != null) { + AtlasEntityHeader assetEntity = entityRetriever.toAtlasEntityHeaderWithClassifications(asset.getGuid()); + verifyAssetAccess(assetEntity, AtlasPrivilege.ENTITY_UPDATE, resourceEntity, AtlasPrivilege.ENTITY_DELETE); + } else { + //No linked asset to the Resource, check for resource delete permission + verifyAccess(resourceEntity, AtlasPrivilege.ENTITY_DELETE); + } + } finally { + RequestContext.get().endMetricRecord(recorder); + } + } + + private AtlasObjectId getAssetRelationAttr(AtlasEntity entity) { + AtlasObjectId ret = null; + + if (entity.hasRelationshipAttribute(ASSET_RELATION_ATTR) && + entity.getRelationshipAttribute(ASSET_RELATION_ATTR) != null) { + ret = (AtlasObjectId) entity.getRelationshipAttribute(ASSET_RELATION_ATTR); + } + + return ret; + } + + private void verifyAssetAccess(AtlasEntityHeader asset, AtlasPrivilege assetPrivilege, + AtlasEntity resource, AtlasPrivilege resourcePrivilege) throws AtlasBaseException { + verifyAccess(asset, assetPrivilege); + verifyAccess(resource, resourcePrivilege); + } + + private void verifyAccess(AtlasEntity entity, AtlasPrivilege privilege) throws AtlasBaseException { + verifyAccess(new AtlasEntityHeader(entity), privilege); + } + + private void verifyAccess(AtlasEntityHeader entityHeader, AtlasPrivilege privilege) throws AtlasBaseException { + String errorMessage = privilege.name() + " entity: " + entityHeader.getTypeName(); + AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, privilege, entityHeader), errorMessage); + } +} diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/resource/LinkPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/resource/LinkPreProcessor.java new file mode 100644 index 00000000000..273359869c3 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/resource/LinkPreProcessor.java @@ -0,0 +1,103 @@ +package org.apache.atlas.repository.store.graph.v2.preprocessor.resource; + +import com.google.common.base.Predicate; +import org.apache.atlas.RequestContext; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.model.instance.AtlasStruct; +import org.apache.atlas.model.instance.EntityMutations; +import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; +import org.apache.atlas.repository.store.graph.v2.EntityMutationContext; +import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.atlas.utils.AtlasPerfMetrics; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.regex.Pattern; + +import static org.apache.atlas.model.instance.EntityMutations.EntityOperation.CREATE; +import static org.apache.atlas.model.instance.EntityMutations.EntityOperation.UPDATE; +import static org.apache.atlas.repository.Constants.ASSET_LINK_EDGE_LABEL; +import static org.apache.atlas.repository.Constants.ATTRIBUTE_LINK; +import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; + +public class LinkPreProcessor extends AbstractResourcePreProcessor { + private static final Logger LOG = LoggerFactory.getLogger(LinkPreProcessor.class); + + private static final Pattern REGEX_ONSITE_URL = Pattern.compile("(?:[\\p{L}\\p{N}\\\\\\.\\#@\\$%\\+&;\\-_~,\\?=/!]+|\\#(\\w)+)"); + private static final Pattern REGEX_OFFSITE_URL = Pattern.compile("\\s*(?:(?:ht|f)tps?://|mailto:)[\\p{L}\\p{N}]" + + "[\\p{L}\\p{N}\\p{Zs}\\.\\#@\\$%\\+&;:\\-_~,\\?=/!\\(\\)]*+\\s*"); + private static final Predicate REGEX_ON_OFFSITE_URL = matchesEither(REGEX_ONSITE_URL, REGEX_OFFSITE_URL); + + public LinkPreProcessor(AtlasTypeRegistry typeRegistry, + EntityGraphRetriever entityRetriever) { + super(typeRegistry, entityRetriever); + } + + @Override + public void processAttributes(AtlasStruct entityStruct, EntityMutationContext context, EntityMutations.EntityOperation operation) throws AtlasBaseException { + if (LOG.isDebugEnabled()) { + LOG.debug("LinkPreProcessor.processAttributes: pre processing {}, {}", entityStruct.getAttribute(QUALIFIED_NAME), operation); + } + + AtlasEntity entity = (AtlasEntity) entityStruct; + + switch (operation) { + case CREATE: + processLinkCreate(entity); + break; + case UPDATE: + processLinkUpdate(entity, context.getVertex(entity.getGuid())); + break; + } + + } + + private void processLinkCreate(AtlasEntity linkEntity) throws AtlasBaseException { + validateLinkAttribute(linkEntity, CREATE.name()); + } + + private void processLinkUpdate(AtlasEntity linkEntity, AtlasVertex vertex) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processLinkUpdate"); + + try { + validateLinkAttribute(linkEntity, UPDATE.name()); + + authorizeResourceUpdate(linkEntity, vertex, ASSET_LINK_EDGE_LABEL); + } finally { + RequestContext.get().endMetricRecord(metricRecorder); + } + } + + /** + * Validate the link based on the provided operation. + * + * @param linkEntity The linkEntity to be processed. + * @param operation The operation to be performed. + * @throws AtlasBaseException If the link is not valid or empty. + */ + private void validateLinkAttribute(AtlasEntity linkEntity, String operation) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord(operation); + + String link = (String) linkEntity.getAttribute(ATTRIBUTE_LINK); + + if (link == null || link.isEmpty()) { + throw new AtlasBaseException("Link is empty for operation: " + operation); + } + if (!REGEX_ON_OFFSITE_URL.apply(link)) { + throw new AtlasBaseException("Please provide a valid URL for operation: " + operation); + } + + RequestContext.get().endMetricRecord(metricRecorder); + } + + private static Predicate matchesEither(final Pattern a, final Pattern b) { + return input -> a.matcher(input).matches() || b.matcher(input).matches(); + } + + @Override + public void processDelete(AtlasVertex vertex) throws AtlasBaseException { + authorizeResourceDelete(vertex); + } +} diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/resource/ReadmePreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/resource/ReadmePreProcessor.java new file mode 100644 index 00000000000..b2ca8ca11e7 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/resource/ReadmePreProcessor.java @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.repository.store.graph.v2.preprocessor.resource; + +import org.apache.atlas.RequestContext; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.model.instance.AtlasStruct; +import org.apache.atlas.model.instance.EntityMutations; +import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; +import org.apache.atlas.repository.store.graph.v2.EntityMutationContext; +import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.atlas.utils.AtlasPerfMetrics; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.atlas.repository.Constants.ASSET_README_EDGE_LABEL; +import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; + +public class ReadmePreProcessor extends AbstractResourcePreProcessor { + private static final Logger LOG = LoggerFactory.getLogger(ReadmePreProcessor.class); + + public ReadmePreProcessor(AtlasTypeRegistry typeRegistry, + EntityGraphRetriever entityRetriever) { + super(typeRegistry, entityRetriever); + } + + @Override + public void processAttributes(AtlasStruct entityStruct, EntityMutationContext context, EntityMutations.EntityOperation operation) throws AtlasBaseException { + if (LOG.isDebugEnabled()) { + LOG.debug("ReadmePreProcessor.processAttributes: pre processing {}, {}", entityStruct.getAttribute(QUALIFIED_NAME), operation); + } + + AtlasEntity entity = (AtlasEntity) entityStruct; + + switch (operation) { + case UPDATE: + processUpdateReadme(entity, context.getVertex(entity.getGuid())); + break; + } + } + + private void processUpdateReadme(AtlasStruct struct, AtlasVertex vertex) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processUpdateReadme"); + + AtlasEntity readmeEntity = (AtlasEntity) struct; + + try { + authorizeResourceUpdate(readmeEntity, vertex, ASSET_README_EDGE_LABEL); + } finally { + RequestContext.get().endMetricRecord(metricRecorder); + } + } + + @Override + public void processDelete(AtlasVertex vertex) throws AtlasBaseException { + authorizeResourceDelete(vertex); + } +} diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/sql/QueryCollectionPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/sql/QueryCollectionPreProcessor.java index 42af09f8bba..63e8ba42502 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/sql/QueryCollectionPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/sql/QueryCollectionPreProcessor.java @@ -18,34 +18,86 @@ package org.apache.atlas.repository.store.graph.v2.preprocessor.sql; +import org.apache.atlas.RequestContext; import org.apache.atlas.authorize.AtlasAuthorizationUtils; +import org.apache.atlas.discovery.EntityDiscoveryService; import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.featureflag.FeatureFlagStore; +import org.apache.atlas.model.discovery.AtlasSearchResult; +import org.apache.atlas.model.discovery.IndexSearchParams; import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.model.instance.AtlasEntityHeader; import org.apache.atlas.model.instance.AtlasStruct; import org.apache.atlas.model.instance.EntityMutations; +import org.apache.atlas.repository.graph.GraphHelper; import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.repository.store.graph.AtlasEntityStore; +import org.apache.atlas.repository.store.graph.v2.AtlasEntityStream; import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; import org.apache.atlas.repository.store.graph.v2.EntityMutationContext; +import org.apache.atlas.repository.store.graph.v2.EntityStream; import org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessor; +import org.apache.atlas.repository.store.users.KeycloakStore; +import org.apache.atlas.transformer.PreProcessorPoliciesTransformer; import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.atlas.utils.AtlasPerfMetrics; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; +import org.keycloak.representations.idm.RoleRepresentation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.HashSet; +import java.util.Arrays; +import java.util.stream.Collectors; + +import static org.apache.atlas.authorize.AtlasAuthorizerFactory.ATLAS_AUTHORIZER_IMPL; +import static org.apache.atlas.authorize.AtlasAuthorizerFactory.CURRENT_AUTHORIZER_IMPL; +import static org.apache.atlas.keycloak.client.AtlasKeycloakClient.getKeycloakClient; +import static org.apache.atlas.repository.Constants.ATTR_ADMIN_GROUPS; +import static org.apache.atlas.repository.Constants.ATTR_ADMIN_USERS; +import static org.apache.atlas.repository.Constants.ATTR_VIEWER_GROUPS; +import static org.apache.atlas.repository.Constants.ATTR_VIEWER_USERS; +import static org.apache.atlas.repository.Constants.CREATED_BY_KEY; +import static org.apache.atlas.repository.Constants.POLICY_ENTITY_TYPE; import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.PREFIX_QUERY_QN; import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.getUUID; +import static org.apache.atlas.repository.util.AtlasEntityUtils.mapOf; public class QueryCollectionPreProcessor implements PreProcessor { private static final Logger LOG = LoggerFactory.getLogger(QueryCollectionPreProcessor.class); - private static String qualifiedNameFormat = PREFIX_QUERY_QN + "%s/%s"; + private static final String qualifiedNameFormat = PREFIX_QUERY_QN + "%s/%s"; + private static final String COLL_ADMIN_ROLE_PATTERN = "collection_admins_%s"; + private static final String COLL_VIEWER_ROLE_PATTERN = "collection_viewer_%s"; private final AtlasTypeRegistry typeRegistry; private final EntityGraphRetriever entityRetriever; + private AtlasEntityStore entityStore; + private EntityDiscoveryService discovery; + private PreProcessorPoliciesTransformer transformer; + private FeatureFlagStore featureFlagStore; + private KeycloakStore keycloakStore; - public QueryCollectionPreProcessor(AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever) { + public QueryCollectionPreProcessor(AtlasTypeRegistry typeRegistry, + EntityDiscoveryService discovery, + EntityGraphRetriever entityRetriever, + FeatureFlagStore featureFlagStore, + AtlasEntityStore entityStore) { this.entityRetriever = entityRetriever; this.typeRegistry = typeRegistry; + this.entityStore = entityStore; + this.featureFlagStore = featureFlagStore; + this.discovery = discovery; + + transformer = new PreProcessorPoliciesTransformer(); + keycloakStore = new KeycloakStore(); } @Override @@ -68,17 +120,200 @@ public void processAttributes(AtlasStruct entityStruct, EntityMutationContext co } } - private void processCreate(AtlasStruct entity) { - entity.setAttribute(QUALIFIED_NAME, createQualifiedName()); + private void processCreate(AtlasStruct entity) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processCreateCollection"); + + try { + entity.setAttribute(QUALIFIED_NAME, createQualifiedName()); + + AtlasEntity collection = (AtlasEntity) entity; + + if (ATLAS_AUTHORIZER_IMPL.equalsIgnoreCase(CURRENT_AUTHORIZER_IMPL)) { + + createCollectionAdminRole(collection); + createCollectionViewerRole(collection); + + //create bootstrap policies + AtlasEntity.AtlasEntitiesWithExtInfo policies = transformer.transform(collection); + + try { + RequestContext.get().setSkipAuthorizationCheck(true); + EntityStream entityStream = new AtlasEntityStream(policies); + entityStore.createOrUpdate(entityStream, false); + LOG.info("Created bootstrap policies for collection {}", entity.getAttribute(QUALIFIED_NAME)); + } finally { + RequestContext.get().setSkipAuthorizationCheck(false); + } + } + } finally { + RequestContext.get().endMetricRecord(metricRecorder); + } } - private void processUpdate(AtlasStruct entity, AtlasVertex vertex) { + private void processUpdate(AtlasStruct entity, AtlasVertex vertex) throws AtlasBaseException { String vertexQnName = vertex.getProperty(QUALIFIED_NAME, String.class); + Set userGroupAttributesNames = new HashSet<>(Arrays.asList(ATTR_ADMIN_USERS, ATTR_ADMIN_GROUPS, ATTR_VIEWER_USERS, ATTR_VIEWER_GROUPS)); + + if (ATLAS_AUTHORIZER_IMPL.equalsIgnoreCase(CURRENT_AUTHORIZER_IMPL)) { + AtlasEntity collection = (AtlasEntity) entity; + AtlasEntityHeader existingCollEntity = entityRetriever.toAtlasEntityHeader(vertex, userGroupAttributesNames); + + updateCollectionAdminRole(collection, existingCollEntity, vertex); + updateCollectionViewerRole(collection, existingCollEntity); + } entity.setAttribute(QUALIFIED_NAME, vertexQnName); } - public static String createQualifiedName() { + @Override + public void processDelete(AtlasVertex vertex) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processDeleteCollection"); + + try { + AtlasEntity.Status collectionStatus = GraphHelper.getStatus(vertex); + + if (!AtlasEntity.Status.ACTIVE.equals(collectionStatus)) { + throw new AtlasBaseException("Collection is already deleted/purged"); + } + + if (ATLAS_AUTHORIZER_IMPL.equalsIgnoreCase(CURRENT_AUTHORIZER_IMPL)) { + String collectionGuid = GraphHelper.getGuid(vertex); + + //delete collection policies + List policies = getCollectionPolicies(collectionGuid); + RequestContext.get().setSkipAuthorizationCheck(true); + entityStore.deleteByIds(policies.stream().map(x -> x.getGuid()).collect(Collectors.toList())); + + //delete collection roles + String adminRoleName = String.format(COLL_ADMIN_ROLE_PATTERN, collectionGuid); + String viewerRoleName = String.format(COLL_VIEWER_ROLE_PATTERN, collectionGuid); + + keycloakStore.removeRoleByName(adminRoleName); + keycloakStore.removeRoleByName(viewerRoleName); + } + } finally { + RequestContext.get().endMetricRecord(metricRecorder); + RequestContext.get().setSkipAuthorizationCheck(false); + } + } + + private static String createQualifiedName() { return String.format(qualifiedNameFormat, AtlasAuthorizationUtils.getCurrentUserName(), getUUID()); } + + private RoleRepresentation createCollectionAdminRole(AtlasEntity collection) throws AtlasBaseException { + //create Admin role + List adminUsers = (List) collection.getAttribute(ATTR_ADMIN_USERS); + List adminGroups = (List) collection.getAttribute(ATTR_ADMIN_GROUPS); + //List adminRoles = (List) collection.getAttribute(ATTR_ADMIN_ROLES); + List adminRoles = new ArrayList<>(0); + + if (adminUsers == null) { + adminUsers = new ArrayList<>(); + } + String creatorUser = RequestContext.get().getUser(); + if (StringUtils.isNotEmpty(creatorUser)) { + adminUsers.add(creatorUser); + } + collection.setAttribute(ATTR_ADMIN_USERS, adminUsers); + + String adminRoleName = String.format(COLL_ADMIN_ROLE_PATTERN, collection.getGuid()); + return keycloakStore.createRoleForConnection(adminRoleName, true, adminUsers, adminGroups, adminRoles); + } + + private RoleRepresentation createCollectionViewerRole(AtlasEntity collection) throws AtlasBaseException { + //create viewers role + String viewerRoleName = String.format(COLL_VIEWER_ROLE_PATTERN, collection.getGuid()); + List viewerUsers = (List) collection.getAttribute(ATTR_VIEWER_USERS); + List viewerGroups = (List) collection.getAttribute(ATTR_VIEWER_GROUPS); + + return keycloakStore.createRoleForConnection(viewerRoleName, true, viewerUsers, viewerGroups, null); + } + + private void updateCollectionAdminRole(AtlasEntity collection, AtlasEntityHeader existingCollEntity, AtlasVertex vertex) throws AtlasBaseException { + String adminRoleName = String.format(COLL_ADMIN_ROLE_PATTERN, collection.getGuid()); + + RoleRepresentation representation = getKeycloakClient().getRoleByName(adminRoleName); + String creatorUser = vertex.getProperty(CREATED_BY_KEY, String.class); + + if (collection.hasAttribute(ATTR_ADMIN_USERS)) { + List newAdminUsers = (List) collection.getAttribute(ATTR_ADMIN_USERS); + List currentAdminUsers = (List) existingCollEntity.getAttribute(ATTR_ADMIN_USERS); + + if (CollectionUtils.isNotEmpty(newAdminUsers) || CollectionUtils.isNotEmpty(currentAdminUsers)) { + if (StringUtils.isNotEmpty(creatorUser) && !newAdminUsers.contains(creatorUser)) { + newAdminUsers.add(creatorUser); + } + collection.setAttribute(ATTR_ADMIN_USERS, newAdminUsers); + keycloakStore.updateRoleUsers(adminRoleName, currentAdminUsers, newAdminUsers, representation); + } + } + + if (collection.hasAttribute(ATTR_ADMIN_GROUPS)) { + List newAdminGroups = (List) collection.getAttribute(ATTR_ADMIN_GROUPS); + List currentAdminGroups =(List) existingCollEntity.getAttribute(ATTR_ADMIN_GROUPS); + + if (CollectionUtils.isNotEmpty(newAdminGroups) || CollectionUtils.isNotEmpty(currentAdminGroups)) { + keycloakStore.updateRoleGroups(adminRoleName, currentAdminGroups, newAdminGroups, representation); + } + } + + /*if (collection.hasAttribute(ATTR_ADMIN_ROLES)) { + List newAdminRoles = (List) collection.getAttribute(ATTR_ADMIN_ROLES); + List currentAdminRoles = (List) existingCollEntity.getAttribute(ATTR_ADMIN_ROLES); + + if (CollectionUtils.isNotEmpty(newAdminRoles) || CollectionUtils.isNotEmpty(currentAdminRoles)) { + keycloakStore.updateRoleRoles(adminRoleName, currentAdminRoles, newAdminRoles, rolesResource, representation); + } + }*/ + } + + private void updateCollectionViewerRole(AtlasEntity collection, AtlasEntityHeader existingCollEntity) throws AtlasBaseException { + String viewerRoleName = String.format(COLL_VIEWER_ROLE_PATTERN, collection.getGuid()); + RoleRepresentation representation = getKeycloakClient().getRoleByName(viewerRoleName); + + if (collection.hasAttribute(ATTR_VIEWER_USERS)) { + List newViewerUsers = (List) collection.getAttribute(ATTR_VIEWER_USERS); + List currentViewerUsers = (List) existingCollEntity.getAttribute(ATTR_VIEWER_USERS); + + if (CollectionUtils.isNotEmpty(newViewerUsers) || CollectionUtils.isNotEmpty(currentViewerUsers)) { + keycloakStore.updateRoleUsers(viewerRoleName, currentViewerUsers, newViewerUsers, representation); + } + } + + if (collection.hasAttribute(ATTR_VIEWER_GROUPS)) { + List newViewerGroups = (List) collection.getAttribute(ATTR_VIEWER_GROUPS); + List currentViewerGroups =(List) existingCollEntity.getAttribute(ATTR_VIEWER_GROUPS); + + if (CollectionUtils.isNotEmpty(newViewerGroups) || CollectionUtils.isNotEmpty(currentViewerGroups)) { + keycloakStore.updateRoleGroups(viewerRoleName, currentViewerGroups, newViewerGroups, representation); + } + } + } + + private List getCollectionPolicies(String guid) throws AtlasBaseException { + List ret = new ArrayList<>(); + + IndexSearchParams indexSearchParams = new IndexSearchParams(); + Map dsl = new HashMap<>(); + + List mustClauseList = new ArrayList(); + mustClauseList.add(mapOf("term", mapOf("__typeName.keyword", POLICY_ENTITY_TYPE))); + mustClauseList.add(mapOf("term", mapOf("__state", "ACTIVE"))); + + + mustClauseList.add(mapOf("wildcard", mapOf(QUALIFIED_NAME, guid + "/*"))); + + dsl.put("query", mapOf("bool", mapOf("must", mustClauseList))); + + indexSearchParams.setDsl(dsl); + indexSearchParams.setSuppressLogs(true); + + AtlasSearchResult result = discovery.directIndexSearch(indexSearchParams); + if (result != null) { + ret = result.getEntities(); + } + + return ret; + } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/sql/QueryFolderPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/sql/QueryFolderPreProcessor.java index c0100ac175c..41a78e5992e 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/sql/QueryFolderPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/sql/QueryFolderPreProcessor.java @@ -18,24 +18,36 @@ package org.apache.atlas.repository.store.graph.v2.preprocessor.sql; +import org.apache.atlas.RequestContext; +import org.apache.atlas.model.instance.*; +import org.apache.atlas.repository.graph.GraphHelper; import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.authorize.AtlasAuthorizationUtils; import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.model.instance.AtlasEntity; -import org.apache.atlas.model.instance.AtlasStruct; -import org.apache.atlas.model.instance.EntityMutations; +import org.apache.atlas.repository.graphdb.AtlasEdgeDirection; import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2; import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; import org.apache.atlas.repository.store.graph.v2.EntityMutationContext; import org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessor; +import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.atlas.utils.AtlasPerfMetrics; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; -import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.COLLECTION_QUALIFIED_NAME; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; + +import static org.apache.atlas.repository.Constants.*; +import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.updateQueryResourceAttributes; import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.getUUID; +import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.COLLECTION_QUALIFIED_NAME; +import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.CHILDREN_FOLDERS; +import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.CHILDREN_QUERIES; +import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.PARENT_QUALIFIED_NAME; public class QueryFolderPreProcessor implements PreProcessor { private static final Logger LOG = LoggerFactory.getLogger(QueryFolderPreProcessor.class); @@ -48,6 +60,7 @@ public class QueryFolderPreProcessor implements PreProcessor { public QueryFolderPreProcessor(AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever) { this.entityRetriever = entityRetriever; this.typeRegistry = typeRegistry; + } @Override @@ -65,7 +78,7 @@ public void processAttributes(AtlasStruct entityStruct, EntityMutationContext co processCreate(entity); break; case UPDATE: - processUpdate(entity, vertex); + processUpdate(entity, vertex, context); break; } } @@ -80,12 +93,180 @@ private void processCreate(AtlasStruct entity) throws AtlasBaseException { entity.setAttribute(QUALIFIED_NAME, createQualifiedName(collectionQualifiedName)); } - private void processUpdate(AtlasStruct entity, AtlasVertex vertex) { - String vertexQnName = vertex.getProperty(QUALIFIED_NAME, String.class); + private void processUpdate(AtlasEntity entity, AtlasVertex vertex, EntityMutationContext context) throws AtlasBaseException { + String currentCollectionQualifiedName = vertex.getProperty(COLLECTION_QUALIFIED_NAME, String.class); + String newCollectionQualifiedName = updateQueryResourceAttributes(typeRegistry, entityRetriever, entity, vertex, context); + + if(StringUtils.isNotEmpty(newCollectionQualifiedName) && !currentCollectionQualifiedName.equals(newCollectionQualifiedName)) { + processParentCollectionUpdation(vertex, currentCollectionQualifiedName, newCollectionQualifiedName); + LOG.debug("Moved folder {} from collection {} to collection {}", entity.getAttribute(QUALIFIED_NAME), currentCollectionQualifiedName, newCollectionQualifiedName); + } + + } + + + private void processParentCollectionUpdation(AtlasVertex folderVertex, String currentCollectionQualifiedName, String newCollectionQualifiedName) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder folderProcessMetric = RequestContext.get().startMetricRecord("processParentCollectionUpdation"); + + String folderQualifiedName = folderVertex.getProperty(QUALIFIED_NAME, String.class); + String updatedFolderQualifiedName = folderQualifiedName.replaceAll(currentCollectionQualifiedName, newCollectionQualifiedName); + + /** + * 1. Move all the queries to new parent first + * 2. Move all the child folders to new parent + * 3. Update the qualified name of current folder + * 4. Recursively find the child folders and move child queries to new collection + */ + moveQueriesToDifferentCollection(folderVertex, currentCollectionQualifiedName, newCollectionQualifiedName, updatedFolderQualifiedName); + + Iterator childrenFolders = getActiveChildren(folderVertex, CHILDREN_FOLDERS); + + while (childrenFolders.hasNext()) { + AtlasVertex nestedFolderVertex = childrenFolders.next(); + if (nestedFolderVertex != null) { + updateChildAttributesOnUpdatingCollection(nestedFolderVertex, currentCollectionQualifiedName, newCollectionQualifiedName, updatedFolderQualifiedName); + /** + * Recursively find the child folders and move child queries to new collection + * folder1 -> folder2 -> query1 + * When we will move folder1 to new collection, recursively it will find folder2 + * Then it will move all the children of folder2 also to new collection + */ + processParentCollectionUpdation(nestedFolderVertex, currentCollectionQualifiedName, newCollectionQualifiedName); + + LOG.info("Moved nested folder into new collection {}", newCollectionQualifiedName); + } + } + + LOG.info("Moved current folder with qualified name {} into new collection {}", folderQualifiedName, newCollectionQualifiedName); + + RequestContext.get().endMetricRecord(folderProcessMetric); + } + + /** + * Move all child queries to new collection and update the qualified name of folder + * @param folderVertex Parent folder vertex + * @param currentCollectionQualifiedName Current collection qualified name + * @param newCollectionQualifiedName New collection qualified name + * @param folderQualifiedName Qualified name of folder + * @throws AtlasBaseException + */ + private void moveQueriesToDifferentCollection(AtlasVertex folderVertex, String currentCollectionQualifiedName, + String newCollectionQualifiedName, String folderQualifiedName) throws AtlasBaseException { + + AtlasPerfMetrics.MetricRecorder queryProcessMetric = RequestContext.get().startMetricRecord("moveQueriesToDifferentCollection"); + Iterator childrenQueriesIterator = getActiveChildren(folderVertex, CHILDREN_QUERIES); + + //Update all the children query attribute + while (childrenQueriesIterator.hasNext()) { + AtlasVertex queryVertex = childrenQueriesIterator.next(); + if(queryVertex != null) { + updateChildAttributesOnUpdatingCollection(queryVertex, currentCollectionQualifiedName, + newCollectionQualifiedName, folderQualifiedName); + } + } + + RequestContext.get().endMetricRecord(queryProcessMetric); + } + + /** + * Get all the active children of folder + * @param folderVertex Parent folder vertex + * @param childrenEdgeLabel Edge label of children + * @return Iterator of children vertices + */ + private Iterator getActiveChildren(AtlasVertex folderVertex, String childrenEdgeLabel) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("getActiveChildren"); + try { + return folderVertex.query() + .direction(AtlasEdgeDirection.OUT) + .label(childrenEdgeLabel) + .has(STATE_PROPERTY_KEY, ACTIVE_STATE_VALUE) + .vertices() + .iterator(); + } catch (Exception e) { + LOG.error("Error while getting active children of folder", e); + throw new AtlasBaseException(AtlasErrorCode.INTERNAL_ERROR, e); + } + finally { + RequestContext.get().endMetricRecord(metricRecorder); + } - entity.setAttribute(QUALIFIED_NAME, vertexQnName); } + /** + * Update the child attributes on updating collection of parent folder + * @param childVertex Child vertex, could be query or folder + * @param currentCollectionQualifiedName Collection qualified name of parent folder / current collection + * @param newCollectionQualifiedName New collection qualified name of parent folder/ new collection + * @param folderQualifiedName Qualified name of parent folder + */ + private void updateChildAttributesOnUpdatingCollection(AtlasVertex childVertex, String currentCollectionQualifiedName, String newCollectionQualifiedName, + String folderQualifiedName) { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("updateChildAttributesOnUpdatingCollection"); + Map updatedAttributes = new HashMap<>(); + String qualifiedName = childVertex.getProperty(QUALIFIED_NAME, String.class); + String updatedQualifiedName = qualifiedName.replaceAll(currentCollectionQualifiedName, newCollectionQualifiedName); + + if (!qualifiedName.equals(updatedQualifiedName)) { + AtlasGraphUtilsV2.setEncodedProperty(childVertex, QUALIFIED_NAME, updatedQualifiedName); + updatedAttributes.put(QUALIFIED_NAME, updatedQualifiedName); + } + + if(!currentCollectionQualifiedName.equals(newCollectionQualifiedName)) { + AtlasGraphUtilsV2.setEncodedProperty(childVertex, COLLECTION_QUALIFIED_NAME, newCollectionQualifiedName); + updatedAttributes.put(COLLECTION_QUALIFIED_NAME, newCollectionQualifiedName); + } + + AtlasGraphUtilsV2.setEncodedProperty(childVertex, PARENT_QUALIFIED_NAME, folderQualifiedName); + + //update system properties + GraphHelper.setModifiedByAsString(childVertex, RequestContext.get().getUser()); + GraphHelper.setModifiedTime(childVertex, System.currentTimeMillis()); + + updatedAttributes.put(PARENT_QUALIFIED_NAME, folderQualifiedName); + + //Record the updated child entities, it will be used to send notification and store audit logs + recordUpdatedChildEntities(childVertex, updatedAttributes); + + RequestContext.get().endMetricRecord(metricRecorder); + } + + /** + * Record the updated child entities, it will be used to send notification and store audit logs + * @param entityVertex Child entity vertex + * @param updatedAttributes Updated attributes while updating required attributes on updating collection + */ + private void recordUpdatedChildEntities(AtlasVertex entityVertex, Map updatedAttributes) { + RequestContext requestContext = RequestContext.get(); + AtlasPerfMetrics.MetricRecorder metricRecorder = requestContext.startMetricRecord("recordUpdatedChildEntities"); + AtlasEntity entity = new AtlasEntity(); + entity = entityRetriever.mapSystemAttributes(entityVertex, entity); + entity.setAttributes(updatedAttributes); + requestContext.cacheDifferentialEntity(new AtlasEntity(entity)); + + AtlasEntityType entityType = typeRegistry.getEntityTypeByName(entity.getTypeName()); + + //Add the min info attributes to entity header to be sent as part of notification + if(entityType != null) { + AtlasEntity finalEntity = entity; + entityType.getMinInfoAttributes().values().stream().filter(attribute -> !updatedAttributes.containsKey(attribute.getName())).forEach(attribute -> { + Object attrValue = null; + try { + attrValue = entityRetriever.getVertexAttribute(entityVertex, attribute); + } catch (AtlasBaseException e) { + LOG.error("Error while getting vertex attribute", e); + } + if(attrValue != null) { + finalEntity.setAttribute(attribute.getName(), attrValue); + } + }); + requestContext.recordEntityUpdate(new AtlasEntityHeader(finalEntity)); + } + + requestContext.endMetricRecord(metricRecorder); + } + + public static String createQualifiedName(String collectionQualifiedName) { return String.format(qualifiedNameFormat, collectionQualifiedName, AtlasAuthorizationUtils.getCurrentUserName(), getUUID()); } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/sql/QueryPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/sql/QueryPreProcessor.java index 7e0dc5456a8..02bcd71ae69 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/sql/QueryPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/sql/QueryPreProcessor.java @@ -35,6 +35,7 @@ import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.COLLECTION_QUALIFIED_NAME; +import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.updateQueryResourceAttributes; import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.getUUID; public class QueryPreProcessor implements PreProcessor { @@ -65,7 +66,7 @@ public void processAttributes(AtlasStruct entityStruct, EntityMutationContext co processCreateQueryCollection(entity); break; case UPDATE: - processUpdateQueryCollection(entity, vertex); + processUpdateQueryCollection(entity, vertex, context); break; } } @@ -80,10 +81,8 @@ private void processCreateQueryCollection(AtlasStruct entity) throws AtlasBaseEx entity.setAttribute(QUALIFIED_NAME, createQualifiedName(collectionQualifiedName)); } - private void processUpdateQueryCollection(AtlasStruct entity, AtlasVertex vertex) { - String vertexQnName = vertex.getProperty(QUALIFIED_NAME, String.class); - - entity.setAttribute(QUALIFIED_NAME, vertexQnName); + private void processUpdateQueryCollection(AtlasEntity entity, AtlasVertex vertex, EntityMutationContext context) throws AtlasBaseException { + updateQueryResourceAttributes(typeRegistry, entityRetriever, entity, vertex, context); } public static String createQualifiedName(String collectionQualifiedName) { diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/tasks/ClassificationPropagateTaskFactory.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/tasks/ClassificationPropagateTaskFactory.java index b3320c60763..ca32e234087 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/tasks/ClassificationPropagateTaskFactory.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/tasks/ClassificationPropagateTaskFactory.java @@ -37,13 +37,33 @@ public class ClassificationPropagateTaskFactory implements TaskFactory { private static final Logger LOG = LoggerFactory.getLogger(ClassificationPropagateTaskFactory.class); public static final String CLASSIFICATION_PROPAGATION_ADD = "CLASSIFICATION_PROPAGATION_ADD"; + + //This should be used when referencing vertex to which classification is directly attached public static final String CLASSIFICATION_PROPAGATION_DELETE = "CLASSIFICATION_PROPAGATION_DELETE"; + + /* This should be used when referencing vertex to which classification is not directly attached but it is propagated + * e.g. t0 -> p0 -> t1 + * tag is on t0 propagating to p0,t1, + * deleting p0 should remove all propagations further to p0 for tag which is propagating from t0 + */ + public static final String CLASSIFICATION_ONLY_PROPAGATION_DELETE = "CLASSIFICATION_ONLY_PROPAGATION_DELETE"; + + public static final String CLASSIFICATION_ONLY_PROPAGATION_DELETE_ON_HARD_DELETE = "CLASSIFICATION_ONLY_PROPAGATION_DELETE_ON_HARD_DELETE"; + + public static final String CLASSIFICATION_REFRESH_PROPAGATION = "CLASSIFICATION_REFRESH_PROPAGATION"; + public static final String CLASSIFICATION_PROPAGATION_RELATIONSHIP_UPDATE = "CLASSIFICATION_PROPAGATION_RELATIONSHIP_UPDATE"; - private static final List supportedTypes = new ArrayList() {{ + + + public static final List supportedTypes = new ArrayList() {{ add(CLASSIFICATION_PROPAGATION_ADD); add(CLASSIFICATION_PROPAGATION_DELETE); + add(CLASSIFICATION_ONLY_PROPAGATION_DELETE); + add(CLASSIFICATION_ONLY_PROPAGATION_DELETE_ON_HARD_DELETE); + add(CLASSIFICATION_REFRESH_PROPAGATION); add(CLASSIFICATION_PROPAGATION_RELATIONSHIP_UPDATE); + }}; private final AtlasGraph graph; @@ -70,6 +90,15 @@ public org.apache.atlas.tasks.AbstractTask create(AtlasTask task) { case CLASSIFICATION_PROPAGATION_DELETE: return new ClassificationPropagationTasks.Delete(task, graph, entityGraphMapper, deleteDelegate, relationshipStore); + case CLASSIFICATION_ONLY_PROPAGATION_DELETE: + return new ClassificationPropagationTasks.DeleteOnlyPropagations(task, graph, entityGraphMapper, deleteDelegate, relationshipStore); + + case CLASSIFICATION_ONLY_PROPAGATION_DELETE_ON_HARD_DELETE: + return new ClassificationPropagationTasks.DeleteOnlyPropagationsOnHardDelete(task, graph, entityGraphMapper, deleteDelegate, relationshipStore); + + case CLASSIFICATION_REFRESH_PROPAGATION: + return new ClassificationPropagationTasks.RefreshPropagation(task, graph, entityGraphMapper, deleteDelegate, relationshipStore); + case CLASSIFICATION_PROPAGATION_RELATIONSHIP_UPDATE: return new ClassificationPropagationTasks.UpdateRelationship(task, graph, entityGraphMapper, deleteDelegate, relationshipStore); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/tasks/ClassificationPropagationTasks.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/tasks/ClassificationPropagationTasks.java index aec5b0cf7bb..6cb3b85d834 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/tasks/ClassificationPropagationTasks.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/tasks/ClassificationPropagationTasks.java @@ -20,7 +20,6 @@ import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.instance.AtlasRelationship; import org.apache.atlas.model.tasks.AtlasTask; -import org.apache.atlas.repository.graphdb.AtlasEdge; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.store.graph.AtlasRelationshipStore; import org.apache.atlas.repository.store.graph.v1.DeleteHandlerDelegate; @@ -28,6 +27,7 @@ import org.apache.atlas.type.AtlasType; import java.util.Map; +import java.util.Set; public class ClassificationPropagationTasks { public static class Add extends ClassificationTask { @@ -37,11 +37,12 @@ public Add(AtlasTask task, AtlasGraph graph, EntityGraphMapper entityGraphMapper @Override protected void run(Map parameters) throws AtlasBaseException { - String entityGuid = (String) parameters.get(PARAM_ENTITY_GUID); - String classificationVertexId = (String) parameters.get(PARAM_CLASSIFICATION_VERTEX_ID); - String relationshipGuid = (String) parameters.get(PARAM_RELATIONSHIP_GUID); + String entityGuid = (String) parameters.get(PARAM_ENTITY_GUID); + String classificationVertexId = (String) parameters.get(PARAM_CLASSIFICATION_VERTEX_ID); + String relationshipGuid = (String) parameters.get(PARAM_RELATIONSHIP_GUID); + Boolean previousRestrictPropagationThroughLineage = (Boolean) parameters.get(PARAM_PREVIOUS_CLASSIFICATION_RESTRICT_PROPAGATE_THROUGH_LINEAGE); - entityGraphMapper.propagateClassification(entityGuid, classificationVertexId, relationshipGuid); + entityGraphMapper.propagateClassification(entityGuid, classificationVertexId, relationshipGuid, previousRestrictPropagationThroughLineage); } } @@ -59,6 +60,54 @@ protected void run(Map parameters) throws AtlasBaseException { } } + // TODO: Will be deprecated + public static class DeleteOnlyPropagations extends ClassificationTask { + public DeleteOnlyPropagations(AtlasTask task, AtlasGraph graph, EntityGraphMapper entityGraphMapper, DeleteHandlerDelegate deleteDelegate, AtlasRelationshipStore relationshipStore) { + super(task, graph, entityGraphMapper, deleteDelegate, relationshipStore); + } + + @Override + protected void run(Map parameters) throws AtlasBaseException { + if (parameters.get(PARAM_DELETED_EDGE_IDS) != null) { + Set deletedEdgeIds = AtlasType.fromJson((String) parameters.get(PARAM_DELETED_EDGE_IDS), Set.class); + entityGraphMapper.deleteClassificationOnlyPropagation(deletedEdgeIds); + } else { + String deletedEdgeId = (String) parameters.get(PARAM_DELETED_EDGE_ID); + String classificationVertexId = (String) parameters.get(PARAM_CLASSIFICATION_VERTEX_ID); + entityGraphMapper.deleteClassificationOnlyPropagation(deletedEdgeId, classificationVertexId); + } + } + } + + // TODO: Will be deprecated + public static class DeleteOnlyPropagationsOnHardDelete extends ClassificationTask { + public DeleteOnlyPropagationsOnHardDelete(AtlasTask task, AtlasGraph graph, EntityGraphMapper entityGraphMapper, DeleteHandlerDelegate deleteDelegate, AtlasRelationshipStore relationshipStore) { + super(task, graph, entityGraphMapper, deleteDelegate, relationshipStore); + } + + @Override + protected void run(Map parameters) throws AtlasBaseException { + String classificationVertexId = (String) parameters.get(PARAM_CLASSIFICATION_VERTEX_ID); + String referencedVertexId = (String) parameters.get(PARAM_REFERENCED_VERTEX_ID); + boolean isTermEntityEdge = (boolean) parameters.get(PARAM_IS_TERM_ENTITY_EDGE); + + entityGraphMapper.deleteClassificationOnlyPropagation(classificationVertexId, referencedVertexId, isTermEntityEdge); + } + } + + public static class RefreshPropagation extends ClassificationTask { + public RefreshPropagation(AtlasTask task, AtlasGraph graph, EntityGraphMapper entityGraphMapper, DeleteHandlerDelegate deleteDelegate, AtlasRelationshipStore relationshipStore) { + super(task, graph, entityGraphMapper, deleteDelegate, relationshipStore); + } + + @Override + protected void run(Map parameters) throws AtlasBaseException { + String classificationVertexId = (String) parameters.get(PARAM_CLASSIFICATION_VERTEX_ID); + + entityGraphMapper.classificationRefreshPropagation(classificationVertexId); + } + } + public static class UpdateRelationship extends ClassificationTask { public UpdateRelationship(AtlasTask task, AtlasGraph graph, EntityGraphMapper entityGraphMapper, DeleteHandlerDelegate deleteDelegate, AtlasRelationshipStore relationshipStore) { super(task, graph, entityGraphMapper, deleteDelegate, relationshipStore); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/tasks/ClassificationTask.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/tasks/ClassificationTask.java index 00c9caaa7a6..03094d8de56 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/tasks/ClassificationTask.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/tasks/ClassificationTask.java @@ -22,15 +22,13 @@ import org.apache.atlas.exception.EntityNotFoundException; import org.apache.atlas.model.instance.AtlasRelationship; import org.apache.atlas.model.tasks.AtlasTask; -import org.apache.atlas.repository.graphdb.AtlasEdge; -import org.apache.atlas.repository.graphdb.AtlasElement; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.store.graph.AtlasRelationshipStore; import org.apache.atlas.repository.store.graph.v1.DeleteHandlerDelegate; -import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2; import org.apache.atlas.repository.store.graph.v2.EntityGraphMapper; import org.apache.atlas.tasks.AbstractTask; import org.apache.atlas.type.AtlasType; +import org.apache.atlas.utils.AtlasPerfMetrics; import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; @@ -39,20 +37,23 @@ import java.util.HashMap; import java.util.Map; -import static org.apache.atlas.model.tasks.AtlasTask.Status.COMPLETE; -import static org.apache.atlas.model.tasks.AtlasTask.Status.FAILED; +import static org.apache.atlas.model.tasks.AtlasTask.Status.*; import static org.apache.atlas.repository.store.graph.v2.tasks.ClassificationPropagateTaskFactory.CLASSIFICATION_PROPAGATION_RELATIONSHIP_UPDATE; -import static org.apache.atlas.type.Constants.PENDING_TASKS_PROPERTY_KEY; public abstract class ClassificationTask extends AbstractTask { private static final Logger LOG = LoggerFactory.getLogger(ClassificationTask.class); - protected static final String PARAM_ENTITY_GUID = "entityGuid"; - protected static final String PARAM_CLASSIFICATION_VERTEX_ID = "classificationVertexId"; - protected static final String PARAM_RELATIONSHIP_GUID = "relationshipGuid"; - protected static final String PARAM_RELATIONSHIP_OBJECT = "relationshipObject"; - protected static final String PARAM_RELATIONSHIP_EDGE_ID = "relationshipEdgeId"; - + public static final String PARAM_ENTITY_GUID = "entityGuid"; + public static final String PARAM_DELETED_EDGE_IDS = "deletedEdgeIds"; // TODO: Will be deprecated + public static final String PARAM_DELETED_EDGE_ID = "deletedEdgeId"; + public static final String PARAM_CLASSIFICATION_VERTEX_ID = "classificationVertexId"; + public static final String PARAM_RELATIONSHIP_GUID = "relationshipGuid"; + public static final String PARAM_RELATIONSHIP_OBJECT = "relationshipObject"; + public static final String PARAM_RELATIONSHIP_EDGE_ID = "relationshipEdgeId"; + public static final String PARAM_REFERENCED_VERTEX_ID = "referencedVertexId"; + public static final String PARAM_IS_TERM_ENTITY_EDGE = "isTermEntityEdge"; + public static final String PARAM_PREVIOUS_CLASSIFICATION_RESTRICT_PROPAGATE_THROUGH_LINEAGE = "previousRestrictPropagationThroughLineage"; + protected final AtlasGraph graph; protected final EntityGraphMapper entityGraphMapper; protected final DeleteHandlerDelegate deleteDelegate; @@ -72,8 +73,9 @@ public ClassificationTask(AtlasTask task, } @Override - public AtlasTask.Status perform() throws Exception { + public AtlasTask.Status perform() throws AtlasBaseException { Map params = getTaskDef().getParameters(); + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord(getTaskGuid()); if (MapUtils.isEmpty(params)) { LOG.warn("Task: {}: Unable to process task: Parameters is not readable!", getTaskGuid()); @@ -92,22 +94,34 @@ public AtlasTask.Status perform() throws Exception { RequestContext.get().setUser(userName, null); try { + setStatus(IN_PROGRESS); + run(params); setStatus(COMPLETE); - } catch (Exception e) { + } catch (AtlasBaseException e) { LOG.error("Task: {}: Error performing task!", getTaskGuid(), e); setStatus(FAILED); throw e; } finally { + RequestContext.get().endMetricRecord(metricRecorder); graph.commit(); } return getStatus(); } + public static Map toParameters(String entityGuid, String classificationVertexId, String relationshipGuid, Boolean restrictPropagationThroughLineage) { + return new HashMap() {{ + put(PARAM_ENTITY_GUID, entityGuid); + put(PARAM_CLASSIFICATION_VERTEX_ID, classificationVertexId); + put(PARAM_RELATIONSHIP_GUID, relationshipGuid); + put(PARAM_PREVIOUS_CLASSIFICATION_RESTRICT_PROPAGATE_THROUGH_LINEAGE, restrictPropagationThroughLineage); + }}; + } + public static Map toParameters(String entityGuid, String classificationVertexId, String relationshipGuid) { return new HashMap() {{ put(PARAM_ENTITY_GUID, entityGuid); @@ -116,6 +130,21 @@ public static Map toParameters(String entityGuid, String classif }}; } + public static Map toParameters(String deletedEdgeId, String classificationVertexId) { + return new HashMap() {{ + put(PARAM_DELETED_EDGE_ID, deletedEdgeId); + put(PARAM_CLASSIFICATION_VERTEX_ID, classificationVertexId); + }}; + } + + public static Map toParameters(String classificationVertexId, String referencedVertexId, boolean isTermEntityEdge) { + return new HashMap() {{ + put(PARAM_CLASSIFICATION_VERTEX_ID, classificationVertexId); + put(PARAM_REFERENCED_VERTEX_ID, referencedVertexId); + put(PARAM_IS_TERM_ENTITY_EDGE, isTermEntityEdge); + }}; + } + public static Map toParameters(String relationshipEdgeId, AtlasRelationship relationship) { return new HashMap() {{ put(PARAM_RELATIONSHIP_EDGE_ID, relationshipEdgeId); @@ -123,18 +152,26 @@ public static Map toParameters(String relationshipEdgeId, AtlasR }}; } + public static Map toParameters(String classificationId) { + return new HashMap() {{ + put(PARAM_CLASSIFICATION_VERTEX_ID, classificationId); + }}; + } + protected void setStatus(AtlasTask.Status status) { super.setStatus(status); + LOG.info(String.format("ClassificationTask status is set %s for the task: %s ", status, super.getTaskGuid())); try { - if (getTaskType() == CLASSIFICATION_PROPAGATION_RELATIONSHIP_UPDATE) { + if (CLASSIFICATION_PROPAGATION_RELATIONSHIP_UPDATE.equals(getTaskType())) { entityGraphMapper.removePendingTaskFromEdge((String) getTaskDef().getParameters().get(PARAM_RELATIONSHIP_EDGE_ID), getTaskGuid()); } else { entityGraphMapper.removePendingTaskFromEntity((String) getTaskDef().getParameters().get(PARAM_ENTITY_GUID), getTaskGuid()); } } catch (EntityNotFoundException | AtlasBaseException e) { - LOG.error("Error updating associated element for: {}", getTaskGuid(), e); + LOG.warn("Error updating associated element for: {}", getTaskGuid(), e); } + graph.commit(); } protected abstract void run(Map parameters) throws AtlasBaseException; diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/tasks/MeaningsTask.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/tasks/MeaningsTask.java new file mode 100644 index 00000000000..42b2bf76d7e --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/tasks/MeaningsTask.java @@ -0,0 +1,116 @@ +package org.apache.atlas.repository.store.graph.v2.tasks; + +import org.apache.atlas.RequestContext; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.exception.EntityNotFoundException; +import org.apache.atlas.model.tasks.AtlasTask; +import org.apache.atlas.repository.graphdb.AtlasGraph; +import org.apache.atlas.repository.store.graph.v2.AtlasEntityStoreV2; +import org.apache.atlas.repository.store.graph.v2.EntityGraphMapper; +import org.apache.atlas.repository.store.graph.v2.preprocessor.glossary.TermPreProcessor; +import org.apache.atlas.tasks.AbstractTask; +import org.apache.commons.collections4.MapUtils; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashMap; +import java.util.Map; + +import static org.apache.atlas.model.tasks.AtlasTask.Status.*; +import static org.apache.atlas.repository.store.graph.v2.tasks.MeaningsTaskFactory.UPDATE_ENTITY_MEANINGS_ON_TERM_HARD_DELETE; + +public abstract class MeaningsTask extends AbstractTask { + private static final Logger LOG = LoggerFactory.getLogger(MeaningsTask.class); + protected static final String PARAM_ENTITY_GUID = "entityGuid"; + protected static final String PARAM_ENTITY_QUALIFIED_NAME = "entityQName"; + protected static final String PARAM_ENTITY_UPDATED_QUALIFIED_NAME = "updatedEntityQName"; + protected static final String PARAM_UPDATED_TERM_NAME = "updatedTermName"; + protected static final String PARAM_CURRENT_TERM_NAME = "currentTermName"; + + + protected final EntityGraphMapper entityGraphMapper; + protected final AtlasGraph graph; + protected final TermPreProcessor preprocessor; + protected final AtlasEntityStoreV2 entityStoreV2; + + + public MeaningsTask(AtlasTask task, EntityGraphMapper entityGraphMapper, + AtlasGraph graph, TermPreProcessor preprocessor, AtlasEntityStoreV2 entityStoreV2) { + super(task); + this.entityGraphMapper = entityGraphMapper; + this.graph = graph; + this.preprocessor = preprocessor; + this.entityStoreV2 = entityStoreV2; + } + + @Override + public AtlasTask.Status perform() throws Exception { + Map params; + params = getTaskDef().getParameters(); + if (!MapUtils.isEmpty(params)) { + String userName = getTaskDef().getCreatedBy(); + + if (StringUtils.isEmpty(userName)) { + LOG.warn("Task: {}: Unable to process task as user name is empty!", getTaskGuid()); + + return FAILED; + } + + RequestContext.get().setUser(userName, null); + try { + setStatus(IN_PROGRESS); + + run(params); + + setStatus(COMPLETE); + } catch (Exception e) { + LOG.error("Task: {}: Error performing task!", getTaskGuid(), e); + + setStatus(FAILED); + + throw e; + } finally { + graph.commit(); + } + return getStatus(); + } else { + LOG.warn("Task: {}: Unable to process task: Parameters is not readable!", getTaskGuid()); + + return FAILED; + } + } + + public static Map toParameters(String currentTerm, String updateTerm, String termQName, String updatedTermQName, String termGuid) { + return new HashMap() {{ + put(PARAM_ENTITY_GUID, termGuid); + put(PARAM_ENTITY_QUALIFIED_NAME, termQName); + put(PARAM_ENTITY_UPDATED_QUALIFIED_NAME, updatedTermQName); + put(PARAM_CURRENT_TERM_NAME, currentTerm); + put(PARAM_UPDATED_TERM_NAME, updateTerm); + }}; + } + public static Map toParameters(String termName, String termQName, String termGuid){ + return new HashMap(){{ + put(PARAM_ENTITY_QUALIFIED_NAME, termQName); + put(PARAM_ENTITY_GUID, termGuid); + put(PARAM_CURRENT_TERM_NAME, termName); + }}; + } + + protected void setStatus(AtlasTask.Status status) { + super.setStatus(status); + try { + if(UPDATE_ENTITY_MEANINGS_ON_TERM_HARD_DELETE.equals(getTaskType())){ + LOG.info("Entity Vertex Deleted, No Need to remove pending task for: {} ",getTaskGuid()); + }else { + entityGraphMapper.removePendingTaskFromEntity((String) getTaskDef().getParameters().get(PARAM_ENTITY_GUID), getTaskGuid()); + } + } catch (EntityNotFoundException e) { + LOG.error("Error updating associated element for: {}", getTaskGuid(), e); + } + + } + + protected abstract void run(Map parameters) throws AtlasBaseException; +} diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/tasks/MeaningsTaskFactory.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/tasks/MeaningsTaskFactory.java new file mode 100644 index 00000000000..9892362ecb0 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/tasks/MeaningsTaskFactory.java @@ -0,0 +1,67 @@ +package org.apache.atlas.repository.store.graph.v2.tasks; + +import org.apache.atlas.model.tasks.AtlasTask; +import org.apache.atlas.repository.graphdb.AtlasGraph; +import org.apache.atlas.repository.store.graph.v2.AtlasEntityStoreV2; +import org.apache.atlas.repository.store.graph.v2.EntityGraphMapper; +import org.apache.atlas.repository.store.graph.v2.preprocessor.glossary.TermPreProcessor; +import org.apache.atlas.tasks.TaskFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import javax.inject.Inject; +import java.util.ArrayList; +import java.util.List; + +@Component +public class MeaningsTaskFactory implements TaskFactory { + private static final Logger LOG = LoggerFactory.getLogger(MeaningsTaskFactory.class); + + public static final String UPDATE_ENTITY_MEANINGS_ON_TERM_UPDATE = "UPDATE_ENTITY_MEANINGS_ON_TERM_UPDATE"; + public static final String UPDATE_ENTITY_MEANINGS_ON_TERM_SOFT_DELETE = "UPDATE_ENTITY_MEANINGS_ON_TERM_SOFT_DELETE"; + public static final String UPDATE_ENTITY_MEANINGS_ON_TERM_HARD_DELETE = "UPDATE_ENTITY_MEANINGS_ON_TERM_HARD_DELETE"; + + public static final List supportedTypes = new ArrayList() {{ + add(UPDATE_ENTITY_MEANINGS_ON_TERM_UPDATE); + add(UPDATE_ENTITY_MEANINGS_ON_TERM_SOFT_DELETE); + add(UPDATE_ENTITY_MEANINGS_ON_TERM_HARD_DELETE); + }}; + + + protected final EntityGraphMapper entityGraphMapper; + protected final TermPreProcessor preprocessor; + protected final AtlasEntityStoreV2 entityStoreV2; + protected final AtlasGraph graph; + + @Inject + public MeaningsTaskFactory(EntityGraphMapper entityGraphMapper, + TermPreProcessor preprocessor, AtlasEntityStoreV2 entityStoreV2, AtlasGraph graph) { + this.entityGraphMapper = entityGraphMapper; + this.preprocessor = preprocessor; + this.entityStoreV2 = entityStoreV2; + this.graph = graph; + } + + @Override + public org.apache.atlas.tasks.AbstractTask create(AtlasTask atlasTask) { + String taskType = atlasTask.getType(); + String taskGuid = atlasTask.getGuid(); + switch (taskType) { + case UPDATE_ENTITY_MEANINGS_ON_TERM_UPDATE: + return new MeaningsTasks.Update(atlasTask, entityGraphMapper, graph, preprocessor); + case UPDATE_ENTITY_MEANINGS_ON_TERM_SOFT_DELETE: + return new MeaningsTasks.Delete(atlasTask, entityGraphMapper, graph, entityStoreV2); + case UPDATE_ENTITY_MEANINGS_ON_TERM_HARD_DELETE: + return new MeaningsTasks.Delete(atlasTask, entityGraphMapper, graph, entityStoreV2); + } + LOG.warn("Type: {} - {} not found!. The task will be ignored.", taskType, taskGuid); + return null; + } + + + @Override + public List getSupportedTypes() { + return supportedTypes; + } +} diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/tasks/MeaningsTasks.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/tasks/MeaningsTasks.java new file mode 100644 index 00000000000..163748aba53 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/tasks/MeaningsTasks.java @@ -0,0 +1,48 @@ +package org.apache.atlas.repository.store.graph.v2.tasks; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.tasks.AtlasTask; +import org.apache.atlas.repository.graphdb.AtlasGraph; +import org.apache.atlas.repository.store.graph.v2.AtlasEntityStoreV2; +import org.apache.atlas.repository.store.graph.v2.EntityGraphMapper; +import org.apache.atlas.repository.store.graph.v2.preprocessor.glossary.TermPreProcessor; + +import java.util.Map; + +public class MeaningsTasks { + public static class Update extends MeaningsTask { + public Update(AtlasTask task, EntityGraphMapper entityGraphMapper, AtlasGraph graph, + TermPreProcessor preprocessor) { + super(task,entityGraphMapper, graph, preprocessor,null); + } + + @Override + protected void run(Map parameters) throws AtlasBaseException { + String termGuid = (String) parameters.get(PARAM_ENTITY_GUID); + String termQName = (String) parameters.get(PARAM_ENTITY_QUALIFIED_NAME); + String updatedTermQName = (String) parameters.get(PARAM_ENTITY_UPDATED_QUALIFIED_NAME); + String currentTermName = (String) parameters.get(PARAM_CURRENT_TERM_NAME); + String updatedTermName = (String) parameters.get(PARAM_UPDATED_TERM_NAME); + + preprocessor.updateMeaningsAttributesInEntitiesOnTermUpdate(currentTermName, updatedTermName, termQName, updatedTermQName, termGuid); + + } + } + + public static class Delete extends MeaningsTask { + public Delete(AtlasTask task, EntityGraphMapper entityGraphMapper,AtlasGraph graph, + AtlasEntityStoreV2 entityStoreV2) { + super(task,entityGraphMapper, graph, null,entityStoreV2); + } + + @Override + protected void run(Map parameters) throws AtlasBaseException { + String termGuid = (String) parameters.get(PARAM_ENTITY_GUID); + String termQName = (String) parameters.get(PARAM_ENTITY_QUALIFIED_NAME); + String termName = (String) parameters.get(PARAM_CURRENT_TERM_NAME); + + + entityStoreV2.updateMeaningsNamesInEntitiesOnTermDelete(termName, termQName, termGuid); + + } + } +} diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/tasks/TaskUtil.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/tasks/TaskUtil.java new file mode 100644 index 00000000000..3b59d5feb0f --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/tasks/TaskUtil.java @@ -0,0 +1,61 @@ +package org.apache.atlas.repository.store.graph.v2.tasks; + +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.tasks.TaskSearchResult; +import org.apache.atlas.repository.graphdb.AtlasGraph; +import org.apache.atlas.tasks.AtlasTaskService; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.apache.atlas.repository.Constants.*; + +public class TaskUtil { + final public AtlasTaskService taskService; + final public AtlasGraph graph; + final public static String TASK_STATUS_PENDING = "PENDING"; + + public TaskUtil(AtlasGraph graph) { + this.graph = graph; + taskService = new AtlasTaskService(graph); + } + + public TaskSearchResult findPendingTasksByClassificationId(int from, int size, String classificationId, List types, List excludeTypes) throws AtlasBaseException { + List> mustConditions = new ArrayList<>(); + List> shouldConditions = new ArrayList<>(); + List> excludeConditions = new ArrayList<>(); + + if (StringUtils.isNotEmpty(classificationId)) + mustConditions.add(getMap("term", getMap(TASK_CLASSIFICATION_ID, classificationId))); + + if (CollectionUtils.isNotEmpty(types)) { + List> typeQueries = types.stream().map(type -> getMap("match", getMap(TASK_TYPE, type))).collect(Collectors.toList()); + shouldConditions.addAll(typeQueries); + } + + if(CollectionUtils.isNotEmpty(excludeConditions)) { + List> excludeTypeQueries = excludeTypes.stream().map(type -> getMap("match", getMap(TASK_TYPE, type))).collect(Collectors.toList()); + excludeConditions.addAll(excludeTypeQueries); + } + + Map statusQuery = getMap("match", getMap(TASK_STATUS, TASK_STATUS_PENDING)); + mustConditions.add(statusQuery); + + return taskService.getTasksByCondition(from, size, mustConditions, shouldConditions, excludeConditions); + + } + + private Map getMap(String key, Object value) { + Map map = new HashMap<>(); + map.put(key, value); + return map; + } + + + +} diff --git a/repository/src/main/java/org/apache/atlas/repository/store/users/KeycloakStore.java b/repository/src/main/java/org/apache/atlas/repository/store/users/KeycloakStore.java new file mode 100644 index 00000000000..3a5f1782f0d --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/store/users/KeycloakStore.java @@ -0,0 +1,400 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.repository.store.users; + +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.type.AtlasType; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang.StringUtils; +import org.keycloak.representations.idm.GroupRepresentation; +import org.keycloak.representations.idm.RoleRepresentation; +import org.keycloak.representations.idm.UserRepresentation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; +import java.util.stream.Collectors; + +import static org.apache.atlas.AtlasErrorCode.RESOURCE_NOT_FOUND; +import static org.apache.atlas.keycloak.client.AtlasKeycloakClient.getKeycloakClient; +import static org.apache.atlas.repository.util.AccessControlUtils.INSTANCE_DOMAIN_KEY; + +public class KeycloakStore { + private static final Logger LOG = LoggerFactory.getLogger(KeycloakStore.class); + + private boolean saveUsersToAttributes = false; + private boolean saveGroupsToAttributes = false; + + public KeycloakStore() {} + + public KeycloakStore(boolean saveUsersToAttributes, boolean saveGroupsToAttributes) { + this.saveUsersToAttributes = saveUsersToAttributes; + this.saveGroupsToAttributes = saveGroupsToAttributes; + } + + public RoleRepresentation createRole(String name) throws AtlasBaseException { + return createRole(name, false, null, null, null, null); + } + + public RoleRepresentation createRole(String name, + List users, List groups, List roles) throws AtlasBaseException { + return createRole(name, false, users, groups, roles, null); + } + + public RoleRepresentation createRoleForConnection(String name, boolean isComposite, + List users, List groups, List roles) throws AtlasBaseException { + + List roleUsers = new ArrayList<>(); + + if (CollectionUtils.isNotEmpty(users)) { + for (String userName : users) { + List matchedUsers = getKeycloakClient().searchUserByUserName(userName); + Optional keyUserOptional = matchedUsers.stream().filter(x -> userName.equals(x.getUsername())).findFirst(); + + if (keyUserOptional.isPresent()) { + roleUsers.add(keyUserOptional.get()); + } else { + throw new AtlasBaseException("Keycloak user not found with userName " + userName); + } + } + } + + List roleGroups = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(groups)) { + for (String groupName : groups) { + List matchedGroups = getKeycloakClient().searchGroupByName(groupName, 0, 100); + Optional keyGroupOptional = matchedGroups.stream().filter(x -> groupName.equals(x.getName())).findFirst(); + + if (keyGroupOptional.isPresent()) { + roleGroups.add(keyGroupOptional.get()); + } else { + throw new AtlasBaseException("Keycloak group not found with name " + groupName); + } + } + } + + List roleRoles = new ArrayList<>(); + + if (CollectionUtils.isNotEmpty(roles)) { + for (String roleId : roles) { + RoleRepresentation role = getRoleById(roleId); + roleRoles.add(role); + } + } + + RoleRepresentation role = new RoleRepresentation(); + role.setName(name); + role.setComposite(isComposite); + + RoleRepresentation createdRole = createRole(role); + if (createdRole == null) { + throw new AtlasBaseException("Failed to create a keycloak role " + name); + } + LOG.info("Created keycloak role with name {}", name); + + //add realm role into users + if (CollectionUtils.isNotEmpty(roleUsers)) { + for (UserRepresentation kUser : roleUsers) { + getKeycloakClient().addRealmLevelRoleMappingsForUser(kUser.getId(), Collections.singletonList(createdRole)); + } + } + + //add realm role into groups + if (CollectionUtils.isNotEmpty(roleGroups)) { + for (GroupRepresentation kGroup : roleGroups) { + getKeycloakClient().addRealmLevelRoleMappingsForGroup(kGroup.getId(), Collections.singletonList(createdRole)); + } + } + + //add realm role into roles + if (CollectionUtils.isNotEmpty(roleRoles)) { + RoleRepresentation connRole = getKeycloakClient().getRoleByName(createdRole.getName()); + for (RoleRepresentation kRole : roleRoles) { + getKeycloakClient().addComposites(kRole.getName(), Collections.singletonList(connRole)); + } + } + + return createdRole; + } + + public RoleRepresentation createRole(String name, boolean isComposite, + List users, List groups, List roles, + Map> attributes) throws AtlasBaseException { + + List roleUsers = new ArrayList<>(); + + if (CollectionUtils.isNotEmpty(users)) { + + for (String userName : users) { + List matchedUsers = getKeycloakClient().searchUserByUserName(userName); + Optional keyUserOptional = matchedUsers.stream().filter(x -> userName.equals(x.getUsername())).findFirst(); + + if (keyUserOptional.isPresent()) { + roleUsers.add(keyUserOptional.get()); + } else { + throw new AtlasBaseException("Keycloak user not found with userName " + userName); + } + } + } + + List roleGroups = new ArrayList<>(); + + if (CollectionUtils.isNotEmpty(groups)) { + for (String groupName : groups) { + List matchedGroups = getKeycloakClient().searchGroupByName(groupName, 0, 100); + Optional keyGroupOptional = matchedGroups.stream().filter(x -> groupName.equals(x.getName())).findFirst(); + + if (keyGroupOptional.isPresent()) { + roleGroups.add(keyGroupOptional.get()); + } else { + throw new AtlasBaseException("Keycloak group not found with name " + groupName); + } + } + } + + List roleRoles = new ArrayList<>(); + + if (CollectionUtils.isNotEmpty(roles)) { + for (String roleName : roles) { + LOG.info("Searching role {}", roleName); + RoleRepresentation roleRepresentation = getKeycloakClient().getRoleByName(roleName); + + if (roleRepresentation != null) { + roleRoles.add(roleRepresentation); + } else { + throw new AtlasBaseException("Keycloak role not found with name " + roleName); + } + } + } + + RoleRepresentation role = new RoleRepresentation(); + role.setName(name); + role.setComposite(isComposite); + + if (attributes == null) { + attributes = new HashMap<>(); + } + + if (saveUsersToAttributes) { + attributes.put("users", Collections.singletonList(AtlasType.toJson(roleUsers.stream().map(x -> x.getId()).collect(Collectors.toList())))); + } + + if (saveGroupsToAttributes) { + attributes.put("groups", Collections.singletonList(AtlasType.toJson(roleGroups.stream().map(x -> x.getId()).collect(Collectors.toList())))); + } + + if (MapUtils.isNotEmpty(attributes)) { + role.setAttributes(attributes); + } + + RoleRepresentation createdRole = createRole(role); + if (createdRole == null) { + throw new AtlasBaseException("Failed to create a keycloak role " + name); + } + LOG.info("Created keycloak role with name {}", name); + + //add realm role into users + if (CollectionUtils.isNotEmpty(roleUsers)) { + for (UserRepresentation kUser : roleUsers) { + getKeycloakClient().addRealmLevelRoleMappingsForUser(kUser.getId(), Collections.singletonList(createdRole)); + } + } + + //add realm role into groups + if (CollectionUtils.isNotEmpty(roleGroups)) { + for (GroupRepresentation kGroup : roleGroups) { + getKeycloakClient().addRealmLevelRoleMappingsForGroup(kGroup.getId(), Collections.singletonList(createdRole)); + } + } + + //add realm role into roles + if (CollectionUtils.isNotEmpty(roleRoles)) { + for (RoleRepresentation kRole : roleRoles) { + RoleRepresentation rr = getKeycloakClient().getRoleByName(kRole.getName()); + getKeycloakClient().addComposites(createdRole.getName(), Collections.singletonList(rr)); + } + } + + return createdRole; + } + + public RoleRepresentation createRole(RoleRepresentation role) throws AtlasBaseException { + getKeycloakClient().createRole(role); + return getKeycloakClient().getRoleByName(role.getName()); + } + + public RoleRepresentation getRole(String roleName) throws AtlasBaseException { + RoleRepresentation roleRepresentation = null; + try{ + roleRepresentation = getKeycloakClient().getRoleByName(roleName); + } catch (AtlasBaseException e) { + return null; + } + return roleRepresentation; + } + + public void updateRoleUsers(String roleName, + List existingUsers, List newUsers, + RoleRepresentation roleRepresentation) throws AtlasBaseException { + + if (roleRepresentation == null) { + throw new AtlasBaseException("Failed to updateRoleUsers as roleRepresentation is null"); + } + + if (newUsers == null) { + newUsers = new ArrayList<>(); + } + + if (existingUsers == null) { + existingUsers = new ArrayList<>(); + } + + List usersToAdd = (List) CollectionUtils.removeAll(newUsers, existingUsers); + List usersToRemove = (List) CollectionUtils.removeAll(existingUsers, newUsers); + + for (String userName : usersToAdd) { + LOG.info("Adding user {} to role {}", userName, roleName); + List matchedUsers = getKeycloakClient().searchUserByUserName(userName); + Optional keyUserOptional = matchedUsers.stream().filter(x -> userName.equals(x.getUsername())).findFirst(); + + if (keyUserOptional.isPresent()) { + getKeycloakClient().addRealmLevelRoleMappingsForUser(keyUserOptional.get().getId(), Collections.singletonList(roleRepresentation)); + } else { + throw new AtlasBaseException("Keycloak user not found with userName " + userName); + } + } + + for (String userName : usersToRemove) { + LOG.info("Removing user {} from role {}", userName, roleName); + List matchedUsers = getKeycloakClient().searchUserByUserName(userName); + Optional keyUserOptional = matchedUsers.stream().filter(x -> userName.equals(x.getUsername())).findFirst(); + + if (keyUserOptional.isPresent()) { + getKeycloakClient().deleteRealmLevelRoleMappingsForUser(keyUserOptional.get().getId(), Collections.singletonList(roleRepresentation)); + } else { + LOG.warn("Keycloak user not found with userName " + userName); + } + } + } + + public void updateRoleGroups(String roleName, + List existingGroups, List newGroups, + RoleRepresentation roleRepresentation) throws AtlasBaseException { + + if (roleRepresentation == null) { + throw new AtlasBaseException("Failed to updateRoleGroups as roleRepresentation is null"); + } + + if (newGroups == null) { + newGroups = new ArrayList<>(); + } + + if (existingGroups == null) { + existingGroups = new ArrayList<>(); + } + + List groupsToAdd = (List) CollectionUtils.removeAll(newGroups, existingGroups); + List groupsToRemove = (List) CollectionUtils.removeAll(existingGroups, newGroups); + + for (String groupName : groupsToAdd) { + LOG.info("Adding group {} to role {}", groupName, roleName); + List matchedGroups = getKeycloakClient().searchGroupByName(groupName, 0, 100); + Optional keyGroupOptional = matchedGroups.stream().filter(x -> groupName.equals(x.getName())).findFirst(); + + if (keyGroupOptional.isPresent()) { + getKeycloakClient().addRealmLevelRoleMappingsForGroup(keyGroupOptional.get().getId(), Collections.singletonList(roleRepresentation)); + } else { + throw new AtlasBaseException("Keycloak group not found with userName " + groupName); + } + } + + for (String groupName : groupsToRemove) { + LOG.info("removing group {} from role {}", groupName, roleName); + List matchedGroups = getKeycloakClient().searchGroupByName(groupName, 0, 100); + Optional keyGroupOptional = matchedGroups.stream().filter(x -> groupName.equals(x.getName())).findFirst(); + + if (keyGroupOptional.isPresent()) { + getKeycloakClient().deleteRealmLevelRoleMappingsForGroup(keyGroupOptional.get().getId(), Collections.singletonList(roleRepresentation)); + } else { + LOG.warn("Keycloak group not found with userName " + groupName); + } + } + } + + public void updateRoleRoles(String roleName, + List existingRoles, List newRoles, + RoleRepresentation roleRepresentation) throws AtlasBaseException { + + if (roleRepresentation == null) { + throw new AtlasBaseException("Failed to updateRoleRoles as roleRepresentation is null"); + } + + if (newRoles == null) { + newRoles = new ArrayList<>(); + } + + if (existingRoles == null) { + existingRoles = new ArrayList<>(); + } + + List rolesToAdd = (List) CollectionUtils.removeAll(newRoles, existingRoles); + List rolesToRemove = (List) CollectionUtils.removeAll(existingRoles, newRoles); + + for (String subRoleId : rolesToAdd) { + LOG.info("Adding role {} to role {}", roleName, subRoleId); + RoleRepresentation keyRole = getRoleById(subRoleId); + getKeycloakClient().addComposites(keyRole.getName(), Collections.singletonList(roleRepresentation)); + } + + for (String subRoleId : rolesToRemove) { + LOG.info("removing role {} from role {}", roleName, subRoleId); + RoleRepresentation keyRole = getRoleById(subRoleId); + getKeycloakClient().deleteComposites(keyRole.getName(), Collections.singletonList(roleRepresentation)); + } + } + + public void removeRole(String roleId) throws AtlasBaseException { + if (StringUtils.isNotEmpty(roleId)) { + getKeycloakClient().deleteRoleById(roleId); + LOG.info("Removed keycloak role with id {}", roleId); + } + } + public void removeRoleByName(String roleName) throws AtlasBaseException { + if (StringUtils.isNotEmpty(roleName)) { + getKeycloakClient().deleteRoleByName(roleName); + LOG.info("Removed keycloak role with name {}", roleName); + } + } + + private RoleRepresentation getRoleById(String roleId) throws AtlasBaseException { + + try { + return getKeycloakClient().getRoleById(roleId); + } catch (Exception e) { + if(e instanceof AtlasBaseException && Objects.equals(RESOURCE_NOT_FOUND.getErrorCode(), ((AtlasBaseException) e).getAtlasErrorCode().getErrorCode())) + { + LOG.error("Role not found with id {}", roleId); + throw new AtlasBaseException(RESOURCE_NOT_FOUND, "Role with id " + roleId); + } + LOG.error("Role not found with id/name {}: {}", roleId, e.getMessage()); + throw new AtlasBaseException(e); + } + } +} diff --git a/repository/src/main/java/org/apache/atlas/repository/util/AccessControlUtils.java b/repository/src/main/java/org/apache/atlas/repository/util/AccessControlUtils.java new file mode 100644 index 00000000000..e0aef354253 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/util/AccessControlUtils.java @@ -0,0 +1,429 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.repository.util; + +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.featureflag.FeatureFlagStore; +import org.apache.atlas.model.discovery.IndexSearchParams; +import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.model.instance.AtlasEntityHeader; +import org.apache.atlas.model.instance.AtlasObjectId; +import org.apache.atlas.model.instance.AtlasStruct; +import org.apache.atlas.repository.graphdb.AtlasGraph; +import org.apache.atlas.repository.graphdb.AtlasIndexQuery; +import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.repository.graphdb.DirectIndexQueryResult; +import org.apache.atlas.repository.store.graph.AtlasEntityStore; +import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; +import org.apache.atlas.util.NanoIdUtils; +import org.apache.commons.collections.CollectionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.apache.atlas.AtlasErrorCode.ACCESS_CONTROL_ALREADY_EXISTS; +import static org.apache.atlas.AtlasErrorCode.DISABLED_OPERATION; +import static org.apache.atlas.AtlasErrorCode.OPERATION_NOT_SUPPORTED; +import static org.apache.atlas.repository.Constants.ATTR_ADMIN_GROUPS; +import static org.apache.atlas.repository.Constants.ATTR_ADMIN_ROLES; +import static org.apache.atlas.repository.Constants.ATTR_ADMIN_USERS; +import static org.apache.atlas.repository.Constants.ATTR_TENANT_ID; +import static org.apache.atlas.repository.Constants.CONNECTION_ENTITY_TYPE; +import static org.apache.atlas.repository.Constants.DEFAULT_TENANT_ID; +import static org.apache.atlas.repository.Constants.NAME; +import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; +import static org.apache.atlas.repository.Constants.VERTEX_INDEX_NAME; +import static org.apache.atlas.repository.util.AtlasEntityUtils.getListAttribute; +import static org.apache.atlas.repository.util.AtlasEntityUtils.getQualifiedName; +import static org.apache.atlas.repository.util.AtlasEntityUtils.getStringAttribute; +import static org.apache.atlas.repository.util.AtlasEntityUtils.mapOf; + +public final class AccessControlUtils { + private static final Logger LOG = LoggerFactory.getLogger(AccessControlUtils.class); + + public static final String ATTR_ACCESS_CONTROL_ENABLED = "isAccessControlEnabled"; + public static final String ATTR_ACCESS_CONTROL_DENY_CM_GUIDS = "denyCustomMetadataGuids"; + public static final String ATTR_ACCESS_CONTROL_DENY_ASSET_TABS = "denyAssetTabs"; + + public static final String ATTR_PERSONA_ROLE_ID = "roleId"; + public static final String ATTR_PERSONA_USERS = "personaUsers"; + public static final String ATTR_PERSONA_GROUPS = "personaGroups"; + + public static final String ATTR_PURPOSE_CLASSIFICATIONS = "purposeClassifications"; + + public static final String ATTR_POLICY_TYPE = "policyType"; + public static final String ATTR_POLICY_USERS = "policyUsers"; + public static final String ATTR_POLICY_GROUPS = "policyGroups"; + public static final String ATTR_POLICY_ROLES = "policyRoles"; + public static final String ATTR_POLICY_ACTIONS = "policyActions"; + public static final String ATTR_POLICY_CATEGORY = "policyCategory"; + public static final String ATTR_POLICY_SUB_CATEGORY = "policySubCategory"; + public static final String ATTR_POLICY_RESOURCES = "policyResources"; + public static final String ATTR_POLICY_IS_ENABLED = "isPolicyEnabled"; + public static final String ATTR_POLICY_CONNECTION_QN = "connectionQualifiedName"; + public static final String ATTR_POLICY_RESOURCES_CATEGORY = "policyResourceCategory"; + public static final String ATTR_POLICY_SERVICE_NAME = "policyServiceName"; + public static final String ATTR_POLICY_PRIORITY = "policyPriority"; + + public static final String REL_ATTR_ACCESS_CONTROL = "accessControl"; + public static final String REL_ATTR_POLICIES = "policies"; + + public static final String POLICY_TYPE_ALLOW = "allow"; + public static final String POLICY_TYPE_DENY = "deny"; + + public static final String ACCESS_READ_PURPOSE_METADATA = "entity-read"; + public static final String ACCESS_READ_PERSONA_METADATA = "persona-asset-read"; + public static final String ACCESS_READ_PERSONA_GLOSSARY = "persona-glossary-read"; + + public static final String POLICY_CATEGORY_PERSONA = "persona"; + public static final String POLICY_CATEGORY_PURPOSE = "purpose"; + public static final String POLICY_CATEGORY_BOOTSTRAP = "bootstrap"; + + public static final String POLICY_SUB_CATEGORY_COLLECTION = "collection"; + + public static final String POLICY_RESOURCE_CATEGORY_PERSONA_CUSTOM = "CUSTOM"; + public static final String POLICY_RESOURCE_CATEGORY_PERSONA_ENTITY = "ENTITY"; + public static final String POLICY_RESOURCE_CATEGORY_PURPOSE = "TAG"; + + public static final String POLICY_SUB_CATEGORY_METADATA = "metadata"; + public static final String POLICY_SUB_CATEGORY_GLOSSARY = "glossary"; + public static final String POLICY_SUB_CATEGORY_DATA = "data"; + + public static final String RESOURCES_ENTITY = "entity:"; + public static final String RESOURCES_ENTITY_TYPE = "entity-type:"; + public static final String RESOURCES_TAG = "tag:"; + public static final String RESOURCES_SPLITTER = ":"; + + private static final String CONNECTION_QN = "%s/%s/%s"; + public static final String CONN_NAME_PATTERN = "connection_admins_%s"; + public static final String ARGO_SERVICE_USER_NAME = "service-account-atlan-argo"; + public static final String BACKEND_SERVICE_USER_NAME = "service-account-atlan-backend"; + + public static final String INSTANCE_DOMAIN_KEY = "instance"; + + private AccessControlUtils() {} + + public static String getEntityName(AtlasEntity entity) { + return (String) entity.getAttribute(NAME); + } + + public static String getEntityQualifiedName(AtlasEntity entity) { + return getStringAttribute(entity, QUALIFIED_NAME); + } + + public static List getPolicyAssets(AtlasEntity policyEntity) throws AtlasBaseException { + List resources = getPolicyResources(policyEntity); + + return getFilteredPolicyResources(resources, RESOURCES_ENTITY); + } + + public static List getFilteredPolicyResources(List resources, String resourcePrefix) { + return resources.stream() + .filter(x -> x.startsWith(resourcePrefix)) + .map(x -> x.substring(resourcePrefix.length())) + .collect(Collectors.toList()); + } + + public static String getPolicyConnectionQN(AtlasEntity policyEntity) { + return getStringAttribute(policyEntity, ATTR_POLICY_CONNECTION_QN); + } + + public static String getPolicyConnectionQN(AtlasEntityHeader policyEntity) { + return getStringAttribute(policyEntity, ATTR_POLICY_CONNECTION_QN); + } + + public static List getPolicyResources(AtlasEntity policyEntity) throws AtlasBaseException { + return getListAttribute(policyEntity, ATTR_POLICY_RESOURCES); + } + + public static List getPolicyResources(AtlasEntityHeader policyEntity) { + return getListAttribute(policyEntity, ATTR_POLICY_RESOURCES); + } + + public static List getPolicyActions(AtlasEntity policyEntity) { + return getListAttribute(policyEntity, ATTR_POLICY_ACTIONS); + } + + public static List getPolicyActions(AtlasEntityHeader policyEntity) { + return getListAttribute(policyEntity, ATTR_POLICY_ACTIONS); + } + + public static String getPolicyCategory(AtlasEntity policyEntity) { + return getStringAttribute(policyEntity, ATTR_POLICY_CATEGORY); + } + + public static String getPolicyResourceCategory(AtlasEntity policyEntity) { + return getStringAttribute(policyEntity, ATTR_POLICY_RESOURCES_CATEGORY); + } + + public static String getPolicyResourceCategory(AtlasEntityHeader policyEntity) { + return getStringAttribute(policyEntity, ATTR_POLICY_RESOURCES_CATEGORY); + } + + public static String getPolicyCategory(AtlasEntityHeader policyEntity) { + return getStringAttribute(policyEntity, ATTR_POLICY_CATEGORY); + } + + public static String getPolicySubCategory(AtlasEntity policyEntity) { + return getStringAttribute(policyEntity, ATTR_POLICY_SUB_CATEGORY); + } + + public static String getPolicySubCategory(AtlasEntityHeader policyEntity) { + return getStringAttribute(policyEntity, ATTR_POLICY_SUB_CATEGORY); + } + + public static String getPolicyServiceName(AtlasEntity policyEntity) { + return getStringAttribute(policyEntity, ATTR_POLICY_SERVICE_NAME); + } + + public static String getPolicyType(AtlasEntity policyEntity) { + return getStringAttribute(policyEntity, ATTR_POLICY_TYPE); + } + + public static List getPolicyRoles(AtlasEntity policyEntity) { + return getListAttribute(policyEntity, ATTR_POLICY_ROLES); + } + + + public static boolean getIsAllowPolicy(AtlasEntity policyEntity) throws AtlasBaseException { + String policyType = (String) policyEntity.getAttribute(ATTR_POLICY_TYPE); + + if (POLICY_TYPE_ALLOW.equals(policyType)) { + return true; + } else if (POLICY_TYPE_DENY.equals(policyType)) { + return false; + } else { + throw new AtlasBaseException("Unsupported policy type while creating index alias filters"); + } + } + + public static AtlasEntity getEntityByQualifiedName(EntityGraphRetriever entityRetriever, String connectionQualifiedName) throws AtlasBaseException { + AtlasObjectId objectId = new AtlasObjectId(CONNECTION_ENTITY_TYPE, mapOf(QUALIFIED_NAME, connectionQualifiedName)); + + AtlasEntity entity = entityRetriever.toAtlasEntity(objectId); + + return entity; + } + + public static String getConnectionQualifiedNameFromPolicyAssets(EntityGraphRetriever entityRetriever, List assets) throws AtlasBaseException { + if (CollectionUtils.isEmpty(assets)) { + throw new AtlasBaseException("Policy assets could not be null"); + } + + AtlasEntity connection = extractConnectionFromResource(entityRetriever, assets.get(0)); + + return getQualifiedName(connection); + } + + public static AtlasEntity extractConnectionFromResource(EntityGraphRetriever entityRetriever, String assetQName) throws AtlasBaseException { + AtlasEntity connection = null; + + String[] splitted = assetQName.split("/"); + String connectionQName; + try { + connectionQName = String.format(CONNECTION_QN, splitted[0], splitted[1], splitted[2]); + } catch (ArrayIndexOutOfBoundsException aib) { + LOG.error("Failed to extract qualifiedName of the connection: " + assetQName); + return null; + } + + connection = getEntityByQualifiedName(entityRetriever, connectionQName); + + return connection; + } + + public static String getPersonaRoleName(AtlasEntity persona) { + String qualifiedName = getStringAttribute(persona, QUALIFIED_NAME); + + String[] parts = qualifiedName.split("/"); + + return "persona_" + parts[parts.length - 1]; + } + + public static String getESAliasName(AtlasEntity entity) { + String qualifiedName = getStringAttribute(entity, QUALIFIED_NAME); + + String[] parts = qualifiedName.split("/"); + + return parts[parts.length - 1]; + } + + public static List getPolicies(AtlasEntity.AtlasEntityWithExtInfo accessControl) { + List policies = (List) accessControl.getEntity().getRelationshipAttribute(REL_ATTR_POLICIES); + + return objectToEntityList(accessControl, policies); + } + + public static List objectToEntityList(AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo, List policies) { + List ret = new ArrayList<>(); + + Set referredGuids = entityWithExtInfo.getReferredEntities().keySet(); + if (policies != null) { + ret = policies.stream() + .filter(x -> referredGuids.contains(x.getGuid())) + .map(x -> entityWithExtInfo.getReferredEntity(x.getGuid())) + .filter(x -> x.getStatus() == null || x.getStatus() == AtlasEntity.Status.ACTIVE) + .collect(Collectors.toList()); + } + + return ret; + } + + public static List getPurposeTags(AtlasStruct entity) { + return getListAttribute(entity, ATTR_PURPOSE_CLASSIFICATIONS); + } + + public static boolean getIsAccessControlEnabled(AtlasEntity entity) { + return (boolean) entity.getAttribute(ATTR_ACCESS_CONTROL_ENABLED); + } + + public static boolean getIsPolicyEnabled(AtlasEntityHeader entity) { + if (entity.hasAttribute(ATTR_POLICY_IS_ENABLED)) { + return (boolean) entity.getAttribute(ATTR_POLICY_IS_ENABLED); + } + return true; + } + + public static List getPersonaUsers(AtlasStruct entity) { + return getListAttribute(entity, ATTR_PERSONA_USERS); + } + + public static List getPersonaGroups(AtlasStruct entity) { + return getListAttribute(entity, ATTR_PERSONA_GROUPS); + } + + public static String getPersonaRoleId(AtlasEntity entity) { + String roleId = (String) entity.getAttribute(ATTR_PERSONA_ROLE_ID); + if (roleId == null) { + LOG.warn("roleId not found for Persona with GUID " + entity.getGuid()); + } + return roleId; + } + + public static String getTenantId(AtlasStruct entity) { + String ret = DEFAULT_TENANT_ID; + + Object tenantId = entity.getAttribute(ATTR_TENANT_ID); + + if (tenantId != null) { + String tenantIdAsString = (String) tenantId; + if (tenantIdAsString.length() > 0) { + ret = tenantIdAsString; + } + } + + return ret; + } + + public static void validateNoPoliciesAttached(AtlasEntity entity) throws AtlasBaseException { + List policies = (List) entity.getRelationshipAttribute(REL_ATTR_POLICIES); + if (CollectionUtils.isNotEmpty(policies)) { + throw new AtlasBaseException(OPERATION_NOT_SUPPORTED, "Can not attach a policy while creating/updating Persona/Purpose"); + } + } + + public static String getUUID(){ + return NanoIdUtils.randomNanoId(22); + } + + public static void validateUniquenessByTags(AtlasGraph graph, List tags, String typeName) throws AtlasBaseException { + IndexSearchParams indexSearchParams = new IndexSearchParams(); + Map dsl = mapOf("size", 1); + + List mustClauseList = new ArrayList(); + mustClauseList.add(mapOf("term", mapOf("__typeName.keyword", typeName))); + mustClauseList.add(mapOf("term", mapOf("__state", "ACTIVE"))); + mustClauseList.add(mapOf("terms", mapOf(ATTR_PURPOSE_CLASSIFICATIONS, tags))); + + Map scriptMap = mapOf("inline", "doc['" + ATTR_PURPOSE_CLASSIFICATIONS + "'].length == params.list_length"); + scriptMap.put("lang", "painless"); + scriptMap.put("params", mapOf("list_length", tags.size())); + + mustClauseList.add(mapOf("script", mapOf("script", scriptMap))); + + dsl.put("query", mapOf("bool", mapOf("must", mustClauseList))); + + indexSearchParams.setDsl(dsl); + + if (hasMatchingVertex(graph, tags, indexSearchParams)){ + throw new AtlasBaseException(String.format("Entity already exists, typeName:tags, %s:%s", typeName, tags)); + } + } + + private static boolean hasMatchingVertex(AtlasGraph graph, List newTags, + IndexSearchParams indexSearchParams) throws AtlasBaseException { + AtlasIndexQuery indexQuery = graph.elasticsearchQuery(VERTEX_INDEX_NAME); + + DirectIndexQueryResult indexQueryResult = indexQuery.vertices(indexSearchParams); + Iterator iterator = indexQueryResult.getIterator(); + + while (iterator.hasNext()) { + AtlasVertex vertex = iterator.next().getVertex(); + if (vertex != null) { + List tags = (List) vertex.getPropertyValues(ATTR_PURPOSE_CLASSIFICATIONS, String.class); + + //TODO: handle via ES query if possible -> match exact tags list + if (CollectionUtils.isEqualCollection(tags, newTags)) { + return true; + } + } + } + + return false; + } + + public static void validateUniquenessByName(AtlasGraph graph, String name, String typeName) throws AtlasBaseException { + IndexSearchParams indexSearchParams = new IndexSearchParams(); + Map dsl = mapOf("size", 1); + + List mustClauseList = new ArrayList(); + mustClauseList.add(mapOf("term", mapOf("__typeName.keyword", typeName))); + mustClauseList.add(mapOf("term", mapOf("__state", "ACTIVE"))); + mustClauseList.add(mapOf("term", mapOf("name.keyword", name))); + + dsl.put("query", mapOf("bool", mapOf("must", mustClauseList))); + + indexSearchParams.setDsl(dsl); + + if (checkEntityExists(graph, indexSearchParams)){ + throw new AtlasBaseException(ACCESS_CONTROL_ALREADY_EXISTS, typeName, name); + } + } + + private static boolean checkEntityExists(AtlasGraph graph, IndexSearchParams indexSearchParams) throws AtlasBaseException { + AtlasIndexQuery indexQuery = graph.elasticsearchQuery(VERTEX_INDEX_NAME); + + DirectIndexQueryResult indexQueryResult = indexQuery.vertices(indexSearchParams); + Iterator iterator = indexQueryResult.getIterator(); + + while (iterator.hasNext()) { + AtlasVertex vertex = iterator.next().getVertex(); + if (vertex != null) { + return true; + } + } + + return false; + } +} diff --git a/repository/src/main/java/org/apache/atlas/repository/util/AtlasEntityUtils.java b/repository/src/main/java/org/apache/atlas/repository/util/AtlasEntityUtils.java new file mode 100644 index 00000000000..093e616d603 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/util/AtlasEntityUtils.java @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.repository.util; + +import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.model.instance.AtlasEntityHeader; +import org.apache.atlas.model.instance.AtlasStruct; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.apache.atlas.repository.Constants.NAME; +import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; + +public final class AtlasEntityUtils { + private static final Logger LOG = LoggerFactory.getLogger(AtlasEntityUtils.class); + + private AtlasEntityUtils() { + } + + public static String getQualifiedName(AtlasEntity entity) { + return getStringAttribute(entity, QUALIFIED_NAME); + } + + public static String getName(AtlasEntity entity) { + return getStringAttribute(entity, NAME); + } + + public static List getListAttribute(AtlasStruct entity, String attrName) { + List ret = new ArrayList<>(); + + Object valueObj = entity.getAttribute(attrName); + if (valueObj != null) { + ret = (List) valueObj; + } + + return ret; + } + + public static String getStringAttribute(AtlasEntity entity, String attrName) { + Object obj = entity.getAttribute(attrName); + return obj == null ? null : (String) obj; + } + + public static String getStringAttribute(AtlasEntityHeader entity, String attrName) { + Object obj = entity.getAttribute(attrName); + return obj == null ? null : (String) obj; + } + + public static Map mapOf(String key, Object value) { + Map map = new HashMap<>(); + map.put(key, value); + return map; + } +} diff --git a/repository/src/main/java/org/apache/atlas/searchlog/ESSearchLogger.java b/repository/src/main/java/org/apache/atlas/searchlog/ESSearchLogger.java new file mode 100644 index 00000000000..cbb52090f32 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/searchlog/ESSearchLogger.java @@ -0,0 +1,180 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.searchlog; + +import org.apache.atlas.AtlasConfiguration; +import org.apache.atlas.AtlasException; +import org.apache.atlas.model.searchlog.SearchRequestLogData; +import org.apache.atlas.service.Service; +import org.apache.atlas.type.AtlasType; +import org.apache.commons.lang.StringUtils; +import org.apache.http.HttpEntity; +import org.apache.http.HttpHost; +import org.apache.http.entity.ContentType; +import org.apache.http.nio.entity.NStringEntity; +import org.apache.http.util.EntityUtils; +import org.elasticsearch.client.RestClientBuilder; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.RestClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.apache.atlas.repository.audit.ESBasedAuditRepository.getHttpHosts; + +@Component +public class ESSearchLogger implements SearchLogger, Service { + private static final Logger LOG = LoggerFactory.getLogger(ESSearchLogger.class); + + public static final String INDEX_NAME = "search_logs"; + public static final String MAPPINGS_FILE_NAME = "es-search-logs-mappings.json"; + public static final String ENDPOINT_CREATE_DOC = INDEX_NAME + "/_doc"; + + private RestClient lowLevelClient; + + @Override + public void log(SearchRequestLogData searchRequestLogData) { + try { + searchRequestLogData.setCreatedAt(System.currentTimeMillis()); + + HttpEntity entity = new NStringEntity(AtlasType.toJson(searchRequestLogData), ContentType.APPLICATION_JSON); + + Request request = new Request("POST", ENDPOINT_CREATE_DOC); + request.setEntity(entity); + + Response response = lowLevelClient.performRequest(request); + int responseCode = response.getStatusLine().getStatusCode(); + + if (responseCode != 200 && responseCode != 201) { + String responseString = EntityUtils.toString(response.getEntity()); + Map responseMap = AtlasType.fromJson(responseString, Map.class); + if ((boolean) responseMap.get("errors")) { + List errors = new ArrayList<>(); + List> resultItems = (List>) responseMap.get("items"); + for (Map resultItem : resultItems) { + if (resultItem.get("index") != null) { + Map resultIndex = (Map) resultItem.get("index"); + if (resultIndex.get("error") != null) { + errors.add(resultIndex.get("error").toString()); + } + } + } + throw new AtlasException(errors.toString()); + } + throw new Exception(); + } + + } catch (Exception e) { + LOG.error("Unable to push search log to ES: {}", e.getMessage()); + } + } + + @Override + public void start() throws AtlasException { + LOG.info("ESSearchLogger: start!"); + setLowLevelClient(); + try { + if (!indexExists()) { + LOG.info("Create ES index for entity search logging in ES based logger"); + if (createIndex()) { + LOG.info("Create ES index with name {}", INDEX_NAME); + } else { + LOG.info("Failed to create ES index with name {}", INDEX_NAME); + } + } + } catch (IOException e) { + LOG.error("ESSearchLogger: Failed to start", e); + throw new AtlasException(e); + } + } + + @Override + public void stop() throws AtlasException { + try { + LOG.info("ESSearchLogger: stop!"); + if (lowLevelClient != null) { + lowLevelClient.close(); + lowLevelClient = null; + } + } catch (IOException e) { + LOG.error("ESSearchLogger: Failed to close ES client", e); + throw new AtlasException(e); + } + } + + private void setLowLevelClient() throws AtlasException { + synchronized (ESSearchLogger.class) { + if (lowLevelClient == null) { + try { + List httpHosts = getHttpHosts(); + + RestClientBuilder builder = RestClient.builder(httpHosts.get(0)); + builder.setRequestConfigCallback(requestConfigBuilder -> requestConfigBuilder + .setConnectTimeout(AtlasConfiguration.INDEX_CLIENT_CONNECTION_TIMEOUT.getInt()) + .setSocketTimeout(AtlasConfiguration.INDEX_CLIENT_SOCKET_TIMEOUT.getInt())); + + lowLevelClient = builder.build(); + } catch (AtlasException e) { + LOG.error("Failed to initialize low level rest client for ES"); + throw new AtlasException(e); + } + } + } + } + + private boolean indexExists() throws IOException { + Request request = new Request("HEAD", INDEX_NAME); + Response response = lowLevelClient.performRequest(request); + int statusCode = response.getStatusLine().getStatusCode(); + if (statusCode == 200) { + LOG.info("ESSearchLogger: {} index exists!", INDEX_NAME); + return true; + } + LOG.info("ESSearchLogger: {} index does not exist!", INDEX_NAME); + return false; + } + + private boolean createIndex() throws IOException { + LOG.info("ESSearchLogger: createIndex"); + String esMappingsString = getIndexMappings(); + + HttpEntity entity = new NStringEntity(esMappingsString, ContentType.APPLICATION_JSON); + Request request = new Request("PUT", INDEX_NAME); + request.setEntity(entity); + Response response = lowLevelClient.performRequest(request); + + return response.getStatusLine().getStatusCode() == 200; + } + + private String getIndexMappings() throws IOException { + String atlasHomeDir = System.getProperty("atlas.home"); + String elasticsearchSettingsFilePath = (StringUtils.isEmpty(atlasHomeDir) ? "." : atlasHomeDir) + File.separator + "elasticsearch" + File.separator + MAPPINGS_FILE_NAME; + File elasticsearchSettingsFile = new File(elasticsearchSettingsFilePath); + return new String(Files.readAllBytes(elasticsearchSettingsFile.toPath()), StandardCharsets.UTF_8); + } +} + diff --git a/repository/src/main/java/org/apache/atlas/searchlog/SearchLogger.java b/repository/src/main/java/org/apache/atlas/searchlog/SearchLogger.java new file mode 100644 index 00000000000..a6dd80b2a2c --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/searchlog/SearchLogger.java @@ -0,0 +1,26 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.searchlog; + +import org.apache.atlas.model.searchlog.SearchRequestLogData; + +public interface SearchLogger { + + void log(SearchRequestLogData searchRequestData); +} \ No newline at end of file diff --git a/repository/src/main/java/org/apache/atlas/searchlog/SearchLoggingConsumer.java b/repository/src/main/java/org/apache/atlas/searchlog/SearchLoggingConsumer.java new file mode 100644 index 00000000000..b3cfa24cb7f --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/searchlog/SearchLoggingConsumer.java @@ -0,0 +1,26 @@ +package org.apache.atlas.searchlog; + +import org.apache.atlas.model.searchlog.SearchRequestLogData; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; + +public class SearchLoggingConsumer implements Runnable { + private static final Logger LOG = LoggerFactory.getLogger(SearchLoggingConsumer.class); + + private final List esSearchLoggers; + private final SearchRequestLogData searchRequestLogData; + + public SearchLoggingConsumer(List esSearchLoggers, SearchRequestLogData searchRequestLogData) { + this.esSearchLoggers = esSearchLoggers; + this.searchRequestLogData = searchRequestLogData; + } + + @Override + public void run() { + for (SearchLogger esSearchLogger : esSearchLoggers) { + esSearchLogger.log(searchRequestLogData); + } + } +} \ No newline at end of file diff --git a/repository/src/main/java/org/apache/atlas/searchlog/SearchLoggingManagement.java b/repository/src/main/java/org/apache/atlas/searchlog/SearchLoggingManagement.java new file mode 100644 index 00000000000..52710352706 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/searchlog/SearchLoggingManagement.java @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.searchlog; + +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.atlas.ApplicationProperties; +import org.apache.atlas.AtlasConfiguration; +import org.apache.atlas.model.searchlog.SearchRequestLogData; +import org.apache.atlas.type.AtlasType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import javax.inject.Inject; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +@Component +public class SearchLoggingManagement { + private static final Logger LOG = LoggerFactory.getLogger(SearchLoggingManagement.class); + + private final List esSearchLoggers; + private final ExecutorService executorService; + + @Inject + public SearchLoggingManagement(List esSearchLoggers) { + this.esSearchLoggers = esSearchLoggers; + + this.executorService = Executors.newFixedThreadPool(AtlasConfiguration.SEARCH_LOGGER_MAX_THREADS.getInt(), + new ThreadFactoryBuilder() + .setDaemon(true) + .setNameFormat("atlas-search-logger-%d") + .build() + ); + } + + public void log(SearchRequestLogData searchRequestLogData) { + SearchLoggingConsumer loggerConsumer = new SearchLoggingConsumer(esSearchLoggers, searchRequestLogData); + this.executorService.submit(loggerConsumer); + } +} \ No newline at end of file diff --git a/repository/src/main/java/org/apache/atlas/stats/EntityStatsListenerV2.java b/repository/src/main/java/org/apache/atlas/stats/EntityStatsListenerV2.java index f1f80cbb472..dd35c24b8d9 100644 --- a/repository/src/main/java/org/apache/atlas/stats/EntityStatsListenerV2.java +++ b/repository/src/main/java/org/apache/atlas/stats/EntityStatsListenerV2.java @@ -17,6 +17,7 @@ */ package org.apache.atlas.stats; +import org.apache.atlas.annotation.EnableConditional; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.listener.EntityChangeListenerV2; import org.apache.atlas.model.glossary.AtlasGlossaryTerm; @@ -25,6 +26,7 @@ import org.apache.atlas.model.instance.AtlasRelatedObjectId; import org.apache.atlas.model.instance.AtlasRelationship; import org.apache.atlas.repository.Constants; +import org.springframework.core.annotation.Order; import org.springframework.stereotype.Component; import javax.inject.Inject; @@ -33,6 +35,8 @@ import java.util.Set; @Component +@EnableConditional(property = "atlas.enable.entity.stats") +@Order(8) public class EntityStatsListenerV2 implements EntityChangeListenerV2 { private final StatsClient statsClient; @@ -79,7 +83,7 @@ public void onClassificationsAdded(AtlasEntity entity, List } @Override - public void onClassificationsAdded(List entities, List classifications) throws AtlasBaseException { + public void onClassificationsAdded(List entities, List classifications, boolean forceInline) throws AtlasBaseException { if (classifications != null) { classifications.forEach(e -> this.statsClient.increment(Constants.CLASSIFICATIONS_ADDED_METRIC)); } diff --git a/repository/src/main/java/org/apache/atlas/stats/StatsClient.java b/repository/src/main/java/org/apache/atlas/stats/StatsClient.java index 277454c2d96..f79304fedcf 100644 --- a/repository/src/main/java/org/apache/atlas/stats/StatsClient.java +++ b/repository/src/main/java/org/apache/atlas/stats/StatsClient.java @@ -26,12 +26,14 @@ import org.apache.commons.configuration.Configuration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.core.annotation.Order; import org.springframework.stereotype.Component; import javax.inject.Singleton; @Singleton @Component +@Order(7) public class StatsClient implements Service { private static final Logger LOG = LoggerFactory.getLogger(StatsClient.class); diff --git a/repository/src/main/java/org/apache/atlas/store/AtlasTypeDefStore.java b/repository/src/main/java/org/apache/atlas/store/AtlasTypeDefStore.java index 00ba85ca1eb..fa9abeca106 100644 --- a/repository/src/main/java/org/apache/atlas/store/AtlasTypeDefStore.java +++ b/repository/src/main/java/org/apache/atlas/store/AtlasTypeDefStore.java @@ -104,9 +104,8 @@ AtlasClassificationDef updateClassificationDefByGuid(String guid, AtlasClassific void deleteTypesDef(AtlasTypesDef typesDef) throws AtlasBaseException; AtlasTypesDef searchTypesDef(SearchFilter searchFilter) throws AtlasBaseException; + boolean hasBuiltInTypeName(AtlasBaseTypeDef typeDef); - - /* Generic operation */ AtlasBaseTypeDef getByName(String name) throws AtlasBaseException; AtlasBaseTypeDef getByGuid(String guid) throws AtlasBaseException; diff --git a/repository/src/main/java/org/apache/atlas/tasks/AbstractTask.java b/repository/src/main/java/org/apache/atlas/tasks/AbstractTask.java index 33e1d6a477b..6a64e91a736 100644 --- a/repository/src/main/java/org/apache/atlas/tasks/AbstractTask.java +++ b/repository/src/main/java/org/apache/atlas/tasks/AbstractTask.java @@ -18,10 +18,13 @@ package org.apache.atlas.tasks; import org.apache.atlas.model.tasks.AtlasTask; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import static org.apache.atlas.model.tasks.AtlasTask.Status; public abstract class AbstractTask { + private static final Logger LOG = LoggerFactory.getLogger(AbstractTask.class); private final AtlasTask task; public AbstractTask(AtlasTask task) { @@ -32,7 +35,7 @@ public void run() throws Exception { try { perform(); } catch (Exception exception) { - task.setStatusPending(); + task.setStatus(Status.FAILED); task.setErrorMessage(exception.getMessage()); diff --git a/repository/src/main/java/org/apache/atlas/tasks/AtlasTaskService.java b/repository/src/main/java/org/apache/atlas/tasks/AtlasTaskService.java new file mode 100644 index 00000000000..ba0fe1a4da7 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/tasks/AtlasTaskService.java @@ -0,0 +1,332 @@ +package org.apache.atlas.tasks; + +import org.apache.atlas.AtlasErrorCode; +import org.apache.atlas.DeleteType; +import org.apache.atlas.RequestContext; +import org.apache.atlas.annotation.GraphTransaction; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.tasks.AtlasTask; +import org.apache.atlas.model.tasks.TaskSearchParams; +import org.apache.atlas.model.tasks.TaskSearchResult; +import org.apache.atlas.repository.Constants; +import org.apache.atlas.repository.graph.GraphHelper; +import org.apache.atlas.repository.graphdb.*; +import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2; +import org.apache.atlas.repository.store.graph.v2.tasks.ClassificationPropagateTaskFactory; +import org.apache.atlas.repository.store.graph.v2.tasks.MeaningsTaskFactory; +import org.apache.atlas.utils.AtlasJson; +import org.apache.atlas.utils.AtlasPerfMetrics; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import javax.inject.Inject; +import java.lang.reflect.Field; +import java.util.*; + +import static org.apache.atlas.repository.Constants.TASK_GUID; +import static org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.setEncodedProperty; +import static org.apache.atlas.repository.store.graph.v2.tasks.ClassificationTask.PARAM_CLASSIFICATION_VERTEX_ID; +import static org.apache.atlas.tasks.TaskRegistry.toAtlasTask; + +@Component +public class AtlasTaskService implements TaskService { + private static final Logger LOG = LoggerFactory.getLogger(AtlasTaskService.class); + + private final AtlasGraph graph; + + private final List retryAllowedStatuses; + + @Inject + public AtlasTaskService(AtlasGraph graph) { + this.graph = graph; + retryAllowedStatuses = new ArrayList<>(); + retryAllowedStatuses.add(AtlasTask.Status.COMPLETE.toString()); + retryAllowedStatuses.add(AtlasTask.Status.FAILED.toString()); + // Since classification vertex is deleted after the task gets deleted, no need to retry the DELETED task + } + + @Override + public TaskSearchResult getTasks(TaskSearchParams searchParams) throws AtlasBaseException { + TaskSearchResult ret = new TaskSearchResult(); + List tasks = new ArrayList<>(); + AtlasIndexQuery indexQuery = null; + DirectIndexQueryResult indexQueryResult; + + try { + indexQuery = searchTask(searchParams); + indexQueryResult = indexQuery.vertices(searchParams); + + if (indexQueryResult != null) { + Iterator iterator = indexQueryResult.getIterator(); + + while (iterator.hasNext()) { + AtlasVertex vertex = iterator.next().getVertex(); + + if (vertex != null) { + tasks.add(toAtlasTask(vertex)); + } else { + LOG.warn("Null vertex while fetching tasks"); + } + + } + + ret.setTasks(tasks); + ret.setApproximateCount(indexQuery.vertexTotals()); + ret.setAggregations(indexQueryResult.getAggregationMap()); + } + } catch (AtlasBaseException e) { + LOG.error("Failed to fetch tasks: {}", e.getMessage()); + throw e; + } catch (Exception e) { + e.printStackTrace(); + throw new AtlasBaseException(AtlasErrorCode.RUNTIME_EXCEPTION, e); + } + + return ret; + } + + @Override + public TaskSearchResult getTasksByCondition(int from, int size, List> mustConditions, List> shouldConditions, + List> mustNotConditions) throws AtlasBaseException { + Map dsl = getMap("from", from); + dsl.put("size", size); + Map> boolCondition = Collections.singletonMap("bool", new HashMap<>()); + Map shouldQuery = getMap("bool", getMap("should", shouldConditions)); + mustConditions.add(shouldQuery); + boolCondition.get("bool").put("must", mustConditions); + boolCondition.get("bool").put("must_not", mustNotConditions); + + dsl.put("query", boolCondition); + TaskSearchParams taskSearchParams = new TaskSearchParams(); + taskSearchParams.setDsl(dsl); + TaskSearchResult tasks = getTasks(taskSearchParams); + return tasks; + } + + private Map getMap(String key, Object value) { + Map map = new HashMap<>(); + map.put(key, value); + return map; + } + @Override + @GraphTransaction + public void retryTask(String taskGuid) throws AtlasBaseException { + TaskSearchParams taskSearchParams = getMatchQuery(taskGuid); + AtlasIndexQuery atlasIndexQuery = searchTask(taskSearchParams); + DirectIndexQueryResult indexQueryResult = atlasIndexQuery.vertices(taskSearchParams); + + AtlasVertex atlasVertex = getTaskVertex(indexQueryResult.getIterator(), taskGuid); + + String status = atlasVertex.getProperty(Constants.TASK_STATUS, String.class); + + // Retrial ability of the task is not limited to FAILED ones due to testing/debugging + if (! retryAllowedStatuses.contains(status)) { + throw new AtlasBaseException(AtlasErrorCode.TASK_STATUS_NOT_APPROPRIATE, taskGuid, status); + } + + setEncodedProperty(atlasVertex, Constants.TASK_STATUS, AtlasTask.Status.PENDING); + int attemptCount = atlasVertex.getProperty(Constants.TASK_ATTEMPT_COUNT, Integer.class); + setEncodedProperty(atlasVertex, Constants.TASK_ATTEMPT_COUNT, attemptCount+1); + } + + @Override + @GraphTransaction + public List createAtlasTasks(List tasks) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("createAtlasTasks"); + List supportedTypes = new ArrayList<>(); + supportedTypes.addAll(ClassificationPropagateTaskFactory.supportedTypes); + supportedTypes.addAll(MeaningsTaskFactory.supportedTypes); + + List createdTasks = new ArrayList<>(); + + if (CollectionUtils.isNotEmpty(tasks)) { + for (AtlasTask task : tasks) { + String taskType = task.getType(); + if (!supportedTypes.contains(taskType)) { + throw new AtlasBaseException(AtlasErrorCode.TASK_TYPE_NOT_SUPPORTED, task.getType()); + } + if (isClassificationTaskType(taskType)) { + String classificationName = task.getClassificationName(); + String entityGuid = task.getEntityGuid(); + String classificationId = StringUtils.isEmpty(task.getClassificationId()) ? resolveAndReturnClassificationId(classificationName, entityGuid) : task.getClassificationId(); + if (StringUtils.isEmpty(classificationId)) { + throw new AtlasBaseException(AtlasErrorCode.TASK_INVALID_PARAMETERS, task.toString()); + } + task.getParameters().put(PARAM_CLASSIFICATION_VERTEX_ID, classificationId); + } + task.setUpdatedTime(new Date()); + task.setCreatedTime(new Date()); + task.setStatusPending(); + task.setAttemptCount(0); + task.setGuid(UUID.randomUUID().toString()); + task.setCreatedBy(RequestContext.getCurrentUser()); + + createTaskVertex(task); + createdTasks.add(task); + } + } + + graph.commit(); + RequestContext.get().endMetricRecord(metric); + return createdTasks; + } + + private boolean isClassificationTaskType(String taskType) { + return ClassificationPropagateTaskFactory.supportedTypes.contains(taskType); + } + + private String resolveAndReturnClassificationId(String classificationName, String entityGuid) throws AtlasBaseException { + String ret = null; + AtlasVertex entityVertex = AtlasGraphUtilsV2.findByGuid(entityGuid); + if (entityVertex == null) { + throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND, entityGuid); + } + + AtlasVertex classificationVertex = GraphHelper.getClassificationVertex(entityVertex, classificationName); + + if (classificationVertex != null) { + ret = classificationVertex.getIdForDisplay(); + } + + return ret; + } + + @Override + @GraphTransaction + public List deleteAtlasTasks(List tasks) { + List deletedTasks = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(tasks)) { + for (AtlasTask task : tasks) { + String taskGuid = task.getGuid(); + try { + deleteTask(taskGuid); + + deletedTasks.add(task); + } catch (AtlasBaseException e) { + LOG.error("Failed to delete task {}", taskGuid); + } + } + } + return deletedTasks; + } + + private void deleteTask(String taskGuid) throws AtlasBaseException { + DeleteType deleteType = RequestContext.get().getDeleteType(); + if (deleteType == DeleteType.SOFT || deleteType == DeleteType.DEFAULT) { + softDelete(taskGuid); + } else if (deleteType == DeleteType.HARD) { + hardDelete(taskGuid); + } + } + + @Override + public AtlasVertex createTaskVertex(AtlasTask task) { + AtlasVertex ret = graph.addVertex(); + + setEncodedProperty(ret, Constants.TASK_GUID, task.getGuid()); + setEncodedProperty(ret, Constants.TASK_TYPE_PROPERTY_KEY, Constants.TASK_TYPE_NAME); + setEncodedProperty(ret, Constants.TASK_STATUS, task.getStatus().toString()); + setEncodedProperty(ret, Constants.TASK_TYPE, task.getType()); + setEncodedProperty(ret, Constants.TASK_CREATED_BY, task.getCreatedBy()); + setEncodedProperty(ret, Constants.TASK_CREATED_TIME, task.getCreatedTime()); + setEncodedProperty(ret, Constants.TASK_UPDATED_TIME, task.getUpdatedTime()); + if (task.getClassificationId() != null) { + setEncodedProperty(ret, Constants.TASK_CLASSIFICATION_ID, task.getClassificationId()); + } + + if(task.getEntityGuid() != null) { + setEncodedProperty(ret, Constants.TASK_ENTITY_GUID, task.getEntityGuid()); + } + + if (task.getStartTime() != null) { + setEncodedProperty(ret, Constants.TASK_START_TIME, task.getStartTime().getTime()); + } + + if (task.getEndTime() != null) { + setEncodedProperty(ret, Constants.TASK_END_TIME, task.getEndTime().getTime()); + } + + setEncodedProperty(ret, Constants.TASK_PARAMETERS, AtlasJson.toJson(task.getParameters())); + setEncodedProperty(ret, Constants.TASK_ATTEMPT_COUNT, task.getAttemptCount()); + setEncodedProperty(ret, Constants.TASK_ERROR_MESSAGE, task.getErrorMessage()); + + LOG.info("Creating task vertex: {}: {}, {}: {}, {}: {} ", + Constants.TASK_TYPE, task.getType(), + Constants.TASK_PARAMETERS, AtlasJson.toJson(task.getParameters()), + TASK_GUID, task.getGuid()); + + return ret; + } + + @Override + public void hardDelete(String guid) throws AtlasBaseException { + try { + AtlasGraphQuery query = graph.query() + .has(Constants.TASK_TYPE_PROPERTY_KEY, Constants.TASK_TYPE_NAME) + .has(TASK_GUID, guid); + + Iterator results = query.vertices().iterator(); + + if (results.hasNext()) { + graph.removeVertex(results.next()); + } + } catch (Exception exception) { + LOG.error("Error: deletingByGuid: {}", guid); + + throw new AtlasBaseException(exception); + } + } + + @Override + public void softDelete(String guid) throws AtlasBaseException{ + try { + AtlasGraphQuery query = graph.query() + .has(Constants.TASK_TYPE_PROPERTY_KEY, Constants.TASK_TYPE_NAME) + .has(TASK_GUID, guid); + + Iterator results = query.vertices().iterator(); + + if (results.hasNext()) { + AtlasVertex taskVertex = results.next(); + + setEncodedProperty(taskVertex, Constants.TASK_STATUS, AtlasTask.Status.DELETED); + setEncodedProperty(taskVertex, Constants.TASK_UPDATED_TIME, System.currentTimeMillis()); + } + } + catch (Exception exception) { + LOG.error("Error: on soft delete: {}", guid); + + throw new AtlasBaseException(exception); + } + } + + private AtlasVertex getTaskVertex(Iterator iterator, String taskGuid) throws AtlasBaseException { + while(iterator.hasNext()) { + AtlasVertex atlasVertex = iterator.next().getVertex(); + if (atlasVertex.getProperty(Constants.TASK_GUID, String.class).equals(taskGuid)) { + return atlasVertex; + } + } + throw new AtlasBaseException(AtlasErrorCode.TASK_NOT_FOUND, taskGuid); + } + + private TaskSearchParams getMatchQuery(String guid) { + TaskSearchParams params = new TaskSearchParams(); + params.setDsl(mapOf("query", mapOf("match", mapOf(TASK_GUID, guid)))); + return params; + } + + private Map mapOf(String key, Object value) { + Map map = new HashMap<>(); + map.put(key, value); + + return map; + } + + private AtlasIndexQuery searchTask(TaskSearchParams searchParams) throws AtlasBaseException { + return graph.elasticsearchQuery(Constants.VERTEX_INDEX, searchParams); + } +} diff --git a/repository/src/main/java/org/apache/atlas/tasks/TaskExecutor.java b/repository/src/main/java/org/apache/atlas/tasks/TaskExecutor.java index 81957d8fc2e..e224840e94f 100644 --- a/repository/src/main/java/org/apache/atlas/tasks/TaskExecutor.java +++ b/repository/src/main/java/org/apache/atlas/tasks/TaskExecutor.java @@ -17,54 +17,71 @@ */ package org.apache.atlas.tasks; -import com.google.common.annotations.VisibleForTesting; import com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.atlas.ICuratorFactory; +import org.apache.atlas.RequestContext; import org.apache.atlas.model.tasks.AtlasTask; import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.service.redis.RedisService; import org.apache.atlas.type.AtlasType; +import org.apache.atlas.utils.AtlasPerfTracer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.List; import java.util.Map; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; + public class TaskExecutor { + private static final Logger PERF_LOG = AtlasPerfTracer.getPerfLogger("atlas.task"); private static final Logger LOG = LoggerFactory.getLogger(TaskExecutor.class); private static final TaskLogger TASK_LOG = TaskLogger.getLogger(); private static final String TASK_NAME_FORMAT = "atlas-task-%d-"; - private final TaskRegistry registry; - private final Map taskTypeFactoryMap; + private static final boolean perfEnabled = AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG); + + private final ExecutorService taskExecutorService; + private final TaskRegistry registry; + private final Map taskTypeFactoryMap; private final TaskManagement.Statistics statistics; - private final ExecutorService executorService; + private final ICuratorFactory curatorFactory; + private final boolean isActiveActiveHAEnabled; + private final String zkRoot; - public TaskExecutor(TaskRegistry registry, Map taskTypeFactoryMap, TaskManagement.Statistics statistics) { - this.registry = registry; - this.taskTypeFactoryMap = taskTypeFactoryMap; - this.statistics = statistics; - this.executorService = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder() + private TaskQueueWatcher watcher; + private Thread watcherThread; + private RedisService redisService; + + public TaskExecutor(TaskRegistry registry, Map taskTypeFactoryMap, TaskManagement.Statistics statistics, + ICuratorFactory curatorFactory, RedisService redisService, final String zkRoot, boolean isActiveActiveHAEnabled) { + this.taskExecutorService = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder() .setDaemon(true) .setNameFormat(TASK_NAME_FORMAT + Thread.currentThread().getName()) .build()); - } - public void addAll(List tasks) { - for (AtlasTask task : tasks) { - if (task == null) { - continue; - } + this.registry = registry; + this.statistics = statistics; + this.taskTypeFactoryMap = taskTypeFactoryMap; + this.curatorFactory = curatorFactory; + this.redisService = redisService; + this.isActiveActiveHAEnabled = isActiveActiveHAEnabled; + this.zkRoot = zkRoot; + } - TASK_LOG.log(task); + public Thread startWatcherThread() { - this.executorService.submit(new TaskConsumer(task, this.registry, this.taskTypeFactoryMap, this.statistics)); - } + watcher = new TaskQueueWatcher(taskExecutorService, registry, taskTypeFactoryMap, statistics, curatorFactory, redisService, zkRoot, isActiveActiveHAEnabled); + watcherThread = new Thread(watcher); + watcherThread.start(); + return watcherThread; } - @VisibleForTesting - void waitUntilDone() throws InterruptedException { - Thread.sleep(5000); + public void stopQueueWatcher() { + if (watcher != null) { + watcher.shutdown(); + } } static class TaskConsumer implements Runnable { @@ -74,12 +91,17 @@ static class TaskConsumer implements Runnable { private final TaskRegistry registry; private final TaskManagement.Statistics statistics; private final AtlasTask task; + private CountDownLatch latch; - public TaskConsumer(AtlasTask task, TaskRegistry registry, Map taskTypeFactoryMap, TaskManagement.Statistics statistics) { + AtlasPerfTracer perf = null; + + public TaskConsumer(AtlasTask task, TaskRegistry registry, Map taskTypeFactoryMap, TaskManagement.Statistics statistics, + CountDownLatch latch) { this.task = task; this.registry = registry; this.taskTypeFactoryMap = taskTypeFactoryMap; this.statistics = statistics; + this.latch = latch; } @Override @@ -88,12 +110,23 @@ public void run() { int attemptCount; try { + if (task == null) { + TASK_LOG.info("Task not scheduled as it was not found"); + return; + } + + TASK_LOG.info("Task guid = "+task.getGuid()); taskVertex = registry.getVertex(task.getGuid()); + if (taskVertex == null) { + TASK_LOG.warn("Task not scheduled as vertex not found", task); + } - if (task == null || taskVertex == null || task.getStatus() == AtlasTask.Status.COMPLETE) { - TASK_LOG.warn("Task not scheduled as it was not found or status was COMPLETE!", task); + if (task.getStatus() == AtlasTask.Status.COMPLETE) { + TASK_LOG.warn("Task not scheduled as status was COMPLETE!", task); + } - return; + if (perfEnabled) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, String.format("atlas.task:%s", task.getGuid(), task.getType())); } statistics.increment(1); @@ -103,24 +136,25 @@ public void run() { if (attemptCount >= MAX_ATTEMPT_COUNT) { TASK_LOG.warn("Max retry count for task exceeded! Skipping!", task); + task.setStatus(AtlasTask.Status.FAILED); + registry.updateStatus(taskVertex, task); + return; } + LOG.info(String.format("Started performing task with guid: %s", task.getGuid())); + performTask(taskVertex, task); - } catch (InterruptedException exception) { - if (task != null) { - registry.updateStatus(taskVertex, task); - TASK_LOG.error("{}: {}: Interrupted!", task, exception); - } else { - LOG.error("Interrupted!", exception); - } + LOG.info(String.format("Finished task with guid: %s", task.getGuid())); + + } catch (InterruptedException exception) { + registry.updateStatus(taskVertex, task); + TASK_LOG.error("{}: {}: Interrupted!", task, exception); statistics.error(); } catch (Exception exception) { if (task != null) { - task.updateStatusFromAttemptCount(); - registry.updateStatus(taskVertex, task); TASK_LOG.error("Error executing task. Please perform the operation again!", task, exception); @@ -135,6 +169,10 @@ public void run() { TASK_LOG.log(task); } + + latch.countDown(); + RequestContext.get().clearCache(); + AtlasPerfTracer.log(perf); } } @@ -147,9 +185,11 @@ private void performTask(AtlasVertex taskVertex, AtlasTask task) throws Exceptio AbstractTask runnableTask = factory.create(task); + registry.inProgress(taskVertex, task); + runnableTask.run(); - registry.deleteComplete(taskVertex, task); + registry.complete(taskVertex, task); statistics.successPrint(); } diff --git a/repository/src/main/java/org/apache/atlas/tasks/TaskFactoryRegistry.java b/repository/src/main/java/org/apache/atlas/tasks/TaskFactoryRegistry.java index 4c93f0eeeb8..621d48681c6 100644 --- a/repository/src/main/java/org/apache/atlas/tasks/TaskFactoryRegistry.java +++ b/repository/src/main/java/org/apache/atlas/tasks/TaskFactoryRegistry.java @@ -45,8 +45,8 @@ public TaskFactoryRegistry(TaskManagement taskManagement, Set facto @PostConstruct public void startTaskManagement() throws AtlasException { try { - if (!taskManagement.hasStarted()) { - LOG.info("TaskFactoryRegistry: TaskManagement start skipped! Someone else will start it."); + if (taskManagement.isWatcherActive()) { + LOG.info("TaskFactoryRegistry: TaskManagement already started!"); return; } diff --git a/repository/src/main/java/org/apache/atlas/tasks/TaskManagement.java b/repository/src/main/java/org/apache/atlas/tasks/TaskManagement.java index 9a519bacc50..da9d0c72f95 100644 --- a/repository/src/main/java/org/apache/atlas/tasks/TaskManagement.java +++ b/repository/src/main/java/org/apache/atlas/tasks/TaskManagement.java @@ -20,17 +20,19 @@ import com.google.common.annotations.VisibleForTesting; import org.apache.atlas.AtlasConfiguration; import org.apache.atlas.AtlasException; +import org.apache.atlas.ICuratorFactory; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.ha.HAConfiguration; import org.apache.atlas.listener.ActiveStateChangeHandler; import org.apache.atlas.model.tasks.AtlasTask; import org.apache.atlas.service.Service; +import org.apache.atlas.service.redis.RedisService; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.configuration.Configuration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.core.annotation.Order; import org.springframework.stereotype.Component; -import org.springframework.util.CollectionUtils; import javax.inject.Inject; import java.util.ArrayList; @@ -44,48 +46,62 @@ public class TaskManagement implements Service, ActiveStateChangeHandler { private static final Logger LOG = LoggerFactory.getLogger(TaskManagement.class); - private final ThreadLocal taskExecutorThreadLocal = new ThreadLocal<>(); + private TaskExecutor taskExecutor; private final Configuration configuration; private final TaskRegistry registry; private final Statistics statistics; private final Map taskTypeFactoryMap; - private boolean hasStarted; + private final ICuratorFactory curatorFactory; + private final RedisService redisService; + private Thread watcherThread = null; + + public enum DeleteType { + SOFT, + HARD + } @Inject - public TaskManagement(Configuration configuration, TaskRegistry taskRegistry) { + public TaskManagement(Configuration configuration, TaskRegistry taskRegistry, ICuratorFactory curatorFactory, RedisService redisService) { this.configuration = configuration; this.registry = taskRegistry; + this.redisService = redisService; this.statistics = new Statistics(); this.taskTypeFactoryMap = new HashMap<>(); + this.curatorFactory = curatorFactory; } @VisibleForTesting - TaskManagement(Configuration configuration, TaskRegistry taskRegistry, TaskFactory taskFactory) { + TaskManagement(Configuration configuration, TaskRegistry taskRegistry, TaskFactory taskFactory, ICuratorFactory curatorFactory, RedisService redisService) { this.configuration = configuration; this.registry = taskRegistry; + this.redisService = redisService; this.statistics = new Statistics(); this.taskTypeFactoryMap = new HashMap<>(); + this.curatorFactory = curatorFactory; createTaskTypeFactoryMap(taskTypeFactoryMap, taskFactory); } @Override public void start() throws AtlasException { - if (configuration == null || !HAConfiguration.isHAEnabled(configuration)) { - startInternal(); - } else { - LOG.info("TaskManagement.start(): deferring until instance activation"); + try { + if (configuration == null || !HAConfiguration.isHAEnabled(configuration)) { + startInternal(); + } else { + LOG.info("TaskManagement.start(): deferring until instance activation"); + } + } catch (Exception e) { + throw e; } - - this.hasStarted = true; } - public boolean hasStarted() { - return this.hasStarted; + public boolean isWatcherActive() { + return watcherThread != null; } @Override public void stop() throws AtlasException { + stopQueueWatcher(); LOG.info("TaskManagement: Stopped!"); } @@ -93,13 +109,18 @@ public void stop() throws AtlasException { public void instanceIsActive() throws AtlasException { LOG.info("==> TaskManagement.instanceIsActive()"); - startInternal(); + try { + startInternal(); + } catch (Exception e) { + throw e; + } LOG.info("<== TaskManagement.instanceIsActive()"); } @Override public void instanceIsPassive() throws AtlasException { + stopQueueWatcher(); LOG.info("TaskManagement.instanceIsPassive(): no action needed"); } @@ -113,19 +134,45 @@ public void addFactory(TaskFactory taskFactory) { } public AtlasTask createTask(String taskType, String createdBy, Map parameters) { - return this.registry.createVertex(taskType, createdBy, parameters); + return this.registry.createVertex(taskType, createdBy, parameters, null, null); + } + + public AtlasTask createTask(String taskType, String createdBy, Map parameters, String classificationId, String entityGuid) { + return this.registry.createVertex(taskType, createdBy, parameters, classificationId, entityGuid); } public List getAll() { return this.registry.getAll(); } - public void addAll(List tasks) { - if (CollectionUtils.isEmpty(tasks)) { - return; + public List getAll(List statusList, int offset, int limit) { + return this.registry.getAll(statusList, offset, limit); + } + + public List getQueuedTasks() { + return this.registry.getTasksForReQueue(); + } + + public void retryTasks(List taskGuids) throws AtlasBaseException { + List taskToRetry = new ArrayList<>(); + for (String taskGuid : taskGuids) { + AtlasTask task = getByGuid(taskGuid); + + if (task != null && + (task.getStatus().equals(AtlasTask.Status.FAILED) || task.getStatus().equals(AtlasTask.Status.IN_PROGRESS))) { + /* Allowing IN_PROGRESS task retry for following scenario + -> Started a task + -> before task gets completed Cassandra gets completely down + -> task is still in IN_PROGRESS state & as Cassandra write is not possible it will never change + -> Once cassandra is up & Atlas started communicating to Cassandra again such task may be retried + */ + taskToRetry.add(task); + } } - dispatchTasks(tasks); + if (CollectionUtils.isNotEmpty(taskToRetry)) { + //addAll(taskToRetry); + } } public AtlasTask getByGuid(String guid) throws AtlasBaseException { @@ -152,6 +199,14 @@ public List getByGuids(List guids) throws AtlasBaseException return ret; } + public List getByGuidsES(List guids) throws AtlasBaseException { + return registry.getByIdsES(guids); + } + + public List getInProgressTasks() { + return registry.getInProgressTasks(); + } + public void deleteByGuid(String guid) throws AtlasBaseException { try { this.registry.deleteByGuid(guid); @@ -160,6 +215,19 @@ public void deleteByGuid(String guid) throws AtlasBaseException { } } + public void deleteByGuid(String guid, DeleteType deleteType) throws AtlasBaseException { + try { + if (deleteType == DeleteType.SOFT) { + this.registry.softDelete(guid); + } + else { + this.registry.deleteByGuid(guid); + } + } catch (Exception exception) { + throw new AtlasBaseException(exception); + } + } + public void deleteByGuids(List guids) throws AtlasBaseException { if (CollectionUtils.isEmpty(guids)) { return; @@ -170,16 +238,17 @@ public void deleteByGuids(List guids) throws AtlasBaseException { } } - private void dispatchTasks(List tasks) { - if (CollectionUtils.isEmpty(tasks)) { - return; - } + private synchronized void startWatcherThread() { - if (this.taskExecutorThreadLocal.get() == null) { - this.taskExecutorThreadLocal.set(new TaskExecutor(registry, taskTypeFactoryMap, statistics)); + if (this.taskExecutor == null) { + final boolean isActiveActiveHAEnabled = HAConfiguration.isActiveActiveHAEnabled(configuration); + final String zkRoot = HAConfiguration.getZookeeperProperties(configuration).getZkRoot(); + this.taskExecutor = new TaskExecutor(registry, taskTypeFactoryMap, statistics, curatorFactory, redisService, zkRoot,isActiveActiveHAEnabled); } - this.taskExecutorThreadLocal.get().addAll(tasks); + if (watcherThread == null) { + watcherThread = this.taskExecutor.startWatcherThread(); + } this.statistics.print(); } @@ -195,19 +264,12 @@ private void startInternal() { return; } - queuePendingTasks(); - } - - private void queuePendingTasks() { - if (AtlasConfiguration.TASKS_USE_ENABLED.getBoolean() == false) { - return; + try { + startWatcherThread(); + } catch (Exception e) { + LOG.error("TaskManagement: Error while re queue tasks"); + e.printStackTrace(); } - - List pendingTasks = this.registry.getPendingTasks(); - - LOG.info("TaskManagement: Found: {}: Tasks in pending state.", pendingTasks.size()); - - addAll(pendingTasks); } @VisibleForTesting @@ -227,6 +289,11 @@ static Map createTaskTypeFactoryMap(Map + * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.tasks; + +import org.apache.atlas.AtlasConfiguration; +import org.apache.atlas.AtlasConstants; +import org.apache.atlas.ICuratorFactory; +import org.apache.atlas.model.tasks.AtlasTask; +import org.apache.atlas.service.redis.RedisService; +import org.apache.commons.collections.CollectionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.PreDestroy; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +public class TaskQueueWatcher implements Runnable { + private static final Logger LOG = LoggerFactory.getLogger(TaskQueueWatcher.class); + private static final TaskExecutor.TaskLogger TASK_LOG = TaskExecutor.TaskLogger.getLogger(); + private final String zkRoot; + private final boolean isActiveActiveHAEnabled; + + private TaskRegistry registry; + private final ExecutorService executorService; + private final Map taskTypeFactoryMap; + private final TaskManagement.Statistics statistics; + private final ICuratorFactory curatorFactory; + private final RedisService redisService; + + private static long pollInterval = AtlasConfiguration.TASKS_REQUEUE_POLL_INTERVAL.getLong(); + private static final String TASK_LOCK = "/task-lock"; + private static final String ATLAS_TASK_LOCK = "atlas:task:lock"; + + private final AtomicBoolean shouldRun = new AtomicBoolean(false); + + public TaskQueueWatcher(ExecutorService executorService, TaskRegistry registry, + Map taskTypeFactoryMap, TaskManagement.Statistics statistics, + ICuratorFactory curatorFactory, RedisService redisService, final String zkRoot, boolean isActiveActiveHAEnabled) { + + this.registry = registry; + this.executorService = executorService; + this.taskTypeFactoryMap = taskTypeFactoryMap; + this.statistics = statistics; + this.curatorFactory = curatorFactory; + this.redisService = redisService; + this.zkRoot = zkRoot; + this.isActiveActiveHAEnabled = isActiveActiveHAEnabled; + } + + public void shutdown() { + shouldRun.set(false); + LOG.info("TaskQueueWatcher: Shutdown"); + } + + @Override + public void run() { + shouldRun.set(true); + + if (LOG.isDebugEnabled()) { + LOG.debug("TaskQueueWatcher: running {}:{}", Thread.currentThread().getName(), Thread.currentThread().getId()); + } + while (shouldRun.get()) { + try { + if (!redisService.acquireDistributedLock(ATLAS_TASK_LOCK)) { + Thread.sleep(AtlasConstants.TASK_WAIT_TIME_MS); + continue; + } + + TasksFetcher fetcher = new TasksFetcher(registry); + + Thread tasksFetcherThread = new Thread(fetcher); + tasksFetcherThread.start(); + tasksFetcherThread.join(); + + List tasks = fetcher.getTasks(); + if (CollectionUtils.isNotEmpty(tasks)) { + final CountDownLatch latch = new CountDownLatch(tasks.size()); + submitAll(tasks, latch); + LOG.info("Submitted {} tasks to the queue", tasks.size()); + waitForTasksToComplete(latch); + } else { + redisService.releaseDistributedLock(ATLAS_TASK_LOCK); + } + Thread.sleep(pollInterval); + } catch (InterruptedException interruptedException) { + LOG.error("TaskQueueWatcher: Interrupted: thread is terminated, new tasks will not be loaded into the queue until next restart"); + break; + } catch (Exception e) { + LOG.error("TaskQueueWatcher: Exception occurred " + e.getMessage(), e); + } finally { + redisService.releaseDistributedLock(ATLAS_TASK_LOCK); + } + } + } + + private void waitForTasksToComplete(final CountDownLatch latch) throws InterruptedException { + if (latch.getCount() != 0) { + LOG.info("TaskQueueWatcher: Waiting on Latch, current count: {}", latch.getCount()); + latch.await(); + LOG.info("TaskQueueWatcher: Waiting completed on Latch, current count: {}", latch.getCount()); + } + } + + private void submitAll(List tasks, CountDownLatch latch) { + if (CollectionUtils.isNotEmpty(tasks)) { + + for (AtlasTask task : tasks) { + if (task != null) { + TASK_LOG.log(task); + } + + this.executorService.submit(new TaskExecutor.TaskConsumer(task, this.registry, this.taskTypeFactoryMap, this.statistics, latch)); + } + + LOG.info("TasksFetcher: Submitted {} tasks to the queue", tasks.size()); + } else { + LOG.info("TasksFetcher: No task to queue"); + } + } + + static class TasksFetcher implements Runnable { + private TaskRegistry registry; + private List tasks = new ArrayList<>(); + + public TasksFetcher(TaskRegistry registry) { + this.registry = registry; + } + + @Override + public void run() { + if (LOG.isDebugEnabled()) { + LOG.debug("TasksFetcher: Fetching tasks for queuing"); + } + + this.tasks = registry.getTasksForReQueue(); + } + + public List getTasks() { + return tasks; + } + } + + @PreDestroy + public void cleanUp() { + if (!Objects.isNull(this.executorService)) { + this.redisService.releaseDistributedLock(ATLAS_TASK_LOCK); + this.executorService.shutdownNow(); + try { + this.executorService.awaitTermination(1, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + } +} diff --git a/repository/src/main/java/org/apache/atlas/tasks/TaskRegistry.java b/repository/src/main/java/org/apache/atlas/tasks/TaskRegistry.java index 32e0ad91b44..1537fcd5760 100644 --- a/repository/src/main/java/org/apache/atlas/tasks/TaskRegistry.java +++ b/repository/src/main/java/org/apache/atlas/tasks/TaskRegistry.java @@ -17,27 +17,41 @@ */ package org.apache.atlas.tasks; +import org.apache.atlas.AtlasConfiguration; +import org.apache.atlas.RequestContext; import org.apache.atlas.annotation.GraphTransaction; import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.discovery.IndexSearchParams; import org.apache.atlas.model.tasks.AtlasTask; +import org.apache.atlas.model.tasks.TaskSearchParams; +import org.apache.atlas.model.tasks.TaskSearchResult; import org.apache.atlas.repository.Constants; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasGraphQuery; +import org.apache.atlas.repository.graphdb.AtlasIndexQuery; import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.repository.graphdb.DirectIndexQueryResult; import org.apache.atlas.type.AtlasType; -import org.apache.atlas.utils.AtlasJson; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections4.ListUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; import javax.inject.Inject; import java.util.ArrayList; +import java.util.Collections; import java.util.Date; +import java.util.HashMap; import java.util.Iterator; +import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import static org.apache.atlas.repository.Constants.TASK_GUID; +import static org.apache.atlas.repository.Constants.TASK_STATUS; import static org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.setEncodedProperty; @Component @@ -45,10 +59,16 @@ public class TaskRegistry { private static final Logger LOG = LoggerFactory.getLogger(TaskRegistry.class); private AtlasGraph graph; + private TaskService taskService; + private int queueSize; + private boolean useGraphQuery; @Inject - public TaskRegistry(AtlasGraph graph) { + public TaskRegistry(AtlasGraph graph, TaskService taskService) { this.graph = graph; + this.taskService = taskService; + queueSize = AtlasConfiguration.TASKS_QUEUE_SIZE.getInt(); + useGraphQuery = AtlasConfiguration.TASKS_REQUEUE_GRAPH_QUERY.getBoolean(); } @GraphTransaction @@ -58,7 +78,6 @@ public AtlasTask save(AtlasTask task) { return toAtlasTask(vertex); } - @GraphTransaction public List getPendingTasks() { List ret = new ArrayList<>(); @@ -73,12 +92,37 @@ public List getPendingTasks() { while (results.hasNext()) { AtlasVertex vertex = results.next(); - ret.add(toAtlasTask(vertex)); + if(vertex != null) { + ret.add(toAtlasTask(vertex)); + } } } catch (Exception exception) { LOG.error("Error fetching pending tasks!", exception); - } finally { - graph.commit(); + } + + return ret; + } + + public List getInProgressTasks() { + List ret = new ArrayList<>(); + + try { + AtlasGraphQuery query = graph.query() + .has(Constants.TASK_TYPE_PROPERTY_KEY, Constants.TASK_TYPE_NAME) + .has(Constants.TASK_STATUS, AtlasTask.Status.IN_PROGRESS) + .orderBy(Constants.TASK_CREATED_TIME, AtlasGraphQuery.SortOrder.ASC); + + Iterator results = query.vertices().iterator(); + + while (results.hasNext()) { + AtlasVertex vertex = results.next(); + + if(vertex != null) { + ret.add(toAtlasTask(vertex)); + } + } + } catch (Exception exception) { + LOG.error("Error fetching in progress tasks!", exception); } return ret; @@ -99,22 +143,59 @@ public void updateStatus(AtlasVertex taskVertex, AtlasTask task) { @GraphTransaction public void deleteByGuid(String guid) throws AtlasBaseException { try { - AtlasGraphQuery query = graph.query() - .has(Constants.TASK_TYPE_PROPERTY_KEY, Constants.TASK_TYPE_NAME) - .has(TASK_GUID, guid); - - Iterator results = query.vertices().iterator(); - - if (results.hasNext()) { - graph.removeVertex(results.next()); - } + taskService.hardDelete(guid); } catch (Exception exception) { LOG.error("Error: deletingByGuid: {}", guid); throw new AtlasBaseException(exception); + } finally { + graph.commit(); } } + @GraphTransaction + public void softDelete(String guid) throws AtlasBaseException{ + try { + taskService.softDelete(guid); + } + catch (Exception exception) { + LOG.error("Error: on soft delete: {}", guid); + + throw new AtlasBaseException(exception); + } + } + + public List getByIdsES(List guids) throws AtlasBaseException { + List ret = new ArrayList<>(); + + List> chunkedGuids = ListUtils.partition(guids, 50); + + for (List chunkedGuidList : chunkedGuids) { + List should = new ArrayList<>(); + for (String guid : chunkedGuidList) { + should.add(mapOf("match", mapOf(TASK_GUID, guid))); + } + + TaskSearchParams params = new TaskSearchParams(); + params.setDsl(mapOf("query", mapOf("bool", mapOf("should", should)))); + + TaskSearchResult result = taskService.getTasks(params); + if (result == null) { + return null; + } + + // as __task_guid is text field, might result multiple results due to "-" tokenizing in ES + // adding filtering layer to filter exact tasks + ret.addAll(filterTasksByGuids(result.getTasks(), chunkedGuidList)); + } + + return ret; + } + + private List filterTasksByGuids(List tasks, List guidList) { + return tasks.stream().filter(task -> guidList.contains(task.getGuid())).collect(Collectors.toList()); + } + @GraphTransaction public void deleteComplete(AtlasVertex taskVertex, AtlasTask task) { updateStatus(taskVertex, task); @@ -122,6 +203,36 @@ public void deleteComplete(AtlasVertex taskVertex, AtlasTask task) { deleteVertex(taskVertex); } + public void inProgress(AtlasVertex taskVertex, AtlasTask task) { + RequestContext.get().setCurrentTask(task); + + task.setStartTime(new Date()); + + setEncodedProperty(taskVertex, Constants.TASK_START_TIME, task.getStartTime()); + setEncodedProperty(taskVertex, Constants.TASK_STATUS, AtlasTask.Status.IN_PROGRESS); + setEncodedProperty(taskVertex, Constants.TASK_UPDATED_TIME, System.currentTimeMillis()); + graph.commit(); + } + + @GraphTransaction + public void complete(AtlasVertex taskVertex, AtlasTask task) { + if (task.getEndTime() != null) { + setEncodedProperty(taskVertex, Constants.TASK_END_TIME, task.getEndTime()); + + if (task.getStartTime() == null) { + LOG.warn("Task start time was not recorded since could not calculate task's total take taken"); + } else { + long timeTaken = task.getEndTime().getTime() - task.getStartTime().getTime(); + timeTaken = TimeUnit.MILLISECONDS.toSeconds(timeTaken); + setEncodedProperty(taskVertex, Constants.TASK_TIME_TAKEN_IN_SECONDS, timeTaken); + } + } + + updateStatus(taskVertex, task); + + LOG.info(String.format("TaskRegistry complete %s", task.toString())); + } + @GraphTransaction public AtlasTask getById(String guid) { AtlasGraphQuery query = graph.query() @@ -158,12 +269,159 @@ public List getAll() { return ret; } + /* + * This returns tasks which has status IN statusList + * If not specified, return all tasks + * */ + @GraphTransaction + public List getAll(List statusList, int offset, int limit) { + List ret = new ArrayList<>(); + AtlasGraphQuery query = graph.query() + .has(Constants.TASK_TYPE_PROPERTY_KEY, Constants.TASK_TYPE_NAME); + + if (CollectionUtils.isNotEmpty(statusList)) { + List orConditions = new LinkedList<>(); + + for (String status : statusList) { + orConditions.add(query.createChildQuery().has(Constants.TASK_STATUS, AtlasTask.Status.from(status))); + } + + query.or(orConditions); + } + + query.orderBy(Constants.TASK_CREATED_TIME, AtlasGraphQuery.SortOrder.DESC); + + Iterator results = query.vertices(offset, limit).iterator(); + + while (results.hasNext()) { + ret.add(toAtlasTask(results.next())); + } + + return ret; + } + + public List getTasksForReQueue() { + List ret = null; + + if (useGraphQuery) { + ret = getTasksForReQueueGraphQuery(); + } else { + ret = getTasksForReQueueIndexSearch(); + } + + return ret; + } + + public List getTasksForReQueueGraphQuery() { + + List ret = new ArrayList<>(); + AtlasGraphQuery query = graph.query() + .has(Constants.TASK_TYPE_PROPERTY_KEY, Constants.TASK_TYPE_NAME); + + List orConditions = new LinkedList<>(); + orConditions.add(query.createChildQuery().has(Constants.TASK_STATUS, AtlasTask.Status.IN_PROGRESS)); + orConditions.add(query.createChildQuery().has(Constants.TASK_STATUS, AtlasTask.Status.PENDING)); + query.or(orConditions); + + query.orderBy(Constants.TASK_CREATED_TIME, AtlasGraphQuery.SortOrder.ASC); + + Iterator results = query.vertices(queueSize).iterator(); + + while (results.hasNext()) { + AtlasVertex vertex = results.next(); + + if (vertex != null) { + ret.add(toAtlasTask(vertex)); + } else { + LOG.error("Null vertex while re-queuing tasks"); + } + } + + return ret; + } + + public List getTasksForReQueueIndexSearch() { + DirectIndexQueryResult indexQueryResult = null; + List ret = new ArrayList<>(); + + int size = 1000; + int from = 0; + + IndexSearchParams indexSearchParams = new IndexSearchParams(); + + List statusClauseList = new ArrayList(); + statusClauseList.add(mapOf("match", mapOf(TASK_STATUS, AtlasTask.Status.IN_PROGRESS.toString()))); + statusClauseList.add(mapOf("match", mapOf(TASK_STATUS, AtlasTask.Status.PENDING.toString()))); + + Map dsl = mapOf("query", mapOf("bool", mapOf("should", statusClauseList))); + dsl.put("sort", Collections.singletonList(mapOf(Constants.TASK_CREATED_TIME, mapOf("order", "asc")))); + dsl.put("size", size); + int totalFetched = 0; + while (true) { + int fetched = 0; + try { + if (totalFetched + size > queueSize) { + size = queueSize - totalFetched; + } + + dsl.put("from", from); + dsl.put("size", size); + + indexSearchParams.setDsl(dsl); + + AtlasIndexQuery indexQuery = graph.elasticsearchQuery(Constants.VERTEX_INDEX, indexSearchParams); + + try { + indexQueryResult = indexQuery.vertices(indexSearchParams); + } catch (AtlasBaseException e) { + LOG.error("Failed to fetch pending/in-progress task vertices to re-que"); + e.printStackTrace(); + break; + } + + if (indexQueryResult != null) { + Iterator iterator = indexQueryResult.getIterator(); + + while (iterator.hasNext()) { + AtlasVertex vertex = iterator.next().getVertex(); + + if (vertex != null) { + AtlasTask atlasTask = toAtlasTask(vertex); + if (atlasTask.getStatus().equals(AtlasTask.Status.PENDING) || + atlasTask.getStatus().equals(AtlasTask.Status.IN_PROGRESS) ){ + LOG.info(String.format("Fetched task from index search: %s", atlasTask.toString())); + ret.add(atlasTask); + } + else { + LOG.warn(String.format("There is a mismatch on tasks status between ES and Cassandra for guid: %s", atlasTask.getGuid())); + } + } else { + LOG.warn("Null vertex while re-queuing tasks at index {}", fetched); + } + + fetched++; + } + } + + totalFetched += fetched; + from += size; + if (fetched < size || totalFetched >= queueSize) { + break; + } + } catch (Exception e){ + break; + } + } + + return ret; + } + public void commit() { this.graph.commit(); } - public AtlasTask createVertex(String taskType, String createdBy, Map parameters) { - AtlasTask ret = new AtlasTask(taskType, createdBy, parameters); + public AtlasTask createVertex(String taskType, String createdBy, Map parameters, String classificationId, String entityGuid) { + AtlasTask ret = new AtlasTask(taskType, createdBy, parameters, classificationId, entityGuid); createVertex(ret); @@ -178,13 +436,28 @@ private void deleteVertex(AtlasVertex taskVertex) { graph.removeVertex(taskVertex); } - private AtlasTask toAtlasTask(AtlasVertex v) { + public static AtlasTask toAtlasTask(AtlasVertex v) { AtlasTask ret = new AtlasTask(); - ret.setGuid(v.getProperty(Constants.TASK_GUID, String.class)); - ret.setType(v.getProperty(Constants.TASK_TYPE, String.class)); - ret.setStatus(v.getProperty(Constants.TASK_STATUS, String.class)); - ret.setCreatedBy(v.getProperty(Constants.TASK_CREATED_BY, String.class)); + String guid = v.getProperty(Constants.TASK_GUID, String.class); + if (guid != null) { + ret.setGuid(guid); + } + + String type = v.getProperty(Constants.TASK_TYPE, String.class); + if (type != null) { + ret.setType(type); + } + + String status = v.getProperty(Constants.TASK_STATUS, String.class); + if (status != null) { + ret.setStatus(status); + } + + String createdBy = v.getProperty(Constants.TASK_CREATED_BY, String.class); + if (createdBy != null) { + ret.setCreatedBy(createdBy); + } Long createdTime = v.getProperty(Constants.TASK_CREATED_TIME, Long.class); if (createdTime != null) { @@ -204,40 +477,50 @@ private AtlasTask toAtlasTask(AtlasVertex v) { Long endTime = v.getProperty(Constants.TASK_END_TIME, Long.class); if (endTime != null) { ret.setEndTime(new Date(endTime)); + + Long timeTaken = v.getProperty(Constants.TASK_TIME_TAKEN_IN_SECONDS, Long.class); + if (timeTaken != null) { + ret.setTimeTakenInSeconds(timeTaken); + } } String parametersJson = v.getProperty(Constants.TASK_PARAMETERS, String.class); - ret.setParameters(AtlasType.fromJson(parametersJson, Map.class)); - - ret.setAttemptCount(v.getProperty(Constants.TASK_ATTEMPT_COUNT, Integer.class)); - ret.setErrorMessage(v.getProperty(Constants.TASK_ERROR_MESSAGE, String.class)); - - return ret; - } + if (parametersJson != null) { + ret.setParameters(AtlasType.fromJson(parametersJson, Map.class)); + } - private AtlasVertex createVertex(AtlasTask task) { - AtlasVertex ret = graph.addVertex(); + String classificationId = v.getProperty(Constants.TASK_CLASSIFICATION_ID, String.class); + if (classificationId != null) { + ret.setClassificationId(classificationId); + } - setEncodedProperty(ret, Constants.TASK_GUID, task.getGuid()); - setEncodedProperty(ret, Constants.TASK_TYPE_PROPERTY_KEY, Constants.TASK_TYPE_NAME); - setEncodedProperty(ret, Constants.TASK_STATUS, task.getStatus().toString()); - setEncodedProperty(ret, Constants.TASK_TYPE, task.getType()); - setEncodedProperty(ret, Constants.TASK_CREATED_BY, task.getCreatedBy()); - setEncodedProperty(ret, Constants.TASK_CREATED_TIME, task.getCreatedTime()); - setEncodedProperty(ret, Constants.TASK_UPDATED_TIME, task.getUpdatedTime()); + String entityGuid = v.getProperty(Constants.TASK_ENTITY_GUID, String.class); + if(entityGuid != null) { + ret.setEntityGuid(entityGuid); + } - if (task.getStartTime() != null) { - setEncodedProperty(ret, Constants.TASK_START_TIME, task.getStartTime().getTime()); + Integer attemptCount = v.getProperty(Constants.TASK_ATTEMPT_COUNT, Integer.class); + if (attemptCount != null) { + ret.setAttemptCount(attemptCount); } - if (task.getEndTime() != null) { - setEncodedProperty(ret, Constants.TASK_END_TIME, task.getEndTime().getTime()); + String errorMessage = v.getProperty(Constants.TASK_ERROR_MESSAGE, String.class); + if (errorMessage != null) { + ret.setErrorMessage(errorMessage); } - setEncodedProperty(ret, Constants.TASK_PARAMETERS, AtlasJson.toJson(task.getParameters())); - setEncodedProperty(ret, Constants.TASK_ATTEMPT_COUNT, task.getAttemptCount()); - setEncodedProperty(ret, Constants.TASK_ERROR_MESSAGE, task.getErrorMessage()); return ret; } + + private AtlasVertex createVertex(AtlasTask task) { + return taskService.createTaskVertex(task); + } + + private Map mapOf(String key, Object value) { + Map map = new HashMap<>(); + map.put(key, value); + + return map; + } } \ No newline at end of file diff --git a/repository/src/main/java/org/apache/atlas/tasks/TaskService.java b/repository/src/main/java/org/apache/atlas/tasks/TaskService.java new file mode 100644 index 00000000000..0234605fc3a --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/tasks/TaskService.java @@ -0,0 +1,92 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.tasks; + + +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.tasks.AtlasTask; +import org.apache.atlas.model.tasks.TaskSearchParams; +import org.apache.atlas.model.tasks.TaskSearchResult; +import org.apache.atlas.repository.graphdb.AtlasVertex; + +import java.util.List; +import java.util.Map; + +public interface TaskService { + + /** + * Search for direct ES query to fetch Tasks + * @param searchParams Search criteria + * @return Matching tasks + * @throws AtlasBaseException + */ + TaskSearchResult getTasks(TaskSearchParams searchParams) throws AtlasBaseException; + + /** + * + * @param from + * @param size + * @param mustConditions + * @param shouldConditions + * @param mustNotConditions + * @return + * @throws AtlasBaseException + */ + TaskSearchResult getTasksByCondition(int from, int size, List> mustConditions, List> shouldConditions, + List> mustNotConditions) throws AtlasBaseException; + /** + * Retry the task by changing its status to PENDING and increment attempt count + * @param taskGuid Guid of the task + * @throws AtlasBaseException + */ + void retryTask(String taskGuid) throws AtlasBaseException; + + /** + * Create a new task + * @param tasks to create + * @return Created task + * @throws AtlasBaseException + */ + List createAtlasTasks(List tasks) throws AtlasBaseException; + + List deleteAtlasTasks(List tasks); + + /** + * Update a task + * @param task to create + * @return created task + * @throws AtlasBaseException + */ + AtlasVertex createTaskVertex(AtlasTask task); + + /** + * Delete a task + * @param guid task to delete + * @throws AtlasBaseException + */ + void hardDelete(String guid) throws AtlasBaseException; + + /** + * Delete a task + * @param guid task to soft delete + * @throws AtlasBaseException + */ + void softDelete(String guid) throws AtlasBaseException; + +} diff --git a/repository/src/main/java/org/apache/atlas/transformer/PreProcessorPoliciesTransformer.java b/repository/src/main/java/org/apache/atlas/transformer/PreProcessorPoliciesTransformer.java new file mode 100644 index 00000000000..5893c48cc81 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/transformer/PreProcessorPoliciesTransformer.java @@ -0,0 +1,132 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.transformer; + +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo; +import org.apache.atlas.repository.store.graph.v2.preprocessor.ConnectionPreProcessor; +import org.apache.atlas.type.AtlasType; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.apache.atlas.AtlasErrorCode.RESOURCE_NOT_FOUND; +import static org.apache.atlas.repository.Constants.NAME; +import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; +import static org.apache.atlas.repository.Constants.getStaticFileAsString; +import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_RESOURCES; +import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_ROLES; +import static org.apache.atlas.repository.util.AccessControlUtils.getPolicyRoles; +import static org.apache.atlas.repository.util.AtlasEntityUtils.getListAttribute; +import static org.apache.atlas.repository.util.AtlasEntityUtils.getName; +import static org.apache.atlas.repository.util.AtlasEntityUtils.getQualifiedName; + +public final class PreProcessorPoliciesTransformer { + private static final Logger LOG = LoggerFactory.getLogger(ConnectionPreProcessor.class); + + static final String PLACEHOLDER_ENTITY = "{entity}"; + static final String PLACEHOLDER_GUID = "{guid}"; + static final String PLACEHOLDER_NAME = "{name}"; + + public PreProcessorPoliciesTransformer() { + + } + + public AtlasEntitiesWithExtInfo transform(AtlasEntity entity) throws AtlasBaseException { + LOG.info("transforming preprocessor bootstrap policies"); + String qualifiedName = getQualifiedName(entity); + String guid = entity.getGuid(); + String name = getName(entity); + + AtlasEntitiesWithExtInfo policiesExtInfo = new AtlasEntitiesWithExtInfo(); + + for (AtlasEntity templatePolicy : TemplateHelper.getTemplate(entity.getTypeName())) { + AtlasEntity policy = new AtlasEntity(templatePolicy); + String bootPolicyName = getName(policy); + String bootPolicyQn = getQualifiedName(policy); + + policy.setAttribute(NAME, bootPolicyName.replace(PLACEHOLDER_NAME, name)); + policy.setAttribute(QUALIFIED_NAME, bootPolicyQn.replace(PLACEHOLDER_GUID, guid)); + + List policyRoles = getPolicyRoles(policy); + List finalRoles = new ArrayList<>(); + for (String role : policyRoles) { + finalRoles.add(role.replace(PLACEHOLDER_GUID, guid)); + } + policy.setAttribute(ATTR_POLICY_ROLES, finalRoles); + + List resources = getListAttribute(policy, ATTR_POLICY_RESOURCES); + List resourcesFinal = new ArrayList<>(); + for (String resource : resources) { + if (resource.contains(PLACEHOLDER_ENTITY)) { + resource = resource.replace(PLACEHOLDER_ENTITY, qualifiedName); + } + + resourcesFinal.add(resource); + } + policy.setAttribute(ATTR_POLICY_RESOURCES, resourcesFinal); + + policiesExtInfo.addEntity(policy); + } + LOG.info("transformed preprocessor bootstrap policies"); + + return policiesExtInfo; + } + + static class TemplateHelper { + static final String TEMPLATE_FILE_NAME_PATTERN = "templates/%s_bootstrap_policies.json"; + + private static Map> templates = new HashMap<>(); + + public static List getTemplate(String entityTypeName) throws AtlasBaseException { + if (StringUtils.isEmpty(entityTypeName)) { + throw new AtlasBaseException("Please provide entityTypeName to fetch template"); + } + + if (MapUtils.isEmpty(templates) || !templates.containsKey(entityTypeName)) { + templates.put(entityTypeName, loadTemplate(entityTypeName)); + } + + return templates.get(entityTypeName); + } + + private static List loadTemplate(String entityTypeName) throws AtlasBaseException { + String jsonTemplate = null; + String templateName = String.format(TEMPLATE_FILE_NAME_PATTERN, entityTypeName.toLowerCase()); + + try { + jsonTemplate = getStaticFileAsString(templateName); + } catch (IOException e) { + LOG.error("Failed to load template for policies: {}", templateName); + throw new AtlasBaseException(RESOURCE_NOT_FOUND, "Template file " + templateName); + } + + AtlasEntity[] entities = AtlasType.fromJson(jsonTemplate, AtlasEntity[].class); + return Arrays.asList(entities); + } + } +} diff --git a/repository/src/main/java/org/apache/atlas/util/NanoIdUtils.java b/repository/src/main/java/org/apache/atlas/util/NanoIdUtils.java index 2be07c0be7f..773671b87f3 100644 --- a/repository/src/main/java/org/apache/atlas/util/NanoIdUtils.java +++ b/repository/src/main/java/org/apache/atlas/util/NanoIdUtils.java @@ -77,6 +77,10 @@ public static String randomNanoId() { return randomNanoId(DEFAULT_NUMBER_GENERATOR, DEFAULT_ALPHABET, DEFAULT_SIZE); } + public static String randomNanoId(int size) { + return randomNanoId(DEFAULT_NUMBER_GENERATOR, DEFAULT_ALPHABET, size); + } + /** * Static factory to retrieve a NanoId String. * diff --git a/repository/src/test/java/org/apache/atlas/TestModules.java b/repository/src/test/java/org/apache/atlas/TestModules.java index 8dda208256e..aa3e5e01ba1 100644 --- a/repository/src/test/java/org/apache/atlas/TestModules.java +++ b/repository/src/test/java/org/apache/atlas/TestModules.java @@ -189,7 +189,7 @@ protected void configure() { bind(TaskManagement.class).asEagerSingleton(); bind(ClassificationPropagateTaskFactory.class).asEagerSingleton(); - final GraphTransactionInterceptor graphTransactionInterceptor = new GraphTransactionInterceptor(new AtlasGraphProvider().get(), null); + final GraphTransactionInterceptor graphTransactionInterceptor = new GraphTransactionInterceptor(new AtlasGraphProvider().get()); requestInjection(graphTransactionInterceptor); bindInterceptor(Matchers.any(), Matchers.annotatedWith(GraphTransaction.class), graphTransactionInterceptor); } diff --git a/repository/src/test/java/org/apache/atlas/discovery/ShouldTerminateTestCases.java b/repository/src/test/java/org/apache/atlas/discovery/ShouldTerminateTestCases.java new file mode 100644 index 00000000000..7629bf8eda3 --- /dev/null +++ b/repository/src/test/java/org/apache/atlas/discovery/ShouldTerminateTestCases.java @@ -0,0 +1,248 @@ +package org.apache.atlas.discovery; + +import org.apache.atlas.model.instance.AtlasEntityHeader; +import org.apache.atlas.model.lineage.AtlasLineageInfo; +import org.apache.atlas.repository.graphdb.AtlasEdge; +import org.testng.annotations.Test; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; + +public class ShouldTerminateTestCases { + + private final EntityLineageService entityLineageService = new EntityLineageService(); + + @Test + public void when_it_is_input_and_there_are_more_vertices_it_should_return_false() { + AtlasLineageInfo lineageInfo = new AtlasLineageInfo(); + lineageInfo.setGuidEntityMap(new HashMap<>()); + AtlasLineageContext context = new AtlasLineageContext(); + context.setLimit(3); + context.setDirection(AtlasLineageInfo.LineageDirection.INPUT); + List currentVertexEdges = fillEdgedWithNull(6000); + List processEdges = fillEdgedWithNull(3); + + boolean shouldTerminate = entityLineageService.shouldTerminate(true, lineageInfo, context, currentVertexEdges, 0, 50, processEdges, 1); + + assertFalse(shouldTerminate); + assertFalse(lineageInfo.getHasMoreUpstreamVertices()); + assertFalse(lineageInfo.getHasMoreDownstreamVertices()); + + } + + @Test + public void when_it_is_input_and_there_are_not_any_vertices_it_should_return_true() { + AtlasLineageInfo lineageInfo = createFullLineageInfo(); + AtlasLineageContext context = new AtlasLineageContext(); + context.setLimit(3); + context.setDirection(AtlasLineageInfo.LineageDirection.INPUT); + List currentVertexEdges = fillEdgedWithNull(6000); + List processEdges = fillEdgedWithNull(3); + + boolean shouldTerminate = entityLineageService.shouldTerminate(true, lineageInfo, context, currentVertexEdges, 0, 50, processEdges, 1); + + assertTrue(shouldTerminate); + assertTrue(lineageInfo.getHasMoreUpstreamVertices()); + } + + @Test + public void when_it_should_terminate_and_there_are_no_more_vertices_has_more_upstream_value_should_be_false() { + AtlasLineageInfo lineageInfo = createFullLineageInfo(); + AtlasLineageContext context = new AtlasLineageContext(); + context.setLimit(3); + context.setDirection(AtlasLineageInfo.LineageDirection.INPUT); + List currentVertexEdges = fillEdgedWithNull(6000); + List processEdges = fillEdgedWithNull(3); + + boolean shouldTerminate = entityLineageService.shouldTerminate(true, lineageInfo, context, currentVertexEdges, 0, 5999, processEdges, 2); + + assertTrue(shouldTerminate); + assertFalse(lineageInfo.getHasMoreUpstreamVertices()); + assertFalse(lineageInfo.getHasMoreDownstreamVertices()); + } + + @Test + public void when_it_is_output_and_there_are_more_vertices_it_should_return_true() { + AtlasLineageInfo lineageInfo = new AtlasLineageInfo(); + lineageInfo.setGuidEntityMap(new HashMap<>()); + AtlasLineageContext context = new AtlasLineageContext(); + context.setLimit(3); + context.setDirection(AtlasLineageInfo.LineageDirection.OUTPUT); + List currentVertexEdges = fillEdgedWithNull(6000); + List processEdges = fillEdgedWithNull(3); + boolean shouldTerminate = entityLineageService.shouldTerminate(false, lineageInfo, context, currentVertexEdges, 0, 50, processEdges, 1); + assertFalse(shouldTerminate); + assertFalse(lineageInfo.getHasMoreUpstreamVertices()); + assertFalse(lineageInfo.getHasMoreDownstreamVertices()); + } + + @Test + public void when_it_is_output_and_there_are_not_any_more_vertices_it_should_return_false() { + AtlasLineageInfo lineageInfo = createFullLineageInfo(); + AtlasLineageContext context = new AtlasLineageContext(); + context.setLimit(3); + context.setDirection(AtlasLineageInfo.LineageDirection.OUTPUT); + List currentVertexEdges = fillEdgedWithNull(6000); + List processEdges = fillEdgedWithNull(3); + boolean shouldTerminate = entityLineageService.shouldTerminate(false, lineageInfo, context, currentVertexEdges, 0, 50, processEdges, 1); + assertTrue(shouldTerminate); + assertTrue(lineageInfo.getHasMoreDownstreamVertices()); + } + + @Test + public void when_it_should_terminate_and_there_are_no_more_vertices_has_more_downstream_value_should_be_false() { + AtlasLineageInfo lineageInfo = createFullLineageInfo(); + AtlasLineageContext context = new AtlasLineageContext(); + context.setLimit(3); + context.setDirection(AtlasLineageInfo.LineageDirection.OUTPUT); + List currentVertexEdges = fillEdgedWithNull(6000); + List processEdges = fillEdgedWithNull(3); + boolean shouldTerminate = entityLineageService.shouldTerminate(false, lineageInfo, context, currentVertexEdges, 0, 5999, processEdges, 2); + assertTrue(shouldTerminate); + assertFalse(lineageInfo.getHasMoreDownstreamVertices()); + assertFalse(lineageInfo.getHasMoreUpstreamVertices()); + } + + @Test + public void when_direction_is_both_type_is_input_and_there_are_more_vertices_it_should_return_false() { + AtlasLineageInfo lineageInfo = new AtlasLineageInfo(); + lineageInfo.setGuidEntityMap(new HashMap<>()); + AtlasLineageContext context = new AtlasLineageContext(); + context.setLimit(3); + context.setDirection(AtlasLineageInfo.LineageDirection.BOTH); + List currentVertexEdges = fillEdgedWithNull(6000); + List processEdges = fillEdgedWithNull(3); + + boolean shouldTerminate = entityLineageService.shouldTerminate(true, lineageInfo, context, currentVertexEdges, 0, 50, processEdges, 1); + + assertFalse(shouldTerminate); + assertFalse(lineageInfo.getHasMoreUpstreamVertices()); + assertFalse(lineageInfo.getHasMoreDownstreamVertices()); + } + + @Test + public void when_direction_is_both_type_is_input_and_there_are_not_any_vertices_it_should_return_true() { + AtlasLineageInfo lineageInfo = createFullLineageInfo(); + AtlasLineageContext context = new AtlasLineageContext(); + context.setLimit(3); + context.setDirection(AtlasLineageInfo.LineageDirection.BOTH); + List currentVertexEdges = fillEdgedWithNull(6000); + List processEdges = fillEdgedWithNull(3); + + boolean shouldTerminate = entityLineageService.shouldTerminate(true, lineageInfo, context, currentVertexEdges, 0, 50, processEdges, 1); + + assertTrue(shouldTerminate); + assertTrue(lineageInfo.getHasMoreUpstreamVertices()); + assertFalse(lineageInfo.getHasMoreDownstreamVertices()); + } + + @Test + public void when_direction_is_both_type_is_input_and_it_should_terminate_and_there_are_no_vertices_left_has_more_upstream_value_should_be_false() { + AtlasLineageInfo lineageInfo = createFullLineageInfo(); + AtlasLineageContext context = new AtlasLineageContext(); + context.setLimit(3); + context.setDirection(AtlasLineageInfo.LineageDirection.BOTH); + List currentVertexEdges = fillEdgedWithNull(6000); + List processEdges = fillEdgedWithNull(3); + + boolean shouldTerminate = entityLineageService.shouldTerminate(true, lineageInfo, context, currentVertexEdges, 0, 5999, processEdges, 2); + + assertTrue(shouldTerminate); + assertFalse(lineageInfo.getHasMoreUpstreamVertices()); + assertFalse(lineageInfo.getHasMoreDownstreamVertices()); + } + + @Test + public void when_direction_is_both_type_is_output_and_there_are_more_vertices_it_should_return_false() { + AtlasLineageInfo lineageInfo = new AtlasLineageInfo(); + lineageInfo.setGuidEntityMap(new HashMap<>()); + AtlasLineageContext context = new AtlasLineageContext(); + context.setLimit(3); + context.setDirection(AtlasLineageInfo.LineageDirection.BOTH); + List currentVertexEdges = fillEdgedWithNull(6000); + List processEdges = fillEdgedWithNull(3); + + boolean shouldTerminate = entityLineageService.shouldTerminate(false, lineageInfo, context, currentVertexEdges, 0, 50, processEdges, 1); + + assertFalse(shouldTerminate); + assertFalse(lineageInfo.getHasMoreUpstreamVertices()); + assertFalse(lineageInfo.getHasMoreDownstreamVertices()); + } + + @Test + public void when_direction_is_both_type_is_output_and_there_are_not_any_vertices_it_should_return_true() { + AtlasLineageInfo lineageInfo = createFullLineageInfo(); + AtlasLineageContext context = new AtlasLineageContext(); + context.setLimit(3); + context.setDirection(AtlasLineageInfo.LineageDirection.BOTH); + List currentVertexEdges = fillEdgedWithNull(6000); + List processEdges = fillEdgedWithNull(3); + + boolean shouldTerminate = entityLineageService.shouldTerminate(false, lineageInfo, context, currentVertexEdges, 0, 50, processEdges, 1); + + assertTrue(shouldTerminate); + assertTrue(lineageInfo.getHasMoreDownstreamVertices()); + assertFalse(lineageInfo.getHasMoreUpstreamVertices()); + } + + @Test + public void when_direction_is_both_type_is_output_there_are_not_any_vertices_and_input_vertex_count_is_non_zero_it_should_return_true() { + AtlasLineageInfo lineageInfo = createFullLineageInfo(); + lineageInfo.getGuidEntityMap().put("key6", new AtlasEntityHeader("Table")); + lineageInfo.getGuidEntityMap().put("key7", new AtlasEntityHeader("Table")); + lineageInfo.getGuidEntityMap().put("key8", new AtlasEntityHeader("Table")); + AtlasLineageContext context = new AtlasLineageContext(); + context.setLimit(3); + context.setDirection(AtlasLineageInfo.LineageDirection.BOTH); + List currentVertexEdges = fillEdgedWithNull(6000); + List processEdges = fillEdgedWithNull(3); + + boolean shouldTerminate = entityLineageService.shouldTerminate(false, lineageInfo, context, currentVertexEdges, 3, 50, processEdges, 1); + + assertTrue(shouldTerminate); + assertTrue(lineageInfo.getHasMoreDownstreamVertices()); + assertFalse(lineageInfo.getHasMoreUpstreamVertices()); + } + + @Test + public void when_direction_is_both_type_is_outputs_it_should_terminate_and_there_are_no_vertices_left_has_more_downstream_value_should_be_false() { + AtlasLineageInfo lineageInfo = createFullLineageInfo(); + AtlasLineageContext context = new AtlasLineageContext(); + context.setLimit(3); + context.setDirection(AtlasLineageInfo.LineageDirection.BOTH); + List currentVertexEdges = fillEdgedWithNull(6000); + List processEdges = fillEdgedWithNull(3); + + boolean shouldTerminate = entityLineageService.shouldTerminate(false, lineageInfo, context, currentVertexEdges, 0, 5999, processEdges, 2); + + assertTrue(shouldTerminate); + assertFalse(lineageInfo.getHasMoreDownstreamVertices()); + assertFalse(lineageInfo.getHasMoreUpstreamVertices()); + } + + private AtlasLineageInfo createFullLineageInfo() { + AtlasLineageInfo lineageInfo = new AtlasLineageInfo(); + HashMap guidEntityMap = new HashMap<>(); + guidEntityMap.put("key1", new AtlasEntityHeader("Process")); + guidEntityMap.put("key2", new AtlasEntityHeader("Table")); + guidEntityMap.put("key3", new AtlasEntityHeader("Table")); + guidEntityMap.put("key4", new AtlasEntityHeader("Table")); + guidEntityMap.put("key5", new AtlasEntityHeader("Table")); + lineageInfo.setGuidEntityMap(guidEntityMap); + return lineageInfo; + } + + private List fillEdgedWithNull(int edgeCount) { + List edges = new ArrayList<>(); + + for (int i = 0; i < edgeCount; i++) { + edges.add(null); + } + return edges; + } + +} diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ExportSkipLineageTest.java b/repository/src/test/java/org/apache/atlas/repository/impexp/ExportSkipLineageTest.java index 163ce59f00f..de9fb0718a5 100644 --- a/repository/src/test/java/org/apache/atlas/repository/impexp/ExportSkipLineageTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ExportSkipLineageTest.java @@ -27,11 +27,10 @@ import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.repository.AtlasTestBase; import org.apache.atlas.repository.graphdb.AtlasGraph; +import org.apache.atlas.repository.store.graph.AtlasRelationshipStore; import org.apache.atlas.repository.store.graph.v1.DeleteHandlerDelegate; import org.apache.atlas.repository.store.graph.v1.RestoreHandlerV1; -import org.apache.atlas.repository.store.graph.v2.AtlasEntityChangeNotifier; -import org.apache.atlas.repository.store.graph.v2.AtlasEntityStoreV2; -import org.apache.atlas.repository.store.graph.v2.EntityGraphMapper; +import org.apache.atlas.repository.store.graph.v2.*; import org.apache.atlas.store.AtlasTypeDefStore; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.utils.TestResourceFileUtils; @@ -68,6 +67,9 @@ public class ExportSkipLineageTest extends AtlasTestBase { @Inject ExportService exportService; + @Inject + private AtlasRelationshipStore atlasRelationshipStore; + @Inject AtlasGraph atlasGraph; @@ -82,7 +84,7 @@ public void setup() throws IOException, AtlasBaseException { loadHiveModel(typeDefStore, typeRegistry); RequestContext.get().setImportInProgress(true); - entityStore = new AtlasEntityStoreV2(atlasGraph, deleteDelegate, restoreHandlerV1, typeRegistry, mockChangeNotifier, graphMapper); + entityStore = new AtlasEntityStoreV2(atlasGraph, deleteDelegate, restoreHandlerV1, typeRegistry, mockChangeNotifier, graphMapper, null, atlasRelationshipStore, null); createEntities(entityStore, ENTITIES_SUB_DIR, new String[]{"db", "table-columns", "table-view", "table-table-lineage"}); final String[] entityGuids = {DB_GUID, TABLE_GUID, TABLE_TABLE_GUID, TABLE_VIEW_GUID}; verifyCreatedEntities(entityStore, entityGuids, 4); diff --git a/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2Test.java b/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2Test.java index ebbd672f683..cda9d5658ba 100644 --- a/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2Test.java +++ b/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2Test.java @@ -128,7 +128,7 @@ public void setUp() throws Exception { } @BeforeTest public void init() throws Exception { - entityStore = new AtlasEntityStoreV2(graph, deleteDelegate, restoreHandlerV1, typeRegistry, mockChangeNotifier, graphMapper); + entityStore = new AtlasEntityStoreV2(graph, deleteDelegate, restoreHandlerV1, typeRegistry, mockChangeNotifier, graphMapper, null, atlasRelationshipStore, null); RequestContext.clear(); RequestContext.get().setUser(TestUtilsV2.TEST_USER, null); diff --git a/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityTestBase.java b/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityTestBase.java index 08aba7f18c7..60973b308bc 100644 --- a/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityTestBase.java +++ b/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityTestBase.java @@ -39,6 +39,7 @@ import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer; import org.apache.atlas.repository.store.graph.AtlasEntityStore; +import org.apache.atlas.repository.store.graph.AtlasRelationshipStore; import org.apache.atlas.repository.store.graph.v1.DeleteHandlerDelegate; import org.apache.atlas.repository.store.graph.v1.RestoreHandlerV1; import org.apache.atlas.runner.LocalSolrRunner; @@ -90,6 +91,9 @@ public class AtlasEntityTestBase extends AtlasTestBase { @Inject protected AtlasGraph graph; + @Inject + protected AtlasRelationshipStore atlasRelationshipStore; + AtlasEntityChangeNotifier mockChangeNotifier = mock(AtlasEntityChangeNotifier.class); @BeforeClass @@ -111,7 +115,7 @@ public void clear() throws Exception { @BeforeTest public void init() throws Exception { - entityStore = new AtlasEntityStoreV2(graph, deleteDelegate, restoreHandlerV1, typeRegistry, mockChangeNotifier, graphMapper); + entityStore = new AtlasEntityStoreV2(graph, deleteDelegate, restoreHandlerV1, typeRegistry, mockChangeNotifier, graphMapper, null, atlasRelationshipStore, null); RequestContext.clear(); RequestContext.get().setUser(TestUtilsV2.TEST_USER, null); diff --git a/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2Test.java b/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2Test.java index 5d47a8569f5..796da31b65b 100644 --- a/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2Test.java +++ b/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2Test.java @@ -39,7 +39,6 @@ import org.apache.atlas.repository.store.graph.AtlasRelationshipStore; import org.apache.atlas.repository.store.graph.v1.DeleteHandlerDelegate; import org.apache.atlas.repository.store.graph.v1.RestoreHandlerV1; -import org.apache.atlas.runner.LocalSolrRunner; import org.apache.atlas.store.AtlasTypeDefStore; import org.apache.atlas.DeleteType; import org.apache.atlas.type.AtlasEntityType; @@ -135,8 +134,8 @@ public void setUp() throws Exception { @BeforeTest public void init() throws Exception { - entityStore = new AtlasEntityStoreV2(atlasGraph, deleteDelegate, restoreHandlerV1, typeRegistry, mockChangeNotifier, graphMapper); relationshipStore = new AtlasRelationshipStoreV2(atlasGraph, typeRegistry, deleteDelegate, entityNotifier); + entityStore = new AtlasEntityStoreV2(atlasGraph, deleteDelegate, restoreHandlerV1, typeRegistry, mockChangeNotifier, graphMapper, null, relationshipStore, null); RequestContext.clear(); RequestContext.get().setUser(TestUtilsV2.TEST_USER, null); diff --git a/repository/src/test/java/org/apache/atlas/repository/tagpropagation/ClassificationPropagationWithTasksTest.java b/repository/src/test/java/org/apache/atlas/repository/tagpropagation/ClassificationPropagationWithTasksTest.java index d440f2f2dd9..94b344bbc4a 100644 --- a/repository/src/test/java/org/apache/atlas/repository/tagpropagation/ClassificationPropagationWithTasksTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/tagpropagation/ClassificationPropagationWithTasksTest.java @@ -17,6 +17,7 @@ */ package org.apache.atlas.repository.tagpropagation; +import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.RequestContext; import org.apache.atlas.TestModules; import org.apache.atlas.exception.AtlasBaseException; @@ -54,6 +55,7 @@ import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; +import static org.testng.AssertJUnit.assertEquals; @Guice(modules = TestModules.TestOnlyModule.class) public class ClassificationPropagationWithTasksTest extends AtlasTestBase { @@ -120,22 +122,22 @@ public static InputStream getZipSource(String fileName) throws IOException { @Test public void parameterValidation() throws AtlasBaseException { try { - entityGraphMapper.propagateClassification(null, null, null); - entityGraphMapper.propagateClassification("unknown", "abcd", "xyz"); + entityGraphMapper.propagateClassification(null, null, null, null); + entityGraphMapper.propagateClassification("unknown", "abcd", "xyz", null); } catch (AtlasBaseException e) { assertNotNull(e.getCause()); assertTrue(e.getCause() instanceof EntityNotFoundException); } - List ret = entityGraphMapper.propagateClassification(HDFS_PATH_EMPLOYEES, StringUtils.EMPTY, StringUtils.EMPTY); + List ret = entityGraphMapper.propagateClassification(HDFS_PATH_EMPLOYEES, StringUtils.EMPTY, StringUtils.EMPTY, null); assertNull(ret); ret = entityGraphMapper.deleteClassificationPropagation(StringUtils.EMPTY, StringUtils.EMPTY); assertNull(ret); AtlasEntity hdfs_employees = getEntity(HDFS_PATH_EMPLOYEES); - ret = entityGraphMapper.propagateClassification(hdfs_employees.getGuid(), StringUtils.EMPTY, StringUtils.EMPTY); + ret = entityGraphMapper.propagateClassification(hdfs_employees.getGuid(), StringUtils.EMPTY, StringUtils.EMPTY, null); assertNull(ret); } @@ -165,7 +167,7 @@ public void add() throws AtlasBaseException { AtlasEntity entityUpdated = getEntity(HDFS_PATH_EMPLOYEES); assertNotNull(entityUpdated.getPendingTasks()); - List impactedEntities = entityGraphMapper.propagateClassification(hdfs_employees.getGuid(), classificationVertex.getId().toString(), StringUtils.EMPTY); + List impactedEntities = entityGraphMapper.propagateClassification(hdfs_employees.getGuid(), classificationVertex.getId().toString(), StringUtils.EMPTY, null); assertNotNull(impactedEntities); } @@ -195,7 +197,7 @@ public void delete() throws AtlasBaseException { final String TAG_NAME = "tagX"; AtlasEntity hdfs_employees = getEntity(HDFS_PATH_EMPLOYEES); - entityGraphMapper.propagateClassification(hdfs_employees.getGuid(), StringUtils.EMPTY, StringUtils.EMPTY); + entityGraphMapper.propagateClassification(hdfs_employees.getGuid(), StringUtils.EMPTY, StringUtils.EMPTY, null); AtlasClassification tagX = new AtlasClassification(TAG_NAME); tagX.setEntityGuid(hdfs_employees.getGuid()); @@ -204,7 +206,11 @@ public void delete() throws AtlasBaseException { AtlasVertex entityVertex = AtlasGraphUtilsV2.findByGuid(hdfs_employees.getGuid()); AtlasVertex classificationVertex = GraphHelper.getClassificationVertex(entityVertex, TAG_NAME); - entityStore.deleteClassification(HDFS_PATH_EMPLOYEES, tagX.getTypeName()); + try { + entityStore.deleteClassification(HDFS_PATH_EMPLOYEES, tagX.getTypeName()); + } catch (AtlasBaseException e) { + assertEquals(e.getAtlasErrorCode(), AtlasErrorCode.DELETE_TAG_PROPAGATION_NOT_ALLOWED); + } assertNotNull(entityVertex); assertNotNull(classificationVertex); diff --git a/repository/src/test/java/org/apache/atlas/tasks/BaseTaskFixture.java b/repository/src/test/java/org/apache/atlas/tasks/BaseTaskFixture.java index 51e9bf7a725..935c22a43af 100644 --- a/repository/src/test/java/org/apache/atlas/tasks/BaseTaskFixture.java +++ b/repository/src/test/java/org/apache/atlas/tasks/BaseTaskFixture.java @@ -111,6 +111,6 @@ public SpyErrorThrowingTask getErrorTask() { } protected AtlasTask createTask(TaskManagement taskManagement, String type) { - return taskManagement.createTask(type, "testUser", Collections.singletonMap("params", "params")); + return taskManagement.createTask(type, "testUser", Collections.singletonMap("params", "params"),"classificationId", "id-ref"); } } \ No newline at end of file diff --git a/repository/src/test/java/org/apache/atlas/tasks/TaskExecutorTest.java b/repository/src/test/java/org/apache/atlas/tasks/TaskExecutorTest.java index f8d131cc741..979c4236a8e 100644 --- a/repository/src/test/java/org/apache/atlas/tasks/TaskExecutorTest.java +++ b/repository/src/test/java/org/apache/atlas/tasks/TaskExecutorTest.java @@ -17,6 +17,7 @@ */ package org.apache.atlas.tasks; +import org.apache.atlas.AtlasConfiguration; import org.apache.atlas.TestModules; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.tasks.AtlasTask; @@ -27,10 +28,8 @@ import org.testng.Assert; import javax.inject.Inject; -import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; -import java.util.List; import java.util.Map; @Guice(modules = TestModules.TestOnlyModule.class) @@ -44,6 +43,10 @@ public class TaskExecutorTest extends BaseTaskFixture { @Inject private TaskManagement taskManagement; + private long pollingInterval = AtlasConfiguration.TASKS_REQUEUE_POLL_INTERVAL.getLong(); + + private final String defaultZkRoot = "/apache-atlas"; + @Test public void noTasksExecuted() { TaskManagementTest.SpyingFactory spyingFactory = new TaskManagementTest.SpyingFactory(); @@ -51,7 +54,7 @@ public void noTasksExecuted() { TaskManagement.createTaskTypeFactoryMap(new HashMap<>(), spyingFactory); TaskManagement.Statistics statistics = new TaskManagement.Statistics(); - new TaskExecutor(taskRegistry, taskFactoryMap, statistics); + new TaskExecutor(taskRegistry, taskFactoryMap, statistics, null, null, defaultZkRoot, false); Assert.assertEquals(statistics.getTotal(), 0); } @@ -63,11 +66,11 @@ public void tasksNotPersistedIsNotExecuted() throws InterruptedException { TaskManagement.createTaskTypeFactoryMap(taskFactoryMap, spyingFactory); TaskManagement.Statistics statistics = new TaskManagement.Statistics(); - TaskExecutor taskExecutor = new TaskExecutor(taskRegistry, taskFactoryMap, statistics); + TaskExecutor taskExecutor = new TaskExecutor(taskRegistry, taskFactoryMap, statistics, null, null, defaultZkRoot,false); - taskExecutor.addAll(Collections.singletonList(new AtlasTask(SPYING_TASK_ADD, "test", Collections.emptyMap()))); + taskManagement.createTask(SPYING_TASK_ADD, "test", Collections.emptyMap(), "testId", "testGuid"); - taskExecutor.waitUntilDone(); + Thread.sleep(pollingInterval + 5000); Assert.assertEquals(statistics.getTotal(), 0); } @@ -78,20 +81,15 @@ public void persistedIsExecuted() throws AtlasBaseException, InterruptedExceptio Map taskFactoryMap = new HashMap<>(); TaskManagement.createTaskTypeFactoryMap(taskFactoryMap, spyingFactory); - AtlasTask addTask = taskManagement.createTask("add", "test", Collections.emptyMap()); - AtlasTask errorThrowingTask = taskManagement.createTask("errorThrowingTask", "test", Collections.emptyMap()); + AtlasTask addTask = taskManagement.createTask("add", "test", Collections.emptyMap(),"testId", "testGuid"); + AtlasTask errorThrowingTask = taskManagement.createTask("errorThrowingTask", "test", Collections.emptyMap(),"testId", "testGuid"); TaskManagement.Statistics statistics = new TaskManagement.Statistics(); - List tasks = new ArrayList() {{ - add(addTask); - add(errorThrowingTask); - }}; graph.commit(); - TaskExecutor taskExecutor = new TaskExecutor(taskRegistry, taskFactoryMap, statistics); - taskExecutor.addAll(tasks); + TaskExecutor taskExecutor = new TaskExecutor(taskRegistry, taskFactoryMap, statistics, null, null, defaultZkRoot,false); - taskExecutor.waitUntilDone(); + Thread.sleep(pollingInterval + 5000); Assert.assertEquals(statistics.getTotal(), 2); Assert.assertEquals(statistics.getTotalSuccess(), 1); Assert.assertEquals(statistics.getTotalError(), 1); @@ -113,10 +111,10 @@ private void assertTaskUntilFail(AtlasTask errorThrowingTask, TaskExecutor taskE Assert.assertEquals(errorTaskFromDB.getStatus(), AtlasTask.Status.PENDING); for (int i = errorTaskFromDB.getAttemptCount(); i <= AtlasTask.MAX_ATTEMPT_COUNT; i++) { - taskExecutor.addAll(Collections.singletonList(errorThrowingTask)); + taskManagement.createTask(errorThrowingTask.getType(), errorThrowingTask.getCreatedBy(), errorThrowingTask.getParameters(), "testId", "testGuid"); } - taskExecutor.waitUntilDone(); + Thread.sleep(pollingInterval + 5000); graph.commit(); Assert.assertEquals(errorThrowingTask.getStatus(), AtlasTask.Status.FAILED); } diff --git a/repository/src/test/java/org/apache/atlas/tasks/TaskManagementTest.java b/repository/src/test/java/org/apache/atlas/tasks/TaskManagementTest.java index 6f0b89c6e84..68a88c06300 100644 --- a/repository/src/test/java/org/apache/atlas/tasks/TaskManagementTest.java +++ b/repository/src/test/java/org/apache/atlas/tasks/TaskManagementTest.java @@ -17,6 +17,7 @@ */ package org.apache.atlas.tasks; +import org.apache.atlas.AtlasConfiguration; import org.apache.atlas.AtlasException; import org.apache.atlas.TestModules; import org.apache.atlas.exception.AtlasBaseException; @@ -25,13 +26,13 @@ import org.testng.annotations.Guice; import org.testng.annotations.Test; -import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.concurrent.TimeUnit; @Guice(modules = TestModules.TestOnlyModule.class) public class TaskManagementTest extends BaseTaskFixture { + long pollingInterval = AtlasConfiguration.TASKS_REQUEUE_POLL_INTERVAL.getLong(); private static class NullFactory implements TaskFactory { @Override @@ -47,23 +48,21 @@ public List getSupportedTypes() { @Test public void factoryReturningNullIsHandled() throws AtlasException { - TaskManagement taskManagement = new TaskManagement(null, taskRegistry, new NullFactory()); + TaskManagement taskManagement = new TaskManagement(null, taskRegistry, new NullFactory(),null, null); taskManagement.start(); } @Test public void taskSucceedsTaskVertexRemoved() throws AtlasException, InterruptedException, AtlasBaseException { SpyingFactory spyingFactory = new SpyingFactory(); - TaskManagement taskManagement = new TaskManagement(null, taskRegistry, spyingFactory); + TaskManagement taskManagement = new TaskManagement(null, taskRegistry, spyingFactory,null, null); taskManagement.start(); AtlasTask spyTask = createTask(taskManagement, SPYING_TASK_ADD); AtlasTask spyTaskError = createTask(taskManagement, SPYING_TASK_ERROR_THROWING); graph.commit(); - taskManagement.addAll(Arrays.asList(spyTask, spyTaskError)); - - TimeUnit.SECONDS.sleep(5); + TimeUnit.MILLISECONDS.sleep(pollingInterval + 5000); Assert.assertTrue(spyingFactory.getAddTask().taskPerformed()); Assert.assertTrue(spyingFactory.getErrorTask().taskPerformed()); @@ -75,22 +74,19 @@ public void taskSucceedsTaskVertexRemoved() throws AtlasException, InterruptedEx public void severalTaskAdds() throws AtlasException, InterruptedException { int MAX_THREADS = 5; - TaskManagement taskManagement = new TaskManagement(null, taskRegistry); + TaskManagement taskManagement = new TaskManagement(null, taskRegistry,null,null); taskManagement.start(); Thread[] threads = new Thread[MAX_THREADS]; for (int i = 0; i < MAX_THREADS; i++) { threads[i] = new Thread(() -> { try { - AtlasTask spyAdd = taskManagement.createTask(SPYING_TASK_ADD, "test", Collections.emptyMap()); - AtlasTask spyErr = taskManagement.createTask(SPYING_TASK_ERROR_THROWING, "test", Collections.emptyMap()); - - taskManagement.addAll(Collections.singletonList(spyAdd)); - taskManagement.addAll(Collections.singletonList(spyErr)); + AtlasTask spyAdd = taskManagement.createTask(SPYING_TASK_ADD, "test", Collections.emptyMap(), "testId", "testGuid"); + AtlasTask spyErr = taskManagement.createTask(SPYING_TASK_ERROR_THROWING, "test", Collections.emptyMap(), "testId", "testGuid"); Thread.sleep(10000); for (int j = 0; j <= AtlasTask.MAX_ATTEMPT_COUNT; j++) { - taskManagement.addAll(Collections.singletonList(spyErr)); + taskManagement.createTask(SPYING_TASK_ERROR_THROWING, "test", Collections.emptyMap(), "testId", "testGuid"); } } catch (InterruptedException e) { e.printStackTrace(); diff --git a/repository/src/test/java/org/apache/atlas/tasks/TaskRegistryTest.java b/repository/src/test/java/org/apache/atlas/tasks/TaskRegistryTest.java index 7f139dcb49e..7198d5e901c 100644 --- a/repository/src/test/java/org/apache/atlas/tasks/TaskRegistryTest.java +++ b/repository/src/test/java/org/apache/atlas/tasks/TaskRegistryTest.java @@ -41,7 +41,7 @@ public class TaskRegistryTest { @Test public void basic() throws AtlasException, AtlasBaseException { - AtlasTask task = new AtlasTask("abcd", "test", Collections.singletonMap("p1", "p1")); + AtlasTask task = new AtlasTask("abcd", "test", Collections.singletonMap("p1", "p1"), "testParam", "testRefId"); Assert.assertNull(registry.getById(task.getGuid())); AtlasTask taskFromVertex = registry.save(task); @@ -80,7 +80,7 @@ public void pendingTasks() throws AtlasBaseException { final String TASK_TYPE_FORMAT = "abcd:%d"; for (int i = 0; i < MAX_TASKS; i++) { - AtlasTask task = new AtlasTask(String.format(TASK_TYPE_FORMAT, i), "test", Collections.singletonMap("p1", "p1")); + AtlasTask task = new AtlasTask(String.format(TASK_TYPE_FORMAT, i), "test", Collections.singletonMap("p1", "p1"), "testParam", "testRefId"); registry.save(task); } diff --git a/sem-ver-bump.sh b/sem-ver-bump.sh new file mode 100644 index 00000000000..18468d815f1 --- /dev/null +++ b/sem-ver-bump.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +# Increment a version string using Semantic Versioning (SemVer) terminology. + +# Parse command line options. +:' +while getopts ":Mmp" Option +do + case $Option in + M ) major=true;; + m ) minor=true;; + p ) patch=true;; + esac +done + +shift $(($OPTIND - 1)) + +version=$1 + +# Build array from version string. + +a=( ${version//./ } ) + +# If version string is missing or has the wrong number of members, show usage message. + +if [ ${#a[@]} -ne 3 ] +then + echo "usage: $(basename $0) [-Mmp] major.minor.patch" + exit 1 +fi + +# Increment version numbers as requested. + +if [ ! -z $major ] +then + ((a[0]++)) + a[1]=0 + a[2]=0 +fi + +if [ ! -z $minor ] +then + ((a[1]++)) + a[2]=0 +fi + +if [ ! -z $patch ] +then + ((a[2]++)) +fi + +echo "${a[0]}.${a[1]}.${a[2]}" +' diff --git a/server-api/src/main/java/org/apache/atlas/RequestContext.java b/server-api/src/main/java/org/apache/atlas/RequestContext.java index a63cc14e096..053af6560a1 100644 --- a/server-api/src/main/java/org/apache/atlas/RequestContext.java +++ b/server-api/src/main/java/org/apache/atlas/RequestContext.java @@ -18,12 +18,10 @@ package org.apache.atlas; -import org.apache.atlas.model.instance.AtlasClassification; -import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.model.instance.*; import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo; -import org.apache.atlas.model.instance.AtlasEntityHeader; -import org.apache.atlas.model.instance.AtlasObjectId; import org.apache.atlas.model.tasks.AtlasTask; +import org.apache.atlas.service.metrics.MetricsRegistry; import org.apache.atlas.utils.AtlasPerfMetrics; import org.apache.atlas.utils.AtlasPerfMetrics.MetricRecorder; import org.apache.commons.collections.CollectionUtils; @@ -32,19 +30,14 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import static org.apache.atlas.model.instance.AtlasObjectId.KEY_GUID; public class RequestContext { private static final Logger METRICS = LoggerFactory.getLogger("METRICS"); + private static final Logger LOG = LoggerFactory.getLogger(RequestContext.class); private static final ThreadLocal CURRENT_CONTEXT = new ThreadLocal<>(); private static final Set ACTIVE_REQUESTS = new HashSet<>(); @@ -55,34 +48,57 @@ public class RequestContext { private final Map deletedEntities = new HashMap<>(); private final Map restoreEntities = new HashMap<>(); private final Map entityCache = new HashMap<>(); + private final Map entityHeaderCache = new HashMap<>(); private final Map entityExtInfoCache = new HashMap<>(); private final Map diffEntityCache = new HashMap<>(); private final Map> addedPropagations = new HashMap<>(); private final Map> removedPropagations = new HashMap<>(); - private final AtlasPerfMetrics metrics = isMetricsEnabled ? new AtlasPerfMetrics() : null; - private List entityGuidInRequest = null; - private final Set entitiesToSkipUpdate = new HashSet<>(); - private final Set onlyCAUpdateEntities = new HashSet<>(); - private final Set onlyBAUpdateEntities = new HashSet<>(); - private final List queuedTasks = new ArrayList<>(); + private final Map requestContextHeaders= new HashMap<>(); + private final Set deletedEdgesIds = new HashSet<>(); + private final Set processGuidIds = new HashSet<>(); + + private final AtlasPerfMetrics metrics = isMetricsEnabled ? new AtlasPerfMetrics() : null; + private List entityGuidInRequest = null; + private final Set entitiesToSkipUpdate = new HashSet<>(); + private final Set onlyCAUpdateEntities = new HashSet<>(); + private final Set onlyBAUpdateEntities = new HashSet<>(); + private final List queuedTasks = new ArrayList<>(); private final Set relationAttrsForSearch = new HashSet<>(); + private static String USERNAME = ""; + private final Map> removedElementsMap = new HashMap<>(); + private final Map> newElementsCreatedMap = new HashMap<>(); - private String user; - private Set userGroups; - private String clientIPAddress; + private final Map> relationshipMutationMap = new HashMap<>(); + + private String user; + private Set userGroups; + private String clientIPAddress; private List forwardedAddresses; - private DeleteType deleteType = DeleteType.DEFAULT; - private boolean isPurgeRequested = false; - private int maxAttempts = 1; - private int attemptCount = 1; - private boolean isImportInProgress = false; + private DeleteType deleteType = DeleteType.DEFAULT; + private boolean isPurgeRequested = false; + private int maxAttempts = 1; + private int attemptCount = 1; + private boolean isImportInProgress = false; private boolean isInNotificationProcessing = false; private boolean isInTypePatching = false; private boolean createShellEntityForNonExistingReference = false; private boolean skipFailedEntities = false; private boolean allowDeletedRelationsIndexsearch = false; +<<<<<<< HEAD +======= + private boolean includeMeanings = true; + private boolean includeClassifications = true; +>>>>>>> 7f3c811fb15361ba82bfb4edd7a8393798863ff0 private String currentTypePatchAction = ""; + private AtlasTask currentTask; + private String traceId; + private final Map relationshipEndToVertexIdMap = new HashMap<>(); + private boolean allowDuplicateDisplayName; + private MetricsRegistry metricsRegistry; + private boolean skipAuthorizationCheck = false; + private Set deletedEdgesIdsForResetHasLineage = new HashSet<>(0); + private String requestUri; private RequestContext() { } @@ -122,6 +138,7 @@ public void clearCache() { this.updatedEntities.clear(); this.deletedEntities.clear(); this.entityCache.clear(); + this.entityHeaderCache.clear(); this.entityExtInfoCache.clear(); this.diffEntityCache.clear(); this.addedPropagations.clear(); @@ -131,12 +148,25 @@ public void clearCache() { this.onlyBAUpdateEntities.clear(); this.relationAttrsForSearch.clear(); this.queuedTasks.clear(); + this.newElementsCreatedMap.clear(); + this.removedElementsMap.clear(); + this.deletedEdgesIds.clear(); + this.processGuidIds.clear(); + this.deletedEdgesIdsForResetHasLineage.clear(); + this.requestContextHeaders.clear(); + this.relationshipEndToVertexIdMap.clear(); + this.relationshipMutationMap.clear(); + this.currentTask = null; + this.skipAuthorizationCheck = false; if (metrics != null && !metrics.isEmpty()) { METRICS.debug(metrics.toString()); - + if (Objects.nonNull(this.metricsRegistry)){ + this.metricsRegistry.collect(traceId, this.requestUri, metrics); + } metrics.clear(); } + setTraceId(null); if (this.entityGuidInRequest != null) { this.entityGuidInRequest.clear(); @@ -153,6 +183,14 @@ public void setRelationAttrsForSearch(Set relationAttrsForSearch) { } } + public Map> getRemovedElementsMap() { + return removedElementsMap; + } + + public Map> getNewElementsCreatedMap() { + return newElementsCreatedMap; + } + public static String getCurrentUser() { RequestContext context = CURRENT_CONTEXT.get(); String ret = context != null ? context.getUser() : null; @@ -173,6 +211,16 @@ public static String getCurrentUser() { } public String getUser() { + if (isImportInProgress) { + if (StringUtils.isEmpty(USERNAME)){ + try { + USERNAME = ApplicationProperties.get().getString("atlas.migration.user.name", ""); + } catch (AtlasException e) { + LOG.error("Failed to find value for atlas.migration.user.name from properties"); + } + } + return USERNAME; + } return user; } @@ -265,6 +313,16 @@ public void setAllowDeletedRelationsIndexsearch(boolean allowDeletedRelationsInd this.allowDeletedRelationsIndexsearch = allowDeletedRelationsIndexsearch; } +<<<<<<< HEAD +======= + public void setAllowDuplicateDisplayName(boolean allowDuplicateDisplayName){ + this.allowDuplicateDisplayName = allowDuplicateDisplayName; + } + public boolean getAllowDuplicateDisplayName(){ + return allowDuplicateDisplayName; + } + +>>>>>>> 7f3c811fb15361ba82bfb4edd7a8393798863ff0 public String getCurrentTypePatchAction() { return currentTypePatchAction; } @@ -332,10 +390,51 @@ public void recordAddedPropagation(String guid, AtlasClassification classificati } } + public void addToDeletedEdgesIds(String edgeId) { + deletedEdgesIds.add(edgeId); + } + + public Set getDeletedEdgesIds() { + return deletedEdgesIds; + } + + public void addToDeletedEdgesIdsForResetHasLineage(String edgeId) { + deletedEdgesIdsForResetHasLineage.add(edgeId); + } + + public Set getDeletedEdgesIdsForResetHasLineage() { + return deletedEdgesIdsForResetHasLineage; + } + + public Set getProcessGuidIds() { + return processGuidIds; + } + + public void addProcessGuidIds(String guid) { + processGuidIds.add(guid); + } + + + public AtlasTask getCurrentTask() { + return currentTask; + } + + public void setCurrentTask(AtlasTask currentTask) { + this.currentTask = currentTask; + } + public static int getActiveRequestsCount() { return ACTIVE_REQUESTS.size(); } + public boolean isSkipAuthorizationCheck() { + return skipAuthorizationCheck; + } + + public void setSkipAuthorizationCheck(boolean skipAuthorizationCheck) { + this.skipAuthorizationCheck = skipAuthorizationCheck; + } + public static long earliestActiveRequestTime() { long ret = System.currentTimeMillis(); @@ -395,12 +494,27 @@ public void cacheDifferentialEntity(AtlasEntity entity) { } } + public void setEntityHeaderCache(AtlasEntityHeader headerCache){ + if(headerCache != null && headerCache.getGuid() != null){ + entityHeaderCache.put(headerCache.getGuid(), headerCache); + } + } + + public AtlasEntityHeader getCachedEntityHeader(String guid){ + if(guid == null){ + return null; + } + return entityHeaderCache.get(guid); + } + public AtlasEntity getDifferentialEntity(String guid) { return diffEntityCache.get(guid); } public Collection getDifferentialEntities() { return diffEntityCache.values(); } + public Map getDifferentialEntitiesMap() { return diffEntityCache; } + public Collection getUpdatedEntities() { return updatedEntities.values(); } @@ -416,7 +530,7 @@ public void clearRemovePropagations() { public void clearAddedPropagations() { addedPropagations.clear(); } - + public Collection getRestoredEntities() { return restoreEntities.values(); } @@ -452,6 +566,16 @@ public boolean isRestoredEntity(String guid) { return restoreEntities.containsKey(guid); } + public void addRequestContextHeader(String headerName, String headerValue) { + if (StringUtils.isNotEmpty(headerName)) { + requestContextHeaders.put(headerName, headerValue); + } + } + + public Map getRequestContextHeaders() { + return requestContextHeaders; + } + public MetricRecorder startMetricRecord(String name) { return metrics != null ? metrics.getMetricRecorder(name) : null; } public void endMetricRecord(MetricRecorder recorder) { @@ -496,6 +620,42 @@ public List getQueuedTasks() { return this.queuedTasks; } + public String getTraceId() { + return traceId; + } + + public void setTraceId(String traceId) { + this.traceId = traceId; + } + + public void setIncludeMeanings(boolean includeMeanings) { + this.includeMeanings = includeMeanings; + } + + public boolean includeMeanings() { + return this.includeMeanings; + } + + public void setIncludeClassifications(boolean includeClassifications) { + this.includeClassifications = includeClassifications; + } + + public boolean includeClassifications() { + return this.includeClassifications; + } + + public void setMetricRegistry(MetricsRegistry metricsRegistry) { + this.metricsRegistry = metricsRegistry; + } + + public void setUri(String uri) { + this.requestUri = uri; + } + + public String getRequestUri() { + return this.requestUri; + } + public class EntityGuidPair { private final Object entity; private final String guid; @@ -533,4 +693,26 @@ public List getForwardedAddresses() { public void setForwardedAddresses(List forwardedAddresses) { this.forwardedAddresses = forwardedAddresses; } + + public void addRelationshipEndToVertexIdMapping(AtlasObjectId atlasObjectId, Object vertexId) { + this.relationshipEndToVertexIdMap.put(atlasObjectId, vertexId); + } + + public Map getRelationshipEndToVertexIdMap() { + return this.relationshipEndToVertexIdMap; + } + + public void saveRelationshipsMutationContext(String event, AtlasRelationship relationship) { + Set deletedRelationships = this.relationshipMutationMap.getOrDefault(event, new HashSet<>()); + deletedRelationships.add(relationship); + this.relationshipMutationMap.put(event, deletedRelationships); + } + + public void clearMutationContext(String event) { + this.relationshipMutationMap.remove(event); + } + + public Map> getRelationshipMutationMap() { + return relationshipMutationMap; + } } \ No newline at end of file diff --git a/server-api/src/main/java/org/apache/atlas/listener/ActiveStateChangeHandler.java b/server-api/src/main/java/org/apache/atlas/listener/ActiveStateChangeHandler.java index 2916d6b46f4..8fe8afe35a7 100644 --- a/server-api/src/main/java/org/apache/atlas/listener/ActiveStateChangeHandler.java +++ b/server-api/src/main/java/org/apache/atlas/listener/ActiveStateChangeHandler.java @@ -34,7 +34,8 @@ public enum HandlerOrder { DEFAULT_METADATA_SERVICE(4), NOTIFICATION_HOOK_CONSUMER(5), TASK_MANAGEMENT(6), - INDEX_RECOVERY(7); + INDEX_RECOVERY(7), + AUTH_POLICIES_INITIALIZER(8); private final int order; diff --git a/tools/atlas-index-repair/src/main/java/org/apache/atlas/tools/RepairIndex.java b/tools/atlas-index-repair/src/main/java/org/apache/atlas/tools/RepairIndex.java index 37565188e62..1ee7248f1c8 100644 --- a/tools/atlas-index-repair/src/main/java/org/apache/atlas/tools/RepairIndex.java +++ b/tools/atlas-index-repair/src/main/java/org/apache/atlas/tools/RepairIndex.java @@ -119,7 +119,7 @@ private static CommandLine getCommandLine(String[] args) throws ParseException { return new DefaultParser().parse(options, args); } - private static void setupGraph() { + public static void setupGraph() { display("Initializing graph: "); graph = AtlasJanusGraphDatabase.getGraphInstance(); displayCrlf("Graph Initialized!"); @@ -247,4 +247,45 @@ private static AtlasClientV2 getAtlasClientV2(String[] atlasEndpoint, String[] u } return atlasClientV2; } + + private static Set getEntityAndReferenceGuids(String guid, Map referredEntities) throws Exception { + Set set = new HashSet<>(); + set.add(guid); + if (referredEntities == null || referredEntities.isEmpty()) { + return set; + } + set.addAll(referredEntities.keySet()); + return set; + } + + public void restoreSelective(String guid, Map referredEntities) throws Exception { + Set referencedGUIDs = new HashSet<>(getEntityAndReferenceGuids(guid, referredEntities)); + displayCrlf("processing referencedGuids => " + referencedGUIDs); + + StandardJanusGraph janusGraph = (StandardJanusGraph) graph; + IndexSerializer indexSerializer = janusGraph.getIndexSerializer(); + + for (String indexName : getIndexes()) { + LOG.info("Restoring: " + indexName); + long startTime = System.currentTimeMillis(); + reindexVertex(indexName, indexSerializer, referencedGUIDs); + + LOG.info(": Time taken: " + (System.currentTimeMillis() - startTime) + " ms"); + } + } + + public void restoreByIds(Set guids) throws Exception { + + StandardJanusGraph janusGraph = (StandardJanusGraph) graph; + IndexSerializer indexSerializer = janusGraph.getIndexSerializer(); + + for (String indexName : getIndexes()) { + LOG.info("Restoring: " + indexName); + long startTime = System.currentTimeMillis(); + reindexVertex(indexName, indexSerializer, guids); + + LOG.info(": Time taken: " + (System.currentTimeMillis() - startTime) + " ms"); + LOG.info(": Done!"); + } + } } diff --git a/webapp/pom.xml b/webapp/pom.xml index 858ab9aafda..35a0e8a010b 100755 --- a/webapp/pom.xml +++ b/webapp/pom.xml @@ -121,6 +121,13 @@ atlas-client-v2 + + + com.googlecode.owasp-java-html-sanitizer + owasp-java-html-sanitizer + ${owasp-html-sanitizer.version} + + org.apache.atlas atlas-authorization @@ -130,6 +137,12 @@ org.apache.atlas atlas-notification + + + com.yammer.metrics + metrics-core + + @@ -137,6 +150,12 @@ atlas-intg + + org.apache.atlas + auth-plugin-atlas + ${project.version} + + org.apache.atlas atlas-janusgraph-hbase2 @@ -189,6 +208,10 @@ io.netty netty-transport-native-epoll + + log4j + log4j + @@ -211,8 +234,12 @@ org.apache.zookeeper zookeeper - - + + + log4j + log4j + + @@ -504,6 +531,23 @@ + + + io.sentry + sentry-log4j + 1.7.30 + + + log4j + log4j + + + + + ch.qos.reload4j + reload4j + ${reload4j.version} + org.apache.atlas atlas-testtools @@ -533,6 +577,13 @@ jackson-core ${jackson.version} + + org.apache.atlas + atlas-index-repair-tool + 3.0.0-SNAPSHOT + compile + + @@ -547,22 +598,24 @@ org.apache.atlas atlas-dashboardv2 + ${skipOverlay} org.apache.atlas atlas-dashboardv3 + ${skipOverlay} - + diff --git a/webapp/src/main/java/org/apache/atlas/Atlas.java b/webapp/src/main/java/org/apache/atlas/Atlas.java index fda7d7ab9bc..548ef6e4e7a 100755 --- a/webapp/src/main/java/org/apache/atlas/Atlas.java +++ b/webapp/src/main/java/org/apache/atlas/Atlas.java @@ -20,6 +20,7 @@ import org.apache.atlas.repository.graphdb.janus.AtlasElasticsearchDatabase; import org.apache.atlas.security.SecurityProperties; +import org.apache.atlas.util.AccessAuditLogsIndexCreator; import org.apache.atlas.web.service.EmbeddedServer; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.GnuParser; @@ -35,7 +36,7 @@ import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.client.indices.IndexTemplatesExistRequest; import org.elasticsearch.client.indices.PutIndexTemplateRequest; -import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.xcontent.XContentType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.bridge.SLF4JBridgeHandler; @@ -143,6 +144,10 @@ public static void main(String[] args) throws Exception { initElasticsearch(); } + if (configuration.getString("atlas.authorizer.impl").equalsIgnoreCase("atlas")) { + initAccessAuditElasticSearch(configuration); + } + server = EmbeddedServer.newServer(appHost, appPort, appPath, enableTLS); installLogBridge(); @@ -243,6 +248,11 @@ private static void installLogBridge() { SLF4JBridgeHandler.install(); } + private static void initAccessAuditElasticSearch(Configuration configuration) throws IOException { + AccessAuditLogsIndexCreator indexCreator = new AccessAuditLogsIndexCreator(configuration); + indexCreator.start(); + } + private static void initElasticsearch() throws IOException { RestHighLevelClient esClient = AtlasElasticsearchDatabase.getClient(); IndexTemplatesExistRequest indexTemplateExistsRequest = new IndexTemplatesExistRequest("atlan-template"); @@ -266,6 +276,12 @@ private static void initElasticsearch() throws IOException { File elasticsearchSettingsFile = new File(elasticsearchSettingsFilePath); String jsonString = new String(Files.readAllBytes(elasticsearchSettingsFile.toPath()), StandardCharsets.UTF_8); request.settings(jsonString, XContentType.JSON); + + String elasticsearchMappingsFilePath = (org.apache.commons.lang3.StringUtils.isEmpty(atlasHomeDir) ? "." : atlasHomeDir) + File.separator + "elasticsearch" + File.separator + "es-mappings.json"; + File elasticsearchMappingsFile = new File(elasticsearchMappingsFilePath); + String mappingsJsonString = new String(Files.readAllBytes(elasticsearchMappingsFile.toPath()), StandardCharsets.UTF_8); + request.mapping(mappingsJsonString, XContentType.JSON); + try { AcknowledgedResponse putTemplateResponse = esClient.indices().putTemplate(request, RequestOptions.DEFAULT); if (putTemplateResponse.isAcknowledged()) { diff --git a/webapp/src/main/java/org/apache/atlas/notification/EntityNotificationListenerV2.java b/webapp/src/main/java/org/apache/atlas/notification/EntityNotificationListenerV2.java index 55f67d9889c..a5e90431bee 100644 --- a/webapp/src/main/java/org/apache/atlas/notification/EntityNotificationListenerV2.java +++ b/webapp/src/main/java/org/apache/atlas/notification/EntityNotificationListenerV2.java @@ -19,6 +19,7 @@ import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.RequestContext; +import org.apache.atlas.annotation.EnableConditional; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.listener.EntityChangeListenerV2; import org.apache.atlas.model.glossary.AtlasGlossaryTerm; @@ -34,6 +35,7 @@ import org.apache.atlas.type.AtlasClassificationType; import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasStructType.AtlasAttribute; +import org.apache.atlas.type.AtlasType; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.utils.AtlasPerfMetrics.MetricRecorder; import org.apache.commons.collections.CollectionUtils; @@ -44,23 +46,9 @@ import org.springframework.stereotype.Component; import javax.inject.Inject; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.Date; - -import static org.apache.atlas.model.notification.EntityNotification.EntityNotificationV2.OperationType.CLASSIFICATION_ADD; -import static org.apache.atlas.model.notification.EntityNotification.EntityNotificationV2.OperationType.CLASSIFICATION_DELETE; -import static org.apache.atlas.model.notification.EntityNotification.EntityNotificationV2.OperationType.CLASSIFICATION_UPDATE; -import static org.apache.atlas.model.notification.EntityNotification.EntityNotificationV2.OperationType.ENTITY_CREATE; -import static org.apache.atlas.model.notification.EntityNotification.EntityNotificationV2.OperationType.ENTITY_DELETE; -import static org.apache.atlas.model.notification.EntityNotification.EntityNotificationV2.OperationType.ENTITY_UPDATE; -import static org.apache.atlas.model.notification.EntityNotification.EntityNotificationV2.OperationType.RELATIONSHIP_CREATE; -import static org.apache.atlas.model.notification.EntityNotification.EntityNotificationV2.OperationType.RELATIONSHIP_DELETE; -import static org.apache.atlas.model.notification.EntityNotification.EntityNotificationV2.OperationType.RELATIONSHIP_UPDATE; +import java.util.*; + +import static org.apache.atlas.model.notification.EntityNotification.EntityNotificationV2.OperationType.*; import static org.apache.atlas.repository.graph.GraphHelper.isInternalType; import static org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever.CREATE_TIME; import static org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever.DESCRIPTION; @@ -69,11 +57,13 @@ import static org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever.QUALIFIED_NAME; @Component +@EnableConditional(property = "atlas.enable.entity.notifications") public class EntityNotificationListenerV2 implements EntityChangeListenerV2 { private static final Logger LOG = LoggerFactory.getLogger(EntityNotificationListenerV2.class); private final AtlasTypeRegistry typeRegistry; private final EntityNotificationSender notificationSender; + private final EntityNotificationSender inlineNotificationSender; @Inject public EntityNotificationListenerV2(AtlasTypeRegistry typeRegistry, @@ -81,6 +71,7 @@ public EntityNotificationListenerV2(AtlasTypeRegistry typeRegistry, Configuration configuration) { this.typeRegistry = typeRegistry; this.notificationSender = new EntityNotificationSender<>(notificationInterface, configuration); + this.inlineNotificationSender = new EntityNotificationSender<>(notificationInterface, false); } @Override @@ -105,12 +96,21 @@ public void onEntitiesPurged(List entities) throws AtlasBaseExcepti @Override public void onClassificationsAdded(AtlasEntity entity, List classifications) throws AtlasBaseException { +<<<<<<< HEAD notifyEntityEvents(Collections.singletonList(entity), classifications, CLASSIFICATION_ADD); } @Override public void onClassificationsAdded(List entities, List classifications) throws AtlasBaseException { notifyEntityEvents(entities, classifications, CLASSIFICATION_ADD); +======= + notifyClassificationEvents(Collections.singletonList(entity), CLASSIFICATION_ADD, classifications); + } + + @Override + public void onClassificationsAdded(List entities, List classifications, boolean forceInline) throws AtlasBaseException { + notifyClassificationEvents(entities, CLASSIFICATION_ADD, classifications, forceInline); +>>>>>>> 7f3c811fb15361ba82bfb4edd7a8393798863ff0 } @Override @@ -119,20 +119,34 @@ public void onClassificationsUpdated(AtlasEntity entity, List> removedPropagations = RequestContext.get().getRemovedPropagations(); if (addedPropagations.containsKey(entity.getGuid())) { +<<<<<<< HEAD notifyEntityEvents(Collections.singletonList(entity), classifications, CLASSIFICATION_ADD); } else if (!removedPropagations.containsKey(entity.getGuid())) { notifyEntityEvents(Collections.singletonList(entity), classifications, CLASSIFICATION_UPDATE); +======= + notifyClassificationEvents(Collections.singletonList(entity), CLASSIFICATION_ADD, classifications); + } else if (!removedPropagations.containsKey(entity.getGuid())) { + notifyClassificationEvents(Collections.singletonList(entity), CLASSIFICATION_UPDATE, classifications); +>>>>>>> 7f3c811fb15361ba82bfb4edd7a8393798863ff0 } } @Override public void onClassificationsDeleted(AtlasEntity entity, List classifications) throws AtlasBaseException { +<<<<<<< HEAD notifyEntityEvents(Collections.singletonList(entity), classifications, CLASSIFICATION_DELETE); +======= + notifyClassificationEvents(Collections.singletonList(entity), CLASSIFICATION_DELETE, classifications); +>>>>>>> 7f3c811fb15361ba82bfb4edd7a8393798863ff0 } @Override public void onClassificationsDeleted(List entities, List classifications) throws AtlasBaseException { +<<<<<<< HEAD notifyEntityEvents(entities, classifications, CLASSIFICATION_DELETE); +======= + notifyClassificationEvents(entities, CLASSIFICATION_DELETE, classifications); +>>>>>>> 7f3c811fb15361ba82bfb4edd7a8393798863ff0 } @Override @@ -157,40 +171,106 @@ public void onLabelsAdded(AtlasEntity entity, Set labels) throws AtlasBa private void notifyEntityEvents(List entities, List classifications, OperationType operationType) throws AtlasBaseException { MetricRecorder metric = RequestContext.get().startMetricRecord("entityNotification"); + + Map differentialEntities = RequestContext.get().getDifferentialEntitiesMap(); + Map requestContextHeaders = RequestContext.get().getRequestContextHeaders(); + + List messages = new ArrayList<>(); + + for (AtlasEntity entity : entities) { + if (isInternalType(entity.getTypeName())) { + continue; + } + String entityGuid = entity.getGuid(); + + if(differentialEntities != null){ + if (differentialEntities.containsKey(entityGuid)) { + messages.add(new EntityNotificationV2(toNotificationHeader(entity), differentialEntities.get(entityGuid), + operationType, RequestContext.get().getRequestTime(), requestContextHeaders)); + }else { + messages.add(new EntityNotificationV2(toNotificationHeader(entity), null, + operationType, RequestContext.get().getRequestTime(), requestContextHeaders)); + } + }else{ + messages.add(new EntityNotificationV2(toNotificationHeader(entity), null, + operationType, RequestContext.get().getRequestTime(), requestContextHeaders)); + } + + } + + sendNotifications(operationType, messages); + RequestContext.get().endMetricRecord(metric); + } + + private void notifyClassificationEvents(List entities, OperationType operationType, Object mutatedObj) throws AtlasBaseException { + notifyClassificationEvents(entities, operationType, mutatedObj, false); + } + + private void notifyClassificationEvents(List entities, OperationType operationType, Object mutatedObj, boolean forceInline) throws AtlasBaseException { + MetricRecorder metric = RequestContext.get().startMetricRecord("classificationNotification"); List messages = new ArrayList<>(); + Map requestContextHeaders = RequestContext.get().getRequestContextHeaders(); for (AtlasEntity entity : entities) { if (isInternalType(entity.getTypeName())) { continue; } +<<<<<<< HEAD messages.add(new EntityNotificationV2(toNotificationHeader(entity), classifications, operationType, RequestContext.get().getRequestTime())); +======= + messages.add(new EntityNotificationV2(toNotificationHeader(entity), mutatedObj, operationType, + RequestContext.get().getRequestTime(), requestContextHeaders)); +>>>>>>> 7f3c811fb15361ba82bfb4edd7a8393798863ff0 } - sendNotifications(operationType, messages); + sendNotifications(operationType, messages, forceInline); + RequestContext.get().endMetricRecord(metric); } private void notifyRelationshipEvents(List relationships, OperationType operationType) throws AtlasBaseException { MetricRecorder metric = RequestContext.get().startMetricRecord("entityNotification"); List messages = new ArrayList<>(); + Map requestContextHeaders = RequestContext.get().getRequestContextHeaders(); for (AtlasRelationship relationship : relationships) { if (isInternalType(relationship.getTypeName())) { continue; } - - messages.add(new EntityNotificationV2(toNotificationHeader(relationship), operationType, RequestContext.get().getRequestTime())); + messages.add(new EntityNotificationV2(toNotificationHeader(relationship), operationType, + RequestContext.get().getRequestTime(), requestContextHeaders)); } sendNotifications(operationType, messages); RequestContext.get().endMetricRecord(metric); } + private void notifyBusinessMetadataEvents(AtlasEntity entity, OperationType operationType, Map> updatedBusinessAttributes) throws AtlasBaseException { + MetricRecorder metric = RequestContext.get().startMetricRecord("entityBMNotification"); + List messages = new ArrayList<>(); + Map requestContextHeaders = RequestContext.get().getRequestContextHeaders(); + + messages.add(new EntityNotificationV2(toNotificationHeader(entity), updatedBusinessAttributes, operationType, + RequestContext.get().getRequestTime(), requestContextHeaders)); + + sendNotifications(operationType, messages); + RequestContext.get().endMetricRecord(metric); + } + private void sendNotifications(OperationType operationType, List messages) throws AtlasBaseException { + sendNotifications(operationType, messages, false); + } + + private void sendNotifications(OperationType operationType, List messages, boolean forceInline) throws AtlasBaseException { if (!messages.isEmpty()) { try { - notificationSender.send(messages); + if (forceInline) { + inlineNotificationSender.send(operationType, messages); + } + else { + notificationSender.send(operationType, messages); + } } catch (NotificationException e) { throw new AtlasBaseException(AtlasErrorCode.ENTITY_NOTIFICATION_FAILED, e, operationType.name()); } @@ -264,7 +344,6 @@ private AtlasEntityHeaderWithRelations toNotificationHeader(AtlasEntity entity) return ret; } - private AtlasRelationshipHeader toNotificationHeader(AtlasRelationship relationship) { return new AtlasRelationshipHeader(relationship); } @@ -341,6 +420,6 @@ public void onRelationshipsPurged(List relationships) throws @Override public void onBusinessAttributesUpdated(AtlasEntity entity, Map> updatedBusinessAttributes) throws AtlasBaseException{ - // do nothing -> notification not sent out for business metadata attribute updation from entities + notifyBusinessMetadataEvents(entity, BUSINESS_ATTRIBUTE_UPDATE, updatedBusinessAttributes); } } \ No newline at end of file diff --git a/webapp/src/main/java/org/apache/atlas/notification/EntityNotificationSender.java b/webapp/src/main/java/org/apache/atlas/notification/EntityNotificationSender.java index c2ae5122b8e..c039cc837ac 100644 --- a/webapp/src/main/java/org/apache/atlas/notification/EntityNotificationSender.java +++ b/webapp/src/main/java/org/apache/atlas/notification/EntityNotificationSender.java @@ -18,6 +18,7 @@ package org.apache.atlas.notification; import org.apache.atlas.GraphTransactionInterceptor; +import org.apache.atlas.model.notification.EntityNotification; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.configuration.Configuration; import org.slf4j.Logger; @@ -27,6 +28,7 @@ import java.util.List; import static org.apache.atlas.notification.NotificationInterface.NotificationType.ENTITIES; +import static org.apache.atlas.notification.NotificationInterface.NotificationType.RELATIONSHIPS; public class EntityNotificationSender { @@ -52,13 +54,13 @@ public EntityNotificationSender(NotificationInterface notificationInterface, boo } } - public void send(List notifications) throws NotificationException { - this.notificationSender.send(notifications); + public void send(EntityNotification.EntityNotificationV2.OperationType operationType, List notifications) throws NotificationException { + this.notificationSender.send(operationType, notifications); } private interface NotificationSender { - void send(List notifications) throws NotificationException; + void send(EntityNotification.EntityNotificationV2.OperationType operationType, List notifications) throws NotificationException; } private class InlineNotificationSender implements NotificationSender { @@ -69,8 +71,11 @@ public InlineNotificationSender(NotificationInterface notificationInterface) { } @Override - public void send(List notifications) throws NotificationException { - notificationInterface.send(ENTITIES, notifications); + public void send(EntityNotification.EntityNotificationV2.OperationType operationType, List notifications) throws NotificationException { + if (isRelationshipEvent(operationType)) + notificationInterface.send(RELATIONSHIPS, notifications); + else + notificationInterface.send(ENTITIES, notifications); } } @@ -83,23 +88,29 @@ public PostCommitNotificationSender(NotificationInterface notificationInterface) } @Override - public void send(List notifications) throws NotificationException { + public void send(EntityNotification.EntityNotificationV2.OperationType operationType, List notifications) throws NotificationException { PostCommitNotificationHook notificationHook = postCommitNotificationHooks.get(); if (notificationHook == null) { - notificationHook = new PostCommitNotificationHook(notifications); - + notificationHook = new PostCommitNotificationHook(operationType, notifications); postCommitNotificationHooks.set(notificationHook); } else { - notificationHook.addNotifications(notifications); + if (isRelationshipEvent(operationType)) + notificationHook.addRelationshipNotifications(notifications); + else + notificationHook.addNotifications(notifications); } } class PostCommitNotificationHook extends GraphTransactionInterceptor.PostTransactionHook { private final List notifications = new ArrayList<>(); + private final List relationshipNotifications = new ArrayList<>(); - public PostCommitNotificationHook(List notifications) { - this.addNotifications(notifications); + public PostCommitNotificationHook(EntityNotification.EntityNotificationV2.OperationType operationType, List notifications) { + if (isRelationshipEvent(operationType)) + this.addRelationshipNotifications(notifications); + else + this.addNotifications(notifications); } public void addNotifications(List notifications) { @@ -108,14 +119,21 @@ public void addNotifications(List notifications) { } } + public void addRelationshipNotifications(List notifications) { + if (notifications != null) { + this.relationshipNotifications.addAll(notifications); + } + } + @Override public void onComplete(boolean isSuccess) { postCommitNotificationHooks.remove(); - if (CollectionUtils.isNotEmpty(notifications)) { + if (CollectionUtils.isNotEmpty(notifications) || CollectionUtils.isNotEmpty(relationshipNotifications)) { if (isSuccess) { try { notificationInterface.send(ENTITIES, notifications); + notificationInterface.send(RELATIONSHIPS, relationshipNotifications); } catch (NotificationException excp) { LOG.error("failed to send entity notifications", excp); } @@ -125,7 +143,12 @@ public void onComplete(boolean isSuccess) { } } } + } } } + + private static boolean isRelationshipEvent(EntityNotification.EntityNotificationV2.OperationType operationType) { + return EntityNotification.EntityNotificationV2.OperationType.RELATIONSHIP_CREATE.equals(operationType) || EntityNotification.EntityNotificationV2.OperationType.RELATIONSHIP_UPDATE.equals(operationType) || EntityNotification.EntityNotificationV2.OperationType.RELATIONSHIP_DELETE.equals(operationType); + } } \ No newline at end of file diff --git a/webapp/src/main/java/org/apache/atlas/notification/NotificationEntityChangeListener.java b/webapp/src/main/java/org/apache/atlas/notification/NotificationEntityChangeListener.java index 520d1ea870f..ab61da66ff3 100644 --- a/webapp/src/main/java/org/apache/atlas/notification/NotificationEntityChangeListener.java +++ b/webapp/src/main/java/org/apache/atlas/notification/NotificationEntityChangeListener.java @@ -188,7 +188,7 @@ private void notifyOfEntityEvent(Collection entityDefinitions, } if (!messages.isEmpty()) { - notificationSender.send(messages); + notificationSender.send(null, messages); } RequestContext.get().endMetricRecord(metric); diff --git a/webapp/src/main/java/org/apache/atlas/util/AccessAuditLogsIndexCreator.java b/webapp/src/main/java/org/apache/atlas/util/AccessAuditLogsIndexCreator.java new file mode 100644 index 00000000000..f5786cf7d39 --- /dev/null +++ b/webapp/src/main/java/org/apache/atlas/util/AccessAuditLogsIndexCreator.java @@ -0,0 +1,356 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.util; + +import org.apache.atlas.AtlasConfiguration; +import org.apache.atlas.authorization.credutils.CredentialsProviderUtil; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.configuration.Configuration; +import org.apache.commons.lang.StringUtils; +import org.apache.http.HttpEntity; +import org.apache.http.HttpHost; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.entity.ContentType; +import org.apache.http.nio.entity.NStringEntity; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestClientBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.StringTokenizer; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +import static org.apache.atlas.repository.audit.ESBasedAuditRepository.INDEX_BACKEND_CONF; + +public class AccessAuditLogsIndexCreator extends Thread { + private static final Logger LOG = LoggerFactory.getLogger(AccessAuditLogsIndexCreator.class); + + private static final String ES_CONFIG_USERNAME = "atlas.audit.elasticsearch.user"; + private static final String ES_CONFIG_PASSWORD = "atlas.audit.elasticsearch.password"; + private static final String ES_CONFIG_PORT = "atlas.audit.elasticsearch.port"; + private static final String ES_CONFIG_PROTOCOL = "atlas.audit.elasticsearch.protocol"; + private static final String ES_CONFIG_INDEX = "atlas.audit.elasticsearch.index"; + private static final String ES_TIME_INTERVAL = "atlas.audit.elasticsearch.time.interval"; + private static final String ES_NO_SHARDS = "atlas.audit.elasticsearch.no.shards"; + private static final String ES_NO_REPLICA = "atlas.audit.elasticsearch.no.replica"; + private static final String ES_CREDENTIAL_PROVIDER_PATH = "atlas.credential.provider.path"; + private static final String ES_CREDENTIAL_ALIAS = "atlas.audit.elasticsearch.credential.alias"; + private static final String ES_BOOTSTRAP_MAX_RETRY = "atlas.audit.elasticsearch.max.retry"; + + private static final String DEFAULT_INDEX_NAME = "ranger-audit"; + private static final String ES_RANGER_AUDIT_SCHEMA_FILE = "atlas-auth-es-schema.json"; + + private static final long DEFAULT_ES_TIME_INTERVAL_MS = 30000L; + private static final int TRY_UNTIL_SUCCESS = -1; + private static final int DEFAULT_ES_BOOTSTRAP_MAX_RETRY = 3; + + private final AtomicLong lastLoggedAt = new AtomicLong(0); + private volatile RestClient client = null; + private Long time_interval; + + private String user; + private String password; + List hosts; + private String protocol; + private String index; + private String es_ranger_audit_schema_json; + + private int port; + private int max_retry; + private int retry_counter = 0; + private int no_of_replicas; + private int no_of_shards; + private boolean is_completed = false; + + public AccessAuditLogsIndexCreator(Configuration configuration) throws IOException { + LOG.info("Starting Ranger audit schema setup in ElasticSearch."); + time_interval = configuration.getLong(ES_TIME_INTERVAL, DEFAULT_ES_TIME_INTERVAL_MS); + user = configuration.getString(ES_CONFIG_USERNAME); + + hosts = getHttpHosts(configuration); + port = getPort(configuration); + + protocol = configuration.getString(ES_CONFIG_PROTOCOL, "http"); + index = configuration.getString(ES_CONFIG_INDEX, DEFAULT_INDEX_NAME); + password = configuration.getString(ES_CONFIG_PASSWORD); + + no_of_replicas = configuration.getInt(ES_NO_REPLICA, 1); + no_of_shards = configuration.getInt(ES_NO_SHARDS, 1); + max_retry = configuration.getInt(ES_BOOTSTRAP_MAX_RETRY, DEFAULT_ES_BOOTSTRAP_MAX_RETRY); + + String atlasHomeDir = System.getProperty("atlas.home"); + String elasticsearchFilePath = (StringUtils.isEmpty(atlasHomeDir) ? "." : atlasHomeDir) + File.separator + "elasticsearch" + File.separator + ES_RANGER_AUDIT_SCHEMA_FILE; + Path es_schema_path = Paths.get(elasticsearchFilePath); + es_ranger_audit_schema_json = new String(Files.readAllBytes(es_schema_path), StandardCharsets.UTF_8); + + String providerPath = configuration.getString(ES_CREDENTIAL_PROVIDER_PATH); + String credentialAlias = configuration.getString(ES_CREDENTIAL_ALIAS, ES_CONFIG_PASSWORD); + if (providerPath != null && credentialAlias != null) { + if (StringUtils.isBlank(password) || "none".equalsIgnoreCase(password.trim())) { + password = configuration.getString(ES_CONFIG_PASSWORD); + } + } + } + + private String connectionString() { + return String.format(Locale.ROOT,"User:%s, %s://%s:%s/%s", user, protocol, hosts, port, index); + } + + @Override + public void run() { + LOG.info("Started run method"); + if (CollectionUtils.isNotEmpty(hosts)) { + LOG.info("Elastic search hosts=" + hosts + ", index=" + index); + while (!is_completed && (max_retry == TRY_UNTIL_SUCCESS || retry_counter < max_retry)) { + try { + LOG.info("Trying to acquire elastic search connection"); + if (connect()) { + LOG.info("Connection to elastic search established successfully"); + if (createIndex()) { + is_completed = true; + break; + } else { + logErrorMessageAndWait("Error while performing operations on elasticsearch. ", null); + } + } else { + logErrorMessageAndWait( + "Cannot connect to elasticsearch kindly check the elasticsearch related configs. ", + null); + } + } catch (Exception ex) { + logErrorMessageAndWait("Error while validating elasticsearch index ", ex); + } finally { + try { + if (client != null) { + client.close(); + } + } catch (IOException e) { + LOG.warn("AccessAuditLogsIndexCreator: Failed to close ES client", e); + } + } + } + } else { + LOG.error("elasticsearch hosts values are empty. Please set property " + INDEX_BACKEND_CONF); + } + + } + + private synchronized boolean connect() { + if (client == null) { + synchronized (AccessAuditLogsIndexCreator.class) { + if (client == null) { + try { + createClient(); + } catch (Exception ex) { + LOG.error("Can't connect to elasticsearch server. host=" + hosts + ", index=" + index + ex); + } + } + } + } + return client != null; + } + + private void createClient() { + try { + RestClientBuilder builder = RestClient.builder(hosts.get(0)); + builder.setRequestConfigCallback(requestConfigBuilder -> requestConfigBuilder + .setConnectTimeout(AtlasConfiguration.INDEX_CLIENT_CONNECTION_TIMEOUT.getInt()) + .setSocketTimeout(AtlasConfiguration.INDEX_CLIENT_SOCKET_TIMEOUT.getInt())); + + client = builder.build(); + } catch (Throwable t) { + lastLoggedAt.updateAndGet(lastLoggedAt -> { + long now = System.currentTimeMillis(); + long elapsed = now - lastLoggedAt; + if (elapsed > TimeUnit.MINUTES.toMillis(1)) { + LOG.error("Can't connect to ElasticSearch server: " + connectionString() + t); + return now; + } else { + return lastLoggedAt; + } + }); + } + } + + public static RestClientBuilder getRestClientBuilder(String urls, String protocol, String user, String password, int port) { + RestClientBuilder restClientBuilder = RestClient.builder( + toArray(urls, ",").stream() + .map(x -> new HttpHost(x, port, protocol)) + .toArray(i -> new HttpHost[i]) + ); + if (StringUtils.isNotBlank(user) && StringUtils.isNotBlank(password) && !user.equalsIgnoreCase("NONE") && !password.equalsIgnoreCase("NONE")) { + + final CredentialsProvider credentialsProvider = + CredentialsProviderUtil.getBasicCredentials(user, password); + restClientBuilder.setHttpClientConfigCallback(clientBuilder -> + clientBuilder.setDefaultCredentialsProvider(credentialsProvider)); + + } else { + LOG.error("ElasticSearch Credentials not provided!!"); + final CredentialsProvider credentialsProvider = null; + restClientBuilder.setHttpClientConfigCallback(clientBuilder -> + clientBuilder.setDefaultCredentialsProvider(credentialsProvider)); + } + return restClientBuilder; + } + + private boolean createIndex() { + boolean exists = false; + if (client == null) { + connect(); + } + if (client != null) { + try { + Request request = new Request("HEAD", index); + Response response = client.performRequest(request); + + int statusCode = response.getStatusLine().getStatusCode(); + if (statusCode == 200) { + LOG.info("Entity audits index exists!"); + exists = true; + } else { + LOG.info("Entity audits index does not exist!"); + exists = false; + } + + } catch (Exception e) { + LOG.info("Index " + this.index + " not available."); + } + if (!exists) { + LOG.info("Index does not exist. Attempting to create index:" + this.index); + try { + HttpEntity entity = new NStringEntity(es_ranger_audit_schema_json, ContentType.APPLICATION_JSON); + Request request = new Request("PUT", index); + + /*if (this.no_of_shards >= 0 && this.no_of_replicas >= 0) { + request.settings(Settings.builder().put("number_of_shards", this.no_of_shards) + .put("number_of_replicas", this.no_of_replicas)); + }*/ + + request.setEntity(entity); + Response response = client.performRequest(request); + + if (response != null && response.getStatusLine().getStatusCode() == 200) { + LOG.info("Index " + this.index + " created successfully."); + exists = true; + } + } catch (Exception e) { + LOG.error("Unable to create Index. Reason:" + e.toString()); + e.printStackTrace(); + } + } else { + LOG.info("Index " + this.index + " is already created."); + } + } + return exists; + } + + private void logErrorMessageAndWait(String msg, Exception exception) { + retry_counter++; + String attemptMessage; + if (max_retry != TRY_UNTIL_SUCCESS) { + attemptMessage = (retry_counter == max_retry) ? ("Maximum attempts reached for setting up elasticsearch.") + : ("[retrying after " + time_interval + " ms]. No. of attempts left : " + + (max_retry - retry_counter) + " . Maximum attempts : " + max_retry); + } else { + attemptMessage = "[retrying after " + time_interval + " ms]"; + } + StringBuilder errorBuilder = new StringBuilder(); + errorBuilder.append(msg); + if (exception != null) { + errorBuilder.append("Error : ".concat(exception.getMessage() + ". ")); + } + errorBuilder.append(attemptMessage); + LOG.error(errorBuilder.toString()); + try { + Thread.sleep(time_interval); + } catch (InterruptedException ex) { + LOG.info("sleep interrupted: " + ex.getMessage()); + } + } + + private static String getHosts(Configuration configuration) { + StringBuilder urls = new StringBuilder(); + String indexConf = configuration.getString(INDEX_BACKEND_CONF); + String[] hosts = indexConf.split(","); + for (String host : hosts) { + host = host.trim(); + String[] hostAndPort = host.split(":"); + urls.append(hostAndPort[0]); + + } + return urls.toString(); + } + + public static List getHttpHosts(Configuration configuration) { + List httpHosts = new ArrayList<>(); + + String indexConf = configuration.getString(INDEX_BACKEND_CONF); + String[] hosts = indexConf.split(","); + for (String host : hosts) { + host = host.trim(); + String[] hostAndPort = host.split(":"); + if (hostAndPort.length == 1) { + httpHosts.add(new HttpHost(hostAndPort[0])); + } else if (hostAndPort.length == 2) { + httpHosts.add(new HttpHost(hostAndPort[0], Integer.parseInt(hostAndPort[1]))); + } + } + return httpHosts; + } + + + private static int getPort(Configuration configuration) { + int port = 9200; + StringBuilder urls = new StringBuilder(); + String indexConf = configuration.getString(INDEX_BACKEND_CONF); + try { + String[] hosts = indexConf.split(","); + String host = hosts[0]; + host = host.trim(); + String[] hostAndPort = host.split(":"); + port = Integer.parseInt(hostAndPort[1]); + } catch (Exception e) { + LOG.warn("Setting ES port to default {}", port); + } + + return port; + } + + public static List toArray(String destListStr, String delim) { + List list = new ArrayList(); + if (StringUtils.isNotBlank(destListStr)) { + StringTokenizer tokenizer = new StringTokenizer(destListStr, delim.trim()); + while (tokenizer.hasMoreTokens()) { + list.add(tokenizer.nextToken()); + } + } + return list; + } +} diff --git a/webapp/src/main/java/org/apache/atlas/web/errors/AtlasBaseExceptionMapper.java b/webapp/src/main/java/org/apache/atlas/web/errors/AtlasBaseExceptionMapper.java index bb68e03c453..a411ad545c4 100755 --- a/webapp/src/main/java/org/apache/atlas/web/errors/AtlasBaseExceptionMapper.java +++ b/webapp/src/main/java/org/apache/atlas/web/errors/AtlasBaseExceptionMapper.java @@ -21,6 +21,7 @@ import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.type.AtlasType; +import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.StringUtils; import org.springframework.stereotype.Component; @@ -50,7 +51,7 @@ public Response toResponse(AtlasBaseException exception) { } protected Response buildAtlasBaseExceptionResponse(AtlasBaseException baseException) { - Map errorJsonMap = new LinkedHashMap<>(); + Map errorJsonMap = new LinkedHashMap<>(); AtlasErrorCode errorCode = baseException.getAtlasErrorCode(); errorJsonMap.put("errorCode", errorCode.getErrorCode()); errorJsonMap.put("errorMessage", baseException.getMessage()); @@ -59,6 +60,10 @@ protected Response buildAtlasBaseExceptionResponse(AtlasBaseException baseExcept errorJsonMap.put("entityGuid", baseException.getEntityGuid()); } + if (MapUtils.isNotEmpty(baseException.getErrorDetailsMap())) { + errorJsonMap.put("errorDetailsMap", baseException.getErrorDetailsMap()); + } + if (baseException.getCause() != null) { errorJsonMap.put("errorCause", baseException.getCause().getMessage()); } diff --git a/webapp/src/main/java/org/apache/atlas/web/filters/AtlasXSSPreventionFilter.java b/webapp/src/main/java/org/apache/atlas/web/filters/AtlasXSSPreventionFilter.java new file mode 100644 index 00000000000..8d6fe8a4547 --- /dev/null +++ b/webapp/src/main/java/org/apache/atlas/web/filters/AtlasXSSPreventionFilter.java @@ -0,0 +1,208 @@ +package org.apache.atlas.web.filters; + +import org.apache.atlas.AtlasConfiguration; +import org.apache.atlas.web.util.CachedBodyHttpServletRequest; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringEscapeUtils; +import org.apache.commons.lang3.StringUtils; +import org.owasp.html.HtmlPolicyBuilder; +import org.owasp.html.PolicyFactory; +import org.owasp.html.Sanitizers; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import javax.inject.Inject; +import javax.servlet.*; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import com.google.common.base.Predicate; + +import java.util.regex.Pattern; + +@Component +public class AtlasXSSPreventionFilter implements Filter { + + private static final Logger LOG = LoggerFactory.getLogger(AtlasXSSPreventionFilter.class); + private static Pattern pattern; + private PolicyFactory policy; + private static final String MASK_STRING = "##ATLAN##"; + private static final String CONTENT_TYPE_JSON = "application/json"; + private static final String ERROR_INVALID_CHARACTERS = "invalid characters in the request body (XSS Filter)"; + private static final Pattern REGEX_NUMBER = Pattern.compile("^[-+]?[0-9]*\\.?[0-9]+([eE][-+]?[0-9]+)?$"); + public static final Pattern REGEX_INTEGER = Pattern.compile("^[0-9]+$"); + public static final Pattern REGEX_ISO8601 = Pattern.compile("^^[0-9]{4}(-[0-9]{2}(-[0-9]{2}([ T][0-9]{2}(:[0-9]{2}){1,2}(.[0-9]{1,6})" + +"?Z?([\\+-][0-9]{2}:[0-9]{2})?)?)?)?$"); + public static final Pattern REGEX_PARAGRAPH = Pattern.compile("^[\\p{L}\\p{N}\\s\\-_',\\[\\]!\\./\\\\\\(\\)]*$"); + public static final Pattern REGEX_SPACE_SEPARATED_TOKENS = Pattern.compile("^([\\s\\p{L}\\p{N}_-]+)$"); + public static final Pattern REGEX_DIRECTION = Pattern.compile("(?i)^(rtl|ltr)$"); + public static final Pattern REGEX_IMAGE_ALIGNMENT = Pattern.compile("(?i)^(left|right|top|texttop|middle|absmiddle|baseline|bottom|absbottom)$"); + public static final Pattern REGEX_NUMBER_OR_PERCENT = Pattern.compile("^[0-9]+[%]?$"); + public static final Pattern REGEX_LIST_TYPE = Pattern.compile("(?i)^(circle|disc|square|a|A|i|I|1)$"); + public static final Pattern REGEX_CELL_ALIGN = Pattern.compile("(?i)^(center|justify|left|right|char)$"); + public static final Pattern REGEX_CELL_VERTICAL_ALIGN = Pattern.compile("(?i)^(top|middle|bottom|baseline)$"); + public static final Pattern REGEX_SHAPE = Pattern.compile("(?i)^(rect|circle|poly|default)$"); + public static final Pattern REGEX_LANG = Pattern.compile("^[a-zA-Z]{2,20}"); + public static final Pattern REGEX_ID = Pattern.compile("[a-zA-Z0-9\\:\\-_\\.]+"); + public static final Pattern REGEX_NAME = Pattern.compile("^([\\p{L}\\p{N}_-]+)$"); + public static final Pattern REGEX_USEMAP = Pattern.compile("(?i)^#[\\p{L}\\p{N}_-]+$"); + public static final Pattern REGEX_OPEN = Pattern.compile("(?i)^(|open)$"); + public static final Pattern REGEX_COORDS = Pattern.compile("^([0-9]+,)+[0-9]+$"); + public static final Pattern REGEX_SCOPE = Pattern.compile("(?i)(?:row|col)(?:group)?"); + public static final Pattern REGEX_NOWRAP = Pattern.compile("(?i)|nowrap"); + public static final Pattern REGEX_ONSITE_URL = Pattern.compile("(?:[\\p{L}\\p{N}\\\\\\.\\#@\\$%\\+&;\\-_~,\\?=/!]+|\\#(\\w)+)"); + public static final Pattern REGEX_OFFSITE_URL = Pattern.compile("\\s*(?:(?:ht|f)tps?://|mailto:)[\\p{L}\\p{N}]" + + "[\\p{L}\\p{N}\\p{Zs}\\.\\#@\\$%\\+&;:\\-_~,\\?=/!\\(\\)]*+\\s*"); + public static final Predicate REGEX_ON_OFFSITE_URL = matchesEither(REGEX_ONSITE_URL, REGEX_OFFSITE_URL); + + @Inject + public AtlasXSSPreventionFilter() throws ServletException { + LOG.info("AtlasXSSPreventionFilter initialized"); + try { + init(null); + } catch (ServletException e) { + LOG.error("Error while initializing AtlasXSSPreventionFilter", e); + throw e; + } + } + + @Override + public void init(FilterConfig filterConfig) throws ServletException { + pattern = Pattern.compile(AtlasConfiguration.REST_API_XSS_FILTER_MASK_STRING.getString()); + + policy = new HtmlPolicyBuilder() + .allowStandardUrlProtocols() + .requireRelNofollowOnLinks() + .allowCommonInlineFormattingElements() + .allowCommonBlockElements() + .allowElements("article", "aside","details","figure","section","summary","hgroup") + .allowAttributes("open").matching(REGEX_OPEN).onElements("details") + //Allow common header elements + .allowElements("h1","h2","h3","h4","h5","h6") + .allowElements("blockquote") + .allowAttributes("cite").onElements("blockquote") + //allow common html tags + .allowElements("br", "div", "hr", "p", "span", "wbr") + //Links + .allowElements("a", "map", "area") + .allowAttributes("href").matching(REGEX_OFFSITE_URL).onElements("a", "area") + .allowAttributes("name").matching(REGEX_NAME).onElements("map") + .allowAttributes("alt").matching(REGEX_PARAGRAPH).onElements("area") + .allowAttributes("coords").matching(REGEX_COORDS).onElements("area") + .allowAttributes("shape").matching(REGEX_SHAPE).onElements("area") + .allowAttributes("rel").matching(REGEX_SPACE_SEPARATED_TOKENS).onElements("area") + .allowAttributes("target").matching(REGEX_SPACE_SEPARATED_TOKENS).onElements("a") + .allowAttributes("usemap").matching(REGEX_USEMAP).onElements("img") + //phrase elements + .allowElements("abbr", "acronym", "cite", "code", "dfn", "em", + "figcaption", "mark", "s", "samp", "strong", "sub", "sup", "var", "q", "time") + .allowAttributes("cite").onElements( "q") + .allowAttributes("datetime").matching(REGEX_ISO8601).onElements("time") + + //Style elements + .allowElements("b", "i", "pre", "small", "strike", "tt", "u") + //HTML5 formatting elements + .allowElements("bdi","bdo", "rp", "rt", "ruby", "wbr", "del", "ins") + .allowAttributes("cite").matching(REGEX_PARAGRAPH).onElements("del", "ins") + .allowAttributes("datetime").matching(REGEX_ISO8601).onElements("del", "ins") + + //Lists + .allowElements("dl", "dt", "dd", "ol", "ul", "li") + .allowAttributes("type").matching(REGEX_LIST_TYPE).onElements("ol", "ul", "li") + .allowAttributes("value").matching(REGEX_INTEGER).onElements("li") + //allow Tables + .allowElements("table", "caption", "col", "colgroup", "tbody", "td", "tfoot", "th", "thead", "tr") + .allowAttributes("height","width").matching(REGEX_NUMBER_OR_PERCENT).onElements("table","col", "colgroup", "tbody", "td", "tfoot", "th", "thead", "tr") + .allowAttributes("summary").matching(REGEX_PARAGRAPH).onElements("table") + .allowAttributes("align").matching(REGEX_CELL_ALIGN).onElements("col", "colgroup", "td", "th", "tbody", "tfoot") + .allowAttributes("valign").matching(REGEX_CELL_VERTICAL_ALIGN).onElements("col", "colgroup", "td", "th", "tbody", "tfoot") + .allowAttributes("abbr").matching(REGEX_PARAGRAPH).onElements("td", "th") + .allowAttributes("colspan","rowspan").matching(REGEX_INTEGER).onElements("td", "th") + .allowAttributes("headers").matching(REGEX_SPACE_SEPARATED_TOKENS).onElements("td", "th") + .allowAttributes("scope").matching(REGEX_SCOPE).onElements("td", "th") + .allowAttributes("nowrap").matching(REGEX_NOWRAP).onElements("td", "th") + //allow Forms + //By and large, forms are not permitted. However there are some form + // elements that can be used to present data, and we do permit those + .allowElements("meter", "progress") + .allowAttributes("value", "min", "max", "low", "high", "optimum").matching(REGEX_NUMBER).onElements("meter", "progress") + .allowAttributes("value","max").matching(REGEX_NUMBER).onElements("progress") + //Allow Image + .allowElements("img") + .allowAttributes("align").matching(REGEX_IMAGE_ALIGNMENT).onElements("img") + .allowAttributes("alt").matching(REGEX_PARAGRAPH).onElements("img") + .allowAttributes("height", "width").matching(REGEX_NUMBER_OR_PERCENT).onElements("img") + .allowAttributes("src").matching(REGEX_ON_OFFSITE_URL).onElements("img") + //Allow Global Attributes + .allowAttributes("dir").matching(REGEX_DIRECTION).globally() + .allowAttributes("lang").matching(REGEX_LANG).globally() + .allowAttributes("id").matching(REGEX_ID).globally() + .allowAttributes("title").matching(REGEX_PARAGRAPH).globally() + + .allowWithoutAttributes("a", "span") + .toFactory() + .and(Sanitizers.IMAGES); + + } + + @Override + public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain) throws IOException, ServletException { + HttpServletResponse response = (HttpServletResponse) servletResponse; + HttpServletRequest request = (HttpServletRequest) servletRequest; + + String serverName = request.getServerName(); + if (StringUtils.isNotEmpty(serverName) && serverName.contains(AtlasConfiguration.REST_API_XSS_FILTER_EXLUDE_SERVER_NAME.getString())) { + LOG.debug("AtlasXSSPreventionFilter: skipping filter for serverName: {}", serverName); + filterChain.doFilter(request, response); + return; + } + + String method = request.getMethod(); + if(!method.equals("POST") && !method.equals("PUT")) { + filterChain.doFilter(request, response); + return; + } + + String contentType = request.getContentType(); + if(StringUtils.isEmpty(contentType) || !contentType.contains(CONTENT_TYPE_JSON)) { + filterChain.doFilter(request, response); + return; + } + + CachedBodyHttpServletRequest cachedBodyHttpServletRequest = new CachedBodyHttpServletRequest(request); + String body = IOUtils.toString(cachedBodyHttpServletRequest.getInputStream(), "UTF-8"); + String reqBodyStr = StringEscapeUtils.unescapeJava(pattern.matcher(body).replaceAll(MASK_STRING)); + String sanitizedBody = policy.sanitize(reqBodyStr); + + if(!StringUtils.equals(reqBodyStr, StringEscapeUtils.unescapeHtml4(sanitizedBody))) { + response.setHeader("Content-Type", CONTENT_TYPE_JSON); + response.setStatus(400); + response.getWriter().write(getErrorMessages(ERROR_INVALID_CHARACTERS)); + return; + } + + filterChain.doFilter(cachedBodyHttpServletRequest, response); + } + + private static Predicate matchesEither( + final Pattern a, final Pattern b) { + return new Predicate() { + public boolean apply(String s) { + return a.matcher(s).matches() || b.matcher(s).matches(); + } + }; + } + + @Override + public void destroy() { + LOG.debug("AtlasXSSPreventionFilter destroyed"); + } + + + + private String getErrorMessages(String err) { + return "{\"code\":1000,\"error\":\"XSS\",\"message\":\"" + err + "\"}"; + } +} diff --git a/webapp/src/main/java/org/apache/atlas/web/filters/AuditFilter.java b/webapp/src/main/java/org/apache/atlas/web/filters/AuditFilter.java index 329d209a79f..d3cb21b94dc 100755 --- a/webapp/src/main/java/org/apache/atlas/web/filters/AuditFilter.java +++ b/webapp/src/main/java/org/apache/atlas/web/filters/AuditFilter.java @@ -18,11 +18,10 @@ package org.apache.atlas.web.filters; -import org.apache.atlas.AtlasClient; -import org.apache.atlas.AtlasException; -import org.apache.atlas.RequestContext; +import org.apache.atlas.*; import org.apache.atlas.authorize.AtlasAuthorizationUtils; -import org.apache.atlas.DeleteType; +import org.apache.atlas.service.metrics.MetricUtils; +import org.apache.atlas.service.metrics.MetricsRegistry; import org.apache.atlas.util.AtlasRepositoryConfiguration; import org.apache.atlas.web.util.DateTimeHelper; import org.apache.atlas.web.util.Servlets; @@ -30,8 +29,11 @@ import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.slf4j.MDC; import org.springframework.stereotype.Component; +import org.springframework.web.context.support.SpringBeanAutowiringSupport; +import javax.inject.Inject; import javax.servlet.Filter; import javax.servlet.FilterChain; import javax.servlet.FilterConfig; @@ -45,8 +47,9 @@ import java.util.Set; import java.util.UUID; +import static java.util.Optional.ofNullable; import static org.apache.atlas.AtlasConfiguration.*; - +import static org.apache.commons.lang.StringUtils.EMPTY; /** * This records audit information as part of the filter after processing the request * and also introduces a UUID into request and response for tracing requests in logs. @@ -55,17 +58,21 @@ public class AuditFilter implements Filter { private static final Logger LOG = LoggerFactory.getLogger(AuditFilter.class); private static final Logger AUDIT_LOG = LoggerFactory.getLogger("AUDIT"); - + public static final String TRACE_ID = "trace_id"; + public static final String X_ATLAN_REQUEST_ID = "X-Atlan-Request-Id"; private boolean deleteTypeOverrideEnabled = false; private boolean createShellEntityForNonExistingReference = false; + @Inject + private MetricsRegistry metricsRegistry; + @Override public void init(FilterConfig filterConfig) throws ServletException { LOG.info("AuditFilter initialization started"); deleteTypeOverrideEnabled = REST_API_ENABLE_DELETE_TYPE_OVERRIDE.getBoolean(); createShellEntityForNonExistingReference = REST_API_CREATE_SHELL_ENTITY_FOR_NON_EXISTING_REF.getBoolean(); - + SpringBeanAutowiringSupport.processInjectionBasedOnCurrentContext(this); LOG.info("REST_API_ENABLE_DELETE_TYPE_OVERRIDE={}", deleteTypeOverrideEnabled); } @@ -76,7 +83,7 @@ public void doFilter(ServletRequest request, ServletResponse response, FilterCha final Date requestTime = new Date(); final HttpServletRequest httpRequest = (HttpServletRequest) request; final HttpServletResponse httpResponse = (HttpServletResponse) response; - final String requestId = UUID.randomUUID().toString(); + final String internalRequestId = UUID.randomUUID().toString(); final Thread currentThread = Thread.currentThread(); final String oldName = currentThread.getName(); final String user = AtlasAuthorizationUtils.getCurrentUserName(); @@ -85,24 +92,33 @@ public void doFilter(ServletRequest request, ServletResponse response, FilterCha final boolean skipFailedEntities = Boolean.parseBoolean(httpRequest.getParameter("skipFailedEntities")); try { - currentThread.setName(formatName(oldName, requestId)); + currentThread.setName(formatName(oldName, internalRequestId)); RequestContext.clear(); RequestContext requestContext = RequestContext.get(); + requestContext.setUri(MetricUtils.matchCanonicalPattern(httpRequest.getRequestURI()).orElse(EMPTY)); + requestContext.setTraceId(internalRequestId); requestContext.setUser(user, userGroups); requestContext.setClientIPAddress(AtlasAuthorizationUtils.getRequestIpAddress(httpRequest)); requestContext.setCreateShellEntityForNonExistingReference(createShellEntityForNonExistingReference); requestContext.setForwardedAddresses(AtlasAuthorizationUtils.getForwardedAddressesFromRequest(httpRequest)); requestContext.setSkipFailedEntities(skipFailedEntities); - + requestContext.setMetricRegistry(metricsRegistry); + MDC.put(TRACE_ID, internalRequestId); + MDC.put(X_ATLAN_REQUEST_ID, ofNullable(httpRequest.getHeader(X_ATLAN_REQUEST_ID)).orElse(EMPTY)); if (StringUtils.isNotEmpty(deleteType)) { if (deleteTypeOverrideEnabled) { + if(DeleteType.PURGE.name().equals(deleteType)) { + requestContext.setPurgeRequested(true); + } requestContext.setDeleteType(DeleteType.from(deleteType)); } else { LOG.warn("Override of deleteType is not enabled. Ignoring parameter deleteType={}, in request from user={}", deleteType, user); } } + HeadersUtil.setRequestContextHeaders((HttpServletRequest)request); + filterChain.doFilter(request, response); } finally { long timeTaken = System.currentTimeMillis() - startTime; @@ -110,9 +126,11 @@ public void doFilter(ServletRequest request, ServletResponse response, FilterCha recordAudit(httpRequest, requestTime, user, httpResponse.getStatus(), timeTaken); // put the request id into the response so users can trace logs for this request - httpResponse.setHeader(AtlasClient.REQUEST_ID, requestId); + httpResponse.setHeader(TRACE_ID, internalRequestId); + httpResponse.setHeader(X_ATLAN_REQUEST_ID, MDC.get(X_ATLAN_REQUEST_ID)); currentThread.setName(oldName); RequestContext.clear(); + MDC.clear(); } } diff --git a/webapp/src/main/java/org/apache/atlas/web/filters/HeadersUtil.java b/webapp/src/main/java/org/apache/atlas/web/filters/HeadersUtil.java index 1f8845da6b2..8f48e8beab5 100644 --- a/webapp/src/main/java/org/apache/atlas/web/filters/HeadersUtil.java +++ b/webapp/src/main/java/org/apache/atlas/web/filters/HeadersUtil.java @@ -18,12 +18,16 @@ package org.apache.atlas.web.filters; import org.apache.atlas.AtlasConfiguration; +import org.apache.atlas.RequestContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; +import javax.servlet.http.HttpServletRequest; +import java.util.Enumeration; import java.util.HashMap; import java.util.Map; + @Component public class HeadersUtil { @@ -49,6 +53,9 @@ public class HeadersUtil { public static final String X_REQUESTED_WITH_VALUE = "XMLHttpRequest"; public static final int SC_AUTHENTICATION_TIMEOUT = 419; + private static final String ATLAN_HEADER_PREFIX_PATTERN = "x-atlan-"; + + HeadersUtil() { headerMap.put(X_FRAME_OPTIONS_KEY, X_FRAME_OPTIONS_VAL); headerMap.put(X_CONTENT_TYPE_OPTIONS_KEY, X_CONTENT_TYPE_OPTIONS_VAL); @@ -68,4 +75,17 @@ public static void setSecurityHeaders(AtlasResponseRequestWrapper responseWrappe responseWrapper.setHeader(entry.getKey(), entry.getValue()); } } + + public static void setRequestContextHeaders(HttpServletRequest request) { + Enumeration headerNames = request.getHeaderNames(); + RequestContext context = RequestContext.get(); + + while (headerNames.hasMoreElements()) { + String headerName = headerNames.nextElement(); + + if (headerName.startsWith(ATLAN_HEADER_PREFIX_PATTERN)) { + context.addRequestContextHeader(headerName, request.getHeader(headerName)); + } + } + } } diff --git a/webapp/src/main/java/org/apache/atlas/web/filters/MetricsFilter.java b/webapp/src/main/java/org/apache/atlas/web/filters/MetricsFilter.java new file mode 100644 index 00000000000..0e8a44f8518 --- /dev/null +++ b/webapp/src/main/java/org/apache/atlas/web/filters/MetricsFilter.java @@ -0,0 +1,40 @@ +package org.apache.atlas.web.filters; + +import io.micrometer.core.instrument.Timer; +import org.apache.atlas.service.metrics.MetricUtils; +import org.jetbrains.annotations.NotNull; +import org.springframework.stereotype.Component; +import org.springframework.web.context.support.SpringBeanAutowiringSupport; +import org.springframework.web.filter.OncePerRequestFilter; + +import javax.inject.Inject; +import javax.servlet.FilterChain; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; + +/** + * Filter used to record HTTP request & response metrics + */ +@Component +public class MetricsFilter extends OncePerRequestFilter { + + @Inject + private MetricUtils metricUtils; + + public MetricsFilter() { + SpringBeanAutowiringSupport.processInjectionBasedOnCurrentContext(this); + } + + @Override + protected void doFilterInternal(@NotNull HttpServletRequest request, @NotNull HttpServletResponse response, @NotNull FilterChain filterChain) throws ServletException, IOException { + Timer.Sample timerSample = null; + try { + timerSample = metricUtils.start(request.getRequestURI()); + filterChain.doFilter(request, response); + } finally { + metricUtils.recordHttpTimer(timerSample, request.getMethod(), request.getRequestURI(), response.getStatus()); + } + } +} diff --git a/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java b/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java index b9849763df3..3afd2b451d3 100755 --- a/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java +++ b/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java @@ -32,8 +32,6 @@ import org.apache.atlas.model.audit.AtlasAuditEntry; import org.apache.atlas.model.audit.AtlasAuditEntry.AuditOperation; import org.apache.atlas.model.audit.AuditSearchParameters; -import org.apache.atlas.model.audit.EntityAuditEventV2; -import org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditActionV2; import org.apache.atlas.model.impexp.AtlasExportRequest; import org.apache.atlas.model.impexp.AtlasExportResult; import org.apache.atlas.model.impexp.AtlasImportRequest; @@ -50,7 +48,6 @@ import org.apache.atlas.model.patches.AtlasPatch.AtlasPatches; import org.apache.atlas.model.tasks.AtlasTask; import org.apache.atlas.repository.audit.AtlasAuditService; -import org.apache.atlas.repository.audit.CassandraBasedAuditRepository; import org.apache.atlas.repository.impexp.AtlasServerService; import org.apache.atlas.repository.impexp.ExportImportAuditService; import org.apache.atlas.repository.impexp.ExportService; @@ -59,6 +56,7 @@ import org.apache.atlas.repository.impexp.ZipSink; import org.apache.atlas.repository.patches.AtlasPatchManager; import org.apache.atlas.repository.store.graph.AtlasEntityStore; +import org.apache.atlas.service.metrics.MetricsRegistry; import org.apache.atlas.services.MetricsService; import org.apache.atlas.tasks.TaskManagement; import org.apache.atlas.type.AtlasType; @@ -69,6 +67,7 @@ import org.apache.atlas.web.filters.AtlasCSRFPreventionFilter; import org.apache.atlas.web.service.ActiveInstanceElectorService; import org.apache.atlas.web.service.AtlasDebugMetricsSink; +import org.apache.atlas.web.service.AtlasHealthStatus; import org.apache.atlas.web.service.ServiceState; import org.apache.atlas.web.util.Servlets; import org.apache.commons.collections.CollectionUtils; @@ -109,7 +108,6 @@ import java.io.IOException; import java.io.InputStream; import java.security.SecureRandom; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.HashMap; @@ -122,6 +120,8 @@ import java.util.stream.Collectors; import java.util.*; +import static org.apache.atlas.AtlasErrorCode.DEPRECATED_API; +import static org.apache.atlas.AtlasErrorCode.DISABLED_API; import static org.apache.atlas.web.filters.AtlasCSRFPreventionFilter.CSRF_TOKEN; @@ -160,6 +160,9 @@ public class AdminResource { @Context private HttpServletResponse httpServletResponse; + @Inject + private AtlasHealthStatus atlasHealthStatus; + private Response version; private static Configuration atlasProperties; @@ -178,12 +181,14 @@ public class AdminResource { private final AtlasPatchManager patchManager; private final AtlasAuditService auditService; private final String defaultUIVersion; - private final CassandraBasedAuditRepository auditRepository; private final boolean isTimezoneFormatEnabled; private final String uiDateFormat; private final AtlasDebugMetricsSink debugMetricsRESTSink; private final boolean isDebugMetricsEnabled; private final boolean isTasksEnabled; + private final boolean isOnDemandLineageEnabled; + private final int defaultLineageNodeCount; + private final MetricsRegistry metricsRegistry; static { try { @@ -199,8 +204,8 @@ public AdminResource(ServiceState serviceState, MetricsService metricsService, A MigrationProgressService migrationProgressService, AtlasServerService serverService, ExportImportAuditService exportImportAuditService, AtlasEntityStore entityStore, - AtlasPatchManager patchManager, AtlasAuditService auditService, CassandraBasedAuditRepository auditRepository, - TaskManagement taskManagement, AtlasDebugMetricsSink debugMetricsRESTSink) { + AtlasPatchManager patchManager, AtlasAuditService auditService, + TaskManagement taskManagement, AtlasDebugMetricsSink debugMetricsRESTSink, MetricsRegistry metricsRegistry) { this.serviceState = serviceState; this.metricsService = metricsService; this.exportService = exportService; @@ -214,9 +219,9 @@ public AdminResource(ServiceState serviceState, MetricsService metricsService, A this.importExportOperationLock = new ReentrantLock(); this.patchManager = patchManager; this.auditService = auditService; - this.auditRepository = auditRepository; this.taskManagement = taskManagement; this.debugMetricsRESTSink = debugMetricsRESTSink; + this.metricsRegistry = metricsRegistry; if (atlasProperties != null) { this.defaultUIVersion = atlasProperties.getString(DEFAULT_UI_VERSION, UI_VERSION_V2); @@ -224,12 +229,16 @@ public AdminResource(ServiceState serviceState, MetricsService metricsService, A this.uiDateFormat = atlasProperties.getString(UI_DATE_FORMAT, UI_DATE_DEFAULT_FORMAT); this.isDebugMetricsEnabled = AtlasConfiguration.DEBUG_METRICS_ENABLED.getBoolean(); this.isTasksEnabled = AtlasConfiguration.TASKS_USE_ENABLED.getBoolean(); + this.isOnDemandLineageEnabled = AtlasConfiguration.LINEAGE_ON_DEMAND_ENABLED.getBoolean(); + this.defaultLineageNodeCount = AtlasConfiguration.LINEAGE_ON_DEMAND_DEFAULT_NODE_COUNT.getInt(); } else { this.defaultUIVersion = UI_VERSION_V2; this.isTimezoneFormatEnabled = true; this.uiDateFormat = UI_DATE_DEFAULT_FORMAT; this.isDebugMetricsEnabled = false; this.isTasksEnabled = false; + this.isOnDemandLineageEnabled = false; + this.defaultLineageNodeCount = 3; } } @@ -379,6 +388,8 @@ public Response getUserProfile(@Context HttpServletRequest request) { responseData.put(UI_DATE_FORMAT, uiDateFormat); responseData.put(AtlasConfiguration.DEBUG_METRICS_ENABLED.getPropertyName(), isDebugMetricsEnabled); responseData.put(AtlasConfiguration.TASKS_USE_ENABLED.getPropertyName(), isTasksEnabled); + responseData.put(AtlasConfiguration.LINEAGE_ON_DEMAND_ENABLED.getPropertyName(), isOnDemandLineageEnabled); + responseData.put(AtlasConfiguration.LINEAGE_ON_DEMAND_DEFAULT_NODE_COUNT.getPropertyName(), defaultLineageNodeCount); if (AtlasConfiguration.SESSION_TIMEOUT_SECS.getInt() != -1) { responseData.put(AtlasConfiguration.SESSION_TIMEOUT_SECS.getPropertyName(), AtlasConfiguration.SESSION_TIMEOUT_SECS.getInt()); @@ -415,6 +426,11 @@ public Response healthCheck() { boolean cassandraFailed = false; try { + List healthStatuses = atlasHealthStatus.getHealthStatuses(); + for (final HealthStatus healthStatus : healthStatuses) { + result.put(healthStatus.name, healthStatus); + } + GraphTraversal t = graph.V().limit(1); t.hasNext(); result.put("cassandra", new HealthStatus("cassandra", "ok", true, new Date().toString(), "")); @@ -427,7 +443,7 @@ public Response healthCheck() { LOG.debug("<== AdminResource.healthCheck()"); } - if (cassandraFailed) { + if (cassandraFailed || atlasHealthStatus.isAtleastOneComponentUnHealthy()) { return Response.status(500).entity(result).build(); } @@ -487,6 +503,24 @@ public AtlasMetrics getMetrics() { return metrics; } + @GET + @Path("metrics/prometheus") + public void scrapMetrics(@Context HttpServletResponse httpServletResponse) { + try { + if (LOG.isDebugEnabled()) { + LOG.debug("==> AdminResource.scrapMetrics()"); + } + this.metricsRegistry.scrape(httpServletResponse.getWriter()); + } catch (IOException e) { + //do nothing + LOG.error("Failed to scrap metrics for prometheus"); + } finally { + if (LOG.isDebugEnabled()) { + LOG.debug("<== AdminResource.scrapMetrics()"); + } + } + } + @GET @Path("pushMetricsToStatsd") @Produces(Servlets.JSON_MEDIA_TYPE) @@ -778,39 +812,7 @@ public List getAtlasAudits(AuditSearchParameters auditSearchPar public List getAuditDetails(@PathParam("auditGuid") String auditGuid, @QueryParam("limit") @DefaultValue("10") int limit, @QueryParam("offset") @DefaultValue("0") int offset) throws AtlasBaseException { - AtlasPerfTracer perf = null; - - try { - if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { - perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "AdminResource.getAuditDetails(" + auditGuid + ", " + limit + ", " + offset + ")"); - } - - List ret = new ArrayList<>(); - - AtlasAuditEntry auditEntry = auditService.toAtlasAuditEntry(entityStore.getById(auditGuid, false, true)); - - if(auditEntry != null && StringUtils.isNotEmpty(auditEntry.getResult())) { - String[] listOfResultGuid = auditEntry.getResult().split(","); - EntityAuditActionV2 auditAction = auditEntry.getOperation().toEntityAuditActionV2(); - - if(offset <= listOfResultGuid.length) { - for(int index=offset; index < listOfResultGuid.length && index < (offset + limit); index++) { - List events = auditRepository.listEventsV2(listOfResultGuid[index], auditAction, null, (short)1); - - for (EntityAuditEventV2 event : events) { - AtlasEntityHeader entityHeader = event.getEntityHeader(); - if(entityHeader != null) { - ret.add(entityHeader); - } - } - } - } - } - - return ret; - } finally { - AtlasPerfTracer.log(perf); - } + throw new AtlasBaseException(DEPRECATED_API, "/entity/auditSearch"); } @GET @@ -848,6 +850,25 @@ public AtlasCheckStateResult checkState(AtlasCheckStateRequest request) throws A } } + @POST + @Path("repairmeanings") + @Produces(Servlets.JSON_MEDIA_TYPE) + @Consumes(Servlets.JSON_MEDIA_TYPE) + public void repairmeanings(List termGuid) throws AtlasBaseException { + AtlasPerfTracer perf = null; + + try { + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "repairmeanings(" + termGuid + ")"); + } + + entityStore.repairMeaningAttributeForTerms(termGuid); + + } finally { + AtlasPerfTracer.log(perf); + } + } + @GET @Path("patches") @Produces(Servlets.JSON_MEDIA_TYPE) @@ -865,11 +886,32 @@ public AtlasPatches getAtlasPatches() { return ret; } + /* + * This returns all tasks + * Filtering support: Either filter by statusList or guids + * @param guids filter tasks with specified list of guids, If not specified, return all tasks, will not have any effect if statusList is specified + * @param statusList filter tasks with specified list of status, If not specified, return all tasks, will not have any effect if guids is specified + * guids takes preference + * */ @GET @Path("/tasks") @Produces(Servlets.JSON_MEDIA_TYPE) - public List getTaskStatus(@QueryParam("guids") List guids) throws AtlasBaseException { - return CollectionUtils.isNotEmpty(guids) ? taskManagement.getByGuids(guids) : taskManagement.getAll(); + public List getTaskStatus(@QueryParam("status") List statusList, @QueryParam("guids") List guids, + @QueryParam("offset") @DefaultValue("0") int offset, + @QueryParam("limit") @DefaultValue("20") int limit) throws AtlasBaseException { + return CollectionUtils.isNotEmpty(guids) ? taskManagement.getByGuids(guids) : taskManagement.getAll(statusList, offset, limit); + } + + /* + * Retry failed/ in_progress tasks on demand (for a very special case of cassandra went down) + * @Param taskGuids list of task guids to retry + * */ + @POST + @Path("/tasks/retry") + @Produces(Servlets.JSON_MEDIA_TYPE) + public void retryFailedTasks(@QueryParam("guid") List taskGuids) throws AtlasBaseException { + //taskManagement.retryTasks(taskGuids); + throw new AtlasBaseException(DISABLED_API, "META-2979: Limit tasks queue"); } @DELETE diff --git a/webapp/src/main/java/org/apache/atlas/web/resources/DataSetLineageResource.java b/webapp/src/main/java/org/apache/atlas/web/resources/DataSetLineageResource.java deleted file mode 100644 index 5660c5bcb74..00000000000 --- a/webapp/src/main/java/org/apache/atlas/web/resources/DataSetLineageResource.java +++ /dev/null @@ -1,224 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.web.resources; - -import org.apache.atlas.AtlasErrorCode; -import org.apache.atlas.discovery.AtlasLineageService; -import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.model.instance.AtlasEntity; -import org.apache.atlas.model.lineage.AtlasLineageInfo; -import org.apache.atlas.model.lineage.AtlasLineageInfo.LineageDirection; -import org.apache.atlas.repository.store.graph.AtlasEntityStore; -import org.apache.atlas.type.AtlasEntityType; -import org.apache.atlas.type.AtlasTypeRegistry; -import org.apache.atlas.utils.AtlasPerfTracer; -import org.apache.atlas.v1.model.lineage.DataSetLineageResponse; -import org.apache.atlas.v1.model.lineage.SchemaResponse; -import org.apache.atlas.web.util.LineageUtils; -import org.apache.atlas.web.util.Servlets; -import org.apache.commons.lang.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Service; - -import javax.inject.Inject; -import javax.inject.Singleton; -import javax.servlet.http.HttpServletRequest; -import javax.ws.rs.Consumes; -import javax.ws.rs.GET; -import javax.ws.rs.Path; -import javax.ws.rs.PathParam; -import javax.ws.rs.Produces; -import javax.ws.rs.WebApplicationException; -import javax.ws.rs.core.Context; -import javax.ws.rs.core.Response; -import java.util.HashMap; -import java.util.Map; - -import static org.apache.atlas.v1.model.lineage.SchemaResponse.SchemaDetails; - -/** - * Jersey Resource for Hive Table Lineage. - */ -@Path("lineage/hive") -@Singleton -@Service -@Deprecated -public class DataSetLineageResource { - - private static final Logger LOG = LoggerFactory.getLogger(DataSetLineageResource.class); - private static final Logger PERF_LOG = AtlasPerfTracer.getPerfLogger("rest.DataSetLineageResource"); - - private final AtlasLineageService atlasLineageService; - private final AtlasTypeRegistry typeRegistry; - private final AtlasEntityStore atlasEntityStore; - - @Inject - public DataSetLineageResource(final AtlasLineageService atlasLineageService, final AtlasTypeRegistry typeRegistry, final AtlasEntityStore atlasEntityStore) { - this.atlasLineageService = atlasLineageService; - this.typeRegistry = typeRegistry; - this.atlasEntityStore = atlasEntityStore; - } - - /** - * Returns the inputs graph for a given entity. - * - * @param tableName table name - */ - @GET - @Path("table/{tableName}/inputs/graph") - @Consumes(Servlets.JSON_MEDIA_TYPE) - @Produces(Servlets.JSON_MEDIA_TYPE) - public DataSetLineageResponse inputsGraph(@Context HttpServletRequest request, @PathParam("tableName") String tableName) { - if (LOG.isDebugEnabled()) { - LOG.debug("==> DataSetLineageResource.inputsGraph({})", tableName); - } - - DataSetLineageResponse ret = new DataSetLineageResponse(); - AtlasPerfTracer perf = null; - - try { - if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { - perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "DataSetLineageResource.inputsGraph(tableName=" + tableName + ")"); - } - - String guid = getGuid(tableName); - - AtlasLineageInfo lineageInfo = atlasLineageService.getAtlasLineageInfo(guid, LineageDirection.INPUT, -1); - ret.setTableName(tableName); - ret.setRequestId(Servlets.getRequestId()); - ret.setResults(LineageUtils.toLineageStruct(lineageInfo, typeRegistry)); - - return ret; - } catch (IllegalArgumentException e) { - LOG.error("Unable to get lineage inputs graph for table {}", tableName, e); - throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST)); - } catch (WebApplicationException e) { - LOG.error("Unable to get lineage inputs graph for table {}", tableName, e); - throw e; - } catch (Throwable e) { - LOG.error("Unable to get lineage inputs graph for table {}", tableName, e); - throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); - } finally { - AtlasPerfTracer.log(perf); - } - } - - /** - * Returns the outputs graph for a given entity. - * - * @param tableName table name - */ - @GET - @Path("table/{tableName}/outputs/graph") - @Consumes(Servlets.JSON_MEDIA_TYPE) - @Produces(Servlets.JSON_MEDIA_TYPE) - public DataSetLineageResponse outputsGraph(@Context HttpServletRequest request, @PathParam("tableName") String tableName) { - if (LOG.isDebugEnabled()) { - LOG.debug("==> DataSetLineageResource.outputsGraph({})", tableName); - } - - DataSetLineageResponse ret = new DataSetLineageResponse(); - AtlasPerfTracer perf = null; - - try { - if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { - perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "DataSetLineageResource.outputsGraph(tableName=" + tableName + ")"); - } - - String guid = getGuid(tableName); - - AtlasLineageInfo lineageInfo = atlasLineageService.getAtlasLineageInfo(guid, LineageDirection.OUTPUT, -1); - ret.setTableName(tableName); - ret.setRequestId(Servlets.getRequestId()); - ret.setResults(LineageUtils.toLineageStruct(lineageInfo, typeRegistry)); - - return ret; - } catch (IllegalArgumentException e) { - LOG.error("Unable to get lineage outputs graph for table {}", tableName, e); - throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST)); - } catch (WebApplicationException e) { - LOG.error("Unable to get lineage outputs graph for table {}", tableName, e); - throw e; - } catch (Throwable e) { - LOG.error("Unable to get lineage outputs graph for table {}", tableName, e); - throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); - } finally { - AtlasPerfTracer.log(perf); - } - } - - /** - * Return the schema for the given tableName. - * - * @param tableName table name - */ - @GET - @Path("table/{tableName}/schema") - @Consumes(Servlets.JSON_MEDIA_TYPE) - @Produces(Servlets.JSON_MEDIA_TYPE) - public SchemaResponse schema(@Context HttpServletRequest request, @PathParam("tableName") String tableName) { - if (LOG.isDebugEnabled()) { - LOG.debug("==> DataSetLineageResource.schema({})", tableName); - } - - AtlasPerfTracer perf = null; - SchemaResponse ret = new SchemaResponse(); - - try { - if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { - perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "DataSetLineageResource.schema(tableName=" + tableName + ")"); - } - - SchemaDetails schemaDetails = atlasLineageService.getSchemaForHiveTableByName(tableName); - - ret.setRequestId(Servlets.getRequestId()); - ret.setTableName(tableName); - ret.setResults(schemaDetails); - return ret; - } catch (IllegalArgumentException e) { - LOG.error("Unable to get schema for table {}", tableName, e); - throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST)); - } catch (WebApplicationException e) { - LOG.error("Unable to get schema for table {}", tableName, e); - throw e; - } catch (Throwable e) { - LOG.error("Unable to get schema for table {}", tableName, e); - throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); - } finally { - AtlasPerfTracer.log(perf); - } - } - - private String getGuid(String tableName) throws AtlasBaseException { - if (StringUtils.isEmpty(tableName)) { - // TODO: Fix the error code if mismatch - throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST); - } - Map lookupAttributes = new HashMap<>(); - lookupAttributes.put("qualifiedName", tableName); - AtlasEntityType entityType = typeRegistry.getEntityTypeByName("hive_table"); - AtlasEntity.AtlasEntityWithExtInfo hive_table = atlasEntityStore.getByUniqueAttributes(entityType, lookupAttributes); - if (hive_table != null) { - return hive_table.getEntity().getGuid(); - } else { - throw new AtlasBaseException(AtlasErrorCode.INSTANCE_NOT_FOUND, tableName); - } - } -} diff --git a/webapp/src/main/java/org/apache/atlas/web/resources/EntityResource.java b/webapp/src/main/java/org/apache/atlas/web/resources/EntityResource.java deleted file mode 100755 index 60610bdc27e..00000000000 --- a/webapp/src/main/java/org/apache/atlas/web/resources/EntityResource.java +++ /dev/null @@ -1,1206 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.web.resources; - -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import org.apache.atlas.AtlasClient; -import org.apache.atlas.AtlasConstants; -import org.apache.atlas.AtlasErrorCode; -import org.apache.atlas.AtlasException; -import org.apache.atlas.CreateUpdateEntitiesResult; -import org.apache.atlas.EntityAuditEvent; -import org.apache.atlas.model.audit.EntityAuditEventV2; -import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.model.instance.AtlasClassification; -import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo; -import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo; -import org.apache.atlas.model.instance.EntityMutationResponse; -import org.apache.atlas.model.instance.GuidMapping; -import org.apache.atlas.model.legacy.EntityResult; -import org.apache.atlas.repository.audit.CassandraBasedAuditRepository; -import org.apache.atlas.repository.converters.AtlasInstanceConverter; -import org.apache.atlas.repository.store.graph.AtlasEntityStore; -import org.apache.atlas.repository.store.graph.v2.AtlasEntityStream; -import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2; -import org.apache.atlas.type.AtlasEntityType; -import org.apache.atlas.type.AtlasType; -import org.apache.atlas.type.AtlasTypeRegistry; -import org.apache.atlas.type.AtlasTypeUtil; -import org.apache.atlas.utils.AtlasJson; -import org.apache.atlas.utils.AtlasPerfTracer; -import org.apache.atlas.utils.ParamChecker; -import org.apache.atlas.v1.model.instance.Id; -import org.apache.atlas.v1.model.instance.Referenceable; -import org.apache.atlas.v1.model.instance.Struct; -import org.apache.atlas.web.rest.EntityREST; -import org.apache.atlas.web.util.Servlets; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Service; - -import javax.inject.Inject; -import javax.inject.Singleton; -import javax.servlet.http.HttpServletRequest; -import javax.ws.rs.*; -import javax.ws.rs.core.Context; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; -import javax.ws.rs.core.UriBuilder; -import javax.ws.rs.core.UriInfo; -import java.io.IOException; -import java.net.URI; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - - -/** - * Entity management operations as REST API. - * - * An entity is an "instance" of a Type. Entities conform to the definition - * of the Type they correspond with. - */ -@Singleton -@Path("entities") -@Service -@Deprecated -public class EntityResource { - - private static final Logger LOG = LoggerFactory.getLogger(EntityResource.class); - private static final Logger PERF_LOG = AtlasPerfTracer.getPerfLogger("rest.EntityResource"); - - private static final String TRAIT_NAME = "traitName"; - - private final AtlasInstanceConverter restAdapters; - private final AtlasEntityStore entitiesStore; - private final AtlasTypeRegistry typeRegistry; - private final EntityREST entityREST; - private final CassandraBasedAuditRepository entityAuditRepository; - private final AtlasInstanceConverter instanceConverter; - - @Context - UriInfo uriInfo; - - @Inject - public EntityResource(final AtlasInstanceConverter restAdapters, - final AtlasEntityStore entitiesStore, - final AtlasTypeRegistry typeRegistry, - final EntityREST entityREST, - final CassandraBasedAuditRepository entityAuditRepository, - final AtlasInstanceConverter instanceConverter) { - this.restAdapters = restAdapters; - this.entitiesStore = entitiesStore; - this.typeRegistry = typeRegistry; - this.entityREST = entityREST; - this.entityAuditRepository = entityAuditRepository; - this.instanceConverter = instanceConverter; - } - - /** - * Submits the entity definitions (instances). - * The body contains the JSONArray of entity json. The service takes care of de-duping the entities based on any - * unique attribute for the give type. - */ - @POST - @Consumes({Servlets.JSON_MEDIA_TYPE, MediaType.APPLICATION_JSON}) - @Produces(Servlets.JSON_MEDIA_TYPE) - public Response submit(@Context HttpServletRequest request) { - if (LOG.isDebugEnabled()) { - LOG.debug("==> EntityResource.submit()"); - } - - String entityJson = null; - AtlasPerfTracer perf = null; - try { - if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { - perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "EntityResource.submit()"); - } - - entityJson = Servlets.getRequestPayload(request); - - //Handle backward compatibility - if entities is not JSONArray, convert to JSONArray - String[] jsonStrings; - - try { - ArrayNode jsonEntities = AtlasJson.parseToV1ArrayNode(entityJson); - - jsonStrings = new String[jsonEntities.size()]; - - for (int i = 0; i < jsonEntities.size(); i++) { - jsonStrings[i] = AtlasJson.toV1Json(jsonEntities.get(i)); - } - } catch (IOException e) { - jsonStrings = new String[1]; - - jsonStrings[0] = entityJson; - } - - if (LOG.isDebugEnabled()) { - LOG.debug("submitting entities: count={}; entities-json={}", jsonStrings.length, entityJson); - } - - AtlasEntitiesWithExtInfo entitiesInfo = restAdapters.toAtlasEntities(jsonStrings); - EntityMutationResponse mutationResponse = entityREST.createOrUpdate(entitiesInfo, false, false); - - final List guids = restAdapters.getGuids(mutationResponse.getCreatedEntities()); - - if (LOG.isDebugEnabled()) { - LOG.debug("Created entities {}", guids); - } - - final CreateUpdateEntitiesResult result = restAdapters.toCreateUpdateEntitiesResult(mutationResponse); - - String response = getResponse(result); - URI locationURI = getLocationURI(guids); - - return Response.created(locationURI).entity(response).build(); - - } catch (AtlasBaseException e) { - LOG.error("Unable to persist entity instance entityDef={}", entityJson, e); - throw toWebApplicationException(e); - } catch (AtlasException | IllegalArgumentException e) { - LOG.error("Unable to persist entity instance entityDef={}", entityJson, e); - throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST)); - } catch (WebApplicationException e) { - LOG.error("Unable to persist entity instance entityDef={}", entityJson, e); - throw e; - } catch (Throwable e) { - LOG.error("Unable to persist entity instance entityDef={}", entityJson, e); - throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); - } finally { - AtlasPerfTracer.log(perf); - - if (LOG.isDebugEnabled()) { - LOG.debug("<== EntityResource.submit()"); - } - - } - } - - @VisibleForTesting - public URI getLocationURI(List guids) { - URI locationURI = null; - if (uriInfo != null) { - UriBuilder ub = uriInfo.getAbsolutePathBuilder(); - locationURI = CollectionUtils.isEmpty(guids) ? null : ub.path(guids.get(0)).build(); - } else { - String uriPath = AtlasClient.API_V1.GET_ENTITY.getNormalizedPath(); - locationURI = guids.isEmpty() ? null : UriBuilder - .fromPath(AtlasConstants.DEFAULT_ATLAS_REST_ADDRESS) - .path(uriPath).path(guids.get(0)).build(); - - } - return locationURI; - } - - private String getResponse(EntityResult entityResult) throws AtlasBaseException, AtlasException { - CreateUpdateEntitiesResult result = new CreateUpdateEntitiesResult(); - result.setEntityResult(entityResult); - return getResponse(result); - - } - private String getResponse(CreateUpdateEntitiesResult result) throws AtlasBaseException, AtlasException { - Map response = new HashMap<>(); - EntityResult entityResult = result.getEntityResult(); - GuidMapping mapping = result.getGuidMapping(); - - response.put(AtlasClient.REQUEST_ID, Servlets.getRequestId()); - - if(entityResult != null) { - response.put(AtlasClient.ENTITIES, entityResult.getEntities()); - - String sampleEntityId = getSample(result.getEntityResult()); - if (sampleEntityId != null) { - response.put(AtlasClient.DEFINITION, getEntity(sampleEntityId)); - } - } - - if(mapping != null) { - response.put(AtlasClient.GUID_ASSIGNMENTS, mapping); - } - return AtlasJson.toV1Json(response); - } - - /** - * Complete update of a set of entities - the values not specified will be replaced with null/removed - * Adds/Updates given entities identified by its GUID or unique attribute - * @return response payload as json - */ - @PUT - @Consumes({Servlets.JSON_MEDIA_TYPE, MediaType.APPLICATION_JSON}) - @Produces(Servlets.JSON_MEDIA_TYPE) - public Response updateEntities(@Context HttpServletRequest request) { - if (LOG.isDebugEnabled()) { - LOG.debug("==> EntityResource.updateEntities()"); - } - - String entityJson = null; - AtlasPerfTracer perf = null; - try { - if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { - perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "EntityResource.updateEntities()"); - } - - entityJson = Servlets.getRequestPayload(request); - - //Handle backward compatibility - if entities is not JSONArray, convert to JSONArray - String[] jsonStrings; - - try { - ArrayNode jsonEntities = AtlasJson.parseToV1ArrayNode(entityJson); - - jsonStrings = new String[jsonEntities.size()]; - - for (int i = 0; i < jsonEntities.size(); i++) { - jsonStrings[i] = AtlasJson.toV1Json(jsonEntities.get(i)); - } - } catch (IOException e) { - jsonStrings = new String[1]; - - jsonStrings[0] = entityJson; - } - - if (LOG.isDebugEnabled()) { - LOG.debug("Updating entities: count={}; entities-json={}", jsonStrings.length, entityJson); - } - - AtlasEntitiesWithExtInfo entitiesInfo = restAdapters.toAtlasEntities(jsonStrings); - EntityMutationResponse mutationResponse = entityREST.createOrUpdate(entitiesInfo, false, false); - CreateUpdateEntitiesResult result = restAdapters.toCreateUpdateEntitiesResult(mutationResponse); - - if (LOG.isDebugEnabled()) { - LOG.debug("Updated entities: {}", result.getEntityResult()); - } - - String response = getResponse(result); - return Response.ok(response).build(); - } catch (AtlasBaseException e) { - LOG.error("Unable to persist entity instance entityDef={}", entityJson, e); - throw toWebApplicationException(e); - } catch (AtlasException | IllegalArgumentException e) { - LOG.error("Unable to persist entity instance entityDef={}", entityJson, e); - throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST)); - } catch (WebApplicationException e) { - LOG.error("Unable to persist entity instance entityDef={}", entityJson, e); - throw e; - } catch (Throwable e) { - LOG.error("Unable to persist entity instance entityDef={}", entityJson, e); - throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); - } finally { - AtlasPerfTracer.log(perf); - - if (LOG.isDebugEnabled()) { - LOG.debug("<== EntityResource.updateEntities()"); - } - } - } - - private String getSample(EntityResult entityResult) { - String sample = getSample(entityResult.getCreatedEntities()); - if (sample == null) { - sample = getSample(entityResult.getUpdateEntities()); - } - return sample; - } - - - private String getSample(List list) { - if (list != null && list.size() > 0) { - return list.get(0); - } - return null; - } - - /** - * Adds/Updates given entity identified by its unique attribute( entityType, attributeName and value) - * Updates support only partial update of an entity - Adds/updates any new values specified - * Updates do not support removal of attribute values - * - * @param entityType the entity type - * @param attribute the unique attribute used to identify the entity - * @param value the unique attributes value - * @param request The updated entity json - * @return response payload as json - * The body contains the JSONArray of entity json. The service takes care of de-duping the entities based on any - * unique attribute for the give type. - */ - @POST - @Path("qualifiedName") - @Consumes({Servlets.JSON_MEDIA_TYPE, MediaType.APPLICATION_JSON}) - @Produces(Servlets.JSON_MEDIA_TYPE) - public Response updateByUniqueAttribute(@QueryParam("type") String entityType, - @QueryParam("property") String attribute, - @QueryParam("value") String value, @Context HttpServletRequest request) { - if (LOG.isDebugEnabled()) { - LOG.debug("==> EntityResource.updateByUniqueAttribute({}, {}, {})", entityType, attribute, value); - } - - AtlasPerfTracer perf = null; - String entityJson = null; - try { - if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { - perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "EntityResource.updateByUniqueAttribute(" + entityType + ", " + attribute + ", " + value + ")"); - } - - entityJson = Servlets.getRequestPayload(request); - - if (LOG.isDebugEnabled()) { - LOG.debug("Partially updating entity by unique attribute {} {} {} {} ", entityType, attribute, value, entityJson); - } - - Referenceable updatedEntity = AtlasType.fromV1Json(entityJson, Referenceable.class); - - entityType = ParamChecker.notEmpty(entityType, "Entity type cannot be null"); - attribute = ParamChecker.notEmpty(attribute, "attribute name cannot be null"); - value = ParamChecker.notEmpty(value, "attribute value cannot be null"); - - Map attributes = new HashMap<>(); - attributes.put(attribute, value); - - // update referenceable with Id if not specified in payload - Id updateId = updatedEntity.getId(); - - if (updateId != null && !AtlasTypeUtil.isAssignedGuid(updateId.getId())) { - String guid = AtlasGraphUtilsV2.getGuidByUniqueAttributes(getEntityType(entityType), attributes); - - updatedEntity.setId(new Id(guid, 0, updatedEntity.getTypeName())); - } - - AtlasEntitiesWithExtInfo entitiesInfo = restAdapters.toAtlasEntity(updatedEntity); - EntityMutationResponse mutationResponse = entitiesStore.createOrUpdate(new AtlasEntityStream(entitiesInfo), true); - CreateUpdateEntitiesResult result = restAdapters.toCreateUpdateEntitiesResult(mutationResponse); - - if (LOG.isDebugEnabled()) { - LOG.debug("Updated entities: {}", result.getEntityResult()); - } - - String response = getResponse(result); - return Response.ok(response).build(); - } catch (AtlasBaseException e) { - LOG.error("Unable to partially update entity {} {}:{}.{}", entityJson, entityType, attribute, value, e); - throw toWebApplicationException(e); - } catch (AtlasException | IllegalArgumentException e) { - LOG.error("Unable to partially update entity {} {}:{}.{}", entityJson, entityType, attribute, value, e); - throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST)); - } catch (WebApplicationException e) { - LOG.error("Unable to partially update entity {} {}:{}.{}", entityJson, entityType, attribute, value, e); - throw e; - } catch (Throwable e) { - LOG.error("Unable to partially update entity {} {}:{}.{}", entityJson, entityType, attribute, value, e); - throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); - } finally { - AtlasPerfTracer.log(perf); - - if (LOG.isDebugEnabled()) { - LOG.debug("<== EntityResource.updateByUniqueAttribute({}, {}, {})", entityType, attribute, value); - } - } - } - - /** - * Updates entity identified by its GUID - * Support Partial update of an entity - Adds/updates any new values specified - * Does not support removal of attribute values - * - * @param guid - * @param request The updated entity json - * @return - */ - @POST - @Path("{guid}") - @Consumes({Servlets.JSON_MEDIA_TYPE, MediaType.APPLICATION_JSON}) - @Produces(Servlets.JSON_MEDIA_TYPE) - public Response updateEntityByGuid(@PathParam("guid") String guid, @QueryParam("property") String attribute, - @Context HttpServletRequest request) { - if (LOG.isDebugEnabled()) { - LOG.debug("==> EntityResource.updateEntityByGuid({}, {})", guid, attribute); - } - - AtlasPerfTracer perf = null; - try { - if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { - perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "EntityResource.updateEntityByGuid(" + guid + ", " + attribute + ")"); - } - - if (StringUtils.isEmpty(attribute)) { - return partialUpdateEntityByGuid(guid, request); - } else { - return partialUpdateEntityAttrByGuid(guid, attribute, request); - } - } finally { - AtlasPerfTracer.log(perf); - - if (LOG.isDebugEnabled()) { - LOG.debug("<== EntityResource.updateEntityByGuid({}, {})", guid, attribute); - } - } - } - - private Response partialUpdateEntityByGuid(String guid, HttpServletRequest request) { - String entityJson = null; - try { - guid = ParamChecker.notEmpty(guid, "Guid property cannot be null"); - entityJson = Servlets.getRequestPayload(request); - - if (LOG.isDebugEnabled()) { - LOG.debug("partially updating entity for guid {} : {} ", guid, entityJson); - } - - Referenceable updatedEntity = AtlasType.fromV1Json(entityJson, Referenceable.class); - - // update referenceable with Id if not specified in payload - Id updateId = updatedEntity.getId(); - - if (updateId != null && !AtlasTypeUtil.isAssignedGuid(updateId.getId())) { - updatedEntity.setId(new Id(guid, 0, updatedEntity.getTypeName())); - } - - AtlasEntitiesWithExtInfo entitiesInfo = restAdapters.toAtlasEntity(updatedEntity); - EntityMutationResponse mutationResponse = entitiesStore.createOrUpdate(new AtlasEntityStream(entitiesInfo), true); - CreateUpdateEntitiesResult result = restAdapters.toCreateUpdateEntitiesResult(mutationResponse); - - if (LOG.isDebugEnabled()) { - LOG.debug("Updated entities: {}", result.getEntityResult()); - } - - String response = getResponse(result); - return Response.ok(response).build(); - } catch (AtlasBaseException e) { - LOG.error("Unable to update entity by GUID {} {} ", guid, entityJson, e); - throw toWebApplicationException(e); - } catch (AtlasException | IllegalArgumentException e) { - LOG.error("Unable to update entity by GUID {} {}", guid, entityJson, e); - throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST)); - } catch (WebApplicationException e) { - LOG.error("Unable to update entity by GUID {} {} ", guid, entityJson, e); - throw e; - } catch (Throwable e) { - LOG.error("Unable to update entity by GUID {} {} ", guid, entityJson, e); - throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); - } - } - - /** - * Supports Partial updates - * Adds/Updates given entity specified by its GUID - * Supports updation of only simple primitive attributes like strings, ints, floats, enums, class references and - * does not support updation of complex types like arrays, maps - * @param guid entity id - * @param property property to add - * @postbody property's value - * @return response payload as json - */ - private Response partialUpdateEntityAttrByGuid(String guid, String property, HttpServletRequest request) { - String value = null; - try { - Preconditions.checkNotNull(property, "Entity property cannot be null"); - value = Servlets.getRequestPayload(request); - Preconditions.checkNotNull(value, "Entity value cannot be null"); - - if (LOG.isDebugEnabled()) { - LOG.debug("Updating entity {} for property {} = {}", guid, property, value); - } - - EntityMutationResponse mutationResponse = entitiesStore.updateEntityAttributeByGuid(guid, property, value); - CreateUpdateEntitiesResult result = restAdapters.toCreateUpdateEntitiesResult(mutationResponse); - - if (LOG.isDebugEnabled()) { - LOG.debug("Updated entities: {}", result.getEntityResult()); - } - - String response = getResponse(result); - return Response.ok(response).build(); - } catch (AtlasBaseException e) { - LOG.error("Unable to add property {} to entity id {} {} ", property, guid, value, e); - throw toWebApplicationException(e); - } catch (AtlasException | IllegalArgumentException e) { - LOG.error("Unable to add property {} to entity id {} {} ", property, guid, value, e); - throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST)); - } catch (WebApplicationException e) { - LOG.error("Unable to add property {} to entity id {} {} ", property, guid, value, e); - throw e; - } catch (Throwable e) { - LOG.error("Unable to add property {} to entity id {} {} ", property, guid, value, e); - throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); - } - } - - /** - * Delete entities from the repository identified by their guids (including their composite references) - * or - * Deletes a single entity identified by its type and unique attribute value from the repository (including their composite references) - * - * @param guids list of deletion candidate guids - * or - * @param entityType the entity type - * @param attribute the unique attribute used to identify the entity - * @param value the unique attribute value used to identify the entity - * @return response payload as json - including guids of entities(including composite references from that entity) that were deleted - */ - @DELETE - @Produces(Servlets.JSON_MEDIA_TYPE) - public Response deleteEntities(@QueryParam("guid") List guids, - @QueryParam("type") String entityType, - @QueryParam("property") final String attribute, - @QueryParam("value") final String value) { - if (LOG.isDebugEnabled()) { - LOG.debug("==> EntityResource.deleteEntities({}, {}, {}, {})", guids, entityType, attribute, value); - } - - AtlasPerfTracer perf = null; - try { - if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { - perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "EntityResource.deleteEntities(" + guids + ", " + entityType + ", " + attribute + ", " + value + ")"); - } - - EntityResult entityResult; - - if (guids != null && !guids.isEmpty()) { - if (LOG.isDebugEnabled()) { - LOG.debug("Deleting entities {}", guids); - } - - EntityMutationResponse mutationResponse = entityREST.deleteByGuids(guids); - entityResult = restAdapters.toCreateUpdateEntitiesResult(mutationResponse).getEntityResult(); - } else { - if (LOG.isDebugEnabled()) { - LOG.debug("Deleting entity type={} with property {}={}", entityType, attribute, value); - } - - Map attributes = new HashMap<>(); - attributes.put(attribute, value); - - EntityMutationResponse mutationResponse = entitiesStore.deleteByUniqueAttributes(getEntityType(entityType), attributes); - entityResult = restAdapters.toCreateUpdateEntitiesResult(mutationResponse).getEntityResult(); - } - - if (LOG.isDebugEnabled()) { - LOG.debug("Deleted entity result: {}", entityResult); - } - - String response = getResponse(entityResult); - return Response.ok(response).build(); - } catch (AtlasBaseException e) { - LOG.error("Unable to delete entities {} {} {} {} ", guids, entityType, attribute, value, e); - throw toWebApplicationException(e); - } catch (AtlasException | IllegalArgumentException e) { - LOG.error("Unable to delete entities {} {} {} {} ", guids, entityType, attribute, value, e); - throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST)); - } catch (WebApplicationException e) { - LOG.error("Unable to delete entities {} {} {} {} ", guids, entityType, attribute, value, e); - throw e; - } catch (Throwable e) { - LOG.error("Unable to delete entities {} {} {} {} ", guids, entityType, attribute, value, e); - throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); - } finally { - AtlasPerfTracer.log(perf); - - if (LOG.isDebugEnabled()) { - LOG.debug("<== EntityResource.deleteEntities({}, {}, {}, {})", guids, entityType, attribute, value); - } - } - } - - /** - * Fetch the complete definition of an entity given its GUID. - * - * @param guid GUID for the entity - */ - @GET - @Path("{guid}") - @Produces(Servlets.JSON_MEDIA_TYPE) - public Response getEntityDefinition(@PathParam("guid") String guid) { - if (LOG.isDebugEnabled()) { - LOG.debug("==> EntityResource.getEntityDefinition({})", guid); - } - - AtlasPerfTracer perf = null; - try { - if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { - perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "EntityResource.getEntityDefinition(" + guid + ")"); - } - - if (LOG.isDebugEnabled()) { - LOG.debug("Fetching entity definition for guid={} ", guid); - } - - guid = ParamChecker.notEmpty(guid, "guid cannot be null"); - - Referenceable entity = getEntity(guid); - - Map response = new HashMap<>(); - - response.put(AtlasClient.REQUEST_ID, Servlets.getRequestId()); - - Response.Status status = Response.Status.NOT_FOUND; - if (entity != null) { - response.put(AtlasClient.DEFINITION, entity); - status = Response.Status.OK; - } else { - response.put(AtlasClient.ERROR, - Servlets.escapeJsonString(String.format("An entity with GUID={%s} does not exist", guid))); - } - - return Response.status(status).entity(AtlasJson.toV1Json(response)).build(); - } catch (IllegalArgumentException e) { - LOG.error("Bad GUID={} ", guid, e); - throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST)); - } catch (AtlasBaseException e) { - LOG.error("Unable to get instance definition for GUID {}", guid, e); - throw toWebApplicationException(e); - } catch (WebApplicationException e) { - LOG.error("Unable to get instance definition for GUID {}", guid, e); - throw e; - } catch (Throwable e) { - LOG.error("Unable to get instance definition for GUID {}", guid, e); - throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); - } finally { - AtlasPerfTracer.log(perf); - - if (LOG.isDebugEnabled()) { - LOG.debug("<== EntityResource.getEntityDefinition({})", guid); - } - } - } - - /** - * Gets the list of entities for a given entity type. - * - * @param entityType name of a type which is unique - */ - public Response getEntityListByType(String entityType) { - try { - Preconditions.checkNotNull(entityType, "Entity type cannot be null"); - - if (LOG.isDebugEnabled()) { - LOG.debug("Fetching entity list for type={} ", entityType); - } - - List entityGUIDS = entitiesStore.getEntityGUIDS(entityType); - - Map response = new HashMap<>(); - response.put(AtlasClient.REQUEST_ID, Servlets.getRequestId()); - response.put(AtlasClient.TYPENAME, entityType); - response.put(AtlasClient.RESULTS, entityGUIDS); - response.put(AtlasClient.COUNT, entityGUIDS.size()); - - return Response.ok(AtlasJson.toV1Json(response)).build(); - } catch (NullPointerException e) { - LOG.error("Entity type cannot be null", e); - throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST)); - } catch (IllegalArgumentException e) { - LOG.error("Unable to get entity list for type {}", entityType, e); - throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST)); - } catch (WebApplicationException e) { - LOG.error("Unable to get entity list for type {}", entityType, e); - throw e; - } catch (AtlasBaseException e) { - LOG.error("Unable to get entity list for type {}", entityType, e); - throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST)); - } catch (Throwable e) { - LOG.error("Unable to get entity list for type {}", entityType, e); - throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); - } - } - - @GET - @Consumes({Servlets.JSON_MEDIA_TYPE, MediaType.APPLICATION_JSON}) - @Produces(Servlets.JSON_MEDIA_TYPE) - public Response getEntity(@QueryParam("type") String entityType, - @QueryParam("property") String attribute, - @QueryParam("value") final String value) { - if (LOG.isDebugEnabled()) { - LOG.debug("==> EntityResource.getEntity({}, {}, {})", entityType, attribute, value); - } - - AtlasPerfTracer perf = null; - try { - if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { - perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "EntityResource.getEntity(" + entityType + ", " + attribute + ", " + value + ")"); - } - - if (StringUtils.isEmpty(attribute)) { - //List API - return getEntityListByType(entityType); - } else { - //Get entity by unique attribute - return getEntityDefinitionByAttribute(entityType, attribute, value); - } - } finally { - AtlasPerfTracer.log(perf); - - if (LOG.isDebugEnabled()) { - LOG.debug("<== EntityResource.getEntity({}, {}, {})", entityType, attribute, value); - } - } - } - - /** - * Fetch the complete definition of an entity given its qualified name. - * - * @param entityType - * @param attribute - * @param value - */ - public Response getEntityDefinitionByAttribute(String entityType, String attribute, String value) { - try { - if (LOG.isDebugEnabled()) { - LOG.debug("Fetching entity definition for type={}, qualified name={}", entityType, value); - } - - entityType = ParamChecker.notEmpty(entityType, "Entity type cannot be null"); - attribute = ParamChecker.notEmpty(attribute, "attribute name cannot be null"); - value = ParamChecker.notEmpty(value, "attribute value cannot be null"); - - Map attributes = new HashMap<>(); - attributes.put(attribute, value); - - AtlasEntityWithExtInfo entityInfo; - - try { - entityInfo = entitiesStore.getByUniqueAttributes(getEntityType(entityType), attributes); - } catch (AtlasBaseException e) { - LOG.error("Cannot find entity with type: {}, attribute: {} and value: {}", entityType, attribute, value); - throw toWebApplicationException(e); - } - - Referenceable entity = null; - - if (entityInfo != null) { - entity = restAdapters.getReferenceable(entityInfo); - } - - Map response = new HashMap<>(); - - response.put(AtlasClient.REQUEST_ID, Servlets.getRequestId()); - - Response.Status status = Response.Status.NOT_FOUND; - if (entity != null) { - response.put(AtlasClient.DEFINITION, entity); - status = Response.Status.OK; - } else { - response.put(AtlasClient.ERROR, Servlets.escapeJsonString(String.format("An entity with type={%s}, " + - "qualifiedName={%s} does not exist", entityType, value))); - } - - return Response.status(status).entity(AtlasJson.toV1Json(response)).build(); - } catch (AtlasBaseException e) { - LOG.error("Unable to get instance definition for type={}, qualifiedName={}", entityType, value, e); - throw toWebApplicationException(e); - } catch (IllegalArgumentException e) { - LOG.error("Bad type={}, qualifiedName={}", entityType, value, e); - throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST)); - } catch (WebApplicationException e) { - LOG.error("Unable to get instance definition for type={}, qualifiedName={}", entityType, value, e); - throw e; - } catch (Throwable e) { - LOG.error("Unable to get instance definition for type={}, qualifiedName={}", entityType, value, e); - throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); - } - } - - - // Trait management functions - - /** - * Gets the list of trait names for a given entity represented by a guid. - * - * @param guid globally unique identifier for the entity - * @return a list of trait names for the given entity guid - */ - @GET - @Path("{guid}/traits") - @Produces(Servlets.JSON_MEDIA_TYPE) - public Response getTraitNames(@PathParam("guid") String guid) { - if (LOG.isDebugEnabled()) { - LOG.debug("==> EntityResource.getTraitNames({})", guid); - } - - AtlasPerfTracer perf = null; - try { - if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { - perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "EntityResource.getTraitNames(" + guid + ")"); - } - - if (LOG.isDebugEnabled()) { - LOG.debug("Fetching trait names for entity={}", guid); - } - - final List classifications = entitiesStore.getClassifications(guid); - - List traitNames = new ArrayList<>(); - for (AtlasClassification classification : classifications) { - traitNames.add(classification.getTypeName()); - } - - Map response = new HashMap<>(); - response.put(AtlasClient.REQUEST_ID, Servlets.getRequestId()); - response.put(AtlasClient.RESULTS, traitNames); - response.put(AtlasClient.COUNT, traitNames.size()); - - return Response.ok(AtlasJson.toV1Json(response)).build(); - } catch (AtlasBaseException e) { - LOG.error("Unable to get trait definition for entity {}", guid, e); - throw toWebApplicationException(e); - } catch (IllegalArgumentException e) { - LOG.error("Unable to get trait definition for entity {}", guid, e); - throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST)); - } catch (WebApplicationException e) { - LOG.error("Unable to get trait names for entity {}", guid, e); - throw e; - } catch (Throwable e) { - LOG.error("Unable to get trait names for entity {}", guid, e); - throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); - } finally { - AtlasPerfTracer.log(perf); - - if (LOG.isDebugEnabled()) { - LOG.debug("<== EntityResource.getTraitNames({})", guid); - } - } - } - - /** - * Fetches the trait definitions of all the traits associated to the given entity - * @param guid globally unique identifier for the entity - */ - @GET - @Path("{guid}/traitDefinitions") - @Produces(Servlets.JSON_MEDIA_TYPE) - public Response getTraitDefinitionsForEntity(@PathParam("guid") String guid){ - if (LOG.isDebugEnabled()) { - LOG.debug("==> EntityResource.getTraitDefinitionsForEntity({})", guid); - } - - AtlasPerfTracer perf = null; - try { - if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { - perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "EntityResource.getTraitDefinitionsForEntity(" + guid + ")"); - } - - if (LOG.isDebugEnabled()) { - LOG.debug("Fetching all trait definitions for entity={}", guid); - } - - final List classifications = entitiesStore.getClassifications(guid); - - List traits = new ArrayList<>(classifications.size()); - for (AtlasClassification classification : classifications) { - Struct trait = restAdapters.getTrait(classification); - traits.add(trait); - } - - Map response = new HashMap<>(); - response.put(AtlasClient.REQUEST_ID, Servlets.getRequestId()); - response.put(AtlasClient.RESULTS, traits); - response.put(AtlasClient.COUNT, traits.size()); - - return Response.ok(AtlasJson.toV1Json(response)).build(); - } catch (AtlasBaseException e) { - LOG.error("Unable to get trait definition for entity {}", guid, e); - throw toWebApplicationException(e); - } catch (IllegalArgumentException e) { - LOG.error("Unable to get trait definition for entity {}", guid, e); - throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST)); - } catch (WebApplicationException e) { - LOG.error("Unable to get trait definitions for entity {}", guid, e); - throw e; - } catch (Throwable e) { - LOG.error("Unable to get trait definitions for entity {}", guid, e); - throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); - } finally { - AtlasPerfTracer.log(perf); - - if (LOG.isDebugEnabled()) { - LOG.debug("<== EntityResource.getTraitDefinitionsForEntity({})", guid); - } - } - } - - /** - * Fetches the trait definition for an entity given its guid and trait name - * - * @param guid globally unique identifier for the entity - * @param traitName name of the trait - */ - @GET - @Path("{guid}/traitDefinitions/{traitName}") - @Produces(Servlets.JSON_MEDIA_TYPE) - public Response getTraitDefinitionForEntity(@PathParam("guid") String guid, @PathParam("traitName") String traitName){ - if (LOG.isDebugEnabled()) { - LOG.debug("==> EntityResource.getTraitDefinitionForEntity({}, {})", guid, traitName); - } - - AtlasPerfTracer perf = null; - try { - if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { - perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "EntityResource.getTraitDefinitionForEntity(" + guid + ", " + traitName + ")"); - } - - if (LOG.isDebugEnabled()) { - LOG.debug("Fetching trait definition for entity {} and trait name {}", guid, traitName); - } - - - final AtlasClassification classification = entitiesStore.getClassification(guid, traitName); - - Struct traitDefinition = restAdapters.getTrait(classification); - - Map response = new HashMap<>(); - response.put(AtlasClient.REQUEST_ID, Servlets.getRequestId()); - response.put(AtlasClient.RESULTS, traitDefinition); - - return Response.ok(AtlasJson.toV1Json(response)).build(); - - } catch (AtlasBaseException e) { - LOG.error("Unable to get trait definition for entity {} and trait {}", guid, traitName, e); - throw toWebApplicationException(e); - } catch (IllegalArgumentException e) { - LOG.error("Unable to get trait definition for entity {} and trait {}", guid, traitName, e); - throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST)); - } catch (WebApplicationException e) { - LOG.error("Unable to get trait definition for entity {} and trait {}", guid, traitName, e); - throw e; - } catch (Throwable e) { - LOG.error("Unable to get trait definition for entity {} and trait {}", guid, traitName, e); - throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); - } finally { - AtlasPerfTracer.log(perf); - - if (LOG.isDebugEnabled()) { - LOG.debug("<== EntityResource.getTraitDefinitionForEntity({}, {})", guid, traitName); - } - } - } - - /** - * Adds a new trait to an existing entity represented by a guid. - * - * @param guid globally unique identifier for the entity - */ - @POST - @Path("{guid}/traits") - @Consumes({Servlets.JSON_MEDIA_TYPE, MediaType.APPLICATION_JSON}) - @Produces(Servlets.JSON_MEDIA_TYPE) - public Response addTrait(@Context HttpServletRequest request, @PathParam("guid") final String guid) { - if (LOG.isDebugEnabled()) { - LOG.debug("==> EntityResource.addTrait({})", guid); - } - - String traitDefinition = null; - AtlasPerfTracer perf = null; - try { - if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { - perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "EntityResource.addTrait(" + guid + ")"); - } - - traitDefinition = Servlets.getRequestPayload(request); - - if (LOG.isDebugEnabled()) { - LOG.debug("Adding trait={} for entity={} ", traitDefinition, guid); - } - - List guids = new ArrayList() {{ - add(guid); - }}; - - entitiesStore.addClassification(guids, restAdapters.toAtlasClassification(AtlasType.fromV1Json(traitDefinition, Struct.class))); - - URI locationURI = getLocationURI(new ArrayList() {{ - add(guid); - }}); - - Map response = new HashMap<>(); - response.put(AtlasClient.REQUEST_ID, Servlets.getRequestId()); - - return Response.created(locationURI).entity(AtlasJson.toV1Json(response)).build(); - } catch (AtlasBaseException e) { - LOG.error("Unable to add trait for entity={} traitDef={}", guid, traitDefinition, e); - throw toWebApplicationException(e); - } catch (IllegalArgumentException e) { - LOG.error("Unable to add trait for entity={} traitDef={}", guid, traitDefinition, e); - throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST)); - } catch (WebApplicationException e) { - LOG.error("Unable to add trait for entity={} traitDef={}", guid, traitDefinition, e); - throw e; - } catch (Throwable e) { - LOG.error("Unable to add trait for entity={} traitDef={}", guid, traitDefinition, e); - throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); - } finally { - AtlasPerfTracer.log(perf); - - if (LOG.isDebugEnabled()) { - LOG.debug("<== EntityResource.addTrait({})", guid); - } - } - } - - /** - * Deletes a given trait from an existing entity represented by a guid. - * - * @param guid globally unique identifier for the entity - * @param traitName name of the trait - */ - @DELETE - @Path("{guid}/traits/{traitName}") - @Consumes({Servlets.JSON_MEDIA_TYPE, MediaType.APPLICATION_JSON}) - @Produces(Servlets.JSON_MEDIA_TYPE) - public Response deleteTrait(@Context HttpServletRequest request, @PathParam("guid") String guid, - @PathParam(TRAIT_NAME) final String traitName) { - if (LOG.isDebugEnabled()) { - LOG.debug("==> EntityResource.deleteTrait({}, {})", guid, traitName); - } - - AtlasPerfTracer perf = null; - - if (LOG.isDebugEnabled()) { - LOG.debug("Deleting trait={} from entity={} ", traitName, guid); - } - - try { - if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { - perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "EntityResource.deleteTrait(" + guid + ", " + traitName + ")"); - } - - entitiesStore.deleteClassification(guid, traitName); - - Map response = new HashMap<>(); - response.put(AtlasClient.REQUEST_ID, Servlets.getRequestId()); - response.put(TRAIT_NAME, traitName); - - return Response.ok(AtlasJson.toV1Json(response)).build(); - } catch (AtlasBaseException e) { - LOG.error("Unable to delete trait name={} for entity={}", traitName, guid, e); - throw toWebApplicationException(e); - } catch (IllegalArgumentException e) { - LOG.error("Unable to delete trait name={} for entity={}", traitName, guid, e); - throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST)); - } catch (WebApplicationException e) { - LOG.error("Unable to delete trait name={} for entity={}", traitName, guid, e); - throw e; - } catch (Throwable e) { - LOG.error("Unable to delete trait name={} for entity={}", traitName, guid, e); - throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); - } finally { - AtlasPerfTracer.log(perf); - - if (LOG.isDebugEnabled()) { - LOG.debug("<== EntityResource.deleteTrait({}, {})", guid, traitName); - } - } - } - - /** - * Returns the entity audit events for a given entity id. The events are returned in the decreasing order of timestamp. - * @param guid entity id - * @param startKey used for pagination. Startkey is inclusive, the returned results contain the event with the given startkey. - * First time getAuditEvents() is called for an entity, startKey should be null, - * with count = (number of events required + 1). Next time getAuditEvents() is called for the same entity, - * startKey should be equal to the entityKey of the last event returned in the previous call. - * @param count number of events required - * @return - */ - @GET - @Path("{guid}/audit") - @Produces(Servlets.JSON_MEDIA_TYPE) - public Response getAuditEvents(@PathParam("guid") String guid, @QueryParam("startKey") String startKey, - @QueryParam("count") @DefaultValue("100") short count) { - if (LOG.isDebugEnabled()) { - LOG.debug("==> EntityResource.getAuditEvents({}, {}, {})", guid, startKey, count); - } - - AtlasPerfTracer perf = null; - - try { - if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { - perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "EntityResource.getAuditEvents(" + guid + ", " + startKey + ", " + count + ")"); - } - - List events = entityAuditRepository.listEvents(guid, startKey, count); - List v1Events = new ArrayList<>(); - - for (Object event : events) { - if (event instanceof EntityAuditEvent) { - v1Events.add((EntityAuditEvent) event); - } else if (event instanceof EntityAuditEventV2) { - v1Events.add(instanceConverter.toV1AuditEvent((EntityAuditEventV2) event)); - } else { - LOG.warn("unknown entity-audit event type {}. Ignored", event != null ? event.getClass().getCanonicalName() : "null"); - } - } - - Map response = new HashMap<>(); - response.put(AtlasClient.REQUEST_ID, Servlets.getRequestId()); - response.put(AtlasClient.EVENTS, v1Events); - return Response.ok(AtlasJson.toV1Json(response)).build(); - } catch (IllegalArgumentException e) { - LOG.error("Unable to get audit events for entity guid={} startKey={}", guid, startKey, e); - throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST)); - } catch (WebApplicationException e) { - LOG.error("Unable to get audit events for entity guid={} startKey={}", guid, startKey, e); - throw e; - } catch (Throwable e) { - LOG.error("Unable to get audit events for entity guid={} startKey={}", guid, startKey, e); - throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); - } finally { - AtlasPerfTracer.log(perf); - - if (LOG.isDebugEnabled()) { - LOG.debug("<== EntityResource.getAuditEvents({}, {}, {})", guid, startKey, count); - } - } - } - - private AtlasEntityType getEntityType(String typeName) throws AtlasBaseException { - AtlasEntityType ret = typeRegistry.getEntityTypeByName(typeName); - - if (ret == null) { - throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_NOT_FOUND, typeName); - } - - return ret; - } - - public static WebApplicationException toWebApplicationException(AtlasBaseException e) { - if (e.getAtlasErrorCode() == AtlasErrorCode.CLASSIFICATION_NOT_FOUND - || e.getAtlasErrorCode() == AtlasErrorCode.INSTANCE_GUID_NOT_FOUND - || e.getAtlasErrorCode() == AtlasErrorCode.INSTANCE_BY_UNIQUE_ATTRIBUTE_NOT_FOUND) { - return new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.NOT_FOUND)); - } - - if (e.getAtlasErrorCode() == AtlasErrorCode.INVALID_PARAMETERS - || e.getAtlasErrorCode() == AtlasErrorCode.INSTANCE_CRUD_INVALID_PARAMS) { - return new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST)); - } - - return new WebApplicationException(Servlets.getErrorResponse(e, e.getAtlasErrorCode().getHttpCode())); - } - - private Referenceable getEntity(String guid) throws AtlasBaseException { - AtlasEntityWithExtInfo entity = entitiesStore.getById(guid); - Referenceable referenceable = restAdapters.getReferenceable(entity); - - return referenceable; - } -} diff --git a/webapp/src/main/java/org/apache/atlas/web/resources/LineageResource.java b/webapp/src/main/java/org/apache/atlas/web/resources/LineageResource.java deleted file mode 100644 index 94b525907cc..00000000000 --- a/webapp/src/main/java/org/apache/atlas/web/resources/LineageResource.java +++ /dev/null @@ -1,200 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.web.resources; - -import org.apache.atlas.discovery.AtlasLineageService; -import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.model.lineage.AtlasLineageInfo; -import org.apache.atlas.model.lineage.AtlasLineageInfo.LineageDirection; -import org.apache.atlas.type.AtlasTypeRegistry; -import org.apache.atlas.utils.AtlasPerfTracer; -import org.apache.atlas.v1.model.lineage.LineageResponse; -import org.apache.atlas.v1.model.lineage.SchemaResponse; -import org.apache.atlas.web.util.LineageUtils; -import org.apache.atlas.web.util.Servlets; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Service; - -import javax.inject.Inject; -import javax.inject.Singleton; -import javax.ws.rs.Consumes; -import javax.ws.rs.GET; -import javax.ws.rs.Path; -import javax.ws.rs.PathParam; -import javax.ws.rs.Produces; -import javax.ws.rs.WebApplicationException; -import javax.ws.rs.core.Response; - -@Path("lineage") -@Singleton -@Service -@Deprecated -public class LineageResource { - private static final Logger LOG = LoggerFactory.getLogger(DataSetLineageResource.class); - private static final Logger PERF_LOG = AtlasPerfTracer.getPerfLogger("rest.LineageResource"); - - private final AtlasLineageService atlasLineageService; - private final AtlasTypeRegistry typeRegistry; - - /** - * Created by the Guice ServletModule and injected with the - * configured LineageService. - * - */ - @Inject - public LineageResource(AtlasLineageService atlasLineageService, AtlasTypeRegistry typeRegistry) { - this.atlasLineageService = atlasLineageService; - this.typeRegistry = typeRegistry; - } - - /** - * Returns input lineage graph for the given entity id. - * @param guid dataset entity id - * @return - */ - @GET - @Path("{guid}/inputs/graph") - @Consumes(Servlets.JSON_MEDIA_TYPE) - @Produces(Servlets.JSON_MEDIA_TYPE) - public LineageResponse inputsGraph(@PathParam("guid") String guid) { - if (LOG.isDebugEnabled()) { - LOG.debug("==> LineageResource.inputsGraph({})", guid); - } - - LineageResponse ret = new LineageResponse(); - - AtlasPerfTracer perf = null; - try { - if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { - perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "LineageResource.inputsGraph(" + guid + ")"); - } - - AtlasLineageInfo lineageInfo = atlasLineageService.getAtlasLineageInfo(guid, LineageDirection.INPUT, -1); - ret.setRequestId(Servlets.getRequestId()); - ret.setResults(LineageUtils.toLineageStruct(lineageInfo, typeRegistry)); - - return ret; - } catch (AtlasBaseException e) { - LOG.error("Unable to get lineage inputs graph for entity guid={}", guid, e); - throw new WebApplicationException(Servlets.getErrorResponse(e)); - } catch (WebApplicationException e) { - LOG.error("Unable to get lineage inputs graph for entity guid={}", guid, e); - throw e; - } finally { - AtlasPerfTracer.log(perf); - - if (LOG.isDebugEnabled()) { - LOG.debug("<== LineageResource.inputsGraph({})", guid); - } - } - } - - /** - * Returns the outputs graph for a given entity id. - * - * @param guid dataset entity id - */ - @GET - @Path("{guid}/outputs/graph") - @Consumes(Servlets.JSON_MEDIA_TYPE) - @Produces(Servlets.JSON_MEDIA_TYPE) - public LineageResponse outputsGraph(@PathParam("guid") String guid) { - if (LOG.isDebugEnabled()) { - LOG.debug("==> LineageResource.outputsGraph({})", guid); - } - - LineageResponse ret = new LineageResponse(); - - AtlasPerfTracer perf = null; - - try { - if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { - perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "LineageResource.outputsGraph(" + guid + ")"); - } - - AtlasLineageInfo lineageInfo = atlasLineageService.getAtlasLineageInfo(guid, LineageDirection.OUTPUT, -1); - ret.setRequestId(Servlets.getRequestId()); - ret.setResults(LineageUtils.toLineageStruct(lineageInfo, typeRegistry)); - - return ret; - } catch (AtlasBaseException e) { - LOG.error("Unable to get lineage outputs graph for entity guid={}", guid, e); - throw new WebApplicationException(Servlets.getErrorResponse(e)); - } catch (WebApplicationException e) { - LOG.error("Unable to get lineage outputs graph for entity guid={}", guid, e); - throw e; - } finally { - AtlasPerfTracer.log(perf); - - if (LOG.isDebugEnabled()) { - LOG.debug("<== LineageResource.outputsGraph({})", guid); - } - } - } - - /** - * Returns the schema for the given dataset id. - * - * @param guid dataset entity id - */ - @GET - @Path("{guid}/schema") - @Consumes(Servlets.JSON_MEDIA_TYPE) - @Produces(Servlets.JSON_MEDIA_TYPE) - public SchemaResponse schema(@PathParam("guid") String guid) { - if (LOG.isDebugEnabled()) { - LOG.debug("==> LineageResource.schema({})", guid); - } - - AtlasPerfTracer perf = null; - SchemaResponse ret = new SchemaResponse(); - - try { - if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { - perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "LineageResource.schema(" + guid + ")"); - } - - SchemaResponse.SchemaDetails schemaDetails = atlasLineageService.getSchemaForHiveTableByGuid(guid); - - - ret.setRequestId(Servlets.getRequestId()); - ret.setResults(schemaDetails); - return ret; - } catch (IllegalArgumentException e) { - LOG.error("Unable to get schema for entity guid={}", guid, e); - throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST)); - } catch (WebApplicationException e) { - LOG.error("Unable to get schema for entity guid={}", guid, e); - throw e; - } catch (AtlasBaseException e) { - LOG.error("Unable to get schema for entity={}", guid, e); - throw new WebApplicationException(Servlets.getErrorResponse(e, e.getAtlasErrorCode().getHttpCode())); - } catch (Throwable e) { - LOG.error("Unable to get schema for entity={}", guid, e); - throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); - } finally { - AtlasPerfTracer.log(perf); - - if (LOG.isDebugEnabled()) { - LOG.debug("<== LineageResource.schema({})", guid); - } - } - } -} diff --git a/webapp/src/main/java/org/apache/atlas/web/resources/MetadataDiscoveryResource.java b/webapp/src/main/java/org/apache/atlas/web/resources/MetadataDiscoveryResource.java deleted file mode 100755 index 34ba4da92ca..00000000000 --- a/webapp/src/main/java/org/apache/atlas/web/resources/MetadataDiscoveryResource.java +++ /dev/null @@ -1,318 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.web.resources; - -import org.apache.atlas.AtlasConfiguration; -import org.apache.atlas.discovery.AtlasDiscoveryService; -import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.model.discovery.AtlasSearchResult; -import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo; -import org.apache.atlas.model.instance.AtlasEntityHeader; -import org.apache.atlas.query.QueryParams; -import org.apache.atlas.repository.converters.AtlasInstanceConverter; -import org.apache.atlas.repository.store.graph.AtlasEntityStore; -import org.apache.atlas.utils.AtlasJson; -import org.apache.atlas.utils.AtlasPerfTracer; -import org.apache.atlas.utils.ParamChecker; -import org.apache.atlas.v1.model.discovery.DSLSearchResult; -import org.apache.atlas.v1.model.discovery.FullTextSearchResult; -import org.apache.atlas.v1.model.instance.Referenceable; -import org.apache.atlas.web.util.Servlets; -import org.apache.commons.collections.CollectionUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Service; - -import javax.inject.Inject; -import javax.inject.Singleton; -import javax.ws.rs.Consumes; -import javax.ws.rs.DefaultValue; -import javax.ws.rs.GET; -import javax.ws.rs.Path; -import javax.ws.rs.Produces; -import javax.ws.rs.QueryParam; -import javax.ws.rs.WebApplicationException; -import javax.ws.rs.core.Response; -import java.util.List; - -/** - * Jersey Resource for metadata operations. - */ -@Path("discovery") -@Singleton -@Service -@Deprecated -public class MetadataDiscoveryResource { - - private static final Logger LOG = LoggerFactory.getLogger(MetadataDiscoveryResource.class); - private static final Logger PERF_LOG = AtlasPerfTracer.getPerfLogger("rest.MetadataDiscoveryResource"); - private static final String QUERY_TYPE_DSL = "dsl"; - private static final String QUERY_TYPE_FULLTEXT = "full-text"; - private static final String LIMIT_OFFSET_DEFAULT = "-1"; - - private final AtlasDiscoveryService atlasDiscoveryService; - private final AtlasInstanceConverter restAdapters; - private final AtlasEntityStore entitiesStore; - - /** - * Created by the Guice ServletModule and injected with the - * configured DiscoveryService. - * - * @param atlasDiscoveryService atlasDiscoveryService - * @param restAdapters restAdapters - * @param entitiesStore entitiesStore - */ - @Inject - public MetadataDiscoveryResource(AtlasDiscoveryService atlasDiscoveryService, AtlasInstanceConverter restAdapters, AtlasEntityStore entitiesStore) { - this.atlasDiscoveryService = atlasDiscoveryService; - this.restAdapters = restAdapters; - this.entitiesStore = entitiesStore; - } - - /** - * Search using a given query. - * - * @param query search query in DSL format falling back to full text. - * @param limit number of rows to be returned in the result, used for pagination. maxlimit > limit > 0. -1 maps to atlas.search.defaultlimit property value - * @param offset offset to the results returned, used for pagination. offset >= 0. -1 maps to offset 0 - * @return JSON representing the type and results. - */ - @GET - @Path("search") - @Consumes(Servlets.JSON_MEDIA_TYPE) - @Produces(Servlets.JSON_MEDIA_TYPE) - public Response search(@QueryParam("query") String query, - @DefaultValue(LIMIT_OFFSET_DEFAULT) @QueryParam("limit") int limit, - @DefaultValue(LIMIT_OFFSET_DEFAULT) @QueryParam("offset") int offset) { - boolean dslQueryFailed = false; - Response response = null; - try { - response = searchUsingQueryDSL(query, limit, offset); - if (response.getStatus() != Response.Status.OK.getStatusCode()) { - dslQueryFailed = true; - } - } catch (Exception e) { - if (LOG.isDebugEnabled()) { - LOG.debug("Error while running DSL. Switching to fulltext for query {}", query, e); - } - - dslQueryFailed = true; - } - - if ( dslQueryFailed ) { - response = searchUsingFullText(query, limit, offset); - } - - return response; - } - - /** - * Search using query DSL format. - * - * @param dslQuery search query in DSL format. - * @param limit number of rows to be returned in the result, used for pagination. maxlimit > limit > 0. -1 maps to atlas.search.defaultlimit property value - * @param offset offset to the results returned, used for pagination. offset >= 0. -1 maps to offset 0 - * Limit and offset in API are used in conjunction with limit and offset in DSL query - * Final limit = min(API limit, max(query limit - API offset, 0)) - * Final offset = API offset + query offset - * - * @return JSON representing the type and results. - */ - @GET - @Path("search/dsl") - @Consumes(Servlets.JSON_MEDIA_TYPE) - @Produces(Servlets.JSON_MEDIA_TYPE) - public Response searchUsingQueryDSL(@QueryParam("query") String dslQuery, - @DefaultValue(LIMIT_OFFSET_DEFAULT) @QueryParam("limit") int limit, - @DefaultValue(LIMIT_OFFSET_DEFAULT) @QueryParam("offset") int offset) { - if (LOG.isDebugEnabled()) { - LOG.debug("==> MetadataDiscoveryResource.searchUsingQueryDSL({}, {}, {})", dslQuery, limit, offset); - } - - AtlasPerfTracer perf = null; - try { - if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { - perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "MetadataDiscoveryResource.searchUsingQueryDSL(" + dslQuery + ", " + limit + ", " + offset + ")"); - } - - dslQuery = ParamChecker.notEmpty(dslQuery, "dslQuery cannot be null"); - - QueryParams queryParams = validateQueryParams(limit, offset); - AtlasSearchResult result = atlasDiscoveryService.searchUsingDslQuery(dslQuery, queryParams.limit(), queryParams.offset()); - DSLSearchResult dslResult = new DSLSearchResult(); - - dslResult.setQueryType(QUERY_TYPE_DSL); - dslResult.setRequestId(Servlets.getRequestId()); - dslResult.setDataType(result.getType()); - dslResult.setQuery(result.getQueryText()); - dslResult.setCount(0); - - if (CollectionUtils.isNotEmpty(result.getEntities())) { - for (AtlasEntityHeader entityHeader : result.getEntities()) { - Referenceable entity = getEntity(entityHeader.getGuid()); - - dslResult.addResult(entity); - } - - if (dslResult.getResults() != null) { - dslResult.setCount(dslResult.getResults().size()); - } - } else if (result.getAttributes() != null && CollectionUtils.isNotEmpty(result.getAttributes().getName())) { - List attrNames = result.getAttributes().getName(); - - for (List attrValues : result.getAttributes().getValues()) { - if (attrValues == null) { - continue; - } - - Referenceable entity = new Referenceable(); - - for (int i = 0; i < attrNames.size(); i++) { - String attrName = attrNames.get(i); - Object attrValue = attrValues.size() > i ? attrValues.get(i) : null; - - entity.set(attrName, attrValue); - } - - dslResult.addResult(entity); - } - - if (dslResult.getResults() != null) { - dslResult.setCount(dslResult.getResults().size()); - } - } - - String response = AtlasJson.toV1SearchJson(dslResult); - - return Response.ok(response).build(); - } catch (IllegalArgumentException e) { - LOG.error("Unable to get entity list for dslQuery {}", dslQuery, e); - throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST)); - } catch (WebApplicationException e) { - LOG.error("Unable to get entity list for dslQuery {}", dslQuery, e); - throw e; - } catch (Throwable e) { - LOG.error("Unable to get entity list for dslQuery {}", dslQuery, e); - throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); - } finally { - AtlasPerfTracer.log(perf); - - if (LOG.isDebugEnabled()) { - LOG.debug("<== MetadataDiscoveryResource.searchUsingQueryDSL({}, {}, {})", dslQuery, limit, offset); - } - } - } - - /** - * Search using full text search. - * - * @param query search query. - * @param limit number of rows to be returned in the result, used for pagination. maxlimit > limit > 0. -1 maps to atlas.search.defaultlimit property value - * @param offset offset to the results returned, used for pagination. offset >= 0. -1 maps to offset 0 - * @return JSON representing the type and results. - */ - @GET - @Path("search/fulltext") - @Consumes(Servlets.JSON_MEDIA_TYPE) - @Produces(Servlets.JSON_MEDIA_TYPE) - public Response searchUsingFullText(@QueryParam("query") String query, - @DefaultValue(LIMIT_OFFSET_DEFAULT) @QueryParam("limit") int limit, - @DefaultValue(LIMIT_OFFSET_DEFAULT) @QueryParam("offset") int offset) { - if (LOG.isDebugEnabled()) { - LOG.debug("==> MetadataDiscoveryResource.searchUsingFullText({}, {}, {})", query, limit, offset); - } - - AtlasPerfTracer perf = null; - try { - if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { - perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "MetadataDiscoveryResource.searchUsingFullText(" + query + ", " + limit + ", " + offset + ")"); - } - - query = ParamChecker.notEmpty(query, "query cannot be null or empty"); - - QueryParams queryParams = validateQueryParams(limit, offset); - AtlasSearchResult result = atlasDiscoveryService.searchUsingFullTextQuery(query, false, queryParams.limit(), queryParams.offset()); - FullTextSearchResult fullTextResult = new FullTextSearchResult(); - - fullTextResult.setQueryType(QUERY_TYPE_FULLTEXT); - fullTextResult.setRequestId(Servlets.getRequestId()); - fullTextResult.setDataType(result.getType()); - fullTextResult.setQuery(result.getQueryText()); - fullTextResult.setCount(0); - - if (CollectionUtils.isNotEmpty(result.getFullTextResult())) { - for (AtlasSearchResult.AtlasFullTextResult entity : result.getFullTextResult()) { - fullTextResult.addResult(entity); - } - - fullTextResult.setCount(fullTextResult.getResults().size()); - } - - String response = AtlasJson.toV1SearchJson(fullTextResult); - - return Response.ok(response).build(); - } catch (IllegalArgumentException e) { - LOG.error("Unable to get entity list for query {}", query, e); - throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST)); - } catch (WebApplicationException e) { - LOG.error("Unable to get entity list for query {}", query, e); - throw e; - } catch (Throwable e) { - LOG.error("Unable to get entity list for query {}", query, e); - throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); - } finally { - AtlasPerfTracer.log(perf); - - if (LOG.isDebugEnabled()) { - LOG.debug("<== MetadataDiscoveryResource.searchUsingFullText({}, {}, {})", query, limit, offset); - } - } - } - - private QueryParams validateQueryParams(int limitParam, int offsetParam) { - int maxLimit = AtlasConfiguration.SEARCH_MAX_LIMIT.getInt(); - int defaultLimit = AtlasConfiguration.SEARCH_DEFAULT_LIMIT.getInt(); - int limit = defaultLimit; - boolean limitSet = (limitParam != Integer.valueOf(LIMIT_OFFSET_DEFAULT)); - - if (limitSet) { - ParamChecker.lessThan(limitParam, maxLimit, "limit"); - ParamChecker.greaterThan(limitParam, 0, "limit"); - - limit = limitParam; - } - - int offset = 0; - boolean offsetSet = (offsetParam != Integer.valueOf(LIMIT_OFFSET_DEFAULT)); - - if (offsetSet) { - ParamChecker.greaterThan(offsetParam, -1, "offset"); - offset = offsetParam; - } - - return new QueryParams(limit, offset); - } - - private Referenceable getEntity(String guid) throws AtlasBaseException { - AtlasEntityWithExtInfo entity = entitiesStore.getById(guid); - Referenceable referenceable = restAdapters.getReferenceable(entity); - - return referenceable; - } -} diff --git a/webapp/src/main/java/org/apache/atlas/web/resources/TypesResource.java b/webapp/src/main/java/org/apache/atlas/web/resources/TypesResource.java deleted file mode 100755 index bcc57e0d380..00000000000 --- a/webapp/src/main/java/org/apache/atlas/web/resources/TypesResource.java +++ /dev/null @@ -1,314 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.web.resources; - -import com.sun.jersey.api.client.ClientResponse; -import com.sun.jersey.api.core.ResourceContext; -import org.apache.atlas.AtlasClient; -import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.model.typedef.AtlasTypesDef; -import org.apache.atlas.utils.AtlasJson; -import org.apache.atlas.v1.model.typedef.TypesDef; -import org.apache.atlas.store.AtlasTypeDefStore; -import org.apache.atlas.type.AtlasTypeRegistry; -import org.apache.atlas.repository.converters.TypeConverterUtil; -import org.apache.atlas.utils.AtlasPerfTracer; -import org.apache.atlas.web.rest.TypesREST; -import org.apache.atlas.web.util.Servlets; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Service; - -import javax.inject.Inject; -import javax.inject.Singleton; -import javax.servlet.http.HttpServletRequest; -import javax.ws.rs.Consumes; -import javax.ws.rs.GET; -import javax.ws.rs.POST; -import javax.ws.rs.PUT; -import javax.ws.rs.Path; -import javax.ws.rs.PathParam; -import javax.ws.rs.Produces; -import javax.ws.rs.QueryParam; -import javax.ws.rs.WebApplicationException; -import javax.ws.rs.core.Context; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * This class provides RESTful API for Types. - * - * A type is the description of any representable item; - * e.g. a Hive table - * - * You could represent any meta model representing any domain using these types. - */ -@Path("types") -@Singleton -@Service -@Deprecated -public class TypesResource { - private static final Logger LOG = LoggerFactory.getLogger(TypesResource.class); - private static final Logger PERF_LOG = AtlasPerfTracer.getPerfLogger("rest.TypesResource"); - private static AtlasTypeRegistry typeRegistry; - private final TypesREST typesREST; - private final AtlasTypeDefStore typeDefStore; - - @Inject - public TypesResource(AtlasTypeRegistry typeRegistry, TypesREST typesREST, AtlasTypeDefStore typeDefStore) { - this.typeRegistry = typeRegistry; - this.typesREST = typesREST; - this.typeDefStore = typeDefStore; - } - - @Context - private ResourceContext resourceContext; - - /** - * Submits a type definition corresponding to a given type representing a meta model of a - * domain. Could represent things like Hive Database, Hive Table, etc. - */ - @POST - @Consumes({Servlets.JSON_MEDIA_TYPE, MediaType.APPLICATION_JSON}) - @Produces(Servlets.JSON_MEDIA_TYPE) - public Response submit(@Context HttpServletRequest request) { - if (LOG.isDebugEnabled()) { - LOG.debug("==> TypesResource.submit()"); - } - - AtlasPerfTracer perf = null; - - if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { - perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "TypesResource.submit()"); - } - - try { - final String typeDefinition = Servlets.getRequestPayload(request); - - if (LOG.isDebugEnabled()) { - LOG.debug("Creating type with definition {} ", typeDefinition); - } - - AtlasTypesDef createTypesDef = TypeConverterUtil.toAtlasTypesDef(typeDefinition, typeRegistry); - AtlasTypesDef createdTypesDef = typesREST.createAtlasTypeDefs(createTypesDef); - List typeNames = TypeConverterUtil.getTypeNames(createdTypesDef); - List> typesResponse = new ArrayList<>(typeNames.size()); - - for (String typeName : typeNames) { - typesResponse.add(Collections.singletonMap(AtlasClient.NAME, typeName)); - } - - Map response = new HashMap<>(); - response.put(AtlasClient.REQUEST_ID, Servlets.getRequestId()); - response.put(AtlasClient.TYPES, typesResponse); - return Response.status(ClientResponse.Status.CREATED).entity(AtlasJson.toV1Json(response)).build(); - } catch (AtlasBaseException e) { - LOG.error("Type creation failed", e); - throw new WebApplicationException(Servlets.getErrorResponse(e)); - } catch (IllegalArgumentException e) { - LOG.error("Unable to persist types", e); - throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST)); - } catch (WebApplicationException e) { - LOG.error("Unable to persist types", e); - throw e; - } catch (Throwable e) { - LOG.error("Unable to persist types", e); - throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); - } finally { - AtlasPerfTracer.log(perf); - - if (LOG.isDebugEnabled()) { - LOG.debug("<== TypesResource.submit()"); - } - } - } - - /** - * Update of existing types - if the given type doesn't exist, creates new type - * Allowed updates are: - * 1. Add optional attribute - * 2. Change required to optional attribute - * 3. Add super types - super types shouldn't contain any required attributes - * @param request - * @return - */ - @PUT - @Consumes({Servlets.JSON_MEDIA_TYPE, MediaType.APPLICATION_JSON}) - @Produces(Servlets.JSON_MEDIA_TYPE) - public Response update(@Context HttpServletRequest request) { - if (LOG.isDebugEnabled()) { - LOG.debug("==> TypesResource.update()"); - } - - AtlasPerfTracer perf = null; - - if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { - perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "TypesResource.update()"); - } - - try { - final String typeDefinition = Servlets.getRequestPayload(request); - - if (LOG.isDebugEnabled()) { - LOG.debug("Updating type with definition {} ", typeDefinition); - } - - AtlasTypesDef updateTypesDef = TypeConverterUtil.toAtlasTypesDef(typeDefinition, typeRegistry); - AtlasTypesDef updatedTypesDef = typeDefStore.createUpdateTypesDef(updateTypesDef); - List typeNames = TypeConverterUtil.getTypeNames(updatedTypesDef); - List> typesResponse = new ArrayList<>(typeNames.size()); - - for (String typeName : typeNames) { - typesResponse.add(Collections.singletonMap(AtlasClient.NAME, typeName)); - } - - Map response = new HashMap<>(); - response.put(AtlasClient.REQUEST_ID, Servlets.getRequestId()); - response.put(AtlasClient.TYPES, typesResponse); - return Response.ok().entity(AtlasJson.toV1Json(response)).build(); - } catch (AtlasBaseException e) { - LOG.error("Unable to persist types", e); - throw new WebApplicationException(Servlets.getErrorResponse(e)); - } catch (IllegalArgumentException e) { - LOG.error("Unable to persist types", e); - throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST)); - } catch (WebApplicationException e) { - LOG.error("Unable to persist types", e); - throw e; - } catch (Throwable e) { - LOG.error("Unable to persist types", e); - throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); - } finally { - AtlasPerfTracer.log(perf); - - if (LOG.isDebugEnabled()) { - LOG.debug("<== TypesResource.update()"); - } - } - } - - /** - * Fetch the complete definition of a given type name which is unique. - * - * @param typeName name of a type which is unique. - */ - @GET - @Path("{typeName}") - @Produces(Servlets.JSON_MEDIA_TYPE) - public Response getDefinition(@Context HttpServletRequest request, @PathParam("typeName") String typeName) { - if (LOG.isDebugEnabled()) { - LOG.debug("==> TypesResource.getDefinition({})", typeName); - } - - AtlasPerfTracer perf = null; - - if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { - perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "TypesResource.getDefinition(" + typeName + ")"); - } - - Map response = new HashMap<>(); - - try { - TypesDef typesDef = TypeConverterUtil.toTypesDef(typeRegistry.getType(typeName), typeRegistry);; - - response.put(AtlasClient.TYPENAME, typeName); - response.put(AtlasClient.DEFINITION, typesDef); - response.put(AtlasClient.REQUEST_ID, Servlets.getRequestId()); - - return Response.ok(AtlasJson.toV1Json(response)).build(); - } catch (AtlasBaseException e) { - LOG.error("Unable to get type definition for type {}", typeName, e); - throw new WebApplicationException(Servlets.getErrorResponse(e)); - } catch (IllegalArgumentException e) { - LOG.error("Unable to get type definition for type {}", typeName, e); - throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST)); - } catch (WebApplicationException e) { - LOG.error("Unable to get type definition for type {}", typeName, e); - throw e; - } catch (Throwable e) { - LOG.error("Unable to get type definition for type {}", typeName, e); - throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); - } finally { - AtlasPerfTracer.log(perf); - - if (LOG.isDebugEnabled()) { - LOG.debug("<== TypesResource.getDefinition({})", typeName); - } - } - } - - /** - * Return the list of type names in the type system which match the specified filter. - * - * @return list of type names - * @param typeCategory returns types whose relationshipCategory is the given typeCategory - * @param supertype returns types which contain the given supertype - * @param notsupertype returns types which do not contain the given supertype - * - * Its possible to specify combination of these filters in one request and the conditions are combined with AND - * For example, typeCategory = TRAIT && supertype contains 'X' && supertype !contains 'Y' - * If there is no filter, all the types are returned - */ - @GET - @Produces(Servlets.JSON_MEDIA_TYPE) - public Response getTypesByFilter(@Context HttpServletRequest request, @QueryParam("type") String typeCategory, - @QueryParam("supertype") String supertype, - @QueryParam("notsupertype") String notsupertype) throws AtlasBaseException { - if (LOG.isDebugEnabled()) { - LOG.debug("==> TypesResource.getTypesByFilter({}, {}, {})", typeCategory, supertype, notsupertype); - } - - AtlasPerfTracer perf = null; - - if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { - perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "TypesResource.getTypesByFilter(" + typeCategory + ", " + supertype + ", " + notsupertype + ")"); - } - - Map response = new HashMap<>(); - try { - List result = TypeConverterUtil.getTypeNames(typesREST.getTypeDefHeaders(request)); - - response.put(AtlasClient.RESULTS, result); - response.put(AtlasClient.COUNT, result.size()); - response.put(AtlasClient.REQUEST_ID, Servlets.getRequestId()); - - return Response.ok(AtlasJson.toV1Json(response)).build(); - } catch (AtlasBaseException e) { - LOG.warn("TypesREST exception: {} {}", e.getClass().getSimpleName(), e.getMessage()); - throw new WebApplicationException(Servlets.getErrorResponse(e)); - } catch (WebApplicationException e) { - LOG.error("Unable to get types list", e); - throw e; - } catch (Throwable e) { - LOG.error("Unable to get types list", e); - throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); - } finally { - AtlasPerfTracer.log(perf); - - if (LOG.isDebugEnabled()) { - LOG.debug("<== TypesResource.getTypesByFilter({}, {}, {})", typeCategory, supertype, notsupertype); - } - } - } -} diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java new file mode 100644 index 00000000000..c5868cbfa30 --- /dev/null +++ b/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java @@ -0,0 +1,229 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.web.rest; + +import org.apache.atlas.RequestContext; +import org.apache.atlas.annotation.Timed; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.audit.AuditSearchParams; +import org.apache.atlas.model.audit.EntityAuditSearchResult; +import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.plugin.util.KeycloakUserStore; +import org.apache.atlas.plugin.util.RangerRoles; +import org.apache.atlas.plugin.util.RangerUserStore; +import org.apache.atlas.plugin.util.ServicePolicies; +import org.apache.atlas.policytransformer.CachePolicyTransformerImpl; +import org.apache.atlas.repository.audit.ESBasedAuditRepository; +import org.apache.atlas.repository.store.graph.AtlasEntityStore; +import org.apache.atlas.repository.store.graph.v2.AtlasEntityStream; +import org.apache.atlas.utils.AtlasPerfMetrics; +import org.apache.atlas.utils.AtlasPerfTracer; +import org.apache.atlas.web.util.Servlets; +import org.apache.commons.collections.CollectionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import javax.inject.Inject; +import javax.inject.Singleton; +import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.Consumes; +import javax.ws.rs.DefaultValue; +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.apache.atlas.policytransformer.CachePolicyTransformerImpl.ATTR_SERVICE_LAST_SYNC; +import static org.apache.atlas.repository.Constants.PERSONA_ENTITY_TYPE; +import static org.apache.atlas.repository.Constants.POLICY_ENTITY_TYPE; +import static org.apache.atlas.repository.Constants.PURPOSE_ENTITY_TYPE; + +/** + * REST interface for CRUD operations on tasks + */ +@Path("auth") +@Singleton +@Service +@Consumes({Servlets.JSON_MEDIA_TYPE, MediaType.APPLICATION_JSON}) +@Produces({Servlets.JSON_MEDIA_TYPE, MediaType.APPLICATION_JSON}) +public class AuthREST { + private static final Logger LOG = LoggerFactory.getLogger(AuthREST.class); + private static final Logger PERF_LOG = AtlasPerfTracer.getPerfLogger("rest.AuthREST"); + + private CachePolicyTransformerImpl policyTransformer; + private ESBasedAuditRepository auditRepository; + private AtlasEntityStore entityStore; + + @Inject + public AuthREST(CachePolicyTransformerImpl policyTransformer, + ESBasedAuditRepository auditRepository, AtlasEntityStore entityStore) { + this.entityStore = entityStore; + this.auditRepository = auditRepository; + this.policyTransformer = policyTransformer; + } + + @GET + @Path("download/roles/{serviceName}") + @Timed + public RangerRoles downloadRoles(@PathParam("serviceName") final String serviceName, + @QueryParam("pluginId") String pluginId, + @DefaultValue("0") @QueryParam("lastUpdatedTime") Long lastUpdatedTime, + @Context HttpServletResponse response) throws AtlasBaseException { + AtlasPerfTracer perf = null; + + try { + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "AuthREST.downloadRoles(serviceName="+serviceName+", pluginId="+pluginId+", lastUpdatedTime="+lastUpdatedTime+")"); + } + + KeycloakUserStore keycloakUserStore = new KeycloakUserStore(serviceName); + RangerRoles roles = keycloakUserStore.loadRolesIfUpdated(lastUpdatedTime); + + if (roles == null) { + response.setStatus(HttpServletResponse.SC_NOT_MODIFIED); + } + + return roles; + } finally { + AtlasPerfTracer.log(perf); + } + } + + @GET + @Path("download/users/{serviceName}") + @Timed + public RangerUserStore downloadUserStore(@PathParam("serviceName") final String serviceName, + @QueryParam("pluginId") String pluginId, + @DefaultValue("0") @QueryParam("lastUpdatedTime") Long lastUpdatedTime, + @Context HttpServletResponse response) throws AtlasBaseException { + AtlasPerfTracer perf = null; + + try { + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "AuthREST.downloadUserStore(serviceName="+serviceName+", pluginId="+pluginId+", lastUpdatedTime="+lastUpdatedTime+")"); + } + + KeycloakUserStore keycloakUserStore = new KeycloakUserStore(serviceName); + RangerUserStore userStore = keycloakUserStore.loadUserStoreIfUpdated(lastUpdatedTime); + + if (userStore == null) { + response.setStatus(HttpServletResponse.SC_NOT_MODIFIED); + } + + return userStore; + } finally { + AtlasPerfTracer.log(perf); + } + } + + @GET + @Path("download/policies/{serviceName}") + @Timed + public ServicePolicies downloadPolicies(@PathParam("serviceName") final String serviceName, + @QueryParam("pluginId") String pluginId, + @DefaultValue("0") @QueryParam("lastUpdatedTime") Long lastUpdatedTime) throws AtlasBaseException { + AtlasPerfTracer perf = null; + + try { + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "AuthREST.downloadPolicies(serviceName="+serviceName+", pluginId="+pluginId+", lastUpdatedTime="+lastUpdatedTime+")"); + } + + if (!isPolicyUpdated(serviceName, lastUpdatedTime)) { + return null; + } + + ServicePolicies ret = policyTransformer.getPolicies(serviceName, pluginId, lastUpdatedTime); + + updateLastSync(serviceName); + + return ret; + } finally { + AtlasPerfTracer.log(perf); + } + } + + private void updateLastSync(String serviceName) { + AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("AuthRest.updateLastSync." + serviceName); + + try { + if (policyTransformer.getService() != null) { + AtlasEntity serviceEntity = new AtlasEntity(policyTransformer.getService()); + serviceEntity.setAttribute(ATTR_SERVICE_LAST_SYNC, System.currentTimeMillis()); + try { + entityStore.createOrUpdate(new AtlasEntityStream(serviceEntity), false); + } catch (AtlasBaseException e) { + LOG.error("Failed to update authServicePolicyLastSync time: {}", e.getMessage()); + } + } + } finally { + RequestContext.get().endMetricRecord(recorder); + } + } + + private boolean isPolicyUpdated(String serviceName, long lastUpdatedTime) { + AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("AuthRest.isPolicyUpdated." + serviceName); + + List entityUpdateToWatch = new ArrayList<>(); + entityUpdateToWatch.add(POLICY_ENTITY_TYPE); + entityUpdateToWatch.add(PERSONA_ENTITY_TYPE); + entityUpdateToWatch.add(PURPOSE_ENTITY_TYPE); + + AuditSearchParams parameters = new AuditSearchParams(); + Map dsl = getMap("size", 1); + + List> mustClauseList = new ArrayList<>(); + mustClauseList.add(getMap("terms", getMap("typeName", entityUpdateToWatch))); + + lastUpdatedTime = lastUpdatedTime == -1 ? 0 : lastUpdatedTime; + mustClauseList.add(getMap("range", getMap("timestamp", getMap("gte", lastUpdatedTime)))); + + dsl.put("query", getMap("bool", getMap("must", mustClauseList))); + + parameters.setDsl(dsl); + + try { + EntityAuditSearchResult result = auditRepository.searchEvents(parameters.getQueryString()); + + if (result == null || CollectionUtils.isEmpty(result.getEntityAudits())) { + return false; + } + } catch (AtlasBaseException e) { + LOG.error("ERROR in getPoliciesIfUpdated while fetching entity audits {}: ", e.getMessage()); + return true; + } finally { + RequestContext.get().endMetricRecord(recorder); + } + + return true; + } + + private Map getMap(String key, Object value) { + Map map = new HashMap<>(); + map.put(key, value); + return map; + } +} diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java index 52ceacc2c0e..14375a19727 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java @@ -18,7 +18,9 @@ package org.apache.atlas.web.rest; import org.apache.atlas.AtlasClient; +import org.apache.atlas.AtlasConfiguration; import org.apache.atlas.AtlasErrorCode; +import org.apache.atlas.RequestContext; import org.apache.atlas.SortOrder; import org.apache.atlas.annotation.Timed; import org.apache.atlas.authorize.AtlasAuthorizationUtils; @@ -28,7 +30,11 @@ import org.apache.atlas.model.discovery.*; import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria; import org.apache.atlas.model.profile.AtlasUserSavedSearch; +import org.apache.atlas.model.searchlog.SearchLogSearchParams; +import org.apache.atlas.model.searchlog.SearchLogSearchResult; +import org.apache.atlas.model.searchlog.SearchRequestLogData.SearchRequestLogDataBuilder; import org.apache.atlas.repository.Constants; +import org.apache.atlas.searchlog.SearchLoggingManagement; import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasStructType; import org.apache.atlas.type.AtlasTypeRegistry; @@ -38,6 +44,7 @@ import org.apache.commons.configuration.Configuration; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; import javax.inject.Inject; @@ -55,34 +62,44 @@ import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import java.io.IOException; +import java.util.HashSet; import java.util.List; import java.util.Set; +import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; +import static org.apache.atlas.repository.Constants.REQUEST_HEADER_HOST; +import static org.apache.atlas.repository.Constants.REQUEST_HEADER_USER_AGENT; + /** * REST interface for data discovery using dsl or full text search */ -@Path("v2/search") +@Path("search") @Singleton @Service @Consumes({Servlets.JSON_MEDIA_TYPE, MediaType.APPLICATION_JSON}) @Produces({Servlets.JSON_MEDIA_TYPE, MediaType.APPLICATION_JSON}) public class DiscoveryREST { private static final Logger PERF_LOG = AtlasPerfTracer.getPerfLogger("rest.DiscoveryREST"); - + private static final Logger LOG = LoggerFactory.getLogger(DiscoveryREST.class); @Context private HttpServletRequest httpServletRequest; private final int maxFullTextQueryLength; private final int maxDslQueryLength; + private final boolean enableSearchLogging; private final AtlasTypeRegistry typeRegistry; private final AtlasDiscoveryService discoveryService; + private final SearchLoggingManagement loggerManagement; @Inject - public DiscoveryREST(AtlasTypeRegistry typeRegistry, AtlasDiscoveryService discoveryService, Configuration configuration) { + public DiscoveryREST(AtlasTypeRegistry typeRegistry, AtlasDiscoveryService discoveryService, + SearchLoggingManagement loggerManagement, Configuration configuration) { this.typeRegistry = typeRegistry; this.discoveryService = discoveryService; + this.loggerManagement = loggerManagement; this.maxFullTextQueryLength = configuration.getInt(Constants.MAX_FULLTEXT_QUERY_STR_LENGTH, 4096); this.maxDslQueryLength = configuration.getInt(Constants.MAX_DSL_QUERY_STR_LENGTH, 4096); + this.enableSearchLogging = AtlasConfiguration.ENABLE_SEARCH_LOGGER.getBoolean(); } /** @@ -370,26 +387,87 @@ public AtlasSearchResult searchWithParameters(SearchParameters parameters) throw @Path("indexsearch") @POST @Timed - public AtlasSearchResult indexSearch(IndexSearchParams parameters) throws AtlasBaseException { + public AtlasSearchResult indexSearch(@Context HttpServletRequest servletRequest, IndexSearchParams parameters) throws AtlasBaseException { AtlasPerfTracer perf = null; +<<<<<<< HEAD if (parameters == null) { throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Please provide parameters for search"); } +======= + long startTime = System.currentTimeMillis(); +>>>>>>> 7f3c811fb15361ba82bfb4edd7a8393798863ff0 + RequestContext.get().setIncludeMeanings(!parameters.isExcludeMeanings()); + RequestContext.get().setIncludeClassifications(!parameters.isExcludeClassifications()); try { if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "DiscoveryREST.indexSearch(" + parameters + ")"); } if (StringUtils.isEmpty(parameters.getQuery())) { - throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Please provide query"); + AtlasBaseException abe = new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Invalid search query"); + if (enableSearchLogging && parameters.isSaveSearchLog()) { + logSearchLog(parameters, servletRequest, abe, System.currentTimeMillis() - startTime); + } + throw abe; + } + + if(LOG.isDebugEnabled()){ + LOG.debug("Performing indexsearch for the params ({})", parameters); + } + AtlasSearchResult result = discoveryService.directIndexSearch(parameters); + long endTime = System.currentTimeMillis(); + + if (enableSearchLogging && parameters.isSaveSearchLog()) { + logSearchLog(parameters, result, servletRequest, endTime - startTime); + } + + return result; + } catch (AtlasBaseException abe) { + if (enableSearchLogging && parameters.isSaveSearchLog()) { + logSearchLog(parameters, servletRequest, abe, System.currentTimeMillis() - startTime); + } + throw abe; + } catch (Exception e) { + AtlasBaseException abe = new AtlasBaseException(e.getMessage(), e.getCause()); + if (enableSearchLogging && parameters.isSaveSearchLog()) { + logSearchLog(parameters, servletRequest, abe, System.currentTimeMillis() - startTime); + } + throw abe; + } finally { + AtlasPerfTracer.log(perf); + } + } + + /** + * Index based search for query direct on ES search-logs index + * + * @param parameters Index Search parameters @SearchLogSearchParams.java + * @return search log search result + * @throws AtlasBaseException + * @HTTP 200 On successful search + */ + @Path("searchlog") + @POST + @Timed + public SearchLogSearchResult searchLogs(SearchLogSearchParams parameters) throws AtlasBaseException { + AtlasPerfTracer perf = null; + SearchLogSearchResult result; + + try { + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "DiscoveryREST.searchLogs(" + parameters + ")"); } - return discoveryService.directIndexSearch(parameters); + result = discoveryService.searchLogs(parameters); + + } catch (Exception e) { + throw e; } finally { AtlasPerfTracer.log(perf); } + return result; } /** @@ -417,6 +495,7 @@ public AtlasSearchResult searchRelatedEntities(@QueryParam("guid") @QueryParam("sortOrder") SortOrder sortOrder, @QueryParam("excludeDeletedEntities") boolean excludeDeletedEntities, @QueryParam("includeClassificationAttributes") boolean includeClassificationAttributes, + @QueryParam("excludeMeaningsAttributes") boolean excludeMeaningsAttributes, @QueryParam("getApproximateCount") boolean getApproximateCount, @QueryParam("limit") int limit, @QueryParam("offset") int offset) throws AtlasBaseException { @@ -440,6 +519,10 @@ public AtlasSearchResult searchRelatedEntities(@QueryParam("guid") parameters.setLimit(limit); parameters.setOffset(offset); parameters.setIncludeClassificationAttributes(includeClassificationAttributes); + parameters.setExcludeClassifications(!includeClassificationAttributes); + parameters.setExcludeMeanings(excludeMeaningsAttributes); + RequestContext.get().setIncludeClassifications(includeClassificationAttributes); + RequestContext.get().setIncludeMeanings(!excludeMeaningsAttributes); return discoveryService.searchRelatedEntities(guid, relation, getApproximateCount, parameters); } finally { @@ -788,4 +871,112 @@ private void validateSearchParameters(QuickSearchParameters parameters) throws A validateSearchParameters(EntityDiscoveryService.createSearchParameters(parameters)); } } + + private void logSearchLog(IndexSearchParams parameters, AtlasSearchResult result, + HttpServletRequest servletRequest, long requestTime) { + SearchRequestLogDataBuilder builder = new SearchRequestLogDataBuilder(); + builder.setHasResult(false); + + if (CollectionUtils.isNotEmpty(result.getEntities())) { + builder.setHasResult(true); + builder.setResultsCount(result.getApproximateCount()); + + Set entityGuidsAll = new HashSet<>(); + Set entityQFNamesAll = new HashSet<>(); + Set entityGuidsAllowed = new HashSet<>(); + Set entityQFNamesAllowed = new HashSet<>(); + Set entityGuidsDenied = new HashSet<>(); + Set entityQFNamesDenied = new HashSet<>(); + Set entityTypesAll = new HashSet<>(); + Set entityTypesAllowed = new HashSet<>(); + Set entityTypesDenied = new HashSet<>(); + + result.getEntities().forEach(x -> { + boolean allowed = x.getScrubbed() == null; + + String guid = x.getGuid(); + + if (guid != null) { + entityGuidsAll.add(guid); + entityTypesAll.add(x.getTypeName()); + + if (allowed) { + entityGuidsAllowed.add(guid); + entityTypesAllowed.add(x.getTypeName()); + } else { + entityGuidsDenied.add(guid); + entityTypesDenied.add(x.getTypeName()); + } + } + + try { + String qualifiedName = (String) x.getAttribute(QUALIFIED_NAME); + + if (qualifiedName != null) { + + entityQFNamesAll.add(qualifiedName); + if (allowed) { + entityQFNamesAllowed.add(qualifiedName); + } else { + entityQFNamesDenied.add(qualifiedName); + } + } + } catch (NullPointerException npe) { + //no attributes for entity + } + }); + + builder.setEntityGuidsAll(entityGuidsAll) + .setEntityQFNamesAll(entityQFNamesAll) + .setEntityGuidsAllowed(entityGuidsAllowed) + .setEntityQFNamesAllowed(entityQFNamesAllowed) + .setEntityGuidsDenied(entityGuidsDenied) + .setEntityQFNamesDenied(entityQFNamesDenied) + .setEntityTypeNamesAll(entityTypesAll) + .setEntityTypeNamesAllowed(entityTypesAllowed) + .setEntityTypeNamesDenied(entityTypesDenied); + + } + + logSearchLog(parameters, servletRequest, builder, requestTime); + } + + private void logSearchLog(IndexSearchParams parameters, HttpServletRequest servletRequest, + AtlasBaseException e, long requestTime) { + SearchRequestLogDataBuilder builder = new SearchRequestLogDataBuilder(); + + builder.setErrorDetails(e.getMessage()) + .setErrorCode(e.getAtlasErrorCode().getErrorCode()) + .setFailed(true); + + + logSearchLog(parameters, servletRequest, builder, requestTime); + } + + private void logSearchLog(IndexSearchParams parameters, HttpServletRequest servletRequest, + SearchRequestLogDataBuilder builder, long requestTime) { + + if (StringUtils.isNotEmpty(parameters.getPersona())) { + builder.setPersona(parameters.getPersona()); + } else { + builder.setPurpose(parameters.getPurpose()); + } + + builder.setDsl(parameters.getDsl()) + .setSearchInput(parameters.getSearchInput()) + .setUtmTags(parameters.getUtmTags()) + .setAttributes(parameters.getAttributes()) + .setRelationAttributes(parameters.getRelationAttributes()) + + .setUserAgent(servletRequest.getHeader(REQUEST_HEADER_USER_AGENT)) + .setHost(servletRequest.getHeader(REQUEST_HEADER_HOST)) + + .setIpAddress(AtlasAuthorizationUtils.getRequestIpAddress(servletRequest)) + .setUserName(AtlasAuthorizationUtils.getCurrentUserName()) + + .setTimestamp(RequestContext.get().getRequestTime()) + .setResponseTime(requestTime); + + loggerManagement.log(builder.build()); + } } diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java index 47c6cbbe2d7..3250800a286 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java @@ -17,10 +17,10 @@ */ package org.apache.atlas.web.rest; +import com.google.common.collect.Lists; import com.sun.jersey.core.header.FormDataContentDisposition; import com.sun.jersey.multipart.FormDataParam; import org.apache.atlas.AtlasErrorCode; -import org.apache.atlas.EntityAuditEvent; import org.apache.atlas.RequestContext; import org.apache.atlas.annotation.Timed; import org.apache.atlas.authorize.*; @@ -31,13 +31,12 @@ import org.apache.atlas.model.audit.EntityAuditEventV2; import org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditActionV2; import org.apache.atlas.model.audit.EntityAuditSearchResult; +import org.apache.atlas.model.discovery.AtlasSearchResult; import org.apache.atlas.model.instance.*; import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo; import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo; import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef; -import org.apache.atlas.repository.audit.CassandraBasedAuditRepository; import org.apache.atlas.repository.audit.ESBasedAuditRepository; -import org.apache.atlas.repository.converters.AtlasInstanceConverter; import org.apache.atlas.repository.store.graph.AtlasEntityStore; import org.apache.atlas.repository.store.graph.v2.AtlasEntityStream; import org.apache.atlas.repository.store.graph.v2.ClassificationAssociator; @@ -47,6 +46,7 @@ import org.apache.atlas.type.AtlasType; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.util.FileUtils; +import org.apache.atlas.utils.AtlasPerfMetrics; import org.apache.atlas.utils.AtlasPerfTracer; import org.apache.atlas.web.util.Servlets; import org.apache.commons.collections.CollectionUtils; @@ -55,6 +55,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; +import org.apache.atlas.tools.RepairIndex; import javax.inject.Inject; import javax.inject.Singleton; @@ -70,13 +71,14 @@ import java.util.*; import static org.apache.atlas.AtlasErrorCode.BAD_REQUEST; +import static org.apache.atlas.AtlasErrorCode.DEPRECATED_API; import static org.apache.atlas.authorize.AtlasPrivilege.*; /** * REST for a single entity */ -@Path("v2/entity") +@Path("entity") @Singleton @Service @Consumes({Servlets.JSON_MEDIA_TYPE, MediaType.APPLICATION_JSON}) @@ -88,22 +90,22 @@ public class EntityREST { public static final String PREFIX_ATTR = "attr:"; public static final String PREFIX_ATTR_ = "attr_"; public static final String QUALIFIED_NAME = "qualifiedName"; + private static final int HUNDRED_THOUSAND = 100000; + private static final int TWO_MILLION = HUNDRED_THOUSAND * 10 * 2; + private static final Set ATTRS_WITH_TWO_MILLION_LIMIT = new HashSet() {{ + add("rawQueryText"); + }}; private final AtlasTypeRegistry typeRegistry; private final AtlasEntityStore entitiesStore; - private final CassandraBasedAuditRepository cassandraBasedAuditRepository; private final ESBasedAuditRepository esBasedAuditRepository; - private final AtlasInstanceConverter instanceConverter; @Inject - public EntityREST(AtlasTypeRegistry typeRegistry, AtlasEntityStore entitiesStore, - CassandraBasedAuditRepository cassandraBasedAuditRepository, ESBasedAuditRepository esBasedAuditRepository, AtlasInstanceConverter instanceConverter) { + public EntityREST(AtlasTypeRegistry typeRegistry, AtlasEntityStore entitiesStore, ESBasedAuditRepository esBasedAuditRepository) { this.typeRegistry = typeRegistry; this.entitiesStore = entitiesStore; - this.cassandraBasedAuditRepository = cassandraBasedAuditRepository; this.esBasedAuditRepository = esBasedAuditRepository; - this.instanceConverter = instanceConverter; } /** @@ -155,7 +157,8 @@ public List evaluatePolicies(List evaluatePolicies(List getAccessors(List atlasAccessorRequestList) throws AtlasBaseException { + AtlasPerfTracer perf = null; + List ret; + + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "EntityREST.getAccessorss()"); + } + + try { + validateAccessorRequest(atlasAccessorRequestList); + + ret = entitiesStore.getAccessors(atlasAccessorRequestList); + + } finally { + AtlasPerfTracer.log(perf); + } + return ret; + } + private AtlasEntityHeader getAtlasEntityHeader(String entityGuid, String entityId, String entityType) throws AtlasBaseException { - AtlasEntityHeader entityHeader; + AtlasEntityHeader entityHeader = null; if (StringUtils.isNotEmpty(entityGuid)) { AtlasEntityWithExtInfo ret = entitiesStore.getByIdWithoutAuthorization(entityGuid); entityHeader = new AtlasEntityHeader(ret.getEntity()); } else if (StringUtils.isNotEmpty(entityId) && StringUtils.isNotEmpty(entityType)) { - Map attributes = new HashMap<>(); - attributes.put(QUALIFIED_NAME, entityId); - entityHeader = new AtlasEntityHeader(entityType, attributes); + try { + entityHeader = entitiesStore.getAtlasEntityHeaderWithoutAuthorization(null, entityId, entityType); + } catch (AtlasBaseException abe) { + if (abe.getAtlasErrorCode() == AtlasErrorCode.INSTANCE_BY_UNIQUE_ATTRIBUTE_NOT_FOUND) { + Map attributes = new HashMap<>(); + attributes.put(QUALIFIED_NAME, entityId); + entityHeader = new AtlasEntityHeader(entityType, attributes); + } + } } else { throw new AtlasBaseException(BAD_REQUEST, "requires entityGuid or typeName and qualifiedName for entity authorization"); } @@ -478,15 +518,17 @@ public EntityMutationResponse deleteByUniqueAttribute(@PathParam("typeName") Str @Timed public EntityMutationResponse createOrUpdate(AtlasEntityWithExtInfo entity, @QueryParam("replaceClassifications") @DefaultValue("false") boolean replaceClassifications, - @QueryParam("replaceBusinessAttributes") @DefaultValue("false") boolean replaceBusinessAttributes) throws AtlasBaseException { + @QueryParam("replaceBusinessAttributes") @DefaultValue("false") boolean replaceBusinessAttributes, + @QueryParam("overwriteBusinessAttributes") @DefaultValue("false") boolean isOverwriteBusinessAttributes) throws AtlasBaseException { AtlasPerfTracer perf = null; try { if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "EntityREST.createOrUpdate()"); } + validateAttributeLength(Lists.newArrayList(entity.getEntity())); - return entitiesStore.createOrUpdate(new AtlasEntityStream(entity), replaceClassifications, replaceBusinessAttributes); + return entitiesStore.createOrUpdate(new AtlasEntityStream(entity), replaceClassifications, replaceBusinessAttributes, isOverwriteBusinessAttributes); } finally { AtlasPerfTracer.log(perf); } @@ -879,7 +921,8 @@ public AtlasEntitiesWithExtInfo getByGuids(@QueryParam("guid") List guid @Timed public EntityMutationResponse createOrUpdate(AtlasEntitiesWithExtInfo entities, @QueryParam("replaceClassifications") @DefaultValue("false") boolean replaceClassifications, - @QueryParam("replaceBusinessAttributes") @DefaultValue("false") boolean replaceBusinessAttributes) throws AtlasBaseException { + @QueryParam("replaceBusinessAttributes") @DefaultValue("false") boolean replaceBusinessAttributes, + @QueryParam("overwriteBusinessAttributes") @DefaultValue("false") boolean isOverwriteBusinessAttributes) throws AtlasBaseException { AtlasPerfTracer perf = null; try { @@ -888,14 +931,40 @@ public EntityMutationResponse createOrUpdate(AtlasEntitiesWithExtInfo entities, (CollectionUtils.isEmpty(entities.getEntities()) ? 0 : entities.getEntities().size()) + ")"); } + validateAttributeLength(entities.getEntities()); + EntityStream entityStream = new AtlasEntityStream(entities); - return entitiesStore.createOrUpdate(entityStream, replaceClassifications, replaceBusinessAttributes); + return entitiesStore.createOrUpdate(entityStream, replaceClassifications, replaceBusinessAttributes, isOverwriteBusinessAttributes); } finally { AtlasPerfTracer.log(perf); } } + public static void validateAttributeLength(final List entities) throws AtlasBaseException { + List errorMessages = new ArrayList<>(); + + for (final AtlasEntity atlasEntity : entities) { + for (Map.Entry attribute : atlasEntity.getAttributes().entrySet()) { + + if (attribute.getValue() instanceof String && ((String) attribute.getValue()).length() > HUNDRED_THOUSAND) { + + if (ATTRS_WITH_TWO_MILLION_LIMIT.contains(attribute.getKey())) { + if (((String) attribute.getValue()).length() > TWO_MILLION) { + errorMessages.add("Attribute " + attribute.getKey() + " exceeds limit of " + TWO_MILLION + " characters"); + } + } else { + errorMessages.add("Attribute " + attribute.getKey() + " exceeds limit of " + HUNDRED_THOUSAND + " characters"); + } + } + } + + if (errorMessages.size() > 0) { + throw new AtlasBaseException(AtlasType.toJson(errorMessages)); + } + } + } + /** * Bulk API to delete list of entities identified by its GUIDs */ @@ -1092,52 +1161,7 @@ public List getAuditEvents(@PathParam("guid") String guid, @ @QueryParam("offset") @DefaultValue("-1") int offset, @QueryParam("sortBy") String sortBy, @QueryParam("sortOrder") String sortOrder) throws AtlasBaseException { - AtlasPerfTracer perf = null; - - try { - if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { - perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "EntityREST.getAuditEvents(" + guid + ", " + startKey + ", " + count + ")"); - } - - // Enforces authorization for entity-read - try { - entitiesStore.getHeaderById(guid); - } catch (AtlasBaseException e) { - if (e.getAtlasErrorCode() == AtlasErrorCode.INSTANCE_GUID_NOT_FOUND) { - AtlasEntityHeader entityHeader = getEntityHeaderFromPurgedAudit(guid); - - AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, ENTITY_READ, entityHeader), "read entity audit: guid=", guid); - } else { - throw e; - } - } - - List ret = new ArrayList<>(); - - if (sortBy != null || offset > -1) { - ret = cassandraBasedAuditRepository.listEventsV2(guid, auditAction, sortBy, StringUtils.equalsIgnoreCase(sortOrder, "desc"), offset, count); - } else if(auditAction != null) { - ret = cassandraBasedAuditRepository.listEventsV2(guid, auditAction, startKey, count); - } else { - List events = cassandraBasedAuditRepository.listEvents(guid, startKey, count); - - if (events != null) { - for (Object event : events) { - if (event instanceof EntityAuditEventV2) { - ret.add((EntityAuditEventV2) event); - } else if (event instanceof EntityAuditEvent) { - ret.add(instanceConverter.toV2AuditEvent((EntityAuditEvent) event)); - } else { - LOG.warn("unknown entity-audit event type {}. Ignored", event != null ? event.getClass().getCanonicalName() : "null"); - } - } - } - } - - return ret; - } finally { - AtlasPerfTracer.log(perf); - } + throw new AtlasBaseException(DEPRECATED_API, "/entity/auditSearch"); } @POST @@ -1185,18 +1209,57 @@ public EntityAuditSearchResult searchAuditEvents(AuditSearchParams parameters) t perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "EntityREST.searchAuditEvents"); } - AtlasAuthorizationUtils.verifyAccess(new AtlasAdminAccessRequest(AtlasPrivilege.ADMIN_ENTITY_AUDITS), "search audit events!"); - String dslString = parameters.getQueryString(); EntityAuditSearchResult ret = esBasedAuditRepository.searchEvents(dslString); + scrubAndSetEntityAudits(ret, parameters.getSuppressLogs(), parameters.getAttributes()); + return ret; } finally { AtlasPerfTracer.log(perf); } } + private void scrubAndSetEntityAudits(EntityAuditSearchResult result, boolean suppressLogs, Set attributes) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("scrubEntityAudits"); + for (EntityAuditEventV2 event : result.getEntityAudits()) { + try { + AtlasSearchResult ret = new AtlasSearchResult(); + AtlasEntityWithExtInfo entityWithExtInfo = entitiesStore.getByIdWithoutAuthorization(event.getEntityId()); + AtlasEntityHeader entityHeader = new AtlasEntityHeader(entityWithExtInfo.getEntity()); + ret.addEntity(entityHeader); + AtlasSearchResultScrubRequest request = new AtlasSearchResultScrubRequest(typeRegistry, ret); + AtlasAuthorizationUtils.scrubSearchResults(request, suppressLogs); + if(entityHeader.getScrubbed()!= null && entityHeader.getScrubbed()){ + event.setDetail(null); + } + Map entityAttrs = entityHeader.getAttributes(); + if(attributes == null) entityAttrs.clear(); + else entityAttrs.keySet().retainAll(attributes); + + event.setEntityDetail(entityHeader); + + } catch (AtlasBaseException e) { + if (e.getAtlasErrorCode() == AtlasErrorCode.INSTANCE_GUID_NOT_FOUND) { + try { + AtlasEntityHeader entityHeader = event.getEntityHeader(); + AtlasSearchResult ret = new AtlasSearchResult(); + ret.addEntity(entityHeader); + AtlasSearchResultScrubRequest request = new AtlasSearchResultScrubRequest(typeRegistry, ret); + AtlasAuthorizationUtils.scrubSearchResults(request, suppressLogs); + if(entityHeader.getScrubbed()!= null && entityHeader.getScrubbed()){ + event.setDetail(null); + } + } catch (AtlasBaseException abe) { + throw abe; + } + } + } + } + RequestContext.get().endMetricRecord(metric); + } + @GET @Path("bulk/headers") @Produces(Servlets.JSON_MEDIA_TYPE) @@ -1215,7 +1278,7 @@ public AtlasEntityHeaders getEntityHeaders(@QueryParam("tagUpdateStartTime") lon perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "EntityREST.getEntityHeaders(" + tagUpdateStartTime + ", " + tagUpdateEndTime + ")"); } - ClassificationAssociator.Retriever associator = new ClassificationAssociator.Retriever(typeRegistry, cassandraBasedAuditRepository); + ClassificationAssociator.Retriever associator = new ClassificationAssociator.Retriever(typeRegistry, esBasedAuditRepository); return associator.get(tagUpdateStartTime, tagUpdateEndTime); } finally { AtlasPerfTracer.log(perf); @@ -1630,7 +1693,7 @@ public BulkImportResponse importBMAttributes(@FormDataParam("file") InputStream } private AtlasEntityHeader getEntityHeaderFromPurgedAudit(String guid) throws AtlasBaseException { - List auditEvents = cassandraBasedAuditRepository.listEventsV2(guid, EntityAuditActionV2.ENTITY_PURGE, null, (short)1); + List auditEvents = esBasedAuditRepository.listEventsV2(guid, EntityAuditActionV2.ENTITY_PURGE, null, (short)1); AtlasEntityHeader ret = CollectionUtils.isNotEmpty(auditEvents) ? auditEvents.get(0).getEntityHeader() : null; if (ret == null) { @@ -1661,8 +1724,222 @@ public void repairIndex() throws AtlasBaseException { } + /** + * repairHasLineage API to correct haslineage attribute of entity. + */ + @POST + @Path("/repairhaslineage") + @Consumes(Servlets.JSON_MEDIA_TYPE) + public void repairHasLineage(AtlasHasLineageRequests request) throws AtlasBaseException { + + AtlasPerfTracer perf = null; + + try { + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "EntityREST.repairHasLineage(" + request + ")"); + } + + entitiesStore.repairHasLineage(request); + } finally { + AtlasPerfTracer.log(perf); + } + } + private boolean hasNoGUIDAndTypeNameAttributes(ClassificationAssociateRequest request) { return (request == null || (CollectionUtils.isEmpty(request.getEntityGuids()) && (CollectionUtils.isEmpty(request.getEntitiesUniqueAttributes()) || request.getEntityTypeName() == null))); } + + private void validateAccessorRequest(List atlasAccessorRequestList) throws AtlasBaseException { + + if (CollectionUtils.isEmpty(atlasAccessorRequestList)) { + throw new AtlasBaseException(BAD_REQUEST, "Requires list of AtlasAccessor"); + } else { + + for (AtlasAccessorRequest accessorRequest : atlasAccessorRequestList) { + try { + if (StringUtils.isEmpty(accessorRequest.getAction())) { + throw new AtlasBaseException(BAD_REQUEST, "Requires action parameter"); + } + + AtlasPrivilege action = null; + try { + action = AtlasPrivilege.valueOf(accessorRequest.getAction()); + } catch (IllegalArgumentException el) { + throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Invalid action provided " + accessorRequest.getAction()); + } + + switch (action) { + case ENTITY_READ: + case ENTITY_CREATE: + case ENTITY_UPDATE: + case ENTITY_DELETE: + validateEntityForAccessors(accessorRequest.getGuid(), accessorRequest.getQualifiedName(), accessorRequest.getTypeName()); + break; + + case ENTITY_READ_CLASSIFICATION: + case ENTITY_ADD_CLASSIFICATION: + case ENTITY_UPDATE_CLASSIFICATION: + case ENTITY_REMOVE_CLASSIFICATION: + if (StringUtils.isEmpty(accessorRequest.getClassification())) { + throw new AtlasBaseException(BAD_REQUEST, "Requires classification"); + } + validateEntityForAccessors(accessorRequest.getGuid(), accessorRequest.getQualifiedName(), accessorRequest.getTypeName()); + break; + + case ENTITY_ADD_LABEL: + case ENTITY_REMOVE_LABEL: + if (StringUtils.isEmpty(accessorRequest.getLabel())) { + throw new AtlasBaseException(BAD_REQUEST, "Requires label"); + } + validateEntityForAccessors(accessorRequest.getGuid(), accessorRequest.getQualifiedName(), accessorRequest.getTypeName()); + break; + + case ENTITY_UPDATE_BUSINESS_METADATA: + if (StringUtils.isEmpty(accessorRequest.getBusinessMetadata())) { + throw new AtlasBaseException(BAD_REQUEST, "Requires businessMetadata"); + } + validateEntityForAccessors(accessorRequest.getGuid(), accessorRequest.getQualifiedName(), accessorRequest.getTypeName()); + break; + + + case RELATIONSHIP_ADD: + case RELATIONSHIP_UPDATE: + case RELATIONSHIP_REMOVE: + if (StringUtils.isEmpty(accessorRequest.getRelationshipTypeName())) { + throw new AtlasBaseException(BAD_REQUEST, "Requires relationshipTypeName"); + } + + validateEntityForAccessors(accessorRequest.getEntityGuidEnd1(), accessorRequest.getEntityQualifiedNameEnd1(), accessorRequest.getEntityTypeEnd1()); + validateEntityForAccessors(accessorRequest.getEntityGuidEnd2(), accessorRequest.getEntityQualifiedNameEnd2(), accessorRequest.getEntityTypeEnd2()); + break; + + + case TYPE_READ: + case TYPE_CREATE: + case TYPE_UPDATE: + case TYPE_DELETE: + if (StringUtils.isEmpty(accessorRequest.getTypeName())) { + throw new AtlasBaseException(BAD_REQUEST, "Requires typeName of the asset"); + } + break; + + default: + throw new AtlasBaseException(BAD_REQUEST, "Please add validation support for action {}", accessorRequest.getAction()); + + } + + } catch (AtlasBaseException e) { + e.getErrorDetailsMap().put("accessorRequest", AtlasType.toJson(accessorRequest)); + throw e; + } + } + } + } + + private void validateEntityForAccessors(String guid, String qualifiedName, String typeName) throws AtlasBaseException { + if (StringUtils.isEmpty(typeName) && StringUtils.isNotEmpty(qualifiedName)) { + throw new AtlasBaseException(BAD_REQUEST, "Requires typeName of the asset with qualifiedName"); + } + + if (StringUtils.isEmpty(guid) && StringUtils.isEmpty(qualifiedName)) { + throw new AtlasBaseException(BAD_REQUEST, "Requires either qualifiedName or GUID of the asset"); + } + } + + /** + * Repair index for the entity GUID. + * @param guid GUID for the entity + * @return AtlasEntity + * @throws AtlasBaseException + */ + @POST + @Path("/guid/{guid}/repairindex") + public void repairEntityIndex(@PathParam("guid") String guid) throws AtlasBaseException { + Servlets.validateQueryParamLength("guid", guid); + + AtlasAuthorizationUtils.verifyAccess(new AtlasAdminAccessRequest(AtlasPrivilege.ADMIN_REPAIR_INDEX), "Admin Repair Index"); + + AtlasPerfTracer perf = null; + + try { + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "EntityREST.repairEntityIndex(" + guid + ")"); + } + + AtlasEntityWithExtInfo entity = entitiesStore.getById(guid); + Map referredEntities = entity.getReferredEntities(); + RepairIndex repairIndex = new RepairIndex(); + repairIndex.setupGraph(); + + repairIndex.restoreSelective(guid, referredEntities); + } catch (Exception e) { + LOG.error("Exception while repairEntityIndex ", e); + throw new AtlasBaseException(e); + } finally { + AtlasPerfTracer.log(perf); + } + } + + + @POST + @Path("/repairindex/{typename}") + public void repairIndexByTypeName(@PathParam("typename") String typename, @QueryParam("delay") @DefaultValue("0") int delay, @QueryParam("limit") @DefaultValue("1000") int limit, @QueryParam("offset") @DefaultValue("0") int offset, @QueryParam("batchSize") @DefaultValue("1000") int batchSize) throws AtlasBaseException { + Servlets.validateQueryParamLength("typename", typename); + + AtlasPerfTracer perf = null; + + try { + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "EntityREST.repairAllEntitiesIndex"); + } + + AtlasAuthorizationUtils.verifyAccess(new AtlasAdminAccessRequest(AtlasPrivilege.ADMIN_REPAIR_INDEX), "Admin Repair Index"); + + RepairIndex repairIndex = new RepairIndex(); + repairIndex.setupGraph(); + + LOG.info("Repairing index for entities in " + typename); + + long startTime = System.currentTimeMillis(); + + List entityGUIDs = entitiesStore.getEntityGUIDS(typename); + + LOG.info("Entities to repair in " + typename + " " + entityGUIDs.size()); + + if (entityGUIDs.size() > offset + limit) { + entityGUIDs = entityGUIDs.subList(offset, offset + limit); + LOG.info("Updated - Entities to repair in " + typename + " " + entityGUIDs.size()); + } else if (entityGUIDs.size() > offset) { + entityGUIDs = entityGUIDs.subList(offset, entityGUIDs.size()); + LOG.info("Updated - Entities to repair in " + typename + " " + entityGUIDs.size()); + } else { + LOG.info("No entities to repair"); + return; + } + + List> entityGUIDsChunked = Lists.partition(entityGUIDs, batchSize); + for (List guids : entityGUIDsChunked) { + Set guidsToReIndex = new HashSet<>(guids); + repairIndex.restoreByIds(guidsToReIndex); + if (guids.size() >= batchSize && delay > 0) { + try { + LOG.info("Sleep for " + delay + " ms"); + Thread.sleep(delay); + } catch(InterruptedException ex) { + Thread.currentThread().interrupt(); + LOG.info("Thread interrupted at " + typename); + } + } + LOG.info("Repaired index for " + guids.size() + " entities of type" + typename); + } + LOG.info("Repaired index for entities for typeName " + typename + " in " + (System.currentTimeMillis() - startTime) + " ms"); + + } catch (Exception e) { + LOG.error("Exception while repairIndexByTypeName ", e); + throw new AtlasBaseException(e); + } finally { + AtlasPerfTracer.log(perf); + } + } } diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/GlossaryREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/GlossaryREST.java index fdf9ecc323d..cfd507518c6 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/GlossaryREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/GlossaryREST.java @@ -54,7 +54,7 @@ import java.util.Objects; import java.util.Set; -@Path("v2/glossary") +@Path("glossary") @Service @Consumes({Servlets.JSON_MEDIA_TYPE, MediaType.APPLICATION_JSON}) @Produces({Servlets.JSON_MEDIA_TYPE, MediaType.APPLICATION_JSON}) diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/LineageREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/LineageREST.java index ced6e1e996f..829b9aaf287 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/LineageREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/LineageREST.java @@ -19,32 +19,31 @@ package org.apache.atlas.web.rest; +import org.apache.atlas.AtlasConfiguration; import org.apache.atlas.AtlasErrorCode; +import org.apache.atlas.RequestContext; import org.apache.atlas.annotation.Timed; import org.apache.atlas.discovery.AtlasLineageService; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.TypeCategory; -import org.apache.atlas.model.lineage.AtlasLineageInfo; +import org.apache.atlas.model.lineage.*; import org.apache.atlas.model.lineage.AtlasLineageInfo.LineageDirection; import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2; import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.utils.AtlasPerfTracer; +import org.apache.atlas.web.rest.validator.LineageListRequestValidator; +import org.apache.atlas.web.rest.validator.RequestValidator; import org.apache.atlas.web.util.Servlets; import org.apache.commons.collections.MapUtils; import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; import javax.inject.Inject; import javax.inject.Singleton; import javax.servlet.http.HttpServletRequest; -import javax.ws.rs.Consumes; -import javax.ws.rs.DefaultValue; -import javax.ws.rs.GET; -import javax.ws.rs.Path; -import javax.ws.rs.PathParam; -import javax.ws.rs.Produces; -import javax.ws.rs.QueryParam; +import javax.ws.rs.*; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import java.util.HashMap; @@ -53,34 +52,108 @@ /** * REST interface for an entity's lineage information */ -@Path("v2/lineage") +@Path("lineage") @Singleton @Service @Consumes({Servlets.JSON_MEDIA_TYPE, MediaType.APPLICATION_JSON}) @Produces({Servlets.JSON_MEDIA_TYPE, MediaType.APPLICATION_JSON}) public class LineageREST { + private static final Logger LOG = LoggerFactory.getLogger(LineageREST.class); private static final Logger PERF_LOG = AtlasPerfTracer.getPerfLogger("rest.LineageREST"); private static final String PREFIX_ATTR = "attr:"; private final AtlasTypeRegistry typeRegistry; private final AtlasLineageService atlasLineageService; + private final RequestValidator lineageListRequestValidator; private static final String DEFAULT_DIRECTION = "BOTH"; - private static final String DEFAULT_DEPTH = "3"; + private static final String DEFAULT_DEPTH = "3"; + private static final String DEFAULT_PAGE = "-1"; + private static final String DEFAULT_RECORD_PER_PAGE = "-1"; @Context private HttpServletRequest httpServletRequest; @Inject - public LineageREST(AtlasTypeRegistry typeRegistry, AtlasLineageService atlasLineageService) { + public LineageREST(AtlasTypeRegistry typeRegistry, AtlasLineageService atlasLineageService, LineageListRequestValidator lineageListRequestValidator) { this.typeRegistry = typeRegistry; this.atlasLineageService = atlasLineageService; + this.lineageListRequestValidator = lineageListRequestValidator; } /** * Returns lineage info about entity. - * @param guid - unique entity id + * @return AtlasLineageInfo + * @throws AtlasBaseException + * @HTTP 200 If Lineage exists for the given entity + * @HTTP 400 Bad query parameters + * @HTTP 404 If no lineage is found for the given entity + */ + @POST + @Path("/{guid}") + @Consumes(Servlets.JSON_MEDIA_TYPE) + @Produces(Servlets.JSON_MEDIA_TYPE) + @Timed + public AtlasLineageOnDemandInfo getLineageGraph(@PathParam("guid") String guid, + LineageOnDemandRequest lineageOnDemandRequest) throws AtlasBaseException { + if (!AtlasConfiguration.LINEAGE_ON_DEMAND_ENABLED.getBoolean()) { + LOG.warn("LineageREST: "+ AtlasErrorCode.LINEAGE_ON_DEMAND_NOT_ENABLED.getFormattedErrorMessage(AtlasConfiguration.LINEAGE_ON_DEMAND_ENABLED.getPropertyName())); + + throw new AtlasBaseException(AtlasErrorCode.LINEAGE_ON_DEMAND_NOT_ENABLED, AtlasConfiguration.LINEAGE_ON_DEMAND_ENABLED.getPropertyName()); + } + + Servlets.validateQueryParamLength("guid", guid); + + AtlasPerfTracer perf = null; + + try { + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "LineageREST.getOnDemandLineageGraph(" + guid + "," + lineageOnDemandRequest + ")"); + } + + return atlasLineageService.getAtlasLineageInfo(guid, lineageOnDemandRequest); + } finally { + AtlasPerfTracer.log(perf); + } + } + + /** + * Returns lineage list info about entity. + * @return AtlasLineageListInfo + * @throws AtlasBaseException + * @HTTP 200 If Lineage exists for the given entity + * @HTTP 400 Bad query parameters + */ + @POST + @Path("/list") + @Consumes(Servlets.JSON_MEDIA_TYPE) + @Produces(Servlets.JSON_MEDIA_TYPE) + @Timed + public AtlasLineageListInfo getLineageList(LineageListRequest lineageListRequest) throws AtlasBaseException { + lineageListRequestValidator.validate(lineageListRequest); + + String guid = lineageListRequest.getGuid(); + Servlets.validateQueryParamLength("guid", guid); + AtlasPerfTracer perf = null; + + RequestContext.get().setIncludeMeanings(!lineageListRequest.isExcludeMeanings()); + RequestContext.get().setIncludeClassifications(!lineageListRequest.isExcludeClassifications()); + + try { + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "LineageREST.getLineageList(" + guid + "," + lineageListRequest + ")"); + + return atlasLineageService.getLineageListInfoOnDemand(guid, lineageListRequest); + } finally { + AtlasPerfTracer.log(perf); + } + } + + /** + * Returns lineage info about entity. + * + * @param guid - unique entity id * @param direction - input, output or both - * @param depth - number of hops for lineage + * @param depth - number of hops for lineage * @return AtlasLineageInfo * @throws AtlasBaseException * @HTTP 200 If Lineage exists for the given entity @@ -91,9 +164,12 @@ public LineageREST(AtlasTypeRegistry typeRegistry, AtlasLineageService atlasLine @Path("/{guid}") @Timed public AtlasLineageInfo getLineageGraph(@PathParam("guid") String guid, - @QueryParam("direction") @DefaultValue(DEFAULT_DIRECTION) LineageDirection direction, + @QueryParam("direction") @DefaultValue(DEFAULT_DIRECTION) LineageDirection direction, @QueryParam("depth") @DefaultValue(DEFAULT_DEPTH) int depth, - @QueryParam("hideProcess") @DefaultValue("false") boolean hideProcess) throws AtlasBaseException { + @QueryParam("hideProcess") @DefaultValue("false") boolean hideProcess, + @QueryParam("offset") @DefaultValue(DEFAULT_PAGE) int offset, + @QueryParam("limit") @DefaultValue(DEFAULT_RECORD_PER_PAGE) int limit, + @QueryParam("calculateRemainingVertexCounts") @DefaultValue("false") boolean calculateRemainingVertexCounts) throws AtlasBaseException { Servlets.validateQueryParamLength("guid", guid); AtlasPerfTracer perf = null; @@ -101,27 +177,56 @@ public AtlasLineageInfo getLineageGraph(@PathParam("guid") String guid, try { if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "LineageREST.getLineageGraph(" + guid + "," + direction + - "," + depth + ")"); + "," + depth + ")"); } - return atlasLineageService.getAtlasLineageInfo(guid, direction, depth, hideProcess); + return atlasLineageService.getAtlasLineageInfo(guid, direction, depth, hideProcess, offset, limit, calculateRemainingVertexCounts); } finally { AtlasPerfTracer.log(perf); } } + /** * Returns lineage info about entity. * + * @param request - AtlasLineageRequest + * @return AtlasLineageInfo + * @throws AtlasBaseException + * @HTTP 200 If Lineage exists for the given entity + * @HTTP 400 Bad query parameters + * @HTTP 404 If no lineage is found for the given entity + */ + @POST + @Path("/getlineage") + @Timed + public AtlasLineageInfo getLineageGraph(AtlasLineageRequest request) throws AtlasBaseException { + AtlasPerfTracer perf = null; + request.performValidation(); + + try { + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "LineageREST.getLineageGraph(" + request + ")"); + } + + return atlasLineageService.getAtlasLineageInfo(request); + } finally { + AtlasPerfTracer.log(perf); + } + } + + /** + * Returns lineage info about entity. + *

* In addition to the typeName path parameter, attribute key-value pair(s) can be provided in the following format - * + *

* attr:= - * + *

* NOTE: The attrName and attrValue should be unique across entities, eg. qualifiedName * - * @param typeName - typeName of entity + * @param typeName - typeName of entity * @param direction - input, output or both - * @param depth - number of hops for lineage + * @param depth - number of hops for lineage * @return AtlasLineageInfo * @throws AtlasBaseException * @HTTP 200 If Lineage exists for the given entity @@ -182,4 +287,5 @@ private AtlasEntityType ensureEntityType(String typeName) throws AtlasBaseExcept return ret; } -} + +} \ No newline at end of file diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/MigrationREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/MigrationREST.java new file mode 100644 index 00000000000..b1cb307550c --- /dev/null +++ b/webapp/src/main/java/org/apache/atlas/web/rest/MigrationREST.java @@ -0,0 +1,483 @@ +package org.apache.atlas.web.rest; + +import org.apache.atlas.AtlasErrorCode; +import org.apache.atlas.RequestContext; +import org.apache.atlas.annotation.Timed; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.discovery.IndexSearchParams; +import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.model.instance.EntityMutationResponse; +import org.apache.atlas.repository.graph.GraphHelper; +import org.apache.atlas.repository.graphdb.*; +import org.apache.atlas.repository.store.graph.AtlasEntityStore; +import org.apache.atlas.repository.store.graph.v2.AtlasEntityStream; +import org.apache.atlas.repository.store.graph.v2.EntityStream; +import org.apache.atlas.repository.store.users.KeycloakStore; +import org.apache.atlas.transformer.PreProcessorPoliciesTransformer; +import org.apache.atlas.utils.AtlasPerfTracer; +import org.apache.atlas.v1.model.instance.Id; +import org.apache.atlas.web.util.Servlets; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; +import org.keycloak.representations.idm.GroupRepresentation; +import org.keycloak.representations.idm.RoleRepresentation; +import org.keycloak.representations.idm.UserRepresentation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import javax.inject.Inject; +import javax.inject.Singleton; +import javax.ws.rs.*; +import javax.ws.rs.core.MediaType; +import java.util.*; +import java.util.stream.Collectors; + +import static org.apache.atlas.keycloak.client.AtlasKeycloakClient.getKeycloakClient; +import static org.apache.atlas.repository.Constants.*; + +@Path("migration") +@Singleton +@Service +@Consumes({Servlets.JSON_MEDIA_TYPE, MediaType.APPLICATION_JSON}) +@Produces({Servlets.JSON_MEDIA_TYPE, MediaType.APPLICATION_JSON}) +public class MigrationREST { + private static final Logger LOG = LoggerFactory.getLogger(MigrationREST.class); + private static final Logger PERF_LOG = AtlasPerfTracer.getPerfLogger("rest.MigrationREST"); + + private static final String COLL_ADMIN_ROLE_PATTERN = "collection_admins_%s"; + private static final String COLL_VIEWER_ROLE_PATTERN = "collection_viewer_%s"; + public static final String CONN_NAME_PATTERN = "connection_admins_%s"; + + private final AtlasEntityStore entityStore; + private final PreProcessorPoliciesTransformer transformer; + private KeycloakStore keycloakStore; + private AtlasGraph graph; + + @Inject + public MigrationREST(AtlasEntityStore entityStore, AtlasGraph graph) { + this.entityStore = entityStore; + this.graph = graph; + this.transformer = new PreProcessorPoliciesTransformer(); + keycloakStore = new KeycloakStore(); + } + + @POST + @Path("bootstrap/connections") + @Timed + public EntityMutationResponse bootstrapConnections(AtlasEntity.AtlasEntitiesWithExtInfo entities) throws Exception { + AtlasPerfTracer perf = null; + EntityMutationResponse response = new EntityMutationResponse(); + try { + + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "MigrationREST.bootstrapConnections(entityCount=" + + (CollectionUtils.isEmpty(entities.getEntities()) ? 0 : entities.getEntities().size()) + ")"); + } + + for (AtlasEntity entity : entities.getEntities()) { + if (entity.getTypeName().equalsIgnoreCase(CONNECTION_ENTITY_TYPE)) { + //create connection role + String roleName = String.format(CONN_NAME_PATTERN, entity.getGuid()); + + List adminUsers = (List) entity.getAttribute(ATTR_ADMIN_USERS); + List adminGroups = (List) entity.getAttribute(ATTR_ADMIN_GROUPS); + List adminRoles = (List) entity.getAttribute(ATTR_ADMIN_ROLES); + if (CollectionUtils.isEmpty(adminUsers)) { + adminUsers = new ArrayList<>(); + } + + if (StringUtils.isNotEmpty(entity.getCreatedBy())) { + adminUsers.add(entity.getCreatedBy()); + } + + if (CollectionUtils.isEmpty(adminGroups)) { + adminGroups = new ArrayList<>(); + } + + if (CollectionUtils.isEmpty(adminRoles)) { + adminRoles = new ArrayList<>(); + } + + entity.setAttribute(ATTR_ADMIN_USERS, adminUsers); + + RoleRepresentation role = keycloakStore.getRole(roleName); + if (role == null) { + createCompositeRole(roleName, adminUsers, adminGroups, adminRoles); + } else { + updateCompositeRole(role, adminUsers, adminGroups, adminRoles); + } + AtlasEntity.AtlasEntitiesWithExtInfo policiesExtInfo = transformer.transform(entity); + try { + RequestContext.get().setSkipAuthorizationCheck(true); + EntityStream entityStream = new AtlasEntityStream(policiesExtInfo); + EntityMutationResponse policyResponse = entityStore.createOrUpdate(entityStream, false); + response.setMutatedEntities(policyResponse.getMutatedEntities()); + LOG.info("Created bootstrap policies for connection"); + } finally { + RequestContext.get().setSkipAuthorizationCheck(false); + } + } + } + + return response; + } finally { + AtlasPerfTracer.log(perf); + } + } + + @POST + @Path("bootstrap/collections") + @Timed + public EntityMutationResponse bootstrapCollections(AtlasEntity.AtlasEntitiesWithExtInfo entities) throws Exception { + AtlasPerfTracer perf = null; + EntityMutationResponse response = new EntityMutationResponse(); + try { + + for (AtlasEntity entity : entities.getEntities()) { + if (entity.getTypeName().equalsIgnoreCase(QUERY_COLLECTION_ENTITY_TYPE)) { + createCollectionAdminRole(entity); + createCollectionViewerRole(entity); + + //create bootstrap policies + AtlasEntity.AtlasEntitiesWithExtInfo policies = transformer.transform(entity); + try { + RequestContext.get().setSkipAuthorizationCheck(true); + + EntityStream entityStream = new AtlasEntityStream(policies); + EntityMutationResponse policyResponse = entityStore.createOrUpdate(entityStream, false); + response.setMutatedEntities(policyResponse.getMutatedEntities()); + LOG.info("Created bootstrap policies for connection"); + } finally { + RequestContext.get().setSkipAuthorizationCheck(false); + } + } + } + + return response; + } finally { + AtlasPerfTracer.log(perf); + } + + } + + @GET + @Path("search/{typeName}") + @Timed + public List searchForType(@PathParam("typeName") String typeName, @QueryParam("minExtInfo") @DefaultValue("false") boolean minExtInfo, @QueryParam("ignoreRelationships") @DefaultValue("false") boolean ignoreRelationships) throws Exception { + AtlasPerfTracer perf = null; + try { + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "MigrationREST.searchUsingDslQuery(" + typeName + ")"); + } + + List ret = new ArrayList<>(); + + List allowedTypeNames = Arrays.asList("Persona", "Purpose"); + if (!allowedTypeNames.contains(typeName)) { + throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_INVALID, typeName); + } + + IndexSearchParams indexSearchParams = new IndexSearchParams(); + + Map dsl = getMap("size", 0); + + List> mustClauseList = new ArrayList<>(); + mustClauseList.add(getMap("term", getMap("__typeName.keyword", typeName))); + mustClauseList.add(getMap("match", getMap("__state", Id.EntityState.ACTIVE))); + + dsl.put("query", getMap("bool", getMap("must", mustClauseList))); + + dsl.put("sort", Collections.singleton(getMap("__guid", getMap("order", "desc")))); + + indexSearchParams.setDsl(dsl); + + int from = 0; + int size = 100; + boolean found = true; + + do { + dsl.put("from", from); + dsl.put("size", size); + indexSearchParams.setDsl(dsl); + List entities = getEntitiesByIndexSearch(indexSearchParams, minExtInfo, ignoreRelationships); + + if (CollectionUtils.isNotEmpty(entities)) { + ret.addAll(entities); + } else { + found = false; + } + from += size; + + } while (found && ret.size() % size == 0); + + return ret; + + } finally { + AtlasPerfTracer.log(perf); + } + } + + private List getEntitiesByIndexSearch(IndexSearchParams indexSearchParams, Boolean minExtInfo, boolean ignoreRelationships) throws AtlasBaseException { + List entities = new ArrayList<>(); + String indexName = "janusgraph_vertex_index"; + AtlasIndexQuery indexQuery = graph.elasticsearchQuery(indexName); + DirectIndexQueryResult indexQueryResult = indexQuery.vertices(indexSearchParams); + Iterator iterator = indexQueryResult.getIterator(); + + while (iterator.hasNext()) { + AtlasIndexQuery.Result result = iterator.next(); + AtlasVertex vertex = result.getVertex(); + + if (vertex == null) { + LOG.warn("vertex is null"); + continue; + } + + AtlasEntity entity = new AtlasEntity(); + entity.setGuid(GraphHelper.getGuid(vertex)); + entity.setTypeName(GraphHelper.getTypeName(vertex)); + + // Use a method to extract attributes from vertex + setVertexAttributes(vertex, entity); + + // Use a method to get policy entities + List policyEntities = getPolicyEntities(vertex); + if (!policyEntities.isEmpty()) { + entity.setAttribute("policies", policyEntities); + } + // Check if entity is not null before adding it to the list + if (entity != null) { + entities.add(entity); + } + } + + return entities; + } + + private void setVertexAttributes(AtlasVertex vertex, AtlasEntity entity) { + List attributes = Arrays.asList("name", "qualifiedName", "roleId"); + for (String attribute : attributes) { + entity.setAttribute(attribute, vertex.getProperty(attribute, String.class)); + } + entity.setCustomAttributes(GraphHelper.getCustomAttributes(vertex)); + } + + private List getPolicyEntities(AtlasVertex vertex) { + List policyEntities = new ArrayList<>(); + Iterator vertices = vertex.query().direction(AtlasEdgeDirection.OUT) + .label("__AccessControl.policies").vertices().iterator(); + + while (vertices.hasNext()) { + AtlasVertex policyVertex = vertices.next(); + if (policyVertex != null) { + AtlasEntity policyEntity = new AtlasEntity(); + policyEntity.setGuid(GraphHelper.getGuid(policyVertex)); + policyEntity.setTypeName(GraphHelper.getTypeName(policyVertex)); + + // Use a method to extract attributes from policy vertex + setVertexAttributes(policyVertex, policyEntity); + + policyEntity.setCustomAttributes(GraphHelper.getCustomAttributes(policyVertex)); + policyEntities.add(policyEntity); + } + } + return policyEntities; + } + + private Map getMap(String key, Object value) { + Map map = new HashMap<>(); + map.put(key, value); + return map; + } + + private RoleRepresentation createCollectionAdminRole(AtlasEntity collection) throws AtlasBaseException { + //create Admin role + List adminUsers = (List) collection.getAttribute(ATTR_ADMIN_USERS); + List adminGroups = (List) collection.getAttribute(ATTR_ADMIN_GROUPS); + List adminRoles = (List) collection.getAttribute(ATTR_ADMIN_ROLES); + + if (adminUsers == null) { + adminUsers = new ArrayList<>(); + } + + if (StringUtils.isNotEmpty(collection.getCreatedBy())) { + adminUsers.add(collection.getCreatedBy()); + } + + if (adminGroups == null) { + adminGroups = new ArrayList<>(); + } + + if (adminRoles == null) { + adminRoles = new ArrayList<>(); + } + + String adminRoleName = String.format(COLL_ADMIN_ROLE_PATTERN, collection.getGuid()); + RoleRepresentation role = keycloakStore.getRole(adminRoleName); + if (role == null) { + createCompositeRole(adminRoleName, adminUsers, adminGroups, adminRoles); + } else { + updateCompositeRole(role, adminUsers, adminGroups, adminRoles); + } + + + return role; + } + + private RoleRepresentation createCollectionViewerRole(AtlasEntity collection) throws AtlasBaseException { + //create viewers role + String viewerRoleName = String.format(COLL_VIEWER_ROLE_PATTERN, collection.getGuid()); + List viewerUsers = (List) collection.getAttribute(ATTR_VIEWER_USERS); + List viewerGroups = (List) collection.getAttribute(ATTR_VIEWER_GROUPS); + + if (viewerUsers == null) { + viewerUsers = new ArrayList<>(); + } + + if (viewerGroups == null) { + viewerGroups = new ArrayList<>(); + } + + RoleRepresentation role = keycloakStore.getRole(viewerRoleName); + if (role == null) { + createCompositeRole(viewerRoleName, viewerUsers, viewerGroups, new ArrayList<>()); + } else { + updateCompositeRole(role, viewerUsers, viewerGroups, new ArrayList<>()); + } + return role; + } + + private void updateCompositeRole(RoleRepresentation role, List users, List groups, List roles) throws AtlasBaseException { + List currentUsers = getKeycloakClient().getRoleUserMembers(role.getName()).stream().collect(Collectors.toList()); + List currentGroups = getKeycloakClient().getRoleGroupMembers(role.getName()).stream().collect(Collectors.toList()); + List currentRoles = getKeycloakClient().getRoleComposites(role.getName()).stream().collect(Collectors.toList()); + + //Find users to add and remove from role, if the user is already in the role, don't add it again + List usersToAdd = users.stream().filter(user -> currentUsers.stream().noneMatch(userRep -> userRep.getUsername().equals(user))).collect(Collectors.toList()); + List usersToRemove = currentUsers.stream().filter(user -> users.stream().noneMatch(userRep -> userRep.equals(user.getUsername()))).collect(Collectors.toList()); + + //Find groups to add and remove from role, if the group is already in the role, don't add it again + List groupsToAdd = groups.stream().filter(group -> currentGroups.stream().noneMatch(groupRep -> groupRep.getName().equals(group))).collect(Collectors.toList()); + List groupsToRemove = currentGroups.stream().filter(group -> groups.stream().noneMatch(groupRep -> groupRep.equals(group.getName()))).collect(Collectors.toList()); + + //Find roles to add and remove from role, if the role is already in the role, don't add it again + List rolesToAdd = roles.stream().filter(roleToAdd -> currentRoles.stream().noneMatch(roleRep -> roleRep.getId().equals(roleToAdd))).collect(Collectors.toList()); + List rolesToRemove = currentRoles.stream().filter(roleToRemove -> roles.stream().noneMatch(roleRep -> roleRep.equals(roleToRemove.getId()))).collect(Collectors.toList()); + + //Add users to role + for (String userName : usersToAdd) { + List matchedUsers = getKeycloakClient().searchUserByUserName(userName); + Optional keyUserOptional = matchedUsers.stream().filter(x -> userName.equals(x.getUsername())).findFirst(); + if (keyUserOptional.isPresent()) { + UserRepresentation keyUser = keyUserOptional.get(); + getKeycloakClient().addRealmLevelRoleMappingsForUser(keyUser.getId(), Collections.singletonList(role)); + } + } + //Remove users from role + for (UserRepresentation userToRemove : usersToRemove) { + getKeycloakClient().deleteRealmLevelRoleMappingsForUser(userToRemove.getId(), Collections.singletonList(role)); + } + + //Add groups to role + for (String groupName : groupsToAdd) { + List matchedGroups = getKeycloakClient().searchGroupByName(groupName, 0, 1); + Optional keyGroupOptional = matchedGroups.stream().filter(x -> groupName.equals(x.getName())).findFirst(); + if (keyGroupOptional.isPresent()) { + GroupRepresentation keyGroup = keyGroupOptional.get(); + getKeycloakClient().addRealmLevelRoleMappingsForGroup(keyGroup.getId(), Collections.singletonList(role)); + } + } + //Remove groups from role + for (GroupRepresentation groupToRemove : groupsToRemove) { + getKeycloakClient().deleteRealmLevelRoleMappingsForGroup(groupToRemove.getId(), Collections.singletonList(role)); + } + + //Add roles to role + for (String roleId : rolesToAdd) { + RoleRepresentation roleToAdd = getKeycloakClient().getRoleById(roleId); + getKeycloakClient().addComposites(role.getName(), Collections.singletonList(roleToAdd)); + } + + //Remove roles from role + for (RoleRepresentation roleToRemove : rolesToRemove) { + getKeycloakClient().deleteComposites(role.getName(), Collections.singletonList(roleToRemove)); + } + } + + private void createCompositeRole(String roleName, List users, List groups, List roles) throws AtlasBaseException { + List roleUsers = new ArrayList<>(); + List roleGroups = new ArrayList<>(); + List roleRoles = new ArrayList<>(); + + if (CollectionUtils.isNotEmpty(users)) { + roleUsers = users.stream().map(user -> { + List matchedUsers = null; + try { + matchedUsers = getKeycloakClient().searchUserByUserName(user); + Optional keyUserOptional = matchedUsers.stream().filter(x -> user.equals(x.getUsername())).findFirst(); + if (keyUserOptional.isPresent()) { + return keyUserOptional.get(); + } else { + LOG.warn("User {} not found in keycloak", user); + } + } catch (AtlasBaseException e) { + LOG.error("Failed to get user by name {}", user, e); + } + return null; + }).filter(Objects::nonNull).collect(Collectors.toList()); + } + + if (CollectionUtils.isNotEmpty(groups)) { + roleGroups = new ArrayList<>(); + for(String group : groups) { + List matchedGroups = getKeycloakClient().searchGroupByName(group, 0, 1); + Optional keyGroupOptional = matchedGroups.stream().filter(x -> group.equals(x.getName())).findFirst(); + if (keyGroupOptional.isPresent()) { + roleGroups.add(keyGroupOptional.get()); + } else { + LOG.warn("Group {} not found in keycloak", group); + } + } + } + + if (CollectionUtils.isNotEmpty(roles)) { + roleRoles = new ArrayList<>(); + for(String role: roles) { + List matchedRoles = getKeycloakClient().getAllRoles(); + Optional keyRoleOptional = matchedRoles.stream().filter(x -> role.equals(x.getId())).findFirst(); + if (keyRoleOptional.isPresent()) { + roleRoles.add(keyRoleOptional.get()); + } else { + LOG.warn("Role {} not found in keycloak", role); + } + } + } + + RoleRepresentation role = new RoleRepresentation(); + role.setName(roleName); + role.setComposite(true); + + RoleRepresentation createdRole = keycloakStore.createRole(role); + if (createdRole == null) { + throw new AtlasBaseException(AtlasErrorCode.INTERNAL_ERROR, "Failed to create role " + roleName); + } + + //Add realm role to users + for (UserRepresentation user : roleUsers) { + getKeycloakClient().addRealmLevelRoleMappingsForUser(user.getId(), Collections.singletonList(createdRole)); + } + + //Add realm role to groups + for (GroupRepresentation group : roleGroups) { + getKeycloakClient().addRealmLevelRoleMappingsForGroup(group.getId(), Collections.singletonList(createdRole)); + + } + + //Add realm role to roles + for (RoleRepresentation roleToAdd : roleRoles) { + getKeycloakClient().addComposites(createdRole.getName(), Collections.singletonList(roleToAdd)); + } + + } +} \ No newline at end of file diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/RelationshipREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/RelationshipREST.java index 6bf41f5c260..0f5b2f48f2b 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/RelationshipREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/RelationshipREST.java @@ -39,7 +39,7 @@ /** * REST interface for entity relationships. */ -@Path("v2/relationship") +@Path("relationship") @Singleton @Service @Consumes({Servlets.JSON_MEDIA_TYPE, MediaType.APPLICATION_JSON}) @@ -68,7 +68,6 @@ public AtlasRelationship create(AtlasRelationship relationship) throws AtlasBase } return relationshipStore.create(relationship); - } finally { AtlasPerfTracer.log(perf); } @@ -88,7 +87,6 @@ public List createOrUpdate(List relationsh } return relationshipStore.createOrUpdate(relationships); - } finally { AtlasPerfTracer.log(perf); } @@ -108,7 +106,6 @@ public AtlasRelationship update(AtlasRelationship relationship) throws AtlasBase } return relationshipStore.update(relationship); - } finally { AtlasPerfTracer.log(perf); } @@ -158,9 +155,8 @@ public void deleteById(@PathParam("guid") String guid) throws AtlasBaseException AtlasPerfTracer perf = null; try { - if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "RelationshipREST.deleteById(" + guid + ")"); - } relationshipStore.deleteById(guid); } finally { @@ -190,7 +186,6 @@ public void deleteByIds(List guids) throws AtlasBaseException { if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "RelationshipREST.deleteById(" + guids.size() + ")"); } - relationshipStore.deleteByIds(guids); } finally { AtlasPerfTracer.log(perf); diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/TaskREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/TaskREST.java new file mode 100644 index 00000000000..e51c598c962 --- /dev/null +++ b/webapp/src/main/java/org/apache/atlas/web/rest/TaskREST.java @@ -0,0 +1,140 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.web.rest; + +import org.apache.atlas.annotation.Timed; +import org.apache.atlas.authorize.AtlasAdminAccessRequest; +import org.apache.atlas.authorize.AtlasAuthorizationUtils; +import org.apache.atlas.authorize.AtlasPrivilege; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.tasks.AtlasTask; +import org.apache.atlas.model.tasks.TaskSearchParams; +import org.apache.atlas.model.tasks.TaskSearchResult; +import org.apache.atlas.tasks.TaskService; +import org.apache.atlas.utils.AtlasPerfTracer; +import org.apache.atlas.web.util.Servlets; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.http.HttpStatus; +import org.springframework.stereotype.Service; + +import javax.inject.Inject; +import javax.inject.Singleton; +import javax.ws.rs.*; +import javax.ws.rs.core.MediaType; +import java.util.List; + +/** + * REST interface for CRUD operations on tasks + */ +@Path("task") +@Singleton +@Service +@Consumes({Servlets.JSON_MEDIA_TYPE, MediaType.APPLICATION_JSON}) +@Produces({Servlets.JSON_MEDIA_TYPE, MediaType.APPLICATION_JSON}) +public class TaskREST { + private static final Logger LOG = LoggerFactory.getLogger(TaskREST.class); + private static final Logger PERF_LOG = AtlasPerfTracer.getPerfLogger("rest.TaskREST"); + + private final TaskService taskService; + + @Inject + public TaskREST(TaskService taskService) { + this.taskService = taskService; + } + + @POST + @Path("search") + @Timed + public TaskSearchResult getTasks(TaskSearchParams parameters) throws AtlasBaseException { + AtlasPerfTracer perf = null; + + try { + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "EntityREST.getTasks"); + } + + TaskSearchResult ret = taskService.getTasks(parameters); + + return ret; + } finally { + AtlasPerfTracer.log(perf); + } + } + + @PUT + @Path("retry/{guid}") + @Timed + public HttpStatus retryTask(@PathParam("guid") final String guid) throws AtlasBaseException { + AtlasPerfTracer perf = null; + + try { + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "EntityREST.retryTask"); + } + + taskService.retryTask(guid); + + return HttpStatus.OK; + } finally { + AtlasPerfTracer.log(perf); + } + } + + @POST + @Path("bulk") + @Timed + public List createTasks(List tasks) throws AtlasBaseException { + AtlasPerfTracer perf = null; + + AtlasAuthorizationUtils.verifyAccess(new AtlasAdminAccessRequest(AtlasPrivilege.ADMIN_TASK_CUD), "create task is not allowed"); + + try { + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "EntityREST.createTasks"); + } + + List ret = taskService.createAtlasTasks(tasks); + + return ret; + + } finally { + AtlasPerfTracer.log(perf); + } + } + + @DELETE + @Path("bulk") + @Timed + public List deleteTasks(List tasks) throws AtlasBaseException { + AtlasPerfTracer perf = null; + + AtlasAuthorizationUtils.verifyAccess(new AtlasAdminAccessRequest(AtlasPrivilege.ADMIN_TASK_CUD), "deleteTasks is not allowed"); + + try { + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "EntityREST.deleteTasks"); + } + + return taskService.deleteAtlasTasks(tasks); + + } finally { + AtlasPerfTracer.log(perf); + } + } +} diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/TypeCacheRefreshREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/TypeCacheRefreshREST.java new file mode 100644 index 00000000000..cbc9a188a34 --- /dev/null +++ b/webapp/src/main/java/org/apache/atlas/web/rest/TypeCacheRefreshREST.java @@ -0,0 +1,99 @@ +package org.apache.atlas.web.rest; + +import org.apache.atlas.annotation.Timed; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.repository.RepositoryException; +import org.apache.atlas.repository.graph.IAtlasGraphProvider; +import org.apache.atlas.store.AtlasTypeDefStore; +import org.apache.atlas.web.service.AtlasHealthStatus; +import org.apache.atlas.web.service.ServiceState; +import org.apache.atlas.web.util.Servlets; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import javax.inject.Inject; +import javax.inject.Singleton; +import javax.ws.rs.*; +import javax.ws.rs.core.MediaType; + +import static org.apache.atlas.AtlasErrorCode.FAILED_TO_REFRESH_TYPE_DEF_CACHE; +import static org.apache.atlas.repository.Constants.VERTEX_INDEX; + + +@Path("admin/types") +@Singleton +@Service +@Consumes({Servlets.JSON_MEDIA_TYPE, MediaType.APPLICATION_JSON}) +@Produces({Servlets.JSON_MEDIA_TYPE, MediaType.APPLICATION_JSON}) +public class TypeCacheRefreshREST { + private static final Logger LOG = LoggerFactory.getLogger(TypeCacheRefreshREST.class); + + private final AtlasTypeDefStore typeDefStore; + private final IAtlasGraphProvider provider; + private final ServiceState serviceState; + private final AtlasHealthStatus atlasHealthStatus; + + @Inject + public TypeCacheRefreshREST(AtlasTypeDefStore typeDefStore, IAtlasGraphProvider provider, ServiceState serviceState, AtlasHealthStatus atlasHealthStatus) { + this.typeDefStore = typeDefStore; + this.provider = provider; + this.serviceState = serviceState; + this.atlasHealthStatus = atlasHealthStatus; + } + + /** + * API to refresh type-def cache. + * + * @throws AtlasBaseException + * @HTTP 204 if type def cache is refreshed successfully + * @HTTP 500 if there is an error refreshing type def cache + */ + @POST + @Path("/refresh") + @Timed + public void refreshCache(@QueryParam("expectedFieldKeys") int expectedFieldKeys, @QueryParam("traceId") String traceId) throws AtlasBaseException { + try { + if (serviceState.getState() != ServiceState.ServiceStateValue.ACTIVE) { + LOG.warn("Node is in {} state. skipping refreshing type-def-cache :: traceId {}", serviceState.getState(), traceId); + return; + } + refreshTypeDef(expectedFieldKeys, traceId); + } catch (Exception e) { + LOG.error("Error during refreshing cache :: traceId " + traceId + " " + e.getMessage(), e); + serviceState.setState(ServiceState.ServiceStateValue.PASSIVE, true); + atlasHealthStatus.markUnhealthy(AtlasHealthStatus.Component.TYPE_DEF_CACHE, "type-def-cache is not in sync"); + throw new AtlasBaseException(FAILED_TO_REFRESH_TYPE_DEF_CACHE); + } + } + + private void refreshTypeDef(int expectedFieldKeys,final String traceId) throws RepositoryException, InterruptedException, AtlasBaseException { + LOG.info("Initiating type-def cache refresh with expectedFieldKeys = {} :: traceId {}", expectedFieldKeys,traceId); + int currentSize = provider.get().getManagementSystem().getGraphIndex(VERTEX_INDEX).getFieldKeys().size(); + LOG.info("Size of field keys before refresh = {} :: traceId {}", currentSize,traceId); + + long totalWaitTimeInMillis = 15 * 1000;//15 seconds + long sleepTimeInMillis = 500; + long totalIterationsAllowed = Math.floorDiv(totalWaitTimeInMillis, sleepTimeInMillis); + int counter = 0; + + while (currentSize != expectedFieldKeys && counter++ < totalIterationsAllowed) { + currentSize = provider.get().getManagementSystem().getGraphIndex(VERTEX_INDEX).getFieldKeys().size(); + LOG.info("field keys size found = {} at iteration {} :: traceId {}", currentSize, counter, traceId); + Thread.sleep(sleepTimeInMillis); + } + //This condition will hold true when expected fieldKeys did not appear even after waiting for totalWaitTimeInMillis + if (counter > totalIterationsAllowed) { + final String errorMessage = String.format("Could not find desired count of fieldKeys %d after %d ms of wait. Current size of field keys is %d :: traceId %s", + expectedFieldKeys, totalWaitTimeInMillis, currentSize, traceId); + throw new AtlasBaseException(errorMessage); + } else { + LOG.info("Found desired size of fieldKeys in iteration {} :: traceId {}", counter, traceId); + } + //Reload in-memory cache of type-registry + typeDefStore.init(); + + LOG.info("Size of field keys after refresh = {}", provider.get().getManagementSystem().getGraphIndex(VERTEX_INDEX).getFieldKeys().size()); + LOG.info("Completed type-def cache refresh :: traceId {}", traceId); + } +} diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/TypesREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/TypesREST.java index 8cff1d11210..cd872dc81ca 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/TypesREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/TypesREST.java @@ -17,60 +17,61 @@ */ package org.apache.atlas.web.rest; +import org.apache.atlas.AtlasErrorCode; +import org.apache.atlas.RequestContext; import org.apache.atlas.annotation.Timed; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.SearchFilter; -import org.apache.atlas.model.typedef.AtlasBaseTypeDef; -import org.apache.atlas.model.typedef.AtlasBusinessMetadataDef; -import org.apache.atlas.model.typedef.AtlasClassificationDef; -import org.apache.atlas.model.typedef.AtlasEntityDef; -import org.apache.atlas.model.typedef.AtlasEnumDef; -import org.apache.atlas.model.typedef.AtlasRelationshipDef; -import org.apache.atlas.model.typedef.AtlasStructDef; -import org.apache.atlas.model.typedef.AtlasTypeDefHeader; -import org.apache.atlas.model.typedef.AtlasTypesDef; +import org.apache.atlas.model.typedef.*; +import org.apache.atlas.repository.graph.TypeCacheRefresher; import org.apache.atlas.repository.util.FilterUtil; +import org.apache.atlas.service.redis.RedisService; import org.apache.atlas.store.AtlasTypeDefStore; import org.apache.atlas.type.AtlasTypeUtil; import org.apache.atlas.utils.AtlasPerfTracer; import org.apache.atlas.web.util.Servlets; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.configuration.Configuration; import org.apache.http.annotation.Experimental; import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; +import javax.annotation.PreDestroy; import javax.inject.Inject; import javax.inject.Singleton; import javax.servlet.http.HttpServletRequest; -import javax.ws.rs.Consumes; -import javax.ws.rs.DELETE; -import javax.ws.rs.GET; -import javax.ws.rs.POST; -import javax.ws.rs.PUT; -import javax.ws.rs.Path; -import javax.ws.rs.PathParam; -import javax.ws.rs.Produces; +import javax.ws.rs.*; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import java.util.Arrays; import java.util.List; import java.util.Set; +import java.util.UUID; /** * REST interface for CRUD operations on type definitions */ -@Path("v2/types") +@Path("types") @Singleton @Service @Consumes({Servlets.JSON_MEDIA_TYPE, MediaType.APPLICATION_JSON}) @Produces({Servlets.JSON_MEDIA_TYPE, MediaType.APPLICATION_JSON}) public class TypesREST { private static final Logger PERF_LOG = AtlasPerfTracer.getPerfLogger("rest.TypesREST"); + private static final Logger LOG = LoggerFactory.getLogger(TypesREST.class); + private static final String ATLAS_TYPEDEF_LOCK = "atlas:type-def:lock"; + private final AtlasTypeDefStore typeDefStore; + private final RedisService redisService; + private final TypeCacheRefresher typeCacheRefresher; @Inject - public TypesREST(AtlasTypeDefStore typeDefStore) { + public TypesREST(AtlasTypeDefStore typeDefStore, RedisService redisService, Configuration configuration, TypeCacheRefresher typeCacheRefresher) { this.typeDefStore = typeDefStore; + this.redisService = redisService; + this.typeCacheRefresher = typeCacheRefresher; } /** @@ -373,12 +374,29 @@ public AtlasBusinessMetadataDef getBusinessMetadataDefByName(@PathParam("name") return ret; } + private void attemptAcquiringLock() throws AtlasBaseException { + final String traceId = RequestContext.get().getTraceId(); + try { + if (!redisService.acquireDistributedLock(ATLAS_TYPEDEF_LOCK)) { + LOG.info("Lock is already acquired. Returning now :: traceId {}", traceId); + throw new AtlasBaseException(AtlasErrorCode.FAILED_TO_OBTAIN_TYPE_UPDATE_LOCK); + } + LOG.info("successfully acquired lock :: traceId {}", traceId); + } catch (AtlasBaseException e) { + throw e; + } catch (Exception e) { + LOG.error("Error while acquiring lock on type-defs :: traceId " + traceId + " ." + e.getMessage(), e); + throw new AtlasBaseException("Error while acquiring a lock on type-defs"); + } + } + /* Bulk API operation */ /** * Bulk create APIs for all atlas type definitions, only new definitions will be created. * Any changes to the existing definitions will be discarded * @param typesDef A composite wrapper object with corresponding lists of the type definition + * @param allowDuplicateDisplayName * @return A composite wrapper object with lists of type definitions that were successfully * created * @throws Exception @@ -388,22 +406,54 @@ public AtlasBusinessMetadataDef getBusinessMetadataDefByName(@PathParam("name") @POST @Path("/typedefs") @Timed - public AtlasTypesDef createAtlasTypeDefs(final AtlasTypesDef typesDef) throws AtlasBaseException { + public AtlasTypesDef createAtlasTypeDefs(final AtlasTypesDef typesDef, @QueryParam("allowDuplicateDisplayName") @DefaultValue("false") boolean allowDuplicateDisplayName) throws AtlasBaseException { AtlasPerfTracer perf = null; - + validateTypeCreateOrUpdate(typesDef); + RequestContext.get().setTraceId(UUID.randomUUID().toString()); try { + typeCacheRefresher.verifyCacheRefresherHealth(); if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "TypesREST.createAtlasTypeDefs(" + AtlasTypeUtil.toDebugString(typesDef) + ")"); } + attemptAcquiringLock(); + RequestContext.get().setAllowDuplicateDisplayName(allowDuplicateDisplayName); typesDef.getBusinessMetadataDefs().forEach(AtlasBusinessMetadataDef::setRandomNameForEntityAndAttributeDefs); typesDef.getClassificationDefs().forEach(AtlasClassificationDef::setRandomNameForEntityAndAttributeDefs); - return typeDefStore.createTypesDef(typesDef); - } finally { + AtlasTypesDef atlasTypesDef = typeDefStore.createTypesDef(typesDef); + typeCacheRefresher.refreshAllHostCache(); + return atlasTypesDef; + } catch (AtlasBaseException atlasBaseException) { + LOG.error("TypesREST.createAtlasTypeDefs:: " + atlasBaseException.getMessage(), atlasBaseException); + throw atlasBaseException; + } catch (Exception e) { + LOG.error("TypesREST.createAtlasTypeDefs:: " + e.getMessage(), e); + throw new AtlasBaseException("Error while creating a type definition"); + } + finally { + redisService.releaseDistributedLock(ATLAS_TYPEDEF_LOCK); AtlasPerfTracer.log(perf); } } + private void validateTypeCreateOrUpdate(AtlasTypesDef typesDef) throws AtlasBaseException { + + if (CollectionUtils.isNotEmpty(typesDef.getEnumDefs())) { + for (AtlasBaseTypeDef typeDef : typesDef.getEnumDefs()) + if (typeDefStore.hasBuiltInTypeName(typeDef)) + throw new AtlasBaseException(AtlasErrorCode.FORBIDDEN_TYPENAME, typeDef.getName()); + } + if (CollectionUtils.isNotEmpty(typesDef.getEntityDefs())) { + for (AtlasBaseTypeDef typeDef : typesDef.getEntityDefs()) + if (typeDefStore.hasBuiltInTypeName(typeDef)) + throw new AtlasBaseException(AtlasErrorCode.FORBIDDEN_TYPENAME, typeDef.getName()); + } + if (CollectionUtils.isNotEmpty(typesDef.getStructDefs())) { + for (AtlasBaseTypeDef typeDef : typesDef.getStructDefs()) + if (typeDefStore.hasBuiltInTypeName(typeDef)) + throw new AtlasBaseException(AtlasErrorCode.FORBIDDEN_TYPENAME, typeDef.getName()); + } + } /** * Bulk update API for all types, changes detected in the type definitions would be persisted * @param typesDef A composite object that captures all type definition changes @@ -416,14 +466,19 @@ public AtlasTypesDef createAtlasTypeDefs(final AtlasTypesDef typesDef) throws At @Path("/typedefs") @Experimental @Timed - public AtlasTypesDef updateAtlasTypeDefs(final AtlasTypesDef typesDef) throws AtlasBaseException { + public AtlasTypesDef updateAtlasTypeDefs(final AtlasTypesDef typesDef, @QueryParam("patch") final boolean patch, + @QueryParam("allowDuplicateDisplayName") @DefaultValue("false") boolean allowDuplicateDisplayName) throws AtlasBaseException { AtlasPerfTracer perf = null; - + validateTypeCreateOrUpdate(typesDef); + RequestContext.get().setTraceId(UUID.randomUUID().toString()); try { + typeCacheRefresher.verifyCacheRefresherHealth(); if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "TypesREST.updateAtlasTypeDefs(" + AtlasTypeUtil.toDebugString(typesDef) + ")"); } + attemptAcquiringLock(); + for (AtlasBusinessMetadataDef mb : typesDef.getBusinessMetadataDefs()) { AtlasBusinessMetadataDef existingMB; try{ @@ -444,8 +499,21 @@ public AtlasTypesDef updateAtlasTypeDefs(final AtlasTypesDef typesDef) throws At } classificationDef.setRandomNameForNewAttributeDefs(existingClassificationDef); } - return typeDefStore.updateTypesDef(typesDef); + RequestContext.get().setInTypePatching(patch); + RequestContext.get().setAllowDuplicateDisplayName(allowDuplicateDisplayName); + LOG.info("TypesRest.updateAtlasTypeDefs:: Typedef patch enabled:" + patch); + AtlasTypesDef atlasTypesDef = typeDefStore.updateTypesDef(typesDef); + typeCacheRefresher.refreshAllHostCache(); + return atlasTypesDef; + } catch (AtlasBaseException atlasBaseException) { + LOG.error("TypesREST.updateAtlasTypeDefs:: " + atlasBaseException.getMessage(), atlasBaseException); + throw atlasBaseException; + } catch (Exception e) { + LOG.error("TypesREST.updateAtlasTypeDefs:: " + e.getMessage(), e); + throw new AtlasBaseException("Error while updating a type definition"); } finally { + RequestContext.clear(); + redisService.releaseDistributedLock(ATLAS_TYPEDEF_LOCK); AtlasPerfTracer.log(perf); } } @@ -463,17 +531,24 @@ public AtlasTypesDef updateAtlasTypeDefs(final AtlasTypesDef typesDef) throws At @Timed public void deleteAtlasTypeDefs(final AtlasTypesDef typesDef) throws AtlasBaseException { AtlasPerfTracer perf = null; - + RequestContext.get().setTraceId(UUID.randomUUID().toString()); try { + typeCacheRefresher.verifyCacheRefresherHealth(); if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "TypesREST.deleteAtlasTypeDefs(" + AtlasTypeUtil.toDebugString(typesDef) + ")"); } - - - + attemptAcquiringLock(); typeDefStore.deleteTypesDef(typesDef); + typeCacheRefresher.refreshAllHostCache(); + } catch (AtlasBaseException atlasBaseException) { + LOG.error("TypesREST.deleteAtlasTypeDefs:: " + atlasBaseException.getMessage(), atlasBaseException); + throw atlasBaseException; + } catch (Exception e) { + LOG.error("TypesREST.deleteAtlasTypeDefs:: " + e.getMessage(), e); + throw new AtlasBaseException("Error while deleting a type definition"); } finally { + redisService.releaseDistributedLock(ATLAS_TYPEDEF_LOCK); AtlasPerfTracer.log(perf); } } @@ -490,14 +565,23 @@ public void deleteAtlasTypeDefs(final AtlasTypesDef typesDef) throws AtlasBaseEx @Timed public void deleteAtlasTypeByName(@PathParam("typeName") final String typeName) throws AtlasBaseException { AtlasPerfTracer perf = null; - + RequestContext.get().setTraceId(UUID.randomUUID().toString()); try { + typeCacheRefresher.verifyCacheRefresherHealth(); if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "TypesREST.deleteAtlasTypeByName(" + typeName + ")"); } - + attemptAcquiringLock(); typeDefStore.deleteTypeByName(typeName); + typeCacheRefresher.refreshAllHostCache(); + } catch (AtlasBaseException atlasBaseException) { + LOG.error("TypesREST.deleteAtlasTypeByName:: " + atlasBaseException.getMessage(), atlasBaseException); + throw atlasBaseException; + } catch (Exception e) { + LOG.error("TypesREST.deleteAtlasTypeByName:: " + e.getMessage(), e); + throw new AtlasBaseException("Error while deleting a type definition"); } finally { + redisService.releaseDistributedLock(ATLAS_TYPEDEF_LOCK); AtlasPerfTracer.log(perf); } } @@ -531,4 +615,9 @@ private SearchFilter getSearchFilter(HttpServletRequest httpServletRequest) { return ret; } + + @PreDestroy + public void cleanUp() { + this.redisService.releaseDistributedLock(ATLAS_TYPEDEF_LOCK); + } } diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/validator/LineageListRequestValidator.java b/webapp/src/main/java/org/apache/atlas/web/rest/validator/LineageListRequestValidator.java new file mode 100644 index 00000000000..ccad98e1452 --- /dev/null +++ b/webapp/src/main/java/org/apache/atlas/web/rest/validator/LineageListRequestValidator.java @@ -0,0 +1,42 @@ +package org.apache.atlas.web.rest.validator; + +import org.apache.atlas.AtlasErrorCode; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.lineage.LineageListRequest; +import org.apache.commons.lang.StringUtils; +import org.springframework.stereotype.Component; + +@Component +public class LineageListRequestValidator implements RequestValidator { + + private static final int MAX_DEPTH = Integer.MAX_VALUE / 2; + + @Override + public void validate(LineageListRequest lineageListRequest) throws AtlasBaseException { + if (StringUtils.isEmpty(lineageListRequest.getGuid())) + throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "guid is required"); + + validate(lineageListRequest.getSize() == null, "The 'size' parameter is missing or invalid"); + validate(lineageListRequest.getSize() < 0, "Invalid 'size' value, 'size' should be greater than or equal to zero"); + + validate(lineageListRequest.getFrom() == null, "The 'from' parameter is missing or invalid"); + validate(lineageListRequest.getFrom() < 0, "Invalid 'from' value, 'from' should be greater than or equal to zero"); + + validate(lineageListRequest.getDepth() == null, "The 'depth' parameter is missing or invalid"); + validate(lineageListRequest.getDepth() < 0 || lineageListRequest.getDepth() > MAX_DEPTH, "Invalid 'depth' parameter, constraint not satisfied: 0 <= depth <= 1073741823 (Integer.MAX_VALUE/2)"); + + validate(lineageListRequest.getDirection() == null, "Invalid request, mandatory key 'direction' is required"); + + // By default exclude extra metadata if flags not provided + if (lineageListRequest.isExcludeMeanings() == null) + lineageListRequest.setExcludeMeanings(Boolean.TRUE); + if (lineageListRequest.isExcludeClassifications() == null) + lineageListRequest.setExcludeClassifications(Boolean.TRUE); + } + + private static void validate(boolean isInvalid, String errorMessage) throws AtlasBaseException { + if (isInvalid) + throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, errorMessage); + } + +} diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/validator/RequestValidator.java b/webapp/src/main/java/org/apache/atlas/web/rest/validator/RequestValidator.java new file mode 100644 index 00000000000..62dd01b2404 --- /dev/null +++ b/webapp/src/main/java/org/apache/atlas/web/rest/validator/RequestValidator.java @@ -0,0 +1,9 @@ +package org.apache.atlas.web.rest.validator; + +import org.apache.atlas.exception.AtlasBaseException; + +public interface RequestValidator { + + void validate(T t) throws AtlasBaseException; + +} \ No newline at end of file diff --git a/webapp/src/main/java/org/apache/atlas/web/security/AtlasAuthenticationProvider.java b/webapp/src/main/java/org/apache/atlas/web/security/AtlasAuthenticationProvider.java index dff3d8d31c3..17ca1864fe7 100644 --- a/webapp/src/main/java/org/apache/atlas/web/security/AtlasAuthenticationProvider.java +++ b/webapp/src/main/java/org/apache/atlas/web/security/AtlasAuthenticationProvider.java @@ -130,6 +130,8 @@ public Authentication authenticate(Authentication authentication) } else if (keycloakAuthenticationEnabled) { try { authentication = atlasKeycloakAuthenticationProvider.authenticate(authentication); + } catch (KeycloakAuthenticationException ex) { + throw new AtlasAuthenticationException("Authentication failed."); } catch (Exception ex) { LOG.error("Error while Keycloak authentication", ex); } diff --git a/webapp/src/main/java/org/apache/atlas/web/security/AtlasKeycloakAuthenticationProvider.java b/webapp/src/main/java/org/apache/atlas/web/security/AtlasKeycloakAuthenticationProvider.java index 367839b82aa..88523998cda 100644 --- a/webapp/src/main/java/org/apache/atlas/web/security/AtlasKeycloakAuthenticationProvider.java +++ b/webapp/src/main/java/org/apache/atlas/web/security/AtlasKeycloakAuthenticationProvider.java @@ -16,10 +16,17 @@ */ package org.apache.atlas.web.security; +import io.micrometer.core.instrument.Counter; +import org.apache.atlas.AtlasConfiguration; +import org.apache.atlas.keycloak.client.AtlasKeycloakClient; +import org.apache.atlas.service.metrics.MetricUtils; import org.apache.atlas.ApplicationProperties; import org.apache.commons.configuration.Configuration; import org.keycloak.adapters.springsecurity.authentication.KeycloakAuthenticationProvider; import org.keycloak.adapters.springsecurity.token.KeycloakAuthenticationToken; +import org.keycloak.representations.oidc.TokenMetadataRepresentation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.security.core.Authentication; import org.springframework.security.core.GrantedAuthority; import org.springframework.security.core.authority.SimpleGrantedAuthority; @@ -28,18 +35,25 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Objects; @Component public class AtlasKeycloakAuthenticationProvider extends AtlasAbstractAuthenticationProvider { private final boolean groupsFromUGI; private final String groupsClaim; + private final boolean isTokenIntrospectionEnabled; private final KeycloakAuthenticationProvider keycloakAuthenticationProvider; + private final AtlasKeycloakClient atlasKeycloakClient; + private static final Logger LOG = LoggerFactory.getLogger(AtlasKeycloakAuthenticationProvider.class); public AtlasKeycloakAuthenticationProvider() throws Exception { this.keycloakAuthenticationProvider = new KeycloakAuthenticationProvider(); + this.atlasKeycloakClient = AtlasKeycloakClient.getKeycloakClient(); Configuration configuration = ApplicationProperties.get(); + + this.isTokenIntrospectionEnabled = AtlasConfiguration.ENABLE_KEYCLOAK_TOKEN_INTROSPECTION.getBoolean(); this.groupsFromUGI = configuration.getBoolean("atlas.authentication.method.keycloak.ugi-groups", true); this.groupsClaim = configuration.getString("atlas.authentication.method.keycloak.groups_claim"); } @@ -66,6 +80,29 @@ public Authentication authenticate(Authentication authentication) { } } + if (authentication.getName().startsWith("service-account-apikey")) { + // Increment the counter when the authentication is for a service account. + Counter.builder("service_account_apikey_request_counter").register(MetricUtils.getMeterRegistry()).increment(); + + // Validate the token online with keycloak server if token introspection is enabled + LOG.info("isTokenIntrospectionEnabled: {}", isTokenIntrospectionEnabled); + if (isTokenIntrospectionEnabled) { + LOG.info("Validating request for clientId: {}", authentication.getName().substring("service-account-".length())); + try { + KeycloakAuthenticationToken keycloakToken = (KeycloakAuthenticationToken) authentication; + String bearerToken = keycloakToken.getAccount().getKeycloakSecurityContext().getTokenString(); + TokenMetadataRepresentation introspectToken = atlasKeycloakClient.introspectToken(bearerToken); + if (Objects.nonNull(introspectToken) && introspectToken.isActive()) { + authentication.setAuthenticated(true); + } else { + handleInvalidApiKey(authentication); + } + } catch (Exception e) { + throw new KeycloakAuthenticationException("Keycloak Authentication failed", e.getCause()); + } + } + } + return authentication; } @@ -73,4 +110,10 @@ public Authentication authenticate(Authentication authentication) { public boolean supports(Class aClass) { return keycloakAuthenticationProvider.supports(aClass); } + + private void handleInvalidApiKey(Authentication authentication) { + authentication.setAuthenticated(false); + LOG.error("Invalid clientId: {}", authentication.getName().substring("service-account-".length())); + throw new KeycloakAuthenticationException("Invalid ClientId"); + } } \ No newline at end of file diff --git a/webapp/src/main/java/org/apache/atlas/web/security/AtlasSecurityConfig.java b/webapp/src/main/java/org/apache/atlas/web/security/AtlasSecurityConfig.java index f53ade5c156..ec4b9bef4b5 100644 --- a/webapp/src/main/java/org/apache/atlas/web/security/AtlasSecurityConfig.java +++ b/webapp/src/main/java/org/apache/atlas/web/security/AtlasSecurityConfig.java @@ -56,7 +56,6 @@ import org.springframework.security.web.authentication.logout.LogoutFilter; import org.springframework.security.web.authentication.session.RegisterSessionAuthenticationStrategy; import org.springframework.security.web.authentication.session.SessionAuthenticationStrategy; -import org.springframework.security.web.authentication.www.BasicAuthenticationEntryPoint; import org.springframework.security.web.authentication.www.BasicAuthenticationFilter; import org.springframework.security.web.servletapi.SecurityContextHolderAwareRequestFilter; import org.springframework.security.web.util.matcher.AntPathRequestMatcher; @@ -72,6 +71,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import static org.apache.atlas.AtlasConstants.ATLAS_MIGRATION_MODE_FILENAME; import static org.apache.atlas.web.filters.HeadersUtil.SERVER_KEY; @@ -89,11 +89,14 @@ public class AtlasSecurityConfig extends WebSecurityConfigurerAdapter { private final AtlasAuthenticationFilter atlasAuthenticationFilter; private final AtlasCSRFPreventionFilter csrfPreventionFilter; private final AtlasAuthenticationEntryPoint atlasAuthenticationEntryPoint; + private final AtlasXSSPreventionFilter atlasXSSPreventionFilter; // Our own Atlas filters need to be registered as well private final Configuration configuration; private final StaleTransactionCleanupFilter staleTransactionCleanupFilter; private final ActiveServerFilter activeServerFilter; + private final boolean isAlbEnabled = Boolean.valueOf(System.getenv("ALB_ENABLED")).booleanValue(); + public static final RequestMatcher KEYCLOAK_REQUEST_MATCHER = new OrRequestMatcher(new RequestMatcher[]{new AntPathRequestMatcher("/login.jsp"), new RequestHeaderRequestMatcher("Authorization"), new QueryParamPresenceRequestMatcher("access_token")}); @@ -112,6 +115,7 @@ public AtlasSecurityConfig(AtlasKnoxSSOAuthenticationFilter ssoAuthenticationFil AtlasAuthenticationSuccessHandler successHandler, AtlasAuthenticationFailureHandler failureHandler, AtlasAuthenticationEntryPoint atlasAuthenticationEntryPoint, + AtlasXSSPreventionFilter atlasXSSPreventionFilter, Configuration configuration, StaleTransactionCleanupFilter staleTransactionCleanupFilter, ActiveServerFilter activeServerFilter) { @@ -122,6 +126,7 @@ public AtlasSecurityConfig(AtlasKnoxSSOAuthenticationFilter ssoAuthenticationFil this.successHandler = successHandler; this.failureHandler = failureHandler; this.atlasAuthenticationEntryPoint = atlasAuthenticationEntryPoint; + this.atlasXSSPreventionFilter = atlasXSSPreventionFilter; this.configuration = configuration; this.staleTransactionCleanupFilter = staleTransactionCleanupFilter; this.activeServerFilter = activeServerFilter; @@ -173,10 +178,15 @@ public void configure(WebSecurity web) throws Exception { "/migration-status.html", "/api/atlas/admin/status", "/api/atlas/admin/metrics", + "/api/atlas/admin/metrics/prometheus", "/api/atlas/admin/health", + "/api/atlas/admin/types/refresh", "/api/atlas/admin/isactive", "/api/atlas/admin/killtheleader", - "/api/atlas/admin/pushMetricsToStatsd" + "/api/atlas/admin/pushMetricsToStatsd", + "/api/atlas/v2/auth/download/policies/*", + "/api/atlas/v2/auth/download/roles/*", + "/api/atlas/v2/auth/download/users/*" )); if (!keycloakEnabled) { @@ -224,15 +234,17 @@ protected void configure(HttpSecurity httpSecurity) throws Exception { //@formatter:on boolean configMigrationEnabled = !StringUtils.isEmpty(configuration.getString(ATLAS_MIGRATION_MODE_FILENAME)); - if (configuration.getBoolean("atlas.server.ha.enabled", false) || - configMigrationEnabled) { - if(configMigrationEnabled) { - LOG.info("Atlas is in Migration Mode, enabling ActiveServerFilter"); - } else { - LOG.info("Atlas is in HA Mode, enabling ActiveServerFilter"); - } - httpSecurity.addFilterAfter(activeServerFilter, BasicAuthenticationFilter.class); + if (configMigrationEnabled) { + LOG.info("Atlas is in Migration Mode, enabling ActiveServerFilter"); + } else { + LOG.info("Atlas is in HA or HS Mode, enabling ActiveServerFilter"); } + + // TODO: Enable XSS Filter after solving encoding problems + LOG.warn("XSS filter is disabled from Atlas"); + //Enable activeServerFilter regardless of HA or HS + httpSecurity.addFilterAfter(activeServerFilter, BasicAuthenticationFilter.class); + httpSecurity .addFilterAfter(staleTransactionCleanupFilter, BasicAuthenticationFilter.class) .addFilterBefore(ssoAuthenticationFilter, BasicAuthenticationFilter.class) diff --git a/webapp/src/main/java/org/apache/atlas/web/security/KeycloakAuthenticationException.java b/webapp/src/main/java/org/apache/atlas/web/security/KeycloakAuthenticationException.java new file mode 100644 index 00000000000..01480015036 --- /dev/null +++ b/webapp/src/main/java/org/apache/atlas/web/security/KeycloakAuthenticationException.java @@ -0,0 +1,13 @@ +package org.apache.atlas.web.security; + +import org.springframework.security.core.AuthenticationException; + +public class KeycloakAuthenticationException extends AuthenticationException { + public KeycloakAuthenticationException(String msg, Throwable cause) { + super(msg, cause); + } + + public KeycloakAuthenticationException(String msg) { + super(msg); + } +} diff --git a/webapp/src/main/java/org/apache/atlas/web/service/AtlasHealthStatus.java b/webapp/src/main/java/org/apache/atlas/web/service/AtlasHealthStatus.java new file mode 100644 index 00000000000..dc14c788329 --- /dev/null +++ b/webapp/src/main/java/org/apache/atlas/web/service/AtlasHealthStatus.java @@ -0,0 +1,82 @@ +package org.apache.atlas.web.service; + +import org.apache.atlas.ApplicationProperties; +import org.apache.atlas.AtlasException; +import org.apache.atlas.ha.HAConfiguration; +import org.apache.atlas.model.general.HealthStatus; +import org.apache.commons.configuration.Configuration; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Optional; + +@Component +public class AtlasHealthStatus { + + private static final Logger LOG = LoggerFactory.getLogger(AtlasHealthStatus.class); + private static final String ERROR = "error"; + private static final String OK = "ok"; + + private final List healthStatuses = new ArrayList<>(); + private boolean isActiveActiveHAEnabled; + + @PostConstruct + public void init() throws AtlasException { + Configuration atlasProperties = ApplicationProperties.get(); + this.isActiveActiveHAEnabled = HAConfiguration.isActiveActiveHAEnabled(atlasProperties); + registerComponents(); + } + + private void registerComponents() { + //Initialize with OK status. + if (isActiveActiveHAEnabled) { + healthStatuses.add(new HealthStatus(Component.TYPE_DEF_CACHE.componentName, OK, false, new Date().toString(), StringUtils.EMPTY)); + } + } + + public void markUnhealthy(final Component component, final String errorString) { + HealthStatus health = getHealth(component); + if (health != null) { + health.status = ERROR; + health.errorString = errorString; + health.checkTime = new Date().toString(); + } + } + + public HealthStatus getHealth(final Component component) { + Optional optionalHealthStatus = healthStatuses.stream().filter( + healthStatus -> healthStatus.name.equalsIgnoreCase(component.componentName)).findFirst(); + + if (!optionalHealthStatus.isPresent()) { + LOG.error("Could not find component {} in health status. Should have been registered first", component.componentName); + return null; + } else { + return optionalHealthStatus.get(); + } + } + + public List getHealthStatuses() { + return healthStatuses; + } + + public boolean isAtleastOneComponentUnHealthy() { + return healthStatuses.stream().anyMatch(healthStatus -> healthStatus.status.equalsIgnoreCase(ERROR)); + } + + public enum Component { + TYPE_DEF_CACHE("typeDefCache"); + + private final String componentName; + + Component(String componentName) { + this.componentName = componentName; + } + + } +} \ No newline at end of file diff --git a/webapp/src/main/java/org/apache/atlas/web/service/CuratorFactory.java b/webapp/src/main/java/org/apache/atlas/web/service/CuratorFactory.java index 7c89055e01e..9da49d32efe 100644 --- a/webapp/src/main/java/org/apache/atlas/web/service/CuratorFactory.java +++ b/webapp/src/main/java/org/apache/atlas/web/service/CuratorFactory.java @@ -22,6 +22,7 @@ import com.google.common.base.Charsets; import org.apache.atlas.ApplicationProperties; import org.apache.atlas.AtlasException; +import org.apache.atlas.ICuratorFactory; import org.apache.atlas.ha.HAConfiguration; import org.apache.commons.configuration.Configuration; import org.apache.curator.framework.AuthInfo; @@ -49,7 +50,7 @@ */ @Singleton @Component -public class CuratorFactory { +public class CuratorFactory implements ICuratorFactory { private static final Logger LOG = LoggerFactory.getLogger(CuratorFactory.class); @@ -197,6 +198,11 @@ public LeaderLatch leaderLatchInstance(String serverId, String zkRoot) { } public InterProcessMutex lockInstance(String zkRoot) { - return new InterProcessMutex(curatorFramework, zkRoot+ SETUP_LOCK); + return lockInstance(zkRoot, SETUP_LOCK); + } + + @Override + public InterProcessMutex lockInstance(String zkRoot, String lockName) { + return new InterProcessMutex(curatorFramework, zkRoot + lockName); } } diff --git a/webapp/src/main/java/org/apache/atlas/web/service/EmbeddedServer.java b/webapp/src/main/java/org/apache/atlas/web/service/EmbeddedServer.java index 4c3a1c56993..4fbfb88b7e4 100755 --- a/webapp/src/main/java/org/apache/atlas/web/service/EmbeddedServer.java +++ b/webapp/src/main/java/org/apache/atlas/web/service/EmbeddedServer.java @@ -18,6 +18,8 @@ package org.apache.atlas.web.service; +import io.micrometer.core.instrument.binder.jetty.JettyConnectionMetrics; +import io.micrometer.core.instrument.binder.jetty.JettyServerThreadPoolMetrics; import org.apache.atlas.AtlasConfiguration; import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.util.BeanUtil; @@ -35,11 +37,14 @@ import org.slf4j.LoggerFactory; import java.io.IOException; +import java.util.Collections; import java.util.Date; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import static org.apache.atlas.service.metrics.MetricUtils.getMeterRegistry; + /** * This class embeds a Jetty server and a connector. */ @@ -68,6 +73,8 @@ public EmbeddedServer(String host, int port, String path) throws IOException { server = new Server(pool); Connector connector = getConnector(host, port); + connector.addBean(new JettyConnectionMetrics(getMeterRegistry())); + new JettyServerThreadPoolMetrics(pool, Collections.emptyList()).bindTo(getMeterRegistry()); server.addConnector(connector); WebAppContext application = getWebAppContext(path); diff --git a/webapp/src/main/java/org/apache/atlas/web/service/ServiceState.java b/webapp/src/main/java/org/apache/atlas/web/service/ServiceState.java index ea74d21ad00..78b4f54fc13 100644 --- a/webapp/src/main/java/org/apache/atlas/web/service/ServiceState.java +++ b/webapp/src/main/java/org/apache/atlas/web/service/ServiceState.java @@ -89,14 +89,18 @@ public void becomingActive() { setState(ServiceStateValue.BECOMING_ACTIVE); } - private void setState(ServiceStateValue newState) { - Preconditions.checkState(HAConfiguration.isHAEnabled(configuration), "Cannot change state as requested, as HA is not enabled for this instance."); - + public void setState(ServiceStateValue newState, final boolean skipValidation) { + if (!skipValidation) { + Preconditions.checkState(HAConfiguration.isHAEnabled(configuration), "Cannot change state as requested, as HA is not enabled for this instance."); + } state = newState; - auditServerStatus(); } + private void setState(ServiceStateValue newState) { + this.setState(newState, false); + } + private void auditServerStatus() { if (state == ServiceState.ServiceStateValue.ACTIVE) { diff --git a/webapp/src/main/java/org/apache/atlas/web/util/CachedBodyHttpServletRequest.java b/webapp/src/main/java/org/apache/atlas/web/util/CachedBodyHttpServletRequest.java new file mode 100644 index 00000000000..789fde1f0a6 --- /dev/null +++ b/webapp/src/main/java/org/apache/atlas/web/util/CachedBodyHttpServletRequest.java @@ -0,0 +1,24 @@ +package org.apache.atlas.web.util; + +import org.springframework.util.StreamUtils; + +import javax.servlet.ServletInputStream; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletRequestWrapper; +import java.io.IOException; +import java.io.InputStream; + +public class CachedBodyHttpServletRequest extends HttpServletRequestWrapper { + private byte[] cachedBody; + + public CachedBodyHttpServletRequest(HttpServletRequest request) throws IOException { + super(request); + InputStream requestInputStream = request.getInputStream(); + this.cachedBody = StreamUtils.copyToByteArray(requestInputStream); + } + + @Override + public ServletInputStream getInputStream() throws IOException { + return new CachedBodyServletInputStream(this.cachedBody); + } +} diff --git a/webapp/src/main/java/org/apache/atlas/web/util/CachedBodyServletInputStream.java b/webapp/src/main/java/org/apache/atlas/web/util/CachedBodyServletInputStream.java new file mode 100644 index 00000000000..b4fec04b713 --- /dev/null +++ b/webapp/src/main/java/org/apache/atlas/web/util/CachedBodyServletInputStream.java @@ -0,0 +1,39 @@ +package org.apache.atlas.web.util; + +import javax.servlet.ReadListener; +import javax.servlet.ServletInputStream; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; + +public class CachedBodyServletInputStream extends ServletInputStream { + private InputStream inputStream; + + public CachedBodyServletInputStream(byte[] cachedBody) { + this.inputStream = new ByteArrayInputStream(cachedBody); + } + + @Override + public int read() throws IOException { + return inputStream.read(); + } + + @Override + public boolean isFinished() { + try { + return inputStream.available() == 0; + } catch (IOException e) { + return false; + } + } + + @Override + public boolean isReady() { + return true; + } + + @Override + public void setReadListener(ReadListener readListener) { + + } +} diff --git a/webapp/src/main/resources/spring-security.xml b/webapp/src/main/resources/spring-security.xml index 3020690dca0..72f7a2917b7 100644 --- a/webapp/src/main/resources/spring-security.xml +++ b/webapp/src/main/resources/spring-security.xml @@ -28,6 +28,7 @@ + diff --git a/webapp/src/main/webapp/WEB-INF/web.xml b/webapp/src/main/webapp/WEB-INF/web.xml index 7bc93192058..07092d62ebb 100755 --- a/webapp/src/main/webapp/WEB-INF/web.xml +++ b/webapp/src/main/webapp/WEB-INF/web.xml @@ -60,6 +60,8 @@ jersey-servlet /api/atlas/* + /api/atlas/v2/* + /api/meta/* @@ -72,6 +74,16 @@ /* + + MetricsFilter + org.apache.atlas.web.filters.MetricsFilter + + + + MetricsFilter + /* + + AuditFilter org.apache.atlas.web.filters.AuditFilter @@ -92,6 +104,11 @@ /api/atlas/admin/metrics + + HeaderFilter + /api/atlas/admin/metrics/prometheus + + HeaderFilter /api/atlas/admin/status diff --git a/webapp/src/test/java/org/apache/atlas/web/adapters/TestEntitiesREST.java b/webapp/src/test/java/org/apache/atlas/web/adapters/TestEntitiesREST.java index 51dd3425744..88c702893e4 100644 --- a/webapp/src/test/java/org/apache/atlas/web/adapters/TestEntitiesREST.java +++ b/webapp/src/test/java/org/apache/atlas/web/adapters/TestEntitiesREST.java @@ -162,7 +162,7 @@ public void testCustomAttributesSearch() throws Exception { dbWithCustomAttr.setCustomAttributes(customAttr); AtlasEntitiesWithExtInfo atlasEntitiesWithExtInfo = new AtlasEntitiesWithExtInfo(dbWithCustomAttr); - EntityMutationResponse response = entityREST.createOrUpdate(atlasEntitiesWithExtInfo, false, false); + EntityMutationResponse response = entityREST.createOrUpdate(atlasEntitiesWithExtInfo, false, false, false); Assert.assertNotNull(response.getUpdatedEntitiesByTypeName(DATABASE_TYPE)); @@ -646,7 +646,7 @@ public void testUpdateWithSerializedEntities() throws Exception { newEntities.addReferredEntity(serDeserEntity(column)); } - EntityMutationResponse response2 = entityREST.createOrUpdate(newEntities, false, false); + EntityMutationResponse response2 = entityREST.createOrUpdate(newEntities, false, false, false); List newGuids = response2.getEntitiesByOperation(EntityMutations.EntityOperation.CREATE); Assert.assertNotNull(newGuids); @@ -775,7 +775,7 @@ private void registerEntities() throws Exception { entities.addReferredEntity(column); } - EntityMutationResponse response = entityREST.createOrUpdate(entities, false, false); + EntityMutationResponse response = entityREST.createOrUpdate(entities, false, false, false); List guids = response.getEntitiesByOperation(EntityMutations.EntityOperation.CREATE); Assert.assertNotNull(guids); diff --git a/webapp/src/test/java/org/apache/atlas/web/adapters/TestEntityREST.java b/webapp/src/test/java/org/apache/atlas/web/adapters/TestEntityREST.java index 3c65ab11619..8255b4d4483 100644 --- a/webapp/src/test/java/org/apache/atlas/web/adapters/TestEntityREST.java +++ b/webapp/src/test/java/org/apache/atlas/web/adapters/TestEntityREST.java @@ -87,7 +87,7 @@ public void cleanup() throws Exception { private void createTestEntity() throws Exception { AtlasEntity dbEntity = TestUtilsV2.createDBEntity(); - final EntityMutationResponse response = entityREST.createOrUpdate(new AtlasEntitiesWithExtInfo(dbEntity), false, false); + final EntityMutationResponse response = entityREST.createOrUpdate(new AtlasEntitiesWithExtInfo(dbEntity), false, false, false); Assert.assertNotNull(response); List entitiesMutated = response.getEntitiesByOperation(EntityMutations.EntityOperation.CREATE); @@ -391,7 +391,7 @@ public void testDeleteEntityById() throws Exception { @Test public void testPartialUpdateByUniqueAttribute() throws Exception { AtlasEntity dbEntity = TestUtilsV2.createDBEntity(); - EntityMutationResponse response = entityREST.createOrUpdate(new AtlasEntitiesWithExtInfo(dbEntity), false, false); + EntityMutationResponse response = entityREST.createOrUpdate(new AtlasEntitiesWithExtInfo(dbEntity), false, false, false); String dbGuid = response.getEntitiesByOperation(EntityMutations.EntityOperation.CREATE).get(0).getGuid(); Assert.assertTrue(AtlasTypeUtil.isAssignedGuid(dbGuid)); @@ -422,7 +422,7 @@ public void testPartialUpdateByUniqueAttribute() throws Exception { @Test public void testUpdateGetDeleteEntityByUniqueAttribute() throws Exception { AtlasEntity dbEntity = TestUtilsV2.createDBEntity(); - EntityMutationResponse response = entityREST.createOrUpdate(new AtlasEntitiesWithExtInfo(dbEntity), false, false); + EntityMutationResponse response = entityREST.createOrUpdate(new AtlasEntitiesWithExtInfo(dbEntity), false, false, false); String dbGuid = response.getEntitiesByOperation(EntityMutations.EntityOperation.CREATE).get(0).getGuid(); Assert.assertTrue(AtlasTypeUtil.isAssignedGuid(dbGuid)); diff --git a/webapp/src/test/java/org/apache/atlas/web/adapters/TestEntityRESTDelete.java b/webapp/src/test/java/org/apache/atlas/web/adapters/TestEntityRESTDelete.java index e4c2b8f8d1e..8b919812214 100644 --- a/webapp/src/test/java/org/apache/atlas/web/adapters/TestEntityRESTDelete.java +++ b/webapp/src/test/java/org/apache/atlas/web/adapters/TestEntityRESTDelete.java @@ -90,7 +90,7 @@ private void createEntities() throws Exception { for (int i = 1; i <= 2; i++) { AtlasEntity dbEntity = TestUtilsV2.createDBEntity(); - final EntityMutationResponse response = entityREST.createOrUpdate(new AtlasEntity.AtlasEntitiesWithExtInfo(dbEntity), false, false); + final EntityMutationResponse response = entityREST.createOrUpdate(new AtlasEntity.AtlasEntitiesWithExtInfo(dbEntity), false, false, false); assertNotNull(response); List entitiesMutated = response.getEntitiesByOperation(EntityMutations.EntityOperation.CREATE); diff --git a/webapp/src/test/java/org/apache/atlas/web/adapters/TypeDefsRESTTest.java b/webapp/src/test/java/org/apache/atlas/web/adapters/TypeDefsRESTTest.java index 3560c25078f..fa6705c2c5f 100644 --- a/webapp/src/test/java/org/apache/atlas/web/adapters/TypeDefsRESTTest.java +++ b/webapp/src/test/java/org/apache/atlas/web/adapters/TypeDefsRESTTest.java @@ -80,7 +80,7 @@ public void cleanup() throws Exception { private void createTestEntity() throws AtlasBaseException { AtlasEntity dbEntity = TestUtilsV2.createDBEntity(); - final EntityMutationResponse response = entityREST.createOrUpdate(new AtlasEntity.AtlasEntitiesWithExtInfo(dbEntity), false, false); + final EntityMutationResponse response = entityREST.createOrUpdate(new AtlasEntity.AtlasEntitiesWithExtInfo(dbEntity), false, false, false); Assert.assertNotNull(response); List entitiesMutated = response.getEntitiesByOperation(EntityMutations.EntityOperation.CREATE); diff --git a/webapp/src/test/java/org/apache/atlas/web/integration/BaseResourceIT.java b/webapp/src/test/java/org/apache/atlas/web/integration/BaseResourceIT.java index 9d5a9e5c383..6af234feaed 100755 --- a/webapp/src/test/java/org/apache/atlas/web/integration/BaseResourceIT.java +++ b/webapp/src/test/java/org/apache/atlas/web/integration/BaseResourceIT.java @@ -104,6 +104,7 @@ public abstract class BaseResourceIT { public static final String SEC_TAG = "sec_Tag"; public static final String FINANCE_TAG = "finance_Tag"; public static final String CLASSIFICATION = "classification"; + public static final String ATLAS_LINEAGE_ON_DEMAND_ENABLED = "atlas.lineage.on.demand.enabled"; protected static final int MAX_WAIT_TIME = 60000; @@ -111,6 +112,7 @@ public abstract class BaseResourceIT { protected AtlasClient atlasClientV1; protected AtlasClientV2 atlasClientV2; protected String[] atlasUrls; + protected boolean isLineageOnDemandEnabled; protected NotificationInterface notificationInterface = null; @@ -126,6 +128,11 @@ public void setUp() throws Exception { Configuration configuration = ApplicationProperties.get(); + isLineageOnDemandEnabled = configuration.getBoolean(ATLAS_LINEAGE_ON_DEMAND_ENABLED); + if (!isLineageOnDemandEnabled) { + ApplicationProperties.get().setProperty(ATLAS_LINEAGE_ON_DEMAND_ENABLED, true); + } + atlasUrls = configuration.getStringArray(ATLAS_REST_ADDRESS); if (atlasUrls == null || atlasUrls.length == 0) {