diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..00a51aff5 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,6 @@ +# +# https://help.github.com/articles/dealing-with-line-endings/ +# +# These are explicitly windows files and should use crlf +*.bat text eol=crlf + diff --git a/.github/ISSUE_TEMPLATE/bug_report_template.yaml b/.github/ISSUE_TEMPLATE/bug_report_template.yaml index f6f317370..a23ee6c19 100644 --- a/.github/ISSUE_TEMPLATE/bug_report_template.yaml +++ b/.github/ISSUE_TEMPLATE/bug_report_template.yaml @@ -7,7 +7,7 @@ body: - type: markdown attributes: value: | - Thank you for submitting an issue. Please refer to our [contribution guide](https://github.com/linkedin/feathr/blob/main/docs/dev_guide/new_contributor_guide.md) for additional information. + Thank you for submitting an issue. Please refer to our [contribution guide](https://github.com/feathr-ai/feathr/blob/main/docs/dev_guide/new_contributor_guide.md) for additional information. #### Please fill in this bug report template to ensure a timely and thorough response. - type: dropdown id: contribution diff --git a/.github/ISSUE_TEMPLATE/doc_improvements.yaml b/.github/ISSUE_TEMPLATE/doc_improvements.yaml index bd8703da4..214b11198 100644 --- a/.github/ISSUE_TEMPLATE/doc_improvements.yaml +++ b/.github/ISSUE_TEMPLATE/doc_improvements.yaml @@ -7,7 +7,7 @@ body: - type: markdown attributes: value: | - Thank you for submitting an issue. Please refer to our [contribution guide](https://github.com/linkedin/feathr/blob/main/docs/dev_guide/new_contributor_guide.md) for additional information. + Thank you for submitting an issue. Please refer to our [contribution guide](https://github.com/feathr-ai/feathr/blob/main/docs/dev_guide/new_contributor_guide.md) for additional information. #### Please fill in this non-technical template to ensure a timely and thorough response. - type: dropdown id: contribution diff --git a/.github/ISSUE_TEMPLATE/feature_request_template.yaml b/.github/ISSUE_TEMPLATE/feature_request_template.yaml index ddc3c0405..9e08b470c 100644 --- a/.github/ISSUE_TEMPLATE/feature_request_template.yaml +++ b/.github/ISSUE_TEMPLATE/feature_request_template.yaml @@ -7,7 +7,7 @@ body: - type: markdown attributes: value: | - Thank you for submitting an issue. Please refer to our [contribution guide](https://github.com/linkedin/feathr/blob/main/docs/dev_guide/new_contributor_guide.md) for additional information. + Thank you for submitting an issue. Please refer to our [contribution guide](https://github.com/feathr-ai/feathr/blob/main/docs/dev_guide/new_contributor_guide.md) for additional information. #### Please fill in this bug report template to ensure a timely and thorough response. - type: dropdown id: contribution diff --git a/.github/ISSUE_TEMPLATE/non_technical_request_template.yaml b/.github/ISSUE_TEMPLATE/non_technical_request_template.yaml index c09310514..bd7e90239 100644 --- a/.github/ISSUE_TEMPLATE/non_technical_request_template.yaml +++ b/.github/ISSUE_TEMPLATE/non_technical_request_template.yaml @@ -7,7 +7,7 @@ body: - type: markdown attributes: value: | - Thank you for submitting an issue. Please refer to our [contribution guide](https://github.com/linkedin/feathr/blob/main/docs/dev_guide/new_contributor_guide.md) for additional information. + Thank you for submitting an issue. Please refer to our [contribution guide](https://github.com/feathr-ai/feathr/blob/main/docs/dev_guide/new_contributor_guide.md) for additional information. #### Please fill in this non-technical template to ensure a timely and thorough response. - type: dropdown id: contribution diff --git a/.github/workflows/.coveragerc_db b/.github/workflows/.coveragerc_db new file mode 100644 index 000000000..410ae2191 --- /dev/null +++ b/.github/workflows/.coveragerc_db @@ -0,0 +1,8 @@ +[run] +omit = feathr_project/feathr/registry/_feature_registry_purview.py + feathr_project/feathr/spark_provider/_synapse_submission.py + feathr_project/feathr/spark_provider/_localspark_submission.py +[report] +exclude_lines = + pragma: no cover + @abstract \ No newline at end of file diff --git a/.github/workflows/.coveragerc_local b/.github/workflows/.coveragerc_local new file mode 100644 index 000000000..0f517b928 --- /dev/null +++ b/.github/workflows/.coveragerc_local @@ -0,0 +1,8 @@ +[run] +omit = feathr_project/feathr/registry/_feature_registry_purview.py + feathr_project/feathr/spark_provider/_databricks_submission.py + feathr_project/feathr/spark_provider/_synapse_submission.py +[report] +exclude_lines = + pragma: no cover + @abstract \ No newline at end of file diff --git a/.github/workflows/.coveragerc_sy b/.github/workflows/.coveragerc_sy new file mode 100644 index 000000000..f44e27cef --- /dev/null +++ b/.github/workflows/.coveragerc_sy @@ -0,0 +1,8 @@ +[run] +omit = feathr_project/feathr/registry/_feature_registry_purview.py + feathr_project/feathr/spark_provider/_databricks_submission.py + feathr_project/feathr/spark_provider/_localspark_submission.py +[report] +exclude_lines = + pragma: no cover + @abstract \ No newline at end of file diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 84e99b614..9b96d441c 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -1,5 +1,5 @@ # This workflow builds the docker container and publishes to dockerhub with appropriate tag -# It has two triggers, +# It has two triggers, # 1. daily i.e. runs everyday at specific time. # 2. Anytime a new branch is created under releases @@ -15,7 +15,6 @@ on: branches: - 'releases/**' - jobs: build_and_push_image_to_registry: name: Push Docker image to Docker Hub @@ -23,19 +22,19 @@ jobs: steps: - name: Check out the repo uses: actions/checkout@v3 - + - name: Log in to Docker Hub uses: docker/login-action@v2 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - + - name: Extract metadata (tags, labels) for Docker id: meta uses: docker/metadata-action@v4 with: images: feathrfeaturestore/feathr-registry - + - name: Build and push Docker image uses: docker/build-push-action@v3 with: @@ -45,34 +44,32 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - # Deploy the docker container to the three test environments for feathr + # Trigger Azure Web App webhooks to pull the latest nightly image deploy: runs-on: ubuntu-latest needs: build_and_push_image_to_registry - - + steps: - - name: Deploy to Feathr SQL Registry Azure Web App - id: deploy-to-sql-webapp - uses: azure/webapps-deploy@v2 - with: - app-name: 'feathr-sql-registry' - publish-profile: ${{ secrets.AZURE_WEBAPP_PUBLISH_PROFILE_FEATHR_SQL_REGISTRY }} - images: 'feathrfeaturestore/feathr-registry:nightly' - - - name: Deploy to Feathr Purview Registry Azure Web App - id: deploy-to-purview-webapp - uses: azure/webapps-deploy@v2 - with: - app-name: 'feathr-purview-registry' - publish-profile: ${{ secrets.AZURE_WEBAPP_PUBLISH_PROFILE_FEATHR_PURVIEW_REGISTRY }} - images: 'feathrfeaturestore/feathr-registry:nightly' + - name: Deploy to Azure Web App feathr-registry-purview + id: deploy-to-feathr-registry-purview + uses: distributhor/workflow-webhook@v3.0.1 + env: + webhook_url: ${{ secrets.AZURE_WEBAPP_FEATHR_REGISTRY_PURVIEW_WEBHOOK }} - - name: Deploy to Feathr RBAC Registry Azure Web App - id: deploy-to-rbac-webapp - uses: azure/webapps-deploy@v2 - with: - app-name: 'feathr-rbac-registry' - publish-profile: ${{ secrets.AZURE_WEBAPP_PUBLISH_PROFILE_FEATHR_RBAC_REGISTRY }} - images: 'feathrfeaturestore/feathr-registry:nightly' + - name: Deploy to Azure Web App feathr-registry-purview-rbac + id: deploy-to-feathr-registry-purview-rbac + uses: distributhor/workflow-webhook@v3.0.1 + env: + webhook_url: ${{ secrets.AZURE_WEBAPP_FEATHR_REGISTRY_PURVIEW_RBAC_WEBHOOK }} + + - name: Deploy to Azure Web App feathr-registry-sql + id: deploy-to-feathr-registry-sql + uses: distributhor/workflow-webhook@v3.0.1 + env: + webhook_url: ${{ secrets.AZURE_WEBAPP_FEATHR_REGISTRY_SQL_WEBHOOK }} + - name: Deploy to Azure Web App feathr-registry-sql-rbac + id: deploy-to-feathr-registry-sql-rbac + uses: distributhor/workflow-webhook@v3.0.1 + env: + webhook_url: ${{ secrets.AZURE_WEBAPP_FEATHR_REGISTRY_SQL_RBAC_WEBHOOK }} diff --git a/.github/workflows/document-scan.yml b/.github/workflows/document-scan.yml index 3762ca2af..291a04f44 100644 --- a/.github/workflows/document-scan.yml +++ b/.github/workflows/document-scan.yml @@ -1,6 +1,9 @@ name: Feathr Documents' Broken Link Check -on: [push] +on: + push: + branches: [main] + jobs: check-links: runs-on: ubuntu-latest diff --git a/.github/workflows/publish-to-maven.yml b/.github/workflows/publish-to-maven.yml index ae4d98e68..21bac0108 100644 --- a/.github/workflows/publish-to-maven.yml +++ b/.github/workflows/publish-to-maven.yml @@ -1,18 +1,18 @@ name: Publish package to the Maven Central Repository -on: +on: push: # This pipeline will get triggered everytime there is a new tag created. - # It is required + # It is required tags: ["*"] jobs: publish-to-maven: runs-on: ubuntu-latest - + steps: - name: Checkout source uses: actions/checkout@v2 - + # Setting up JDK 8, this is required to build Feathr - name: Set up JDK 8 uses: actions/setup-java@v2 @@ -27,10 +27,9 @@ jobs: # CI release command defaults to publishSigned # Sonatype release command defaults to sonaTypeBundleRelease - # https://github.com/sbt/sbt-ci-release - - name: Sbt ci release - run: | - sbt ci-release + - name: Gradle publish + if: startsWith(github.head_ref, 'release/v') + run: gradle clean publish env: PGP_PASSPHRASE: ${{ secrets.MAVEN_GPG_PASSPHRASE }} PGP_SECRET: ${{ secrets.MAVEN_GPG_PRIVATE_KEY }} diff --git a/.github/workflows/pull_request_push_test.yml b/.github/workflows/pull_request_push_test.yml index 1102d6028..9d336f804 100644 --- a/.github/workflows/pull_request_push_test.yml +++ b/.github/workflows/pull_request_push_test.yml @@ -23,10 +23,14 @@ on: - "ui/**" - "**/README.md" + schedule: + # Runs daily at 1 PM UTC (9 PM CST), will send notification to TEAMS_WEBHOOK + - cron: '00 13 * * *' + jobs: - sbt_test: + gradle_test: runs-on: ubuntu-latest - if: github.event_name == 'push' || github.event_name == 'pull_request' || (github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'safe to test')) + if: github.event_name == 'schedule' || github.event_name == 'push' || github.event_name == 'pull_request' || (github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'safe to test')) steps: - uses: actions/checkout@v2 with: @@ -37,11 +41,11 @@ jobs: java-version: "8" distribution: "temurin" - name: Run tests - run: sbt clean && sbt test + run: ./gradlew clean && ./gradlew test python_lint: runs-on: ubuntu-latest - if: github.event_name == 'push' || github.event_name == 'pull_request' || (github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'safe to test')) + if: github.event_name == 'schedule' || github.event_name == 'push' || github.event_name == 'pull_request' || (github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'safe to test')) steps: - name: Set up Python 3.8 uses: actions/setup-python@v2 @@ -61,7 +65,7 @@ jobs: databricks_test: runs-on: ubuntu-latest - if: github.event_name == 'push' || github.event_name == 'pull_request' || (github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'safe to test')) + if: github.event_name == 'schedule' || github.event_name == 'push' || github.event_name == 'pull_request' || (github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'safe to test')) steps: - uses: actions/checkout@v2 with: @@ -71,15 +75,15 @@ jobs: with: java-version: "8" distribution: "temurin" - - name: Build JAR + - name: Gradle build run: | - sbt assembly + ./gradlew build # remote folder for CI upload echo "CI_SPARK_REMOTE_JAR_FOLDER=feathr_jar_github_action_$(date +"%H_%M_%S")" >> $GITHUB_ENV # get local jar name without paths so version change won't affect it - echo "FEATHR_LOCAL_JAR_NAME=$(ls target/scala-2.12/*.jar| xargs -n 1 basename)" >> $GITHUB_ENV + echo "FEATHR_LOCAL_JAR_NAME=$(ls build/libs/*.jar| xargs -n 1 basename)" >> $GITHUB_ENV # get local jar name without path - echo "FEATHR_LOCAL_JAR_FULL_NAME_PATH=$(ls target/scala-2.12/*.jar)" >> $GITHUB_ENV + echo "FEATHR_LOCAL_JAR_FULL_NAME_PATH=$(ls build/libs/*.jar)" >> $GITHUB_ENV - name: Set up Python 3.8 uses: actions/setup-python@v2 with: @@ -87,8 +91,7 @@ jobs: - name: Install Feathr Package run: | python -m pip install --upgrade pip - python -m pip install pytest pytest-xdist databricks-cli - python -m pip install -e ./feathr_project/ + python -m pip install -e ./feathr_project/[all] if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - name: Set env variable and upload jars env: @@ -126,13 +129,12 @@ jobs: AWS_ACCESS_KEY_ID: ${{secrets.AWS_ACCESS_KEY_ID}} AWS_SECRET_ACCESS_KEY: ${{secrets.AWS_SECRET_ACCESS_KEY}} run: | - # run only test with databricks. run in 4 parallel jobs - pytest -n 6 feathr_project/test/ - + # run only test with databricks. run in 6 parallel jobs + pytest -n 6 --cov-report term-missing --cov=feathr_project/feathr feathr_project/test --cov-config=.github/workflows/.coveragerc_db azure_synapse_test: # might be a bit duplication to setup both the azure_synapse test and databricks test, but for now we will keep those to accelerate the test speed runs-on: ubuntu-latest - if: github.event_name == 'push' || github.event_name == 'pull_request' || (github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'safe to test')) + if: github.event_name == 'schedule' || github.event_name == 'push' || github.event_name == 'pull_request' || (github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'safe to test')) steps: - uses: actions/checkout@v2 with: @@ -142,15 +144,16 @@ jobs: with: java-version: "8" distribution: "temurin" - - name: Build JAR + + - name: Gradle build run: | - sbt assembly + ./gradlew build # remote folder for CI upload echo "CI_SPARK_REMOTE_JAR_FOLDER=feathr_jar_github_action_$(date +"%H_%M_%S")" >> $GITHUB_ENV # get local jar name without paths so version change won't affect it - echo "FEATHR_LOCAL_JAR_NAME=$(ls target/scala-2.12/*.jar| xargs -n 1 basename)" >> $GITHUB_ENV + echo "FEATHR_LOCAL_JAR_NAME=$(ls build/libs/*.jar| xargs -n 1 basename)" >> $GITHUB_ENV # get local jar name without path - echo "FEATHR_LOCAL_JAR_FULL_NAME_PATH=$(ls target/scala-2.12/*.jar)" >> $GITHUB_ENV + echo "FEATHR_LOCAL_JAR_FULL_NAME_PATH=$(ls build/libs/*.jar)" >> $GITHUB_ENV - name: Set up Python 3.8 uses: actions/setup-python@v2 with: @@ -166,8 +169,7 @@ jobs: - name: Install Feathr Package run: | python -m pip install --upgrade pip - python -m pip install pytest pytest-xdist - python -m pip install -e ./feathr_project/ + python -m pip install -e ./feathr_project/[all] if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - name: Run Feathr with Azure Synapse env: @@ -198,12 +200,12 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{secrets.AWS_SECRET_ACCESS_KEY}} run: | # skip databricks related test as we just ran the test; also seperate databricks and synapse test to make sure there's no write conflict - # run in 4 parallel jobs to make the time shorter - pytest -n 6 feathr_project/test/ + # run in 6 parallel jobs to make the time shorter + pytest -n 6 --cov-report term-missing --cov=feathr_project/feathr feathr_project/test --cov-config=.github/workflows/.coveragerc_sy local_spark_test: runs-on: ubuntu-latest - if: github.event_name == 'push' || github.event_name == 'pull_request' || (github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'safe to test')) + if: github.event_name == 'schedule' || github.event_name == 'push' || github.event_name == 'pull_request' || (github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'safe to test')) steps: - uses: actions/checkout@v2 with: @@ -213,15 +215,16 @@ jobs: with: java-version: "8" distribution: "temurin" - - name: Build JAR + + - name: Gradle build run: | - sbt assembly + ./gradlew build # remote folder for CI upload echo "CI_SPARK_REMOTE_JAR_FOLDER=feathr_jar_github_action_$(date +"%H_%M_%S")" >> $GITHUB_ENV # get local jar name without paths so version change won't affect it - echo "FEATHR_LOCAL_JAR_NAME=$(ls target/scala-2.12/*.jar| xargs -n 1 basename)" >> $GITHUB_ENV + echo "FEATHR_LOCAL_JAR_NAME=$(ls build/libs/*.jar| xargs -n 1 basename)" >> $GITHUB_ENV # get local jar name without path - echo "FEATHR_LOCAL_JAR_FULL_NAME_PATH=$(ls target/scala-2.12/*.jar)" >> $GITHUB_ENV + echo "FEATHR_LOCAL_JAR_FULL_NAME_PATH=$(ls build/libs/*.jar)" >> $GITHUB_ENV - name: Set up Python 3.8 uses: actions/setup-python@v2 with: @@ -229,9 +232,8 @@ jobs: - name: Install Feathr Package run: | python -m pip install --upgrade pip - python -m pip install pytest pytest-xdist - python -m pip install -e ./feathr_project/ - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + python -m pip install -e ./feathr_project/[all] + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - name: Run Feathr with Local Spark env: PROJECT_CONFIG__PROJECT_NAME: "feathr_github_ci_local" @@ -258,4 +260,26 @@ jobs: SQL1_PASSWORD: ${{secrets.SQL1_PASSWORD}} run: | # skip cloud related tests - pytest feathr_project/test/test_local_spark_e2e.py \ No newline at end of file + pytest --cov-report term-missing --cov=feathr_project/feathr/spark_provider feathr_project/test/test_local_spark_e2e.py --cov-config=.github/workflows/.coveragerc_local + + failure_notification: + # If any failure, warning message will be sent + needs: [gradle_test, python_lint, databricks_test, azure_synapse_test, local_spark_test] + runs-on: ubuntu-latest + if: failure() && github.event_name == 'schedule' + steps: + - name: Warning + run: | + curl -H 'Content-Type: application/json' -d '{"text": "[WARNING] Daily CI has failure, please check: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"}' ${{ secrets.TEAMS_WEBHOOK }} + + notification: + # Final Daily Report with all job status + needs: [gradle_test, python_lint, databricks_test, azure_synapse_test, local_spark_test] + runs-on: ubuntu-latest + if: always() && github.event_name == 'schedule' + steps: + - name: Get Date + run: echo "NOW=$(date +'%Y-%m-%d')" >> $GITHUB_ENV + - name: Notification + run: | + curl -H 'Content-Type: application/json' -d '{"text": "${{env.NOW}} Daily Report: 1. Gradle Test ${{needs.gradle_test.result}}, 2. Python Lint Test ${{needs.python_lint.result}}, 3. Databricks Test ${{needs.databricks_test.result}}, 4. Synapse Test ${{needs.azure_synapse_test.result}} , 5. LOCAL SPARK TEST ${{needs.local_spark_test.result}}. Link: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"}' ${{ secrets.TEAMS_WEBHOOK }} \ No newline at end of file diff --git a/.gitignore b/.gitignore index 976c0b239..6d39b31f4 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,7 @@ .AppleDouble .LSOverride metastore_db -src/integTest +feathr-impl/src/integTest test-output temp @@ -189,17 +189,16 @@ cython_debug/ # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml hs_err_pid* -target/ .idea .project/target .project/project .DS_store -.DS_Store *.jar -src/main/scala/META-INF/MANIFEST.MF +feathr-impl/src/main/scala/META-INF/MANIFEST.MF *.MF feathr_project/feathr_cli.egg-info/* *.pyc +*.iml # VS Code .vscode @@ -207,9 +206,20 @@ feathr_project/feathr_cli.egg-info/* #Local Build null/* +# Ignore Gradle project-specific cache directory +.gradle + +# Ignore Gradle build output directory +build + # For Metal Server .metals/ .bloop/ project/.bloop metals.sbt + .bsp/sbt.json + +# Feathr output debug folder +**/debug/ + diff --git a/.husky/pre-commit b/.husky/pre-commit old mode 100755 new mode 100644 index d24fdfc60..0312b7602 --- a/.husky/pre-commit +++ b/.husky/pre-commit @@ -1,4 +1,4 @@ #!/usr/bin/env sh . "$(dirname -- "$0")/_/husky.sh" -npx lint-staged +npx lint-staged \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 76a01bd06..ec137aa03 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -40,7 +40,11 @@ Our open source community strives to: - **Be respectful**: We are a world-wide community of professionals, and we conduct ourselves professionally. Disagreement is no excuse for poor behavior and poor manners. - **Understand disagreements**: Disagreements, both social and technical, are useful learning opportunities. Seek to understand the other viewpoints and resolve differences constructively. - **Remember that we’re different**. The strength of our community comes from its diversity, people from a wide range of backgrounds. Different people have different perspectives on issues. Being unable to understand why someone holds a viewpoint doesn’t mean that they’re wrong. Focus on helping to resolve issues and learning from mistakes. +- ## Attribution & Acknowledgements This code of conduct is based on the Open Code of Conduct from the [TODOGroup](https://todogroup.org/blog/open-code-of-conduct/). + +# Committers +Benjamin Le, David Stein, Edwin Cheung, Hangfei Lin, Jimmy Guo, Jinghui Mo, Li Lu, Rama Ramani, Ray Zhang, Xiaoyong Zhu diff --git a/FeathrRegistry.Dockerfile b/FeathrRegistry.Dockerfile index f3c2d6792..c127b81c6 100644 --- a/FeathrRegistry.Dockerfile +++ b/FeathrRegistry.Dockerfile @@ -11,7 +11,7 @@ RUN npm install && npm run build FROM python:3.9 ## Install dependencies -RUN apt-get update -y && apt-get install -y nginx +RUN apt-get update -y && apt-get install -y nginx freetds-dev COPY ./registry /usr/src/registry WORKDIR /usr/src/registry/sql-registry RUN pip install -r requirements.txt diff --git a/build.gradle b/build.gradle new file mode 100644 index 000000000..250d08422 --- /dev/null +++ b/build.gradle @@ -0,0 +1,173 @@ +import com.vanniktech.maven.publish.SonatypeHost + +buildscript { + ext.junitJupiterVersion = '5.6.1' + ext.pegasusVersion = '29.22.16' + ext.mavenVersion = '3.6.3' + ext.springVersion = '5.3.19' + ext.springBootVersion = '2.5.12' + apply from: './repositories.gradle' + buildscript.repositories.addAll(project.repositories) + dependencies { + classpath 'com.linkedin.pegasus:gradle-plugins:' + pegasusVersion + } +} + +plugins { + id 'java' + // Currently "maven-publish" has some issues with publishing to Nexus repo. So, we will use a different plugin. + // See https://issues.sonatype.org/browse/OSSRH-86507 for more details. + id "com.vanniktech.maven.publish" version "0.22.0" + id 'signing' +} + +repositories { + mavenCentral() + mavenLocal() + maven { + url "https://repository.mulesoft.org/nexus/content/repositories/public/" + } + maven { + url "https://linkedin.jfrog.io/artifactory/open-source/" // GMA, pegasus + } + +} + +configurations { + // configuration that holds jars to include in the jar + extraLibs + + // Dependencies that will be provided at runtime in the cloud execution + provided + + compileOnly.extendsFrom(provided) + testImplementation.extendsFrom provided +} + +jar { + archivesBaseName = "feathr_2.12" + duplicatesStrategy = DuplicatesStrategy.EXCLUDE + manifest { + attributes('Class-Path': [project.configurations.runtimeClasspath], + 'Main-Class': 'com.linkedin.feathr.offline.job.FeatureJoinJob', + "Implementation-title": "Build jar for local experimentation") + } + from { + configurations.runtimeClasspath.collect { it.isDirectory() ? it : zipTree(it) } + } +} + +dependencies { + implementation project(":feathr-compute") + implementation project(":feathr-config") + implementation project(":feathr-data-models") + implementation project(":feathr-impl") + // needed to include data models in jar + extraLibs project(path: ':feathr-data-models', configuration: 'dataTemplate') +} + +ext { + // Version numbers shared between multiple dependencies + // FUTURE consider version catalogs https://docs.gradle.org/current/userguide/platforms.html + ver = [ + scala : '2.12.15', + scala_rt: '2.12', + spark : '3.1.3' + ] +} + +project.ext.spec = [ + 'product' : [ + 'pegasus' : [ + 'd2' : 'com.linkedin.pegasus:d2:29.33.3', + 'data' : 'com.linkedin.pegasus:data:29.33.3', + 'dataAvro1_6' : 'com.linkedin.pegasus:data-avro-1_6:29.33.3', + 'generator': 'com.linkedin.pegasus:generator:29.33.3', + ], + 'jackson' : [ + 'dataformat_csv' : "com.fasterxml.jackson.dataformat:jackson-dataformat-csv:2.12.6", + 'dataformat_yaml' : "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.6", + 'dataformat_hocon' : "com.jasonclawson:jackson-dataformat-hocon:1.1.0", + 'module_scala' : "com.fasterxml.jackson.module:jackson-module-scala_$ver.scala_rt:2.12.6", + 'jackson_databind' : "com.fasterxml.jackson.core:jackson-databind:2.12.6.1", + 'jackson_core': "com.fasterxml.jackson.core:jackson-core:2.12.6", + 'jackson_module_caseclass' : "com.github.changvvb:jackson-module-caseclass_$ver.scala_rt:1.1.1", + ], + 'spark_redis' : "com.redislabs:spark-redis_$ver.scala_rt:3.0.0", + 'typesafe_config' : "com.typesafe:config:1.3.4", + 'hadoop' : [ + 'mapreduce_client_core' : "org.apache.hadoop:hadoop-mapreduce-client-core:2.7.7", + 'common' : "org.apache.hadoop:hadoop-common:2.7.7", + ], + 'spark' : [ + 'spark_core' : "org.apache.spark:spark-core_$ver.scala_rt:$ver.spark", + 'spark_avro' : "org.apache.spark:spark-avro_$ver.scala_rt:$ver.spark", + 'spark_hive' : "org.apache.spark:spark-hive_$ver.scala_rt:$ver.spark", + 'spark_sql' : "org.apache.spark:spark-sql_$ver.scala_rt:$ver.spark", + 'spark_catalyst' : "org.apache.spark:spark-catalyst_$ver.scala_rt:$ver.spark", + ], + 'scala' : [ + 'scala_library' : "org.scala-lang:scala-library:$ver.scala", + 'scalatest' : "org.scalatest:scalatest_$ver.scala_rt:3.0.0", + ], + 'avro' : "org.apache.avro:avro:1.10.2", + "avroUtil": "com.linkedin.avroutil1:helper-all:0.2.100", + 'fastutil' : "it.unimi.dsi:fastutil:8.1.1", + 'mvel' : "org.mvel:mvel2:2.2.8.Final", + 'protobuf' : "com.google.protobuf:protobuf-java:3.19.4", + 'guava' : "com.google.guava:guava:25.0-jre", + 'xbean' : "org.apache.xbean:xbean-asm6-shaded:4.10", + 'log4j' : "log4j:log4j:1.2.17", + 'json' : "org.json:json:20180130", + 'equalsverifier' : "nl.jqno.equalsverifier:equalsverifier:3.1.12", + 'mockito' : "org.mockito:mockito-core:3.1.0", + "mockito_inline": "org.mockito:mockito-inline:2.28.2", + 'testing' : "org.testng:testng:6.14.3", + 'jdiagnostics' : "org.anarres.jdiagnostics:jdiagnostics:1.0.7", + 'jsonSchemaVali': "com.github.everit-org.json-schema:org.everit.json.schema:1.9.1", + "antlr": "org.antlr:antlr4:4.8", + "antlrRuntime": "org.antlr:antlr4-runtime:4.8", + "jsqlparser": "com.github.jsqlparser:jsqlparser:3.1", + + ] +] + +if (hasProperty('buildScan')) { + buildScan { + termsOfServiceUrl = 'https://gradle.com/terms-of-service' + termsOfServiceAgree = 'yes' + } +} + +allprojects { + plugins.withId("com.vanniktech.maven.publish.base") { + group = "com.linkedin.feathr" + version = project.version + mavenPublishing { + publishToMavenCentral(SonatypeHost.DEFAULT) + signAllPublications() + pom { + name = 'Feathr' + description = 'An Enterprise-Grade, High Performance Feature Store' + url = 'https://github.com/linkedin/feathr' + licenses { + license { + name = 'APL2' + url = 'http://www.apache.org/licenses/LICENSE-2.0.txt' + } + } + developers { + developer { + id = 'feathr_dev' + name = 'Feathr Dev' + email = 'feathrai@gmail.com' + } + } + scm { + connection = 'scm:git@github.com:linkedin/feathr.git' + url = 'https://github.com/linkedin/feathr' + } + } + } + } +} diff --git a/build.sbt b/build.sbt deleted file mode 100644 index 2919ddae6..000000000 --- a/build.sbt +++ /dev/null @@ -1,107 +0,0 @@ -import sbt.Keys.publishLocalConfiguration - -ThisBuild / resolvers += Resolver.mavenLocal -ThisBuild / scalaVersion := "2.12.15" -ThisBuild / version := "0.8.0" -ThisBuild / organization := "com.linkedin.feathr" -ThisBuild / organizationName := "linkedin" -val sparkVersion = "3.1.3" - -publishLocalConfiguration := publishLocalConfiguration.value.withOverwrite(true) - -val localAndCloudDiffDependencies = Seq( - "org.apache.spark" %% "spark-avro" % sparkVersion, - "org.apache.spark" %% "spark-sql" % sparkVersion, - "org.apache.spark" %% "spark-hive" % sparkVersion, - "org.apache.spark" %% "spark-catalyst" % sparkVersion, - "org.apache.logging.log4j" % "log4j-core" % "2.17.2", - "com.typesafe" % "config" % "1.3.4", - "com.fasterxml.jackson.core" % "jackson-databind" % "2.12.6.1", - "org.apache.hadoop" % "hadoop-mapreduce-client-core" % "2.7.7", - "org.apache.hadoop" % "hadoop-common" % "2.7.7", - "org.apache.avro" % "avro" % "1.8.2", - "org.apache.xbean" % "xbean-asm6-shaded" % "4.10", - "org.apache.spark" % "spark-sql-kafka-0-10_2.12" % "3.1.3" -) - -val cloudProvidedDeps = localAndCloudDiffDependencies.map(x => x % "provided") - -val localAndCloudCommonDependencies = Seq( - "com.microsoft.azure" % "azure-eventhubs-spark_2.12" % "2.3.21", - "org.apache.kafka" % "kafka-clients" % "3.1.0", - "com.google.guava" % "guava" % "31.1-jre", - "org.testng" % "testng" % "6.14.3" % Test, - "org.mockito" % "mockito-core" % "3.1.0" % Test, - "nl.jqno.equalsverifier" % "equalsverifier" % "3.1.13" % Test, - "org.scalatest" %% "scalatest" % "3.0.9" % Test, - "it.unimi.dsi" % "fastutil" % "8.1.1", - "org.mvel" % "mvel2" % "2.2.8.Final", - "com.fasterxml.jackson.module" %% "jackson-module-scala" % "2.12.6", - "com.fasterxml.jackson.dataformat" % "jackson-dataformat-yaml" % "2.12.6", - "com.fasterxml.jackson.dataformat" % "jackson-dataformat-csv" % "2.12.6", - "com.jasonclawson" % "jackson-dataformat-hocon" % "1.1.0", - "com.redislabs" %% "spark-redis" % "3.0.0", - "org.scalatest" %% "scalatest" % "3.0.9" % "test", - "org.apache.xbean" % "xbean-asm6-shaded" % "4.10", - "com.google.protobuf" % "protobuf-java" % "2.6.1", - "net.snowflake" % "snowflake-jdbc" % "3.13.18", - "net.snowflake" % "spark-snowflake_2.12" % "2.10.0-spark_3.2", - "org.apache.commons" % "commons-lang3" % "3.12.0", - "org.xerial" % "sqlite-jdbc" % "3.36.0.3", - "com.github.changvvb" %% "jackson-module-caseclass" % "1.1.1", - "com.azure.cosmos.spark" % "azure-cosmos-spark_3-1_2-12" % "4.11.1", - "org.eclipse.jetty" % "jetty-util" % "9.3.24.v20180605" -) // Common deps - -val jdbcDrivers = Seq( - "com.microsoft.sqlserver" % "mssql-jdbc" % "10.2.0.jre8", - "net.snowflake" % "snowflake-jdbc" % "3.13.18", - "org.postgresql" % "postgresql" % "42.3.4", -) - -// For azure -lazy val root = (project in file(".")) - .settings( - name := "feathr", - // To assemble, run sbt assembly -java-home /Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home - assembly / mainClass := Some("com.linkedin.feathr.offline.job.FeatureJoinJob"), - libraryDependencies ++= cloudProvidedDeps, - libraryDependencies ++= localAndCloudCommonDependencies, - libraryDependencies ++= jdbcDrivers, - libraryDependencies ++= Seq( - "org.apache.spark" %% "spark-core" % sparkVersion % "provided" - ) - ) - -// If you want to build jar for feathr test, enable this and comment out root -//lazy val localCliJar = (project in file(".")) -// .settings( -// name := "feathr-cli", -// // To assemble, run sbt assembly -java-home /Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home -// assembly / mainClass := Some("com.linkedin.feathr.cli.FeatureExperimentEntryPoint"), -// // assembly / mainClass := Some("com.linkedin.feathr.offline.job.FeatureJoinJob"), -// libraryDependencies ++= localAndCloudDiffDependencies, -// libraryDependencies ++= localAndCloudCommonDependencies, -// libraryDependencies ++= Seq( -// // See https://stackoverflow.com/questions/55923943/how-to-fix-unsupported-class-file-major-version-55-while-executing-org-apache -// "org.apache.spark" %% "spark-core" % sparkVersion exclude("org.apache.xbean","xbean-asm6-shaded") -// ) -// ) - - -// To assembly with certain java version: sbt assembly -java-home "/Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home" -// Please specify the feathr version feathr-assembly-X.X.X-SNAPSHOT.jar -// To execute the jar: java -jar target/scala-2.12/feathr-assembly-0.5.0-SNAPSHOT.jar (Please use the latest version of the jar) - -assembly / assemblyMergeStrategy := { - // See https://stackoverflow.com/questions/17265002/hadoop-no-filesystem-for-scheme-file - // See https://stackoverflow.com/questions/62232209/classnotfoundexception-caused-by-java-lang-classnotfoundexception-csv-default - case PathList("META-INF","services",xs @ _*) => MergeStrategy.filterDistinctLines - case PathList("META-INF",xs @ _*) => MergeStrategy.discard - case _ => MergeStrategy.first -} - -// Some systems(like Hadoop) use different versinos of protobuf(like v2) so we have to shade it. -assemblyShadeRules in assembly := Seq( - ShadeRule.rename("com.google.protobuf.**" -> "shade.protobuf.@1").inAll, -) \ No newline at end of file diff --git a/docker/Dockerfile b/docker/Dockerfile index 2735306c5..4149a7521 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -18,7 +18,7 @@ RUN apt-get update && \ RUN mkdir /feathr WORKDIR /feathr -RUN pip install git+https://github.com/linkedin/feathr.git#subdirectory=feathr_project +RUN pip install git+https://github.com/feathr-ai/feathr.git#subdirectory=feathr_project # install code-server RUN mkdir -p /opt/code-server && \ diff --git a/docs/README.md b/docs/README.md index 1a797ce48..ca67ed446 100644 --- a/docs/README.md +++ b/docs/README.md @@ -159,7 +159,7 @@ Read [Point-in-time Correctness and Point-in-time Join in Feathr](https://feathr ### Running Feathr Examples -Follow the [quick start Jupyter Notebook](./samples/product_recommendation_demo.ipynb) to try it out. There is also a companion [quick start guide](https://feathr-ai.github.io/feathr/quickstart_synapse.html) containing a bit more explanation on the notebook. +Follow the [quick start Jupyter Notebook](https://github.com/feathr-ai/feathr/blob/main/docs/samples/azure_synapse/product_recommendation_demo.ipynb) to try it out. There is also a companion [quick start guide](https://feathr-ai.github.io/feathr/quickstart_synapse.html) containing a bit more explanation on the notebook. ## 🗣️ Tech Talks on Feathr diff --git a/docs/concepts/feature-registry.md b/docs/concepts/feature-registry.md index 112fa1c4b..e78c0e605 100644 --- a/docs/concepts/feature-registry.md +++ b/docs/concepts/feature-registry.md @@ -80,7 +80,7 @@ Please avoid applying a same name to different features under a certain project. The feature producers can just let the feature consumers know which features exist so the feature consumers can reuse them. For feature consumers, they can reuse existing features from the registry. The whole project can be retrieved to local environment by calling this API `client.get_features_from_registry` with a project name. This encourage feature reuse across organizations. For example, end users of a feature just need to read all feature definitions from the existing projects, then use a few features from the projects and join those features with a new dataset you have. -For example, in the [product recommendation demo notebook](./../samples/product_recommendation_demo.ipynb), some other team members have already defined a few features, such as `feature_user_gift_card_balance` and `feature_user_has_valid_credit_card`. If we want to reuse those features for anti-abuse purpose in a new dataset, what you can do is like this, i.e. just call `get_features_from_registry` to get the features, then put the features you want to query to the anti-abuse dataset you have. +For example, in the [product recommendation demo notebook](https://github.com/feathr-ai/feathr/blob/main/docs/samples/azure_synapse/product_recommendation_demo.ipynb), some other team members have already defined a few features, such as `feature_user_gift_card_balance` and `feature_user_has_valid_credit_card`. If we want to reuse those features for anti-abuse purpose in a new dataset, what you can do is like this, i.e. just call `get_features_from_registry` to get the features, then put the features you want to query to the anti-abuse dataset you have. ```python registered_features_dict = client.get_features_from_registry(client.project_name) diff --git a/docs/dev_guide/cloud_integration_testing.md b/docs/dev_guide/cloud_integration_testing.md index 3ce5ea206..ed558d6c2 100644 --- a/docs/dev_guide/cloud_integration_testing.md +++ b/docs/dev_guide/cloud_integration_testing.md @@ -7,7 +7,7 @@ parent: Developer Guides We use [GitHub Actions](https://github.com/feathr-ai/feathr/tree/main/.github/workflows) to do cloud integration test. Currently the integration test has 4 jobs: -- running `sbt test` to verify if the scala/spark related code has passed all the test +- running `./gradlew test` to verify if the scala/spark related code has passed all the test - running `flake8` to lint python scripts and make sure there are no obvious syntax errors - running the built jar in databricks environment with end to end test to make sure it passed the end to end test - running the built jar in Azure Synapse environment with end to end test to make sure it passed the end to end test diff --git a/docs/dev_guide/feathr_overall_release_guide.md b/docs/dev_guide/feathr_overall_release_guide.md index 0174c8dae..323d5d697 100644 --- a/docs/dev_guide/feathr_overall_release_guide.md +++ b/docs/dev_guide/feathr_overall_release_guide.md @@ -10,63 +10,88 @@ This document describes all the release process for the development team. ## Prerequisites -- Make sure the CI tests are passing so there are no surprises on the release day. +- Make sure the CI tests are passing prior to bug bash. - Make sure all the active PRs related to the release are merged. - ## When to Release -- For each major and minor version release, please follow these steps. -- For patch versions, there should be no releases. +The release process is triggered by the release manager. The release manager will decide when to release with following steps: + +1. Ensure Prerequisites are met. +2. Creation of Release Candidate(rc) on GitHub. +3. Bug Bash. +4. Creation of Release on GitHub. +5. Post Release announcement. + +## Release Versioning + +- Major and minor version: X.Y.Z +- Release Candidate: X.Y.Z-rcN ## Writing Release Note Write a release note following past examples [here](https://github.com/feathr-ai/feathr/releases). Read through the [commit log](https://github.com/feathr-ai/feathr/commits/main) to identify the commits after last release to include in the release note. Here are the major things to include -- highlights of the release -- improvements and changes of this release -- new contributors of this release +- Highlights of the release +- Improvements and changes of this release +- New contributors of this release ## Code Changes -Before the release is made, the version needs to be updated in following places -- [build.sbt](https://github.com/feathr-ai/feathr/blob/main/build.sbt#L3) - For Maven release version + +Before the release candidate or release is made, the version needs to be updated in following places + +- [build.gradle](https://github.com/feathr-ai/feathr/blob/main/gradle.properties#L3) - For Maven release version - [version.py](https://github.com/feathr-ai/feathr/blob/main/feathr_project/feathr/version.py#L1) - For Feathr version - [conf.py](https://github.com/feathr-ai/feathr/blob/main/feathr_project/docs/conf.py#L27) - For documentation version -- [feathr_config.yaml](https://github.com/feathr-ai/feathr/blob/main/feathr_project/test/test_user_workspace/feathr_config.yaml#L84) - To set the spark runtime location for Azure Synapse and Azure Databricks used by test suite. Please update all .yaml files under this path. +- [feathr_config.yaml](https://github.com/feathr-ai/feathr/blob/main/feathr_project/test/test_user_workspace/feathr_config.yaml#L84) - To set the spark runtime location for Azure Synapse and Azure Databricks used by test suite. Please update all .yaml files under this path. +- [package.json](https://github.com/feathr-ai/feathr/blob/main/ui/package.json#L3) - For Feathr UI version + +Following file should only be updated for release, which means should be skipped for release candidate. + - [azure_resource_provision.json](https://github.com/feathr-ai/feathr/blob/main/docs/how-to-guides/azure_resource_provision.json#L114) - To set the deployment template to pull the latest release image. -- [constants.py](https://github.com/feathr-ai/feathr/blob/main/feathr_project/feathr/constants.py#L31) - To set the default maven artifact version (Only needed when maven version is **NOT** the same as python sdk version) -## Triggering automated release pipelines -Our goal is to automate the release process as much as possible. So far, we have automated the following steps -1. Automated [workflow](https://github.com/feathr-ai/feathr/blob/main/.github/workflows/docker-publish.yml) to build and publish for our UI and API container to [dockerhub](https://hub.docker.com/r/feathrfeaturestore/feathr-registry/tags). - **Triggers** - Nightly, branch with name pattern "releases/*" +## Release Branches + +Each major and minor release should have a release branch. The release branch should be named as `releases/vX.Y.Z` or `releases/vX.Y.Z-rcN` where `X.Y.Z` is the release version. The release branch should be created from the `main` branch. See past release branches [here](https://github.com/feathr-ai/feathr/branches/all?query=releases). -1. Automated [workflow](https://github.com/feathr-ai/feathr/blob/main/.github/workflows/publish-to-pypi.yml) for publishing Python package to [PyPi](https://pypi.org/project/feathr/). +## Release Tags - **Triggers** - branch with name pattern "releases/*" +Once the release branch is created, a release tag should be created from the release branch. The release tag should be named as `vX.Y.Z` or `vX.Y.Z-rcN` where `X.Y.Z` is the release version. See past release tags [here](https://github.com/feathr-ai/feathr/tags). -1. Automated Maven workflow - Coming soon. +## Triggering automated release pipelines -**PLEASE NOTE: To trigger the above workflows as part of release, create a new branch with pattern releases/v0.x.0**. See past release branches [here](https://github.com/feathr-ai/feathr/branches/all?query=releases). +Once the release branch and release tag are created, the release pipelines will be triggered automatically. The release pipelines will build the release artifacts and publish them to Maven and PyPI. +1. Automated [workflow](https://github.com/feathr-ai/feathr/blob/main/.github/workflows/docker-publish.yml) to build and publish for Feathr Registry docker images to [DockerHub](https://hub.docker.com/r/feathrfeaturestore/feathr-registry/tags). -## Release Maven + **Triggers** - Nightly or branch with name pattern "releases/*" + +2. Automated [workflow](https://github.com/feathr-ai/feathr/blob/main/.github/workflows/publish-to-pypi.yml) for publishing Python package to [PyPi](https://pypi.org/project/feathr/). -See [Developer Guide for publishing to maven](publish_to_maven.md) + **Triggers** - branch with name pattern "releases/*" + +3. Automated [workflow](https://github.com/feathr-ai/feathr/blob/main/.github/workflows/publish-to-maven.yml) for publishing the jar to [maven/sonatype repository](https://oss.sonatype.org/). ## Upload Feathr Jar Run the command to generate the Java jar. After the jar is generated, please upload to [Azure storage](https://ms.portal.azure.com/#view/Microsoft_Azure_Storage/ContainerMenuBlade/~/overview/storageAccountId/%2Fsubscriptions%2Fa6c2a7cc-d67e-4a1a-b765-983f08c0423a%2FresourceGroups%2Fazurefeathrintegration%2Fproviders%2FMicrosoft.Storage%2FstorageAccounts%2Fazurefeathrstorage/path/public/etag/%220x8D9E6F64D62D599%22/defaultEncryptionScope/%24account-encryption-key/denyEncryptionScopeOverride//defaultId//publicAccessVal/Container) for faster access. ## Release PyPi -The automated workflow should take care of this, you can check under [actions](https://github.com/feathr-ai/feathr/actions/workflows/publish-to-pypi.yml) to see the triggered run and results. For manual steps, see [Python Package Release Note](https://feathr-ai.github.io/feathr/dev_guide/python_package_release.html) + +The automated workflow should take care of this, you can check under [actions](https://github.com/feathr-ai/feathr/actions/workflows/publish-to-pypi.yml) to see the triggered run and results. For manual steps, see [Python Package Release Guide](https://feathr-ai.github.io/feathr/dev_guide/python_package_release.html) ## Updating docker image for API and Registry + The automated workflow should take care of this as well, you can check under [actions](https://github.com/feathr-ai/feathr/actions/workflows/docker-publish.yml) to see the triggered run and results. For manual steps, see [Feathr Registry docker image](https://feathr-ai.github.io/feathr/dev_guide/build-and-push-feathr-registry-docker-image.html) +## Release Maven + +The automated workflow should take of this too, you can check under [actions](https://github.com/feathr-ai/feathr/blob/main/.github/workflows/publish-to-maven.yml) to see the triggered run and results. For manual steps, see [Feathr Developer Guide for publishing to maven](https://feathr-ai.github.io/feathr/dev_guide/publish_to_maven.html) + ## Testing -Run one of the sample [notebook](https://github.com/feathr-ai/feathr/blob/main/docs/samples/product_recommendation_demo.ipynb) as it uses the latest package from Maven and PyPi. + +Run one of the sample [notebook](https://github.com/feathr-ai/feathr/blob/main/docs/samples/azure_synapse/product_recommendation_demo.ipynb) as it uses the latest package from Maven and PyPi. ## Announcement diff --git a/docs/dev_guide/images/coverage_res.png b/docs/dev_guide/images/coverage_res.png new file mode 100644 index 000000000..db7b0316f Binary files /dev/null and b/docs/dev_guide/images/coverage_res.png differ diff --git a/docs/dev_guide/new_contributor_guide.md b/docs/dev_guide/new_contributor_guide.md index 1856ffd84..223b7d91b 100644 --- a/docs/dev_guide/new_contributor_guide.md +++ b/docs/dev_guide/new_contributor_guide.md @@ -6,11 +6,11 @@ parent: Feathr Developer Guides # What can I contribute? All forms of contributions are welcome, including and not limited to: -* Improve or contribute new [notebook samples](https://github.com/feathr-ai/feathr/tree/main/feathr_project/feathrcli/data/feathr_user_workspace) +* Improve or contribute new [notebook samples](https://github.com/feathr-ai/feathr/tree/main/docs/samples) * Add tutorial, blog posts, tech talks etc * Increase media coverage and exposure * Improve user-facing documentation or developer-facing documentation -* Add testing code +* Add testing code * Add new features * Refactor and improve architecture * For any other forms of contribution and collaboration, don't hesitate to reach out to us. @@ -18,7 +18,7 @@ All forms of contributions are welcome, including and not limited to: # I am interested, how can I start? If you are new to this project, we recommend start with [`good-first-issue`](https://github.com/feathr-ai/feathr/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22). -The issues are also labled with what types of programming language the task need. +The issues are also labled with what types of programming language the task need. * [`good-first-issue` and `Python`](https://github.com/feathr-ai/feathr/issues?q=is%3Aopen+label%3A%22good+first+issue%22+label%3Apython) * [`good-first-issue` and `Scala`](https://github.com/feathr-ai/feathr/issues?q=is%3Aopen+label%3A%22good+first+issue%22+label%3Ascala) * [`good-first-issue` and `Java`](https://github.com/feathr-ai/feathr/issues?q=is%3Aopen+label%3A%22good+first+issue%22+label%3Ajava) diff --git a/docs/dev_guide/publish_to_maven.md b/docs/dev_guide/publish_to_maven.md index 02eab16bb..75baf3f01 100644 --- a/docs/dev_guide/publish_to_maven.md +++ b/docs/dev_guide/publish_to_maven.md @@ -10,8 +10,10 @@ parent: Developer Guides --- ### Prerequisites -- Install JDK8, for macOS: `brew install --cask adoptopenjdk` -- Install SBT, for macOS: `brew install sbt` +- Install JDK8, for macOS: + `brew tap adoptopenjdk/openjdk + brew install --cask adoptopenjdk8` +- Install Gradle, for macOS: `brew install gradle` - Install GPG, for macOS: `brew install gpg` - Sonatype account credential @@ -27,7 +29,7 @@ parent: Developer Guides "Central Repo Test " Change (N)ame, (E)mail, or (O)kay/(Q)uit? O ``` - * Save key passphrase, which is needed during the sbt publishSigned step + * Save key passphrase, which is needed during the gradle publishSigned step * Verify your gpg metadata, and note the uid. In this example it is `CA925CD6C9E8D064FF05B4728190C4130ABA0F98` * ``` $ gpg --list-keys @@ -47,45 +49,49 @@ parent: Developer Guides * upload to http://keyserver.ubuntu.com/ via `submit key` * Upload via command line. Currently this hasn't succeeded, if succeeded, please alter the steps here with your fix. - * ``` + * ``` $ gpg --keyserver keyserver.ubuntu.com --recv-keys CA925CD6C9E8D064FF05B4728190C4130ABA0F98 ``` + * Export your keyring file to somewhere on your disk (not to be checked in). + * ``` + $ gpg --export-secret-keys --armor + ``` --- 2. Set up `Sonatype` credentials * Get account details to login to https://oss.sonatype.org/. Reachout to Feathr team, such as @jaymo001, @hangfei or @blrchen - * Setup the credentials locally - * Create sonatype configuration file - * ``` - vim $HOME/.sbt/1.0/sonatype.sbt - ``` - * Paste the following with the sonatype credentials - * ``` - credentials += Credentials("Sonatype Nexus Repository Manager", - "oss.sonatype.org", - "", - "") - ``` + * Setup the credentials locally + ``` + * Paste the following with the sonatype credentials to your gradle.properties file + * ``` + signing.keyId= + signing.password= + signing.secretKeyRingFile= + mavenCentralUsername= + mavenCentralPassword= + + ``` --- -3. Increase version number in build.sbt, search for `ThisBuild / version` and replace the version number with the next version number. +3. Increase version number in gradle.properties and build.gradle files, and replace the version number with the next version number. * ``` - ThisBuild / version := "0.6.0" + version="0.6.0" ``` - ---- -4. Publish to sonatype/maven via sbt +4. Publish to sonatype/maven via gradle * In your feathr directory, clear your cache to prevent stale errors * ``` - rm -rf target/sonatype-staging/ + rm -rf build/ ``` - * Start sbt console by running - * ``` - sbt -java-home /Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home - ``` - * Execute command in sbt console to publish to maven - * ``` - reload; publishSigned; sonatypeBundleRelease + * Execute command in your terminal to publish to sonatype staging + * ``` + ./gradlew publish -Dorg.gradle.java.home= ``` + * Execute command in your terminal release the staged artifact into central maven. + * ``` + ./gradlew closeAndReleaseRepository -Dorg.gradle.java.home= + * To publish to local maven, execute the below command + * ``` + ./gradlew publishToMavenLocal -Dorg.gradle.java.home= + ``` --- 5. Upon release, new version will be published to Central: this typically occurs within 30 minutes, though updates to search can take up to 24 hours. See the [Sonatype documentation](https://central.sonatype.org/publish/publish-guide/#releasing-to-central) for more information. @@ -95,8 +101,9 @@ parent: Developer Guides 6. After new version is released via Maven, use the released version to run a test to ensure it actually works. You can do this by running a codebase that imports Feathr scala code. ## Troubleshooting -- If you get something like `[error] gpg: signing failed: Inappropriate ioctl for device`, run `export GPG_TTY=$(tty)` in your terminal and restart sbt console. -- If the published jar fails to run in Spark with error `java.lang.UnsupportedClassVersionError: com/feathr-ai/feathr/common/exception/FeathrInputDataException has been compiled by a more recent version of the Java Runtime (class file version 62.0), this version of the Java Runtime only recognizes class file versions up to 52.0`, make sure you complied with the right Java version with -java-home parameter in sbt console. +- If you get something like `[error] gpg: signing failed: Inappropriate ioctl for device`, run `export GPG_TTY=$(tty)` in your terminal and restart console. +- If the published jar fails to run in Spark with error `java.lang.UnsupportedClassVersionError: com/feathr-ai/feathr/common/exception/FeathrInputDataException has been compiled by a more recent version of the Java Runtime (class file version 62.0), this version of the Java Runtime only recognizes class file versions up to 52.0`, + make sure you complied with the right Java version with -Dorg.gradle.java.home parameter in your console. ## CI Automatic Publishing There is a Github Action that automates the above process, you can find it [here](../../.github/workflows/publish-to-maven.yml). This action is triggered anytime a new tag is created, which is usually for release purposes. To manually trigger the pipeline for testing purposes tag can be created using following commands @@ -138,28 +145,18 @@ Following are some of the things to keep in mind while attempting to do somethin uid [ultimate] YOUR NAME ssb abc123 2022-08-24 [E] [expires: 2024-08-23] ``` -1. Make sure you are using the right credential host in [sonatype.sbt](../../sonatype.sbt) +1. Make sure you are using the right credential host in [build.gradle](../../build.gradle) - For accounts created before Feb 2021 use __oss.sonatype.org__ - For accounts created after Feb 2021 use __s01.oss.sonatype.org__ - - -1. Make sure you are using latest release of sbt-pgp package, or atleast the one close to the dev box on which gpg keypair is generated. You can change the version in [build.sbt](../../build.sbt) - ```bash - addSbtPlugin("com.github.sbt" % "sbt-pgp" % "2.1.2") - ``` - -1. We are using sbt-ci-release plugin, that makes the publishing process easier. Read more about it [here](https://github.com/sbt/sbt-ci-release). You can add this in [build.sbt](../../build.sbt) - ```bash - addSbtPlugin("com.github.sbt" % "sbt-ci-release" % "1.5.10") ``` ### References -- https://github.com/xerial/sbt-sonatype +- https://github.com/johnsonlee/sonatype-publish-plugin - https://www.linuxbabe.com/security/a-practical-guide-to-gpg-part-1-generate-your-keypair - https://central.sonatype.org/publish/publish-guide/#deployment -- https://www.scala-sbt.org/1.x/docs/Using-Sonatype.html +- https://blog.sonatype.com/new-sonatype-scan-gradle-plugin -- https://github.com/sbt/sbt-ci-release +- https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-java-with-gradle diff --git a/docs/dev_guide/scala_dev_guide.md b/docs/dev_guide/scala_dev_guide.md index d743ebff0..8d79f0e2a 100644 --- a/docs/dev_guide/scala_dev_guide.md +++ b/docs/dev_guide/scala_dev_guide.md @@ -13,10 +13,9 @@ IntelliJ is the recommended IDE to use when developing Feathr. Please visit Inte in your local machine. To import Feathr as a new project: 1. Git clone Feathr into your local machine. i.e. via https `git clone https://github.com/feathr-ai/feathr.git` or ssh `git clone git@github.com:feathr-ai/feathr.git` 2. In IntelliJ, select `File` > `New` > `Project from Existing Sources...` and select `feathr` from the directory you cloned. -3. Under `Import project from external model` select `sbt`. Click `Next`. -4. Under `Project JDK` specify a valid Java `1.8` JDK and select SBT shell for `project reload` and `builds`. +3. Under `Import project from external model` select `gradle`. Click `Next`. +4. Under `Project JDK` specify a valid Java `1.8` JDK. 5. Click `Finish`. -6. You should see something like `[success] Total time: 5 s, completed Jun 1, 2022 9:43:26 PM` in sbt shell. ### Setup Verification @@ -34,28 +33,28 @@ Please checkout [Databricks' Scala Style Guide](https://github.com/databricks/sc ## Building and Testing -Feathr is compiled using [SBT](https://www.scala-sbt.org/1.x/docs/Command-Line-Reference.html). +Feathr is compiled using [Gradle](https://docs.gradle.org/current/userguide/command_line_interface.html). To compile, run ``` -sbt assembly +./gradlew build ``` To compile with certain java version, run ``` -sbt assembly -java-home "/Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home" +./gradlew build -Dorg.gradle.java.home=/JDK_PATH ``` -The jar files are compiled and placed in `feathr/target/scala-2.12/feathr-assembly-X.X.X.jar `. +The jar files are compiled and placed in `feathr/build/libs/feathr-X.X.X.jar `. To execute tests, run ``` -sbt test +./gradlew test ``` To execute a single test suite, run ``` -sbt 'testOnly com.linkedin.feathr.offline.AnchoredFeaturesIntegTest' +./gradlew test --tests com.linkedin.feathr.offline.AnchoredFeaturesIntegTest ``` -Refer to [SBT docs](https://www.scala-sbt.org/1.x/docs/Command-Line-Reference.html) for more commands. +Refer to [Gradle docs](https://docs.gradle.org/current/userguide/command_line_interface.html) for more commands. diff --git a/docs/dev_guide/test_coverage_guide.md b/docs/dev_guide/test_coverage_guide.md new file mode 100644 index 000000000..54ef20979 --- /dev/null +++ b/docs/dev_guide/test_coverage_guide.md @@ -0,0 +1,47 @@ +--- +layout: default +title: Feathr Test Coverage Guide +parent: Developer Guides +--- + +# Feathr Test Coverage Check Guide + +## Background +To maintain and improve codes quality of feathr, we expect test coverage ratio to be equal to or above 90% in general. For any code changes, please make sure related test cases are added and check that test coverage can meet our request. + +## How to conduct test coverage +### Through github workflows pipeline: + We already added this coverage checking into our CI pipeline. For every pull request, push and scheduled jobs, github will check the coverage when runing 'pytest' automatically. You can find the result for 'azure_synapse', 'databricks' and 'local spark', respectively from each PR and commit. + + An example of test coverage result: +![test coverage example](./images/coverage_res.png) + + From the example above we can see the result can show coverage ratio for each file and also the missing lines of codes that current test cases fail to cover. Based on the result, we can add related test cases to improve the total test coverage ratio. + +### Test locally: + We can also check the coverage locally by simply running the command under `feathr/` directory: + `pytest --cov-report term-missing --cov=feathr_project/feathr feathr_project/test/` + This will provide a coverage report for all codes of python client by running all python test cases. + + You may need to install `pytest-cov` locally: + `pip install pytest-cov` + + If you just want to check coverage situation under some folders by running some of test cases, you can also specify paths for each of them. Eg. if you only want to check coverage ratio of local spark, you may run: + `pytest --cov-report term-missing --cov=feathr_project/feathr/spark_provider feathr_project/test/test_local_spark_e2e.py` + + In addition, if you want the report ignore some files under the specified folder, you can set it through config file. Eg: + ``` + [run] + omit = feathr_project/feathr/registry/_feature_registry_purview.py + feathr_project/feathr/spark_provider/_synapse_submission.py + feathr_project/feathr/spark_provider/_localspark_submission.py + ``` + + The default config file is `.coveragerc`. You can also specify one by adding `--cov-config={your_coverage_config}` into the above `pytest` command. + + Coverage config example: + [coverage config to ignore synapse when running by databricks](../../.github/workflows/.coveragerc_db) + + More reference for pytest coverage: + [pytest coverage](https://pypi.org/project/pytest-cov/) + \ No newline at end of file diff --git a/docs/how-to-guides/azure-deployment-arm.md b/docs/how-to-guides/azure-deployment-arm.md index 7bc9a926f..170e419ea 100644 --- a/docs/how-to-guides/azure-deployment-arm.md +++ b/docs/how-to-guides/azure-deployment-arm.md @@ -17,7 +17,7 @@ The provided Azure Resource Manager (ARM) template deploys the following resourc 7. Azure Event Hub 8. Azure Redis -### Please Note: you need to have the **Owner Role** in the resource group you are deploying this in. Owner access is required to assign role to managed identity within the ARM template so it can access key vault and store secrets. It is also required by the permission section in our sample notebooks. If you don't have such permission, you might want to contact your IT admin to see if they can do that. +** Please Note: you need to have the *Owner Role* in the resource group you are deploying this in. Owner access is required to assign role to managed identity within the ARM template so it can access key vault and store secrets. It is also required by the permission section in our sample notebooks. If you don't have such permission, you might want to contact your IT admin to see if they can do that. ** Although we recommend end users deploy the resources using the ARM template, we understand that in many situations where users want to reuse existing resources instead of creating new resources; or users may have permission issues. See [Manually connecting existing resources](#manually-connecting-existing-resources) for more details. @@ -34,7 +34,7 @@ Feathr has native cloud integration and getting started with Feathr is very stra The very first step is to create an Azure Active Directory (AAD) application to enable authentication on the Feathr UI (which gets created as part of the deployment script). Currently it is not possible to create one through ARM template but you can easily create one by running the following CLI commands in the [Cloud Shell](https://shell.azure.com/bash). -### Please make note of the Client ID and Tenant ID for the AAD app, you will need it in the ARM template deployment section. +** Please make note of the Client ID and Tenant ID for the AAD app, you will need it in the ARM template deployment section.** ```bash # This is the prefix you want to name your resources with, make a note of it, you will need it during deployment. diff --git a/docs/how-to-guides/azure_resource_provision.json b/docs/how-to-guides/azure_resource_provision.json index 03d175052..f08771ad5 100644 --- a/docs/how-to-guides/azure_resource_provision.json +++ b/docs/how-to-guides/azure_resource_provision.json @@ -111,7 +111,7 @@ "destinationBacpacBlobUrl": "[concat('https://',variables('dlsName'),'.blob.core.windows.net/',variables('dlsFsName'),'/',variables('bacpacBlobName'))]", "bacpacDeploymentScriptName": "CopyBacpacFile", "bacpacDbExtensionName": "registryRbacDbImport", - "preBuiltdockerImage": "feathrfeaturestore/feathr-registry:releases-v0.8.0" + "preBuiltdockerImage": "feathrfeaturestore/feathr-registry:releases-v0.9.0" }, "functions": [], "resources": [ diff --git a/docs/how-to-guides/deployment/deployFeathr.ps1 b/docs/how-to-guides/deployment/deployFeathr.ps1 index 71dd769fb..1644c8715 100644 --- a/docs/how-to-guides/deployment/deployFeathr.ps1 +++ b/docs/how-to-guides/deployment/deployFeathr.ps1 @@ -16,5 +16,5 @@ New-AzDeployment ` -Name feathrDeployment ` -location $AzureRegion ` -principalId $UserObjectID ` - -TemplateUri https://raw.githubusercontent.com/linkedin/feathr/main/docs/how-to-guides/deploy.json ` + -TemplateUri https://raw.githubusercontent.com/feathr-ai/feathr/main/docs/how-to-guides/deployment/deploy.json ` -DeploymentDebugLogLevel All \ No newline at end of file diff --git a/docs/how-to-guides/deployment/requirements.txt b/docs/how-to-guides/deployment/requirements.txt index 8506251af..dc7fe0c82 100644 --- a/docs/how-to-guides/deployment/requirements.txt +++ b/docs/how-to-guides/deployment/requirements.txt @@ -1,4 +1,4 @@ pip -git+https://github.com/linkedin/feathr.git#subdirectory=feathr_project +git+https://github.com/feathr-ai/feathr.git#subdirectory=feathr_project pandavro aiohttp \ No newline at end of file diff --git a/docs/how-to-guides/feathr-advanced-topic.md b/docs/how-to-guides/feathr-advanced-topic.md new file mode 100644 index 000000000..d3b85bb77 --- /dev/null +++ b/docs/how-to-guides/feathr-advanced-topic.md @@ -0,0 +1,31 @@ +--- +layout: default +title: Advanced Usages for Feathr +parent: How-to Guides +--- + +# Advanced Usage on Feathr + +This document describes various advanced usages on Feathr + +# Adding Additional Users to your Feathr environment + +They are all optional steps are are for reference only. Some of the steps are optional if you are not using those services (such as Synapse) + +1. Update the key vault permission as well as the Synapse cluster permission: + +```bash +userId= +resource_prefix= +synapse_workspace_name="${resource_prefix}syws" +keyvault_name="${resource_prefix}kv" +objectId=$(az ad user show --id $userId --query id -o tsv) +az keyvault update --name $keyvault_name --enable-rbac-authorization false +az keyvault set-policy -n $keyvault_name --secret-permissions get list --object-id $objectId +az role assignment create --assignee $userId --role "Storage Blob Data Contributor" +az synapse role assignment create --workspace-name $synapse_workspace_name --role "Synapse Contributor" --assignee $userId +``` + +2. Grant users access control in the Feathr UI by going to the "management" page, as below shows: + +![Feathr Registry Update](../images/feathr-add-users.jpg) \ No newline at end of file diff --git a/docs/how-to-guides/feathr-configuration-and-env.md b/docs/how-to-guides/feathr-configuration-and-env.md index b8745b332..dfdc29707 100644 --- a/docs/how-to-guides/feathr-configuration-and-env.md +++ b/docs/how-to-guides/feathr-configuration-and-env.md @@ -84,60 +84,69 @@ feathr_client = FeathrClient(..., secret_manager_client = cache) # A list of environment variables that Feathr uses -| Environment Variable | Description | Required? | -| ----------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------- | -| SECRETS__AZURE_KEY_VAULT__NAME | Name of the Azure Key Vault service so that Feathr can get credentials from that service. | Optional | -| AZURE_CLIENT_ID | Client ID for authentication into Azure Services. Read [here](https://docs.microsoft.com/en-us/python/api/azure-identity/azure.identity.environmentcredential?view=azure-python) for more details. | This is required if you are using Service Principal to login with Feathr. | -| AZURE_TENANT_ID | Client ID for authentication into Azure Services. Read [here](https://docs.microsoft.com/en-us/python/api/azure-identity/azure.identity.environmentcredential?view=azure-python) for more details. | This is required if you are using Service Principal to login with Feathr. | -| AZURE_CLIENT_SECRET | Client ID for authentication into Azure Services. Read [here](https://docs.microsoft.com/en-us/python/api/azure-identity/azure.identity.environmentcredential?view=azure-python) for more details. | This is required if you are using Service Principal to login with Feathr. | -| OFFLINE_STORE__ADLS__ADLS_ENABLED | Whether to enable ADLS as offline store or not. | Optional | -| ADLS_ACCOUNT | ADLS account that you connect to. | Required if using ADLS as an offline store. | -| ADLS_KEY | ADLS key that you connect to. | Required if using ADLS as an offline store. | -| OFFLINE_STORE__WASB__WASB_ENABLED | Whether to enable Azure BLOB storage as offline store or not. | -| WASB_ACCOUNT | Azure BLOB Storage account that you connect to. | Required if using Azure BLOB Storage as an offline store. | -| WASB_KEY | Azure BLOB Storage key that you connect to. | Required if using Azure BLOB Storage as an offline store. | -| S3_ACCESS_KEY | AWS S3 access key for the S3 account. | Required if using AWS S3 Storage as an offline store. | -| S3_SECRET_KEY | AWS S3 secret key for the S3 account. | Required if using AWS S3 Storage as an offline store. | -| OFFLINE_STORE__S3__S3_ENABLED | Whether to enable S3 as offline store or not. | Optional | -| OFFLINE_STORE__S3__S3_ENDPOINT | S3 endpoint. If you use S3 endpoint, then you need to provide access key and secret key in the environment variable as well. | Required if using AWS S3 Storage as an offline store. | -| OFFLINE_STORE__JDBC__JDBC_ENABLED | Whether to enable JDBC as offline store or not. | Optional | -| OFFLINE_STORE__JDBC__JDBC_DATABASE | If using JDBC endpoint as offline store, this config specifies the JDBC database to read from. | Required if using JDBC sources as offline store | -| OFFLINE_STORE__JDBC__JDBC_TABLE | If using JDBC endpoint as offline store, this config specifies the JDBC table to read from. Same as `JDBC_TABLE`. | Required if using JDBC sources as offline store | -| JDBC_TABLE | If using JDBC endpoint as offline store, this config specifies the JDBC table to read from | Required if using JDBC sources as offline store | -| JDBC_USER | If using JDBC endpoint as offline store, this config specifies the JDBC user | Required if using JDBC sources as offline store | -| JDBC_PASSWORD | If using JDBC endpoint as offline store, this config specifies the JDBC password | Required if using JDBC sources as offline store | -| KAFKA_SASL_JAAS_CONFIG | see [here](#KAFKA_SASL_JAAS_CONFIG) for more details. | Required if using Kafka/EventHub as streaming source input. | -| PROJECT_CONFIG__PROJECT_NAME | Configures the project name. | Required | -| OFFLINE_STORE__SNOWFLAKE__URL | Configures the Snowflake URL. Usually it's something like `dqllago-ol19457.snowflakecomputing.com`. | Required if using Snowflake as an offline store. | -| OFFLINE_STORE__SNOWFLAKE__USER | Configures the Snowflake user. | Required if using Snowflake as an offline store. | -| OFFLINE_STORE__SNOWFLAKE__ROLE | Configures the Snowflake role. Usually it's something like `ACCOUNTADMIN`. | Required if using Snowflake as an offline store. | -| JDBC_SF_PASSWORD | Configurations for Snowflake password | Required if using Snowflake as an offline store. | -| SPARK_CONFIG__SPARK_CLUSTER | Choice for spark runtime. Currently support: `azure_synapse`, `databricks`. The `databricks` configs will be ignored if `azure_synapse` is set and vice versa. | Required | -| SPARK_CONFIG__SPARK_RESULT_OUTPUT_PARTS | Configure number of parts for the spark output for feature generation job | Required | -| SPARK_CONFIG__AZURE_SYNAPSE__DEV_URL | Dev URL to the synapse cluster. Usually it's something like `https://yourclustername.dev.azuresynapse.net` | Required if using Azure Synapse | -| SPARK_CONFIG__AZURE_SYNAPSE__POOL_NAME | name of the spark pool that you are going to use | Required if using Azure Synapse | -| SPARK_CONFIG__AZURE_SYNAPSE__WORKSPACE_DIR | A location that Synapse has access to. This workspace dir stores all the required configuration files and the jar resources. All the feature definitions will be uploaded here. Suggest to use an empty dir for a new spark job to avoid conflicts. | Required if using Azure Synapse | -| SPARK_CONFIG__AZURE_SYNAPSE__EXECUTOR_SIZE | Specifies the executor size for the Azure Synapse cluster. Currently the options are `Small`, `Medium`, `Large`. | Required if using Azure Synapse | -| SPARK_CONFIG__AZURE_SYNAPSE__EXECUTOR_NUM | Specifies the number of executors for the Azure Synapse cluster | Required if using Azure Synapse | -| SPARK_CONFIG__AZURE_SYNAPSE__FEATHR_RUNTIME_LOCATION | Specifies the Feathr runtime location. Support local paths, path start with `http(s)://`, and paths start with `abfss:/`. If not set, will use the [Feathr package published in Maven](https://search.maven.org/artifact/com.linkedin.feathr/feathr_2.12). | Required if using Azure Synapse | -| SPARK_CONFIG__DATABRICKS__WORKSPACE_INSTANCE_URL | Workspace instance URL for your databricks cluster. Will be something like this: `https://adb-6885802458123232.12.azuredatabricks.net/` | Required if using Databricks | -| SPARK_CONFIG__DATABRICKS__CONFIG_TEMPLATE | Config string including run time information, spark version, machine size, etc. See [below](#SPARK_CONFIG__DATABRICKS__CONFIG_TEMPLATE) for more details. | Required if using Databricks | -| SPARK_CONFIG__DATABRICKS__WORK_DIR | Workspace dir for storing all the required configuration files and the jar resources. All the feature definitions will be uploaded here. | Required if using Databricks | -| SPARK_CONFIG__DATABRICKS__FEATHR_RUNTIME_LOCATION | Feathr runtime location. Support local paths, path start with `http(s)://`, and paths start with `dbfs:/`. If not set, will use the [Feathr package published in Maven](https://search.maven.org/artifact/com.linkedin.feathr/feathr_2.12). | Required if using Databricks | -| ONLINE_STORE__REDIS__HOST | Redis host name to access Redis cluster. | Required if using Redis as online store. | -| ONLINE_STORE__REDIS__PORT | Redis port number to access Redis cluster. | Required if using Redis as online store. | -| ONLINE_STORE__REDIS__SSL_ENABLED | Whether SSL is enabled to access Redis cluster. | Required if using Redis as online store. | -| REDIS_PASSWORD | Password for the Redis cluster. | Required if using Redis as online store. | -| FEATURE_REGISTRY__API_ENDPOINT | Specifies registry endpoint. | Required if using registry service. | -| FEATURE_REGISTRY__PURVIEW__PURVIEW_NAME (Deprecated Soon) | Configure the name of the purview endpoint. | Required if using Purview directly without registry service. Deprecate soon, see [here](#deprecation) for more details.| -| FEATURE_REGISTRY__PURVIEW__DELIMITER (Deprecated Soon) | See [here](#FEATURE_REGISTRY__PURVIEW__DELIMITER) for more details. | Required if using Purview directly without registry service. Deprecate soon, see [here](#deprecation) for more details.| -| FEATURE_REGISTRY__PURVIEW__TYPE_SYSTEM_INITIALIZATION (Deprecated Soon)| Controls whether the type system (think this as the "schema" for the registry) will be initialized or not. Usually this is only required to be set to `True` to initialize schema, and then you can set it to `False` to shorten the initialization time. | Required if using Purview directly without registry service. Deprecate soon, see [here](#deprecation) for more details.| +| Environment Variable | Description | Required? | +| ----------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------- | +| SECRETS__AZURE_KEY_VAULT__NAME | Name of the Azure Key Vault service so that Feathr can get credentials from that service. | Optional | +| AZURE_CLIENT_ID | Client ID for authentication into Azure Services. Read [here](https://docs.microsoft.com/en-us/python/api/azure-identity/azure.identity.environmentcredential?view=azure-python) for more details. | This is required if you are using Service Principal to login with Feathr. | +| AZURE_TENANT_ID | Tenant ID for authentication into Azure Services. Read [here](https://docs.microsoft.com/en-us/python/api/azure-identity/azure.identity.environmentcredential?view=azure-python) for more details. | This is required if you are using Service Principal to login with Feathr. | +| AZURE_CLIENT_SECRET | Client secret for authentication into Azure Services. Read [here](https://docs.microsoft.com/en-us/python/api/azure-identity/azure.identity.environmentcredential?view=azure-python) for more details. | This is required if you are using Service Principal to login with Feathr. | +| OFFLINE_STORE__ADLS__ADLS_ENABLED | Whether to enable ADLS as offline store or not. Available value: "True" or "False". Equivalent to "False" if not set. | Optional | +| ADLS_ACCOUNT | ADLS account that you connect to. | Required if using ADLS as an offline store. | +| ADLS_KEY | ADLS key that you connect to. | Required if using ADLS as an offline store. | +| OFFLINE_STORE__WASB__WASB_ENABLED | Whether to enable Azure BLOB storage as offline store or not. Available value: "True" or "False". Equivalent to "False" if not set. | +| BLOB_ACCOUNT | Azure BLOB Storage account that you connect to. | Required if using Azure BLOB Storage as an offline store. | +| BLOB_KEY | Azure BLOB Storage key that you connect to. | Required if using Azure BLOB Storage as an offline store. | +| S3_ACCESS_KEY | AWS S3 access key for the S3 account. | Required if using AWS S3 Storage as an offline store. | +| S3_SECRET_KEY | AWS S3 secret key for the S3 account. | Required if using AWS S3 Storage as an offline store. | +| OFFLINE_STORE__S3__S3_ENABLED | Whether to enable S3 as offline store or not. Available value: "True" or "False". Equivalent to "False" if not set. | Optional | +| OFFLINE_STORE__S3__S3_ENDPOINT | S3 endpoint. If you use S3 endpoint, then you need to provide access key and secret key in the environment variable as well. | Required if using AWS S3 Storage as an offline store. | +| OFFLINE_STORE__JDBC__JDBC_ENABLED | Whether to enable JDBC as offline store or not. Available value: "True" or "False". Equivalent to "False" if not set. | Optional | +| OFFLINE_STORE__JDBC__JDBC_DATABASE | If using JDBC endpoint as offline store, this config specifies the JDBC database to read from. | Required if using JDBC sources as offline store | +| OFFLINE_STORE__JDBC__JDBC_TABLE | If using JDBC endpoint as offline store, this config specifies the JDBC table to read from. Same as `JDBC_TABLE`. | Required if using JDBC sources as offline store | +| JDBC_TABLE | If using JDBC endpoint as offline store, this config specifies the JDBC table to read from | Required if using JDBC sources as offline store | +| JDBC_USER | If using JDBC endpoint as offline store, this config specifies the JDBC user | Required if using JDBC sources as offline store | +| JDBC_PASSWORD | If using JDBC endpoint as offline store, this config specifies the JDBC password | Required if using JDBC sources as offline store | +| KAFKA_SASL_JAAS_CONFIG | see [here](#KAFKA_SASL_JAAS_CONFIG) for more details. | Required if using Kafka/EventHub as streaming source input. | +| PROJECT_CONFIG__PROJECT_NAME | Configures the project name. | Required | +| OFFLINE_STORE__SNOWFLAKE__SNOWFLAKE_ENABLED | Configures whether Snowflake as offline store is enabled or not. Available value: "True" or "False". Equivalent to "False" if not set. | Required if using Snowflake as an offline store. | +| OFFLINE_STORE__SNOWFLAKE__URL | Configures the Snowflake URL. Usually it's something like `dqllago-ol19457.snowflakecomputing.com`. | Required if using Snowflake as an offline store. | +| OFFLINE_STORE__SNOWFLAKE__USER | Configures the Snowflake user. | Required if using Snowflake as an offline store. | +| OFFLINE_STORE__SNOWFLAKE__ROLE | Configures the Snowflake role. Usually it's something like `ACCOUNTADMIN`. | Required if using Snowflake as an offline store. +| OFFLINE_STORE__SNOWFLAKE__WAREHOUSE | Configures the Snowflake Warehouse. | Required if using Snowflake as an offline store. | +| JDBC_SF_PASSWORD | Configurations for Snowflake password | Required if using Snowflake as an offline store. | +| SPARK_CONFIG__SPARK_CLUSTER | Choice for spark runtime. Currently support: `azure_synapse`, `databricks`. The `databricks` configs will be ignored if `azure_synapse` is set and vice versa. | Required | +| SPARK_CONFIG__SPARK_RESULT_OUTPUT_PARTS | Configure number of parts for the spark output for feature generation job | Required | +| SPARK_CONFIG__AZURE_SYNAPSE__DEV_URL | Dev URL to the synapse cluster. Usually it's something like `https://yourclustername.dev.azuresynapse.net` | Required if using Azure Synapse | +| SPARK_CONFIG__AZURE_SYNAPSE__POOL_NAME | name of the spark pool that you are going to use | Required if using Azure Synapse | +| SPARK_CONFIG__AZURE_SYNAPSE__WORKSPACE_DIR | A location that Synapse has access to. This workspace dir stores all the required configuration files and the jar resources. All the feature definitions will be uploaded here. Suggest to use an empty dir for a new spark job to avoid conflicts. | Required if using Azure Synapse | +| SPARK_CONFIG__AZURE_SYNAPSE__EXECUTOR_SIZE | Specifies the executor size for the Azure Synapse cluster. Currently the options are `Small`, `Medium`, `Large`. | Required if using Azure Synapse | +| SPARK_CONFIG__AZURE_SYNAPSE__EXECUTOR_NUM | Specifies the number of executors for the Azure Synapse cluster | Required if using Azure Synapse | +| SPARK_CONFIG__AZURE_SYNAPSE__FEATHR_RUNTIME_LOCATION | Specifies the Feathr runtime location. Support local paths, path start with `http(s)://`, and paths start with `abfss:/`. If not set, will use the [Feathr package published in Maven](https://search.maven.org/artifact/com.linkedin.feathr/feathr_2.12). | Required if using Azure Synapse | +| SPARK_CONFIG__DATABRICKS__WORKSPACE_INSTANCE_URL | Workspace instance URL for your databricks cluster. Will be something like this: `https://adb-6885802458123232.12.azuredatabricks.net/` | Required if using Databricks | +| SPARK_CONFIG__DATABRICKS__CONFIG_TEMPLATE | Config string including run time information, spark version, machine size, etc. See [below](#SPARK_CONFIG__DATABRICKS__CONFIG_TEMPLATE) for more details. | Required if using Databricks | +| SPARK_CONFIG__DATABRICKS__WORK_DIR | Workspace dir for storing all the required configuration files and the jar resources. All the feature definitions will be uploaded here. | Required if using Databricks | +| SPARK_CONFIG__DATABRICKS__FEATHR_RUNTIME_LOCATION | Feathr runtime location. Support local paths, path start with `http(s)://`, and paths start with `dbfs:/`. If not set, will use the [Feathr package published in Maven](https://search.maven.org/artifact/com.linkedin.feathr/feathr_2.12). | Required if using Databricks | +| DATABRICKS_WORKSPACE_TOKEN_VALUE | Token value to access databricks workspace. More details can be found at [Authentication using Databricks personal access tokens](https://docs.databricks.com/dev-tools/api/latest/authentication.html) | Required if using Databricks | +| ONLINE_STORE__REDIS__HOST | Redis host name to access Redis cluster. | Required if using Redis as online store. | +| ONLINE_STORE__REDIS__PORT | Redis port number to access Redis cluster. | Required if using Redis as online store. | +| ONLINE_STORE__REDIS__SSL_ENABLED | Whether SSL is enabled to access Redis cluster. | Required if using Redis as online store. | +| REDIS_PASSWORD | Password for the Redis cluster. | Required if using Redis as online store. | +| FEATURE_REGISTRY__API_ENDPOINT | Specifies registry endpoint. | Required if using registry service. | +| FEATURE_REGISTRY__PURVIEW__PURVIEW_NAME (Deprecated Soon) | Configure the name of the purview endpoint. | Required if using Purview directly without registry service. Deprecate soon, see [here](#deprecation) for more details. | +| FEATURE_REGISTRY__PURVIEW__DELIMITER (Deprecated Soon) | See [here](#FEATURE_REGISTRY__PURVIEW__DELIMITER) for more details. | Required if using Purview directly without registry service. Deprecate soon, see [here](#deprecation) for more details. | +| FEATURE_REGISTRY__PURVIEW__TYPE_SYSTEM_INITIALIZATION (Deprecated Soon) | Controls whether the type system (think this as the "schema" for the registry) will be initialized or not. Usually this is only required to be set to `True` to initialize schema, and then you can set it to `False` to shorten the initialization time. | Required if using Purview directly without registry service. Deprecate soon, see [here](#deprecation) for more details. | +| MAVEN_ARTIFACT_VERSION | Version number like `0.9.0`. Used to define maven package version when main jar is not defined. | Optional | # Explanation for selected configurations +## MAVEN_ARTIFACT_VERSION +By default, Feathr client will use the same version runtime jar from Maven for Spark job submission. If you want to use a different version jar from Maven, you need to explicitly set system env MAVEN_ARTIFACT_VERSION. + +For example, if you want to use Feathr 0.9.0, you can set `os.environ["MAVEN_ARTIFACT_VERSION"] = "0.9.0"`. + ## KAFKA_SASL_JAAS_CONFIG -Feathr uses Kafka behind the scene for streaming input, and Kafka uses the Java Authentication and Authorization Service (JAAS) for SASL ([Simple Authentication and Security Layer](https://en.wikipedia.org/wiki/Simple_Authentication_and_Security_Layer)) configuration. You must provide JAAS configurations for all SASL authentication. +Feathr uses Kafka behind the scene for streaming input, and Kafka uses the Java Authentication and Authorization Service (JAAS) for SASL ([Simple Authentication and Security Layer](https://en.wikipedia.org/wiki/Simple_Authentication_and_Security_Layer)) configuration. You must provide JAAS configurations for all SASL authentication. For cloud services such as Azure EventHub or AWS Managed Streaming for Apache Kafka (MSK), they usually use `ConnectionString` as user name, and the password will be the exact content of the connection string. Feathr will automatically fill that part in so you don't have to worry about it. @@ -145,7 +154,7 @@ In order to get the exact value of the `password` part (i.e. connection string), ![EventHub Config](../images/eventhub_config.png) -For Azure EventHub, read [here](https://github.com/Azure/azure-event-hubs-for-kafka#updating-your-kafka-client-configuration) for how to get this string from the existing string in Azure Portal. The value will be something like: `Endpoint=sb://feathrazureci.servicebus.windows.net/;SharedAccessKeyName=feathrcipolicy;SharedAccessKey=aaaaaaaa=;EntityPath=feathrcieventhub`, and note that you don't need the `EntityPath=feathrcieventhub` part, as this represents the Kafka topic, which you will specify in the code in other places. +For Azure EventHub, read [here](https://github.com/Azure/azure-event-hubs-for-kafka#updating-your-kafka-client-configuration) for how to get this string from the existing string in Azure Portal. The value will be something like: `Endpoint=sb://feathrazureci.servicebus.windows.net/;SharedAccessKeyName=feathrcipolicy;SharedAccessKey=aaaaaaaa=;EntityPath=feathrcieventhub`, and note that you don't need the `EntityPath=feathrcieventhub` part, as this represents the Kafka topic, which you will specify in the code in other places. So finally the configuration in Python will be something like: diff --git a/docs/how-to-guides/feathr-credential-passthru.md b/docs/how-to-guides/feathr-credential-passthru.md new file mode 100644 index 000000000..8473b01c8 --- /dev/null +++ b/docs/how-to-guides/feathr-credential-passthru.md @@ -0,0 +1,46 @@ +--- +layout: default +title: Passing Through Credentials in Feathr +parent: How-to Guides +--- + +# Passing Through Credentials in Feathr + +Sometimes, instead of using key-based credential to access the underlying storage (such as Azure Data Lake Storage), it makes more sense to use a user/service principal to access it, usually for security reasons. + +Feathr has native support for this use case. For example, if you are currently using Databricks and want to access Azure Data Lake Storage using a certain user/principal credential, here are the steps: + +1. Setup an Azure Data Lake Storage account and the corresponding Service Principals. More instructions can be found in this [Tutorial: Azure Data Lake Storage Gen2, Azure Databricks & Spark](https://learn.microsoft.com/en-us/azure/storage/blobs/data-lake-storage-use-databricks-spark). + + + +2. After the first step, you should have an Azure Data Lake Storage account and a Service Principal. The second step is to pass those credentials to Feathr's spark settings, like below: +```python +execution_configs = {"fs.azure.account.auth.type": "OAuth", + "fs.azure.account.oauth.provider.type": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "fs.azure.account.oauth2.client.id": "", + "fs.azure.account.oauth2.client.secret": "", + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com//oauth2/token", + "fs.azure.createRemoteFileSystemDuringInitialization": "true"} + +# if running `get_offline_features` job +client.get_offline_features(observation_settings=settings, + feature_query=feature_query, + execution_configurations=execution_configs, + output_path=output_path) +# if running feature materialization job +client.materialize_features(settings, allow_materialize_non_agg_feature=True, execution_configurations=execution_configs) +``` + +In this code block, replace the `appId`, `clientSecret`, and `tenant` placeholder values in this code block with the values that you collected while completing the first step. + +3. Don't forget your other configuration settings, such as the ones that are specific to Feathr in [Feathr Job Configuration during Run Time](./feathr-job-configuration.md). + +4. Azure SQL Database Credential pass through is also supported. To achieve so you need to pass your token to environment variables and set `auth` parameter to `TOKEN` in `JdbcSource` or `JdbcSink`. For example: +```python +output_name = 'output' +sink = client.JdbcSink(name=output_name, url="some_jdbc_url", dbtable="table_name", auth="TOKEN") + +os.environ[f"{output_name.upper()}_TOKEN"] = self.credential.get_token("https://management.azure.com/.default").token +client.get_offline_features(..., output_path=sink) +``` diff --git a/docs/how-to-guides/feathr-input-format.md b/docs/how-to-guides/feathr-input-format.md index 3266942a3..3ef7b4eb6 100644 --- a/docs/how-to-guides/feathr-input-format.md +++ b/docs/how-to-guides/feathr-input-format.md @@ -1,6 +1,6 @@ --- layout: default -title: Input File Format for Feathr +title: Input File for Feathr parent: How-to Guides --- @@ -18,3 +18,10 @@ Many Spark users will use delta lake format to store the results. In those cases ![Spark Output](../images/spark-output.png) Please note that although the results are shown as "parquet", you should use the path of the parent folder and use `delta` format to read the folder. + +# TimePartitionPattern for input files +When data sources are defined by 'HdfsSource', feathr supports 'time_partition_pattern' to match paths of input files. For example, given time_partition_pattern = 'yyyy/MM/dd' and a 'base_path', all available input files under paths 'base_path'/yyyy/MM/dd will be visited and used as data sources. + +More reference on the APIs: + +- [MaterializationSettings API doc](https://feathr.readthedocs.io/en/latest/feathr.html#feathr.MaterializationSettings) \ No newline at end of file diff --git a/docs/how-to-guides/feathr-registry-client-update.md b/docs/how-to-guides/feathr-registry-client-update.md new file mode 100644 index 000000000..f7094e536 --- /dev/null +++ b/docs/how-to-guides/feathr-registry-client-update.md @@ -0,0 +1,25 @@ +--- +layout: default +title: Updating Feathr Registry and Feathr Client +parent: How-to Guides +--- + +# Updating Feathr Registry and Feathr Client + +Feathr has monthly releases, and usually the release contains 3 major components: + +- Feathr python client, where you can install via `pip install feathr` +- Feathr spark runtime, which is published to [Maven Central](https://search.maven.org/artifact/com.linkedin.feathr/feathr_2.12) +- Feathr Registry Server, which is a docker container that is published in [DockerHub with name feathrfeaturestore/feathr-registry](https://hub.docker.com/r/feathrfeaturestore/feathr-registry/tags) + + +When updating Feathr, there are two steps: +1. Update the Feathr client into a specific version. You can do this by executing `pip install feathr==0.9` to a specific version, or `pip install feathr -U` to update to the latest version. Usually when end users update the Python client, the associated Spark runtime will also be updated, so end users usually don't have to update the Spark runtime unless there are specific reasons. +2. Update the Feature Registry Server. You should go to the webapp that is hosting the UI, and find the "Deployment Center" part, and update the `Full Image Name and Tag` to the DockerHub image that you want to use, for example `feathrfeaturestore/feathr-registry:releases-v0.9.0`. Note that the "Continuous Deployment" setting needs to be set to "on", as below. + + +![Feathr Registry Update](../images/feathr-update.jpg) + + + + \ No newline at end of file diff --git a/docs/how-to-guides/feathr-snowflake-guide.md b/docs/how-to-guides/feathr-snowflake-guide.md new file mode 100644 index 000000000..b3baa9ce5 --- /dev/null +++ b/docs/how-to-guides/feathr-snowflake-guide.md @@ -0,0 +1,38 @@ +--- +layout: default +title: Using Snowflake with Feathr +parent: Feathr How-to Guides +--- + +# Using Snowflake with Feathr + +Currently, feathr supports using Snowflake as a source. + +# Using Snowflake as a source + +To use Snowflake as a source, we need to create a `SnowflakeSource` in projects. + +``` +source = feathr.SnowflakeSource(name: str, database: str, schema: str, dbtable: optional[str], query: Optional[str]) +``` + +* `name` is the source name, same as other sources. +* `database` is SF database that stores the table of interest +* `schema` is SF schema that stores the table of interest +* `dbtable` or `query`, `dbtable` is the table name in the database and `query` is a SQL `SELECT` statement, only one of them should be specified at the same time. + +For more information on how Snowflake uses Databases and Schemas to organize data, please refer to [Snowflake Datatabase and Schema](https://docs.snowflake.com/en/sql-reference/ddl-database.html) + +There are some other parameters such as `preprocessing`, they're same as other sources like `HdfsSource`. + +After creating the `SnowflakeSource`, you can use it in the same way as other kinds of sources. + +# Specifying Snowflake Source in Observation Settings + +`ObservationSettings` requires an observation path. In order to generate the snowflake path, feathr exposes client functionality that exposes the same arguments as SnowflakeSource. + +To generate snowflake path to pass into `ObservationSettings`, we need to call `client.get_snowflake_path()` functionality. + +``` +observation_path = client.get_snowflake_path(database: str, schema: str, dbtable: Optional[str], query: Optional[str]) +``` \ No newline at end of file diff --git a/docs/how-to-guides/jdbc-cosmos-notes.md b/docs/how-to-guides/jdbc-cosmos-notes.md index 49d5c74d1..52fb493e8 100644 --- a/docs/how-to-guides/jdbc-cosmos-notes.md +++ b/docs/how-to-guides/jdbc-cosmos-notes.md @@ -62,6 +62,32 @@ client.get_offline_features(...) These values will be automatically passed to the Feathr core when submitting the job. +If you want to use token, the code will be like this: +Step 1: Define the source JdbcSource +```python +src_name="source_name" +source = JdbcSource(name=src_name, url="jdbc:...", dbtable="table_name", auth="TOKEN") +anchor = FeatureAnchor(name="anchor_name", + source=source, + features=[some_features, some_other_features]) +``` +Step 2: Set the environment variable before submitting the job +```python +os.environ[f"{src_name.upper()}_TOKEN"] = "some_token" +``` +To enable Azure AD authentication in Azure SQL database, please refer to [this document](https://learn.microsoft.com/en-us/azure/azure-sql/database/authentication-aad-overview?view=azuresql#overview). + +There are several ways to obtain Azure AD access token, please refer to [this document](https://docs.microsoft.com/en-us/azure/active-directory/develop/access-tokens) for more details. + +If you want to leverage existing credential in python client, you could try: +```python +from azure.identity import DefaultAzureCredential + +credential = DefaultAzureCredential() +token = credential.get_token("https://management.azure.com/.default").token() +``` + + ## Using SQL database as the offline store To use SQL database as the offline store, you can use `JdbcSink` as the `output_path` parameter of `FeathrClient.get_offline_features`, e.g.: @@ -76,6 +102,7 @@ os.environ[f"{name.upper()}_USER"] = "some_user_name" os.environ[f"{name.upper()}_PASSWORD"] = "some_magic_word" client.get_offline_features(..., output_path=sink) ``` +"TOKEN" auth type is also supported in `JdbcSink`. ## Using SQL database as the online store diff --git a/docs/how-to-guides/local-spark-provider.md b/docs/how-to-guides/local-spark-provider.md index 433af64f3..b5a3b25b0 100644 --- a/docs/how-to-guides/local-spark-provider.md +++ b/docs/how-to-guides/local-spark-provider.md @@ -16,6 +16,7 @@ The local spark provider only requires users to have a [local spark environment] ### Environment Setup Please make sure that `Spark` and `feathr` are installed and the `SPARK_LOCAL_IP` is set. +`JAVA_HOME` and Java environment is also required. ### Local Feathr Config To use local spark environment, user need to set `spark_cluster: 'local'`. If `feathr_runtime_location` is not set, Feathr will use default Maven package instead. @@ -36,7 +37,7 @@ A spark-submit script will auto generated in your workspace under `debug` folder spark-submit \ --master local[*] \ --name project_feathr_local_spark_test \ - --packages "org.apache.spark:spark-avro_2.12:3.3.0,com.microsoft.sqlserver:mssql-jdbc:10.2.0.jre8,com.microsoft.azure:spark-mssql-connector_2.12:1.2.0,org.apache.logging.log4j:log4j-core:2.17.2,com.typesafe:config:1.3.4,com.fasterxml.jackson.core:jackson-databind:2.12.6.1,org.apache.hadoop:hadoop-mapreduce-client-core:2.7.7,org.apache.hadoop:hadoop-common:2.7.7,org.apache.avro:avro:1.8.2,org.apache.xbean:xbean-asm6-shaded:4.10,org.apache.spark:spark-sql-kafka-0-10_2.12:3.1.3,com.microsoft.azure:azure-eventhubs-spark_2.12:2.3.21,org.apache.kafka:kafka-clients:3.1.0,com.google.guava:guava:31.1-jre,it.unimi.dsi:fastutil:8.1.1,org.mvel:mvel2:2.2.8.Final,com.fasterxml.jackson.module:jackson-module-scala_2.12:2.13.3,com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.6,com.fasterxml.jackson.dataformat:jackson-dataformat-csv:2.12.6,com.jasonclawson:jackson-dataformat-hocon:1.1.0,com.redislabs:spark-redis_2.12:3.1.0,org.apache.xbean:xbean-asm6-shaded:4.10,com.google.protobuf:protobuf-java:3.19.4,net.snowflake:snowflake-jdbc:3.13.18,net.snowflake:spark-snowflake_2.12:2.10.0-spark_3.2,org.apache.commons:commons-lang3:3.12.0,org.xerial:sqlite-jdbc:3.36.0.3,com.github.changvvb:jackson-module-caseclass_2.12:1.1.1,com.azure.cosmos.spark:azure-cosmos-spark_3-1_2-12:4.11.1,org.eclipse.jetty:jetty-util:9.3.24.v20180605,commons-io:commons-io:2.6,org.apache.hadoop:hadoop-azure:2.7.4,com.microsoft.azure:azure-storage:8.6.4,com.linkedin.feathr:feathr_2.12:0.8.0" \ + --packages "org.apache.spark:spark-avro_2.12:3.3.0,com.microsoft.sqlserver:mssql-jdbc:10.2.0.jre8,com.microsoft.azure:spark-mssql-connector_2.12:1.2.0,org.apache.logging.log4j:log4j-core:2.17.2,com.typesafe:config:1.3.4,com.fasterxml.jackson.core:jackson-databind:2.12.6.1,org.apache.hadoop:hadoop-mapreduce-client-core:2.7.7,org.apache.hadoop:hadoop-common:2.7.7,org.apache.avro:avro:1.8.2,org.apache.xbean:xbean-asm6-shaded:4.10,org.apache.spark:spark-sql-kafka-0-10_2.12:3.1.3,com.microsoft.azure:azure-eventhubs-spark_2.12:2.3.21,org.apache.kafka:kafka-clients:3.1.0,com.google.guava:guava:31.1-jre,it.unimi.dsi:fastutil:8.1.1,org.mvel:mvel2:2.2.8.Final,com.fasterxml.jackson.module:jackson-module-scala_2.12:2.13.3,com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.6,com.fasterxml.jackson.dataformat:jackson-dataformat-csv:2.12.6,com.jasonclawson:jackson-dataformat-hocon:1.1.0,com.redislabs:spark-redis_2.12:3.1.0,org.apache.xbean:xbean-asm6-shaded:4.10,com.google.protobuf:protobuf-java:3.19.4,net.snowflake:snowflake-jdbc:3.13.18,net.snowflake:spark-snowflake_2.12:2.10.0-spark_3.2,org.apache.commons:commons-lang3:3.12.0,org.xerial:sqlite-jdbc:3.36.0.3,com.github.changvvb:jackson-module-caseclass_2.12:1.1.1,com.azure.cosmos.spark:azure-cosmos-spark_3-1_2-12:4.11.1,org.eclipse.jetty:jetty-util:9.3.24.v20180605,commons-io:commons-io:2.6,org.apache.hadoop:hadoop-azure:2.7.4,com.microsoft.azure:azure-storage:8.6.4,com.linkedin.feathr:feathr_2.12:0.9.0" \ --conf "spark.driver.extraClassPath=../target/scala-2.12/classes:jars/config-1.3.4.jar:jars/jackson-dataformat-hocon-1.1.0.jar:jars/jackson-module-caseclass_2.12-1.1.1.jar:jars/mvel2-2.2.8.Final.jar:jars/fastutil-8.1.1.jar" \ --conf "spark.hadoop.fs.wasbs.impl=org.apache.hadoop.fs.azure.NativeAzureFileSystem" \ --class com.linkedin.feathr.offline.job.FeatureJoinJob \ @@ -72,8 +73,9 @@ In this version of local spark provider, users are only able to test `get_offlin `local-spark-provider` enable users to test features without deploying any cloud resources. However, please use it ONLY in test or trial scenarios. For production usage, cloud spark providers are highly recommended. ### Tips: -If you want to submit more customized params to Spark, a workaround is to generate a sample script and then update it with your own params. - +- If you want to submit more customized params to Spark, a workaround is to generate a sample script and then update it with your own params. +- Cold start will be slow since it needs to download quite a few Maven packages to local environment. But after that it should be very fast to use it locally +- Windows is currently not supported. Linux/MacOS is fully tested. If you are on Windows machine, consider using WSL. ### Use Cases: Following use cases are covered in CI test: - `get_offline_features()` without UDFs @@ -85,3 +87,4 @@ Following use cases are covered in CI test: - `materialize_features()` into online store with local spark environment. - advanced `udf` support - more data sources + diff --git a/docs/images/feathr-add-users.jpg b/docs/images/feathr-add-users.jpg new file mode 100644 index 000000000..a622fae1c Binary files /dev/null and b/docs/images/feathr-add-users.jpg differ diff --git a/docs/images/feathr-update.jpg b/docs/images/feathr-update.jpg new file mode 100644 index 000000000..3b62b3cbf Binary files /dev/null and b/docs/images/feathr-update.jpg differ diff --git a/docs/quickstart_databricks.md b/docs/quickstart_databricks.md index dff5b5f0f..30eaaa835 100644 --- a/docs/quickstart_databricks.md +++ b/docs/quickstart_databricks.md @@ -5,13 +5,13 @@ title: Quick Start Guide with Databricks # Feathr Quick Start Guide with Databricks -For Databricks, you can simply upload [this notebook](./samples/databricks/databricks_quickstart_nyc_taxi_driver.ipynb) to your Databricks cluster and just run it in the Databricks cluster. It has been pre-configured to use the current Databricks cluster to submit jobs. +For Databricks, you can simply upload [this notebook](./samples/databricks/databricks_quickstart_nyc_taxi_demo.ipynb) to your Databricks cluster and just run it in the Databricks cluster. It has been pre-configured to use the current Databricks cluster to submit jobs. 1. Import Notebooks in your Databricks cluster: ![Import Notebooks](./images/databricks_quickstart1.png) -2. Paste the [link to Databricks getting started notebook](./samples/databricks/databricks_quickstart_nyc_taxi_driver.ipynb): +2. Paste the [link to Databricks getting started notebook](./samples/databricks/databricks_quickstart_nyc_taxi_demo.ipynb): ![Import Notebooks](./images/databricks_quickstart2.png) @@ -21,7 +21,7 @@ For Databricks, you can simply upload [this notebook](./samples/databricks/datab Although Databricks Notebooks are great tools, there are also large developer communities that prefer the usage of Visual Studio Code, where [it has native support for Python and Jupyter Notebooks](https://code.visualstudio.com/docs/datascience/jupyter-notebooks) with many great features such as syntax highlight and IntelliSense. -In [this notebook](./samples/databricks/databricks_quickstart_nyc_taxi_driver.ipynb), there are a few lines of code like this: +In [this notebook](./samples/databricks/databricks_quickstart_nyc_taxi_demo.ipynb), there are a few lines of code like this: ```python # Get current databricks notebook context diff --git a/docs/quickstart_synapse.md b/docs/quickstart_synapse.md index 0a66a96bb..c310dd789 100644 --- a/docs/quickstart_synapse.md +++ b/docs/quickstart_synapse.md @@ -24,7 +24,7 @@ Feathr has native cloud integration. Here are the steps to use Feathr on Azure: 1. Follow the [Feathr ARM deployment guide](https://feathr-ai.github.io/feathr/how-to-guides/azure-deployment-arm.html) to run Feathr on Azure. This allows you to quickly get started with automated deployment using Azure Resource Manager template. Alternatively, if you want to set up everything manually, you can checkout the [Feathr CLI deployment guide](https://feathr-ai.github.io/feathr/how-to-guides/azure-deployment-cli.html) to run Feathr on Azure. This allows you to understand what is going on and set up one resource at a time. -2. Once the deployment is complete,run the Feathr Jupyter Notebook by clicking this button: [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/feathr-ai/feathr/main?labpath=feathr_project%2Ffeathrcli%2Fdata%2Ffeathr_user_workspace%2Fnyc_driver_demo.ipynb). +2. Once the deployment is complete,run the Feathr Jupyter Notebook by clicking this button: [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/feathr-ai/feathr/main?labpath=docs%2Fsamples%2Fnyc_taxi_demo.ipynb). 3. You only need to change the specified `Resource Prefix`. ## Step 2: Install Feathr @@ -43,7 +43,7 @@ pip install git+https://github.com/feathr-ai/feathr.git#subdirectory=feathr_proj ## Step 3: Run the sample notebook -We've provided a self-contained [sample notebook](./samples/product_recommendation_demo.ipynb) to act as the main content of this getting started guide. This documentation should be used more like highlights and further explanations of that demo notebook. +We've provided a self-contained [sample notebook](https://github.com/feathr-ai/feathr/blob/main/docs/samples/azure_synapse/product_recommendation_demo.ipynb) to act as the main content of this getting started guide. This documentation should be used more like highlights and further explanations of that demo notebook. ## Step 4: Update Feathr config @@ -88,7 +88,7 @@ os.environ['ONLINE_STORE__REDIS__HOST'] = 'feathrazure.redis.cache.windows.net' ## Step 5: Setup environment variables -In the self-contained [sample notebook](./samples/product_recommendation_demo.ipynb), you also have to setup a few environment variables like below in order to access those cloud resources. You should be able to get those values from the first step. +In the self-contained [sample notebook](https://github.com/feathr-ai/feathr/blob/main/docs/samples/azure_synapse/product_recommendation_demo.ipynb), you also have to setup a few environment variables like below in order to access those cloud resources. You should be able to get those values from the first step. These values can also be retrieved by using cloud key value store, such as [Azure Key Vault](https://azure.microsoft.com/en-us/services/key-vault/): @@ -181,7 +181,7 @@ client.multi_get_online_features("nycTaxiDemoFeature", ["239", "265"], ['f_locat ## Next steps -- Run the [demo notebook](./samples/product_recommendation_demo.ipynb) to understand the workflow of Feathr. +- Run the [demo notebook](https://github.com/feathr-ai/feathr/blob/main/docs/samples/azure_synapse/product_recommendation_demo.ipynb) to understand the workflow of Feathr. - Read the [Feathr Documentation Page](https://feathr-ai.github.io/feathr/) page to understand the Feathr abstractions. - Read guide to understand [how to setup Feathr on Azure using Azure Resource Manager template](https://feathr-ai.github.io/feathr/how-to-guides/azure-deployment-arm.html). - Read guide to understand [how to setup Feathr step by step on Azure using Azure CLI](https://feathr-ai.github.io/feathr/how-to-guides/azure-deployment-cli.html). diff --git a/docs/samples/azure_synapse/product_recommendation_demo.ipynb b/docs/samples/azure_synapse/product_recommendation_demo.ipynb index e93860269..dc6eddf88 100644 --- a/docs/samples/azure_synapse/product_recommendation_demo.ipynb +++ b/docs/samples/azure_synapse/product_recommendation_demo.ipynb @@ -485,7 +485,7 @@ " key=user_id,\n", " feature_type=FLOAT,\n", " input_features=[feature_user_gift_card_balance, feature_user_has_valid_credit_card],\n", - " transform=\"feature_user_gift_card_balance + if_else(toBoolean(feature_user_has_valid_credit_card), 100, 0)\")" + " transform=\"feature_user_gift_card_balance + if(boolean(feature_user_has_valid_credit_card), 100, 0)\")" ] }, { @@ -675,7 +675,7 @@ " sinks=[redisSink],\n", " feature_names=[\"feature_user_age\", \"feature_user_gift_card_balance\"])\n", "\n", - "feathr_client.materialize_features(settings)\n", + "feathr_client.materialize_features(settings, allow_materialize_non_agg_feature =True)\n", "feathr_client.wait_job_to_finish(timeout_sec=500)" ] }, diff --git a/docs/samples/customer360/Customer360.ipynb b/docs/samples/customer360/Customer360.ipynb index db042011b..ad9431d66 100644 --- a/docs/samples/customer360/Customer360.ipynb +++ b/docs/samples/customer360/Customer360.ipynb @@ -55,12 +55,12 @@ "\n", "First step is to provision required cloud resources if you want to use Feathr. Feathr provides a python based client to interact with cloud resources.\n", "\n", - "Please follow the steps [here](https://feathr-ai.github.io/feathr/how-to-guides/azure-deployment-arm.html) to provision required cloud resources. Due to the complexity of the possible cloud environment, it is almost impossible to create a script that works for all the use cases. Because of this, [azure_resource_provision.sh](https://github.com/linkedin/feathr/blob/main/docs/how-to-guides/azure_resource_provision.sh) is a full end to end command line to create all the required resources, and you can tailor the script as needed, while [the companion documentation](https://feathr-ai.github.io/feathr/how-to-guides/azure-deployment-cli.html) can be used as a complete guide for using that shell script.\n", + "Please follow the steps [here](https://feathr-ai.github.io/feathr/how-to-guides/azure-deployment-arm.html) to provision required cloud resources. Due to the complexity of the possible cloud environment, it is almost impossible to create a script that works for all the use cases. Because of this, [azure_resource_provision.sh](https://github.com/feathr-ai/feathr/blob/main/docs/how-to-guides/azure_resource_provision.sh) is a full end to end command line to create all the required resources, and you can tailor the script as needed, while [the companion documentation](https://feathr-ai.github.io/feathr/how-to-guides/azure-deployment-cli.html) can be used as a complete guide for using that shell script.\n", "\n", "\n", "And the architecture is as below:\n", "\n", - "![Architecture](https://github.com/linkedin/feathr/blob/main/docs/images/architecture.png?raw=true)" + "![Architecture](https://github.com/feathr-ai/feathr/blob/main/docs/images/architecture.png?raw=true)" ] }, { @@ -150,7 +150,7 @@ }, "outputs": [], "source": [ - "! pip install --force-reinstall git+https://github.com/linkedin/feathr.git@registry_fix#subdirectory=feathr_project pandavro scikit-learn" + "! pip install --force-reinstall git+https://github.com/feathr-ai/feathr.git@registry_fix#subdirectory=feathr_project pandavro scikit-learn" ] }, { @@ -168,7 +168,7 @@ "\n", "In the first step (Provision cloud resources), you should have provisioned all the required cloud resources. If you use Feathr CLI to create a workspace, you should have a folder with a file called `feathr_config.yaml` in it with all the required configurations. Otherwise, update the configuration below.\n", "\n", - "The code below will write this configuration string to a temporary location and load it to Feathr. Please still refer to [feathr_config.yaml](https://github.com/linkedin/feathr/blob/main/feathr_project/feathrcli/data/feathr_user_workspace/feathr_config.yaml) and use that as the source of truth. It should also have more explanations on the meaning of each variable." + "The code below will write this configuration string to a temporary location and load it to Feathr. Please still refer to [feathr_config.yaml](https://github.com/feathr-ai/feathr/blob/main/feathr_project/feathrcli/data/feathr_user_workspace/feathr_config.yaml) and use that as the source of truth. It should also have more explanations on the meaning of each variable." ] }, { @@ -194,8 +194,8 @@ " - 'REDIS_PASSWORD'\n", " - 'ADLS_ACCOUNT'\n", " - 'ADLS_KEY'\n", - " - 'WASB_ACCOUNT'\n", - " - 'WASB_KEY'\n", + " - 'BLOB_ACCOUNT'\n", + " - 'BLOB_KEY'\n", " - 'DATABRICKS_WORKSPACE_TOKEN_VALUE '\n", " \n", "offline_store:\n", @@ -215,6 +215,7 @@ " url: \".snowflakecomputing.com\"\n", " user: \"\"\n", " role: \"\"\n", + " warehouse: \"\"\n", "spark_config:\n", " spark_cluster: 'databricks'\n", " spark_result_output_parts: '1'\n", @@ -307,7 +308,7 @@ "source": [ "#### Setup necessary environment variables\n", "\n", - "You have to setup the environment variables in order to run this sample. More environment variables can be set by referring to [feathr_config.yaml](https://github.com/linkedin/feathr/blob/main/feathr_project/feathrcli/data/feathr_user_workspace/feathr_config.yaml) and use that as the source of truth. It should also have more explanations on the meaning of each variable." + "You have to setup the environment variables in order to run this sample. More environment variables can be set by referring to [feathr_config.yaml](https://github.com/feathr-ai/feathr/blob/main/feathr_project/feathrcli/data/feathr_user_workspace/feathr_config.yaml) and use that as the source of truth. It should also have more explanations on the meaning of each variable." ] }, { @@ -327,8 +328,8 @@ "os.environ['REDIS_PASSWORD'] = ''\n", "os.environ['ADLS_ACCOUNT'] = ''\n", "os.environ['ADLS_KEY'] = ''\n", - "os.environ['WASB_ACCOUNT'] = \"\"\n", - "os.environ['WASB_KEY'] = ''\n", + "os.environ['BLOB_ACCOUNT'] = \"\"\n", + "os.environ['BLOB_KEY'] = ''\n", "os.environ['DATABRICKS_WORKSPACE_TOKEN_VALUE'] = ''" ] }, @@ -854,7 +855,7 @@ " sinks=[redisSink],\n", " feature_names=[\"f_avg_item_ordered_by_customer\",\"f_avg_customer_discount_amount\"])\n", "\n", - "client.materialize_features(settings)\n", + "client.materialize_features(settings, allow_materialize_non_agg_feature =True)\n", "client.wait_job_to_finish(timeout_sec=500)" ] }, diff --git a/docs/samples/databricks/databricks_quickstart_nyc_taxi_demo.ipynb b/docs/samples/databricks/databricks_quickstart_nyc_taxi_demo.ipynb new file mode 100644 index 000000000..7d41696e8 --- /dev/null +++ b/docs/samples/databricks/databricks_quickstart_nyc_taxi_demo.ipynb @@ -0,0 +1,1216 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "inputWidgets": {}, + "nuid": "843d3142-24ca-4bd1-9e31-b55163804fe3", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "dbutils.widgets.text(\"RESOURCE_PREFIX\", \"\")\n", + "dbutils.widgets.text(\"REDIS_KEY\", \"\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "inputWidgets": {}, + "nuid": "384e5e16-7213-4186-9d04-09d03b155534", + "showTitle": false, + "title": "" + } + }, + "source": [ + "# Feathr Feature Store on Databricks Demo Notebook\n", + "\n", + "This notebook illustrates the use of Feature Store to create a model that predicts NYC Taxi fares. The dataset comes from [here](https://www1.nyc.gov/site/tlc/about/tlc-trip-record-data.page).\n", + "\n", + "This notebook is specifically written for Databricks and is relying on some of the Databricks packages such as `dbutils`. The intention here is to provide a \"one click run\" example with minimum configuration. For example:\n", + "- This notebook skips feature registry which requires running Azure Purview. \n", + "- To make the online feature query work, you will need to configure the Redis endpoint. \n", + "\n", + "The full-fledged notebook can be found from [here](https://github.com/feathr-ai/feathr/blob/main/docs/samples/nyc_taxi_demo.ipynb)." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "inputWidgets": {}, + "nuid": "c2ce58c7-9263-469a-bbb7-43364ddb07b8", + "showTitle": false, + "title": "" + } + }, + "source": [ + "## Prerequisite\n", + "\n", + "To use feathr materialization for online scoring with Redis cache, you may deploy a Redis cluster and set `RESOURCE_PREFIX` and `REDIS_KEY` via Databricks widgets. Note that the deployed Redis host address should be `{RESOURCE_PREFIX}redis.redis.cache.windows.net`. More details about how to deploy the Redis cluster can be found [here](https://feathr-ai.github.io/feathr/how-to-guides/azure-deployment-cli.html#configurure-redis-cluster).\n", + "\n", + "To run this notebook, you'll need to install `feathr` pip package. Here, we install notebook-scoped library. For details, please see [Azure Databricks dependency management document](https://learn.microsoft.com/en-us/azure/databricks/libraries/)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "inputWidgets": {}, + "nuid": "4609d7ad-ad74-40fc-b97e-f440a0fa0737", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "# Install feathr from the latest codes in the repo. You may use `pip install feathr` as well.\n", + "!pip install \"git+https://github.com/feathr-ai/feathr#subdirectory=feathr_project\"" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "inputWidgets": {}, + "nuid": "c81fa80c-bca6-4ae5-84ad-659a036977bd", + "showTitle": false, + "title": "" + } + }, + "source": [ + "## Notebook Steps\n", + "\n", + "This tutorial demonstrates the key capabilities of Feathr, including:\n", + "\n", + "1. Install Feathr and necessary dependencies.\n", + "1. Create shareable features with Feathr feature definition configs.\n", + "1. Create training data using point-in-time correct feature join\n", + "1. Train and evaluate a prediction model.\n", + "1. Materialize feature values for online scoring.\n", + "\n", + "The overall data flow is as follows:\n", + "\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "inputWidgets": {}, + "nuid": "80223a02-631c-40c8-91b3-a037249ffff9", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "from datetime import timedelta\n", + "import os\n", + "from pathlib import Path\n", + "\n", + "from pyspark.ml import Pipeline\n", + "from pyspark.ml.evaluation import RegressionEvaluator\n", + "from pyspark.ml.feature import VectorAssembler\n", + "from pyspark.ml.regression import GBTRegressor\n", + "from pyspark.sql import DataFrame\n", + "import pyspark.sql.functions as F\n", + "\n", + "import feathr\n", + "from feathr import (\n", + " FeathrClient,\n", + " # Feature data types\n", + " BOOLEAN,\n", + " FLOAT,\n", + " INT32,\n", + " ValueType,\n", + " # Feature data sources\n", + " INPUT_CONTEXT,\n", + " HdfsSource,\n", + " # Feature aggregations\n", + " TypedKey,\n", + " WindowAggTransformation,\n", + " # Feature types and anchor\n", + " DerivedFeature,\n", + " Feature,\n", + " FeatureAnchor,\n", + " # Materialization\n", + " BackfillTime,\n", + " MaterializationSettings,\n", + " RedisSink,\n", + " # Offline feature computation\n", + " FeatureQuery,\n", + " ObservationSettings,\n", + ")\n", + "from feathr.datasets import nyc_taxi\n", + "from feathr.spark_provider.feathr_configurations import SparkExecutionConfiguration\n", + "from feathr.utils.config import generate_config\n", + "from feathr.utils.job_utils import get_result_df\n", + "\n", + "\n", + "print(\n", + " f\"\"\"Feathr version: {feathr.__version__}\n", + "Databricks runtime version: {spark.conf.get(\"spark.databricks.clusterUsageTags.sparkVersion\")}\"\"\"\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "inputWidgets": {}, + "nuid": "ab35fa01-b392-457e-8fde-7e445a3c39b5", + "showTitle": false, + "title": "" + } + }, + "source": [ + "## 2. Create Shareable Features with Feathr Feature Definition Configs\n", + "\n", + "In this notebook, we define all the necessary resource key values for authentication. We use the values passed by the databricks widgets at the top of this notebook. Instead of manually entering the values to the widgets, we can also use [Azure Key Vault](https://azure.microsoft.com/en-us/services/key-vault/) to retrieve them.\n", + "Please refer to [how-to guide documents for granting key-vault access](https://feathr-ai.github.io/feathr/how-to-guides/azure-deployment-arm.html#3-grant-key-vault-and-synapse-access-to-selected-users-optional) and [Databricks' Azure Key Vault-backed scopes](https://learn.microsoft.com/en-us/azure/databricks/security/secrets/secret-scopes) for more details." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "inputWidgets": {}, + "nuid": "09f93a9f-7b33-4d91-8f31-ee3b20991696", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "RESOURCE_PREFIX = dbutils.widgets.get(\"RESOURCE_PREFIX\")\n", + "PROJECT_NAME = \"feathr_getting_started\"\n", + "\n", + "REDIS_KEY = dbutils.widgets.get(\"REDIS_KEY\")\n", + "\n", + "# Use a databricks cluster\n", + "SPARK_CLUSTER = \"databricks\"\n", + "\n", + "# Databricks file system path\n", + "DATA_STORE_PATH = f\"dbfs:/{PROJECT_NAME}\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "inputWidgets": {}, + "nuid": "331753d6-1850-47b5-ad97-84b7c01d79d1", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "# Redis credential\n", + "os.environ[\"REDIS_PASSWORD\"] = REDIS_KEY" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "inputWidgets": {}, + "nuid": "08bc3b7e-bbf5-4e3a-9978-fe1aef8c1aee", + "showTitle": false, + "title": "" + } + }, + "source": [ + "### Configurations\n", + "\n", + "Feathr uses a yaml file to define configurations. Please refer to [feathr_config.yaml]( https://github.com//feathr-ai/feathr/blob/main/feathr_project/feathrcli/data/feathr_user_workspace/feathr_config.yaml) for the meaning of each field.\n", + "\n", + "In the following cell, we set required databricks credentials automatically by using a databricks notebook context object as well as new job cluster spec." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ctx = dbutils.notebook.entry_point.getDbutils().notebook().getContext()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "inputWidgets": {}, + "nuid": "8cd64e3a-376c-48e6-ba41-5197f3591d48", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "config_path = generate_config(\n", + " resource_prefix=RESOURCE_PREFIX,\n", + " project_name=PROJECT_NAME,\n", + " spark_config__spark_cluster=SPARK_CLUSTER,\n", + " # You may set an existing cluster id here, but Databricks recommend to use new clusters for greater reliability.\n", + " databricks_cluster_id=None, # Set None to create a new job cluster\n", + " databricks_workspace_token_value=ctx.apiToken().get(),\n", + " spark_config__databricks__workspace_instance_url=f\"https://{ctx.tags().get('browserHostName').get()}\",\n", + ")\n", + "\n", + "with open(config_path, \"r\") as f:\n", + " print(f.read())" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "inputWidgets": {}, + "nuid": "58d22dc1-7590-494d-94ca-3e2488c31c8e", + "showTitle": false, + "title": "" + } + }, + "source": [ + "All the configurations can be overwritten by environment variables with concatenation of `__` for different layers of the config file. For example, `feathr_runtime_location` for databricks config can be overwritten by setting `spark_config__databricks__feathr_runtime_location` environment variable." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "inputWidgets": {}, + "nuid": "3fef7f2f-df19-4f53-90a5-ff7999ed983d", + "showTitle": false, + "title": "" + } + }, + "source": [ + "### Initialize Feathr Client" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "inputWidgets": {}, + "nuid": "9713a2df-c7b2-4562-88b0-b7acce3cc43a", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "client = FeathrClient(config_path=config_path)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "inputWidgets": {}, + "nuid": "c3b64bda-d42c-4a64-b976-0fb604cf38c5", + "showTitle": false, + "title": "" + } + }, + "source": [ + "### View the NYC taxi fare dataset" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "inputWidgets": {}, + "nuid": "c4ccd7b3-298a-4e5a-8eec-b7e309db393e", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "DATA_FILE_PATH = str(Path(DATA_STORE_PATH, \"nyc_taxi.csv\"))\n", + "\n", + "# Download the data file\n", + "df_raw = nyc_taxi.get_spark_df(spark=spark, local_cache_path=DATA_FILE_PATH)\n", + "df_raw.limit(5).toPandas()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "inputWidgets": {}, + "nuid": "7430c942-64e5-4b70-b823-16ce1d1b3cee", + "showTitle": false, + "title": "" + } + }, + "source": [ + "### Defining features with Feathr\n", + "\n", + "In Feathr, a feature is viewed as a function, mapping a key and timestamp to a feature value. For more details, please see [Feathr Feature Definition Guide](https://github.com/feathr-ai/feathr/blob/main/docs/concepts/feature-definition.md).\n", + "\n", + "* The feature key (a.k.a. entity id) identifies the subject of feature, e.g. a user_id or location_id.\n", + "* The feature name is the aspect of the entity that the feature is indicating, e.g. the age of the user.\n", + "* The feature value is the actual value of that aspect at a particular time, e.g. the value is 30 at year 2022.\n", + "\n", + "Note that, in some cases, a feature could be just a transformation function that has no entity key or timestamp involved, e.g. *the day of week of the request timestamp*.\n", + "\n", + "There are two types of features -- anchored features and derivated features:\n", + "\n", + "* **Anchored features**: Features that are directly extracted from sources. Could be with or without aggregation. \n", + "* **Derived features**: Features that are computed on top of other features.\n", + "\n", + "#### Define anchored features\n", + "\n", + "A feature source is needed for anchored features that describes the raw data in which the feature values are computed from. A source value should be either `INPUT_CONTEXT` (the features that will be extracted from the observation data directly) or `feathr.source.Source` object." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "inputWidgets": {}, + "nuid": "75b8d2ed-84df-4446-ae07-5f715434f3ea", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "TIMESTAMP_COL = \"lpep_dropoff_datetime\"\n", + "TIMESTAMP_FORMAT = \"yyyy-MM-dd HH:mm:ss\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "inputWidgets": {}, + "nuid": "93abbcc2-562b-47e4-ad4c-1fedd7cc64df", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "# We define f_trip_distance and f_trip_time_duration features separately\n", + "# so that we can reuse them later for the derived features.\n", + "f_trip_distance = Feature(\n", + " name=\"f_trip_distance\",\n", + " feature_type=FLOAT,\n", + " transform=\"trip_distance\",\n", + ")\n", + "f_trip_time_duration = Feature(\n", + " name=\"f_trip_time_duration\",\n", + " feature_type=FLOAT,\n", + " transform=\"cast_float((to_unix_timestamp(lpep_dropoff_datetime) - to_unix_timestamp(lpep_pickup_datetime)) / 60)\",\n", + ")\n", + "\n", + "features = [\n", + " f_trip_distance,\n", + " f_trip_time_duration,\n", + " Feature(\n", + " name=\"f_is_long_trip_distance\",\n", + " feature_type=BOOLEAN,\n", + " transform=\"trip_distance > 30.0\",\n", + " ),\n", + " Feature(\n", + " name=\"f_day_of_week\",\n", + " feature_type=INT32,\n", + " transform=\"dayofweek(lpep_dropoff_datetime)\",\n", + " ),\n", + " Feature(\n", + " name=\"f_day_of_month\",\n", + " feature_type=INT32,\n", + " transform=\"dayofmonth(lpep_dropoff_datetime)\",\n", + " ),\n", + " Feature(\n", + " name=\"f_hour_of_day\",\n", + " feature_type=INT32,\n", + " transform=\"hour(lpep_dropoff_datetime)\",\n", + " ),\n", + "]\n", + "\n", + "# After you have defined features, bring them together to build the anchor to the source.\n", + "feature_anchor = FeatureAnchor(\n", + " name=\"feature_anchor\",\n", + " source=INPUT_CONTEXT, # Pass through source, i.e. observation data.\n", + " features=features,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "inputWidgets": {}, + "nuid": "728d2d5f-c11f-4941-bdc5-48507f5749f1", + "showTitle": false, + "title": "" + } + }, + "source": [ + "We can define the source with a preprocessing python function." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "inputWidgets": {}, + "nuid": "3cc59a0e-a41b-480e-a84e-ca5443d63143", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "def preprocessing(df: DataFrame) -> DataFrame:\n", + " import pyspark.sql.functions as F\n", + "\n", + " df = df.withColumn(\n", + " \"fare_amount_cents\", (F.col(\"fare_amount\") * 100.0).cast(\"float\")\n", + " )\n", + " return df\n", + "\n", + "\n", + "batch_source = HdfsSource(\n", + " name=\"nycTaxiBatchSource\",\n", + " path=DATA_FILE_PATH,\n", + " event_timestamp_column=TIMESTAMP_COL,\n", + " preprocessing=preprocessing,\n", + " timestamp_format=TIMESTAMP_FORMAT,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "inputWidgets": {}, + "nuid": "46f863c4-bb81-434a-a448-6b585031a221", + "showTitle": false, + "title": "" + } + }, + "source": [ + "For the features with aggregation, the supported functions are as follows:\n", + "\n", + "| Aggregation Function | Input Type | Description |\n", + "| --- | --- | --- |\n", + "|SUM, COUNT, MAX, MIN, AVG\t|Numeric|Applies the the numerical operation on the numeric inputs. |\n", + "|MAX_POOLING, MIN_POOLING, AVG_POOLING\t| Numeric Vector | Applies the max/min/avg operation on a per entry bassis for a given a collection of numbers.|\n", + "|LATEST| Any |Returns the latest not-null values from within the defined time window |" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "inputWidgets": {}, + "nuid": "a373ecbe-a040-4cd3-9d87-0d5f4c5ba553", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "agg_key = TypedKey(\n", + " key_column=\"DOLocationID\",\n", + " key_column_type=ValueType.INT32,\n", + " description=\"location id in NYC\",\n", + " full_name=\"nyc_taxi.location_id\",\n", + ")\n", + "\n", + "agg_window = \"90d\"\n", + "\n", + "# Anchored features with aggregations\n", + "agg_features = [\n", + " Feature(\n", + " name=\"f_location_avg_fare\",\n", + " key=agg_key,\n", + " feature_type=FLOAT,\n", + " transform=WindowAggTransformation(\n", + " agg_expr=\"fare_amount_cents\",\n", + " agg_func=\"AVG\",\n", + " window=agg_window,\n", + " ),\n", + " ),\n", + " Feature(\n", + " name=\"f_location_max_fare\",\n", + " key=agg_key,\n", + " feature_type=FLOAT,\n", + " transform=WindowAggTransformation(\n", + " agg_expr=\"fare_amount_cents\",\n", + " agg_func=\"MAX\",\n", + " window=agg_window,\n", + " ),\n", + " ),\n", + "]\n", + "\n", + "agg_feature_anchor = FeatureAnchor(\n", + " name=\"agg_feature_anchor\",\n", + " source=batch_source, # External data source for feature. Typically a data table.\n", + " features=agg_features,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "inputWidgets": {}, + "nuid": "149f85e2-fa3c-4895-b0c5-de5543ca9b6d", + "showTitle": false, + "title": "" + } + }, + "source": [ + "#### Define derived features\n", + "\n", + "We also define a derived feature, `f_trip_time_distance`, from the anchored features `f_trip_distance` and `f_trip_time_duration` as follows:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "inputWidgets": {}, + "nuid": "05633bc3-9118-449b-9562-45fc437576c2", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "derived_features = [\n", + " DerivedFeature(\n", + " name=\"f_trip_time_distance\",\n", + " feature_type=FLOAT,\n", + " input_features=[\n", + " f_trip_distance,\n", + " f_trip_time_duration,\n", + " ],\n", + " transform=\"f_trip_distance / f_trip_time_duration\",\n", + " )\n", + "]" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "inputWidgets": {}, + "nuid": "ad102c45-586d-468c-85f0-9454401ef10b", + "showTitle": false, + "title": "" + } + }, + "source": [ + "### Build features\n", + "\n", + "Finally, we build the features." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "inputWidgets": {}, + "nuid": "91bb5ebb-87e4-470b-b8eb-1c89b351740e", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "client.build_features(\n", + " anchor_list=[feature_anchor, agg_feature_anchor],\n", + " derived_feature_list=derived_features,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "inputWidgets": {}, + "nuid": "632d5f46-f9e2-41a8-aab7-34f75206e2aa", + "showTitle": false, + "title": "" + } + }, + "source": [ + "## 3. Create Training Data Using Point-in-Time Correct Feature Join\n", + "\n", + "After the feature producers have defined the features (as described in the Feature Definition part), the feature consumers may want to consume those features. Feature consumers will use observation data to query from different feature tables using Feature Query.\n", + "\n", + "To create a training dataset using Feathr, one needs to provide a feature join configuration file to specify\n", + "what features and how these features should be joined to the observation data. \n", + "\n", + "To learn more on this topic, please refer to [Point-in-time Correctness](https://github.com//feathr-ai/feathr/blob/main/docs/concepts/point-in-time-join.md)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "inputWidgets": {}, + "nuid": "02feabc9-2f2f-43e8-898d-b28082798e98", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "feature_names = [feature.name for feature in features + agg_features + derived_features]\n", + "feature_names" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "inputWidgets": {}, + "nuid": "e438e6d8-162e-4aa3-b3b3-9d1f3b0d2b7f", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "DATA_FORMAT = \"parquet\"\n", + "offline_features_path = str(\n", + " Path(DATA_STORE_PATH, \"feathr_output\", f\"features.{DATA_FORMAT}\")\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "inputWidgets": {}, + "nuid": "67e81466-c736-47ba-b122-e640642c01cf", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "# Features that we want to request. Can use a subset of features\n", + "query = FeatureQuery(\n", + " feature_list=feature_names,\n", + " key=agg_key,\n", + ")\n", + "settings = ObservationSettings(\n", + " observation_path=DATA_FILE_PATH,\n", + " event_timestamp_column=TIMESTAMP_COL,\n", + " timestamp_format=TIMESTAMP_FORMAT,\n", + ")\n", + "client.get_offline_features(\n", + " observation_settings=settings,\n", + " feature_query=query,\n", + " # Note, execution_configurations argument only works when using a new job cluster\n", + " # For more details, see https://feathr-ai.github.io/feathr/how-to-guides/feathr-job-configuration.html\n", + " execution_configurations=SparkExecutionConfiguration(\n", + " {\n", + " \"spark.feathr.outputFormat\": DATA_FORMAT,\n", + " }\n", + " ),\n", + " output_path=offline_features_path,\n", + ")\n", + "\n", + "client.wait_job_to_finish(timeout_sec=500)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "inputWidgets": {}, + "nuid": "9871af55-25eb-41ee-a58a-fda74b1a174e", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "# Show feature results\n", + "df = get_result_df(\n", + " spark=spark,\n", + " client=client,\n", + " data_format=\"parquet\",\n", + " res_url=offline_features_path,\n", + ")\n", + "df.select(feature_names).limit(5).toPandas()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "inputWidgets": {}, + "nuid": "dcbf17fc-7f79-4a65-a3af-9cffbd0b5d1f", + "showTitle": false, + "title": "" + } + }, + "source": [ + "## 4. Train and Evaluate a Prediction Model\n", + "\n", + "After generating all the features, we train and evaluate a machine learning model to predict the NYC taxi fare prediction. In this example, we use Spark MLlib's [GBTRegressor](https://spark.apache.org/docs/latest/ml-classification-regression.html#gradient-boosted-tree-regression).\n", + "\n", + "Note that designing features, training prediction models and evaluating them are an iterative process where the models' performance maybe used to modify the features as a part of the modeling process." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "inputWidgets": {}, + "nuid": "5a226026-1c7b-48db-8f91-88d5c2ddf023", + "showTitle": false, + "title": "" + } + }, + "source": [ + "### Load Train and Test Data from the Offline Feature Values" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "inputWidgets": {}, + "nuid": "bd2cdc83-0920-46e8-9454-e5e6e7832ce0", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "# Train / test split\n", + "train_df, test_df = (\n", + " df.withColumn( # Dataframe that we generated from get_offline_features call.\n", + " \"label\", F.col(\"fare_amount\").cast(\"double\")\n", + " )\n", + " .where(F.col(\"f_trip_time_duration\") > 0)\n", + " .fillna(0)\n", + " .randomSplit([0.8, 0.2])\n", + ")\n", + "\n", + "print(f\"Num train samples: {train_df.count()}\")\n", + "print(f\"Num test samples: {test_df.count()}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "inputWidgets": {}, + "nuid": "6a3e2ab1-5c66-4d27-a737-c5e2af03b1dd", + "showTitle": false, + "title": "" + } + }, + "source": [ + "### Build a ML Pipeline\n", + "\n", + "Here, we use Spark ML Pipeline to aggregate feature vectors and feed them to the model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "inputWidgets": {}, + "nuid": "2a254361-63e9-45b2-8c19-40549762eacb", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "# Generate a feature vector column for SparkML\n", + "vector_assembler = VectorAssembler(\n", + " inputCols=[x for x in df.columns if x in feature_names],\n", + " outputCol=\"features\",\n", + ")\n", + "\n", + "# Define a model\n", + "gbt = GBTRegressor(\n", + " featuresCol=\"features\",\n", + " maxIter=100,\n", + " maxDepth=5,\n", + " maxBins=16,\n", + ")\n", + "\n", + "# Create a ML pipeline\n", + "ml_pipeline = Pipeline(\n", + " stages=[\n", + " vector_assembler,\n", + " gbt,\n", + " ]\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "inputWidgets": {}, + "nuid": "bef93538-9591-4247-97b6-289d2055b7b1", + "showTitle": false, + "title": "" + } + }, + "source": [ + "### Train and Evaluate the Model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "inputWidgets": {}, + "nuid": "0c3d5f35-11a3-4644-9992-5860169d8302", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "# Train a model\n", + "model = ml_pipeline.fit(train_df)\n", + "\n", + "# Make predictions\n", + "predictions = model.transform(test_df)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "inputWidgets": {}, + "nuid": "1f9b584c-6228-4a02-a6c3-9b8dd2b78091", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "# Evaluate\n", + "evaluator = RegressionEvaluator(\n", + " labelCol=\"label\",\n", + " predictionCol=\"prediction\",\n", + ")\n", + "\n", + "rmse = evaluator.evaluate(predictions, {evaluator.metricName: \"rmse\"})\n", + "mae = evaluator.evaluate(predictions, {evaluator.metricName: \"mae\"})\n", + "print(f\"RMSE: {rmse}\\nMAE: {mae}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "inputWidgets": {}, + "nuid": "25c33abd-6e87-437d-a6a1-86435f065a1e", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "# predicted fare vs actual fare plots -- will this work for databricks / synapse / local ?\n", + "predictions_pdf = predictions.select([\"label\", \"prediction\"]).toPandas().reset_index()\n", + "\n", + "predictions_pdf.plot(\n", + " x=\"index\",\n", + " y=[\"label\", \"prediction\"],\n", + " style=[\"-\", \":\"],\n", + " figsize=(20, 10),\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "inputWidgets": {}, + "nuid": "664d78cc-4a92-430c-9e05-565ba904558e", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "predictions_pdf.plot.scatter(\n", + " x=\"label\",\n", + " y=\"prediction\",\n", + " xlim=(0, 100),\n", + " ylim=(0, 100),\n", + " figsize=(10, 10),\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "inputWidgets": {}, + "nuid": "8a56d165-c813-4ce0-8ae6-9f4d313c463d", + "showTitle": false, + "title": "" + } + }, + "source": [ + "## 5. Materialize Feature Values for Online Scoring\n", + "\n", + "While we computed feature values on-the-fly at request time via Feathr, we can pre-compute the feature values and materialize them to offline or online storages such as Redis.\n", + "\n", + "Note, only the features anchored to offline data source can be materialized." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "inputWidgets": {}, + "nuid": "751fa72e-8f94-40a1-994e-3e8315b51d37", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "materialized_feature_names = [feature.name for feature in agg_features]\n", + "materialized_feature_names" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "inputWidgets": {}, + "nuid": "4d4699ed-42e6-408f-903d-2f799284f4b6", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "if REDIS_KEY and RESOURCE_PREFIX:\n", + " FEATURE_TABLE_NAME = \"nycTaxiDemoFeature\"\n", + "\n", + " # Get the last date from the dataset\n", + " backfill_timestamp = (\n", + " df_raw.select(\n", + " F.to_timestamp(F.col(TIMESTAMP_COL), TIMESTAMP_FORMAT).alias(TIMESTAMP_COL)\n", + " )\n", + " .agg({TIMESTAMP_COL: \"max\"})\n", + " .collect()[0][0]\n", + " )\n", + "\n", + " # Time range to materialize\n", + " backfill_time = BackfillTime(\n", + " start=backfill_timestamp,\n", + " end=backfill_timestamp,\n", + " step=timedelta(days=1),\n", + " )\n", + "\n", + " # Destinations:\n", + " # For online store,\n", + " redis_sink = RedisSink(table_name=FEATURE_TABLE_NAME)\n", + "\n", + " # For offline store,\n", + " # adls_sink = HdfsSink(output_path=)\n", + "\n", + " settings = MaterializationSettings(\n", + " name=FEATURE_TABLE_NAME + \".job\", # job name\n", + " backfill_time=backfill_time,\n", + " sinks=[redis_sink], # or adls_sink\n", + " feature_names=materialized_feature_names,\n", + " )\n", + "\n", + " client.materialize_features(\n", + " settings=settings,\n", + " # Note, execution_configurations argument only works when using a new job cluster\n", + " execution_configurations={\"spark.feathr.outputFormat\": \"parquet\"},\n", + " )\n", + "\n", + " client.wait_job_to_finish(timeout_sec=500)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "inputWidgets": {}, + "nuid": "5aa13acd-58ec-4fc2-86bb-dc1d9951ebb9", + "showTitle": false, + "title": "" + } + }, + "source": [ + "Now, you can retrieve features for online scoring as follows:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "inputWidgets": {}, + "nuid": "424bc9eb-a47f-4b46-be69-8218d55e66ad", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "if REDIS_KEY and RESOURCE_PREFIX:\n", + " # Note, to get a single key, you may use client.get_online_features instead\n", + " materialized_feature_values = client.multi_get_online_features(\n", + " feature_table=FEATURE_TABLE_NAME,\n", + " keys=[\"239\", \"265\"],\n", + " feature_names=materialized_feature_names,\n", + " )\n", + " materialized_feature_values" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "inputWidgets": {}, + "nuid": "3596dc71-a363-4b6a-a169-215c89978558", + "showTitle": false, + "title": "" + } + }, + "source": [ + "## Cleanup" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "inputWidgets": {}, + "nuid": "b5fb292e-bbb6-4dd7-8e79-c62d9533e820", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "# Remove temporary files\n", + "dbutils.fs.rm(\"dbfs:/tmp/\", recurse=True)" + ] + } + ], + "metadata": { + "application/vnd.databricks.v1+notebook": { + "dashboards": [], + "language": "python", + "notebookMetadata": { + "pythonIndentUnit": 4 + }, + "notebookName": "databricks_quickstart_nyc_taxi_demo", + "notebookOrigID": 2365994027381987, + "widgets": { + "REDIS_KEY": { + "currentValue": "", + "nuid": "d39ce0d5-bcfe-47ef-b3d9-eff67e5cdeca", + "widgetInfo": { + "defaultValue": "", + "label": null, + "name": "REDIS_KEY", + "options": { + "validationRegex": null, + "widgetType": "text" + }, + "widgetType": "text" + } + }, + "RESOURCE_PREFIX": { + "currentValue": "", + "nuid": "87a26035-86fc-4dbd-8dd0-dc546c1c63c1", + "widgetInfo": { + "defaultValue": "", + "label": null, + "name": "RESOURCE_PREFIX", + "options": { + "validationRegex": null, + "widgetType": "text" + }, + "widgetType": "text" + } + } + } + }, + "kernelspec": { + "display_name": "Python 3.10.4 ('feathr')", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.4" + }, + "vscode": { + "interpreter": { + "hash": "e34a1a57d2e174682770a82d94a178aa36d3ccfaa21227c5d2308e319b7ae532" + } + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/docs/samples/databricks/databricks_quickstart_nyc_taxi_driver.ipynb b/docs/samples/databricks/databricks_quickstart_nyc_taxi_driver.ipynb deleted file mode 100644 index 52790f884..000000000 --- a/docs/samples/databricks/databricks_quickstart_nyc_taxi_driver.ipynb +++ /dev/null @@ -1,1442 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "inputWidgets": {}, - "nuid": "384e5e16-7213-4186-9d04-09d03b155534", - "showTitle": false, - "title": "" - } - }, - "source": [ - "# Feathr Feature Store on Databricks Demo Notebook\n", - "\n", - "This notebook illustrates the use of Feature Store to create a model that predicts NYC Taxi fares. This is a notebook that's specially designed for databricks clusters and is relying on some of the databricks packages such as dbutils.\n", - "\n", - "The intent of this notebook is like \"one click run\" without configuring anything, so it has relatively limited capability. \n", - "\n", - "- For example, in this notebook there's no feature registry available since that requires running Azure Purview. \n", - "- Also for online store (Redis), you need to configure the Redis endpoint, otherwise that part will not work. \n", - "\n", - "However, the core part of Feathr, especially defining features, get offline features, point-in-time joins etc., should \"just work\". The full-fledged notebook is [located here](https://github.com/linkedin/feathr/blob/main/feathr_project/feathrcli/data/feathr_user_workspace/nyc_driver_demo.ipynb)." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "# Notebook Steps\n", - "\n", - "This tutorial demonstrates the key capabilities of Feathr, including:\n", - "\n", - "1. Install and set up Feathr with Azure\n", - "2. Create shareable features with Feathr feature definition configs.\n", - "3. Create a training dataset via point-in-time feature join.\n", - "4. Compute and write features.\n", - "5. Train a model using these features to predict fares.\n", - "6. Materialize feature value to online store.\n", - "7. Fetch feature value in real-time from online store for online scoring.\n", - "\n", - "In this tutorial, we use Feathr Feature Store to create a model that predicts NYC Taxi fares. The dataset comes from [here](https://www1.nyc.gov/site/tlc/about/tlc-trip-record-data.page). The feature flow is as below:\n", - "\n", - "![Feature Flow](https://github.com/linkedin/feathr/blob/main/docs/images/feature_flow.png?raw=true)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "inputWidgets": {}, - "nuid": "f00b9d0b-94d1-418f-89b9-25bbacb8b068", - "showTitle": false, - "title": "" - } - }, - "outputs": [ - { - "data": { - "text/html": [ - "" - ] - }, - "metadata": { - "application/vnd.databricks.v1+output": { - "arguments": {}, - "data": "", - "errorSummary": "", - "errorTraceType": null, - "metadata": {}, - "type": "ipynbError" - } - }, - "output_type": "display_data" - } - ], - "source": [ - "! pip install feathr pandavro scikit-learn" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "inputWidgets": {}, - "nuid": "80223a02-631c-40c8-91b3-a037249ffff9", - "showTitle": false, - "title": "" - } - }, - "outputs": [ - { - "data": { - "text/html": [ - "" - ] - }, - "metadata": { - "application/vnd.databricks.v1+output": { - "arguments": {}, - "data": "", - "errorSummary": "", - "errorTraceType": null, - "metadata": {}, - "type": "ipynbError" - } - }, - "output_type": "display_data" - } - ], - "source": [ - "import glob\n", - "import os\n", - "import tempfile\n", - "from datetime import datetime, timedelta\n", - "from math import sqrt\n", - "\n", - "import pandas as pd\n", - "import pandavro as pdx\n", - "from feathr import FeathrClient\n", - "from feathr import BOOLEAN, FLOAT, INT32, ValueType\n", - "from feathr import Feature, DerivedFeature, FeatureAnchor\n", - "from feathr import BackfillTime, MaterializationSettings\n", - "from feathr import FeatureQuery, ObservationSettings\n", - "from feathr import RedisSink\n", - "from feathr import INPUT_CONTEXT, HdfsSource\n", - "from feathr import WindowAggTransformation\n", - "from feathr import TypedKey\n", - "from sklearn.metrics import mean_squared_error\n", - "from sklearn.model_selection import train_test_split\n", - "from azure.identity import DefaultAzureCredential\n", - "from azure.keyvault.secrets import SecretClient\n", - "import json\n", - "import requests" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "inputWidgets": {}, - "nuid": "41d3648a-9bc9-40dc-90da-bc82b21ef9b3", - "showTitle": false, - "title": "" - } - }, - "source": [ - "Get the required databricks credentials automatically:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "inputWidgets": {}, - "nuid": "331753d6-1850-47b5-ad97-84b7c01d79d1", - "showTitle": false, - "title": "" - } - }, - "outputs": [ - { - "data": { - "text/html": [ - "" - ] - }, - "metadata": { - "application/vnd.databricks.v1+output": { - "arguments": {}, - "data": "", - "errorSummary": "", - "errorTraceType": null, - "metadata": {}, - "type": "ipynbError" - } - }, - "output_type": "display_data" - } - ], - "source": [ - "# Get current databricks notebook context\n", - "ctx = dbutils.notebook.entry_point.getDbutils().notebook().getContext()\n", - "host_name = ctx.tags().get(\"browserHostName\").get()\n", - "host_token = ctx.apiToken().get()\n", - "cluster_id = ctx.tags().get(\"clusterId\").get()\n", - "\n", - "\n", - "\n", - "# databricks_config = {'run_name':'FEATHR_FILL_IN','existing_cluster_id':cluster_id,'libraries':[{'jar':'FEATHR_FILL_IN'}],'spark_jar_task':{'main_class_name':'FEATHR_FILL_IN','parameters':['FEATHR_FILL_IN']}}\n", - "os.environ['spark_config__databricks__workspace_instance_url'] = \"https://\" + host_name\n", - "os.environ['spark_config__databricks__config_template']='{\"run_name\":\"FEATHR_FILL_IN\",\"new_cluster\":{\"spark_version\":\"10.4.x-scala2.12\",\"node_type_id\":\"Standard_D3_v2\",\"num_workers\":2,\"spark_conf\":{\"FEATHR_FILL_IN\":\"FEATHR_FILL_IN\"}},\"libraries\":[{\"jar\":\"FEATHR_FILL_IN\"}],\"spark_jar_task\":{\"main_class_name\":\"FEATHR_FILL_IN\",\"parameters\":[\"FEATHR_FILL_IN\"]}}'\n", - "# os.environ['spark_config__databricks__config_template']=json.dumps(databricks_config)\n", - "os.environ['spark_config__databricks__work_dir']='dbfs:/feathr_getting_started'\n", - "os.environ['project_config__project_name']='feathr_getting_started'\n", - "os.environ['DATABRICKS_WORKSPACE_TOKEN_VALUE'] = host_token" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "You need to setup the Redis credentials below in order to push features to online store. You can skip this part if you don't have Redis, but there will be failures for `client.materialize_features(settings)` API." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Get redis credentials; This is to parse Redis connection string.\n", - "redis_port=\"\"\n", - "redis_host=\"\"\n", - "redis_password=\"\"\n", - "redis_ssl=\"\"\n", - "\n", - "# Set the resource link\n", - "os.environ['online_store__redis__host'] = redis_host\n", - "os.environ['online_store__redis__port'] = redis_port\n", - "os.environ['online_store__redis__ssl_enabled'] = redis_ssl\n", - "os.environ['REDIS_PASSWORD']=redis_password" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "inputWidgets": {}, - "nuid": "08bc3b7e-bbf5-4e3a-9978-fe1aef8c1aee", - "showTitle": false, - "title": "" - } - }, - "source": [ - "Configure required credentials (skip if you don't use those):" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "inputWidgets": {}, - "nuid": "8cd64e3a-376c-48e6-ba41-5197f3591d48", - "showTitle": false, - "title": "" - } - }, - "outputs": [ - { - "data": { - "text/html": [ - "" - ] - }, - "metadata": { - "application/vnd.databricks.v1+output": { - "arguments": {}, - "data": "", - "errorSummary": "", - "errorTraceType": null, - "metadata": {}, - "type": "ipynbError" - } - }, - "output_type": "display_data" - } - ], - "source": [ - "import tempfile\n", - "yaml_config = \"\"\"\n", - "# Please refer to https://github.com/linkedin/feathr/blob/main/feathr_project/feathrcli/data/feathr_user_workspace/feathr_config.yaml for explanations on the meaning of each field.\n", - "api_version: 1\n", - "project_config:\n", - " project_name: 'feathr_getting_started2'\n", - " required_environment_variables:\n", - " - 'REDIS_PASSWORD'\n", - "offline_store:\n", - " adls:\n", - " adls_enabled: true\n", - " wasb:\n", - " wasb_enabled: true\n", - " s3:\n", - " s3_enabled: false\n", - " s3_endpoint: ''\n", - " jdbc:\n", - " jdbc_enabled: false\n", - " jdbc_database: ''\n", - " jdbc_table: ''\n", - " snowflake:\n", - " snowflake_enabled: false\n", - " url: \".snowflakecomputing.com\"\n", - " user: \"\"\n", - " role: \"\"\n", - "spark_config:\n", - " # choice for spark runtime. Currently support: azure_synapse, databricks\n", - " # The `databricks` configs will be ignored if `azure_synapse` is set and vice versa.\n", - " spark_cluster: \"databricks\"\n", - " spark_result_output_parts: \"1\"\n", - "\n", - "online_store:\n", - " redis:\n", - " host: '.redis.cache.windows.net'\n", - " port: 6380\n", - " ssl_enabled: True\n", - "feature_registry:\n", - " api_endpoint: \"https://.azurewebsites.net/api/v1\"\n", - "\"\"\"\n", - "tmp = tempfile.NamedTemporaryFile(mode='w', delete=False)\n", - "with open(tmp.name, \"w\") as text_file:\n", - " text_file.write(yaml_config)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "inputWidgets": {}, - "nuid": "3fef7f2f-df19-4f53-90a5-ff7999ed983d", - "showTitle": false, - "title": "" - } - }, - "source": [ - "# Initialize Feathr Client" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "inputWidgets": {}, - "nuid": "9713a2df-c7b2-4562-88b0-b7acce3cc43a", - "showTitle": false, - "title": "" - } - }, - "outputs": [ - { - "data": { - "text/html": [ - "" - ] - }, - "metadata": { - "application/vnd.databricks.v1+output": { - "arguments": {}, - "data": "", - "errorSummary": "", - "errorTraceType": null, - "metadata": {}, - "type": "ipynbError" - } - }, - "output_type": "display_data" - } - ], - "source": [ - "client = FeathrClient(config_path=tmp.name)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "inputWidgets": {}, - "nuid": "c3b64bda-d42c-4a64-b976-0fb604cf38c5", - "showTitle": false, - "title": "" - } - }, - "source": [ - "## View the data\n", - "\n", - "In this tutorial, we use Feathr Feature Store to create a model that predicts NYC Taxi fares. The dataset comes from [here](https://www1.nyc.gov/site/tlc/about/tlc-trip-record-data.page). The data is as below" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "inputWidgets": {}, - "nuid": "c4ccd7b3-298a-4e5a-8eec-b7e309db393e", - "showTitle": false, - "title": "" - } - }, - "outputs": [ - { - "data": { - "text/html": [ - "" - ] - }, - "metadata": { - "application/vnd.databricks.v1+output": { - "arguments": {}, - "data": "", - "errorSummary": "", - "errorTraceType": null, - "metadata": {}, - "type": "ipynbError" - } - }, - "output_type": "display_data" - } - ], - "source": [ - "import pandas as pd\n", - "pd.read_csv(\"https://azurefeathrstorage.blob.core.windows.net/public/sample_data/green_tripdata_2020-04_with_index.csv\")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "inputWidgets": {}, - "nuid": "7430c942-64e5-4b70-b823-16ce1d1b3cee", - "showTitle": false, - "title": "" - } - }, - "source": [ - "## Defining Features with Feathr\n", - "\n", - "In Feathr, a feature is viewed as a function, mapping from entity id or key, and timestamp to a feature value. For more details on feature definition, please refer to the [Feathr Feature Definition Guide](https://github.com/linkedin/feathr/blob/main/docs/concepts/feature-definition.md)\n", - "\n", - "\n", - "1. The typed key (a.k.a. entity id) identifies the subject of feature, e.g. a user id, 123.\n", - "2. The feature name is the aspect of the entity that the feature is indicating, e.g. the age of the user.\n", - "3. The feature value is the actual value of that aspect at a particular time, e.g. the value is 30 at year 2022." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "inputWidgets": {}, - "nuid": "16420730-582e-4e11-a343-efc0ddd35108", - "showTitle": false, - "title": "" - } - }, - "source": [ - "Note that, in some cases, such as features defined on top of request data, may have no entity key or timestamp.\n", - "It is merely a function/transformation executing against request data at runtime.\n", - "For example, the day of week of the request, which is calculated by converting the request UNIX timestamp." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "inputWidgets": {}, - "nuid": "728d2d5f-c11f-4941-bdc5-48507f5749f1", - "showTitle": false, - "title": "" - } - }, - "source": [ - "### Define Sources Section with UDFs\n", - "A feature source is needed for anchored features that describes the raw data in which the feature values are computed from. See the python documentation to get the details on each input column." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "inputWidgets": {}, - "nuid": "3cc59a0e-a41b-480e-a84e-ca5443d63143", - "showTitle": false, - "title": "" - } - }, - "outputs": [ - { - "data": { - "text/html": [ - "" - ] - }, - "metadata": { - "application/vnd.databricks.v1+output": { - "arguments": {}, - "data": "", - "errorSummary": "", - "errorTraceType": null, - "metadata": {}, - "type": "ipynbError" - } - }, - "output_type": "display_data" - } - ], - "source": [ - "batch_source = HdfsSource(name=\"nycTaxiBatchSource\",\n", - " path=\"wasbs://public@azurefeathrstorage.blob.core.windows.net/sample_data/green_tripdata_2020-04_with_index.csv\",\n", - " event_timestamp_column=\"lpep_dropoff_datetime\",\n", - " timestamp_format=\"yyyy-MM-dd HH:mm:ss\")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "inputWidgets": {}, - "nuid": "46f863c4-bb81-434a-a448-6b585031a221", - "showTitle": false, - "title": "" - } - }, - "source": [ - "### Define Anchors and Features\n", - "A feature is called an anchored feature when the feature is directly extracted from the source data, rather than computed on top of other features. The latter case is called derived feature." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "inputWidgets": {}, - "nuid": "a373ecbe-a040-4cd3-9d87-0d5f4c5ba553", - "showTitle": false, - "title": "" - } - }, - "outputs": [ - { - "data": { - "text/html": [ - "" - ] - }, - "metadata": { - "application/vnd.databricks.v1+output": { - "arguments": {}, - "data": "", - "errorSummary": "", - "errorTraceType": null, - "metadata": {}, - "type": "ipynbError" - } - }, - "output_type": "display_data" - } - ], - "source": [ - "f_trip_distance = Feature(name=\"f_trip_distance\",\n", - " feature_type=FLOAT, transform=\"trip_distance\")\n", - "\n", - "features = [\n", - " f_trip_distance,\n", - " Feature(name=\"f_is_long_trip_distance\",\n", - " feature_type=BOOLEAN,\n", - " transform=\"cast_float(trip_distance)>30\"),\n", - " Feature(name=\"f_day_of_week\",\n", - " feature_type=INT32,\n", - " transform=\"dayofweek(lpep_dropoff_datetime)\"),\n", - "]\n", - "\n", - "request_anchor = FeatureAnchor(name=\"request_features\",\n", - " source=INPUT_CONTEXT,\n", - " features=features)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "inputWidgets": {}, - "nuid": "149f85e2-fa3c-4895-b0c5-de5543ca9b6d", - "showTitle": false, - "title": "" - } - }, - "source": [ - "### Window aggregation features\n", - "\n", - "For window aggregation features, see the supported fields below:\n", - "\n", - "Note that the `agg_func` should be any of these:\n", - "\n", - "| Aggregation Type | Input Type | Description |\n", - "| --- | --- | --- |\n", - "|SUM, COUNT, MAX, MIN, AVG\t|Numeric|Applies the the numerical operation on the numeric inputs. |\n", - "|MAX_POOLING, MIN_POOLING, AVG_POOLING\t| Numeric Vector | Applies the max/min/avg operation on a per entry bassis for a given a collection of numbers.|\n", - "|LATEST| Any |Returns the latest not-null values from within the defined time window |\n", - "\n", - "\n", - "After you have defined features and sources, bring them together to build an anchor:\n", - "\n", - "\n", - "Note that if the data source is from the observation data, the `source` section should be `INPUT_CONTEXT` to indicate the source of those defined anchors." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "inputWidgets": {}, - "nuid": "05633bc3-9118-449b-9562-45fc437576c2", - "showTitle": false, - "title": "" - } - }, - "outputs": [ - { - "data": { - "text/html": [ - "" - ] - }, - "metadata": { - "application/vnd.databricks.v1+output": { - "arguments": {}, - "data": "", - "errorSummary": "", - "errorTraceType": null, - "metadata": {}, - "type": "ipynbError" - } - }, - "output_type": "display_data" - } - ], - "source": [ - "location_id = TypedKey(key_column=\"DOLocationID\",\n", - " key_column_type=ValueType.INT32,\n", - " description=\"location id in NYC\",\n", - " full_name=\"nyc_taxi.location_id\")\n", - "agg_features = [Feature(name=\"f_location_avg_fare\",\n", - " key=location_id,\n", - " feature_type=FLOAT,\n", - " transform=WindowAggTransformation(agg_expr=\"cast_float(fare_amount)\",\n", - " agg_func=\"AVG\",\n", - " window=\"90d\")),\n", - " Feature(name=\"f_location_max_fare\",\n", - " key=location_id,\n", - " feature_type=FLOAT,\n", - " transform=WindowAggTransformation(agg_expr=\"cast_float(fare_amount)\",\n", - " agg_func=\"MAX\",\n", - " window=\"90d\")),\n", - " ]\n", - "\n", - "agg_anchor = FeatureAnchor(name=\"aggregationFeatures\",\n", - " source=batch_source,\n", - " features=agg_features)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "inputWidgets": {}, - "nuid": "d2ecaca9-057e-4b36-811f-320f66f753ed", - "showTitle": false, - "title": "" - } - }, - "source": [ - "### Derived Features Section\n", - "Derived features are the features that are computed from other features. They could be computed from anchored features, or other derived features." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "inputWidgets": {}, - "nuid": "270fb11e-8a71-404f-9639-ad29d8e6a2c1", - "showTitle": false, - "title": "" - } - }, - "outputs": [ - { - "data": { - "text/html": [ - "" - ] - }, - "metadata": { - "application/vnd.databricks.v1+output": { - "arguments": {}, - "data": "", - "errorSummary": "", - "errorTraceType": null, - "metadata": {}, - "type": "ipynbError" - } - }, - "output_type": "display_data" - } - ], - "source": [ - "\n", - "f_trip_distance_rounded = DerivedFeature(name=\"f_trip_distance_rounded\",\n", - " feature_type=INT32,\n", - " input_features=[f_trip_distance],\n", - " transform=\"f_trip_distance * 10\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "inputWidgets": {}, - "nuid": "ad102c45-586d-468c-85f0-9454401ef10b", - "showTitle": false, - "title": "" - } - }, - "source": [ - "And then we need to build those features so that it can be consumed later. Note that we have to build both the \"anchor\" and the \"derived\" features (which is not anchored to a source)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "inputWidgets": {}, - "nuid": "91bb5ebb-87e4-470b-b8eb-1c89b351740e", - "showTitle": false, - "title": "" - } - }, - "outputs": [ - { - "data": { - "text/html": [ - "" - ] - }, - "metadata": { - "application/vnd.databricks.v1+output": { - "arguments": {}, - "data": "", - "errorSummary": "", - "errorTraceType": null, - "metadata": {}, - "type": "ipynbError" - } - }, - "output_type": "display_data" - } - ], - "source": [ - "client.build_features(anchor_list=[agg_anchor, request_anchor], derived_feature_list=[\n", - " f_trip_distance_rounded])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "inputWidgets": {}, - "nuid": "632d5f46-f9e2-41a8-aab7-34f75206e2aa", - "showTitle": false, - "title": "" - } - }, - "source": [ - "## Create training data using point-in-time correct feature join\n", - "\n", - "A training dataset usually contains entity id columns, multiple feature columns, event timestamp column and label/target column. \n", - "\n", - "To create a training dataset using Feathr, one needs to provide a feature join configuration file to specify\n", - "what features and how these features should be joined to the observation data. \n", - "\n", - "To learn more on this topic, please refer to [Point-in-time Correctness](https://github.com/linkedin/feathr/blob/main/docs/concepts/point-in-time-join.md)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "inputWidgets": {}, - "nuid": "e438e6d8-162e-4aa3-b3b3-9d1f3b0d2b7f", - "showTitle": false, - "title": "" - } - }, - "outputs": [ - { - "data": { - "text/html": [ - "" - ] - }, - "metadata": { - "application/vnd.databricks.v1+output": { - "arguments": {}, - "data": "", - "errorSummary": "", - "errorTraceType": null, - "metadata": {}, - "type": "ipynbError" - } - }, - "output_type": "display_data" - } - ], - "source": [ - "\n", - "output_path = 'dbfs:/feathrazure_test.avro'\n", - "\n", - "\n", - "feature_query = FeatureQuery(\n", - " feature_list=[\"f_location_avg_fare\", \"f_trip_distance_rounded\", \"f_is_long_trip_distance\"], key=location_id)\n", - "settings = ObservationSettings(\n", - " observation_path=\"wasbs://public@azurefeathrstorage.blob.core.windows.net/sample_data/green_tripdata_2020-04_with_index.csv\",\n", - " event_timestamp_column=\"lpep_dropoff_datetime\",\n", - " timestamp_format=\"yyyy-MM-dd HH:mm:ss\")\n", - "client.get_offline_features(observation_settings=settings,\n", - " feature_query=feature_query,\n", - " output_path=output_path\n", - " )\n", - "client.wait_job_to_finish(timeout_sec=500)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "inputWidgets": {}, - "nuid": "51f078e3-3f8f-4f10-b7f1-499ac8a9ff07", - "showTitle": false, - "title": "" - } - }, - "source": [ - "## Download the result and show the result\n", - "\n", - "Let's use the helper function `get_result_df` to download the result and view it:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "inputWidgets": {}, - "nuid": "23c797b2-ac1a-4cf3-b0ed-c05216de3f37", - "showTitle": false, - "title": "" - } - }, - "outputs": [ - { - "data": { - "text/html": [ - "" - ] - }, - "metadata": { - "application/vnd.databricks.v1+output": { - "arguments": {}, - "data": "", - "errorSummary": "", - "errorTraceType": null, - "metadata": {}, - "type": "ipynbError" - } - }, - "output_type": "display_data" - } - ], - "source": [ - "from feathr.utils.job_utils import get_result_df\n", - "df_res = get_result_df(client, format=\"avro\", res_url = output_path)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "inputWidgets": {}, - "nuid": "b9be042e-eb12-46b9-9d91-a0e5dd0c704f", - "showTitle": false, - "title": "" - } - }, - "outputs": [ - { - "data": { - "text/html": [ - "" - ] - }, - "metadata": { - "application/vnd.databricks.v1+output": { - "arguments": {}, - "data": "", - "errorSummary": "", - "errorTraceType": null, - "metadata": {}, - "type": "ipynbError" - } - }, - "output_type": "display_data" - } - ], - "source": [ - "df_res" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "inputWidgets": {}, - "nuid": "dcbf17fc-7f79-4a65-a3af-9cffbd0b5d1f", - "showTitle": false, - "title": "" - } - }, - "source": [ - "## Train a machine learning model\n", - "After getting all the features, let's train a machine learning model with the converted feature by Feathr:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "inputWidgets": {}, - "nuid": "84745f36-5bac-49c0-903b-38828b923c7c", - "showTitle": false, - "title": "" - } - }, - "outputs": [ - { - "data": { - "text/html": [ - "" - ] - }, - "metadata": { - "application/vnd.databricks.v1+output": { - "arguments": {}, - "data": "", - "errorSummary": "", - "errorTraceType": null, - "metadata": {}, - "type": "ipynbError" - } - }, - "output_type": "display_data" - } - ], - "source": [ - "# remove columns\n", - "from sklearn.ensemble import GradientBoostingRegressor\n", - "final_df = df_res\n", - "final_df.drop([\"lpep_pickup_datetime\", \"lpep_dropoff_datetime\",\n", - " \"store_and_fwd_flag\"], axis=1, inplace=True, errors='ignore')\n", - "final_df.fillna(0, inplace=True)\n", - "final_df['fare_amount'] = final_df['fare_amount'].astype(\"float64\")\n", - "\n", - "\n", - "train_x, test_x, train_y, test_y = train_test_split(final_df.drop([\"fare_amount\"], axis=1),\n", - " final_df[\"fare_amount\"],\n", - " test_size=0.2,\n", - " random_state=42)\n", - "model = GradientBoostingRegressor()\n", - "model.fit(train_x, train_y)\n", - "\n", - "y_predict = model.predict(test_x)\n", - "\n", - "y_actual = test_y.values.flatten().tolist()\n", - "rmse = sqrt(mean_squared_error(y_actual, y_predict))\n", - "\n", - "sum_actuals = sum_errors = 0\n", - "\n", - "for actual_val, predict_val in zip(y_actual, y_predict):\n", - " abs_error = actual_val - predict_val\n", - " if abs_error < 0:\n", - " abs_error = abs_error * -1\n", - "\n", - " sum_errors = sum_errors + abs_error\n", - " sum_actuals = sum_actuals + actual_val\n", - "\n", - "mean_abs_percent_error = sum_errors / sum_actuals\n", - "print(\"Model MAPE:\")\n", - "print(mean_abs_percent_error)\n", - "print()\n", - "print(\"Model Accuracy:\")\n", - "print(1 - mean_abs_percent_error)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "inputWidgets": {}, - "nuid": "5a226026-1c7b-48db-8f91-88d5c2ddf023", - "showTitle": false, - "title": "" - } - }, - "source": [ - "## Materialize feature value into offline/online storage\n", - "\n", - "While Feathr can compute the feature value from the feature definition on-the-fly at request time, it can also pre-compute\n", - "and materialize the feature value to offline and/or online storage. \n", - "\n", - "We can push the generated features to the online store like below:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "inputWidgets": {}, - "nuid": "3b924c66-8634-42fe-90f3-c844487d3f75", - "showTitle": false, - "title": "" - } - }, - "outputs": [ - { - "data": { - "text/html": [ - "" - ] - }, - "metadata": { - "application/vnd.databricks.v1+output": { - "arguments": {}, - "data": "", - "errorSummary": "", - "errorTraceType": null, - "metadata": {}, - "type": "ipynbError" - } - }, - "output_type": "display_data" - } - ], - "source": [ - "backfill_time = BackfillTime(start=datetime(\n", - " 2020, 5, 20), end=datetime(2020, 5, 20), step=timedelta(days=1))\n", - "redisSink = RedisSink(table_name=\"nycTaxiDemoFeature\")\n", - "settings = MaterializationSettings(\"nycTaxiTable\",\n", - " backfill_time=backfill_time,\n", - " sinks=[redisSink],\n", - " feature_names=[\"f_location_avg_fare\", \"f_location_max_fare\"])\n", - "\n", - "client.materialize_features(settings)\n", - "client.wait_job_to_finish(timeout_sec=500)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "inputWidgets": {}, - "nuid": "6a3e2ab1-5c66-4d27-a737-c5e2af03b1dd", - "showTitle": false, - "title": "" - } - }, - "source": [ - "We can then get the features from the online store (Redis):" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "inputWidgets": {}, - "nuid": "bef93538-9591-4247-97b6-289d2055b7b1", - "showTitle": false, - "title": "" - } - }, - "source": [ - "## Fetching feature value for online inference\n", - "\n", - "For features that are already materialized by the previous step, their latest value can be queried via the client's\n", - "`get_online_features` or `multi_get_online_features` API." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "inputWidgets": {}, - "nuid": "0c3d5f35-11a3-4644-9992-5860169d8302", - "showTitle": false, - "title": "" - } - }, - "outputs": [ - { - "data": { - "text/html": [ - "" - ] - }, - "metadata": { - "application/vnd.databricks.v1+output": { - "arguments": {}, - "data": "", - "errorSummary": "", - "errorTraceType": null, - "metadata": {}, - "type": "ipynbError" - } - }, - "output_type": "display_data" - } - ], - "source": [ - "res = client.get_online_features('nycTaxiDemoFeature', '265', [\n", - " 'f_location_avg_fare', 'f_location_max_fare'])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "inputWidgets": {}, - "nuid": "4d4699ed-42e6-408f-903d-2f799284f4b6", - "showTitle": false, - "title": "" - } - }, - "outputs": [ - { - "data": { - "text/html": [ - "" - ] - }, - "metadata": { - "application/vnd.databricks.v1+output": { - "arguments": {}, - "data": "", - "errorSummary": "", - "errorTraceType": null, - "metadata": {}, - "type": "ipynbError" - } - }, - "output_type": "display_data" - } - ], - "source": [ - "client.multi_get_online_features(\"nycTaxiDemoFeature\", [\"239\", \"265\"], [\n", - " 'f_location_avg_fare', 'f_location_max_fare'])" - ] - } - ], - "metadata": { - "application/vnd.databricks.v1+notebook": { - "dashboards": [], - "language": "python", - "notebookMetadata": { - "pythonIndentUnit": 4 - }, - "notebookName": "nyc_driver_demo", - "notebookOrigID": 930353059183053, - "widgets": {} - }, - "interpreter": { - "hash": "830c16c5b424e7ff512f67d4056b67cea1a756a7ad6a92c98b9e2b95c5e484ae" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.5" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/docs/samples/fraud_detection_demo.ipynb b/docs/samples/fraud_detection_demo.ipynb index 0f35bc3bb..1e57604ae 100644 --- a/docs/samples/fraud_detection_demo.ipynb +++ b/docs/samples/fraud_detection_demo.ipynb @@ -169,7 +169,7 @@ "adls_fs_name=resource_prefix+\"fs\"\n", "purview_name=resource_prefix+\"purview\"\n", "key_vault_uri = f\"https://{key_vault_name}.vault.azure.net\"\n", - "credential = DefaultAzureCredential(exclude_interactive_browser_credential=False)\n", + "credential = DefaultAzureCredential(exclude_interactive_browser_credential=False, additionally_allowed_tenants=['*'])\n", "client = SecretClient(vault_url=key_vault_uri, credential=credential)\n", "secretName = \"FEATHR-ONLINE-STORE-CONN\"\n", "retrieved_secret = client.get_secret(secretName).value\n", @@ -206,7 +206,7 @@ "source": [ "import tempfile\n", "yaml_config = \"\"\"\n", - "# Please refer to https://github.com/linkedin/feathr/blob/main/feathr_project/feathrcli/data/feathr_user_workspace/feathr_config.yaml for explanations on the meaning of each field.\n", + "# Please refer to https://github.com/feathr-ai/feathr/blob/main/feathr_project/feathrcli/data/feathr_user_workspace/feathr_config.yaml for explanations on the meaning of each field.\n", "api_version: 1\n", "project_config:\n", " project_name: 'fraud_detection_test'\n", @@ -230,6 +230,7 @@ " url: \".snowflakecomputing.com\"\n", " user: \"\"\n", " role: \"\"\n", + " warehouse: \"\"\n", "spark_config:\n", " spark_cluster: 'azure_synapse'\n", " spark_result_output_parts: '1'\n", @@ -899,7 +900,7 @@ " sinks=[redisSink],\n", " feature_names=[\"fraud_status\"])\n", "\n", - "client.materialize_features(settings)\n", + "client.materialize_features(settings, allow_materialize_non_agg_feature =True)\n", "client.wait_job_to_finish(timeout_sec=5000)" ] }, @@ -997,7 +998,7 @@ "widgets": {} }, "kernelspec": { - "display_name": "Python 3.10.4 64-bit", + "display_name": "Python 3.9.14 64-bit", "language": "python", "name": "python3" }, @@ -1011,12 +1012,12 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.4" + "version": "3.9.14" }, "orig_nbformat": 4, "vscode": { "interpreter": { - "hash": "6eea572ac5b43246b7c51fa33510c93fb6df4c34b515a6e4994c858f44841967" + "hash": "a665b5d41d17b532ea9890333293a1b812fa0b73c9c25c950b3cedf1bebd0438" } } }, diff --git a/docs/samples/nyc_taxi_demo.ipynb b/docs/samples/nyc_taxi_demo.ipynb new file mode 100644 index 000000000..31754950e --- /dev/null +++ b/docs/samples/nyc_taxi_demo.ipynb @@ -0,0 +1,1134 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "inputWidgets": {}, + "nuid": "384e5e16-7213-4186-9d04-09d03b155534", + "showTitle": false, + "title": "" + } + }, + "source": [ + "# Feathr Quick Start Notebook\n", + "\n", + "This notebook illustrates the use of Feathr Feature Store to create a model that predicts NYC Taxi fares. The dataset comes from [here](https://www1.nyc.gov/site/tlc/about/tlc-trip-record-data.page).\n", + "\n", + "The major problems Feathr solves are:\n", + "\n", + "1. Create, share and manage useful features from raw source data.\n", + "2. Provide Point-in-time feature join to create training dataset to ensure no data leakage.\n", + "3. Deploy the same feature data to online store to eliminate training and inference data skew." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Prerequisite\n", + "\n", + "Feathr has native cloud integration. First step is to provision required cloud resources if you want to use Feathr.\n", + "\n", + "Follow the [Feathr ARM deployment guide](https://feathr-ai.github.io/feathr/how-to-guides/azure-deployment-arm.html) to run Feathr on Azure. This allows you to quickly get started with automated deployment using Azure Resource Manager template. For more details, please refer [README.md](https://github.com/feathr-ai/feathr#%EF%B8%8F-running-feathr-on-cloud-with-a-few-simple-steps).\n", + "\n", + "Additionally, to run this notebook, you'll need to install `feathr` pip package. For local spark, simply run `pip install feathr` on the machine that runs this notebook. To use Databricks or Azure Synapse Analytics, please see dependency management documents:\n", + "- [Azure Databricks dependency management](https://learn.microsoft.com/en-us/azure/databricks/libraries/)\n", + "- [Azure Synapse Analytics dependency management](https://learn.microsoft.com/en-us/azure/synapse-analytics/spark/apache-spark-azure-portal-add-libraries)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Notebook Steps\n", + "\n", + "This tutorial demonstrates the key capabilities of Feathr, including:\n", + "\n", + "1. Install Feathr and necessary dependencies\n", + "2. Create shareable features with Feathr feature definition configs\n", + "3. Create training data using point-in-time correct feature join\n", + "4. Train a prediction model and evaluate the model and features\n", + "5. Register the features to share across teams\n", + "6. Materialize feature values for online scoring\n", + "\n", + "The overall data flow is as follows:\n", + "\n", + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. Install Feathr and Necessary Dependancies\n", + "\n", + "Install feathr and necessary packages by running `pip install feathr[notebook]` if you haven't installed them already." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%load_ext autoreload\n", + "%autoreload 2" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "inputWidgets": {}, + "nuid": "80223a02-631c-40c8-91b3-a037249ffff9", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "from datetime import timedelta\n", + "from math import sqrt\n", + "import os\n", + "from pathlib import Path\n", + "from tempfile import TemporaryDirectory\n", + "\n", + "from pyspark.ml import Pipeline\n", + "from pyspark.ml.evaluation import RegressionEvaluator\n", + "from pyspark.ml.feature import VectorAssembler\n", + "from pyspark.ml.regression import GBTRegressor\n", + "from pyspark.sql import DataFrame, SparkSession\n", + "import pyspark.sql.functions as F\n", + "\n", + "import feathr\n", + "from feathr import (\n", + " FeathrClient,\n", + " # Feature data types\n", + " BOOLEAN, FLOAT, INT32, ValueType,\n", + " # Feature data sources\n", + " INPUT_CONTEXT, HdfsSource,\n", + " # Feature aggregations\n", + " TypedKey, WindowAggTransformation,\n", + " # Feature types and anchor\n", + " DerivedFeature, Feature, FeatureAnchor,\n", + " # Materialization\n", + " BackfillTime, MaterializationSettings, RedisSink,\n", + " # Offline feature computation\n", + " FeatureQuery, ObservationSettings,\n", + ")\n", + "from feathr.datasets import nyc_taxi\n", + "from feathr.spark_provider.feathr_configurations import SparkExecutionConfiguration\n", + "from feathr.utils.config import generate_config\n", + "from feathr.utils.job_utils import get_result_df\n", + "from feathr.utils.platform import is_databricks, is_jupyter\n", + "\n", + "print(f\"Feathr version: {feathr.__version__}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2. Create Shareable Features with Feathr Feature Definition Configs\n", + "\n", + "First, we define all the necessary resource key values for authentication. These values are retrieved by using [Azure Key Vault](https://azure.microsoft.com/en-us/services/key-vault/) cloud key value store. For authentication, we use Azure CLI credential in this notebook, but you may add secrets' list and get permission for the necessary service principal instead of running `az login --use-device-code`.\n", + "\n", + "Please refer to [A note on using azure key vault to store credentials](https://github.com/feathr-ai/feathr/blob/41e7496b38c43af6d7f8f1de842f657b27840f6d/docs/how-to-guides/feathr-configuration-and-env.md#a-note-on-using-azure-key-vault-to-store-credentials) for more details." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "RESOURCE_PREFIX = None # TODO fill the value used to deploy the resources via ARM template\n", + "PROJECT_NAME = \"feathr_getting_started\"\n", + "\n", + "# Currently support: 'azure_synapse', 'databricks', and 'local' \n", + "SPARK_CLUSTER = \"local\"\n", + "\n", + "# TODO fill values to use databricks cluster:\n", + "DATABRICKS_CLUSTER_ID = None # Set Databricks cluster id to use an existing cluster\n", + "DATABRICKS_URL = None # Set Databricks workspace url to use databricks\n", + "\n", + "# TODO fill values to use Azure Synapse cluster:\n", + "AZURE_SYNAPSE_SPARK_POOL = None # Set Azure Synapse Spark pool name\n", + "AZURE_SYNAPSE_URL = None # Set Azure Synapse workspace url to use Azure Synapse\n", + "\n", + "# Data store root path. Could be a local file system path, dbfs or Azure storage path like abfs or wasbs\n", + "DATA_STORE_PATH = TemporaryDirectory().name\n", + "\n", + "# Feathr config file path to use an existing file\n", + "FEATHR_CONFIG_PATH = None\n", + "\n", + "# If set True, use an interactive browser authentication to get the redis password.\n", + "USE_CLI_AUTH = False\n", + "\n", + "REGISTER_FEATURES = False\n", + "\n", + "# (For the notebook test pipeline) If true, use ScrapBook package to collect the results.\n", + "SCRAP_RESULTS = False" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To use Databricks as the feathr client's target platform, you may need to set a databricks token to an environment variable like:\n", + "\n", + "`export DATABRICKS_WORKSPACE_TOKEN_VALUE=your-token`\n", + "\n", + "or in the notebook cell,\n", + "\n", + "`os.environ[\"DATABRICKS_WORKSPACE_TOKEN_VALUE\"] = your-token`\n", + "\n", + "If you are running this notebook on Databricks, the token will be automatically retrieved by using the current Databricks notebook context.\n", + "\n", + "On the other hand, to use Azure Synapse cluster, you have to specify the synapse workspace storage key:\n", + "\n", + "`export ADLS_KEY=your-key`\n", + "\n", + "or in the notebook cell,\n", + "\n", + "`os.environ[\"ADLS_KEY\"] = your-key`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "if SPARK_CLUSTER == \"azure_synapse\" and not os.environ.get(\"ADLS_KEY\"):\n", + " os.environ[\"ADLS_KEY\"] = add_your_key_here\n", + "elif SPARK_CLUSTER == \"databricks\" and not os.environ.get(\"DATABRICKS_WORKSPACE_TOKEN_VALUE\"):\n", + " os.environ[\"DATABRICKS_WORKSPACE_TOKEN_VALUE\"] = add_your_token_here" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Force to use dbfs if the notebook is running on Databricks\n", + "if is_databricks() and not DATA_STORE_PATH.startswith(\"dbfs:\"):\n", + " DATA_STORE_PATH = f\"dbfs:/{DATA_STORE_PATH.lstrip('/')}\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "if USE_CLI_AUTH:\n", + " !az login --use-device-code" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Redis password\n", + "if 'REDIS_PASSWORD' not in os.environ:\n", + " # Try to get all the required credentials from Azure Key Vault\n", + " from azure.identity import AzureCliCredential, DefaultAzureCredential \n", + " from azure.keyvault.secrets import SecretClient\n", + "\n", + " vault_url = f\"https://{RESOURCE_PREFIX}kv.vault.azure.net\"\n", + " if USE_CLI_AUTH:\n", + " credential = AzureCliCredential(additionally_allowed_tenants=['*'],)\n", + " else:\n", + " credential = DefaultAzureCredential(\n", + " exclude_interactive_browser_credential=False,\n", + " additionally_allowed_tenants=['*'],\n", + " )\n", + " secret_client = SecretClient(vault_url=vault_url, credential=credential)\n", + " retrieved_secret = secret_client.get_secret('FEATHR-ONLINE-STORE-CONN').value\n", + " os.environ['REDIS_PASSWORD'] = retrieved_secret.split(\",\")[1].split(\"password=\", 1)[1]\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "inputWidgets": {}, + "nuid": "41d3648a-9bc9-40dc-90da-bc82b21ef9b3", + "showTitle": false, + "title": "" + } + }, + "source": [ + "### Configurations\n", + "\n", + "Feathr uses a yaml file to define configurations. Please refer to [feathr_config.yaml]( https://github.com//feathr-ai/feathr/blob/main/feathr_project/feathrcli/data/feathr_user_workspace/feathr_config.yaml) for the meaning of each field." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "inputWidgets": {}, + "nuid": "8cd64e3a-376c-48e6-ba41-5197f3591d48", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "if FEATHR_CONFIG_PATH:\n", + " config_path = FEATHR_CONFIG_PATH\n", + "else:\n", + " config_path = generate_config(\n", + " resource_prefix=RESOURCE_PREFIX,\n", + " project_name=PROJECT_NAME,\n", + " spark_config__spark_cluster=SPARK_CLUSTER,\n", + " spark_config__azure_synapse__dev_url=AZURE_SYNAPSE_URL,\n", + " spark_config__azure_synapse__pool_name=AZURE_SYNAPSE_SPARK_POOL,\n", + " spark_config__databricks__workspace_instance_url=DATABRICKS_URL,\n", + " databricks_cluster_id=DATABRICKS_CLUSTER_ID,\n", + " )\n", + "\n", + "with open(config_path, 'r') as f: \n", + " print(f.read())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "All the configurations can be overwritten by environment variables with concatenation of `__` for different layers of the config file. For example, `feathr_runtime_location` for databricks config can be overwritten by setting `spark_config__databricks__feathr_runtime_location` environment variable." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "inputWidgets": {}, + "nuid": "3fef7f2f-df19-4f53-90a5-ff7999ed983d", + "showTitle": false, + "title": "" + } + }, + "source": [ + "### Initialize Feathr client" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "inputWidgets": {}, + "nuid": "9713a2df-c7b2-4562-88b0-b7acce3cc43a", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "client = FeathrClient(config_path=config_path)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "inputWidgets": {}, + "nuid": "c3b64bda-d42c-4a64-b976-0fb604cf38c5", + "showTitle": false, + "title": "" + } + }, + "source": [ + "### Prepare the NYC taxi fare dataset" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# If the notebook is runnong on Jupyter, start a spark session:\n", + "if is_jupyter():\n", + " spark = (\n", + " SparkSession\n", + " .builder\n", + " .appName(\"feathr\")\n", + " .config(\"spark.jars.packages\", \"org.apache.spark:spark-avro_2.12:3.3.0,io.delta:delta-core_2.12:2.1.1\")\n", + " .config(\"spark.sql.extensions\", \"io.delta.sql.DeltaSparkSessionExtension\")\n", + " .config(\"spark.sql.catalog.spark_catalog\", \"org.apache.spark.sql.delta.catalog.DeltaCatalog\")\n", + " .config(\"spark.ui.port\", \"8080\") # Set ui port other than the default one (4040) so that feathr spark job doesn't fail. \n", + " .getOrCreate()\n", + " )\n", + "\n", + "# Else, you must already have a spark session object available in databricks or synapse notebooks." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "inputWidgets": {}, + "nuid": "c4ccd7b3-298a-4e5a-8eec-b7e309db393e", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "DATA_FILE_PATH = str(Path(DATA_STORE_PATH, \"nyc_taxi.csv\"))\n", + "\n", + "# Download the data file\n", + "df_raw = nyc_taxi.get_spark_df(spark=spark, local_cache_path=DATA_FILE_PATH)\n", + "df_raw.limit(5).toPandas()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "inputWidgets": {}, + "nuid": "7430c942-64e5-4b70-b823-16ce1d1b3cee", + "showTitle": false, + "title": "" + } + }, + "source": [ + "### Defining features with Feathr\n", + "\n", + "In Feathr, a feature is viewed as a function, mapping a key and timestamp to a feature value. For more details, please see [Feathr Feature Definition Guide](https://github.com/feathr-ai/feathr/blob/main/docs/concepts/feature-definition.md).\n", + "\n", + "* The feature key (a.k.a. entity id) identifies the subject of feature, e.g. a user_id or location_id.\n", + "* The feature name is the aspect of the entity that the feature is indicating, e.g. the age of the user.\n", + "* The feature value is the actual value of that aspect at a particular time, e.g. the value is 30 at year 2022.\n", + "\n", + "Note that, in some cases, a feature could be just a transformation function that has no entity key or timestamp involved, e.g. *the day of week of the request timestamp*.\n", + "\n", + "There are two types of features -- anchored features and derivated features:\n", + "\n", + "* **Anchored features**: Features that are directly extracted from sources. Could be with or without aggregation. \n", + "* **Derived features**: Features that are computed on top of other features.\n", + "\n", + "#### Define anchored features\n", + "\n", + "A feature source is needed for anchored features that describes the raw data in which the feature values are computed from. A source value should be either `INPUT_CONTEXT` (the features that will be extracted from the observation data directly) or `feathr.source.Source` object." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "TIMESTAMP_COL = \"lpep_dropoff_datetime\"\n", + "TIMESTAMP_FORMAT = \"yyyy-MM-dd HH:mm:ss\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "inputWidgets": {}, + "nuid": "a373ecbe-a040-4cd3-9d87-0d5f4c5ba553", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "# We define f_trip_distance and f_trip_time_duration features separately\n", + "# so that we can reuse them later for the derived features.\n", + "f_trip_distance = Feature(\n", + " name=\"f_trip_distance\",\n", + " feature_type=FLOAT,\n", + " transform=\"trip_distance\",\n", + ")\n", + "f_trip_time_duration = Feature(\n", + " name=\"f_trip_time_duration\",\n", + " feature_type=FLOAT,\n", + " transform=\"cast_float((to_unix_timestamp(lpep_dropoff_datetime) - to_unix_timestamp(lpep_pickup_datetime)) / 60)\",\n", + ")\n", + "\n", + "features = [\n", + " f_trip_distance,\n", + " f_trip_time_duration,\n", + " Feature(\n", + " name=\"f_is_long_trip_distance\",\n", + " feature_type=BOOLEAN,\n", + " transform=\"trip_distance > 30.0\",\n", + " ),\n", + " Feature(\n", + " name=\"f_day_of_week\",\n", + " feature_type=INT32,\n", + " transform=\"dayofweek(lpep_dropoff_datetime)\",\n", + " ),\n", + " Feature(\n", + " name=\"f_day_of_month\",\n", + " feature_type=INT32,\n", + " transform=\"dayofmonth(lpep_dropoff_datetime)\",\n", + " ),\n", + " Feature(\n", + " name=\"f_hour_of_day\",\n", + " feature_type=INT32,\n", + " transform=\"hour(lpep_dropoff_datetime)\",\n", + " ),\n", + "]\n", + "\n", + "# After you have defined features, bring them together to build the anchor to the source.\n", + "feature_anchor = FeatureAnchor(\n", + " name=\"feature_anchor\",\n", + " source=INPUT_CONTEXT, # Pass through source, i.e. observation data.\n", + " features=features,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can define the source with a preprocessing python function. In order to make the source data accessible from the target spark cluster, we upload the data file into either DBFS or Azure Blob Storage if needed." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Define data source path\n", + "if client.spark_runtime == \"local\" or (client.spark_runtime == \"databricks\" and is_databricks()):\n", + " # In local mode, we can use the same data path as the source.\n", + " # If the notebook is running on databricks, DATA_FILE_PATH should be already a dbfs path.\n", + " data_source_path = DATA_FILE_PATH\n", + "else:\n", + " # Otherwise, upload the local file to the cloud storage (either dbfs or adls).\n", + " data_source_path = client.feathr_spark_launcher.upload_or_get_cloud_path(DATA_FILE_PATH) " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def preprocessing(df: DataFrame) -> DataFrame:\n", + " import pyspark.sql.functions as F\n", + " df = df.withColumn(\"fare_amount_cents\", (F.col(\"fare_amount\") * 100.0).cast(\"float\"))\n", + " return df\n", + "\n", + "batch_source = HdfsSource(\n", + " name=\"nycTaxiBatchSource\",\n", + " path=data_source_path,\n", + " event_timestamp_column=TIMESTAMP_COL,\n", + " preprocessing=preprocessing,\n", + " timestamp_format=TIMESTAMP_FORMAT,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "For the features with aggregation, the supported functions are as follows:\n", + "\n", + "| Aggregation Function | Input Type | Description |\n", + "| --- | --- | --- |\n", + "|SUM, COUNT, MAX, MIN, AVG\t|Numeric|Applies the the numerical operation on the numeric inputs. |\n", + "|MAX_POOLING, MIN_POOLING, AVG_POOLING\t| Numeric Vector | Applies the max/min/avg operation on a per entry bassis for a given a collection of numbers.|\n", + "|LATEST| Any |Returns the latest not-null values from within the defined time window |" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "agg_key = TypedKey(\n", + " key_column=\"DOLocationID\",\n", + " key_column_type=ValueType.INT32,\n", + " description=\"location id in NYC\",\n", + " full_name=\"nyc_taxi.location_id\",\n", + ")\n", + "\n", + "agg_window = \"90d\"\n", + "\n", + "# Anchored features with aggregations\n", + "agg_features = [\n", + " Feature(\n", + " name=\"f_location_avg_fare\",\n", + " key=agg_key,\n", + " feature_type=FLOAT,\n", + " transform=WindowAggTransformation(\n", + " agg_expr=\"fare_amount_cents\",\n", + " agg_func=\"AVG\",\n", + " window=agg_window,\n", + " ),\n", + " ),\n", + " Feature(\n", + " name=\"f_location_max_fare\",\n", + " key=agg_key,\n", + " feature_type=FLOAT,\n", + " transform=WindowAggTransformation(\n", + " agg_expr=\"fare_amount_cents\",\n", + " agg_func=\"MAX\",\n", + " window=agg_window,\n", + " ),\n", + " ),\n", + "]\n", + "\n", + "agg_feature_anchor = FeatureAnchor(\n", + " name=\"agg_feature_anchor\",\n", + " source=batch_source, # External data source for feature. Typically a data table.\n", + " features=agg_features,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "inputWidgets": {}, + "nuid": "d2ecaca9-057e-4b36-811f-320f66f753ed", + "showTitle": false, + "title": "" + } + }, + "source": [ + "#### Define derived features\n", + "\n", + "We also define a derived feature, `f_trip_time_distance`, from the anchored features `f_trip_distance` and `f_trip_time_duration` as follows:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "inputWidgets": {}, + "nuid": "270fb11e-8a71-404f-9639-ad29d8e6a2c1", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "derived_features = [\n", + " DerivedFeature(\n", + " name=\"f_trip_time_distance\",\n", + " feature_type=FLOAT,\n", + " input_features=[\n", + " f_trip_distance,\n", + " f_trip_time_duration,\n", + " ],\n", + " transform=\"f_trip_distance / f_trip_time_duration\",\n", + " )\n", + "]" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "inputWidgets": {}, + "nuid": "ad102c45-586d-468c-85f0-9454401ef10b", + "showTitle": false, + "title": "" + } + }, + "source": [ + "### Build features\n", + "\n", + "Finally, we build the features." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "inputWidgets": {}, + "nuid": "91bb5ebb-87e4-470b-b8eb-1c89b351740e", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "client.build_features(\n", + " anchor_list=[feature_anchor, agg_feature_anchor],\n", + " derived_feature_list=derived_features,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "inputWidgets": {}, + "nuid": "632d5f46-f9e2-41a8-aab7-34f75206e2aa", + "showTitle": false, + "title": "" + } + }, + "source": [ + "## 3. Create Training Data Using Point-in-Time Correct Feature Join\n", + "\n", + "After the feature producers have defined the features (as described in the Feature Definition part), the feature consumers may want to consume those features. Feature consumers will use observation data to query from different feature tables using Feature Query.\n", + "\n", + "To create a training dataset using Feathr, one needs to provide a feature join configuration file to specify\n", + "what features and how these features should be joined to the observation data. \n", + "\n", + "To learn more on this topic, please refer to [Point-in-time Correctness](https://github.com//feathr-ai/feathr/blob/main/docs/concepts/point-in-time-join.md)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "feature_names = [feature.name for feature in features + agg_features + derived_features]\n", + "feature_names" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "DATA_FORMAT = \"parquet\"\n", + "offline_features_path = str(Path(DATA_STORE_PATH, \"feathr_output\", f\"features.{DATA_FORMAT}\"))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "inputWidgets": {}, + "nuid": "e438e6d8-162e-4aa3-b3b3-9d1f3b0d2b7f", + "showTitle": false, + "title": "" + }, + "scrolled": false + }, + "outputs": [], + "source": [ + "# Features that we want to request. Can use a subset of features\n", + "query = FeatureQuery(\n", + " feature_list=feature_names,\n", + " key=agg_key,\n", + ")\n", + "settings = ObservationSettings(\n", + " observation_path=data_source_path,\n", + " event_timestamp_column=TIMESTAMP_COL,\n", + " timestamp_format=TIMESTAMP_FORMAT,\n", + ")\n", + "client.get_offline_features(\n", + " observation_settings=settings,\n", + " feature_query=query,\n", + " # For more details, see https://feathr-ai.github.io/feathr/how-to-guides/feathr-job-configuration.html\n", + " execution_configurations=SparkExecutionConfiguration({\n", + " \"spark.feathr.outputFormat\": DATA_FORMAT,\n", + " }),\n", + " output_path=offline_features_path,\n", + ")\n", + "\n", + "client.wait_job_to_finish(timeout_sec=1000)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Show feature results\n", + "df = get_result_df(\n", + " spark=spark,\n", + " client=client,\n", + " data_format=DATA_FORMAT,\n", + " res_url=offline_features_path,\n", + ")\n", + "df.select(feature_names).limit(5).toPandas()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "inputWidgets": {}, + "nuid": "dcbf17fc-7f79-4a65-a3af-9cffbd0b5d1f", + "showTitle": false, + "title": "" + } + }, + "source": [ + "## 4. Train a Prediction Model and Evaluate the Features\n", + "\n", + "After generating all the features, we train and evaluate a machine learning model to predict the NYC taxi fare prediction. In this example, we use Spark MLlib's [GBTRegressor](https://spark.apache.org/docs/latest/ml-classification-regression.html#gradient-boosted-tree-regression).\n", + "\n", + "Note that designing features, training prediction models and evaluating them are an iterative process where the models' performance maybe used to modify the features as a part of the modeling process." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Load Train and Test Data from the Offline Feature Values" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Train / test split\n", + "train_df, test_df = (\n", + " df # Dataframe that we generated from get_offline_features call.\n", + " .withColumn(\"label\", F.col(\"fare_amount\").cast(\"double\"))\n", + " .where(F.col(\"f_trip_time_duration\") > 0)\n", + " .fillna(0)\n", + " .randomSplit([0.8, 0.2])\n", + ")\n", + "\n", + "print(f\"Num train samples: {train_df.count()}\")\n", + "print(f\"Num test samples: {test_df.count()}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Build a ML Pipeline\n", + "\n", + "Here, we use Spark ML Pipeline to aggregate feature vectors and feed them to the model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Generate a feature vector column for SparkML\n", + "vector_assembler = VectorAssembler(\n", + " inputCols=[x for x in df.columns if x in feature_names],\n", + " outputCol=\"features\",\n", + ")\n", + "\n", + "# Define a model\n", + "gbt = GBTRegressor(\n", + " featuresCol=\"features\",\n", + " maxIter=100,\n", + " maxDepth=5,\n", + " maxBins=16,\n", + ")\n", + "\n", + "# Create a ML pipeline\n", + "ml_pipeline = Pipeline(stages=[\n", + " vector_assembler,\n", + " gbt,\n", + "])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Train and Evaluate the Model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Train a model\n", + "model = ml_pipeline.fit(train_df)\n", + "\n", + "# Make predictions\n", + "predictions = model.transform(test_df)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Evaluate\n", + "evaluator = RegressionEvaluator(\n", + " labelCol=\"label\",\n", + " predictionCol=\"prediction\",\n", + ")\n", + "\n", + "rmse = evaluator.evaluate(predictions, {evaluator.metricName: \"rmse\"})\n", + "mae = evaluator.evaluate(predictions, {evaluator.metricName: \"mae\"})\n", + "print(f\"RMSE: {rmse}\\nMAE: {mae}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# predicted fare vs actual fare plots -- will this work for databricks / synapse / local ?\n", + "predictions_pdf = predictions.select([\"label\", \"prediction\"]).toPandas().reset_index()\n", + "\n", + "predictions_pdf.plot(\n", + " x=\"index\",\n", + " y=[\"label\", \"prediction\"],\n", + " style=['-', ':'],\n", + " figsize=(20, 10),\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "predictions_pdf.plot.scatter(\n", + " x=\"label\",\n", + " y=\"prediction\",\n", + " xlim=(0, 100),\n", + " ylim=(0, 100),\n", + " figsize=(10, 10),\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 5. Register the Features to Share Across Teams\n", + "\n", + "You can register your features in the centralized registry and share the corresponding project with other team members who want to consume those features and for further use." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "if REGISTER_FEATURES:\n", + " try:\n", + " client.register_features()\n", + " except KeyError:\n", + " # TODO temporarily go around the \"Already exists\" error\n", + " pass \n", + " print(client.list_registered_features(project_name=PROJECT_NAME))\n", + " # You can get the actual features too by calling client.get_features_from_registry(PROJECT_NAME)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "inputWidgets": {}, + "nuid": "5a226026-1c7b-48db-8f91-88d5c2ddf023", + "showTitle": false, + "title": "" + } + }, + "source": [ + "## 6. Materialize Feature Values for Online Scoring\n", + "\n", + "While we computed feature values on-the-fly at request time via Feathr, we can pre-compute the feature values and materialize them to offline or online storages such as Redis.\n", + "\n", + "Note, only the features anchored to offline data source can be materialized." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Get the last date from the dataset\n", + "backfill_timestamp = (\n", + " df_raw\n", + " .select(F.to_timestamp(F.col(TIMESTAMP_COL), TIMESTAMP_FORMAT).alias(TIMESTAMP_COL))\n", + " .agg({TIMESTAMP_COL: \"max\"})\n", + " .collect()[0][0]\n", + ")\n", + "backfill_timestamp" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "inputWidgets": {}, + "nuid": "3b924c66-8634-42fe-90f3-c844487d3f75", + "showTitle": false, + "title": "" + }, + "scrolled": false + }, + "outputs": [], + "source": [ + "FEATURE_TABLE_NAME = \"nycTaxiDemoFeature\"\n", + "\n", + "# Time range to materialize\n", + "backfill_time = BackfillTime(\n", + " start=backfill_timestamp,\n", + " end=backfill_timestamp,\n", + " step=timedelta(days=1),\n", + ")\n", + "\n", + "# Destinations:\n", + "# For online store,\n", + "redis_sink = RedisSink(table_name=FEATURE_TABLE_NAME)\n", + "\n", + "# For offline store,\n", + "# adls_sink = HdfsSink(output_path=)\n", + "\n", + "settings = MaterializationSettings(\n", + " name=FEATURE_TABLE_NAME + \".job\", # job name\n", + " backfill_time=backfill_time,\n", + " sinks=[redis_sink], # or adls_sink\n", + " feature_names=[feature.name for feature in agg_features],\n", + ")\n", + "\n", + "client.materialize_features(\n", + " settings=settings,\n", + " execution_configurations={\"spark.feathr.outputFormat\": \"parquet\"},\n", + ")\n", + "\n", + "client.wait_job_to_finish(timeout_sec=1000)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now, you can retrieve features for online scoring as follows:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Note, to get a single key, you may use client.get_online_features instead\n", + "materialized_feature_values = client.multi_get_online_features(\n", + " feature_table=FEATURE_TABLE_NAME,\n", + " keys=[\"239\", \"265\"],\n", + " feature_names=[feature.name for feature in agg_features],\n", + ")\n", + "materialized_feature_values" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Cleanup" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# TODO: Unregister, delete cached files or do any other cleanups." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Stop the spark session if it is a local session.\n", + "if is_jupyter():\n", + " spark.stop()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Scrap Variables for Testing" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "if SCRAP_RESULTS:\n", + " # Record results for test pipelines\n", + " import scrapbook as sb\n", + " sb.glue(\"materialized_feature_values\", materialized_feature_values)\n", + " sb.glue(\"rmse\", rmse)\n", + " sb.glue(\"mae\", mae)" + ] + } + ], + "metadata": { + "application/vnd.databricks.v1+notebook": { + "dashboards": [], + "language": "python", + "notebookMetadata": { + "pythonIndentUnit": 4 + }, + "notebookName": "nyc_driver_demo", + "notebookOrigID": 930353059183053, + "widgets": {} + }, + "celltoolbar": "Tags", + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.4" + }, + "vscode": { + "interpreter": { + "hash": "e34a1a57d2e174682770a82d94a178aa36d3ccfaa21227c5d2308e319b7ae532" + } + } + }, + "nbformat": 4, + "nbformat_minor": 1 +} diff --git a/docs/samples/product_recommendation_demo_advanced.ipynb b/docs/samples/product_recommendation_demo_advanced.ipynb index b03dccac6..aafbdf0f0 100644 --- a/docs/samples/product_recommendation_demo_advanced.ipynb +++ b/docs/samples/product_recommendation_demo_advanced.ipynb @@ -20,7 +20,7 @@ "\n", "After the model is trained, given a user_id, product_id pair and features, we should be able to predict the product rating that the user will give for this product_id.\n", "\n", - "(Compared with [the beginner version of product recommendation](product_recommendation_demo.ipynb), this tutorial expanded the example by predicting ratings for all products.)\n", + "(Compared with [the beginner version of product recommendation](https://github.com/feathr-ai/feathr/blob/main/docs/samples/azure_synapse/product_recommendation_demo.ipynb), this tutorial expanded the example by predicting ratings for all products.)\n", "\n", "## Feature Creation Illustration\n", "In this example, our observation data has compound entity key where a record is uniquely identified by user_id and product_id. So there might be 3 types of features:\n", @@ -31,7 +31,7 @@ "We will focus on the first two in our example.\n", "\n", "The feature creation flow is as below:\n", - "![Feature Flow](https://github.com/linkedin/feathr/blob/main/docs/images/product_recommendation_advanced.jpg?raw=true)" + "![Feature Flow](https://github.com/feathr-ai/feathr/blob/main/docs/images/product_recommendation_advanced.jpg?raw=true)" ] }, { @@ -49,10 +49,10 @@ "\n", "First step is to provision required cloud resources if you want to use Feathr. Feathr provides a python based client to interact with cloud resources.\n", "\n", - "Please follow the steps [here](https://feathr-ai.github.io/feathr/how-to-guides/azure-deployment-arm.html) to provision required cloud resources. Due to the complexity of the possible cloud environment, it is almost impossible to create a script that works for all the use cases. Because of this, [azure_resource_provision.sh](https://github.com/linkedin/feathr/blob/main/docs/how-to-guides/azure_resource_provision.sh) is a full end to end command line to create all the required resources, and you can tailor the script as needed, while [the companion documentation](https://feathr-ai.github.io/feathr/how-to-guides/azure-deployment-cli.html) can be used as a complete guide for using that shell script. \n", + "Please follow the steps [here](https://feathr-ai.github.io/feathr/how-to-guides/azure-deployment-arm.html) to provision required cloud resources. Due to the complexity of the possible cloud environment, it is almost impossible to create a script that works for all the use cases. Because of this, [azure_resource_provision.sh](https://github.com/feathr-ai/feathr/blob/main/docs/how-to-guides/azure_resource_provision.sh) is a full end to end command line to create all the required resources, and you can tailor the script as needed, while [the companion documentation](https://feathr-ai.github.io/feathr/how-to-guides/azure-deployment-cli.html) can be used as a complete guide for using that shell script. \n", "\n", "\n", - "![Architecture](https://github.com/linkedin/feathr/blob/main/docs/images/architecture.png?raw=true)" + "![Architecture](https://github.com/feathr-ai/feathr/blob/main/docs/images/architecture.png?raw=true)" ] }, { @@ -135,6 +135,13 @@ "! pip install feathr azure-cli pandavro scikit-learn\n" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "When running this notebook in synapse, you may get some errors or blocks installing above packages in one cell. Suggest to try installing them in seperate cells if meet some issues. Eg. ! pip install feathr, ! pip install azure-cli , ! pip install pandavro, ! pip install scikit-learn" + ] + }, { "cell_type": "markdown", "metadata": { @@ -201,6 +208,13 @@ "from azure.keyvault.secrets import SecretClient\n" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "If you meet errors like 'cannot import FeatherClient from feathr', it may be caused by incompatible version of 'aiohttp'. Please try to install/upgrade it by running: '! pip install -U aiohttp' or '! pip install aiohttp==3.8.3'" + ] + }, { "cell_type": "markdown", "metadata": { @@ -270,7 +284,7 @@ "adls_fs_name=resource_prefix+\"fs\"\n", "purview_name=resource_prefix+\"purview\"\n", "key_vault_uri = f\"https://{key_vault_name}.vault.azure.net\"\n", - "credential = DefaultAzureCredential(exclude_interactive_browser_credential=False)\n", + "credential = DefaultAzureCredential(exclude_interactive_browser_credential=False, additionally_allowed_tenants=['*'])\n", "client = SecretClient(vault_url=key_vault_uri, credential=credential)\n", "secretName = \"FEATHR-ONLINE-STORE-CONN\"\n", "retrieved_secret = client.get_secret(secretName).value\n", @@ -307,7 +321,7 @@ "\n", "In the first step (Provision cloud resources), you should have provisioned all the required cloud resources. If you use Feathr CLI to create a workspace, you should have a folder with a file called `feathr_config.yaml` in it with all the required configurations. Otherwise, update the configuration below.\n", "\n", - "The code below will write this configuration string to a temporary location and load it to Feathr. Please still refer to [feathr_config.yaml](https://github.com/linkedin/feathr/blob/main/feathr_project/feathrcli/data/feathr_user_workspace/feathr_config.yaml) and use that as the source of truth. It should also have more explanations on the meaning of each variable." + "The code below will write this configuration string to a temporary location and load it to Feathr. Please still refer to [feathr_config.yaml](https://github.com/feathr-ai/feathr/blob/main/feathr_project/feathrcli/data/feathr_user_workspace/feathr_config.yaml) and use that as the source of truth. It should also have more explanations on the meaning of each variable." ] }, { @@ -325,7 +339,7 @@ "source": [ "import tempfile\n", "yaml_config = \"\"\"\n", - "# Please refer to https://github.com/linkedin/feathr/blob/main/feathr_project/feathrcli/data/feathr_user_workspace/feathr_config.yaml for explanations on the meaning of each field.\n", + "# Please refer to https://github.com/feathr-ai/feathr/blob/main/feathr_project/feathrcli/data/feathr_user_workspace/feathr_config.yaml for explanations on the meaning of each field.\n", "api_version: 1\n", "project_config:\n", " project_name: 'feathr_getting_started'\n", @@ -348,6 +362,7 @@ " url: \".snowflakecomputing.com\"\n", " user: \"\"\n", " role: \"\"\n", + " warehouse: \"\"\n", "spark_config:\n", " spark_cluster: 'azure_synapse'\n", " spark_result_output_parts: '1'\n", @@ -387,7 +402,7 @@ "source": [ "## Setup necessary environment variables (Skip if using the above Quick Start Template)\n", "\n", - "You should setup the environment variables in order to run this sample. More environment variables can be set by referring to [feathr_config.yaml](https://github.com/linkedin/feathr/blob/main/feathr_project/feathrcli/data/feathr_user_workspace/feathr_config.yaml) and use that as the source of truth. It also has more explanations on the meaning of each variable.\n", + "You should setup the environment variables in order to run this sample. More environment variables can be set by referring to [feathr_config.yaml](https://github.com/feathr-ai/feathr/blob/main/feathr_project/feathrcli/data/feathr_user_workspace/feathr_config.yaml) and use that as the source of truth. It also has more explanations on the meaning of each variable.\n", "\n", "To run this notebook, for Azure users, you need REDIS_PASSWORD.\n", "To run this notebook, for Databricks useres, you need DATABRICKS_WORKSPACE_TOKEN_VALUE and REDIS_PASSWORD." @@ -528,7 +543,7 @@ "source": [ "## Defining Features with Feathr\n", "Let's try to create features from those raw source data.\n", - "In Feathr, a feature is viewed as a function, mapping from entity id or key, and timestamp to a feature value. For more details on feature definition, please refer to the [Feathr Feature Definition Guide](https://github.com/linkedin/feathr/blob/main/docs/concepts/feature-definition.md)\n", + "In Feathr, a feature is viewed as a function, mapping from entity id or key, and timestamp to a feature value. For more details on feature definition, please refer to the [Feathr Feature Definition Guide](https://github.com/feathr-ai/feathr/blob/main/docs/concepts/feature-definition.md)\n", "\n", "\n", "1. The typed key (a.k.a. entity key) identifies the subject of feature, e.g. a user id, 123.\n", @@ -781,7 +796,7 @@ " feature_type=FLOAT,\n", " input_features=[\n", " feature_user_gift_card_balance, feature_user_has_valid_credit_card],\n", - " transform=\"feature_user_gift_card_balance + if_else(toBoolean(feature_user_has_valid_credit_card), 100, 0)\")" + " transform=\"feature_user_gift_card_balance + if(boolean(feature_user_has_valid_credit_card), 100, 0)\")" ] }, { @@ -833,7 +848,7 @@ "To create a training dataset using Feathr, one needs to provide a feature join configuration file to specify\n", "what features and how these features should be joined to the observation data. \n", "\n", - "To learn more on this topic, please refer to [Point-in-time Correctness](https://github.com/linkedin/feathr/blob/main/docs/concepts/point-in-time-join.md)" + "To learn more on this topic, please refer to [Point-in-time Correctness](https://github.com/feathr-ai/feathr/blob/main/docs/concepts/point-in-time-join.md)" ] }, { @@ -1037,7 +1052,7 @@ " sinks=[redisSink],\n", " feature_names=[\"feature_user_age\", \"feature_user_gift_card_balance\"])\n", "\n", - "client.materialize_features(settings)\n", + "client.materialize_features(settings, allow_materialize_non_agg_feature =True)\n", "client.wait_job_to_finish(timeout_sec=1000)" ] }, @@ -1144,7 +1159,7 @@ " sinks=[redisSink],\n", " feature_names=[\"feature_product_price\"])\n", "\n", - "client.materialize_features(settings)\n", + "client.materialize_features(settings, allow_materialize_non_agg_feature =True)\n", "client.wait_job_to_finish(timeout_sec=1000)" ] }, diff --git a/feathr-compute/build.gradle b/feathr-compute/build.gradle new file mode 100644 index 000000000..6be976725 --- /dev/null +++ b/feathr-compute/build.gradle @@ -0,0 +1,72 @@ +apply plugin: 'java' +apply plugin: 'maven-publish' +apply plugin: 'signing' +apply plugin: "com.vanniktech.maven.publish.base" + +repositories { + mavenCentral() + mavenLocal() + maven { + url "https://repository.mulesoft.org/nexus/content/repositories/public/" + } + maven { + url "https://linkedin.jfrog.io/artifactory/open-source/" // GMA, pegasus + } +} +dependencies { + implementation project(":feathr-config") + implementation project(":feathr-data-models") + implementation project(path: ':feathr-data-models', configuration: 'dataTemplate') + implementation spec.product.mvel + implementation spec.product.jsqlparser + + testImplementation spec.product.testing + testImplementation spec.product.mockito + testImplementation spec.product.equalsverifier + testImplementation spec.product.mockito_inline + + implementation spec.product.jackson.dataformat_yaml + implementation spec.product.jackson.jackson_databind + implementation spec.product.guava +} + +javadoc { + options.noQualifiers 'all' +} + +java { + withSourcesJar() + withJavadocJar() +} + +tasks.withType(Javadoc) { + options.addStringOption('Xdoclint:none', '-quiet') + options.addStringOption('encoding', 'UTF-8') + options.addStringOption('charSet', 'UTF-8') +} + +test { + maxParallelForks = 1 + forkEvery = 1 + // need to keep a lower heap size (TOOLS-296596) + minHeapSize = "512m" + useTestNG() +} + +// Required for publishing to local maven +publishing { + publications { + mavenJava(MavenPublication) { + artifactId = 'feathr-compute' + from components.java + versionMapping { + usage('java-api') { + fromResolutionOf('runtimeClasspath') + } + usage('java-runtime') { + fromResolutionResult() + } + } + } + } +} diff --git a/feathr-compute/src/main/java/com/linkedin/feathr/compute/ComputeGraphBuilder.java b/feathr-compute/src/main/java/com/linkedin/feathr/compute/ComputeGraphBuilder.java new file mode 100644 index 000000000..95633494f --- /dev/null +++ b/feathr-compute/src/main/java/com/linkedin/feathr/compute/ComputeGraphBuilder.java @@ -0,0 +1,101 @@ +package com.linkedin.feathr.compute; + +import com.linkedin.data.template.IntegerMap; +import com.linkedin.data.template.LongMap; +import com.linkedin.data.template.RecordTemplate; + + +/** + * Builder class for Compute Graph + */ +@InternalApi +public class ComputeGraphBuilder { + IntegerMap _featureNameMap = new IntegerMap(); + LongMap _dataSourceMap = new LongMap(); + AnyNodeArray _nodes = new AnyNodeArray(); + + /** + * MODIFIES THE INPUT NODE by assigning it a new ID for this graph being built, and adds it to the graph. + * NOTE that this function doesn't/can't update the node's edges/dependencies so that they correctly point to nodes + * in the new graph! The caller is responsible for doing this. + * + * @param node the node to be modified, assigned a new ID, and inserted into the graph + * @return the node's new ID in this graph being built + */ + public int addNode(AnyNode node) { + int newId = _nodes.size(); + PegasusUtils.setNodeId(node, newId); + _nodes.add(node); + return newId; + } + + public DataSource addNewDataSource() { + return addNodeHelper(new DataSource()); + } + + public Transformation addNewTransformation() { + return addNodeHelper(new Transformation()); + } + + public Aggregation addNewAggregation() { + return addNodeHelper(new Aggregation()); + } + + public Lookup addNewLookup() { + return addNodeHelper(new Lookup()); + } + + public External addNewExternal() { + return addNodeHelper(new External()); + } + + public T addNodeHelper(T node) { + addNode(PegasusUtils.wrapAnyNode(node)); + return node; + } + + /** + * Adds a feature name mapping to this graph being built. + * @param featureName the feature name + * @param nodeId node Id + */ + public void addFeatureName(String featureName, Integer nodeId) { + if (nodeId >= _nodes.size()) { + throw new IllegalArgumentException("Node id " + nodeId + " is not defined in the graph being built: " + this); + } + if (_featureNameMap.containsKey(featureName)) { + throw new IllegalArgumentException("Feature " + featureName + " is already defined in the graph being built: " + + this); + } + _featureNameMap.put(featureName, nodeId); + } + + public int peekNextNodeId() { + return _nodes.size(); + } + + public ComputeGraph build() { + return build(new ComputeGraph()); + } + + public ComputeGraph build(ComputeGraph reuse) { + return build(reuse, true); + } + + /** + * Allows to build the graph without validating it. (Internal use case: Build a merged graph first, and remove + * internally-pointing External-feature nodes later.) Be careful. + */ + ComputeGraph build(ComputeGraph reuse, boolean validate) { + reuse.setFeatureNames(_featureNameMap).setNodes(_nodes); + if (validate) { + ComputeGraphs.validate(reuse); + } + return reuse; + } + + @Override + public String toString() { + return "ComputeGraphBuilder{" + "_featureNameMap=" + _featureNameMap + ", _nodes=" + _nodes + '}'; + } +} \ No newline at end of file diff --git a/feathr-compute/src/main/java/com/linkedin/feathr/compute/ComputeGraphs.java b/feathr-compute/src/main/java/com/linkedin/feathr/compute/ComputeGraphs.java new file mode 100644 index 000000000..dab85f2a2 --- /dev/null +++ b/feathr-compute/src/main/java/com/linkedin/feathr/compute/ComputeGraphs.java @@ -0,0 +1,490 @@ +package com.linkedin.feathr.compute; + +import com.linkedin.data.template.IntegerMap; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Deque; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + + +/** + * Functions for working with instances of compute graphs. + */ +@InternalApi +public class ComputeGraphs { + private ComputeGraphs() { } + + /** + * Ensures the input Graph is internally consistent. + * @param graph + * @return + */ + public static ComputeGraph validate(ComputeGraph graph) { + ensureNodeIdsAreSequential(graph); + ensureNodeReferencesExist(graph); + ensureNoDependencyCycles(graph); + ensureNoExternalReferencesToSelf(graph); + return graph; + } + + /** + * Graph 1: + * A + * | + * B + * + * Graph 2: + * A + * | + * C + * + * Merge(Graph1, Graph2): + * A + * / \ + * B C + * + * Other cases: The graphs could have nothing in common, in which case the merged graph is "not fully connected" but + * is still "one graph." + * + * + * Example for "Derived Features" + * e.g. featureC = featureA + featureB + * Assume featureA, featureB are anchored. + * + * What the definitions look like: + * + * myAnchor1: { + * source: "/foo/bar/baz" + * key: "x" + * features: { + * featureA: "source_columnA.nested_field6" + * } + * } + * + * myAnchor2: { + * source: "..." + * key: "foo" + * features: { + * featureB: "field7" + * } + * } + * + * featureC: "featureA + featureB" + * + * Algorithm to read the above: + * * Read 3 subgraphs, one for featureA, one for FeatureB, one for FeatureC + * * Merge them together, + * * Return + * + * + * Loading/translating definition for featureA: + * DataSource for FeatureA + * | + * Transformation (the "extraction function" for FeatureA") + * | + * (FeatureA) + * (FeatureB looks the same way) + * + * For FeatureC's subgraph: + * A B <----- these aren't defined in FeatureC's subgraph! + * \ / + * C <------ C is defined in this graph, with it's operator (+) + * + * ExternalNode(FeatureA) ExternalNode(FeatureB) + * \ / + * TransformationNode(operator=+, inputs=[the above nodes]) + * | + * FeatureC + * + * + * + * @param inputGraphs + * @return + */ + public static ComputeGraph merge(Collection inputGraphs) { + ComputeGraphBuilder builder = new ComputeGraphBuilder(); + inputGraphs.forEach(inputGraph -> { + int offset = builder.peekNextNodeId(); + inputGraph.getNodes().forEach(inputNode -> { + AnyNode copy = PegasusUtils.copy(inputNode); + Dependencies.remapDependencies(copy, i -> i + offset); + builder.addNode(copy); + }); + inputGraph.getFeatureNames().forEach((featureName, nodeId) -> { + builder.addFeatureName(featureName, nodeId + offset); + + }); + }); + ComputeGraph mergedGraph = builder.build(new ComputeGraph(), false); + return validate(removeExternalNodesForFeaturesDefinedInThisGraph(mergedGraph)); + } + + /* + + A B + \ / + C + + There might be more than one way this could be represented as a ComputeGraph. + 0:A 1:B + \ / + 2:C + Another possibility: + 1:A 2:B + \ / + 0:C + + If we wanted to merge: + I: + 0:A 1:B + \ / + 2:C + II: + 1:A 2:B + \ / + 0:C + Assuming the only differences are the arbitrarily chosen IDs, + we still want the output to be: + 0:A 1:B + \ / + 2:C + + Two nodes won't just be the same because they have the same operator (e.g. +), but they also need to have the same + inputs. Recursively. + */ + + /** + * Removes redundant parts of the graph. + * + * Nodes are considered to be "twins" if: + * 1. their contents are the same except for their node ID (just the main node ID, not the dependency node IDs!), + * OR: + * 2. their contents are the same except for their node IDs, and except for any dependency node IDs that are "twins" + * even if their IDs are different. + * + * @param inputGraph an input graph + * @return a equivalent output graph with any duplicate nodes or subgraphs removed and their dependencies updated + */ + public static ComputeGraph removeRedundancies(ComputeGraph inputGraph) throws CloneNotSupportedException { + /* + The intuitive approach is to start by deduplicating all source nodes into a "standardized" set of source nodes, + and recursively updating any nodes that depended on them, to all point to a standardized node ID for each source. + You can then proceed "up one level" to the nodes that depend on the sources, checking them based on criterion (1) + mentioned in the javadoc above, since by this time their dependency node IDs should already have been + standardized. It is slightly more complex in cases where a single node may depend on the same node via multiple + paths, potentially with a different number of edges between (so you cannot actually iterate over the graph "level + by level"). + */ + + /* + Overall algorithm: + 0. Init "unique node set" + 1. Init IN_PROGRESS, VISITED, UNVISITED table (key is node reference) + 2. Put all nodes in a stack. + 3. While stack is not empty, pop a node: + Is the node VISITED? + YES: Do nothing + NO: Does this node have any dependencies that are not VISITED? + YES: Is this node marked as IN_PROGRESS? + YES: Fail – This indicates a cycle in the graph. + NO: 1. Mark this node as IN_PROGRESS + 2. Push this node, and then each of its dependencies, onto the stack. + NO: 1. Is this node in the unique node set IGNORING ID? + YES: Rewire INBOUND REFERENCES to this node, to point to the twin in the unique node set. + NO: Add this node to the unique node set. + 2. Mark this node as VISITED. + + Algorithm for "Is this node in the unique-node set, IGNORING ID? If so rewire INBOUND REFERENCES to this node, + to point to the twin in the unique node set.": + - Create copies of the input nodes, with their IDs set to zero. Keep track of their IDs via a different way, + via a nodeIndex Map. + - Represent the unique-nodes set as a uniqueNodesMap HashMap. The key is the "standardized" + node with its id still zeroed out, and the value is its actual ID. + - To check whether a given node is in the unique-nodes set, just test whether the uniqueNodesMap contains that + node as a "key." If so, use its corresponding value for rewiring the node's dependents. + - To rewire the node's dependents, construct an index of "who-depends-on-me" at the top of the function, and + use it to figure out which nodes need to be rewired. + - Since the feature name map (map of feature names to node IDs) works differently from node-to-node + dependencies, separately keep a "which-feature-names-depend-on-me" index and update that too (same as in + previous step). + */ + + Map> whoDependsOnMeIndex = getReverseDependencyIndex(inputGraph); + // More than one feature name could point to the same node, e.g. if they are aliases. + Map> featureDependencyIndex = getReverseFeatureDependencyIndex(inputGraph); + + // create copies of all nodes, and set their IDs to zero + List nodes = inputGraph.getNodes().stream() + .map(PegasusUtils::copy) + .collect(Collectors.toList()); + nodes.forEach(node -> PegasusUtils.setNodeId(node, 0)); // set node IDs to zero, to facilitate comparison + + IntegerMap featureNameMap = inputGraph.getFeatureNames(); + + // We are going to "standardize" each subgraph. This requires traversing the graph and standardizing each node + // (after its dependencies have been standardized). This requires checking whether a node already exists in the + // standardized set. Instead of a set, we will use a hash map. The keys are the "standardized nodes" (with IDs set + // to zero, since we want to ignore node ID for comparison) and the values are the node's standardized ID. + Map standardizedNodes = new HashMap<>(); + + // init deque with IDs from 0 to N - 1 + Deque deque = IntStream.range(0, nodes.size()).boxed().collect(Collectors.toCollection(ArrayDeque::new)); + // init visited-state vector + List visitedState = new ArrayList<>(Collections.nCopies(nodes.size(), VisitedState.NOT_VISITED)); + + while (!deque.isEmpty()) { + int thisNodeId = deque.pop(); + if (visitedState.get(thisNodeId) == VisitedState.VISITED) { + continue; + } + AnyNode thisNode = nodes.get(thisNodeId); + Set myDependencies = new Dependencies().getDependencies(thisNode); + List unfinishedDependencies = myDependencies.stream() + .filter(i -> visitedState.get(i) != VisitedState.VISITED) + .collect(Collectors.toList()); + if (!unfinishedDependencies.isEmpty()) { + if (visitedState.get(thisNodeId) == VisitedState.IN_PROGRESS) { + // If I am already in-progress, it means I depended on myself (possibly via other dependency nodes). + throw new RuntimeException("Dependency cycle detected at node " + thisNodeId); + } + deque.push(thisNodeId); // Push myself back onto the deque, so that we can reprocess me later after my dependencies. + visitedState.set(thisNodeId, VisitedState.IN_PROGRESS); // Also mark myself as in-progress (prevent infinite loop in + // case of a cycle). + unfinishedDependencies.forEach(deque::push); + } else { + // Time to standardize this node (all of its dependencies [including transitive] have been standardized). + // 1. See if I am already standardized (check if I have a "twin" in the standardized set) + Integer standardizedNodeId = standardizedNodes.get(thisNode); + if (standardizedNodeId != null) { + // 2. If I DO have a twin in the standardized set, then rewire all the nodes who depend on me, to point to + // my standardized twin instead. + whoDependsOnMeIndex.getOrDefault(thisNodeId, Collections.emptySet()).forEach(nodeWhoDependsOnMe -> + Dependencies.remapDependencies(nodes.get(nodeWhoDependsOnMe), + // "If it points to me, remap it to my standardized twin, else leave it unchanged." + id -> id == thisNodeId ? standardizedNodeId : id)); + // Do the same for the feature name map. + featureDependencyIndex.getOrDefault(thisNodeId, Collections.emptySet()).forEach(featureThatPointsToMe -> + featureNameMap.put(featureThatPointsToMe, standardizedNodeId)); + } else { + // 3. If I DON'T have a twin in the standardized set, then put myself into the standardized set. + standardizedNodes.put(thisNode, thisNodeId); + } + // 4. This node ahs been standardized. Mark it as VISITED. + visitedState.set(thisNodeId, VisitedState.VISITED); + } + } + + // Put the IDs back into the nodes. + standardizedNodes.forEach((node, id) -> PegasusUtils.setNodeId(node, id)); + + // Reindex the nodes to ensure IDs are sequential. + return reindexNodes(standardizedNodes.keySet(), featureNameMap); + } + + private static ComputeGraph removeExternalNodesForFeaturesDefinedInThisGraph(ComputeGraph inputGraph) { + Map externalNodeRemappedIds = new HashMap<>(); + for (int id = 0; id < inputGraph.getNodes().size(); id++) { + AnyNode node = inputGraph.getNodes().get(id); + if (node.isExternal()) { + Integer featureNodeId = inputGraph.getFeatureNames().get(node.getExternal().getName()); + if (featureNodeId != null) { + // "any node who depends on me, should actually depend on that other node instead" + externalNodeRemappedIds.put(id, featureNodeId); + } + } + } + if (externalNodeRemappedIds.isEmpty()) { + return inputGraph; + } else { + inputGraph.getNodes().forEach(node -> { + Dependencies.remapDependencies(node, id -> { + Integer remappedId = externalNodeRemappedIds.get(id); + if (remappedId != null) { + return remappedId; + } else { + return id; + } + }); + }); + return removeNodes(inputGraph, externalNodeRemappedIds::containsKey); + } + } + + /** + * Remove nodes from a graph. + * @param computeGraph input graph + * @param predicate nodes for which this predicate is true, will be removed. the predicate must return true or false + * for all valid nodeIds in this graph (but could throw exceptions for other, invalid cases) + * @return new graph with the nodes removed + */ + static ComputeGraph removeNodes(ComputeGraph computeGraph, Predicate predicate) { + List nodesToKeep = IntStream.range(0, computeGraph.getNodes().size()).boxed() + .filter(predicate.negate()) + .map(computeGraph.getNodes()::get) + .collect(Collectors.toList()); + return reindexNodes(nodesToKeep, computeGraph.getFeatureNames()); + } + + /** + * Rebuilds a graph with a new (valid, sequential) set of IDs. The input nodes must form a valid subgraph, e.g. + * all node references (and feature names) must point to nodes within the subgraph. + * + * @param nodes the nodes (WILL BE MODIFIED) + * @param featureNames feature name map + * @return the reindexed compute graph + */ + static ComputeGraph reindexNodes(Collection nodes, IntegerMap featureNames) { + Map indexRemapping = new HashMap<>(); + ComputeGraphBuilder builder = new ComputeGraphBuilder(); + nodes.forEach(node -> { + int oldId = PegasusUtils.getNodeId(node); + int newId = builder.addNode(node); + indexRemapping.put(oldId, newId); + }); + Function remap = oldId -> { + Integer newId = indexRemapping.get(oldId); + if (newId == null) { + throw new RuntimeException("Node " + oldId + " not found in subgraph."); + } + return newId; + }; + // This is taking advantage of the fact that the nodes are mutable. If we switch to using an immutable API e.g. + // with Protobuf, we'd need to change this somewhat. + nodes.forEach(node -> Dependencies.remapDependencies(node, remap)); + featureNames.forEach((featureName, nodeId) -> builder.addFeatureName(featureName, remap.apply(nodeId))); + return builder.build(); + } + + private static Map> getReverseDependencyIndex(ComputeGraph graph) { + Map> reverseDependencies = new HashMap<>(); + for (int nodeId = 0; nodeId < graph.getNodes().size(); nodeId++) { + AnyNode node = graph.getNodes().get(nodeId); + for (int dependencyNodeId : new Dependencies().getDependencies(node)) { + Set dependentNodes = reverseDependencies.computeIfAbsent(dependencyNodeId, x -> new HashSet<>()); + dependentNodes.add(nodeId); + } + } + return reverseDependencies; + } + + /** + * More than one feature name could point to the same node, e.g. if they are aliases. + * @param graph + * @return + */ + static Map> getReverseFeatureDependencyIndex(ComputeGraph graph) { + // More than one feature name could point to the same node, e.g. if they are aliases. + Map> reverseDependencies = new HashMap<>(); + graph.getFeatureNames().forEach((featureName, nodeId) -> { + Set dependentFeatures = reverseDependencies.computeIfAbsent(nodeId, x -> new HashSet<>(1)); + dependentFeatures.add(featureName); + }); + return reverseDependencies; + } + + /** + * Ensures that all the nodes are sequential. + * @param graph + */ + static void ensureNodeIdsAreSequential(ComputeGraph graph) { + for (int i = 0; i < graph.getNodes().size(); i++) { + if (PegasusUtils.getNodeId(graph.getNodes().get(i)) != i) { + throw new RuntimeException("Graph nodes must be ID'd sequentially from 0 to N-1 where N is the number of nodes."); + } + } + } + + /** + * Ensures that all the node references exist for each of the dependencies in the graph + * @param graph + */ + static void ensureNodeReferencesExist(ComputeGraph graph) { + final int minValidId = 0; + final int maxValidId = graph.getNodes().size() - 1; + graph.getNodes().forEach(anyNode -> { + Set dependencies = new Dependencies().getDependencies(anyNode); + List missingDependencies = dependencies.stream() + .filter(id -> id < minValidId || id > maxValidId) + .collect(Collectors.toList()); + if (!missingDependencies.isEmpty()) { + throw new RuntimeException("Encountered missing dependencies " + missingDependencies + " for node " + anyNode + + ". Graph = " + graph); + } + }); + } + + /** + * Ensure that all the nodes have no concrete keys + * @param graph + */ + static void ensureNoConcreteKeys(ComputeGraph graph) { + graph.getNodes().forEach(node -> { + if ((node.isExternal() && (node.getExternal().hasConcreteKey()) || (node.isAggregation() && ( + node.getAggregation().hasConcreteKey())) || (node.isDataSource() && ( + node.getDataSource().hasConcreteKey())) || (node.isLookup() && (node.getLookup().hasConcreteKey())) + || (node.isTransformation() && (node.getTransformation().hasConcreteKey())))) { + throw new RuntimeException("A concrete key has already been set for the node " + node); + } + }); + } + + /** + * Ensure that none of the external nodes points to a requires feature name + * @param graph + */ + static void ensureNoExternalReferencesToSelf(ComputeGraph graph) { + // make sure graph does not reference external features that are actually defined within itself + graph.getNodes().stream().filter(AnyNode::isExternal).forEach(node -> { + String featureName = node.getExternal().getName(); + if (graph.getFeatureNames().containsKey(featureName)) { + throw new RuntimeException("Graph contains External node " + node + " but also contains feature " + featureName + + " in its feature name table: " + graph.getFeatureNames() + ". Graph = " + graph); + } + }); + } + + /** + * Ensures that there are no dependency cycles. + * @param graph + */ + static void ensureNoDependencyCycles(ComputeGraph graph) { + Deque deque = IntStream.range(0, graph.getNodes().size()).boxed() + .collect(Collectors.toCollection(ArrayDeque::new)); + List visitedState = new ArrayList<>(Collections.nCopies(graph.getNodes().size(), + VisitedState.NOT_VISITED)); + + while (!deque.isEmpty()) { + int nodeId = deque.pop(); + if (visitedState.get(nodeId) == VisitedState.VISITED) { + continue; + } + AnyNode node = graph.getNodes().get(nodeId); + Set dependencies = new Dependencies().getDependencies(node); + List unfinishedDependencies = + dependencies.stream().filter(i -> visitedState.get(i) != VisitedState.VISITED).collect(Collectors.toList()); + if (!unfinishedDependencies.isEmpty()) { + if (visitedState.get(nodeId) == VisitedState.IN_PROGRESS) { + throw new RuntimeException("Dependency cycle involving node " + nodeId); + } + deque.push(nodeId); // check me again later, after checking my dependencies. + unfinishedDependencies.forEach(deque::push); // check my dependencies next. + visitedState.set(nodeId, VisitedState.IN_PROGRESS); + } else { + visitedState.set(nodeId, VisitedState.VISITED); + } + } + } + + private enum VisitedState { NOT_VISITED, IN_PROGRESS, VISITED } + +} \ No newline at end of file diff --git a/feathr-compute/src/main/java/com/linkedin/feathr/compute/Dependencies.java b/feathr-compute/src/main/java/com/linkedin/feathr/compute/Dependencies.java new file mode 100644 index 000000000..be930e507 --- /dev/null +++ b/feathr-compute/src/main/java/com/linkedin/feathr/compute/Dependencies.java @@ -0,0 +1,158 @@ +package com.linkedin.feathr.compute; + +import com.google.common.collect.Sets; +import com.linkedin.data.template.IntegerArray; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; + + +/** + * Utility class for working with nodes' dependencies. + * + * If AnyNode had been a interface instead of a Pegasus record, .getDependencies() and .remapDependencies() would + * have been interface methods for it. But since Pegasus records don't have custom methods (and don't have inheritance), + * use this class to deal with nodes' dependencies instead. + */ +@SuppressWarnings("checkstyle:HideUtilityClassConstructor") +@InternalApi +public class Dependencies { + /** + * Get the dependencies for any kind of node. Note that a dependency is a reference to another node. + * + * @param anyNode the node + * @return the set of ids of the nodes the input node depends on + */ + public Set getDependencies(AnyNode anyNode) { + return Sets.union(getKeyDependencies(anyNode), getNodeDependencies(anyNode)); + } + + private Set getKeyDependencies(AnyNode anyNode) { + if (PegasusUtils.hasConcreteKey(anyNode)) { + return new HashSet<>(PegasusUtils.getConcreteKey(anyNode).getKey()); + } else { + return Collections.emptySet(); + } + } + + private static Set getNodeDependencies(AnyNode anyNode) { + if (anyNode.isAggregation()) { + return getNodeDependencies(anyNode.getAggregation()); + } else if (anyNode.isDataSource()) { + return getNodeDependencies(anyNode.getDataSource()); + } else if (anyNode.isLookup()) { + return getNodeDependencies(anyNode.getLookup()); + } else if (anyNode.isTransformation()) { + return getNodeDependencies(anyNode.getTransformation()); + } else if (anyNode.isExternal()) { + return getNodeDependencies(anyNode.getExternal()); + } else { + throw new RuntimeException("Unhandled kind of AnyNode: " + anyNode); + } + } + + private static Set getNodeDependencies(Aggregation node) { + return Collections.singleton(node.getInput().getId()); + } + + private static Set getNodeDependencies(Transformation node) { + return node.getInputs().stream().map(NodeReference::getId).collect(Collectors.toSet()); + } + + private static Set getNodeDependencies(Lookup node) { + Set dependencies = new HashSet<>(); + node.getLookupKey().stream() + // Only NodeReferences matter for determining dependencies on other nodes. + .filter(Lookup.LookupKey::isNodeReference) + .map(Lookup.LookupKey::getNodeReference) + .map(NodeReference::getId) + .forEach(dependencies::add); + dependencies.add(node.getLookupNode()); + return dependencies; + } + + private static Set getNodeDependencies(DataSource node) { + return Collections.emptySet(); + } + + private static Set getNodeDependencies(External node) { + return Collections.emptySet(); + } + + /** + * Modify a node's dependencies' ids based on a given id-mapping function. + * This can be useful for modifying a graph, merging graphs together, removing duplicate parts of graphs, etc. + * + * @param anyNode the nodes whose dependencies (if it has any) should be modified according to the mapping function; + * must not be null. + * @param idMapping a mapping function that converts from "what the nodes' dependencies currently look like" to "what + * they should look like after the change." For any node id that should NOT change, the the function + * must return the input if that node id is passed in. For any node ids that the caller expects will + * never be encountered, it would be ok for the idMapping function to throw an exception if that node + * id is passed in. The idMapping function can assume its input will never be null, and should NOT + * return null. + */ + static void remapDependencies(AnyNode anyNode, Function idMapping) { + remapKeyDependencies(anyNode, idMapping); + remapNodeDependencies(anyNode, idMapping); + } + + private static void remapKeyDependencies(AnyNode anyNode, Function idMapping) { + if (PegasusUtils.hasConcreteKey(anyNode)) { + ConcreteKey concreteKey = PegasusUtils.getConcreteKey(anyNode); + IntegerArray newKeyDependencies = concreteKey.getKey().stream() + .map(idMapping) + .collect(Collectors.toCollection(IntegerArray::new)); + concreteKey.setKey(newKeyDependencies); + } + } + + private static void remapNodeDependencies(AnyNode anyNode, Function idMapping) { + if (anyNode.isAggregation()) { + remapNodeDependencies(anyNode.getAggregation(), idMapping); + } else if (anyNode.isDataSource()) { + // data source has no dependencies + } else if (anyNode.isLookup()) { + remapNodeDependencies(anyNode.getLookup(), idMapping); + } else if (anyNode.isTransformation()) { + remapNodeDependencies(anyNode.getTransformation(), idMapping); + } else if (anyNode.isExternal()) { + // no dependencies + } else { + throw new RuntimeException("Unhandled kind of AnyNode: " + anyNode); + } + } + + private static void remapNodeDependencies(Aggregation node, Function idMapping) { + int oldInputNodeId = node.getInput().getId(); + int newNodeId = idMapping.apply(oldInputNodeId); // An NPE on this line would mean that the mapping is not complete, + // which should be impossible and would indicate a bug in the graph + // processing code. + node.getInput().setId(newNodeId); + } + + private static void remapNodeDependencies(Transformation node, Function idMapping) { + node.getInputs().forEach(input -> { + int oldInputNodeId = input.getId(); + int newNodeId = idMapping.apply(oldInputNodeId); + input.setId(newNodeId); + }); + } + + private static void remapNodeDependencies(Lookup node, Function idMapping) { + int oldLookupNodeId = node.getLookupNode(); + int newLookupNodeId = idMapping.apply(oldLookupNodeId); + node.setLookupNode(newLookupNodeId); + + node.getLookupKey().forEach(lookupKey -> { + if (lookupKey.isNodeReference()) { + NodeReference nodeReference = lookupKey.getNodeReference(); + int oldReferenceNodeId = nodeReference.getId(); + int newReferenceNodeId = idMapping.apply(oldReferenceNodeId); + nodeReference.setId(newReferenceNodeId); + } + }); + } +} \ No newline at end of file diff --git a/feathr-compute/src/main/java/com/linkedin/feathr/compute/InternalApi.java b/feathr-compute/src/main/java/com/linkedin/feathr/compute/InternalApi.java new file mode 100644 index 000000000..893f83ea0 --- /dev/null +++ b/feathr-compute/src/main/java/com/linkedin/feathr/compute/InternalApi.java @@ -0,0 +1,15 @@ +package com.linkedin.feathr.compute; + +import java.lang.annotation.Documented; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + + +/** + * An annotation indicating that the target is is part of a module-private "internal API" and should not be used by + * external modules. + */ +@Documented +@Retention(RetentionPolicy.SOURCE) +public @interface InternalApi { +} \ No newline at end of file diff --git a/feathr-compute/src/main/java/com/linkedin/feathr/compute/Operators.java b/feathr-compute/src/main/java/com/linkedin/feathr/compute/Operators.java new file mode 100644 index 000000000..10784c0ef --- /dev/null +++ b/feathr-compute/src/main/java/com/linkedin/feathr/compute/Operators.java @@ -0,0 +1,178 @@ +package com.linkedin.feathr.compute; + +/** + * In the compute graph, operators are referenced by their names. + * + */ +public class Operators { + private Operators() { + } + + /** + * Name: anchor mvel + * Description: MVEL operator for an anchored feature + * + * Input: Any + * Output: Any + * + * Parameters: + * - expression + */ + public static final String OPERATOR_ID_ANCHOR_MVEL = "feathr:anchor_mvel:0"; + + /** + * Name: derived mvel + * Description: MVEL operator for an anchored feature + * + * Input: Any + * Output: Any + * + * Parameters: + * - expression + */ + public static final String OPERATOR_ID_DERIVED_MVEL = "feathr:derived_mvel:0"; + + /** + * Name: passthrough mvel + * Description: MVEL operator for a passthrough feature + * + * Input: Any + * Output: Any + * + * Parameters: + * - expression + */ + public static final String OPERATOR_ID_PASSTHROUGH_MVEL = "feathr:passthrough_mvel:0"; + + /** + * Name: lookup mvel + * Description: MVEL operator for a lookup key + * + * Input: Any + * Output: Any + * + * Parameters: + * - expression + */ + public static final String OPERATOR_ID_LOOKUP_MVEL = "feathr:lookup_mvel:0"; + + /** + * Name: sliding_window_aggregation + * Description: Configurable sliding window aggregator + * + * Input: Series + * Output: Any + * + * Parameters: + * - target_column + * - aggregation_type + * - window_size + * - window_unit + * - lateral_view_expression_0, lateral_view_expression_1, ... + * - lateral_view_table_alias_0, lateral_view_table_alias_1, ... + * - filter_expression + * - group_by_expression + * - max_number_groups + */ + public static final String OPERATOR_ID_SLIDING_WINDOW_AGGREGATION = "feathr:sliding_window_aggregation:0"; + + /** + * Name: anchor_java_udf_feature_extractor + * Description: Runs a Java UDF for an anchored feature + * + * Input: Any + * Output: Any + * + * Parameters: + * - class + * - userParam_foo, userParam_bar + */ + public static final String OPERATOR_ID_ANCHOR_JAVA_UDF_FEATURE_EXTRACTOR = "feathr:anchor_java_udf_feature_extractor:0"; + + /** + * Name: passthrough_java_udf_feature_extractor + * Description: Runs a Java UDF for a passthrough feature + * + * Input: Any + * Output: Any + * + * Parameters: + * - class + * - userParam_foo, userParam_bar + */ + public static final String OPERATOR_ID_PASSTHROUGH_JAVA_UDF_FEATURE_EXTRACTOR = "feathr:passthrough_java_udf_feature_extractor:0"; + + /** + * Name: derived_java_udf_feature_extractor + * Description: Runs a Java UDF for a derived feature + * + * Input: Any + * Output: Any + * + * Parameters: + * - class + * - userParam_foo, userParam_bar + */ + public static final String OPERATOR_ID_DERIVED_JAVA_UDF_FEATURE_EXTRACTOR = "feathr:derived_java_udf_feature_extractor:0"; + + /** + * Name: anchor_spark_sql_feature_extractor + * Description: SQL operator for an anchored feature + * + * Input: Any + * Output: Any + * + * Parameters: + * - expression + */ + public static final String OPERATOR_ID_ANCHOR_SPARK_SQL_FEATURE_EXTRACTOR = "feathr:anchor_spark_sql_feature_extractor:0"; + + /** + * Name: passthrough_spark_sql_feature_extractor + * Description: SQL operator for a passthrough feature + * + * Input: Any + * Output: Any + * + * Parameters: + * - expression + */ + public static final String OPERATOR_ID_PASSTHROUGH_SPARK_SQL_FEATURE_EXTRACTOR = "feathr:passthrough_spark_sql_feature_extractor:0"; + + /** + * Name: derived_spark_sql_feature_extractor + * Description: SQL operator for a derived feature + * + * Input: Any + * Output: Any + * + * Parameters: + * - expression + */ + public static final String OPERATOR_ID_DERIVED_SPARK_SQL_FEATURE_EXTRACTOR = "feathr:derived_spark_sql_feature_extractor:0"; + + /** + * Name: extract_from_tuple + * Description: select i-th item from tuple + * + * Input: Tuple + * Output: Any + * + * Parameter: + * - index + */ + public static final String OPERATOR_ID_EXTRACT_FROM_TUPLE = "feathr:extract_from_tuple:0"; + + /** + * Name: feature_alias + * Description: given a feature, create another feature with the same values but different feature name. Main usage + * is for intermediate features in sequential join and derived features. Note that no parameters are needed because + * the input node's output feature will be aliases as this transformation node's feature name. + * + * Input: Feature + * Output: Alias Feature + * + * Parameter: None + */ + public static final String OPERATOR_FEATURE_ALIAS = "feathr:feature_alias:0"; +} \ No newline at end of file diff --git a/feathr-compute/src/main/java/com/linkedin/feathr/compute/PegasusUtils.java b/feathr-compute/src/main/java/com/linkedin/feathr/compute/PegasusUtils.java new file mode 100644 index 000000000..d72784399 --- /dev/null +++ b/feathr-compute/src/main/java/com/linkedin/feathr/compute/PegasusUtils.java @@ -0,0 +1,106 @@ +package com.linkedin.feathr.compute; + +import com.linkedin.data.template.RecordTemplate; + + +/** + * Helper functions for dealing with the generated Pegasus APIs for the Compute Model. For example, Pegasus doesn't + * really support inheritance, so we have some helper functions here to give polymorphism-like behavior. + */ +public class PegasusUtils { + private PegasusUtils() { + } + + static AnyNode copy(AnyNode node) { + try { + return node.copy(); + } catch (CloneNotSupportedException e) { + throw new RuntimeException(e); // this should never happen, based on Pegasus's guarantees, AFAIK + } + } + + /** + * Makes an AnyNode, for some given kind of specific node RecordTemplate (any of Aggregation, DataSource, Lookup, + * Transformation, or External). Throws an exception if any other kind of record is passed in. + * @param node the specific node + * @return the node wrapped as an AnyNode + */ + static AnyNode wrapAnyNode(RecordTemplate node) { + if (node instanceof Aggregation) { + return AnyNode.create((Aggregation) node); + } else if (node instanceof DataSource) { + return AnyNode.create((DataSource) node); + } else if (node instanceof Lookup) { + return AnyNode.create((Lookup) node); + } else if (node instanceof Transformation) { + return AnyNode.create((Transformation) node); + } else if (node instanceof External) { + return AnyNode.create((External) node); + } else { + throw new RuntimeException("Unhandled kind of node: " + node); + } + } + + /** + * Unwraps an AnyNode into its specific node type (Aggregation, DataSource, Lookup, Transformation, or External). + * @param anyNode the AnyNode + * @return the specific node that had been wrapped inside + */ + static RecordTemplate unwrapAnyNode(AnyNode anyNode) { + if (anyNode.isAggregation()) { + return anyNode.getAggregation(); + } else if (anyNode.isDataSource()) { + return anyNode.getDataSource(); + } else if (anyNode.isLookup()) { + return anyNode.getLookup(); + } else if (anyNode.isTransformation()) { + return anyNode.getTransformation(); + } else if (anyNode.isExternal()) { + return anyNode.getExternal(); + } else { + throw new RuntimeException("Unhandled kind of AnyNode: " + anyNode); + } + } + + /** + * Gets the id for the node wrapped inside the provided AnyNode + * @param anyNode any node + * @return the id + */ + static int getNodeId(AnyNode anyNode) { + return abstractNode(anyNode).getId(); + } + + public static int getNodeId(RecordTemplate node) { + return abstractNode(node).getId(); + } + + /** + * Sets the id for the node wrapped inside the provided AnyNode + * @param node the node + * @param id the id to set + */ + static void setNodeId(AnyNode node, int id) { + abstractNode(node).setId(id); + } + + static boolean hasConcreteKey(AnyNode anyNode) { + return abstractNode(anyNode).hasConcreteKey(); + } + + static ConcreteKey getConcreteKey(AnyNode anyNode) { + return abstractNode(anyNode).getConcreteKey(); + } + + static void setConcreteKey(AnyNode anyNode, ConcreteKey concreteKey) { + abstractNode(anyNode).setConcreteKey(concreteKey); + } + + private static AbstractNode abstractNode(AnyNode anyNode) { + return new AbstractNode(unwrapAnyNode(anyNode).data()); + } + + private static AbstractNode abstractNode(RecordTemplate anyNode) { + return new AbstractNode(anyNode.data()); + } +} \ No newline at end of file diff --git a/feathr-compute/src/main/java/com/linkedin/feathr/compute/Resolver.java b/feathr-compute/src/main/java/com/linkedin/feathr/compute/Resolver.java new file mode 100644 index 000000000..bb4a4b39a --- /dev/null +++ b/feathr-compute/src/main/java/com/linkedin/feathr/compute/Resolver.java @@ -0,0 +1,305 @@ +package com.linkedin.feathr.compute; + +import com.linkedin.data.template.IntegerArray; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +import static com.linkedin.feathr.compute.ComputeGraphs.*; + + +/** + * Resolves a given compute graph (output by the [[FeatureDefinitionsConverter]] class) by removing redundancies and simplifies the + * graph by taking the join config into account. + */ +public class Resolver { + private final ComputeGraph _definitionGraph; + + public Resolver(ComputeGraph graph) { + ensureNoConcreteKeys(graph); + // Sanity checks for the input graph + _definitionGraph = ComputeGraphs.validate(graph); + } + + public static Resolver create(ComputeGraph graph) { + return new Resolver(graph); + } + + /** + * This method takes in a list of requested features and optimizes the graph. + * @param featureRequestList Input requested features list + * @return An optimized compute graph + * @throws CloneNotSupportedException + */ + public ComputeGraph resolveForRequest(List featureRequestList) throws CloneNotSupportedException { + // preconditions + // 1. all requested features are defined in the graph + // 2. no colliding output-feature-names + // 3. right number of keys for each feature (this would be quite hard to verity! without more info in the model.) + + List graphParts = featureRequestList.stream() + .map(request -> { + try { + return resolveForRequest(request); + } catch (CloneNotSupportedException e) { + e.printStackTrace(); + } + return null; + }) + .collect(Collectors.toList()); + + return ComputeGraphs.removeRedundancies(ComputeGraphs.merge(graphParts)); + } + + public ComputeGraph resolveForRequest(FeatureRequest featureRequest) throws CloneNotSupportedException { + return resolveForFeature(featureRequest._featureName, featureRequest._keys, featureRequest._alias); + } + + /** + * Resolve the unresolved dependencies required to compute a given feature. For example, we need to resolve the join keys + * the feature. The join keys exist as a separate node inside the graph (a context datasource node). Another example is to + * resolve the dependencies of the input feature. + * @param featureName Name of the feature + * @param keys Keys of the observation datasource + * @param alias the feature can be aliased with another name (optional field) + * @return A compute graph with the dependency resolved for this particular feature + * @throws CloneNotSupportedException + */ + public ComputeGraph resolveForFeature(String featureName, List keys, String alias) + throws CloneNotSupportedException { + if (!_definitionGraph.getFeatureNames().containsKey(featureName)) { + throw new IllegalArgumentException("Feature graph does not contain requested feature " + featureName); + } + if (alias == null) { + alias = featureName; + } + ComputeGraphBuilder builder = new ComputeGraphBuilder(); + + ConcreteKey concreteKey = new ConcreteKey().setKey(new IntegerArray()); + keys.forEach(key -> { + DataSource source = builder.addNewDataSource() + .setSourceType(DataSourceType.CONTEXT) + .setExternalSourceRef(key); + concreteKey.getKey().add(source.getId()); + }); + + ConcreteKeyAttacher concreteKeyAttacher = new ConcreteKeyAttacher(builder); + int newNodeId = concreteKeyAttacher.addNodeAndAttachKey(_definitionGraph.getFeatureNames().get(featureName), concreteKey); + builder.addFeatureName(alias, newNodeId); + + return builder.build(); + } + + /** + * Class to attach the concrete key to all the dependencies + */ + private class ConcreteKeyAttacher { + private final ComputeGraphBuilder _builder; + + public ConcreteKeyAttacher(ComputeGraphBuilder builder) { + _builder = builder; + } + + /** + * Set the given concrete key to the given node. Also, attach the same key to all it's dependendent nodes. + * @param nodeId node id in the original (definition) feature graph + * @param key the "concrete key" to attach. references should be into the new (resolved) graph. + * @return the node id of the newly created counterpart node in the new (resolved) graph + */ + int addNodeAndAttachKey(int nodeId, ConcreteKey key) { + AnyNode node = _definitionGraph.getNodes().get(nodeId); + if (PegasusUtils.hasConcreteKey(node)) { + throw new RuntimeException("Assertion failed. Did not expect to encounter key-annotated node"); + } + AnyNode newNode = PegasusUtils.copy(node); + PegasusUtils.setConcreteKey(newNode, key); + attachKeyToDependencies(newNode, key); + return _builder.addNode(newNode); + } + + private void attachKeyToDependencies(AnyNode node, ConcreteKey key) { + if (node.isAggregation()) { + attachKeyToDependencies(node.getAggregation(), key); + } else if (node.isDataSource()) { + attachKeyToDependencies(node.getDataSource(), key); + } else if (node.isLookup()) { + attachKeyToDependencies(node.getLookup(), key); + } else if (node.isTransformation()) { + attachKeyToDependencies(node.getTransformation(), key); + } else if (node.isExternal()) { + attachKeyToDependencies(node.getExternal(), key); + } else { + throw new RuntimeException("Unhandled kind of AnyNode: " + node); + } + } + + private void attachKeyToDependencies(Aggregation node, ConcreteKey key) { + NodeReference childNodeReference = node.getInput(); + + // If the node is a datasource node, we assume it is the terminal node (ie - no dependencies). + if (_definitionGraph.getNodes().get(childNodeReference.getId()).isDataSource()) { + ArrayList keyReferenceArray = new ArrayList(); + for (int i = 0; i < key.getKey().size(); i++) { + keyReferenceArray.add(new KeyReference().setPosition(i)); + } + + KeyReferenceArray keyReferenceArray1 = new KeyReferenceArray(keyReferenceArray); + childNodeReference.setKeyReference(keyReferenceArray1); + } + ConcreteKey childKey = transformConcreteKey(key, childNodeReference.getKeyReference()); + int childDefinitionNodeId = childNodeReference.getId(); + int resolvedChildNodeId = addNodeAndAttachKey(childDefinitionNodeId, childKey); + childNodeReference.setId(resolvedChildNodeId); + } + + private void attachKeyToDependencies(DataSource node, ConcreteKey key) { + if (node.hasSourceType() && node.getSourceType() == DataSourceType.UPDATE) { + node.setConcreteKey(key); + } + } + + /** + * If the node is a lookup node, we will need to attach the appropriate concrete key to the input nodes + * @param node + * @param inputConcreteKey + */ + private void attachKeyToDependencies(Lookup node, ConcreteKey inputConcreteKey) { + ConcreteKey concreteLookupKey = new ConcreteKey().setKey(new IntegerArray()); + IntegerArray concreteKeyClone = new IntegerArray(); + concreteKeyClone.addAll(inputConcreteKey.getKey()); + ConcreteKey inputConcreteKeyClone = new ConcreteKey().setKey(concreteKeyClone); + node.getLookupKey().forEach(lookupKeyPart -> { + if (lookupKeyPart.isKeyReference()) { // We do not support this yet. + int relativeKey = lookupKeyPart.getKeyReference().getPosition(); + concreteLookupKey.getKey().add(inputConcreteKeyClone.getKey().get(relativeKey)); + } else if (lookupKeyPart.isNodeReference()) { + /** + * seq_join_feature: { + * key: {x, y, viewerId} + * base: {key: x, feature: baseFeature} + * expansion: {key: [y, viewerId] feature: expansionFeature} + * } + * + * We need to add the concrete key of 0 (x) to the base feature node (lookup key) and concrete key of 1, 2 (y, viewerId) + * to the expansion feature node (lookup node). + */ + NodeReference childNodeReference = lookupKeyPart.getNodeReference(); + ConcreteKey childConcreteKey = transformConcreteKey(inputConcreteKey, childNodeReference.getKeyReference()); + int childDefinitionNodeId = childNodeReference.getId(); + int resolvedChildNodeId = 0; + resolvedChildNodeId = addNodeAndAttachKey(childDefinitionNodeId, childConcreteKey); + + // Remove all the keys which are not part of the base key features, ie - y in this case. + IntegerArray keysToBeRemoved = childConcreteKey.getKey(); + inputConcreteKey.getKey().removeAll(keysToBeRemoved); + childNodeReference.setId(resolvedChildNodeId); + + // Add the compute base node to the expansion keyset. Now, concreteLookupKey will have the right values. + concreteLookupKey.getKey().add(resolvedChildNodeId); + } else { + throw new RuntimeException("Unhandled kind of LookupKey: " + lookupKeyPart); + } + }); + + // The right concrete node has been calculated for the expansion feature now. We can just set it. + int lookupDefinitionNodeId = node.getLookupNode(); + int resolvedLookupNodeId = addNodeAndAttachKey(lookupDefinitionNodeId, new ConcreteKey().setKey(concreteLookupKey.getKey())); + inputConcreteKey.setKey(concreteKeyClone); + node.setLookupNode(resolvedLookupNodeId); + } + + /** + * Attach the concrete key to all the dependencies of the transformation node. + * @param node + * @param key + */ + private void attachKeyToDependencies(Transformation node, ConcreteKey key) { + /** + * A transformation node can have n dependencies like:- + * derivedFeature: { + * key: {a, b, c} + * input1: {key: a, feature: AA} + * input2: {key: b, feature: BB} + * input3: {key: c, feature: CC} + * defintion: input1 + input2 + input3 + * } + * + * In this case, we need to attach concrete key 0 (a) to the input1 node, key 1 (b) to the input2 node andd key 3 (c) to the input3 node. + */ + node.getInputs().forEach(childNodeReference -> { + if (_definitionGraph.getNodes().get(childNodeReference.getId()).isDataSource()) { + ArrayList keyReferenceArray = new ArrayList(); + for (int i = 0; i < key.getKey().size(); i++) { + keyReferenceArray.add(new KeyReference().setPosition(i)); + } + KeyReferenceArray keyReferenceArray1 = new KeyReferenceArray(keyReferenceArray); + childNodeReference.setKeyReference(keyReferenceArray1); + } + + ConcreteKey childKey = transformConcreteKey(key, childNodeReference.getKeyReference()); + int childDefinitionNodeId = childNodeReference.getId(); + int resolvedChildNodeId = 0; + resolvedChildNodeId = addNodeAndAttachKey(childDefinitionNodeId, childKey); + + childNodeReference.setId(resolvedChildNodeId); + }); + } + + private void attachKeyToDependencies(External node, ConcreteKey key) { + throw new RuntimeException("Internal error: Can't link key to external feature node not defined in this graph."); + } + } + + /** + * Representation class for a feature request. + */ + public static class FeatureRequest { + private final String _featureName; + private final List _keys; + private final Duration _timeDelay; + private final String _alias; + + public FeatureRequest(String featureName, List keys, Duration timeDelay, String alias) { + _featureName = featureName; + _keys = keys; + _timeDelay = timeDelay; + _alias = alias; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof FeatureRequest)) { + return false; + } + FeatureRequest that = (FeatureRequest) o; + return Objects.equals(_featureName, that._featureName) && Objects.equals(_keys, that._keys) && Objects.equals( + _alias, that._alias); + } + + @Override + public int hashCode() { + return Objects.hash(_featureName, _keys, _alias); + } + } + + /** + * In this method, we transform the original concrete key to the necessary concrete key by using a keyReference array. + * For example, if the original key is [1, 2, 3] and the keyReferenceArray is [0,1]. Then, the resultant concrete key would be + * [1, 2] (which is the 0th and 1st index of the original key. + * @param original the original (or parent) key + * @param keyReference the relative key, whose parts refer to relative positions in the parent key + * @return the child key obtained by applying the keyReference to the parent key + */ + private static ConcreteKey transformConcreteKey(ConcreteKey original, KeyReferenceArray keyReference) { + return new ConcreteKey().setKey( + keyReference.stream() + .map(KeyReference::getPosition) + .map(original.getKey()::get) + .collect(Collectors.toCollection(IntegerArray::new))); + } +} \ No newline at end of file diff --git a/feathr-compute/src/main/java/com/linkedin/feathr/compute/SqlUtil.java b/feathr-compute/src/main/java/com/linkedin/feathr/compute/SqlUtil.java new file mode 100644 index 000000000..504bc7d8c --- /dev/null +++ b/feathr-compute/src/main/java/com/linkedin/feathr/compute/SqlUtil.java @@ -0,0 +1,41 @@ +package com.linkedin.feathr.compute; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import net.sf.jsqlparser.JSQLParserException; +import net.sf.jsqlparser.expression.ExpressionVisitorAdapter; +import net.sf.jsqlparser.parser.CCJSqlParserUtil; +import net.sf.jsqlparser.schema.Column; + + +/** + * Class for SQL utilities + */ +public class SqlUtil { + private SqlUtil() { } + + /** + * Try to find the input feature names from a sqlExpr derived feature. + * (Without depending on Spark and Scala.) + * + * @param sql a sql expression + * @return list of input feature names (without any duplicates) + */ + public static List getInputsFromSqlExpression(String sql) { + Set inputs = new HashSet<>(); + ExpressionVisitorAdapter visitor = new ExpressionVisitorAdapter() { + @Override + public void visit(Column column) { + inputs.add(column.getColumnName()); + } + }; + try { + CCJSqlParserUtil.parseExpression(sql).accept(visitor); + } catch (JSQLParserException e) { + throw new RuntimeException(e); + } + return new ArrayList<>(inputs); + } +} \ No newline at end of file diff --git a/feathr-compute/src/main/java/com/linkedin/feathr/compute/builder/AnchorKeyFunctionBuilder.java b/feathr-compute/src/main/java/com/linkedin/feathr/compute/builder/AnchorKeyFunctionBuilder.java new file mode 100644 index 000000000..a48844c37 --- /dev/null +++ b/feathr-compute/src/main/java/com/linkedin/feathr/compute/builder/AnchorKeyFunctionBuilder.java @@ -0,0 +1,98 @@ +package com.linkedin.feathr.compute.builder; + +import com.google.common.base.Preconditions; +import com.linkedin.feathr.compute.MvelExpression; +import com.linkedin.feathr.compute.OfflineKeyFunction; +import com.linkedin.feathr.compute.SqlExpression; +import com.linkedin.feathr.compute.UserDefinedFunction; +import com.linkedin.feathr.core.config.producer.ExprType; +import com.linkedin.feathr.core.config.producer.anchors.AnchorConfig; +import com.linkedin.feathr.core.config.producer.anchors.AnchorConfigWithExtractor; +import com.linkedin.feathr.core.config.producer.anchors.AnchorConfigWithKey; +import com.linkedin.feathr.core.config.producer.anchors.AnchorConfigWithKeyExtractor; +import com.linkedin.feathr.core.config.producer.anchors.TypedKey; +import javax.annotation.Nonnull; + +public class AnchorKeyFunctionBuilder { + AnchorConfig _anchorConfig; + + public AnchorKeyFunctionBuilder(@Nonnull AnchorConfig anchorConfig) { + Preconditions.checkNotNull(anchorConfig); + _anchorConfig = anchorConfig; + } + + /** + * Build key function based on key field, extractor and key extractor of the anchor config. Following is all of the + * combinations that can be provided in the anchor config. + * + * 1. Anchor has key field only. We use the HOCON string of the keys to build Mvel or Spark function. + * 2. Anchor has extractor field only. We build UDF function. + * 3. Anchor has keyExtractor field only. We build UDF function. + * 4. Key field and extractor field co-exist in anchor config, it will be parsed as AnchorConfigWithKeyExtractor. We + * favor the key field to build Mvel/Spark function.. + * 5. Key extractor field and extractor field co-exist in anchor config, it will be parsed as AnchorConfigWithExtractor. + * We favor key extractor field to build UDF function. + * + * Refer to https://iwww.corp.linkedin.com/wiki/cf/display/ENGS/Frame+Offline+User+Guide#FrameOfflineUserGuide-KeyExtraction + * for more details on key extraction. + */ + public OfflineKeyFunction.KeyFunction build() { + if (_anchorConfig instanceof AnchorConfigWithKey) { + return buildFromAnchorConfigWithKey((AnchorConfigWithKey) _anchorConfig); + } else if (_anchorConfig instanceof AnchorConfigWithKeyExtractor) { + return buildFromConfigWithKeyExtractor((AnchorConfigWithKeyExtractor) _anchorConfig); + } else if (_anchorConfig instanceof AnchorConfigWithExtractor) { + return buildFromConfigWithExtractor((AnchorConfigWithExtractor) _anchorConfig); + } else { + throw new IllegalArgumentException(String.format("Anchor config %s has unsupported type %s", _anchorConfig, + _anchorConfig.getClass())); + } + } + + private OfflineKeyFunction.KeyFunction buildFromAnchorConfigWithKey(AnchorConfigWithKey anchorConfigWithKey) { + return buildFromTypedKey(anchorConfigWithKey.getTypedKey()); + } + + /** + * If extractor is present, we still favor the presence of key. If keys not present, we use extractor to build + * UDF function. + */ + private OfflineKeyFunction.KeyFunction buildFromConfigWithExtractor(AnchorConfigWithExtractor anchorConfigWithExtractor) { + if (anchorConfigWithExtractor.getTypedKey().isPresent()) { + return buildFromTypedKey(anchorConfigWithExtractor.getTypedKey().get()); + } else { + String udfClass = anchorConfigWithExtractor.getKeyExtractor().orElse(anchorConfigWithExtractor.getExtractor()); + UserDefinedFunction userDefinedFunction = new UserDefinedFunction().setClazz(udfClass); + OfflineKeyFunction.KeyFunction keyFunction = new OfflineKeyFunction.KeyFunction(); + keyFunction.setUserDefinedFunction(userDefinedFunction); + return keyFunction; + } + } + + private OfflineKeyFunction.KeyFunction buildFromTypedKey(TypedKey typedKey) { + String keyEpr = typedKey.getRawKeyExpr(); + if (typedKey.getKeyExprType() == ExprType.MVEL) { + MvelExpression mvelExpression = new MvelExpression().setMvel(keyEpr); + OfflineKeyFunction.KeyFunction keyFunction = new OfflineKeyFunction.KeyFunction(); + keyFunction.setMvelExpression(mvelExpression); + return keyFunction; + } else if (typedKey.getKeyExprType() == ExprType.SQL) { + SqlExpression sparkSqlExpression = new SqlExpression().setSql(keyEpr); + OfflineKeyFunction.KeyFunction keyFunction = new OfflineKeyFunction.KeyFunction(); + keyFunction.setSqlExpression(sparkSqlExpression); + return keyFunction; + } else { + throw new IllegalArgumentException(String.format("Typed key %s has unsupported expression type %s", + typedKey, typedKey.getKeyExprType())); + } + } + + private OfflineKeyFunction.KeyFunction buildFromConfigWithKeyExtractor(AnchorConfigWithKeyExtractor anchorConfigWithKeyExtractor) { + String keyExtractor = anchorConfigWithKeyExtractor.getKeyExtractor(); + UserDefinedFunction userDefinedFunction = new UserDefinedFunction().setClazz(keyExtractor); + OfflineKeyFunction.KeyFunction keyFunction = new OfflineKeyFunction.KeyFunction(); + keyFunction.setUserDefinedFunction(userDefinedFunction); + + return keyFunction; + } +} diff --git a/feathr-compute/src/main/java/com/linkedin/feathr/compute/builder/DefaultValueBuilder.java b/feathr-compute/src/main/java/com/linkedin/feathr/compute/builder/DefaultValueBuilder.java new file mode 100644 index 000000000..08dfb8d59 --- /dev/null +++ b/feathr-compute/src/main/java/com/linkedin/feathr/compute/builder/DefaultValueBuilder.java @@ -0,0 +1,34 @@ +package com.linkedin.feathr.compute.builder; + + +import com.google.common.base.Preconditions; +import com.linkedin.feathr.compute.FeatureValue; +import javax.annotation.Nonnull; + + +/** + * Builder class that builds {@link FeatureValue} pegasus object that is used as the default value of a feature. This + * default value will be used to populate feature data when missing data or error occurred while reading data. + */ +public class DefaultValueBuilder { + private static final DefaultValueBuilder INSTANCE = new DefaultValueBuilder(); + public static DefaultValueBuilder getInstance() { + return INSTANCE; + } + + /** + * Build default {@link FeatureValue}. Currently, only raw types, e.g., number, boolean, string, are supported. + * + */ + public FeatureValue build(@Nonnull Object featureValueObject) { + Preconditions.checkNotNull(featureValueObject); + FeatureValue featureValue = new FeatureValue(); + if (featureValueObject instanceof String) { + featureValue.setString((String) featureValueObject); + } else { + throw new IllegalArgumentException(String.format("Default value %s has a unsupported type %s." + + " Currently only support HOCON String.")); + } + return featureValue; + } +} diff --git a/feathr-compute/src/main/java/com/linkedin/feathr/compute/builder/FeatureTypeTensorFeatureFormatBuilder.java b/feathr-compute/src/main/java/com/linkedin/feathr/compute/builder/FeatureTypeTensorFeatureFormatBuilder.java new file mode 100644 index 000000000..ea7ef3f42 --- /dev/null +++ b/feathr-compute/src/main/java/com/linkedin/feathr/compute/builder/FeatureTypeTensorFeatureFormatBuilder.java @@ -0,0 +1,122 @@ +package com.linkedin.feathr.compute.builder; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Sets; +import com.linkedin.feathr.compute.Dimension; +import com.linkedin.feathr.compute.DimensionArray; +import com.linkedin.feathr.compute.DimensionType; +import com.linkedin.feathr.compute.TensorCategory; +import com.linkedin.feathr.compute.ValueType; +import com.linkedin.feathr.core.config.producer.definitions.FeatureType; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import javax.annotation.Nonnull; + + +/** + * Builder class for {@link com.linkedin.feathr.compute.TensorFeatureFormat} object given frame feature type. + * In this case, the builder will map feature types to Quince tensor type. For example, frame feature type Numeric will + * be mapped to Dense Tensor, with float value type and empty dimension. Detailed mapping rule is documented in: + * https://iwww.corp.linkedin.com/wiki/cf/display/ENGS/Frame+Auto-Tensorization+Type+Conversion+Rules + */ +class FeatureTypeTensorFeatureFormatBuilder extends TensorFeatureFormatBuilder { + public static final Set VALID_FEATURE_TYPES = Sets.immutableEnumSet(FeatureType.BOOLEAN, + FeatureType.NUMERIC, FeatureType.CATEGORICAL, FeatureType.CATEGORICAL_SET, FeatureType.VECTOR, + FeatureType.DENSE_VECTOR, FeatureType.TERM_VECTOR); + private static final int UNKNOWN_DIMENSION_SIZE = -1; + + private FeatureType _featureType; + private Optional _embeddingSize; + + public FeatureTypeTensorFeatureFormatBuilder(@Nonnull FeatureType featureType) { + super(); + Preconditions.checkNotNull(featureType); + _featureType = featureType; + _embeddingSize = Optional.empty(); + } + + /** + * Constructor with embedding size. This should be used when feature has SlidingWindowEmbeddingAggregation + * transformation function and embedding size is present. + * @param featureType feature type. + * @param embeddingSize embedding size. + */ + public FeatureTypeTensorFeatureFormatBuilder(@Nonnull FeatureType featureType, int embeddingSize) { + super(); + Preconditions.checkNotNull(featureType); + _featureType = featureType; + _embeddingSize = Optional.of(embeddingSize); + } + + + @Override + void validCheck() { + if (!VALID_FEATURE_TYPES.contains(_featureType)) { + throw new IllegalArgumentException(String.format("Invalid feature type %s for TensorFeatureFormat. Valid types " + + "are %s", _featureType, VALID_FEATURE_TYPES)); + } + if (_embeddingSize.isPresent() && _featureType != FeatureType.DENSE_VECTOR) { + throw new IllegalArgumentException(String.format("Dense vector feature type is expected when embedding size" + + " is set. But provided type is %s", _featureType)); + } + } + + @Override + ValueType buildValueType() { + return ValueType.FLOAT; + } + + @Override + DimensionArray buildDimensions() { + List dimensions = new ArrayList<>(); + //For scalar, we set an empty dimension since dimension is pointless in this case. + if (_featureType == FeatureType.NUMERIC || _featureType == FeatureType.BOOLEAN) { + return new DimensionArray(dimensions); + } + Dimension dimension = new Dimension(); + if (_embeddingSize.isPresent()) { + //Set embedding size as shape when present. + dimension.setShape(_embeddingSize.get()); + } else { + //For other feature types, we set dimension as -1, indicating the dimension is unknown. + dimension.setShape(UNKNOWN_DIMENSION_SIZE); + } + switch (_featureType) { + case CATEGORICAL: + case CATEGORICAL_SET: + case TERM_VECTOR: + dimension.setType(DimensionType.STRING); + break; + case VECTOR: + case DENSE_VECTOR: + dimension.setType(DimensionType.INT); + break; + default: + //This should not happen + throw new IllegalArgumentException(String.format("Feature type %s is not supported. Valid types are: %s", + _featureType, VALID_FEATURE_TYPES)); + } + dimensions.add(dimension); + return new DimensionArray(dimensions); + } + + @Override + TensorCategory buildTensorCategory() { + switch (_featureType) { + case BOOLEAN: + case NUMERIC: + case VECTOR: + case DENSE_VECTOR: + return TensorCategory.DENSE; + case CATEGORICAL: + case CATEGORICAL_SET: + case TERM_VECTOR: + return TensorCategory.SPARSE; + default: + throw new IllegalArgumentException(String.format("Feature type %s is not supported. Valid types are: %s", + _featureType, VALID_FEATURE_TYPES)); + } + } +} diff --git a/feathr-compute/src/main/java/com/linkedin/feathr/compute/builder/FeatureVersionBuilder.java b/feathr-compute/src/main/java/com/linkedin/feathr/compute/builder/FeatureVersionBuilder.java new file mode 100644 index 000000000..04dd523b7 --- /dev/null +++ b/feathr-compute/src/main/java/com/linkedin/feathr/compute/builder/FeatureVersionBuilder.java @@ -0,0 +1,82 @@ +package com.linkedin.feathr.compute.builder; + + +import com.google.common.base.Preconditions; +import com.linkedin.feathr.compute.FeatureVersion; +import com.linkedin.feathr.core.config.producer.anchors.FeatureConfig; +import com.linkedin.feathr.core.config.producer.derivations.DerivationConfig; +import java.util.Optional; +import javax.annotation.Nonnull; + + +/** + * Builder class that builds {@link FeatureVersion} pegasus object, which models a specific version of a feature. A + * Feature can have multiple FeatureVersions. Versioning of a feature is declared by feature producers per semantic + * versioning. Every time the definition of a feature changes, a new FeatureVersion should be created. Each + * FeatureVersion enclosed attributes that don't change across environments. + */ +public class FeatureVersionBuilder { + private final TensorFeatureFormatBuilderFactory _tensorFeatureFormatBuilderFactory; + private final DefaultValueBuilder _defaultValueBuilder; + private final FrameFeatureTypeBuilder _featureTypeBuilder; + + public FeatureVersionBuilder(@Nonnull TensorFeatureFormatBuilderFactory tensorFeatureFormatBuilderFactory, + @Nonnull DefaultValueBuilder defaultValueBuilder, @Nonnull FrameFeatureTypeBuilder featureTypeBuilder) { + Preconditions.checkNotNull(tensorFeatureFormatBuilderFactory); + Preconditions.checkNotNull(defaultValueBuilder); + Preconditions.checkNotNull(featureTypeBuilder); + _tensorFeatureFormatBuilderFactory = tensorFeatureFormatBuilderFactory; + _defaultValueBuilder = defaultValueBuilder; + _featureTypeBuilder = featureTypeBuilder; + } + + /** + * Build {@link FeatureVersion} for anchored feature. + */ + public FeatureVersion build(@Nonnull FeatureConfig featureConfig) { + Preconditions.checkNotNull(featureConfig); + FeatureVersion featureVersion = new FeatureVersion(); + Optional tensorFeatureFormatBuilder = + _tensorFeatureFormatBuilderFactory.getBuilder(featureConfig); + tensorFeatureFormatBuilder.ifPresent(builder -> + featureVersion.setFormat(builder.build())); + /* + * Here if the FeatureTypeConfig contains a legacy feature type, set the type of FeatureVersion. + * In downstream usage, if the `type` field exist, it will be used as the user defined feature type. + * If the `type` field does not exist, we use the `format` field as the user defined tensor feature type. + * + * We still want to build the above `format` field even when the feature type is legacy type. + * Because the `format` field contains other information such as embedding size for SWA feature. + */ + featureConfig.getFeatureTypeConfig().flatMap(_featureTypeBuilder::build).ifPresent(featureVersion::setType); + Optional defaultValue = featureConfig.getDefaultValue(); + defaultValue.ifPresent( + value -> featureVersion.setDefaultValue(_defaultValueBuilder.build(value)) + ); + return featureVersion; + } + + /** + * Build {@link FeatureVersion} for derived feature. + */ + public FeatureVersion build(@Nonnull DerivationConfig derivationConfig) { + Preconditions.checkNotNull(derivationConfig); + + FeatureVersion featureVersion = new FeatureVersion(); + Optional tensorFeatureFormatBuilder = + _tensorFeatureFormatBuilderFactory.getBuilder(derivationConfig); + tensorFeatureFormatBuilder.ifPresent(builder -> + featureVersion.setFormat(builder.build())); + /* + * Here if the FeatureTypeConfig contains a legacy feature type, set the type of FeatureVersion. + * In downstream usage, if the `type` field exist, it will be used as the user defined feature type. + * If the `type` field does not exist, we use the `format` field as the user defined tensor feature type. + * + * We still want to build the above `format` field even when the feature type is legacy type. + * Because the `format` field contains other information such as embedding size for SWA feature. + */ + derivationConfig.getFeatureTypeConfig().flatMap(_featureTypeBuilder::build).ifPresent(featureVersion::setType); + // TODO - add default value support for derived feature + return featureVersion; + } +} diff --git a/feathr-compute/src/main/java/com/linkedin/feathr/compute/builder/FrameFeatureTypeBuilder.java b/feathr-compute/src/main/java/com/linkedin/feathr/compute/builder/FrameFeatureTypeBuilder.java new file mode 100644 index 000000000..fe77ca7e7 --- /dev/null +++ b/feathr-compute/src/main/java/com/linkedin/feathr/compute/builder/FrameFeatureTypeBuilder.java @@ -0,0 +1,47 @@ +package com.linkedin.feathr.compute.builder; + +import com.google.common.base.Preconditions; +import com.linkedin.feathr.compute.FrameFeatureType; +import com.linkedin.feathr.core.config.producer.common.FeatureTypeConfig; +import java.util.Optional; +import javax.annotation.Nonnull; + +/** + * Builder class that builds {@link FrameFeatureType} pegasus object that is used as the legacy type of a feature. + */ +public class FrameFeatureTypeBuilder { + + private static final FrameFeatureTypeBuilder INSTANCE = new FrameFeatureTypeBuilder(); + + public static FrameFeatureTypeBuilder getInstance() { + return INSTANCE; + } + + private FrameFeatureTypeBuilder() { + // singleton constructor + } + + /** + * Build {@link FrameFeatureType} pegasus object if [[FeatureTypeConfig]] contains legacy feature types + */ + public Optional build(@Nonnull FeatureTypeConfig featureTypeConfig) { + Preconditions.checkNotNull(featureTypeConfig); + Preconditions.checkNotNull(featureTypeConfig.getFeatureType()); + + FrameFeatureType featureType; + + if (featureTypeConfig.getFeatureType() == com.linkedin.feathr.core.config.producer.definitions.FeatureType.UNSPECIFIED) { + throw new IllegalArgumentException("UNSPECIFIED feature type should not be used in feature config"); + } else if (TensorTypeTensorFeatureFormatBuilder.VALID_FEATURE_TYPES.contains(featureTypeConfig.getFeatureType())) { + // high level type is always TENSOR, for DENSE_TENSOR, SPARSE_TENSOR, and RAGGED_TENSOR + featureType = FrameFeatureType.TENSOR; + } else { + // For legacy type, since there is a 1:1 mapping of the types between com.linkedin.feathr.common.types.FeatureType + // and com.linkedin.feathr.core.config.producer.definitions.FeatureType for the rest types, + // build directly by name + featureType = FrameFeatureType.valueOf(featureTypeConfig.getFeatureType().toString()); + } + + return Optional.of(featureType); + } +} diff --git a/feathr-compute/src/main/java/com/linkedin/feathr/compute/builder/SlidingWindowAggregationBuilder.java b/feathr-compute/src/main/java/com/linkedin/feathr/compute/builder/SlidingWindowAggregationBuilder.java new file mode 100644 index 000000000..8e9590356 --- /dev/null +++ b/feathr-compute/src/main/java/com/linkedin/feathr/compute/builder/SlidingWindowAggregationBuilder.java @@ -0,0 +1,88 @@ +package com.linkedin.feathr.compute.builder; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import com.linkedin.feathr.compute.AggregationType; +import com.linkedin.feathr.compute.LateralViewArray; +import com.linkedin.feathr.compute.SlidingWindowFeature; +import com.linkedin.feathr.compute.SqlExpression; +import com.linkedin.feathr.compute.Window; +import com.linkedin.feathr.core.config.TimeWindowAggregationType; +import java.util.HashMap; +import java.util.Map; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import org.checkerframework.checker.nullness.qual.NonNull; + + +public class SlidingWindowAggregationBuilder extends SlidingWindowOperationBuilder { + private static final SlidingWindowAggregationBuilder + INSTANCE = new SlidingWindowAggregationBuilder(); + + private static final Map AGGREGATION_TYPE_MAP = new HashMap() { + { + put(TimeWindowAggregationType.AVG, AggregationType.AVG); + put(TimeWindowAggregationType.MIN, AggregationType.MIN); + put(TimeWindowAggregationType.MAX, AggregationType.MAX); + put(TimeWindowAggregationType.SUM, AggregationType.SUM); + put(TimeWindowAggregationType.COUNT, AggregationType.COUNT); + put(TimeWindowAggregationType.LATEST, AggregationType.LATEST); + put(TimeWindowAggregationType.AVG_POOLING, AggregationType.AVG_POOLING); + put(TimeWindowAggregationType.MAX_POOLING, AggregationType.MAX_POOLING); + put(TimeWindowAggregationType.MIN_POOLING, AggregationType.MIN_POOLING); + }}; + + private SlidingWindowAggregationBuilder() { + } + + public static SlidingWindowAggregationBuilder getInstance() { + return INSTANCE; + } + + public static boolean isSlidingWindowAggregationType(TimeWindowAggregationType timeWindowAggregationType) { + return AGGREGATION_TYPE_MAP.containsKey(timeWindowAggregationType); + } + + @Override + SlidingWindowFeature buildSlidingWindowOperationObject(@Nullable String filterStr, @Nullable String groupByStr, + @Nullable Integer limit, @Nonnull Window window, @NonNull String targetColumnStr, + @NonNull LateralViewArray lateralViews, @NonNull TimeWindowAggregationType timeWindowAggregationType) { + Preconditions.checkNotNull(window); + Preconditions.checkNotNull(timeWindowAggregationType); + Preconditions.checkNotNull(targetColumnStr); + Preconditions.checkNotNull(lateralViews); + SlidingWindowFeature slidingWindowAggregation = new SlidingWindowFeature(); + if (filterStr != null) { + SqlExpression sparkSqlExpression = new SqlExpression(); + sparkSqlExpression.setSql(filterStr); + SlidingWindowFeature.Filter filter = new SlidingWindowFeature.Filter(); + filter.setSqlExpression(sparkSqlExpression); + slidingWindowAggregation.setFilter(filter); + } + if (groupByStr != null) { + SlidingWindowFeature.GroupBy groupBy = new SlidingWindowFeature.GroupBy(); + SqlExpression sparkSqlExpression = new SqlExpression(); + sparkSqlExpression.setSql(groupByStr); + groupBy.setSqlExpression(sparkSqlExpression); + slidingWindowAggregation.setGroupBy(groupBy); + } + if (limit != null) { + slidingWindowAggregation.setLimit(limit); + } + slidingWindowAggregation.setWindow(window); + AggregationType aggregationType = AGGREGATION_TYPE_MAP.get(timeWindowAggregationType); + if (aggregationType == null) { + throw new IllegalArgumentException(String.format("Unsupported aggregation type %s for SlidingWindowAggregation." + + "Supported types are %s", timeWindowAggregationType, AGGREGATION_TYPE_MAP.keySet())); + } + slidingWindowAggregation.setAggregationType(aggregationType); + SlidingWindowFeature.TargetColumn targetColumn = new SlidingWindowFeature.TargetColumn(); + SqlExpression sparkSqlExpression = new SqlExpression(); + sparkSqlExpression.setSql(targetColumnStr); + targetColumn.setSqlExpression(sparkSqlExpression); + slidingWindowAggregation.setTargetColumn(targetColumn); + slidingWindowAggregation.setLateralViews(lateralViews); + return slidingWindowAggregation; + } +} + diff --git a/feathr-compute/src/main/java/com/linkedin/feathr/compute/builder/SlidingWindowOperationBuilder.java b/feathr-compute/src/main/java/com/linkedin/feathr/compute/builder/SlidingWindowOperationBuilder.java new file mode 100644 index 000000000..04250c5ba --- /dev/null +++ b/feathr-compute/src/main/java/com/linkedin/feathr/compute/builder/SlidingWindowOperationBuilder.java @@ -0,0 +1,142 @@ +package com.linkedin.feathr.compute.builder; + +import com.google.common.annotations.VisibleForTesting; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.feathr.compute.LateralView; +import com.linkedin.feathr.compute.LateralViewArray; +import com.linkedin.feathr.compute.SqlExpression; +import com.linkedin.feathr.compute.Unit; +import com.linkedin.feathr.compute.Window; +import com.linkedin.feathr.core.config.TimeWindowAggregationType; +import com.linkedin.feathr.core.config.producer.ExprType; +import com.linkedin.feathr.core.config.producer.TypedExpr; +import com.linkedin.feathr.core.config.producer.anchors.AnchorConfig; +import com.linkedin.feathr.core.config.producer.anchors.AnchorConfigWithKey; +import com.linkedin.feathr.core.config.producer.anchors.AnchorConfigWithKeyExtractor; +import com.linkedin.feathr.core.config.producer.anchors.LateralViewParams; +import com.linkedin.feathr.core.config.producer.anchors.TimeWindowFeatureConfig; +import java.time.Duration; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import org.checkerframework.checker.nullness.qual.Nullable; + + +/** + * Builder for SlidingWindowOperation (also known as Sliding Window Aggregation). It models how feature value is + * aggregated from a set of data (called fact data) in a certain interval of time. This builder can be used to build. + */ +abstract class SlidingWindowOperationBuilder { + private Optional _filter = Optional.empty(); + private Optional _groupBy = Optional.empty(); + private Optional _limit = Optional.empty(); + private Window _window; + private String _targetColumn; + private LateralViewArray _lateralViews; + private TimeWindowAggregationType _timeWindowAggregationType; + + abstract SLIDING_WINDOW_OPERATION buildSlidingWindowOperationObject(String filter, String groupBy, Integer limit, + Window window, String targetColumn, LateralViewArray lateralViews, TimeWindowAggregationType aggregationType); + + /** + * Build SlidingWindowOperation. It sets window, targetColumn, groupBy, limit and aggregationType given + * {@link TimeWindowFeatureConfig}, and sets lateralViews given {@link AnchorConfig}. Filter comes from either + * TimeWindowFeatureConfig or AnchorConfig. Setting it in both places will cause exception. Currently, Frame only + * supports single laterView, but it is modeled as an array for future extensibility. + */ + public SLIDING_WINDOW_OPERATION build(TimeWindowFeatureConfig timeWindowFeatureConfig, AnchorConfig anchorConfig) { + _timeWindowAggregationType = timeWindowFeatureConfig.getAggregation(); + _filter = timeWindowFeatureConfig.getTypedFilter().map( + typedFilter -> { + if (typedFilter.getExprType() != ExprType.SQL) { + throw new IllegalArgumentException(String.format("Trying to set filter expr %s with an invalid expression " + + "type %s. The only supported type is SQL. Provided feature config is %s", typedFilter.getExpr(), + typedFilter.getExprType(), timeWindowFeatureConfig)); + } + return typedFilter.getExpr(); + } + ); + _groupBy = timeWindowFeatureConfig.getGroupBy(); + _limit = timeWindowFeatureConfig.getLimit(); + _window = buildWindow(timeWindowFeatureConfig.getWindow()); + TypedExpr columnExpr = timeWindowFeatureConfig.getTypedColumnExpr(); + if (columnExpr.getExprType() != ExprType.SQL) { + throw new IllegalArgumentException(String.format("Trying to set target column expr %s with an invalid expression " + + "type %s. The only supported type is SQL. Provided feature config is %s", columnExpr.getExpr(), + columnExpr.getExprType(), timeWindowFeatureConfig)); + } + _targetColumn = columnExpr.getExpr(); + Optional lateralViewParamsOptional; + if (anchorConfig instanceof AnchorConfigWithKey) { + AnchorConfigWithKey anchorConfigWithKey = (AnchorConfigWithKey) anchorConfig; + lateralViewParamsOptional = anchorConfigWithKey.getLateralViewParams(); + } else if (anchorConfig instanceof AnchorConfigWithKeyExtractor) { + AnchorConfigWithKeyExtractor anchorConfigWithKeyExtractor = (AnchorConfigWithKeyExtractor) anchorConfig; + lateralViewParamsOptional = anchorConfigWithKeyExtractor.getLateralViewParams(); + } else { + lateralViewParamsOptional = Optional.empty(); + } + + if (lateralViewParamsOptional.isPresent()) { + _lateralViews = buildLateralViews(lateralViewParamsOptional.get()); + //If filter field of lateralView is present and top level filter in feature config is not set yet, we will use the + //lateralView filter as the SWA filter. + //lateralView filter and top level filters should not be present at the same time. + if (lateralViewParamsOptional.get().getFilter().isPresent()) { + if (_filter.isPresent()) { + throw new IllegalArgumentException(String.format("Filter present in both feature config %s and " + + "lateral view %s", timeWindowFeatureConfig, lateralViewParamsOptional.get())); + } else { + _filter = lateralViewParamsOptional.get().getFilter(); + } + } + } else { + _lateralViews = new LateralViewArray(); + } + + return buildSlidingWindowOperationObject(_filter.orElse(null), _groupBy.orElse(null), + _limit.orElse(null), _window, _targetColumn, _lateralViews, + _timeWindowAggregationType); + } + + @VisibleForTesting + protected Window buildWindow(Duration windowDuration) { + long size = windowDuration.getSeconds(); + Unit unit = Unit.SECOND; + if (size > 0 && size % 60 == 0) { + size = size / 60; + unit = Unit.MINUTE; + if (size % 60 == 0) { + size = size / 60; + unit = Unit.HOUR; + if (size % 24 == 0) { + size = size / 24; + unit = Unit.DAY; + } + } + } + if (size > Integer.MAX_VALUE) { + throw new IllegalArgumentException(String.format("window size %d too big", size)); + } + Window window = new Window(); + window.setSize((int) size); + window.setUnit(unit); + return window; + } + + @VisibleForTesting + protected LateralViewArray buildLateralViews(@Nullable LateralViewParams lateralViewParams) { + if (lateralViewParams == null) { + return new LateralViewArray(); + } + LateralView lateralView = new LateralView(); + lateralView.setVirtualTableAlias(lateralViewParams.getItemAlias()); + LateralView.TableGeneratingFunction tableGeneratingFunction = new LateralView.TableGeneratingFunction(); + SqlExpression sparkSqlExpression = new SqlExpression(); + sparkSqlExpression.setSql(lateralViewParams.getDef()); + tableGeneratingFunction.setSqlExpression(sparkSqlExpression); + lateralView.setTableGeneratingFunction(tableGeneratingFunction); + List lateralViews = Collections.singletonList(lateralView); + return new LateralViewArray(lateralViews); + } +} diff --git a/feathr-compute/src/main/java/com/linkedin/feathr/compute/builder/TensorFeatureFormatBuilder.java b/feathr-compute/src/main/java/com/linkedin/feathr/compute/builder/TensorFeatureFormatBuilder.java new file mode 100644 index 000000000..914d5da0b --- /dev/null +++ b/feathr-compute/src/main/java/com/linkedin/feathr/compute/builder/TensorFeatureFormatBuilder.java @@ -0,0 +1,45 @@ +package com.linkedin.feathr.compute.builder; + +import com.linkedin.feathr.compute.DimensionArray; +import com.linkedin.feathr.compute.TensorCategory; +import com.linkedin.feathr.compute.TensorFeatureFormat; +import com.linkedin.feathr.compute.ValueType; + + +/** + * Builder class that builds {@link TensorFeatureFormat} pegasus object, which define the format of feature data. It + * unifies frame feature type (https://iwww.corp.linkedin.com/wiki/cf/display/ENGS/Feature+Representation+and+Feature+Type+System) + * and Quince Tensor type (https://iwww.corp.linkedin.com/wiki/cf/display/ENGS/Frame+Tensor+Tutorial). + */ +public abstract class TensorFeatureFormatBuilder { + public TensorFeatureFormat build() { + validCheck(); + TensorFeatureFormat tensorFeatureFormat = new TensorFeatureFormat(); + tensorFeatureFormat.setValueType(buildValueType()); + tensorFeatureFormat.setDimensions(buildDimensions()); + tensorFeatureFormat.setTensorCategory(buildTensorCategory()); + return tensorFeatureFormat; + } + + /** + * build {@link ValueType} pegasus object that defines type of the value column. + */ + abstract ValueType buildValueType(); + + /** + * build {@link DimensionArray}. A tensor can have 0 to n dimensions. Each element of this array represent the + * attributes of one dimension. For scalar (rank-0) scalar, this should return an empty array. + */ + abstract DimensionArray buildDimensions(); + + /** + * build {@link TensorCategory}, which defines the type of tensor, for example, dense tensor. + */ + abstract TensorCategory buildTensorCategory(); + + /** + * Valid the arguments passed in from subclass constructor, to make sure a valid {@link TensorFeatureFormat} can be + * built. + */ + abstract void validCheck(); +} diff --git a/feathr-compute/src/main/java/com/linkedin/feathr/compute/builder/TensorFeatureFormatBuilderFactory.java b/feathr-compute/src/main/java/com/linkedin/feathr/compute/builder/TensorFeatureFormatBuilderFactory.java new file mode 100644 index 000000000..db564a4d4 --- /dev/null +++ b/feathr-compute/src/main/java/com/linkedin/feathr/compute/builder/TensorFeatureFormatBuilderFactory.java @@ -0,0 +1,102 @@ +package com.linkedin.feathr.compute.builder; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Sets; +import com.linkedin.feathr.core.config.producer.anchors.FeatureConfig; +import com.linkedin.feathr.core.config.producer.anchors.TimeWindowFeatureConfig; +import com.linkedin.feathr.core.config.producer.common.FeatureTypeConfig; +import com.linkedin.feathr.core.config.producer.definitions.FeatureType; +import com.linkedin.feathr.core.config.producer.derivations.DerivationConfig; +import java.util.Optional; +import java.util.Set; +import javax.annotation.Nonnull; + + +/** + * Factory class of {@link TensorTypeTensorFeatureFormatBuilder}. Given different feature type, it will return + * different implementations or a empty builder. + */ +public class TensorFeatureFormatBuilderFactory { + public TensorFeatureFormatBuilderFactory() { + } + + /** + * Get builder based on the featureType stored in the featureTypeConfig of the FeatureConfig, with one special case: + * If feature type is not provided, but embedding size is set, we will build + * a {@link FeatureTypeTensorFeatureFormatBuilder} with feature type set as DENSE_VECTOR. + * If feature type is not provided, and embedding size is not set, return empty build + */ + public Optional getBuilder(@Nonnull FeatureConfig featureConfig) { + Preconditions.checkNotNull(featureConfig); + Optional featureTypeConfigOptional = featureConfig.getFeatureTypeConfig(); + + // embeddingSize is set only when feature is a Sliding Window Aggregation feature, and that feature contains + // embeddingSize field + Optional embeddingSizeOptional = (featureConfig instanceof TimeWindowFeatureConfig) + ? ((TimeWindowFeatureConfig) featureConfig).getEmbeddingSize() : Optional.empty(); + + // Special case: if feature type is not provided + if (!featureTypeConfigOptional.isPresent()) { + // If embedding size is set in a Sliding Window Aggregation feature, we will build + // a {@link FeatureTypeTensorFeatureFormatBuilder} with feature type set as DENSE_VECTOR, since embedding implies it + // is a DENSE_VECTOR per Frame feature type. + // Else build empty + return embeddingSizeOptional.map( + embeddingSize -> new FeatureTypeTensorFeatureFormatBuilder(FeatureType.DENSE_VECTOR, embeddingSize) + ); + } else { + return Optional.ofNullable( + getBuilder(featureTypeConfigOptional.get(), embeddingSizeOptional.orElse(null), featureConfig.toString()) + ); + } + } + + /** + * Get builder based on the featureType stored in the featureTypeConfig of the derivationConfig + */ + public Optional getBuilder(@Nonnull DerivationConfig derivationConfig) { + Preconditions.checkNotNull(derivationConfig); + return derivationConfig.getFeatureTypeConfig().map( + featureTypeConfig -> getBuilder(featureTypeConfig, null, derivationConfig.toString()) + ); + } + + /** + * Get builder based on the featureType stored in the featureTypeConfig: + * 1. If the feature type is a legacy frame feature type, we will return + * a {@link FeatureTypeTensorFeatureFormatBuilder}, which maps frame feature type to Quince Tensor type and build + * {@link com.linkedin.feathr.compute.TensorFeatureFormat}. + * + * 2. If the feature type is a Quince Tensor type, we return {@link TensorTypeTensorFeatureFormatBuilder}. + * + * 3. If feature type is TENSOR, it means a FML feature, return empty build + * + * 4. If feature type is not supported, throw exception + */ + private TensorFeatureFormatBuilder getBuilder(FeatureTypeConfig featureTypeConfig, Integer embeddingSize, String configRepresentation) { + // embeddingSize can be null + Preconditions.checkNotNull(featureTypeConfig); + Preconditions.checkNotNull(configRepresentation); + + FeatureType featureType = featureTypeConfig.getFeatureType(); + if (FeatureTypeTensorFeatureFormatBuilder.VALID_FEATURE_TYPES.contains(featureType)) { + return embeddingSize != null ? new FeatureTypeTensorFeatureFormatBuilder(featureType, embeddingSize) + : new FeatureTypeTensorFeatureFormatBuilder(featureType); + } else if (TensorTypeTensorFeatureFormatBuilder.VALID_FEATURE_TYPES.contains(featureType)) { + return embeddingSize != null ? new TensorTypeTensorFeatureFormatBuilder(featureTypeConfig, embeddingSize) + : new TensorTypeTensorFeatureFormatBuilder(featureTypeConfig); + } else if (featureType == FeatureType.TENSOR) { + return null; + } else if (featureType == FeatureType.UNSPECIFIED) { + throw new IllegalArgumentException("UNSPECIFIED feature type should not be used in config:" + configRepresentation); + } else { + Set supportedFeatureTypes = Sets.union( + FeatureTypeTensorFeatureFormatBuilder.VALID_FEATURE_TYPES, + TensorTypeTensorFeatureFormatBuilder.VALID_FEATURE_TYPES); + supportedFeatureTypes.add(FeatureType.TENSOR); + throw new IllegalArgumentException(String.format("Feature type %s is not supported. The config is " + + "is %s. Supported feature type are %s", featureType, configRepresentation, + supportedFeatureTypes)); + } + } +} diff --git a/feathr-compute/src/main/java/com/linkedin/feathr/compute/builder/TensorTypeTensorFeatureFormatBuilder.java b/feathr-compute/src/main/java/com/linkedin/feathr/compute/builder/TensorTypeTensorFeatureFormatBuilder.java new file mode 100644 index 000000000..b662eabc8 --- /dev/null +++ b/feathr-compute/src/main/java/com/linkedin/feathr/compute/builder/TensorTypeTensorFeatureFormatBuilder.java @@ -0,0 +1,149 @@ +package com.linkedin.feathr.compute.builder; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Sets; +import com.linkedin.feathr.compute.Dimension; +import com.linkedin.feathr.compute.DimensionArray; +import com.linkedin.feathr.compute.DimensionType; +import com.linkedin.feathr.compute.TensorCategory; +import com.linkedin.feathr.compute.ValueType; +import com.linkedin.feathr.core.config.producer.common.FeatureTypeConfig; +import com.linkedin.feathr.core.config.producer.definitions.FeatureType; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import javax.annotation.Nonnull; + + +/** + * Builder class for {@link com.linkedin.feathr.compute.TensorFeatureFormat} object given + * {@link FeatureTypeConfig}, when a Quince Tensor type is provided in the feature definition. + */ +public class TensorTypeTensorFeatureFormatBuilder extends TensorFeatureFormatBuilder { + public static final Set VALID_FEATURE_TYPES = Sets.immutableEnumSet(FeatureType.DENSE_TENSOR, + FeatureType.SPARSE_TENSOR, FeatureType.RAGGED_TENSOR); + + private static final int UNKNOWN_DIMENSION_SIZE = -1; + private FeatureTypeConfig _featureTypeConfig; + private Optional _embeddingSize; + + public TensorTypeTensorFeatureFormatBuilder(@Nonnull FeatureTypeConfig featureTypeConfig) { + super(); + Preconditions.checkNotNull(featureTypeConfig); + _featureTypeConfig = featureTypeConfig; + _embeddingSize = Optional.empty(); + } + + /** + * Constructor with embedding size. This should be used when feature has SlidingWindowEmbeddingAggregation + * transformation function and embedding size is present. + * @param featureTypeConfig feature type config. + * @param embeddingSize embedding size. + */ + public TensorTypeTensorFeatureFormatBuilder(@Nonnull FeatureTypeConfig featureTypeConfig, int embeddingSize) { + super(); + Preconditions.checkNotNull(featureTypeConfig); + _featureTypeConfig = featureTypeConfig; + _embeddingSize = Optional.ofNullable(embeddingSize); + } + + /** + * Valid if provided {@link FeatureTypeConfig}. shapes and dimension types both need to present or not present at the + * same time. If they both exist, they need to have the same size. The feature type need to be either Dense Tensor, + * Sparse Tenser or Ragged Tensor. If embedding size is set, validate if an one-dimensional shape is provided and if + * shape[0] matches embedding size. + */ + @Override + void validCheck() { + if (!_featureTypeConfig.getDimensionTypes().isPresent() && _featureTypeConfig.getShapes().isPresent()) { + throw new IllegalArgumentException(String.format("Shapes are provided but Dimensions are not provided in config" + + "%s", _featureTypeConfig)); + } + if (_featureTypeConfig.getDimensionTypes().isPresent() && _featureTypeConfig.getShapes().isPresent() + && _featureTypeConfig.getDimensionTypes().get().size() != _featureTypeConfig.getShapes().get().size()) { + throw new IllegalArgumentException(String.format("The size of dimension types %d and size of shapes %d are " + + "unequal in config %s", _featureTypeConfig.getDimensionTypes().get().size(), + _featureTypeConfig.getShapes().get().size(), _featureTypeConfig)); + } + if (_featureTypeConfig.getShapes().isPresent()) { + if (!_featureTypeConfig.getShapes().get() + .stream().allMatch(shape -> shape > 0 || shape == UNKNOWN_DIMENSION_SIZE)) { + throw new IllegalArgumentException(String.format("Shapes should be larger than 0 or -1. Provided shapes: %s", + _featureTypeConfig.getShapes().get())); + } + } + + FeatureType featureType = _featureTypeConfig.getFeatureType(); + if (!VALID_FEATURE_TYPES.contains(featureType)) { + throw new IllegalArgumentException(String.format("Invalid feature type %s for TensorFeatureFormat in config %s. " + + "Valid types are %s", featureType, _featureTypeConfig, VALID_FEATURE_TYPES)); + } + + //Validate shapes when embedding size is set. + if (_embeddingSize.isPresent()) { + if (!_featureTypeConfig.getShapes().isPresent()) { + throw new IllegalArgumentException(String.format("Shapes are not present while embedding size %d is set", + _embeddingSize.get())); + } + if (_featureTypeConfig.getShapes().get().size() != 1) { + throw new IllegalArgumentException(String.format("One dimensional shape is expected when embedding size" + + " is set, but %s is provided", _featureTypeConfig.getShapes().get())); + } + if (!_featureTypeConfig.getShapes().get().get(0).equals(_embeddingSize.get())) { + throw new IllegalArgumentException(String.format("Embedding size %s and shape size %s don't match", + _embeddingSize.get(), _featureTypeConfig.getShapes().get().get(0))); + } + if (_featureTypeConfig.getFeatureType() != FeatureType.DENSE_TENSOR) { + throw new IllegalArgumentException(String.format("Dense tensor feature type is expected when embedding size" + + " is set. But provided type is %s", _featureTypeConfig.getFeatureType())); + } + } + } + + @Override + ValueType buildValueType() { + if (!_featureTypeConfig.getValType().isPresent()) { + throw new IllegalArgumentException(String.format("Value type is not specified in feature type config %s. " + + "This is required to build TensorFeatureFormat", _featureTypeConfig)); + } + return ValueType.valueOf(_featureTypeConfig.getValType().get().toUpperCase()); + } + + @Override + DimensionArray buildDimensions() { + List dimensions = new ArrayList<>(); + if (_featureTypeConfig.getDimensionTypes().isPresent()) { + for (int i = 0; i < _featureTypeConfig.getDimensionTypes().get().size(); i++) { + Dimension dimension = new Dimension(); + //TODO - 11753) set shapes when emebedding size of lateral view is present + if (_featureTypeConfig.getShapes().isPresent()) { + dimension.setShape(_featureTypeConfig.getShapes().get().get(i)); + } else { + dimension.setShape(UNKNOWN_DIMENSION_SIZE); + } + DimensionType dimensionType = DimensionType.valueOf( + _featureTypeConfig.getDimensionTypes().get().get(i).toUpperCase()); + dimension.setType(dimensionType); + dimensions.add(dimension); + } + } + return new DimensionArray(dimensions); + } + + @Override + TensorCategory buildTensorCategory() { + FeatureType featureType = _featureTypeConfig.getFeatureType(); + switch (featureType) { + case DENSE_TENSOR: + return TensorCategory.DENSE; + case SPARSE_TENSOR: + return TensorCategory.SPARSE; + case RAGGED_TENSOR: + return TensorCategory.RAGGED; + default: + throw new IllegalArgumentException(String.format("Invalid feature type %s. Valid types are %s", + featureType, VALID_FEATURE_TYPES)); + } + } +} diff --git a/feathr-compute/src/main/java/com/linkedin/feathr/compute/builder/TransformationFunctionExpressionBuilder.java b/feathr-compute/src/main/java/com/linkedin/feathr/compute/builder/TransformationFunctionExpressionBuilder.java new file mode 100644 index 000000000..2f4cc8434 --- /dev/null +++ b/feathr-compute/src/main/java/com/linkedin/feathr/compute/builder/TransformationFunctionExpressionBuilder.java @@ -0,0 +1,87 @@ +package com.linkedin.feathr.compute.builder; + +import com.linkedin.data.template.StringMap; +import com.linkedin.feathr.compute.MvelExpression; +import com.linkedin.feathr.compute.SqlExpression; +import com.linkedin.feathr.compute.UserDefinedFunction; +import com.linkedin.feathr.core.config.TimeWindowAggregationType; +import com.linkedin.feathr.core.config.producer.ExprType; +import com.linkedin.feathr.core.config.producer.anchors.AnchorConfig; +import com.linkedin.feathr.core.config.producer.anchors.AnchorConfigWithExtractor; +import com.linkedin.feathr.core.config.producer.anchors.ExpressionBasedFeatureConfig; +import com.linkedin.feathr.core.config.producer.anchors.ExtractorBasedFeatureConfig; +import com.linkedin.feathr.core.config.producer.anchors.FeatureConfig; +import com.linkedin.feathr.core.config.producer.anchors.TimeWindowFeatureConfig; +import javax.annotation.Nonnull; + + +/** + * This class is used to build expression in Transform functions for features. + */ + +public class TransformationFunctionExpressionBuilder { + private final SlidingWindowAggregationBuilder _slidingWindowAggregationBuilder; + + public TransformationFunctionExpressionBuilder(@Nonnull SlidingWindowAggregationBuilder slidingWindowAggregationBuilder) { + _slidingWindowAggregationBuilder = slidingWindowAggregationBuilder; + } + + /** + * Build transform function expression for anchored features. + * + * Transform function can be defined in anchor config via extractor field. In this case, we will build + * UserDefined function. + * + * Or it can be defined in the feature config. Feature config can have following formats: + * + * 1. Simple feature. In this case, the expression will be treated as a Mvel transform function and an MvelExpression will be returned. + * + * 2. Complex feature with SparkSql transform function. In this case, will build SparksqlExpression + * + * 3. Complex feature with Mvel transform function. In this case, will build MvelExpression + * + * 4. Time Windowed feature. For now, we will build UnspecifieldFunction + * + */ + public Object buildTransformationExpression(FeatureConfig featureConfig, AnchorConfig anchorConfig) { + if (anchorConfig instanceof AnchorConfigWithExtractor) { + AnchorConfigWithExtractor anchorConfigWithExtractor = (AnchorConfigWithExtractor) anchorConfig; + UserDefinedFunction userDefinedFunction = new UserDefinedFunction(); + userDefinedFunction.setClazz(anchorConfigWithExtractor.getExtractor()); + userDefinedFunction.setParameters(new StringMap(featureConfig.getParameters())); + return userDefinedFunction; + } + if (featureConfig instanceof ExpressionBasedFeatureConfig) { + ExpressionBasedFeatureConfig expressionBasedFeatureConfig = (ExpressionBasedFeatureConfig) featureConfig; + if (expressionBasedFeatureConfig.getExprType() == ExprType.MVEL) { + MvelExpression mvelExpression = new MvelExpression(); + mvelExpression.setMvel(expressionBasedFeatureConfig.getFeatureExpr()); + return mvelExpression; + } else if (expressionBasedFeatureConfig.getExprType() == ExprType.SQL) { + SqlExpression sparkSqlExpression = new SqlExpression(); + sparkSqlExpression.setSql(expressionBasedFeatureConfig.getFeatureExpr()); + return sparkSqlExpression; + } else { + throw new IllegalArgumentException(String.format("Expression type %s is unsupported in feature config %s", + expressionBasedFeatureConfig.getExprType(), featureConfig)); + } + } else if (featureConfig instanceof ExtractorBasedFeatureConfig) { + ExtractorBasedFeatureConfig extractorBasedFeatureConfig = (ExtractorBasedFeatureConfig) featureConfig; + MvelExpression mvelExpression = new MvelExpression(); + mvelExpression.setMvel(extractorBasedFeatureConfig.getFeatureName()); + return mvelExpression; + } else if (featureConfig instanceof TimeWindowFeatureConfig) { + TimeWindowFeatureConfig timeWindowFeatureConfig = (TimeWindowFeatureConfig) featureConfig; + TimeWindowAggregationType timeWindowAggregationType = ((TimeWindowFeatureConfig) featureConfig).getAggregation(); + if (SlidingWindowAggregationBuilder.isSlidingWindowAggregationType(timeWindowAggregationType)) { + return _slidingWindowAggregationBuilder.build(timeWindowFeatureConfig, anchorConfig); + } else { + throw new IllegalArgumentException("Unsupported time window aggregation type " + timeWindowAggregationType); + } + } else { + throw new IllegalArgumentException(String.format("Feature config type %s is not supported in feature " + + "config %s", featureConfig.getClass(), featureConfig)); + } + } +} + diff --git a/feathr-compute/src/main/java/com/linkedin/feathr/compute/converter/AnchorConfigConverter.java b/feathr-compute/src/main/java/com/linkedin/feathr/compute/converter/AnchorConfigConverter.java new file mode 100644 index 000000000..ab32fb824 --- /dev/null +++ b/feathr-compute/src/main/java/com/linkedin/feathr/compute/converter/AnchorConfigConverter.java @@ -0,0 +1,327 @@ +package com.linkedin.feathr.compute.converter; + +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.data.template.StringMap; +import com.linkedin.feathr.compute.AggregationFunction; +import com.linkedin.feathr.compute.ComputeGraph; +import com.linkedin.feathr.compute.ComputeGraphBuilder; +import com.linkedin.feathr.compute.DataSource; +import com.linkedin.feathr.compute.DataSourceType; +import com.linkedin.feathr.compute.FeatureVersion; +import com.linkedin.feathr.compute.KeyExpressionType; +import com.linkedin.feathr.compute.MvelExpression; +import com.linkedin.feathr.compute.NodeReference; +import com.linkedin.feathr.compute.NodeReferenceArray; +import com.linkedin.feathr.compute.OfflineKeyFunction; +import com.linkedin.feathr.compute.Operators; +import com.linkedin.feathr.compute.PegasusUtils; +import com.linkedin.feathr.compute.SlidingWindowFeature; +import com.linkedin.feathr.compute.SqlExpression; +import com.linkedin.feathr.compute.TimestampCol; +import com.linkedin.feathr.compute.TransformationFunction; +import com.linkedin.feathr.compute.Unit; +import com.linkedin.feathr.compute.UserDefinedFunction; +import com.linkedin.feathr.compute.Window; +import com.linkedin.feathr.compute.builder.AnchorKeyFunctionBuilder; +import com.linkedin.feathr.compute.builder.DefaultValueBuilder; +import com.linkedin.feathr.compute.builder.FeatureVersionBuilder; +import com.linkedin.feathr.compute.builder.FrameFeatureTypeBuilder; +import com.linkedin.feathr.compute.builder.SlidingWindowAggregationBuilder; +import com.linkedin.feathr.compute.builder.TensorFeatureFormatBuilderFactory; +import com.linkedin.feathr.compute.builder.TransformationFunctionExpressionBuilder; +import com.linkedin.feathr.core.config.producer.anchors.AnchorConfig; +import com.linkedin.feathr.core.config.producer.sources.HdfsConfig; +import com.linkedin.feathr.core.config.producer.sources.HdfsConfigWithRegularData; +import com.linkedin.feathr.core.config.producer.sources.HdfsConfigWithSlidingWindow; +import com.linkedin.feathr.core.config.producer.sources.PassThroughConfig; +import com.linkedin.feathr.core.config.producer.sources.SourceConfig; +import java.time.Duration; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +import static com.linkedin.feathr.compute.converter.ConverterUtils.*; + + +/** + * Converts a hocon parsed config model [[AnchorConfig]] into the compute model. This class is resposibile for converting + * anchored and swa feature models into the compute model. + */ +class AnchorConfigConverter implements FeatureDefConfigConverter { + private final String _passthrough = "passthrough"; + private final String _anchor = "anchor"; + private final String _swa = "_swa"; + private final String _window_unit = "window_unit"; + private final String _lateral_view_expression_ = "lateral_view_expression_"; + private final String _lateral_view_table_alias_ = "lateral_view_table_alias_"; + private final String _group_by_expression = "group_by_expression"; + private final String _filter_expression = "filter_expression"; + private final String _max_number_groups = "max_number_groups"; + private final String _expression = "expression"; + private final String _class = "class"; + private final String _userParam_ = "userParam_"; + @Override + public ComputeGraph convert(String configElementName, AnchorConfig configObject, + Map sourceMap) { + ComputeGraphBuilder graphBuilder = new ComputeGraphBuilder(); + + String keyExpression; + KeyExpressionType keyExpressionType; + + // Builds a keyFunction. We need this as currently the config can be in different formats, ie - AnchorConfigWithExtractor, + // AnchorConfigWithMvel, AnchorConfigWithKeyExtractor, AnchorConfigWithKey. The below step consoliates into one single entity. + OfflineKeyFunction.KeyFunction offlineKeyFunction = new AnchorKeyFunctionBuilder(configObject).build(); + if (offlineKeyFunction.isMvelExpression()) { + keyExpression = offlineKeyFunction.getMvelExpression().getMvel(); + keyExpressionType = KeyExpressionType.MVEL; + } else if (offlineKeyFunction.isSqlExpression()) { + keyExpression = offlineKeyFunction.getSqlExpression().getSql(); + keyExpressionType = KeyExpressionType.SQL; + } else if (offlineKeyFunction.isUserDefinedFunction()) { + keyExpression = offlineKeyFunction.getUserDefinedFunction().getClazz(); + keyExpressionType = KeyExpressionType.UDF; + } else { + throw new RuntimeException("Unknown key type found in " + configElementName); + } + + String featureType = getTypeOfFeature(sourceMap, configObject); + + DataSource dataSource = buildDataSource(graphBuilder, configObject, keyExpressionType, keyExpression, sourceMap, featureType); + + // Attach the keys correctly to the datasource. + NodeReference referenceToSource = makeNodeReferenceWithSimpleKeyReference(dataSource.getId(), 1); + + configObject.getFeatures().forEach((featureName, featureConfig) -> { + TransformationFunctionExpressionBuilder transformationFunctionExpressionBuilder = + new TransformationFunctionExpressionBuilder(SlidingWindowAggregationBuilder.getInstance()); + // Build a transformation expression by parsing through the different types of transformation expressions. + Object expression = transformationFunctionExpressionBuilder.buildTransformationExpression(featureConfig, configObject); + + RecordTemplate operatorReference = getOperator(expression, featureType); + + RecordTemplate operatorNode; + + // Build the [[FeatureVersion]] object. + FeatureVersionBuilder featureVersionBuilder = + new FeatureVersionBuilder(new TensorFeatureFormatBuilderFactory(), + DefaultValueBuilder.getInstance(), FrameFeatureTypeBuilder.getInstance()); + FeatureVersion featureVersion = featureVersionBuilder.build(featureConfig); + + // Construct the agg/transformation node + if (operatorReference instanceof AggregationFunction) { + operatorNode = graphBuilder.addNewAggregation() + .setFunction((AggregationFunction) operatorReference) + .setInput(referenceToSource) + .setFeatureName(featureName) + .setFeatureVersion(featureVersion); + } else if (operatorReference instanceof TransformationFunction) { + operatorNode = graphBuilder.addNewTransformation() + .setFunction((TransformationFunction) operatorReference) + .setInputs(new NodeReferenceArray(Collections.singleton(referenceToSource))) + .setFeatureName(featureName) + .setFeatureVersion(featureVersion); + } else { + throw new RuntimeException("Unexpected operator reference type " + operatorReference.getClass() + " - data: " + + operatorReference); + } + graphBuilder.addFeatureName(featureName, PegasusUtils.getNodeId(operatorNode)); + }); + return graphBuilder.build(); + } + + // Get the appropriate transformation operator expression. + private RecordTemplate getOperator(Object expression, String finalFeatureType) { + String operator = null; + RecordTemplate operatorReference; + if (expression instanceof MvelExpression) { + if (Objects.equals(finalFeatureType, _anchor)) { + operator = Operators.OPERATOR_ID_ANCHOR_MVEL; + } else if (Objects.equals(finalFeatureType, _passthrough)) { + operator = Operators.OPERATOR_ID_PASSTHROUGH_MVEL; + } + operatorReference = makeTransformationFunction(((MvelExpression) expression), operator); + } else if (expression instanceof SlidingWindowFeature) { + operatorReference = makeAggregationFunction((SlidingWindowFeature) expression); + } else if (expression instanceof SqlExpression) { + if (Objects.equals(finalFeatureType, _anchor)) { + operator = Operators.OPERATOR_ID_ANCHOR_SPARK_SQL_FEATURE_EXTRACTOR; + } else if (Objects.equals(finalFeatureType, _passthrough)) { + operator = Operators.OPERATOR_ID_PASSTHROUGH_SPARK_SQL_FEATURE_EXTRACTOR; + } + operatorReference = makeTransformationFunction((SqlExpression) expression, operator); + } else if (expression instanceof UserDefinedFunction) { + if (Objects.equals(finalFeatureType, _anchor)) { + operator = Operators.OPERATOR_ID_ANCHOR_JAVA_UDF_FEATURE_EXTRACTOR; + } else if (Objects.equals(finalFeatureType, _passthrough)) { + operator = Operators.OPERATOR_ID_PASSTHROUGH_JAVA_UDF_FEATURE_EXTRACTOR; + } + operatorReference = makeTransformationFunction((UserDefinedFunction) expression, operator); + } else { + throw new RuntimeException("No known way to handle " + expression); + } + return operatorReference; + } + + // Get the feature type correctly to attach the right transformation function operator. The featureType depends on the config source class. + private String getTypeOfFeature(Map sourceMap, AnchorConfig configObject) { + String featureType; + if (sourceMap.containsKey(configObject.getSource()) && sourceMap.get(configObject.getSource()).getClass() == PassThroughConfig.class) { + featureType = _passthrough; + } else if (sourceMap.containsKey(configObject.getSource()) && sourceMap.get(configObject.getSource()).getClass() == HdfsConfigWithSlidingWindow.class) { + String swa = _swa; + featureType = swa; + } else { + if (sourceMap.containsKey(configObject.getSource())) { + HdfsConfigWithRegularData sourceConfig = (HdfsConfigWithRegularData) sourceMap.get(configObject.getSource()); + if (sourceConfig.getTimePartitionPattern().isPresent()) { + featureType = _swa; + } else { + featureType = _anchor; + } + } else { + featureType = _anchor; + } + } + return featureType; + } + + /** + * Builds and adds a datasource object into the graphbuilder using the configObject. + * @param graphBuilder The [[GraphBuilder]] object to which the newly created datasource object should get appended to. + * @param configObject The [[AnchorConfig]] object + * @param keyExpressionType The key expression type, ie - mvel, sql or udf + * @param keyExpression The actual key expression + * @param sourceMap Map of source name to source Config + * @param featureType + * @return The created datasource object + */ + private DataSource buildDataSource(ComputeGraphBuilder graphBuilder, AnchorConfig configObject, KeyExpressionType keyExpressionType, + String keyExpression, Map sourceMap, String featureType) { + DataSource dataSourceNode = null; + String sourcePath; + // If the sourceMap contains the sourceName, we know that it is a compound source and we need to read the source information from the + // sourceMap. + if (sourceMap.containsKey(configObject.getSource())) { + if (Objects.equals(featureType, _anchor)) { // simple anchor + HdfsConfigWithRegularData sourceConfig = (HdfsConfigWithRegularData) sourceMap.get(configObject.getSource()); + sourcePath = sourceConfig.getPath(); + dataSourceNode = graphBuilder.addNewDataSource().setExternalSourceRef(sourcePath) + .setSourceType(DataSourceType.UPDATE).setKeyExpression(keyExpression) + .setKeyExpressionType(keyExpressionType); + } else if (Objects.equals(featureType, _swa)) { // SWA source + HdfsConfig sourceConfig = (HdfsConfig) sourceMap.get(configObject.getSource()); + sourcePath = sourceConfig.getPath(); + dataSourceNode = graphBuilder.addNewDataSource().setExternalSourceRef(sourcePath) + .setSourceType(DataSourceType.EVENT).setKeyExpression(keyExpression) + .setKeyExpressionType(keyExpressionType); + + String filePartitionFormat = null; + if (sourceConfig.getTimePartitionPattern().isPresent()) { + filePartitionFormat = sourceConfig.getTimePartitionPattern().get(); + } + + TimestampCol timestampCol = null; + if (sourceConfig.getClass() == HdfsConfigWithSlidingWindow.class) { + HdfsConfigWithSlidingWindow swaConfig = (HdfsConfigWithSlidingWindow) sourceConfig; + if (swaConfig.getSwaConfig().getTimeWindowParams() != null) { + String timestampColFormat = swaConfig.getSwaConfig().getTimeWindowParams().getTimestampFormat(); + String timestampColExpr = swaConfig.getSwaConfig().getTimeWindowParams().getTimestampField(); + timestampCol = new TimestampCol().setExpression(timestampColExpr).setFormat(timestampColFormat); + } + } + + if (filePartitionFormat != null && timestampCol != null) { + dataSourceNode.setSourceType(DataSourceType.EVENT).setFilePartitionFormat(filePartitionFormat).setTimestampColumnInfo(timestampCol); + } else if (timestampCol != null) { + dataSourceNode.setSourceType(DataSourceType.EVENT).setTimestampColumnInfo(timestampCol); + } else { + dataSourceNode.setSourceType(DataSourceType.EVENT).setFilePartitionFormat(filePartitionFormat); + } + } else if (Objects.equals(featureType, _passthrough)) { + dataSourceNode = graphBuilder.addNewDataSource() + .setSourceType(DataSourceType.CONTEXT).setKeyExpression(keyExpression) + .setKeyExpressionType(keyExpressionType); + } + } else { // source is not an object, so it should be a path. + sourcePath = configObject.getSource(); + dataSourceNode = graphBuilder.addNewDataSource().setExternalSourceRef(sourcePath) + .setSourceType(DataSourceType.UPDATE).setKeyExpression(keyExpression) + .setKeyExpressionType(keyExpressionType); + } + return dataSourceNode; + } + + // Builds the aggregation function + private AggregationFunction makeAggregationFunction(SlidingWindowFeature input) { + Map parameterMap = new HashMap<>(); + String target_column = "target_column"; + parameterMap.put(target_column, input.getTargetColumn().getSqlExpression().getSql()); + String aggregation_type = "aggregation_type"; + parameterMap.put(aggregation_type, input.getAggregationType().name()); + Duration window = convert(input.getWindow()); + String window_size = "window_size"; + parameterMap.put(window_size, window.toString()); + parameterMap.put(_window_unit, input.getWindow().getUnit().name()); + // lateral view expression capability should be rethought + for (int i = 0; i < input.getLateralViews().size(); i++) { + parameterMap.put(_lateral_view_expression_ + i, input.getLateralViews().get(i) + .getTableGeneratingFunction().getSqlExpression().getSql()); + parameterMap.put(_lateral_view_table_alias_ + i, input.getLateralViews().get(i) + .getVirtualTableAlias()); + } + if (input.hasFilter()) { + parameterMap.put(_filter_expression, Objects.requireNonNull(input.getFilter()).getSqlExpression().getSql()); + } + if (input.hasGroupBy()) { + parameterMap.put(_group_by_expression, Objects.requireNonNull(input.getGroupBy()).getSqlExpression().getSql()); + } + if (input.hasLimit()) { + parameterMap.put(_max_number_groups, Objects.requireNonNull(input.getLimit()).toString()); + } + return new AggregationFunction() + .setOperator(Operators.OPERATOR_ID_SLIDING_WINDOW_AGGREGATION) + .setParameters(new StringMap(parameterMap)); + } + + // Build the transformation function given an mvel expression + private TransformationFunction makeTransformationFunction(MvelExpression input, String operator) { + return new TransformationFunction() + .setOperator(operator) + .setParameters(new StringMap(Collections.singletonMap(_expression, input.getMvel()))); + } + + // Build the transformation function given a sql expression + private TransformationFunction makeTransformationFunction(SqlExpression input, String operator) { + return new TransformationFunction().setOperator(operator) + .setParameters(new StringMap(Collections.singletonMap(_expression, input.getSql()))); + } + + // Build the transformation function given a java udf expression + private TransformationFunction makeTransformationFunction(UserDefinedFunction input, String operator) { + Map parameterMap = new HashMap<>(); + parameterMap.put(_class, input.getClazz()); + input.getParameters().forEach((userParamName, userParamValue) -> { + parameterMap.put(_userParam_ + userParamName, userParamValue); + }); + return new TransformationFunction() + .setOperator(operator) + .setParameters(new StringMap(parameterMap)); + } + + private Duration convert(Window frWindow) { + int size = frWindow.getSize(); + if (frWindow.getUnit() == Unit.DAY) { + return Duration.ofDays(size); + } else if (frWindow.getUnit() == Unit.HOUR) { + return Duration.ofHours(size); + } else if (frWindow.getUnit() == Unit.MINUTE) { + return Duration.ofMinutes(size); + } else if (frWindow.getUnit() == Unit.SECOND) { + return Duration.ofSeconds(size); + } else { + throw new RuntimeException("'We only support day, hour, minute, and second time units for window field. The correct example \" +\n" + + " \"can be '1d'(1 day) or '2h'(2 hour) or '3m'(3 minute) or '4s'(4 second) "); + } + } +} diff --git a/feathr-compute/src/main/java/com/linkedin/feathr/compute/converter/ConverterUtils.java b/feathr-compute/src/main/java/com/linkedin/feathr/compute/converter/ConverterUtils.java new file mode 100644 index 000000000..7d462f2d1 --- /dev/null +++ b/feathr-compute/src/main/java/com/linkedin/feathr/compute/converter/ConverterUtils.java @@ -0,0 +1,29 @@ +package com.linkedin.feathr.compute.converter; + +import com.linkedin.feathr.compute.KeyReference; +import com.linkedin.feathr.compute.KeyReferenceArray; +import com.linkedin.feathr.compute.NodeReference; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + + +/** + * Common utility methods that can be shared between the different converters. + */ +public class ConverterUtils { + /** + * For a transformation or aggregation node, we need to fix the input node reference. In this method, we will create that + * node reference, which will be updated in the resolver once we have the join config. + * For now, we will only create a placeholder for the number of keys. + * @param nodeId + * @param nKeyParts + * @return + */ + public static NodeReference makeNodeReferenceWithSimpleKeyReference(int nodeId, int nKeyParts) { + return new NodeReference() + .setId(nodeId) + .setKeyReference(IntStream.range(0, nKeyParts) + .mapToObj(i -> new KeyReference().setPosition(i)) + .collect(Collectors.toCollection(KeyReferenceArray::new))); + } +} diff --git a/feathr-compute/src/main/java/com/linkedin/feathr/compute/converter/DerivationConfigWithExprConverter.java b/feathr-compute/src/main/java/com/linkedin/feathr/compute/converter/DerivationConfigWithExprConverter.java new file mode 100644 index 000000000..bcee4e61e --- /dev/null +++ b/feathr-compute/src/main/java/com/linkedin/feathr/compute/converter/DerivationConfigWithExprConverter.java @@ -0,0 +1,116 @@ +package com.linkedin.feathr.compute.converter; + +import com.linkedin.data.template.StringMap; +import com.linkedin.feathr.compute.ComputeGraph; +import com.linkedin.feathr.compute.ComputeGraphBuilder; +import com.linkedin.feathr.compute.External; +import com.linkedin.feathr.compute.FeatureVersion; +import com.linkedin.feathr.compute.KeyReference; +import com.linkedin.feathr.compute.KeyReferenceArray; +import com.linkedin.feathr.compute.NodeReference; +import com.linkedin.feathr.compute.NodeReferenceArray; +import com.linkedin.feathr.compute.Operators; +import com.linkedin.feathr.compute.Transformation; +import com.linkedin.feathr.compute.TransformationFunction; +import com.linkedin.feathr.compute.builder.DefaultValueBuilder; +import com.linkedin.feathr.compute.builder.FeatureVersionBuilder; +import com.linkedin.feathr.compute.builder.FrameFeatureTypeBuilder; +import com.linkedin.feathr.compute.builder.TensorFeatureFormatBuilderFactory; +import com.linkedin.feathr.core.config.producer.derivations.DerivationConfigWithExpr; +import com.linkedin.feathr.core.config.producer.derivations.KeyedFeature; +import com.linkedin.feathr.core.config.producer.sources.SourceConfig; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + + +/** + * Converts a [[DerivationConfigWithExpr]] object into compute model. + */ +class DerivationConfigWithExprConverter implements FeatureDefConfigConverter { + @Override + public ComputeGraph convert(String configElementName, DerivationConfigWithExpr configObject, + Map sourceMap) { + ComputeGraphBuilder graphBuilder = new ComputeGraphBuilder(); + List entityParameters = configObject.getKeys(); + Map externalFeatureNodes = new HashMap<>(); + Set uniqueValues = new HashSet<>(); + for (Map.Entry input : configObject.getInputs().entrySet()) { + String featureName = input.getValue().getFeature(); + if (uniqueValues.add(featureName)) { + if (externalFeatureNodes.put(featureName, graphBuilder.addNewExternal().setName(featureName)) != null) { + throw new IllegalStateException("Duplicate key found in " + configElementName); + } + } + } + + NodeReferenceArray inputs = configObject.getInputs().entrySet().stream().map(mapEntry -> { + String inputFeatureName = mapEntry.getValue().getFeature(); + List entityArgs = mapEntry.getValue().getKey(); + + KeyReferenceArray keyReferenceArray = entityArgs.stream() + .map(entityParameters::indexOf) + .map(position -> new KeyReference().setPosition(position)) + .collect(Collectors.toCollection(KeyReferenceArray::new)); + int inputNodeId = externalFeatureNodes.get(inputFeatureName).getId(); + + /** + * If there is a featureAlias, add a feature alias transformation node on top of the external node which + * represents the input feature. + * Something like:- + * derivedFeature: { + * key: x + * inputs: { + * arg1: { key: viewerId, feature: AA } + * arg2: { key: vieweeId, feature: BB } + * } + * definition: arg1 + arg2 + * } + * + * We will create a new transformation node for arg1 and arg2. + */ + + if (!Objects.equals(mapEntry.getKey(), "")) { + ArrayList regularKeyReferenceArray = new ArrayList(); + for (int i = 0; i < entityArgs.size(); i++) { + regularKeyReferenceArray.add(new KeyReference().setPosition(i)); + } + KeyReferenceArray simpleKeyReferenceArray = new KeyReferenceArray(regularKeyReferenceArray); + NodeReference inputNodeReference = + new NodeReference().setId(inputNodeId).setKeyReference(simpleKeyReferenceArray); + + TransformationFunction featureAliasFunction = new TransformationFunction().setOperator(Operators.OPERATOR_FEATURE_ALIAS); + Transformation transformation = graphBuilder.addNewTransformation() + .setInputs(new NodeReferenceArray(Collections.singleton(inputNodeReference))) + .setFunction(featureAliasFunction) + .setFeatureVersion((new FeatureVersion())) + .setFeatureName(mapEntry.getKey()); + inputNodeId = transformation.getId(); + } + return new NodeReference().setId(inputNodeId).setKeyReference(keyReferenceArray); + }).collect(Collectors.toCollection(NodeReferenceArray::new)); + + List inputParameterNames = new ArrayList<>(configObject.getInputs().keySet()); + TransformationFunction transformationFunction = new TransformationFunction().setOperator(Operators.OPERATOR_ID_EXTRACT_FROM_TUPLE) + .setParameters(new StringMap(Collections.singletonMap("expression", configObject.getTypedDefinition().getExpr())));; + transformationFunction.getParameters().put("parameterNames", String.join(",", inputParameterNames)); + FeatureVersionBuilder featureVersionBuilder = + new FeatureVersionBuilder(new TensorFeatureFormatBuilderFactory(), + DefaultValueBuilder.getInstance(), FrameFeatureTypeBuilder.getInstance()); + FeatureVersion featureVersion = featureVersionBuilder.build(configObject); + + Transformation transformation = graphBuilder.addNewTransformation() + .setInputs(inputs) + .setFunction(transformationFunction) + .setFeatureName(configElementName) + .setFeatureVersion(featureVersion); + graphBuilder.addFeatureName(configElementName, transformation.getId()); + return graphBuilder.build(); + } +} diff --git a/feathr-compute/src/main/java/com/linkedin/feathr/compute/converter/DerivationConfigWithExtractorConverter.java b/feathr-compute/src/main/java/com/linkedin/feathr/compute/converter/DerivationConfigWithExtractorConverter.java new file mode 100644 index 000000000..b1898e329 --- /dev/null +++ b/feathr-compute/src/main/java/com/linkedin/feathr/compute/converter/DerivationConfigWithExtractorConverter.java @@ -0,0 +1,82 @@ +package com.linkedin.feathr.compute.converter; + +import com.linkedin.data.template.StringMap; +import com.linkedin.feathr.compute.ComputeGraph; +import com.linkedin.feathr.compute.ComputeGraphBuilder; +import com.linkedin.feathr.compute.External; +import com.linkedin.feathr.compute.FeatureVersion; +import com.linkedin.feathr.compute.KeyReference; +import com.linkedin.feathr.compute.KeyReferenceArray; +import com.linkedin.feathr.compute.NodeReference; +import com.linkedin.feathr.compute.NodeReferenceArray; +import com.linkedin.feathr.compute.Operators; +import com.linkedin.feathr.compute.Transformation; +import com.linkedin.feathr.compute.TransformationFunction; +import com.linkedin.feathr.compute.builder.DefaultValueBuilder; +import com.linkedin.feathr.compute.builder.FeatureVersionBuilder; +import com.linkedin.feathr.compute.builder.FrameFeatureTypeBuilder; +import com.linkedin.feathr.compute.builder.TensorFeatureFormatBuilderFactory; +import com.linkedin.feathr.core.config.producer.derivations.DerivationConfigWithExtractor; +import com.linkedin.feathr.core.config.producer.derivations.KeyedFeature; +import com.linkedin.feathr.core.config.producer.sources.SourceConfig; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** + * Converts a [[DerivationConfigWithExtractor]] object into compute model. + */ +class DerivationConfigWithExtractorConverter implements FeatureDefConfigConverter { + @Override + public ComputeGraph convert(String configElementName, DerivationConfigWithExtractor configObject, + Map sourceMap) { + ComputeGraphBuilder graphBuilder = new ComputeGraphBuilder(); + List entityParameters = configObject.getKeys(); + // Create an external feature node with this feature name. + Map externalFeatureNodes = configObject.getInputs().stream() + .map(KeyedFeature::getFeature) + .distinct() + .collect(Collectors.toMap( + Function.identity(), + name -> graphBuilder.addNewExternal().setName(name))); + + + NodeReferenceArray inputs = configObject.getInputs().stream().map(keyedFeature -> { + String inputFeatureName = keyedFeature.getFeature(); + List entityArgs = keyedFeature.getKey(); + + // The entity parameters will have a subset of the keys and we need to set the key position correctly. + KeyReferenceArray keyReferenceArray = entityArgs.stream() + .map(entityParameters::indexOf) // entityParameters should always be small (no 10+ dimensional keys etc) + .map(position -> new KeyReference().setPosition(position)) + .collect(Collectors.toCollection(KeyReferenceArray::new)); + int nodeId = externalFeatureNodes.get(inputFeatureName).getId(); + + return new NodeReference().setId(nodeId).setKeyReference(keyReferenceArray); + }).collect(Collectors.toCollection(NodeReferenceArray::new)); + + TransformationFunction transformationFunction = makeTransformationFunction(configObject.getClassName()); + FeatureVersionBuilder featureVersionBuilder = + new FeatureVersionBuilder(new TensorFeatureFormatBuilderFactory(), + DefaultValueBuilder.getInstance(), FrameFeatureTypeBuilder.getInstance()); + FeatureVersion featureVersion = featureVersionBuilder.build(configObject); + + Transformation transformation = graphBuilder.addNewTransformation() + .setInputs(inputs) + .setFunction(transformationFunction) + .setFeatureName(configElementName) + .setFeatureVersion(featureVersion); + graphBuilder.addFeatureName(configElementName, transformation.getId()); + return graphBuilder.build(); + } + + private TransformationFunction makeTransformationFunction(String className) { + Map parameterMap = new HashMap<>(); + parameterMap.put("class", className); + return new TransformationFunction() + .setOperator(Operators.OPERATOR_ID_DERIVED_JAVA_UDF_FEATURE_EXTRACTOR) + .setParameters(new StringMap(parameterMap)); + } +} diff --git a/feathr-compute/src/main/java/com/linkedin/feathr/compute/converter/FeatureDefConfigConverter.java b/feathr-compute/src/main/java/com/linkedin/feathr/compute/converter/FeatureDefConfigConverter.java new file mode 100644 index 000000000..8055e6e63 --- /dev/null +++ b/feathr-compute/src/main/java/com/linkedin/feathr/compute/converter/FeatureDefConfigConverter.java @@ -0,0 +1,20 @@ +package com.linkedin.feathr.compute.converter; + +import com.linkedin.feathr.compute.ComputeGraph; +import com.linkedin.feathr.compute.ComputeGraphs; +import com.linkedin.feathr.core.config.producer.sources.SourceConfig; +import java.util.Map; + + +interface FeatureDefConfigConverter { + /** + * It may be necessary for different "subgraphs" to refer to other subgraphs via nodes that are not actually named + * features. Currently the graph operations e.g. {@link ComputeGraphs#merge} provide useful capabilities to merge + * subgraphs together but expect them to reference each other based on named features (which are the only things + * External node knows how to reference). To take advantage of those capabilities for nodes that aren't actually + * named features, e.g. source nodes, we'll use a prefix to make synthetic feature names for such references. + */ + String SYNTHETIC_SOURCE_FEATURE_NAME_PREFIX = "__SOURCE__"; + + ComputeGraph convert(String configElementName, T configObject, Map sourceMap); +} diff --git a/feathr-compute/src/main/java/com/linkedin/feathr/compute/converter/FeatureDefinitionsConverter.java b/feathr-compute/src/main/java/com/linkedin/feathr/compute/converter/FeatureDefinitionsConverter.java new file mode 100644 index 000000000..9258fe117 --- /dev/null +++ b/feathr-compute/src/main/java/com/linkedin/feathr/compute/converter/FeatureDefinitionsConverter.java @@ -0,0 +1,84 @@ +package com.linkedin.feathr.compute.converter; + +import com.linkedin.feathr.compute.ComputeGraph; +import com.linkedin.feathr.compute.ComputeGraphs; +import com.linkedin.feathr.core.config.producer.FeatureDefConfig; +import com.linkedin.feathr.core.config.producer.anchors.AnchorConfigWithExtractor; +import com.linkedin.feathr.core.config.producer.anchors.AnchorConfigWithKey; +import com.linkedin.feathr.core.config.producer.anchors.AnchorConfigWithKeyExtractor; +import com.linkedin.feathr.core.config.producer.anchors.AnchorConfigWithOnlyMvel; +import com.linkedin.feathr.core.config.producer.derivations.DerivationConfigWithExpr; +import com.linkedin.feathr.core.config.producer.derivations.DerivationConfigWithExtractor; +import com.linkedin.feathr.core.config.producer.derivations.SequentialJoinConfig; +import com.linkedin.feathr.core.config.producer.derivations.SimpleDerivationConfig; +import com.linkedin.feathr.core.config.producer.sources.SourceConfig; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + + +/** + * Converts a {@link FeatureDefConfig} (parsed HOCON feature definitions) into Feathr Compute Model represented as + * {@link ComputeGraph}. + */ +public class FeatureDefinitionsConverter { + Map sourcesMap = new HashMap<>(); + + private final Map, FeatureDefConfigConverter> _configClassConverterMap = new HashMap<>(); + + { + registerConverter(AnchorConfigWithExtractor.class, new AnchorConfigConverter()); + registerConverter(AnchorConfigWithKey.class, new AnchorConfigConverter()); + registerConverter(AnchorConfigWithKeyExtractor.class, new AnchorConfigConverter()); + registerConverter(AnchorConfigWithOnlyMvel.class, new AnchorConfigConverter()); + registerConverter(DerivationConfigWithExpr.class, new DerivationConfigWithExprConverter()); + registerConverter(DerivationConfigWithExtractor.class, new DerivationConfigWithExtractorConverter()); + registerConverter(SimpleDerivationConfig.class, new SimpleDerivationConfigConverter()); + registerConverter(SequentialJoinConfig.class, new SequentialJoinConfigConverter()); + } + + public ComputeGraph convert(FeatureDefConfig featureDefinitions) throws CloneNotSupportedException { + List graphParts = new ArrayList<>(); + + featureDefinitions.getSourcesConfig().map(sourcesConfig -> sourcesConfig.getSources().entrySet()) + .orElse(Collections.emptySet()) + .forEach(entry -> sourcesMap.put(entry.getKey(), entry.getValue())); + + featureDefinitions.getAnchorsConfig().map(anchorsConfig -> anchorsConfig.getAnchors().entrySet()) + .orElse(Collections.emptySet()).stream() + .map(entry -> convert(entry.getKey(), entry.getValue(), sourcesMap)) + .forEach(graphParts::add); + + featureDefinitions.getDerivationsConfig().map(derivationsConfig -> derivationsConfig.getDerivations().entrySet()) + .orElse(Collections.emptySet()).stream() + .map(entry -> convert(entry.getKey(), entry.getValue(), sourcesMap)) + .forEach(graphParts::add); + + return ComputeGraphs.removeRedundancies(ComputeGraphs.merge(graphParts)); + } + + /** + * Register a converter for a particular kind of config object class. The purpose of this private method (which we + * will only use during construction time) is to prevent accidental mismatches. Via the type parameter we guarantee + * that the converter should always match the corresponding class. + */ + private void registerConverter(Class clazz, FeatureDefConfigConverter converter) { + _configClassConverterMap.put(clazz, converter); + } + + @SuppressWarnings("unchecked") + private FeatureDefConfigConverter getConverter(T configObject) { + return (FeatureDefConfigConverter) _configClassConverterMap.get(configObject.getClass()); + } + + private ComputeGraph convert(String name, T config, Map sourcesMap) { + FeatureDefConfigConverter converter = getConverter(config); + if (converter != null) { + return converter.convert(name, config, sourcesMap); + } else { + throw new RuntimeException("Unhandled config class: " + name + ": " + config); + } + } +} diff --git a/feathr-compute/src/main/java/com/linkedin/feathr/compute/converter/SequentialJoinConfigConverter.java b/feathr-compute/src/main/java/com/linkedin/feathr/compute/converter/SequentialJoinConfigConverter.java new file mode 100644 index 000000000..f966f6ba9 --- /dev/null +++ b/feathr-compute/src/main/java/com/linkedin/feathr/compute/converter/SequentialJoinConfigConverter.java @@ -0,0 +1,122 @@ +package com.linkedin.feathr.compute.converter; + +import com.linkedin.data.template.StringMap; +import com.linkedin.feathr.compute.ComputeGraph; +import com.linkedin.feathr.compute.ComputeGraphBuilder; +import com.linkedin.feathr.compute.External; +import com.linkedin.feathr.compute.FeatureVersion; +import com.linkedin.feathr.compute.KeyReference; +import com.linkedin.feathr.compute.KeyReferenceArray; +import com.linkedin.feathr.compute.Lookup; +import com.linkedin.feathr.compute.MvelExpression; +import com.linkedin.feathr.compute.NodeReference; +import com.linkedin.feathr.compute.NodeReferenceArray; +import com.linkedin.feathr.compute.Operators; +import com.linkedin.feathr.compute.Transformation; +import com.linkedin.feathr.compute.TransformationFunction; +import com.linkedin.feathr.compute.builder.DefaultValueBuilder; +import com.linkedin.feathr.compute.builder.FeatureVersionBuilder; +import com.linkedin.feathr.compute.builder.FrameFeatureTypeBuilder; +import com.linkedin.feathr.compute.builder.TensorFeatureFormatBuilderFactory; +import com.linkedin.feathr.core.config.producer.derivations.SequentialJoinConfig; +import com.linkedin.feathr.core.config.producer.sources.SourceConfig; +import com.linkedin.feathr.core.utils.MvelInputsResolver; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * Converts a [[SequentialJoinConfig]] object into compute model. + */ +class SequentialJoinConfigConverter implements FeatureDefConfigConverter { + + @Override + public ComputeGraph convert(String configElementName, SequentialJoinConfig configObject, + Map sourceMap) { + ComputeGraphBuilder graphBuilder = new ComputeGraphBuilder(); + String baseFeatureName = configObject.getBase().getFeature(); + List baseFeatureKeys = configObject.getBase().getKey(); + List entityParameters = configObject.getKeys(); + External baseExternalFeatureNode = graphBuilder.addNewExternal().setName(baseFeatureName); + KeyReferenceArray keyReferenceArray = baseFeatureKeys.stream() + .map(entityParameters::indexOf) + .map(position -> new KeyReference().setPosition(position)) + .collect(Collectors.toCollection(KeyReferenceArray::new)); + int nodeId = baseExternalFeatureNode.getId(); + NodeReference baseNodeReference = new NodeReference().setId(nodeId).setKeyReference(keyReferenceArray); + Lookup.LookupKey lookupKey; + String featureNameAlias; + if (configObject.getBase().getOutputKeys().isPresent()) { + featureNameAlias = configObject.getBase().getOutputKeys().get().get(0); + } else { + featureNameAlias = "__SequentialJoinDefaultOutputKey__0"; + } + // Here we want to check if there is an expansion key function and add a transformation node on top of the + // base external feature node in that case. Note we only support MVEL in this case in the HOCON config. + if (configObject.getBase().getTransformation().isPresent()) { + // We only support mvel expression here. + MvelExpression baseFeatureTransformationExpression = new MvelExpression().setMvel(configObject.getBase().getTransformation().get()); + // Should be just the base feature. + List inputFeatureNames = MvelInputsResolver.getInstance().getInputFeatures(baseFeatureTransformationExpression.getMvel()); + TransformationFunction transformationFunction = makeTransformationFunction(baseFeatureTransformationExpression, + inputFeatureNames, Operators.OPERATOR_ID_LOOKUP_MVEL); + // Note here we specifically do not set the base feature name or add a feature definition because this is not a named feature, + // it is a intermediate feature that will only be used for sequential join so a name will be generated for it. + Transformation transformationNode = graphBuilder.addNewTransformation() + .setInputs(new NodeReferenceArray(Collections.singleton(baseNodeReference))) + .setFunction(transformationFunction) + .setFeatureVersion(new FeatureVersion()) + .setFeatureName(featureNameAlias); + int transformationNodeId = transformationNode.getId(); + + NodeReference baseTransformationNodeReference = new NodeReference().setId(transformationNodeId).setKeyReference(keyReferenceArray); + lookupKey = new Lookup.LookupKey().create(baseTransformationNodeReference); + } else { + lookupKey = new Lookup.LookupKey().create(baseNodeReference); + } + + // Create lookup key array based on key reference and base node reference. + List expansionKeysArray = configObject.getExpansion().getKey(); + Lookup.LookupKeyArray lookupKeyArray = expansionKeysArray.stream() + .map(entityParameters::indexOf) + .map(position -> position == -1 ? lookupKey + : entityParameters.get(position).equals(featureNameAlias) ? lookupKey + : new Lookup.LookupKey().create(new KeyReference().setPosition(position)) + ) + .collect(Collectors.toCollection(Lookup.LookupKeyArray::new)); + + // create an external node without key reference for expansion. + String expansionFeatureName = configObject.getExpansion().getFeature(); + External expansionExternalFeatureNode = graphBuilder.addNewExternal().setName(expansionFeatureName); + + // get aggregation function + String aggType = configObject.getAggregation(); + FeatureVersionBuilder featureVersionBuilder = + new FeatureVersionBuilder(new TensorFeatureFormatBuilderFactory(), + DefaultValueBuilder.getInstance(), FrameFeatureTypeBuilder.getInstance()); + FeatureVersion featureVersion = featureVersionBuilder.build(configObject); + Lookup lookup = graphBuilder.addNewLookup().setLookupNode(expansionExternalFeatureNode.getId()) + .setLookupKey(lookupKeyArray).setAggregation(aggType).setFeatureName(configElementName).setFeatureVersion(featureVersion); + graphBuilder.addFeatureName(configElementName, lookup.getId()); + return graphBuilder.build(); + } + + // This one will operate on a tuple of inputs (the Feature Derivation case). In this case, the transform function + // will consume a tuple. A list of names will inform the transformer about how to apply the elements in the tuple + // (based on their order) to the variable names used in the MVEL expression itself (e.g. feature1, feature2). + private TransformationFunction makeTransformationFunction( + MvelExpression input, List parameterNames, String operator) { + // Treat derivation mvel derived features differently? + TransformationFunction tf = makeTransformationFunction(input, operator); + tf.getParameters().put("parameterNames", String.join(",", parameterNames)); + return tf; + } + + private TransformationFunction makeTransformationFunction( + MvelExpression input, String operator) { + return new TransformationFunction() + .setOperator(operator) + .setParameters(new StringMap(Collections.singletonMap("expression", input.getMvel()))); + } +} diff --git a/feathr-compute/src/main/java/com/linkedin/feathr/compute/converter/SimpleDerivationConfigConverter.java b/feathr-compute/src/main/java/com/linkedin/feathr/compute/converter/SimpleDerivationConfigConverter.java new file mode 100644 index 000000000..01ca255c0 --- /dev/null +++ b/feathr-compute/src/main/java/com/linkedin/feathr/compute/converter/SimpleDerivationConfigConverter.java @@ -0,0 +1,80 @@ +package com.linkedin.feathr.compute.converter; + +import com.linkedin.data.template.StringMap; +import com.linkedin.feathr.compute.ComputeGraph; +import com.linkedin.feathr.compute.ComputeGraphBuilder; +import com.linkedin.feathr.compute.External; +import com.linkedin.feathr.compute.FeatureVersion; +import com.linkedin.feathr.compute.NodeReferenceArray; +import com.linkedin.feathr.compute.Operators; +import com.linkedin.feathr.compute.SqlUtil; +import com.linkedin.feathr.compute.Transformation; +import com.linkedin.feathr.compute.TransformationFunction; +import com.linkedin.feathr.compute.builder.DefaultValueBuilder; +import com.linkedin.feathr.compute.builder.FeatureVersionBuilder; +import com.linkedin.feathr.compute.builder.FrameFeatureTypeBuilder; +import com.linkedin.feathr.compute.builder.TensorFeatureFormatBuilderFactory; +import com.linkedin.feathr.core.config.producer.ExprType; +import com.linkedin.feathr.core.config.producer.derivations.SimpleDerivationConfig; +import com.linkedin.feathr.core.config.producer.sources.SourceConfig; +import com.linkedin.feathr.core.utils.MvelInputsResolver; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +import static com.linkedin.feathr.compute.converter.ConverterUtils.*; + +/** + * Converts a [[SimpleDerivationConfig]] object into compute model. + */ +class SimpleDerivationConfigConverter implements FeatureDefConfigConverter { + @Override + public ComputeGraph convert(String configElementName, SimpleDerivationConfig configObject, + Map sourceMap) { + List inputFeatureNames = null; + TransformationFunction transformationFunction = null; + ComputeGraphBuilder graphBuilder = new ComputeGraphBuilder(); + if (configObject.getFeatureTypedExpr().getExprType().equals(ExprType.MVEL)) { + String mvel = configObject.getFeatureTypedExpr().getExpr(); + inputFeatureNames = MvelInputsResolver.getInstance().getInputFeatures(mvel); + transformationFunction = new TransformationFunction() + .setOperator(Operators.OPERATOR_ID_DERIVED_MVEL) + .setParameters(new StringMap(Collections.singletonMap("expression", mvel))); + transformationFunction.getParameters().put("parameterNames", String.join(",", inputFeatureNames)); + } else if (configObject.getFeatureTypedExpr().getExprType().equals(ExprType.SQL)) { + String sql = configObject.getFeatureTypedExpr().getExpr(); + inputFeatureNames = SqlUtil.getInputsFromSqlExpression(sql); + transformationFunction = new TransformationFunction() + .setOperator(Operators.OPERATOR_ID_DERIVED_SPARK_SQL_FEATURE_EXTRACTOR) + .setParameters(new StringMap(Collections.singletonMap("expression", sql))); + transformationFunction.getParameters().put("parameterNames", String.join(",", inputFeatureNames)); + } + + Map externalFeatureNodes = inputFeatureNames.stream() + .collect(Collectors.toMap(Function.identity(), + name -> graphBuilder.addNewExternal().setName(name))); + NodeReferenceArray nodeReferences = inputFeatureNames.stream().map(inputFeatureName -> { + int featureDependencyNodeId = externalFeatureNodes.get(inputFeatureName).getId(); + // WE HAVE NO WAY OF KNOWING how many keys the feature has. Perhaps this ambiguity should be specifically + // allowed for in the compute model. We assume the number of key part is always 1 as the simple derivation + // does not have a key field. + return makeNodeReferenceWithSimpleKeyReference(featureDependencyNodeId, 1); + } + ).collect(Collectors.toCollection(NodeReferenceArray::new)); + + FeatureVersionBuilder featureVersionBuilder = + new FeatureVersionBuilder(new TensorFeatureFormatBuilderFactory(), + DefaultValueBuilder.getInstance(), FrameFeatureTypeBuilder.getInstance()); + FeatureVersion featureVersion = featureVersionBuilder.build(configObject); + Transformation transformation = graphBuilder.addNewTransformation() + .setInputs(nodeReferences) + .setFunction(transformationFunction) + .setFeatureName(configElementName) + .setFeatureVersion(featureVersion); + graphBuilder.addFeatureName(configElementName, transformation.getId()); + + return graphBuilder.build(); + } +} diff --git a/feathr-compute/src/test/java/com/linkedin/feathr/compute/TestFeatureDefinitionsConverter.java b/feathr-compute/src/test/java/com/linkedin/feathr/compute/TestFeatureDefinitionsConverter.java new file mode 100644 index 000000000..0e5f179d1 --- /dev/null +++ b/feathr-compute/src/test/java/com/linkedin/feathr/compute/TestFeatureDefinitionsConverter.java @@ -0,0 +1,240 @@ +package com.linkedin.feathr.compute; + +import com.linkedin.data.template.StringMap; +import com.linkedin.feathr.compute.converter.FeatureDefinitionsConverter; +import com.linkedin.feathr.config.FeatureDefinitionLoaderFactory; +import com.linkedin.feathr.core.config.producer.FeatureDefConfig; +import com.linkedin.feathr.core.configdataprovider.ResourceConfigDataProvider; +import java.util.Objects; +import java.util.stream.Collectors; +import org.testng.Assert; +import org.testng.annotations.Test; + + /** + * Unit tests for [[FeatureDefinitionsConverter]] class + */ + public class TestFeatureDefinitionsConverter { + @Test(description = "Test simple swa") + public void testSimplesSwa() throws CloneNotSupportedException { + FeatureDefConfig features = FeatureDefinitionLoaderFactory.getInstance() + .loadAllFeatureDefinitions(new ResourceConfigDataProvider("swa.conf")); + ComputeGraph output = new FeatureDefinitionsConverter().convert(features); + Assert.assertEquals(output.getNodes().size(), 2); + Assert.assertEquals(output.getNodes().stream().map(AnyNode::isAggregation).filter(i -> i).count(), 1); + Aggregation aggregationNode = output.getNodes().stream().map(AnyNode::getAggregation).filter(Objects::nonNull).collect( + Collectors.toList()).get(0); + Assert.assertEquals(aggregationNode.getFeatureName(), "memberEmbedding"); + // concrete key should not be set yet, as there is no join config + Assert.assertEquals(aggregationNode.getConcreteKey(), null); + StringMap aggParams = aggregationNode.getFunction().getParameters(); + Assert.assertEquals(aggParams.get("aggregation_type"), "LATEST"); + Assert.assertEquals(aggParams.get("window_size"), "PT72H"); + Assert.assertEquals(aggParams.get("window_unit"), "DAY"); + Assert.assertEquals(aggParams.get("target_column"), "embedding"); + } + + @Test(description = "Test anchored feature") + public void testAnchoredFeature() throws CloneNotSupportedException { + FeatureDefConfig features = FeatureDefinitionLoaderFactory.getInstance() + .loadAllFeatureDefinitions(new ResourceConfigDataProvider("anchoredFeature.conf")); + ComputeGraph output = new FeatureDefinitionsConverter().convert(features); + Assert.assertEquals(output.getNodes().size(), 2); + Assert.assertEquals(output.getNodes().stream().map(AnyNode::isTransformation).filter(i -> i).count(), 1); + Transformation transformationNode = output.getNodes().stream().map(AnyNode::getTransformation).filter(Objects::nonNull).collect(Collectors.toList()).get(0); + Assert.assertEquals(transformationNode.getFeatureName(), "waterloo_member_yearBorn"); + // concrete key should not be set yet, as there is no join config + Assert.assertNull(transformationNode.getConcreteKey()); + Assert.assertEquals(transformationNode.getFunction().getOperator(), "feathr:anchor_mvel:0"); + StringMap aggParams = transformationNode.getFunction().getParameters(); + Assert.assertEquals(aggParams.get("expression"), "yearBorn"); + DataSource dataSourceNode = output.getNodes().stream().map(AnyNode::getDataSource).filter(Objects::nonNull).collect(Collectors.toList()).get(0); + Assert.assertEquals(dataSourceNode.getExternalSourceRef(), "seqJoin/member.avro.json"); + } + + + @Test(description = "Test seq join feature") + public void testSeqJoinFeature() throws CloneNotSupportedException { + FeatureDefConfig features = FeatureDefinitionLoaderFactory.getInstance() + .loadAllFeatureDefinitions(new ResourceConfigDataProvider("seqJoinFeature.conf")); + ComputeGraph output = new FeatureDefinitionsConverter().convert(features); + Assert.assertEquals(output.getNodes().size(), 5); + Assert.assertEquals(output.getNodes().stream().map(AnyNode::isLookup).filter(i -> i).count(), 1); + Lookup lookupNode = output.getNodes().stream().map(AnyNode::getLookup).filter(Objects::nonNull).collect(Collectors.toList()).get(0); + Assert.assertEquals(lookupNode.getFeatureName(), "seq_join_industry_names"); + + // base feature + int baseNodeId = output.getFeatureNames().get("MemberIndustryId"); + + // expansion feature + int expansionNodeId = output.getFeatureNames().get("MemberIndustryName"); + + // concrete key should not be set yet, as there is no join config + Assert.assertNull(lookupNode.getConcreteKey()); + Assert.assertEquals(lookupNode.getAggregation(), "UNION"); + Assert.assertEquals(lookupNode.getLookupKey().get(0).getNodeReference().getId().intValue(), baseNodeId); + + // MemberIndustryId has only one key, and the same key is re-used. + Assert.assertEquals(lookupNode.getLookupKey().get(0).getNodeReference().getKeyReference().size(), 1); + Assert.assertEquals(lookupNode.getLookupKey().get(0).getNodeReference().getKeyReference().get(0).getPosition().intValue(), 0); + Assert.assertEquals(lookupNode.getLookupNode().intValue(), expansionNodeId); + + DataSource dataSourceNode = output.getNodes().stream().map(AnyNode::getDataSource).filter(Objects::nonNull).collect(Collectors.toList()).get(0); + Assert.assertEquals(dataSourceNode.getExternalSourceRef(), "seqJoin/member.avro.json"); + } + + + @Test(description = "Test a simple mvel derived feature") + public void testMvelDerivedFeature() throws CloneNotSupportedException { + FeatureDefConfig features = FeatureDefinitionLoaderFactory.getInstance() + .loadAllFeatureDefinitions(new ResourceConfigDataProvider("mvelDerivedFeature.conf")); + ComputeGraph output = new FeatureDefinitionsConverter().convert(features); + Assert.assertEquals(output.getNodes().size(), 3); + Transformation derivedFeatureNode = output.getNodes().stream().map(AnyNode::getTransformation) + .filter(Objects::nonNull).filter(p -> Objects.equals(p.getFeatureName(), "B")).collect(Collectors.toList()).get(0); + + // concrete key should not be set yet, as there is no join config + Assert.assertNull(derivedFeatureNode.getConcreteKey()); + Assert.assertEquals(derivedFeatureNode.getFunction().getOperator(), "feathr:derived_mvel:0"); + Assert.assertEquals(derivedFeatureNode.getFunction().getParameters().get("parameterNames"), "AA"); + Assert.assertEquals(derivedFeatureNode.getFunction().getParameters().get("expression"), "AA*2"); + + DataSource dataSourceNode = output.getNodes().stream().map(AnyNode::getDataSource).filter(Objects::nonNull).collect(Collectors.toList()).get(0); + Assert.assertEquals(dataSourceNode.getExternalSourceRef(), "%s"); + } + + + @Test(description = "Test a complex derived feature") + public void testComplexDerivedFeature() throws CloneNotSupportedException { + FeatureDefConfig features = FeatureDefinitionLoaderFactory.getInstance() + .loadAllFeatureDefinitions(new ResourceConfigDataProvider("complexDerivedFeature.conf")); + ComputeGraph output = new FeatureDefinitionsConverter().convert(features); + Assert.assertEquals(output.getNodes().size(), 6); + Transformation derivedFeatureNode = output.getNodes().stream().map(AnyNode::getTransformation) + .filter(Objects::nonNull).filter(p -> Objects.equals(p.getFeatureName(), "C")).collect(Collectors.toList()).get(0); + + // input features + int inputFeature1 = output.getNodes().stream().map(AnyNode::getTransformation) + .filter(Objects::nonNull).filter(p -> Objects.equals(p.getFeatureName(), "arg1")).collect(Collectors.toList()).get(0).getId(); + int inputFeature2 = output.getNodes().stream().map(AnyNode::getTransformation) + .filter(Objects::nonNull).filter(p -> Objects.equals(p.getFeatureName(), "arg2")).collect(Collectors.toList()).get(0).getId(); + + // concrete key should not be set yet, as there is no join config + Assert.assertNull(derivedFeatureNode.getConcreteKey()); + Assert.assertEquals(derivedFeatureNode.getFunction().getOperator(), "feathr:extract_from_tuple:0"); + Assert.assertEquals(derivedFeatureNode.getInputs().size(), 2); + Assert.assertTrue(derivedFeatureNode.getInputs().stream().map(NodeReference::getId).collect(Collectors.toList()).contains(inputFeature1)); + Assert.assertTrue(derivedFeatureNode.getInputs().stream().map(NodeReference::getId).collect(Collectors.toList()).contains(inputFeature2)); + Assert.assertEquals(Objects.requireNonNull(derivedFeatureNode.getFunction().getParameters()).get("expression"), + "arg1 + arg2"); + + DataSource dataSourceNode = output.getNodes().stream().map(AnyNode::getDataSource).filter(Objects::nonNull).collect(Collectors.toList()).get(0); + Assert.assertEquals(dataSourceNode.getExternalSourceRef(), "%s"); + } + + @Test(description = "Test an anchored feature with source object") + public void testAnchorWithSourceObject() throws CloneNotSupportedException { + FeatureDefConfig features = FeatureDefinitionLoaderFactory.getInstance() + .loadAllFeatureDefinitions(new ResourceConfigDataProvider("anchoredFeature2.conf")); + ComputeGraph output = new FeatureDefinitionsConverter().convert(features); + Assert.assertEquals(output.getNodes().size(), 2); + Transformation anchoredFeatureNode = output.getNodes().stream().map(AnyNode::getTransformation) + .filter(Objects::nonNull).filter(p -> Objects.equals(p.getFeatureName(), "f1")).collect(Collectors.toList()).get(0); + + // concrete key should not be set yet, as there is no join config + Assert.assertNull(anchoredFeatureNode.getConcreteKey()); + Assert.assertEquals(anchoredFeatureNode.getFunction().getOperator(), "feathr:anchor_mvel:0"); + + DataSource dataSourceNode = output.getNodes().stream().map(AnyNode::getDataSource).filter(Objects::nonNull).collect(Collectors.toList()).get(0); + Assert.assertEquals(dataSourceNode.getExternalSourceRef(), "slidingWindowAgg/localSWAAnchorTestFeatureData/daily"); + Assert.assertEquals(dataSourceNode.getKeyExpression(), "\"x\""); + } + + @Test(description = "Test an anchored feature with key extractor") + public void testAnchorWithKeyExtractor() throws CloneNotSupportedException { + FeatureDefConfig features = FeatureDefinitionLoaderFactory.getInstance() + .loadAllFeatureDefinitions(new ResourceConfigDataProvider("anchorWithKeyExtractor.conf")); + ComputeGraph output = new FeatureDefinitionsConverter().convert(features); + Assert.assertEquals(output.getNodes().size(), 2); + Transformation anchoredFeatureNode = output.getNodes().stream().map(AnyNode::getTransformation) + .filter(Objects::nonNull).filter(p -> Objects.equals(p.getFeatureName(), "cohortActorFeature_base")).collect(Collectors.toList()).get(0); + + // concrete key should not be set yet, as there is no join config + Assert.assertNull(anchoredFeatureNode.getConcreteKey()); + Assert.assertEquals(anchoredFeatureNode.getFunction().getOperator(), "feathr:anchor_spark_sql_feature_extractor:0"); + + DataSource dataSourceNode = output.getNodes().stream().map(AnyNode::getDataSource).filter(Objects::nonNull).collect(Collectors.toList()).get(0); + Assert.assertEquals(dataSourceNode.getExternalSourceRef(), "seqJoin/cohortActorFeatures.avro.json"); + } + + @Test(description = "Test a complex derived feature with udf") + public void testDerivedWithUdf() throws CloneNotSupportedException { + FeatureDefConfig features = FeatureDefinitionLoaderFactory.getInstance() + .loadAllFeatureDefinitions(new ResourceConfigDataProvider("derivedFeatureWithClass.conf")); + ComputeGraph output = new FeatureDefinitionsConverter().convert(features); + Assert.assertEquals(output.getNodes().size(), 4); + Transformation derivedFeatureNode = output.getNodes().stream().map(AnyNode::getTransformation) + .filter(Objects::nonNull).filter(p -> Objects.equals(p.getFeatureName(), "C")).collect(Collectors.toList()).get(0); + + // input features + int inputFeature1 = output.getNodes().stream().map(AnyNode::getTransformation) + .filter(Objects::nonNull).filter(p -> Objects.equals(p.getFeatureName(), "AA")).collect(Collectors.toList()).get(0).getId(); + int inputFeature2 = output.getNodes().stream().map(AnyNode::getTransformation) + .filter(Objects::nonNull).filter(p -> Objects.equals(p.getFeatureName(), "BB")).collect(Collectors.toList()).get(0).getId(); + + // concrete key should not be set yet, as there is no join config + Assert.assertNull(derivedFeatureNode.getConcreteKey()); + Assert.assertEquals(derivedFeatureNode.getFunction().getOperator(), "feathr:derived_java_udf_feature_extractor:0"); + Assert.assertEquals(derivedFeatureNode.getInputs().size(), 2); + Assert.assertTrue(derivedFeatureNode.getInputs().stream().map(NodeReference::getId).collect(Collectors.toList()).contains(inputFeature1)); + Assert.assertTrue(derivedFeatureNode.getInputs().stream().map(NodeReference::getId).collect(Collectors.toList()).contains(inputFeature2)); + Assert.assertEquals(Objects.requireNonNull(derivedFeatureNode.getFunction().getParameters()).get("class"), + "com.linkedin.feathr.offline.anchored.anchorExtractor.TestxGenericSparkFeatureDataExtractor2"); + + DataSource dataSourceNode = output.getNodes().stream().map(AnyNode::getDataSource).filter(Objects::nonNull).collect(Collectors.toList()).get(0); + Assert.assertEquals(dataSourceNode.getExternalSourceRef(), "%s"); + } + + @Test(description = "Test a derived feature with mvel expression") + public void testDerivedWithMvel() throws CloneNotSupportedException { + FeatureDefConfig features = FeatureDefinitionLoaderFactory.getInstance() + .loadAllFeatureDefinitions(new ResourceConfigDataProvider("mvelDerivedFeature.conf")); + ComputeGraph output = new FeatureDefinitionsConverter().convert(features); + Assert.assertEquals(output.getNodes().size(), 3); + Transformation derivedFeatureNode = output.getNodes().stream().map(AnyNode::getTransformation) + .filter(Objects::nonNull).filter(p -> Objects.equals(p.getFeatureName(), "B")).collect(Collectors.toList()).get(0); + + // input features + int inputFeature1 = output.getNodes().stream().map(AnyNode::getTransformation) + .filter(Objects::nonNull).filter(p -> Objects.equals(p.getFeatureName(), "AA")).collect(Collectors.toList()).get(0).getId(); + + // concrete key should not be set yet, as there is no join config + Assert.assertNull(derivedFeatureNode.getConcreteKey()); + Assert.assertEquals(derivedFeatureNode.getFunction().getOperator(), "feathr:derived_mvel:0"); + Assert.assertEquals(derivedFeatureNode.getInputs().size(), 1); + Assert.assertTrue(derivedFeatureNode.getInputs().stream().map(NodeReference::getId).collect(Collectors.toList()).contains(inputFeature1)); + Assert.assertEquals(Objects.requireNonNull(derivedFeatureNode.getFunction().getParameters()).get("expression"), + "AA*2"); + + DataSource dataSourceNode = output.getNodes().stream().map(AnyNode::getDataSource).filter(Objects::nonNull).collect(Collectors.toList()).get(0); + Assert.assertEquals(dataSourceNode.getExternalSourceRef(), "%s"); + } + + @Test(description = "Test a combination of swa features with key extractors") + public void testSwaWithKeyExtractors() throws CloneNotSupportedException { + FeatureDefConfig features = FeatureDefinitionLoaderFactory.getInstance() + .loadAllFeatureDefinitions(new ResourceConfigDataProvider("swaWithExtractor.conf")); + ComputeGraph output = new FeatureDefinitionsConverter().convert(features); + Assert.assertEquals(output.getNodes().size(), 11); + Assert.assertEquals(output.getNodes().stream().map(AnyNode::isAggregation).filter(i -> i).count(), 5); + Aggregation aggregationNode = output.getNodes().stream().map(AnyNode::getAggregation).filter(Objects::nonNull) + .filter(p -> Objects.equals(p.getFeatureName(), "f3")).collect(Collectors.toList()).get(0); + Assert.assertEquals(aggregationNode.getFeatureName(), "f3"); + // concrete key should not be set yet, as there is no join config + Assert.assertEquals(aggregationNode.getConcreteKey(), null); + StringMap aggParams = aggregationNode.getFunction().getParameters(); + Assert.assertEquals(aggParams.get("aggregation_type"), "SUM"); + Assert.assertEquals(aggParams.get("window_size"), "PT72H"); + Assert.assertEquals(aggParams.get("window_unit"), "DAY"); + Assert.assertEquals(aggParams.get("target_column"), "aggregationWindow"); + } + } diff --git a/feathr-compute/src/test/java/com/linkedin/feathr/compute/TestResolver.java b/feathr-compute/src/test/java/com/linkedin/feathr/compute/TestResolver.java new file mode 100644 index 000000000..9edf84277 --- /dev/null +++ b/feathr-compute/src/test/java/com/linkedin/feathr/compute/TestResolver.java @@ -0,0 +1,346 @@ +package com.linkedin.feathr.compute; + +import com.google.common.collect.ImmutableMap; +import com.linkedin.data.template.IntegerArray; +import com.linkedin.data.template.IntegerMap; +import com.linkedin.data.template.StringMap; +import com.linkedin.feathr.compute.converter.FeatureDefinitionsConverter; +import com.linkedin.feathr.config.FeatureDefinitionLoaderFactory; +import com.linkedin.feathr.core.config.producer.FeatureDefConfig; +import com.linkedin.feathr.core.configdataprovider.ResourceConfigDataProvider; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; +import org.testng.Assert; +import org.testng.annotations.Test; + + +/** + * Unit tests for [[Resolver]] and [[ComputeGraphs]] class + */ +public class TestResolver { + + @Test(description = "test simple merge of 2 compute graphs") + public void testMergeGraphs() throws Exception { + DataSource dataSource1 = new DataSource().setId(0).setSourceType(DataSourceType.UPDATE).setExternalSourceRef("foo"); + Transformation transformation1 = new Transformation().setId(1) + .setInputs(new NodeReferenceArray( + new NodeReference().setId(0).setKeyReference(new KeyReferenceArray(new KeyReference().setPosition(0))))) + .setFunction(new TransformationFunction().setOperator("foo:bar:1").setParameters(new StringMap(Collections.singletonMap("foo", "bar")))); + AnyNodeArray nodeArray1 = new AnyNodeArray(AnyNode.create(dataSource1), AnyNode.create(transformation1)); + IntegerMap featureNameMap1 = new IntegerMap(Collections.singletonMap("baz", 1)); + ComputeGraph graph1 = new ComputeGraph().setNodes(nodeArray1).setFeatureNames(featureNameMap1); + + DataSource dataSource2 = new DataSource().setId(0).setSourceType(DataSourceType.UPDATE).setExternalSourceRef("bar"); + Transformation transformation2 = new Transformation().setId(1) + .setInputs(new NodeReferenceArray((new NodeReference().setId(0).setKeyReference(new KeyReferenceArray(new KeyReference().setPosition(0)))))) + .setFunction(new TransformationFunction().setOperator("foo:baz:1")); + Transformation transformation3 = new Transformation().setId(2) + .setInputs(new NodeReferenceArray((new NodeReference().setId(1).setKeyReference(new KeyReferenceArray(new KeyReference().setPosition(0)))))) + .setFunction(new TransformationFunction().setOperator("foo:foo:2")); + AnyNodeArray nodeArray2 = new AnyNodeArray(AnyNode.create(dataSource2), AnyNode.create(transformation2), AnyNode.create(transformation3)); + IntegerMap featureNameMap2 = new IntegerMap( + ImmutableMap.of("fizz", 1, "buzz", 2)); + ComputeGraph graph2 = new ComputeGraph().setNodes(nodeArray2).setFeatureNames(featureNameMap2); + + ComputeGraph merged = ComputeGraphs.merge(Arrays.asList(graph1, graph2)); + Assert.assertEquals(merged.getNodes().size(), 5); + Assert.assertEquals(merged.getFeatureNames().keySet().size(), 3); + } + + @Test + public void testMergeGraphWithFeatureDependencies() { + External featureReference1 = new External().setId(0).setName("feature1"); + Transformation transformation1 = new Transformation().setId(1) + .setInputs(new NodeReferenceArray( + new NodeReference().setId(0).setKeyReference(new KeyReferenceArray(new KeyReference().setPosition(0))))) + .setFunction(new TransformationFunction().setOperator("foobar1")); + AnyNodeArray nodeArray1 = new AnyNodeArray(AnyNode.create(featureReference1), AnyNode.create(transformation1)); + IntegerMap featureNameMap1 = new IntegerMap(Collections.singletonMap("apple", 1)); + ComputeGraph graph1 = new ComputeGraph().setNodes(nodeArray1).setFeatureNames(featureNameMap1); + Assert.assertEquals(graph1.getNodes().size(), 2); + External featureReference2 = new External().setId(0).setName("feature2"); + Transformation transformation2 = new Transformation().setId(1) + .setInputs(new NodeReferenceArray( + new NodeReference().setId(0).setKeyReference(new KeyReferenceArray(new KeyReference().setPosition(0))))) + .setFunction(new TransformationFunction().setOperator("foobar2")); + AnyNodeArray nodeArray2 = new AnyNodeArray(AnyNode.create(featureReference2), AnyNode.create(transformation2)); + IntegerMap featureNameMap2 = new IntegerMap(Collections.singletonMap("feature1", 1)); + ComputeGraph graph2 = new ComputeGraph().setNodes(nodeArray2).setFeatureNames(featureNameMap2); + Assert.assertEquals(graph2.getNodes().size(), 2); + ComputeGraph merged = ComputeGraphs.merge(Arrays.asList(graph1, graph2)); + Assert.assertEquals(merged.getNodes().size(), 3); + } + + @Test(description = "test remove redundant nodes method") + public void testRemoveDuplicates() throws CloneNotSupportedException { + External featureReference1 = new External().setId(0).setName("feature1"); + Transformation transformation1 = new Transformation().setId(1) + .setInputs(new NodeReferenceArray( + new NodeReference().setId(0).setKeyReference(new KeyReferenceArray(new KeyReference().setPosition(0))))) + .setFunction(new TransformationFunction().setOperator("foobar1")); + External featureReference2 = new External().setId(2).setName("feature1"); + Transformation transformation2 = new Transformation().setId(3) + .setInputs(new NodeReferenceArray(new NodeReference().setId(2).setKeyReference(new KeyReferenceArray(new KeyReference().setPosition(0))))) + .setFunction(new TransformationFunction().setOperator("foobar2")); + AnyNodeArray nodeArray = new AnyNodeArray(AnyNode.create(featureReference1), AnyNode.create(featureReference2), + AnyNode.create(transformation1), AnyNode.create(transformation2)); + IntegerMap featureNameMap = new IntegerMap( + ImmutableMap.of("apple", 1, "banana", 3)); + ComputeGraph graph = new ComputeGraph().setNodes(nodeArray).setFeatureNames(featureNameMap); + Assert.assertEquals(graph.getNodes().size(), 4); + ComputeGraph simplified = ComputeGraphs.removeRedundancies(graph); + Assert.assertEquals(simplified.getNodes().size(), 3); + } + + @Test(description = "test with same feature name and different keys") + public void testResolveGraph() throws CloneNotSupportedException { + DataSource dataSource1 = + new DataSource().setId(0).setSourceType(DataSourceType.UPDATE).setExternalSourceRef("dataSource1"); + Transformation transformation1 = new Transformation().setId(1) + .setInputs(new NodeReferenceArray( + new NodeReference().setId(0).setKeyReference(new KeyReferenceArray(new KeyReference().setPosition(0))))) + .setFunction(new TransformationFunction().setOperator("foobar1")); + AnyNodeArray nodeArray1 = new AnyNodeArray(AnyNode.create(dataSource1), AnyNode.create(transformation1)); + IntegerMap featureNameMap1 = new IntegerMap(Collections.singletonMap("apple", 1)); + ComputeGraph graph1 = new ComputeGraph().setNodes(nodeArray1).setFeatureNames(featureNameMap1); + + List requestedFeatures = Arrays.asList( + new Resolver.FeatureRequest("apple", Collections.singletonList("viewer"), Duration.ZERO,"apple__viewer"), + new Resolver.FeatureRequest("apple", Collections.singletonList("viewee"), Duration.ZERO, "apple__viewee")); + ComputeGraph resolved = Resolver.create(graph1).resolveForRequest(requestedFeatures); + Assert.assertTrue(resolved.getFeatureNames().containsKey("apple__viewer")); + Assert.assertTrue(resolved.getFeatureNames().containsKey("apple__viewee")); + } + + @Test(expectedExceptions = RuntimeException.class) + public void testNonSequentialNodes() { + External featureReference1 = new External().setId(0).setName("feature1"); + Transformation transformation1 = new Transformation().setId(1) + .setInputs(new NodeReferenceArray( + new NodeReference().setId(0).setKeyReference(new KeyReferenceArray(new KeyReference().setPosition(0))))) + .setFunction(new TransformationFunction().setOperator("foobar1")); + External featureReference2 = new External().setId(2).setName("feature1"); + + // Node id 6 is not sequential + Transformation transformation2 = new Transformation().setId(6) + .setInputs(new NodeReferenceArray(new NodeReference().setId(2).setKeyReference(new KeyReferenceArray(new KeyReference().setPosition(0))))) + .setFunction(new TransformationFunction().setOperator("foobar2")); + AnyNodeArray nodeArray = new AnyNodeArray(AnyNode.create(featureReference1), AnyNode.create(featureReference2), + AnyNode.create(transformation1), AnyNode.create(transformation2)); + IntegerMap featureNameMap = new IntegerMap( + ImmutableMap.of("apple", 1, "banana", 3)); + ComputeGraph graph = new ComputeGraph().setNodes(nodeArray).setFeatureNames(featureNameMap); + ComputeGraphs.ensureNodeIdsAreSequential(graph); + } + + @Test(expectedExceptions = RuntimeException.class) + public void testDependenciesNotExist() { + External featureReference1 = new External().setId(0).setName("feature1"); + Transformation transformation1 = new Transformation().setId(1) + // node 6 does not exist + .setInputs(new NodeReferenceArray( + new NodeReference().setId(6).setKeyReference(new KeyReferenceArray(new KeyReference().setPosition(0))))) + .setFunction(new TransformationFunction().setOperator("foobar1")); + External featureReference2 = new External().setId(2).setName("feature1"); + + AnyNodeArray nodeArray = new AnyNodeArray(AnyNode.create(featureReference1), AnyNode.create(featureReference2), + AnyNode.create(transformation1)); + IntegerMap featureNameMap = new IntegerMap( + ImmutableMap.of("apple", 1)); + ComputeGraph graph = new ComputeGraph().setNodes(nodeArray).setFeatureNames(featureNameMap); + ComputeGraphs.ensureNodeReferencesExist(graph); + } + + @Test(expectedExceptions = RuntimeException.class) + public void testNoDependencyCycle() { + External featureReference1 = new External().setId(0).setName("feature1"); + + // Dependency cycle created + Transformation transformation1 = new Transformation().setId(1) + .setInputs(new NodeReferenceArray(new NodeReference().setId(1).setKeyReference(new KeyReferenceArray(new KeyReference().setPosition(0))))) + .setFunction(new TransformationFunction().setOperator("foobar1")); + AnyNodeArray nodeArray = new AnyNodeArray(AnyNode.create(featureReference1), AnyNode.create(transformation1)); + IntegerMap featureNameMap = new IntegerMap( + ImmutableMap.of("apple", 1)); + ComputeGraph graph = new ComputeGraph().setNodes(nodeArray).setFeatureNames(featureNameMap); + ComputeGraphs.ensureNoDependencyCycles(graph); + } + + @Test(expectedExceptions = RuntimeException.class) + public void testNoExternalReferencesToSelf() { + External featureReference1 = new External().setId(0).setName("feature1"); + Transformation transformation1 = new Transformation().setId(1) + .setInputs(new NodeReferenceArray(new NodeReference().setId(1).setKeyReference(new KeyReferenceArray(new KeyReference().setPosition(0))))) + .setFunction(new TransformationFunction().setOperator("foobar1")) + .setFeatureName("feature1"); + AnyNodeArray nodeArray = new AnyNodeArray(AnyNode.create(featureReference1), AnyNode.create(transformation1)); + IntegerMap featureNameMap = new IntegerMap( + ImmutableMap.of("feature1", 1)); + ComputeGraph graph = new ComputeGraph().setNodes(nodeArray).setFeatureNames(featureNameMap); + ComputeGraphs.ensureNoExternalReferencesToSelf(graph); + } + + @Test(expectedExceptions = RuntimeException.class) + public void testNoConcreteKeys() { + External featureReference1 = new External().setId(0).setName("feature1"); + IntegerArray array = new IntegerArray(); + array.add(1); + Transformation transformation1 = new Transformation().setId(1) + .setInputs(new NodeReferenceArray(new NodeReference().setId(1).setKeyReference(new KeyReferenceArray(new KeyReference().setPosition(0))))) + .setFunction(new TransformationFunction().setOperator("foobar1")) + .setFeatureName("feature1") + .setConcreteKey(new ConcreteKey().setKey(array)); + AnyNodeArray nodeArray = new AnyNodeArray(AnyNode.create(featureReference1), AnyNode.create(transformation1)); + IntegerMap featureNameMap = new IntegerMap( + ImmutableMap.of("feature1", 1)); + ComputeGraph graph = new ComputeGraph().setNodes(nodeArray).setFeatureNames(featureNameMap); + ComputeGraphs.ensureNoConcreteKeys(graph); + } + + @Test(description = "test attaching of concrete node to dependencies of transformation node") + public void testAddConcreteKeyToTransformationNode() throws CloneNotSupportedException { + DataSource dataSource1 = new DataSource().setId(0).setExternalSourceRef("testPath"); + Transformation transformation1 = new Transformation().setId(1) + .setInputs(new NodeReferenceArray( + new NodeReference().setId(0).setKeyReference(new KeyReferenceArray(new KeyReference().setPosition(0))))) + .setFunction(new TransformationFunction().setOperator("foobar1")) + .setFeatureName("apple"); + Transformation transformation2 = new Transformation().setId(2) + .setInputs(new NodeReferenceArray( + new NodeReference().setId(1).setKeyReference(new KeyReferenceArray(new KeyReference().setPosition(0))))) + .setFunction(new TransformationFunction().setOperator("foobar1")) + .setFeatureName("banana"); + AnyNodeArray nodeArray = new AnyNodeArray(AnyNode.create(dataSource1), AnyNode.create(transformation1), AnyNode.create(transformation2)); + IntegerMap featureNameMap = new IntegerMap( + ImmutableMap.of("apple", 1, "banana", 2)); + ComputeGraph graph = new ComputeGraph().setNodes(nodeArray).setFeatureNames(featureNameMap); + ComputeGraph simplified = ComputeGraphs.removeRedundancies(graph); + List keys = new ArrayList<>(); + keys.add("x"); + + // The same concrete key should get attached to the dependencies + ComputeGraph withConcreteKeyAttached = new Resolver(ComputeGraphs.removeRedundancies(simplified)).resolveForFeature("banana", keys, "banana"); + + DataSource createdKeyNode = withConcreteKeyAttached.getNodes().stream().map(AnyNode::getDataSource) + .filter(Objects::nonNull).filter(p -> Objects.equals(p.getExternalSourceRef(), "x")).collect(Collectors.toList()).get(0); + Transformation appleNode = withConcreteKeyAttached.getNodes().stream().map(AnyNode::getTransformation) + .filter(Objects::nonNull).filter(p -> Objects.equals(p.getFeatureName(), "apple")).collect(Collectors.toList()).get(0); + Transformation bananaNode = withConcreteKeyAttached.getNodes().stream().map(AnyNode::getTransformation) + .filter(Objects::nonNull).filter(p -> Objects.equals(p.getFeatureName(), "banana")).collect(Collectors.toList()).get(0); + Assert.assertEquals(Objects.requireNonNull(appleNode.getConcreteKey()).getKey().get(0), createdKeyNode.getId()); + Assert.assertEquals(Objects.requireNonNull(bananaNode.getConcreteKey()).getKey().get(0), createdKeyNode.getId()); + } + + @Test(description = "test attaching of concrete node to dependencies of aggregation node") + public void testAddConcreteKeyToAggregationNode() throws CloneNotSupportedException { + DataSource dataSource1 = new DataSource().setId(0); + Aggregation aggregation1 = new Aggregation().setId(1) + .setInput(new NodeReference().setId(0).setKeyReference(new KeyReferenceArray(new KeyReference().setPosition(0)))).setFeatureName("apple"); + AnyNodeArray nodeArray = new AnyNodeArray(AnyNode.create(dataSource1), AnyNode.create(aggregation1)); + IntegerMap featureNameMap = new IntegerMap( + ImmutableMap.of("apple", 1)); + ComputeGraph graph = new ComputeGraph().setNodes(nodeArray).setFeatureNames(featureNameMap); + ComputeGraph simplified = ComputeGraphs.removeRedundancies(graph); + List keys = new ArrayList<>(); + keys.add("x"); + + // The same concrete key should get attached to the dependencies + ComputeGraph withConcreteKeyAttached = new Resolver(ComputeGraphs.removeRedundancies(simplified)).resolveForFeature("apple", keys, "apple"); + + Aggregation appleNode = withConcreteKeyAttached.getNodes().stream().map(AnyNode::getAggregation) + .filter(Objects::nonNull).filter(p -> Objects.equals(p.getFeatureName(), "apple")).collect(Collectors.toList()).get(0); + Assert.assertEquals(Objects.requireNonNull(appleNode.getConcreteKey()).getKey().get(0).intValue(), 0); + } + + @Test(description = "test attaching of concrete node to dependencies of seq join node") + public void testAddConcreteKeyToSeqJoinNode() throws CloneNotSupportedException { + DataSource dataSource1 = new DataSource().setId(0).setExternalSourceRef("testpath"); + Transformation transformation1 = new Transformation().setId(1) + .setInputs(new NodeReferenceArray( + new NodeReference().setId(0).setKeyReference(new KeyReferenceArray(new KeyReference().setPosition(0))))) + .setFunction(new TransformationFunction().setOperator("foobar1")) + .setFeatureName("apple"); + Transformation transformation2 = new Transformation().setId(2) + .setInputs(new NodeReferenceArray( + new NodeReference().setId(1).setKeyReference(new KeyReferenceArray(new KeyReference().setPosition(0))))) + .setFunction(new TransformationFunction().setOperator("foobar1")) + .setFeatureName("banana"); + NodeReference nr = new NodeReference().setId(1).setKeyReference(new KeyReferenceArray(new KeyReference().setPosition(0))); + Lookup.LookupKey lookupKey = new Lookup.LookupKey(); + lookupKey.setNodeReference(nr); + Lookup.LookupKeyArray lookupKeyArray = new Lookup.LookupKeyArray(); + lookupKeyArray.add(lookupKey); + Lookup lookupNode1 = new Lookup().setId(3).setLookupNode(2).setLookupKey(lookupKeyArray).setFeatureName("apple-banana"); + AnyNodeArray nodeArray = new AnyNodeArray(AnyNode.create(dataSource1), AnyNode.create(transformation1), + AnyNode.create(transformation2), AnyNode.create(lookupNode1)); + IntegerMap featureNameMap = new IntegerMap( + ImmutableMap.of("apple", 1, "banana", 2, + "apple-banana", 3)); + ComputeGraph graph = new ComputeGraph().setNodes(nodeArray).setFeatureNames(featureNameMap); + ComputeGraph simplified = ComputeGraphs.removeRedundancies(graph); + List keys = new ArrayList<>(); + keys.add("x"); + // The same concrete key should get attached to the dependencies + ComputeGraph withConcreteKeyAttached = new Resolver(ComputeGraphs.removeRedundancies(simplified)).resolveForFeature("apple-banana", keys, "apple"); + + DataSource createdKeyNode = withConcreteKeyAttached.getNodes().stream().map(AnyNode::getDataSource) + .filter(Objects::nonNull).filter(p -> Objects.equals(p.getExternalSourceRef(), "x")).collect(Collectors.toList()).get(0); + Transformation appleNode = withConcreteKeyAttached.getNodes().stream().map(AnyNode::getTransformation) + .filter(Objects::nonNull).filter(p -> Objects.equals(p.getFeatureName(), "apple")).collect(Collectors.toList()).get(0); + Transformation bananaNode = withConcreteKeyAttached.getNodes().stream().map(AnyNode::getTransformation) + .filter(Objects::nonNull).filter(p -> Objects.equals(p.getFeatureName(), "banana")).collect(Collectors.toList()).get(0); + Assert.assertEquals(Objects.requireNonNull(appleNode.getConcreteKey()).getKey().get(0), createdKeyNode.getId()); + + // key of the expansion should be the transformation node of apple. + Assert.assertEquals(Objects.requireNonNull(bananaNode.getConcreteKey()).getKey().get(0).intValue(), 2); + } + + @Test(description = "test attaching of concrete node to dependencies of complex seq join node with multi-key") + public void testAddConcreteKeyToComplexSeqJoinNode() throws CloneNotSupportedException { + DataSource dataSource1 = new DataSource().setId(0).setExternalSourceRef("testpath"); + Transformation transformation1 = new Transformation().setId(1) + .setInputs(new NodeReferenceArray( + new NodeReference().setId(0).setKeyReference(new KeyReferenceArray(new KeyReference().setPosition(0))))) + .setFunction(new TransformationFunction().setOperator("foobar1")) + .setFeatureName("apple"); + Transformation transformation2 = new Transformation().setId(2) + .setInputs(new NodeReferenceArray( + new NodeReference().setId(1).setKeyReference(new KeyReferenceArray(new KeyReference().setPosition(0))))) + .setFunction(new TransformationFunction().setOperator("foobar1")) + .setFeatureName("banana"); + NodeReference nr = new NodeReference().setId(1).setKeyReference(new KeyReferenceArray(new KeyReference().setPosition(0))); + Lookup.LookupKey lookupKey = new Lookup.LookupKey(); + lookupKey.setNodeReference(nr); + Lookup.LookupKeyArray lookupKeyArray = new Lookup.LookupKeyArray(); + lookupKeyArray.add(lookupKey); + Lookup lookupNode1 = new Lookup().setId(3).setLookupNode(2).setLookupKey(lookupKeyArray).setFeatureName("apple-banana"); + AnyNodeArray nodeArray = new AnyNodeArray(AnyNode.create(dataSource1), AnyNode.create(transformation1), + AnyNode.create(transformation2), AnyNode.create(lookupNode1)); + IntegerMap featureNameMap = new IntegerMap( + ImmutableMap.of("apple", 1, "banana", 2, + "apple-banana", 3)); + ComputeGraph graph = new ComputeGraph().setNodes(nodeArray).setFeatureNames(featureNameMap); + ComputeGraph simplified = ComputeGraphs.removeRedundancies(graph); + List keys = new ArrayList<>(); + keys.add("x"); + keys.add("y"); + // The same concrete key should get attached to the dependencies + ComputeGraph withConcreteKeyAttached = new Resolver(ComputeGraphs.removeRedundancies(simplified)).resolveForFeature("apple-banana", keys, "apple"); + + DataSource createdKeyNode = withConcreteKeyAttached.getNodes().stream().map(AnyNode::getDataSource) + .filter(Objects::nonNull).filter(p -> Objects.equals(p.getExternalSourceRef(), "x")).collect(Collectors.toList()).get(0); + Transformation appleNode = withConcreteKeyAttached.getNodes().stream().map(AnyNode::getTransformation) + .filter(Objects::nonNull).filter(p -> Objects.equals(p.getFeatureName(), "apple")).collect(Collectors.toList()).get(0); + Transformation bananaNode = withConcreteKeyAttached.getNodes().stream().map(AnyNode::getTransformation) + .filter(Objects::nonNull).filter(p -> Objects.equals(p.getFeatureName(), "banana")).collect(Collectors.toList()).get(0); + Assert.assertEquals(Objects.requireNonNull(appleNode.getConcreteKey()).getKey().get(0), createdKeyNode.getId()); + + // key of the expansion should be the transformation node of apple. + Assert.assertEquals(Objects.requireNonNull(bananaNode.getConcreteKey()).getKey().get(0), appleNode.getId()); + } +} \ No newline at end of file diff --git a/feathr-compute/src/test/resources/anchorConfigWithMvelConverter.conf b/feathr-compute/src/test/resources/anchorConfigWithMvelConverter.conf new file mode 100644 index 000000000..6bf621a4b --- /dev/null +++ b/feathr-compute/src/test/resources/anchorConfigWithMvelConverter.conf @@ -0,0 +1,10 @@ +anchors: { + member-lix-segment: { + source: "/data/derived/lix/euc/member/#LATEST" + key: "id" + features: { + member_lixSegment_isStudent: "is_student" + member_lixSegment_isJobSeeker: "job_seeker_class == 'active'" + } + } +} \ No newline at end of file diff --git a/feathr-compute/src/test/resources/anchorWithKeyExtractor.conf b/feathr-compute/src/test/resources/anchorWithKeyExtractor.conf new file mode 100644 index 000000000..dfda42619 --- /dev/null +++ b/feathr-compute/src/test/resources/anchorWithKeyExtractor.conf @@ -0,0 +1,12 @@ +anchors: { + cohortActorAnchors: { + source: "seqJoin/cohortActorFeatures.avro.json" + keyExtractor: "com.linkedin.feathr.offline.SeqJoinExpansionKeyExtractor" + features: { + cohortActorFeature_base: { + def.sqlExpr: cohortActorFeature + type: NUMERIC + } + } + } +} \ No newline at end of file diff --git a/feathr-compute/src/test/resources/anchoredFeature.conf b/feathr-compute/src/test/resources/anchoredFeature.conf new file mode 100644 index 000000000..b40c3cdb6 --- /dev/null +++ b/feathr-compute/src/test/resources/anchoredFeature.conf @@ -0,0 +1,12 @@ +anchors: { + waterloo-member-year-born: { + source: "seqJoin/member.avro.json" + key: "x" + features: { + waterloo_member_yearBorn: { + def:"yearBorn" + type: "NUMERIC" + } + } + } +} diff --git a/feathr-compute/src/test/resources/anchoredFeature2.conf b/feathr-compute/src/test/resources/anchoredFeature2.conf new file mode 100644 index 000000000..908514336 --- /dev/null +++ b/feathr-compute/src/test/resources/anchoredFeature2.conf @@ -0,0 +1,18 @@ +sources: { + xyz: { + location: { path: "slidingWindowAgg/localSWAAnchorTestFeatureData/daily" } + } +} + + +anchors: { + waterloo-member-year-born: { + source: xyz + key: "x" + features: { + f1: { + def: f1 + } + } + } +} \ No newline at end of file diff --git a/feathr-compute/src/test/resources/complexDerivedFeature.conf b/feathr-compute/src/test/resources/complexDerivedFeature.conf new file mode 100644 index 000000000..86d4b2e61 --- /dev/null +++ b/feathr-compute/src/test/resources/complexDerivedFeature.conf @@ -0,0 +1,26 @@ +anchors: { + anchor1: { + source: "%s" + key: "xInFeatureData" + features: { + AA: { + def: "a" + default: 2 + }, + BB: { + def: "b" + default: 2 + } + } + } +} +derivations: { + C: { + key: [viewerId, vieweeId] + inputs: { + arg1: { key: viewerId, feature: AA } + arg2: { key: vieweeId, feature: BB } + } + definition: "arg1 + arg2" + } +} \ No newline at end of file diff --git a/feathr-compute/src/test/resources/derivedFeatureWithClass.conf b/feathr-compute/src/test/resources/derivedFeatureWithClass.conf new file mode 100644 index 000000000..596733330 --- /dev/null +++ b/feathr-compute/src/test/resources/derivedFeatureWithClass.conf @@ -0,0 +1,26 @@ +anchors: { + anchor1: { + source: "%s" + key: "xInFeatureData" + features: { + AA: { + def: "a" + default: 2 + }, + BB: { + def: "b" + default: 2 + } + } + } +} +derivations: { + C: { + key: [viewerId, vieweeId] + inputs: [ + { key: viewerId, feature: AA } + { key: vieweeId, feature: BB } + ] + class: "com.linkedin.feathr.offline.anchored.anchorExtractor.TestxGenericSparkFeatureDataExtractor2" + } +} \ No newline at end of file diff --git a/feathr-compute/src/test/resources/mvelDerivedFeature.conf b/feathr-compute/src/test/resources/mvelDerivedFeature.conf new file mode 100644 index 000000000..456c38770 --- /dev/null +++ b/feathr-compute/src/test/resources/mvelDerivedFeature.conf @@ -0,0 +1,15 @@ +anchors: { + anchor1: { + source: "%s" + key: "xInFeatureData" + features: { + AA: { + def: "a" + default: 2 + } + } + } +} +derivations: { + B: "AA*2" +} \ No newline at end of file diff --git a/feathr-compute/src/test/resources/seqJoinFeature.conf b/feathr-compute/src/test/resources/seqJoinFeature.conf new file mode 100644 index 000000000..e7a471e07 --- /dev/null +++ b/feathr-compute/src/test/resources/seqJoinFeature.conf @@ -0,0 +1,30 @@ +anchors: { + industry-local: { + source: "seqJoin/industry.avro.json" + key.sqlExpr: industryId + features: { + MemberIndustryName.def.sqlExpr : industryName + } + } + waterloo-member-geolocation-local: { + source: "seqJoin/member.avro.json" + key.sqlExpr: "concat('',x)" + features: { + MemberIndustryId : { + def.sqlExpr: profileIndustryId + default: 1 + type: NUMERIC + } + } + } +} +derivations: { + seq_join_industry_names: { + key: "x" + join: { + base: { key: x, feature: MemberIndustryId } + expansion: { key: industryId, feature: MemberIndustryName } + } + aggregation: "UNION" + } +} \ No newline at end of file diff --git a/feathr-compute/src/test/resources/swa.conf b/feathr-compute/src/test/resources/swa.conf new file mode 100644 index 000000000..3fc33e5e7 --- /dev/null +++ b/feathr-compute/src/test/resources/swa.conf @@ -0,0 +1,23 @@ +sources: { + swaSource: { + location: { path: "generation/daily" } + timePartitionPattern: "yyyy/MM/dd" + timeWindowParameters: { + timestampColumn: "timestamp" + timestampColumnFormat: "yyyy-MM-dd" + } + } +} +anchors: { + swaAnchor: { + source: "swaSource" + key: "x" + features: { + memberEmbedding: { + def: "embedding" + aggregation: LATEST + window: 3d + } + } + } +} diff --git a/feathr-compute/src/test/resources/swaWithExtractor.conf b/feathr-compute/src/test/resources/swaWithExtractor.conf new file mode 100644 index 000000000..8f9ff84f1 --- /dev/null +++ b/feathr-compute/src/test/resources/swaWithExtractor.conf @@ -0,0 +1,99 @@ +sources: { + ptSource: { + type: "PASSTHROUGH" + } + swaSource: { + location: { path: "slidingWindowAgg/localSWAAnchorTestFeatureData/daily" } + timePartitionPattern: "yyyy/MM/dd" + timeWindowParameters: { + timestampColumn: "timestamp" + timestampColumnFormat: "yyyy-MM-dd" + } + } +} + +anchors: { + ptAnchor: { + source: "ptSource" + key: "x" + features: { + f1f1: { + def: "([$.term:$.value] in passthroughFeatures if $.name == 'f1f1')" + } + } + } + swaAnchor: { + source: "swaSource" + key: "substring(x, 0)" + lateralViewParameters: { + lateralViewDef: explode(features) + lateralViewItemAlias: feature + } + features: { + f1: { + def: "feature.col.value" + filter: "feature.col.name = 'f1'" + aggregation: SUM + groupBy: "feature.col.term" + window: 3d + } + } + } + + swaAnchor2: { + source: "swaSource" + key: "x" + lateralViewParameters: { + lateralViewDef: explode(features) + lateralViewItemAlias: feature + } + features: { + f1Sum: { + def: "feature.col.value" + filter: "feature.col.name = 'f1'" + aggregation: SUM + groupBy: "feature.col.term" + window: 3d + } + } + } + swaAnchorWithKeyExtractor: { + source: "swaSource" + keyExtractor: "com.linkedin.frame.offline.anchored.keyExtractor.SimpleSampleKeyExtractor" + features: { + f3: { + def: "aggregationWindow" + aggregation: SUM + window: 3d + } + } + } + swaAnchorWithKeyExtractor2: { + source: "swaSource" + keyExtractor: "com.linkedin.frame.offline.anchored.keyExtractor.SimpleSampleKeyExtractor" + features: { + f4: { + def: "aggregationWindow" + aggregation: SUM + window: 3d + } + } + } + swaAnchorWithKeyExtractor3: { + source: "swaSource" + keyExtractor: "com.linkedin.frame.offline.anchored.keyExtractor.SimpleSampleKeyExtractor2" + lateralViewParameters: { + lateralViewDef: explode(features) + lateralViewItemAlias: feature + } + features: { + f2: { + def: "feature.col.value" + filter: "feature.col.name = 'f2'" + aggregation: SUM + groupBy: "feature.col.term" + window: 3d + } + } + } +} \ No newline at end of file diff --git a/feathr-config/build.gradle b/feathr-config/build.gradle new file mode 100644 index 000000000..626c58e76 --- /dev/null +++ b/feathr-config/build.gradle @@ -0,0 +1,71 @@ +apply plugin: 'java' +apply plugin: 'pegasus' +apply plugin: 'maven-publish' +apply plugin: 'signing' +apply plugin: "com.vanniktech.maven.publish.base" + +repositories { + mavenCentral() + mavenLocal() + maven { + url "https://repository.mulesoft.org/nexus/content/repositories/public/" + } + maven { + url "https://linkedin.jfrog.io/artifactory/open-source/" // GMA, pegasus + } +} + +dependencies { + implementation project(":feathr-data-models") + implementation project(path: ':feathr-data-models', configuration: 'dataTemplate') + implementation spec.product.avro + implementation spec.product.pegasus.data + implementation spec.product.typesafe_config + implementation spec.product.log4j + implementation spec.product.jsonSchemaVali + implementation spec.product.jackson.jackson_databind + implementation spec.product.mvel + implementation spec.product.json + + testImplementation spec.product.testing + testImplementation spec.product.mockito + testImplementation spec.product.equalsverifier + testImplementation spec.product.mockito_inline +} + +test { + maxParallelForks = 1 + forkEvery = 1 + // need to keep a lower heap size (TOOLS-296596) + minHeapSize = "512m" + useTestNG() +} + +java { + withSourcesJar() + withJavadocJar() +} + +tasks.withType(Javadoc) { + options.addStringOption('Xdoclint:none', '-quiet') + options.addStringOption('encoding', 'UTF-8') + options.addStringOption('charSet', 'UTF-8') +} + +// Required for publishing to local maven +publishing { + publications { + mavenJava(MavenPublication) { + artifactId = 'feathr-config' + from components.java + versionMapping { + usage('java-api') { + fromResolutionOf('runtimeClasspath') + } + usage('java-runtime') { + fromResolutionResult() + } + } + } + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/config/FeatureDefinitionLoader.java b/feathr-config/src/main/java/com/linkedin/feathr/config/FeatureDefinitionLoader.java new file mode 100644 index 000000000..837fcec45 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/config/FeatureDefinitionLoader.java @@ -0,0 +1,35 @@ +package com.linkedin.feathr.config; + +import com.google.common.base.Preconditions; +import com.linkedin.feathr.core.config.producer.FeatureDefConfig; +import com.linkedin.feathr.core.configbuilder.ConfigBuilder; +import com.linkedin.feathr.core.configdataprovider.ConfigDataProvider; +import javax.annotation.Nonnull; + + +/** + * Loader class for hich encloses all characteristics of a feature, such as source and + * transformation. + */ +public class FeatureDefinitionLoader { + private final ConfigBuilder _configBuilder; + + + /** + * Constructor. + * @param configBuilder Interface for building {@link FeatureDefConfig} from a + * HOCON-based Frame config. + */ + public FeatureDefinitionLoader(@Nonnull ConfigBuilder configBuilder) { + Preconditions.checkNotNull(configBuilder); + _configBuilder = configBuilder; + } + + public FeatureDefConfig loadAllFeatureDefinitions(@Nonnull ConfigDataProvider + configDataProvider) { + Preconditions.checkNotNull(configDataProvider); + FeatureDefConfig featureDefConfig = _configBuilder.buildFeatureDefConfig(configDataProvider); + + return featureDefConfig; + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/config/FeatureDefinitionLoaderFactory.java b/feathr-config/src/main/java/com/linkedin/feathr/config/FeatureDefinitionLoaderFactory.java new file mode 100644 index 000000000..92651a682 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/config/FeatureDefinitionLoaderFactory.java @@ -0,0 +1,24 @@ +package com.linkedin.feathr.config; + +import com.linkedin.feathr.core.configbuilder.ConfigBuilder; + + +/** + * Factory of {@link FeatureDefinitionLoader} + */ +public class FeatureDefinitionLoaderFactory { + private static FeatureDefinitionLoader _instance; + + private FeatureDefinitionLoaderFactory() { + } + + /** + * Get an instance of {@link FeatureDefinitionLoader}. + */ + public static FeatureDefinitionLoader getInstance() { + if (_instance == null) { + _instance = new FeatureDefinitionLoader(ConfigBuilder.get()); + } + return _instance; + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/ConfigObj.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/ConfigObj.java new file mode 100644 index 000000000..4b1d68c21 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/ConfigObj.java @@ -0,0 +1,10 @@ +package com.linkedin.feathr.core.config; + +import java.io.Serializable; + + +/** + * Marker interface for all config objects used in Frame + */ +public interface ConfigObj extends Serializable { +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/ConfigType.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/ConfigType.java new file mode 100644 index 000000000..b474d58c9 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/ConfigType.java @@ -0,0 +1,12 @@ +package com.linkedin.feathr.core.config; + + +/** + * Enumeration class for FeatureDef and Join Config classes + */ +public enum ConfigType { + FeatureDef, + Join, + Metadata, + Presentation +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/TimeWindowAggregationType.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/TimeWindowAggregationType.java new file mode 100644 index 000000000..c8b6c780a --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/TimeWindowAggregationType.java @@ -0,0 +1,9 @@ +package com.linkedin.feathr.core.config; + + +/** + * Enumeration class for Sliding time-window aggregation + */ +public enum TimeWindowAggregationType { + SUM, COUNT, AVG, MAX, MIN, TIMESINCE, LATEST, AVG_POOLING, MAX_POOLING, MIN_POOLING +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/WindowType.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/WindowType.java new file mode 100644 index 000000000..2b6cb9eac --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/WindowType.java @@ -0,0 +1,9 @@ +package com.linkedin.feathr.core.config; + + +/** + * Enumeration class for type of window aggregation + */ +public enum WindowType { + SLIDING, FIXED, SESSION +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/common/DateTimeConfig.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/common/DateTimeConfig.java new file mode 100644 index 000000000..a2a0f5113 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/common/DateTimeConfig.java @@ -0,0 +1,141 @@ +package com.linkedin.feathr.core.config.common; + +import com.linkedin.feathr.core.config.ConfigObj; +import java.time.Duration; +import java.time.temporal.ChronoUnit; +import java.util.Objects; +import java.util.TimeZone; + + +/** + * Represent a time period or a time point. + * the startTime is - offset - length + 1, + * the endTime is referenceEndDateTime in timeZone - offset + */ +public class DateTimeConfig implements ConfigObj { + // end time of this time period, it is called reference because it might + // need to shift by _offsetInSeconds to be the actual endTime, e.g., a date, or NOW, or LATEST + private final String _referenceEndTime; + // _referenceEndTime format, e.g., yyyy-MM-dd + private final String _referenceEndTimeFormat; + // daily or hourly + private final ChronoUnit _timeResolution; + // length of the time period, in terms of _timeResolution + private final long _length; + // offset of referenceEndTIme, means the actual end time is <_offset> before referenceEndTIme + private final Duration _offset; + private final TimeZone _timeZone; + + /** + * Constructor + * @param referenceEndTime end time of this time period, it is called reference because it might + * need to shift by _offsetInSeconds to be the actual endTime, e.g., a date, or NOW, or LATEST + * @param referenceEndTimeFormat format, e.g., yyyy-MM-dd + * @param timeResolution daily or hourly + * @param length length of the time period, in terms of _timeResolution + * @param offset offset + * @param timeZone time zone + */ + public DateTimeConfig(String referenceEndTime, String referenceEndTimeFormat, ChronoUnit timeResolution, long length, + Duration offset, TimeZone timeZone) { + _referenceEndTime = referenceEndTime; + _referenceEndTimeFormat = referenceEndTimeFormat; + _timeResolution = timeResolution; + _length = length; + _offset = offset; + _timeZone = timeZone; + } + + /* + * The previously used lombok library auto generates getters with underscore, which is used in production. + * For backward compatibility, we need to keep these getters. + * However, function name with underscore can not pass LinkedIn's style check, here we need suppress the style check + * for the getters only. + * + * For more detail, please refer to the style check wiki: + * https://iwww.corp.linkedin.com/wiki/cf/display/TOOLS/Checking+Java+Coding+Style+with+Gradle+Checkstyle+Plugin + * + * TODO - 7493) remove the ill-named getters + */ + // CHECKSTYLE:OFF + @Deprecated + public String get_referenceEndTime() { + return _referenceEndTime; + } + + @Deprecated + public String get_referenceEndTimeFormat() { + return _referenceEndTimeFormat; + } + + @Deprecated + public ChronoUnit get_timeResolution() { + return _timeResolution; + } + + @Deprecated + public long get_length() { + return _length; + } + + @Deprecated + public Duration get_offset() { + return _offset; + } + + @Deprecated + public TimeZone get_timeZone() { + return _timeZone; + } + // CHECKSTYLE:ON + + public String getReferenceEndTime() { + return _referenceEndTime; + } + + public String getReferenceEndTimeFormat() { + return _referenceEndTimeFormat; + } + + public ChronoUnit getTimeResolution() { + return _timeResolution; + } + + public long getLength() { + return _length; + } + + public Duration getOffset() { + return _offset; + } + + public TimeZone getTimeZone() { + return _timeZone; + } + + @Override + public String toString() { + return "DateTimeConfig{" + "_referenceEndTime='" + _referenceEndTime + '\'' + ", _referenceEndTimeFormat='" + + _referenceEndTimeFormat + '\'' + ", _timeResolution=" + _timeResolution + ", _length=" + _length + + ", _offset=" + _offset + ", _timeZone=" + _timeZone + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof DateTimeConfig)) { + return false; + } + DateTimeConfig that = (DateTimeConfig) o; + return _length == that._length && Objects.equals(_referenceEndTime, that._referenceEndTime) && Objects.equals( + _referenceEndTimeFormat, that._referenceEndTimeFormat) && _timeResolution == that._timeResolution + && Objects.equals(_offset, that._offset) && Objects.equals(_timeZone, that._timeZone); + } + + @Override + public int hashCode() { + return Objects.hash(_referenceEndTime, _referenceEndTimeFormat, _timeResolution, _length, _offset, _timeZone); + } +} \ No newline at end of file diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/common/OutputFormat.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/common/OutputFormat.java new file mode 100644 index 000000000..f654d61bc --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/common/OutputFormat.java @@ -0,0 +1,9 @@ +package com.linkedin.feathr.core.config.common; + +/** + * output format of Frame feature generation, + * name-term-value(NAME_TERM_VALUE), name-listof-term-value(COMPACT_NAME_TERM_VALUE), RAW_DATA(raw dataframe), TENSOR + */ +public enum OutputFormat { + NAME_TERM_VALUE, COMPACT_NAME_TERM_VALUE, RAW_DATA, TENSOR, CUSTOMIZED, QUINCE_FDS +} \ No newline at end of file diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/consumer/AbsoluteTimeRangeConfig.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/consumer/AbsoluteTimeRangeConfig.java new file mode 100644 index 000000000..d0460aef2 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/consumer/AbsoluteTimeRangeConfig.java @@ -0,0 +1,78 @@ +package com.linkedin.feathr.core.config.consumer; + +import com.linkedin.feathr.core.config.ConfigObj; +import java.util.Objects; + +/** + * Represents the temporal fields for the absolute time range object. + * + * @author rkashyap + */ +public class AbsoluteTimeRangeConfig implements ConfigObj { + public static final String START_TIME = "startTime"; + public static final String END_TIME = "endTime"; + public static final String TIME_FORMAT = "timeFormat"; + + private final String _startTime; + private final String _endTime; + private final String _timeFormat; + + private String _configStr; + + /** + * Constructor with all parameters + * @param startTime The start time for the observation data + * @param endTime The end time for the observation data + * @param timeFormat The time format in which the times are specified + */ + public AbsoluteTimeRangeConfig(String startTime, String endTime, String timeFormat) { + _startTime = startTime; + _endTime = endTime; + _timeFormat = timeFormat; + + constructConfigStr(); + } + + private void constructConfigStr() { + StringBuilder sb = new StringBuilder(); + sb.append(START_TIME).append(": ").append(_startTime).append("\n") + .append(END_TIME).append(": ").append(_endTime).append("\n") + .append(TIME_FORMAT).append(": ").append(_timeFormat).append("\n"); + _configStr = sb.toString(); + } + + @Override + public String toString() { + return _configStr; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof AbsoluteTimeRangeConfig)) { + return false; + } + AbsoluteTimeRangeConfig that = (AbsoluteTimeRangeConfig) o; + return Objects.equals(_startTime, that._startTime) && Objects.equals(_endTime, that._endTime) + && Objects.equals(_timeFormat, that._timeFormat); + } + + @Override + public int hashCode() { + return Objects.hash(_startTime, _endTime, _timeFormat); + } + + public String getStartTime() { + return _startTime; + } + + public String getEndTime() { + return _endTime; + } + + public String getTimeFormat() { + return _timeFormat; + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/consumer/DateTimeRange.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/consumer/DateTimeRange.java new file mode 100644 index 000000000..f47dd41a1 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/consumer/DateTimeRange.java @@ -0,0 +1,71 @@ +package com.linkedin.feathr.core.config.consumer; + +import java.time.LocalDateTime; +import java.util.Objects; + + +/** + * Represents the start and end local date-times without regards to timezone in the ISO-8601 calendar system. + * + * @author djaising + * @author cesun + */ +public final class DateTimeRange { + public static final String START_TIME = "start_time"; + public static final String END_TIME = "end_time"; + + private final LocalDateTime _start; + private final LocalDateTime _end; + + private String _configStr; + + /** + * Constructor + * @param start The start date-time + * @param end The end date-time + */ + public DateTimeRange(LocalDateTime start, LocalDateTime end) { + _start = start; + _end = end; + + constructConfigStr(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof DateTimeRange)) { + return false; + } + DateTimeRange that = (DateTimeRange) o; + return Objects.equals(_start, that._start) && Objects.equals(_end, that._end); + } + + + private void constructConfigStr() { + StringBuilder sb = new StringBuilder(); + sb.append(START_TIME).append(": ").append(_start).append("\n") + .append(END_TIME).append(": ").append(_end).append("\n"); + _configStr = sb.toString(); + } + + @Override + public String toString() { + return _configStr; + } + + @Override + public int hashCode() { + return Objects.hash(_start, _end); + } + + public LocalDateTime getStart() { + return _start; + } + + public LocalDateTime getEnd() { + return _end; + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/consumer/FeatureBagConfig.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/consumer/FeatureBagConfig.java new file mode 100644 index 000000000..6747a885f --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/consumer/FeatureBagConfig.java @@ -0,0 +1,55 @@ +package com.linkedin.feathr.core.config.consumer; + +import com.linkedin.feathr.core.config.ConfigObj; +import com.linkedin.feathr.core.utils.Utils; +import java.util.List; +import java.util.Objects; + + +/** + * Represents list of configs for features + */ +public final class FeatureBagConfig implements ConfigObj { + private final List _keyedFeatures; + + private String _configStr; + + /** + * Constructor + * @param keyedFeatures + */ + public FeatureBagConfig(List keyedFeatures) { + Utils.require(!keyedFeatures.isEmpty(), "List of features to be joined can't be empty"); + _keyedFeatures = keyedFeatures; + + StringBuilder sb = new StringBuilder(); + sb.append(Utils.string(keyedFeatures, "\n")).append("\n"); + _configStr = sb.toString(); + } + + @Override + public String toString() { + return _configStr; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof FeatureBagConfig)) { + return false; + } + FeatureBagConfig that = (FeatureBagConfig) o; + return Objects.equals(_keyedFeatures, that._keyedFeatures); + } + + @Override + public int hashCode() { + return Objects.hash(_keyedFeatures); + } + + public List getKeyedFeatures() { + return _keyedFeatures; + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/consumer/JoinConfig.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/consumer/JoinConfig.java new file mode 100644 index 000000000..9008e5917 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/consumer/JoinConfig.java @@ -0,0 +1,77 @@ +package com.linkedin.feathr.core.config.consumer; + +import com.linkedin.feathr.core.config.ConfigObj; +import com.linkedin.feathr.core.utils.Utils; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; + +/** + * Represents the Join Config which specifies the join plan, and is provided by a feature consumer. + * + * @author djaising + * @author cesun + */ +public class JoinConfig implements ConfigObj { + /* + * Represents the fields used in the Join Config file + */ + public static final String SETTINGS = "settings"; + + private final Optional _settings; + private final Map _featureBagConfigs; + + private String _configStr; + + /** + * Constructor with all parameters + * @param settings {@link SettingsConfig} object + * @param featureBagConfigs The {@link FeatureBagConfig} object that specifies the featureBagConfigs to be fetched and the keys in the observation data + */ + public JoinConfig(SettingsConfig settings, Map featureBagConfigs) { + _settings = Optional.ofNullable(settings); + _featureBagConfigs = featureBagConfigs; + constructConfigStr(); + } + + private void constructConfigStr() { + StringBuilder sb = new StringBuilder(); + _settings.ifPresent(s -> sb.append(SETTINGS).append(": ").append(s).append("\n")); + sb.append(Utils.string(_featureBagConfigs, "\n")).append("\n"); + _configStr = sb.toString(); + } + + public Optional getSettings() { + return _settings; + } + + public Map getFeatureBagConfigs() { + return _featureBagConfigs; + } + + public Optional getFeatureBagConfig(String featureBagName) { + return Optional.ofNullable(_featureBagConfigs.get(featureBagName)); + } + + @Override + public String toString() { + return _configStr; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + JoinConfig that = (JoinConfig) o; + return Objects.equals(_settings, that._settings) && Objects.equals(_featureBagConfigs, that._featureBagConfigs); + } + + @Override + public int hashCode() { + return Objects.hash(_settings, _featureBagConfigs); + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/consumer/JoinTimeSettingsConfig.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/consumer/JoinTimeSettingsConfig.java new file mode 100644 index 000000000..ee360a6b7 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/consumer/JoinTimeSettingsConfig.java @@ -0,0 +1,81 @@ +package com.linkedin.feathr.core.config.consumer; + +import com.linkedin.feathr.core.config.ConfigObj; +import java.time.Duration; +import java.util.Objects; +import java.util.Optional; + +/** + * Represents the temporal fields for the observationDataTimeSettings used for loading of observation data. + * + * @author rkashyap + */ +public class JoinTimeSettingsConfig implements ConfigObj { + + public static final String TIMESTAMP_COLUMN = "timestampColumn"; + public static final String SIMULATE_TIME_DELAY = "simulateTimeDelay"; + public static final String USE_LATEST_FEATURE_DATA = "useLatestFeatureData"; + + private final Optional _timestampColumn; + private final Optional _simulateTimeDelay; + private final Optional _useLatestFeatureData; + + private String _configStr; + + /** + * Constructor with all parameters + * @param timestampColumn The timestamp column and format object. + * @param simulateTimeDelay A Duration value that shifts the observation data to the past thus simulating a delay + * on the observation data. + * @param useLatestFeatureData Boolean to indicate using of latest feature data + */ + public JoinTimeSettingsConfig(TimestampColumnConfig timestampColumn, Duration simulateTimeDelay, Boolean useLatestFeatureData) { + _timestampColumn = Optional.ofNullable(timestampColumn); + _simulateTimeDelay = Optional.ofNullable(simulateTimeDelay); + _useLatestFeatureData = Optional.ofNullable(useLatestFeatureData); + constructConfigStr(); + } + + private void constructConfigStr() { + StringBuilder sb = new StringBuilder(); + _timestampColumn.ifPresent(t -> sb.append(TIMESTAMP_COLUMN).append(": ").append(t).append("\n")); + _simulateTimeDelay.ifPresent(t -> sb.append(SIMULATE_TIME_DELAY).append(": ").append(t).append("\n")); + _useLatestFeatureData.ifPresent(t -> sb.append(USE_LATEST_FEATURE_DATA).append(": ").append(t).append("\n")); + _configStr = sb.toString(); + } + + @Override + public String toString() { + return _configStr; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof JoinTimeSettingsConfig)) { + return false; + } + JoinTimeSettingsConfig that = (JoinTimeSettingsConfig) o; + return Objects.equals(_timestampColumn, that._timestampColumn) && Objects.equals(_simulateTimeDelay, that._simulateTimeDelay) + && Objects.equals(_useLatestFeatureData, that._useLatestFeatureData); + } + + @Override + public int hashCode() { + return Objects.hash(_timestampColumn.hashCode(), _useLatestFeatureData, _simulateTimeDelay); + } + + public Optional getTimestampColumn() { + return _timestampColumn; + } + + public Optional getSimulateTimeDelay() { + return _simulateTimeDelay; + } + + public Optional getUseLatestFeatureData() { + return _useLatestFeatureData; + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/consumer/KeyedFeatures.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/consumer/KeyedFeatures.java new file mode 100644 index 000000000..0ac25088c --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/consumer/KeyedFeatures.java @@ -0,0 +1,102 @@ +package com.linkedin.feathr.core.config.consumer; + +import com.linkedin.feathr.core.utils.Utils; +import java.time.Duration; +import java.util.List; +import java.util.Objects; +import java.util.Optional; + + +/** + * Represents tuple of key (which may be a multi or composite key), and the list of features specific to this key. + * + * @author djaising + * @author cesun + */ +public final class KeyedFeatures { + + /* + * Represents the fields used to specify the key, features, and temporal parameters in the Join Config file. + */ + public static final String KEY = "key"; + public static final String FEATURE_LIST = "featureList"; + public static final String START_DATE = "startDate"; + public static final String END_DATE = "endDate"; + public static final String DATE_OFFSET = "dateOffset"; // TODO: verify field name + public static final String NUM_DAYS = "numDays"; // TODO: verify field name + public static final String OVERRIDE_TIME_DELAY = "overrideTimeDelay"; + + // Not a field but is used to specify the timestamp format + public static final String TIMESTAMP_FORMAT = "yyyyMMdd"; + + private final List _key; + private final List _features; + private final Optional _dates; + private final Optional _overrideTimeDelay; + + private String _configStr; + + /** + * Constructor with all parameters + * @param key If the list contains multiple entries, it specifies a composite key else a single key. + * @param features List of features specific to the key. + * @param dates {@link DateTimeRange} object which delimits the start and end times of the feature records to be + * fetched. + */ + public KeyedFeatures(List key, List features, DateTimeRange dates, Duration overrideTimeDelay) { + _key = key; + _features = features; + _dates = Optional.ofNullable(dates); + _overrideTimeDelay = Optional.ofNullable(overrideTimeDelay); + constructConfigStr(); + } + + private void constructConfigStr() { + StringBuilder sb = new StringBuilder(); + sb.append(KEY).append(": ").append(Utils.string(_key)).append("\n") + .append(FEATURE_LIST).append(": ").append(Utils.string(_features)).append("\n"); + _dates.ifPresent(d -> sb.append(START_DATE).append(": ").append(d.getStart()).append("\n") + .append(END_DATE).append(": ").append(d.getEnd()).append("\n")); + _overrideTimeDelay.ifPresent(d -> sb.append(OVERRIDE_TIME_DELAY).append(": ").append(d).append("\n")); + _configStr = sb.toString(); + } + + @Override + public String toString() { + return _configStr; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof KeyedFeatures)) { + return false; + } + KeyedFeatures that = (KeyedFeatures) o; + return Objects.equals(_key, that._key) && Objects.equals(_features, that._features) && Objects.equals(_dates, + that._dates) && Objects.equals(_overrideTimeDelay, that._overrideTimeDelay); + } + + @Override + public int hashCode() { + return Objects.hash(_key, _features, _dates, _overrideTimeDelay); + } + + public List getKey() { + return _key; + } + + public List getFeatures() { + return _features; + } + + public Optional getDates() { + return _dates; + } + + public Optional getOverrideTimeDelay() { + return _overrideTimeDelay; } + +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/consumer/ObservationDataTimeSettingsConfig.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/consumer/ObservationDataTimeSettingsConfig.java new file mode 100644 index 000000000..6d6575134 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/consumer/ObservationDataTimeSettingsConfig.java @@ -0,0 +1,75 @@ +package com.linkedin.feathr.core.config.consumer; + +import com.linkedin.feathr.core.config.ConfigObj; +import java.util.Objects; +import java.util.Optional; + + +/** + * Represents temporal parameters used in observationDataTimeSettings. + * + * @author rkashyap + */ +public class ObservationDataTimeSettingsConfig implements ConfigObj { + + public static final String ABSOLUTE_TIME_RANGE = "absoluteTimeRange"; + public static final String RELATIVE_TIME_RANGE = "relativeTimeRange"; + + private final Optional _absoluteTimeRangeConfig; + private final Optional _relativeTimeRangeConfig; + + private String _configStr; + + /** + * Constructor with all parameters + * @param absoluteTimeRangeConfig The observation data's absolute time range + * @param relativeTimeRangeConfig The observation data's relative time range + */ + public ObservationDataTimeSettingsConfig(AbsoluteTimeRangeConfig absoluteTimeRangeConfig, + RelativeTimeRangeConfig relativeTimeRangeConfig) { + _absoluteTimeRangeConfig = Optional.ofNullable(absoluteTimeRangeConfig); + _relativeTimeRangeConfig = Optional.ofNullable(relativeTimeRangeConfig); + + constructConfigStr(); + } + + private void constructConfigStr() { + StringBuilder sb = new StringBuilder(); + _absoluteTimeRangeConfig.ifPresent(t -> sb.append(t).append(": ").append(t).append("\n")); + _relativeTimeRangeConfig.ifPresent(t -> sb.append(t).append(": ").append(t).append("\n")); + + _configStr = sb.toString(); + } + + @Override + public String toString() { + return _configStr; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof ObservationDataTimeSettingsConfig)) { + return false; + } + ObservationDataTimeSettingsConfig that = (ObservationDataTimeSettingsConfig) o; + return Objects.equals(_absoluteTimeRangeConfig, that._absoluteTimeRangeConfig) + && Objects.equals(_relativeTimeRangeConfig, that._relativeTimeRangeConfig); + } + + @Override + public int hashCode() { + return Objects.hash(_absoluteTimeRangeConfig, _relativeTimeRangeConfig); + } + + public Optional getAbsoluteTimeRange() { + return _absoluteTimeRangeConfig; + } + + public Optional getRelativeTimeRange() { + return _relativeTimeRangeConfig; + } + +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/consumer/RelativeTimeRangeConfig.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/consumer/RelativeTimeRangeConfig.java new file mode 100644 index 000000000..2040a493d --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/consumer/RelativeTimeRangeConfig.java @@ -0,0 +1,71 @@ +package com.linkedin.feathr.core.config.consumer; + +import com.linkedin.feathr.core.config.ConfigObj; +import java.time.Duration; +import java.util.Objects; +import java.util.Optional; + + +/** + * Represents the temporal fields for the relative time range object. + * + * @author rkashyap + */ +public class RelativeTimeRangeConfig implements ConfigObj { + public static final String WINDOW = "window"; + public static final String OFFSET = "offset"; + + private final Duration _window; + private final Optional _offset; + + private String _configStr; + + /** + * Constructor with all parameters + * @param window number of days/hours from the reference date, reference date = current time - offset + * @param offset number of days/hours to look back relative to the current timestamp + */ + public RelativeTimeRangeConfig(Duration window, Duration offset) { + _window = window; + _offset = Optional.ofNullable(offset); + + constructConfigStr(); + } + + private void constructConfigStr() { + StringBuilder sb = new StringBuilder(); + sb.append(WINDOW).append(": ").append(_window).append("\n"); + _offset.ifPresent(t -> sb.append(OFFSET).append(": ").append(t).append("\n")); + _configStr = sb.toString(); + } + + @Override + public String toString() { + return _configStr; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof RelativeTimeRangeConfig)) { + return false; + } + RelativeTimeRangeConfig that = (RelativeTimeRangeConfig) o; + return Objects.equals(_window, that._window) && Objects.equals(_offset, that._offset); + } + + @Override + public int hashCode() { + return Objects.hash(_window, _offset); + } + + public Duration getWindow() { + return _window; + } + + public Optional getOffset() { + return _offset; + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/consumer/SettingsConfig.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/consumer/SettingsConfig.java new file mode 100644 index 000000000..becd8c5bf --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/consumer/SettingsConfig.java @@ -0,0 +1,73 @@ +package com.linkedin.feathr.core.config.consumer; + +import com.linkedin.feathr.core.config.ConfigObj; +import java.util.Objects; +import java.util.Optional; + +/** + * Represents some 'settings' on the observation data. + * + * @author djaising + * @author cesun + */ +public final class SettingsConfig implements ConfigObj { + /* + * Represents the field used to specify the temporal parameter for sliding window aggregation or time aware join + * in the Join Config file. + */ + public static final String OBSERVATION_DATA_TIME_SETTINGS = "observationDataTimeSettings"; + public static final String JOIN_TIME_SETTINGS = "joinTimeSettings"; + + private final Optional _observationDataTimeSettings; + private final Optional _joinTimeSettings; + + private String _configStr; + + /** + * Constructor with parameter timeWindowJoin and observationTimeInfo + * @param observationDataTimeSettings temporal parameters used to load the observation. + * @param joinTimeSettings temporal parameters used for joining the observation with the feature data. + */ + public SettingsConfig(ObservationDataTimeSettingsConfig observationDataTimeSettings, JoinTimeSettingsConfig joinTimeSettings) { + _observationDataTimeSettings = Optional.ofNullable(observationDataTimeSettings); + _joinTimeSettings = Optional.ofNullable(joinTimeSettings); + constructConfigStr(); + } + + private void constructConfigStr() { + StringBuilder sb = new StringBuilder(); + _observationDataTimeSettings.ifPresent(t -> sb.append(OBSERVATION_DATA_TIME_SETTINGS).append(": ").append(t).append("\n")); + _joinTimeSettings.ifPresent(t -> sb.append(JOIN_TIME_SETTINGS).append(": ").append(t).append("\n")); + _configStr = sb.toString(); + } + + @Override + public String toString() { + return _configStr; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof SettingsConfig)) { + return false; + } + SettingsConfig that = (SettingsConfig) o; + return Objects.equals(_observationDataTimeSettings, that._observationDataTimeSettings) && Objects.equals(_joinTimeSettings, that._joinTimeSettings); + } + + @Override + public int hashCode() { + return Objects.hash(_observationDataTimeSettings, _joinTimeSettings); + } + + public Optional getTimeWindowJoin() { + return _observationDataTimeSettings; + } + + public Optional getObservationTimeInfo() { + return _joinTimeSettings; + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/consumer/TimestampColumnConfig.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/consumer/TimestampColumnConfig.java new file mode 100644 index 000000000..a90e4de88 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/consumer/TimestampColumnConfig.java @@ -0,0 +1,69 @@ +package com.linkedin.feathr.core.config.consumer; +import com.linkedin.feathr.core.config.ConfigObj; +import java.util.Objects; + + +/** + * Represents the timestamp column object + * + * @author rkashyap + */ +public class TimestampColumnConfig implements ConfigObj { + public static final String NAME = "def"; + public static final String FORMAT = "format"; + + private final String _name; + private final String _format; + + private String _configStr; + + /** + * Constructor + * @param name name of the timestamp column + * @param format format of the timestamp column + */ + public TimestampColumnConfig(String name, String format) { + _name = name; + _format = format; + + constructConfigStr(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof TimestampColumnConfig)) { + return false; + } + TimestampColumnConfig that = (TimestampColumnConfig) o; + return Objects.equals(_name, that._name) && Objects.equals(_format, that._format); + } + + + private void constructConfigStr() { + StringBuilder sb = new StringBuilder(); + sb.append(NAME).append(": ").append(_name).append("\n") + .append(FORMAT).append(": ").append(_format).append("\n"); + _configStr = sb.toString(); + } + + @Override + public String toString() { + return _configStr; + } + + @Override + public int hashCode() { + return Objects.hash(_name, _format); + } + + public String getName() { + return _name; + } + + public String getFormat() { + return _format; + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/generation/FeatureGenConfig.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/generation/FeatureGenConfig.java new file mode 100644 index 000000000..f43d8e4ef --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/generation/FeatureGenConfig.java @@ -0,0 +1,81 @@ +package com.linkedin.feathr.core.config.generation; + +import com.linkedin.feathr.core.config.ConfigObj; +import java.util.List; +import java.util.Objects; + + +/** + * Define the feature generation specification, i.e., list of features to generate and other settings. + * We introduce env to differentiate between offline and nearline features. If env is not mentioned, + * it defaults to the offline case, and if we have parameter called env: NEARLINE, it represents a nearline feature. + * env can also be specified as env: OFFLINE. + */ + +public class FeatureGenConfig implements ConfigObj { + private final OperationalConfig _operationalConfig; + private final List _features; + + /** + * Constructor + * @param operationalConfig + * @param features + */ + public FeatureGenConfig(OperationalConfig operationalConfig, List features) { + _operationalConfig = operationalConfig; + _features = features; + } + + /* + * The previously used lombok library auto generates getters with underscore, which is used in production. + * For backward compatibility, we need to keep these getters. + * However, function name with underscore can not pass LinkedIn's style check, here we need suppress the style check + * for the getters only. + * + * For more detail, please refer to the style check wiki: + * https://iwww.corp.linkedin.com/wiki/cf/display/TOOLS/Checking+Java+Coding+Style+with+Gradle+Checkstyle+Plugin + * + * TODO - 7493) remove the ill-named getters + */ + // CHECKSTYLE:OFF + @Deprecated + public OperationalConfig get_operationalConfig() { + return _operationalConfig; + } + + @Deprecated + public List get_features() { + return _features; + } + // CHECKSTYLE:ON + + public OperationalConfig getOperationalConfig() { + return _operationalConfig; + } + + public List getFeatures() { + return _features; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof FeatureGenConfig)) { + return false; + } + FeatureGenConfig that = (FeatureGenConfig) o; + return Objects.equals(_operationalConfig, that._operationalConfig) && Objects.equals(_features, that._features); + } + + @Override + public int hashCode() { + return Objects.hash(_operationalConfig, _features); + } + + @Override + public String toString() { + return "FeatureGenConfig{" + "_operationalConfig=" + _operationalConfig + ", _features=" + _features + '}'; + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/generation/NearlineOperationalConfig.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/generation/NearlineOperationalConfig.java new file mode 100644 index 000000000..6b571dfda --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/generation/NearlineOperationalConfig.java @@ -0,0 +1,16 @@ +package com.linkedin.feathr.core.config.generation; + +import java.util.List; + +/* + * Nearline Operational config currently has all the fields as Operational config. + * + * In nearline, we dont have time based configs like timeSetting, retention, simlateTimeDelay, enableIncremental. + * We only have name, outputProcessorsListConfig. + */ +public class NearlineOperationalConfig extends OperationalConfig { + + public NearlineOperationalConfig(List outputProcessorsListConfig, String name) { + super(outputProcessorsListConfig, name); + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/generation/OfflineOperationalConfig.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/generation/OfflineOperationalConfig.java new file mode 100644 index 000000000..3003ea395 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/generation/OfflineOperationalConfig.java @@ -0,0 +1,107 @@ +package com.linkedin.feathr.core.config.generation; + +import com.linkedin.feathr.core.config.common.DateTimeConfig; +import java.time.Duration; +import java.util.List; +import java.util.Objects; + + +/** + * Operational section in feature generation config + * + * Feature generation config contains two major sections, i.e., operational and feature list sections, + * feature list specify the features to generate, + * operational section contains all the related settings. + */ +public class OfflineOperationalConfig extends OperationalConfig { + private final DateTimeConfig _timeSetting; + private final Duration _retention; + private final Duration _simulateTimeDelay; + private final Boolean _enableIncremental; + + public OfflineOperationalConfig(List outputProcessorsListConfig, String name, DateTimeConfig timeSetting, + Duration retention, Duration simulateTimeDelay, Boolean enableIncremental) { + super(outputProcessorsListConfig, name); + _timeSetting = timeSetting; + _retention = retention; + _simulateTimeDelay = simulateTimeDelay; + _enableIncremental = enableIncremental; + } + + /* + * The previously used lombok library auto generates getters with underscore, which is used in production. + * For backward compatibility, we need to keep these getters. + * However, function name with underscore can not pass LinkedIn's style check, here we need suppress the style check + * for the getters only. + * + * For more detail, please refer to the style check wiki: + * https://iwww.corp.linkedin.com/wiki/cf/display/TOOLS/Checking+Java+Coding+Style+with+Gradle+Checkstyle+Plugin + * + * TODO - 7493) remove the ill-named getters + */ + // CHECKSTYLE:OFF + @Deprecated + public DateTimeConfig get_timeSetting() { + return _timeSetting; + } + + @Deprecated + public Duration get_retention() { + return _retention; + } + + @Deprecated + public Duration get_simulateTimeDelay() { + return _simulateTimeDelay; + } + + @Deprecated + public Boolean get_enableIncremental() { + return _enableIncremental; + } + // CHECKSTYLE:ON + + public DateTimeConfig getTimeSetting() { + return _timeSetting; + } + + public Duration getRetention() { + return _retention; + } + + public Duration getSimulateTimeDelay() { + return _simulateTimeDelay; + } + + public Boolean getEnableIncremental() { + return _enableIncremental; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof OfflineOperationalConfig)) { + return false; + } + if (!super.equals(o)) { + return false; + } + OfflineOperationalConfig that = (OfflineOperationalConfig) o; + return Objects.equals(_timeSetting, that._timeSetting) && Objects.equals(_retention, that._retention) + && Objects.equals(_simulateTimeDelay, that._simulateTimeDelay) && Objects.equals(_enableIncremental, + that._enableIncremental); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), _timeSetting, _retention, _simulateTimeDelay, _enableIncremental); + } + + @Override + public String toString() { + return "OfflineOperationalConfig{" + "_timeSetting=" + _timeSetting + ", _retention=" + _retention + + ", _simulateTimeDelay=" + _simulateTimeDelay + ", _enableIncremental=" + _enableIncremental + '}'; + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/generation/OperationalConfig.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/generation/OperationalConfig.java new file mode 100644 index 000000000..beadfcdae --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/generation/OperationalConfig.java @@ -0,0 +1,76 @@ +package com.linkedin.feathr.core.config.generation; + +import com.linkedin.feathr.core.config.ConfigObj; +import java.util.List; +import java.util.Objects; + + +/** + * Operational section in feature generation config + * + * This abstract class is extended by offline and nearline Operational Config. + */ +public abstract class OperationalConfig implements ConfigObj { + private final List _outputProcessorsListConfig; + private final String _name; + + public OperationalConfig(List outputProcessorsListConfig, String name) { + _outputProcessorsListConfig = outputProcessorsListConfig; + _name = name; + } + + /* + * The previously used lombok library auto generates getters with underscore, which is used in production. + * For backward compatibility, we need to keep these getters. + * However, function name with underscore can not pass LinkedIn's style check, here we need suppress the style check + * for the getters only. + * + * For more detail, please refer to the style check wiki: + * https://iwww.corp.linkedin.com/wiki/cf/display/TOOLS/Checking+Java+Coding+Style+with+Gradle+Checkstyle+Plugin + * + * TODO - 7493) remove the ill-named getters + */ + // CHECKSTYLE:OFF + @Deprecated + public List get_outputProcessorsListConfig() { + return _outputProcessorsListConfig; + } + + @Deprecated + public String get_name() { + return _name; + } + // CHECKSTYLE:ON + + public List getOutputProcessorsListConfig() { + return _outputProcessorsListConfig; + } + + public String getName() { + return _name; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof OperationalConfig)) { + return false; + } + OperationalConfig that = (OperationalConfig) o; + return Objects.equals(_outputProcessorsListConfig, that._outputProcessorsListConfig) && Objects.equals(_name, + that._name); + } + + @Override + public int hashCode() { + return Objects.hash(_outputProcessorsListConfig, _name); + } + + @Override + public String toString() { + return "OperationalConfig{" + "_outputProcessorsListConfig=" + _outputProcessorsListConfig + ", _name='" + _name + + '\'' + '}'; + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/generation/OutputProcessorConfig.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/generation/OutputProcessorConfig.java new file mode 100644 index 000000000..c9c8023b0 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/generation/OutputProcessorConfig.java @@ -0,0 +1,93 @@ +package com.linkedin.feathr.core.config.generation; + +import com.linkedin.feathr.core.config.ConfigObj; +import com.linkedin.feathr.core.config.common.OutputFormat; +import com.typesafe.config.Config; +import java.util.Objects; + + +/** + * Output processor config, e.g., write to HDFS processor or push to Venice processor + */ +public class OutputProcessorConfig implements ConfigObj { + private final String _name; + private final OutputFormat _outputFormat; + // other params, e.g, venice params or hdfs specific parameters + private final Config _params; + + /** + * Constructor + * @param name + * @param outputFormat + * @param params + */ + public OutputProcessorConfig(String name, OutputFormat outputFormat, Config params) { + _name = name; + _outputFormat = outputFormat; + _params = params; + } + + /* + * The previously used lombok library auto generates getters with underscore, which is used in production. + * For backward compatibility, we need to keep these getters. + * However, function name with underscore can not pass LinkedIn's style check, here we need suppress the style check + * for the getters only. + * + * For more detail, please refer to the style check wiki: + * https://iwww.corp.linkedin.com/wiki/cf/display/TOOLS/Checking+Java+Coding+Style+with+Gradle+Checkstyle+Plugin + * + * TODO - 7493) remove the ill-named getters + */ + // CHECKSTYLE:OFF + @Deprecated + public String get_name() { + return _name; + } + + @Deprecated + public OutputFormat get_outputFormat() { + return _outputFormat; + } + + @Deprecated + public Config get_params() { + return _params; + } + // CHECKSTYLE:ON + + public String getName() { + return _name; + } + + public OutputFormat getOutputFormat() { + return _outputFormat; + } + + public Config getParams() { + return _params; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof OutputProcessorConfig)) { + return false; + } + OutputProcessorConfig that = (OutputProcessorConfig) o; + return Objects.equals(_name, that._name) && _outputFormat == that._outputFormat && Objects.equals(_params, + that._params); + } + + @Override + public int hashCode() { + return Objects.hash(_name, _outputFormat, _params); + } + + @Override + public String toString() { + return "OutputProcessorConfig{" + "_name='" + _name + '\'' + ", _outputFormat=" + _outputFormat + ", _params=" + + _params + '}'; + } +} \ No newline at end of file diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/ExprType.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/ExprType.java new file mode 100644 index 000000000..e27006525 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/ExprType.java @@ -0,0 +1,9 @@ +package com.linkedin.feathr.core.config.producer; + +/** + * Enumeration class for key and feature expression type defined in FeatureDef + */ +public enum ExprType { + MVEL, + SQL +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/FeatureDefConfig.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/FeatureDefConfig.java new file mode 100644 index 000000000..d5c4a3841 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/FeatureDefConfig.java @@ -0,0 +1,90 @@ +package com.linkedin.feathr.core.config.producer; + +import com.linkedin.feathr.core.config.ConfigObj; +import com.linkedin.feathr.core.config.producer.derivations.DerivationsConfig; +import com.linkedin.feathr.core.config.producer.anchors.AnchorsConfig; +import com.linkedin.feathr.core.config.producer.sources.SourcesConfig; +import java.util.Objects; +import java.util.Optional; + + +/** + * Represents the FeatureDef configuration + * + * @author djaising + * @author cesun + */ +public final class FeatureDefConfig implements ConfigObj { + /* + * Fields used to specify each of the six sections in a FeatureDef config + */ + public static final String SOURCES = "sources"; + public static final String ANCHORS = "anchors"; + public static final String DERIVATIONS = "derivations"; + public static final String FEATURES = "features"; + + private final Optional _sourcesConfig; + private final Optional _anchorsConfig; + private final Optional _derivationsConfig; + + private String _configStr; + + /** + * Constructor with full parameters + * @param sourcesConfig {@link SourcesConfig} + * @param anchorsConfig {@link AnchorsConfig} + * @param derivationsConfig {@link DerivationsConfig} + */ + public FeatureDefConfig(SourcesConfig sourcesConfig, + AnchorsConfig anchorsConfig, DerivationsConfig derivationsConfig) { + _sourcesConfig = Optional.ofNullable(sourcesConfig); + _anchorsConfig = Optional.ofNullable(anchorsConfig); + _derivationsConfig = Optional.ofNullable(derivationsConfig); + + constructConfigStr(); + } + + private void constructConfigStr() { + StringBuilder strBldr = new StringBuilder(); + _sourcesConfig.ifPresent(cfg -> strBldr.append(SOURCES).append(": ").append(cfg).append("\n")); + _anchorsConfig.ifPresent(cfg -> strBldr.append(ANCHORS).append(": ").append(cfg).append("\n")); + _derivationsConfig.ifPresent(cfg -> strBldr.append(DERIVATIONS).append(": ").append(cfg).append("\n")); + _configStr = strBldr.toString(); + } + + public Optional getSourcesConfig() { + return _sourcesConfig; + } + + public Optional getAnchorsConfig() { + return _anchorsConfig; + } + + public Optional getDerivationsConfig() { + return _derivationsConfig; + } + + @Override + public String toString() { + return _configStr; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + FeatureDefConfig that = (FeatureDefConfig) o; + return Objects.equals(_sourcesConfig, that._sourcesConfig) + && Objects.equals(_anchorsConfig, that._anchorsConfig) && Objects.equals(_derivationsConfig, + that._derivationsConfig); + } + + @Override + public int hashCode() { + return Objects.hash(_sourcesConfig, _anchorsConfig, _derivationsConfig); + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/TypedExpr.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/TypedExpr.java new file mode 100644 index 000000000..666b0444b --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/TypedExpr.java @@ -0,0 +1,53 @@ +package com.linkedin.feathr.core.config.producer; + +import java.util.Objects; + + +/** + * expression with {@link ExprType} type + */ +public class TypedExpr { + private final String _expr; + private final ExprType _exprType; + private String _configStr; + + public TypedExpr(String expr, ExprType exprType) { + _expr = expr; + _exprType = exprType; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof TypedExpr)) { + return false; + } + TypedExpr typedExpr = (TypedExpr) o; + return Objects.equals(_expr, typedExpr._expr) && _exprType == typedExpr._exprType; + } + + @Override + public int hashCode() { + return Objects.hash(_expr, _exprType); + } + + public String getExpr() { + return _expr; + } + + public ExprType getExprType() { + return _exprType; + } + + @Override + public String toString() { + if (_configStr == null) { + _configStr = String.join("\n", + String.join(": ", "expression", _expr), + String.join(": ", "expression type", _exprType.toString())); + } + return _configStr; + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/anchors/AnchorConfig.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/anchors/AnchorConfig.java new file mode 100644 index 000000000..a070f18d9 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/anchors/AnchorConfig.java @@ -0,0 +1,62 @@ +package com.linkedin.feathr.core.config.producer.anchors; + +import com.linkedin.feathr.core.config.ConfigObj; +import java.util.Map; +import java.util.Objects; + + +/** + * Represents the general anchor definition + */ +public abstract class AnchorConfig implements ConfigObj { + + private final String _source; + private final Map _features; + + public static final String SOURCE = "source"; + public static final String KEY = "key"; + public static final String KEY_ALIAS = "keyAlias"; + public static final String KEY_MVEL = "key.mvel"; + public static final String KEY_SQL_EXPR = "key.sqlExpr"; + public static final String KEY_EXTRACTOR = "keyExtractor"; + public static final String EXTRACTOR = "extractor"; + public static final String TRANSFORMER = "transformer"; // TODO: field is deprecated. Remove once client featureDef configs modified. + public static final String LATERAL_VIEW_PARAMS = "lateralViewParameters"; + public static final String FEATURES = "features"; + + /** + * Constructor + * @param source source definition + * @param features map of feature name to {@link FeatureConfig} object + */ + protected AnchorConfig(String source, Map features) { + _source = source; + _features = features; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof AnchorConfig)) { + return false; + } + AnchorConfig that = (AnchorConfig) o; + return Objects.equals(_source, that._source) && Objects.equals(_features, that._features); + } + + @Override + public int hashCode() { + return Objects.hash(_source, _features); + } + + public String getSource() { + return _source; + } + + public Map getFeatures() { + return _features; + } +} + diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/anchors/AnchorConfigWithExtractor.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/anchors/AnchorConfigWithExtractor.java new file mode 100644 index 000000000..eff114cf8 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/anchors/AnchorConfigWithExtractor.java @@ -0,0 +1,176 @@ +package com.linkedin.feathr.core.config.producer.anchors; + +import com.linkedin.feathr.core.utils.Utils; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.StringJoiner; + + +/** + * Represents the anchor definition (the object part) for the anchors that have the extractor specified (in lieu of the + * key). + * The features may be specified in two ways as shown below, + * where the keyExtractor and (keyAlias and/or key) fields are mutually exclusive. + * If using keyAlias or keys, the extractor can only be of AnchorExtractor type. + * If using keyExtractor, the extractor can only be of SimpleAnchorExtractorSpark or GenericAnchorExtractorSpark. + * 
+ *{@code
+ * : {
+ *   source: 
+ *   keyExtractor: 
+ *   extractor: 
+ *   features: {
+ *      : {
+ *       default: 
+ *     },
+ *     : {
+ *       default: 
+ *     },
+ *     ...
+ *   }
+ * }
+ *}
+ *
+ * + * A concise format when there is no default value defined for each feature on this anchor + * 
+ * {@code
+ * : {
+ *   source: 
+ *   keyExtractor: 
+ *   extractor: 
+ *   features: [
+ *     ,
+ *     ,
+ *     ...
+ *   ]
+ * }
+ *}
+ *
+ * + * One example of using keyAlias + * 
+ * {@code
+ * : {
+ *   source: 
+ *   key: 
+ *   keyAlias: 
+ *   extractor: 
+ *   features: [
+ *     ,
+ *     ,
+ *     ...
+ *   ]
+ * }
+ *}
+ *
+ * + * @author djaising + * @author cesun + */ +public class AnchorConfigWithExtractor extends AnchorConfig { + private final Optional _keyExtractor; + private final Optional> _keyAlias; + private final Optional _typedKey; + private final String _extractor; + private String _configStr; + + /** + * Constructor + * @param source Source name (defined in sources section) or HDFS/Dali path + * @param keyExtractor name of Java class that is used to extract the key(s) + * @param typedKey the {@link TypedKey} object + * @param keyAlias list of key alias + * @param extractor Name of Java class that is used to extract the feature(s) + * @param features Map of feature names to {@link FeatureConfig} object + */ + public AnchorConfigWithExtractor(String source, String keyExtractor, TypedKey typedKey, + List keyAlias, String extractor, Map features) { + super(source, features); + _keyExtractor = Optional.ofNullable(keyExtractor); + _keyAlias = Optional.ofNullable(keyAlias); + _typedKey = Optional.ofNullable(typedKey); + _extractor = extractor; + } + + /** + * Constructor + * @param source Source name (defined in sources section) or HDFS/Dali path + * @param keyExtractor name of Java class that is used to extract the key(s) + * @param extractor Name of Java class that is used to extract the feature(s) + * @param features Map of feature names to {@link FeatureConfig} object + */ + public AnchorConfigWithExtractor(String source, String keyExtractor, String extractor, + Map features) { + this(source, keyExtractor, null, null, extractor, features); + } + /** + * Constructor + * @param source Source name (defined in sources section) or HDFS/Dali path + * @param extractor Name of Java class that is used to extract the feature(s) + * @param features Map of feature names to {@link FeatureConfig} object + */ + public AnchorConfigWithExtractor(String source, String extractor, Map features) { + this(source, null, null, null, extractor, features); + } + + public Optional getKeyExtractor() { + return _keyExtractor; + } + + public Optional> getKeyAlias() { + return _keyAlias; + } + + public Optional getTypedKey() { + return _typedKey; + } + + public String getExtractor() { + return _extractor; + } + + @Override + public String toString() { + if (_configStr == null) { + StringJoiner stringJoiner = new StringJoiner("\n"); + + stringJoiner.add(String.join(": ", SOURCE, getSource())) + .add(String.join(": ", EXTRACTOR, getExtractor())) + .add(FEATURES + ":{\n" + Utils.string(getFeatures()) + "\n}"); + + _keyExtractor.ifPresent(ke -> stringJoiner.add(String.join(": ", KEY_EXTRACTOR, ke))); + _keyAlias.ifPresent(ka -> stringJoiner.add(String.join(": ", KEY_ALIAS, Utils.string(ka)))); + _typedKey.ifPresent(tk -> stringJoiner.add(_typedKey.toString())); + + _configStr = stringJoiner.toString(); + } + + return _configStr; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof AnchorConfigWithExtractor)) { + return false; + } + if (!super.equals(o)) { + return false; + } + AnchorConfigWithExtractor that = (AnchorConfigWithExtractor) o; + return Objects.equals(_extractor, that._extractor) + && Objects.equals(_keyAlias, that._keyAlias) + && Objects.equals(_typedKey, that._typedKey) + && Objects.equals(_keyExtractor, that._keyExtractor); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), _extractor, _keyAlias, _typedKey, _keyExtractor); + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/anchors/AnchorConfigWithKey.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/anchors/AnchorConfigWithKey.java new file mode 100644 index 000000000..9001d35e6 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/anchors/AnchorConfigWithKey.java @@ -0,0 +1,183 @@ +package com.linkedin.feathr.core.config.producer.anchors; + +import com.linkedin.feathr.core.utils.Utils; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; + + +/** + * Represents the anchor definition (the object part) for the anchors that have the key specified. + * The anchors may be specified in the following ways: + * + * In the following, the fields {@code type} and {@code default} are optional. + * + * 
+ * {@code
+ * : {
+ *   source: 
+ *   key: 
+ *   keyAlias: 
+ *   features: {
+ *     : {
+ *       def: ,
+ *       type: ,
+ *       default: 
+ *     }
+ *     ...
+ *   }
+ * }
+ *
+ * : {
+ *   source: 
+ *   key: 
+ *   keyAlias: 
+ *   features: {
+ *     : ,
+ *     ...
+ *   }
+ * }
+ * }
+ *
+ * + * + * In the following, the fields {@code key.sqlExpr} and {@code def.sqlExpr} should be used simultaneously. + * The fields {@code type} and {@code default} are optional. + * + * 
+ * {@code
+ * : {
+ *   source: 
+ *   key.sqlExpr: 
+ *   keyAlias: 
+ *   features: {
+ *     : {
+ *       def.sqlExpr: ,
+ *       type: ,
+ *       default: 
+ *     }
+ *     ...
+ *   }
+ * }
+ * }
+ *
+ * + * In the following, the fields 'lateralViewParameters', 'filter', 'groupBy' and 'limit' are optional. + * Further, within 'lateralViewParameters', 'lateralViewFilter' is optional as well. + * 
+ * {@code
+ * : {
+ *    source: 
+ *    key: 
+ *    keyAlias: 
+ *    lateralViewParameters: {
+ *      lateralViewDef: 
+ *      lateralViewItemAlias: 
+ *      lateralViewFilter: 
+ *    }
+ *    features: {
+ *      : {
+ *        def: 
+ *        aggregation: 
+ *        window: 
+ *        filter: 
+ *        groupBy: 
+ *        limit: 
+ *      }
+ *    }
+ * }
+ * }
+ *
+ */ +public final class AnchorConfigWithKey extends AnchorConfig { + private final TypedKey _typedKey; + private final Optional> _keyAlias; + private final Optional _lateralViewParams; + private String _configStr; + + /** + * Constructor + * @param source source name (defined in sources section) or HDFS/Dali path + * @param typedKey the {@link TypedKey} object + * @param keyAlias the list of key alias + * @param lateralViewParams {@link LateralViewParams} object + * @param features Map of feature names to {@link FeatureConfig} + */ + public AnchorConfigWithKey(String source, TypedKey typedKey, List keyAlias, + LateralViewParams lateralViewParams, Map features) { + super(source, features); + _typedKey = typedKey; + _keyAlias = Optional.ofNullable(keyAlias); + _lateralViewParams = Optional.ofNullable(lateralViewParams); + } + + /** + * Constructor + * @param source source name (defined in sources section) or HDFS/Dali path + * @param typedKey the {@link TypedKey} object + * @param lateralViewParams {@link LateralViewParams} object + * @param features Map of feature names to {@link FeatureConfig} + */ + public AnchorConfigWithKey(String source, TypedKey typedKey, LateralViewParams lateralViewParams, + Map features) { + this(source, typedKey, null, lateralViewParams, features); + } + + public List getKey() { + return _typedKey.getKey(); + } + + public TypedKey getTypedKey() { + return _typedKey; + } + + public Optional> getKeyAlias() { + return _keyAlias; + } + + public Optional getLateralViewParams() { + return _lateralViewParams; + } + + @Override + public String toString() { + if (_configStr == null) { + _configStr = String.join("\n", + String.join(": ", SOURCE, getSource()), + _typedKey.toString(), + FEATURES + ":{\n" + Utils.string(getFeatures()) + "\n}"); + + _keyAlias.ifPresent(ka -> _configStr = String.join("\n", _configStr, + String.join(": ", KEY_ALIAS, Utils.string(ka)))); + + _lateralViewParams.ifPresent(lvp -> _configStr = String.join("\n", _configStr, + LATERAL_VIEW_PARAMS + ": {\n" + lvp + "\n}")); + } + + return _configStr; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + AnchorConfigWithKey that = (AnchorConfigWithKey) o; + + return Objects.equals(_typedKey, that._typedKey) + && Objects.equals(_keyAlias, that._keyAlias) + && Objects.equals(_lateralViewParams, that._lateralViewParams); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), _typedKey, _keyAlias, _lateralViewParams); + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/anchors/AnchorConfigWithKeyExtractor.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/anchors/AnchorConfigWithKeyExtractor.java new file mode 100644 index 000000000..1b78e725a --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/anchors/AnchorConfigWithKeyExtractor.java @@ -0,0 +1,136 @@ +package com.linkedin.feathr.core.config.producer.anchors; + +import com.linkedin.feathr.core.utils.Utils; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; + + +/** + * Represents the anchor definition (the object part) for the anchors that have ONLY keyExtractor specified. + * It is mutually exclusive with {@link AnchorConfigWithExtractor} + * The anchors may be specified in the following ways: + * + * In the following, the fields {@code lateralViewParameters}, {@code type}, and {@code default} are optional. + * + * 
+ * {@code
+ * : {
+ *   source: 
+ *   keyExtractor: 
+ *    lateralViewParameters: {
+ *      lateralViewDef: 
+ *      lateralViewItemAlias: 
+ *      lateralViewFilter: 
+ *    }
+ *   features: {
+ *     : {
+ *       def: ,
+ *       type: ,
+ *       default: 
+ *     }
+ *     ...
+ *   }
+ * }
+ *
+ * : {
+ *   source: 
+ *   keyExtractor: 
+ *   features: {
+ *     : ,
+ *     ...
+ *   }
+ * }
+ * }
+ *
+ * + * + * 
+ * {@code
+ * : {
+ *   source: 
+ *   keyExtractor: 
+ *   features: {
+ *     : {
+ *       def.sqlExpr: ,
+ *       type: ,
+ *       default: 
+ *     }
+ *     ...
+ *   }
+ * }
+ * }
+ *
+ * + */ +public final class AnchorConfigWithKeyExtractor extends AnchorConfig { + private final String _keyExtractor; + private final Optional _lateralViewParams; + private String _configStr; + + /** + * Constructor + * @param source source name (defined in sources section) or HDFS/Dali path + * @param keyExtractor entity id + * @param features Map of feature names to {@link FeatureConfig} + * @param lateralViewParams {@link LateralViewParams} object + */ + public AnchorConfigWithKeyExtractor(String source, String keyExtractor, Map features, LateralViewParams lateralViewParams) { + super(source, features); + _keyExtractor = keyExtractor; + _lateralViewParams = Optional.ofNullable(lateralViewParams); + } + + /** + * Constructor + * @param source source name (defined in sources section) or HDFS/Dali path + * @param keyExtractor entity id + * @param features Map of feature names to {@link FeatureConfig} + */ + public AnchorConfigWithKeyExtractor(String source, String keyExtractor, Map features) { + this(source, keyExtractor, features, null); + } + + public String getKeyExtractor() { + return _keyExtractor; + } + + public Optional getLateralViewParams() { + return _lateralViewParams; + } + + @Override + public String toString() { + if (_configStr == null) { + _configStr = String.join("\n", + String.join(": ", SOURCE, getSource()), + String.join(": ", KEY_EXTRACTOR, getKeyExtractor()), + FEATURES + ":{\n" + Utils.string(getFeatures()) + "\n}"); + + _lateralViewParams.ifPresent(lvp -> _configStr = String.join("\n", _configStr, + LATERAL_VIEW_PARAMS + ": {\n" + lvp + "\n}")); + } + + return _configStr; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + AnchorConfigWithKeyExtractor that = (AnchorConfigWithKeyExtractor) o; + return Objects.equals(_keyExtractor, that._keyExtractor) && Objects.equals(_lateralViewParams, that._lateralViewParams); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), _keyExtractor, _lateralViewParams); + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/anchors/AnchorConfigWithOnlyMvel.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/anchors/AnchorConfigWithOnlyMvel.java new file mode 100644 index 000000000..acf330e91 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/anchors/AnchorConfigWithOnlyMvel.java @@ -0,0 +1,37 @@ +package com.linkedin.feathr.core.config.producer.anchors; + +import com.linkedin.feathr.core.utils.Utils; +import java.util.Map; + + +/** + * Represents the anchor definition (the object part) for the anchors that have neither the key nor the extractor + * specified. + * + * @author djaising + * @author cesun + */ +// TODO: This seems to be valid only for online anchors. Verify. +public class AnchorConfigWithOnlyMvel extends AnchorConfig { + + private String _configStr; + + /** + * Constructor + * @param source Source name as defined in the sources section + * @param features Map of feature names to {@link FeatureConfig} + */ + public AnchorConfigWithOnlyMvel(String source, Map features) { + super(source, features); + + StringBuilder sb = new StringBuilder(); + sb.append(SOURCE).append(": ").append(source).append("\n") + .append(FEATURES).append(": ").append(Utils.string(features)).append("\n"); + _configStr = sb.toString(); + } + + @Override + public String toString() { + return _configStr; + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/anchors/AnchorsConfig.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/anchors/AnchorsConfig.java new file mode 100644 index 000000000..e0b79ac10 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/anchors/AnchorsConfig.java @@ -0,0 +1,53 @@ +package com.linkedin.feathr.core.config.producer.anchors; + +import com.linkedin.feathr.core.config.ConfigObj; +import com.linkedin.feathr.core.utils.Utils; +import java.util.Map; +import java.util.Objects; + + +/** + * Container class for the Anchors. + * + * @author djaising + * @author cesun + */ +public class AnchorsConfig implements ConfigObj { + private final Map _anchors; + private String _anchorStr; + + /** + * Constructor + * @param anchors map of anchor name to {@link AnchorConfig} + */ + public AnchorsConfig(Map anchors) { + _anchors = anchors; + _anchorStr = Utils.string(anchors, "\n"); + } + + @Override + public String toString() { + return _anchorStr; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof AnchorsConfig)) { + return false; + } + AnchorsConfig that = (AnchorsConfig) o; + return Objects.equals(_anchors, that._anchors); + } + + @Override + public int hashCode() { + return Objects.hash(_anchors); + } + + public Map getAnchors() { + return _anchors; + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/anchors/ComplexFeatureConfig.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/anchors/ComplexFeatureConfig.java new file mode 100644 index 000000000..e675061a6 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/anchors/ComplexFeatureConfig.java @@ -0,0 +1,164 @@ +package com.linkedin.feathr.core.config.producer.anchors; + +import com.linkedin.feathr.core.config.producer.ExprType; +import com.linkedin.feathr.core.config.producer.common.FeatureTypeConfig; +import com.linkedin.feathr.core.config.producer.definitions.FeatureType; +import java.util.Objects; +import java.util.Optional; + + +/** + * + * Represents an expression based feature configuration by specifying the object part in the following fragment: + * 
+ * {@code
+ *   : {
+ *     def: 
+ *     type: 
+ *     default: 
+ *   }
+ * }
+ *
+ * + * 
+ * {@code
+ *   : {
+ *     def.sqlExpr: 
+ *     type: 
+ *     default: 
+ *   }
+ * }
+ *
+ */ +// TODO - 17615): Rename this to ExpressionBasedFeatureConfigs +// This class is still used by Galene. We should renamed it in next major version bump. +public final class ComplexFeatureConfig extends FeatureConfig { + private final String _featureExpr; + private final ExprType _exprType; + private final Optional _defaultValue; + private final Optional _featureTypeConfig; + + private String _configStr; + + /** + * Constructor with full parameters + * @param featureExpr An expression for the feature + * @param exprType expression type of {@link ExprType} + * @param defaultValue A default value for the feature + * @param featureTypeConfig A detailed feature type information for the feature + */ + public ComplexFeatureConfig(String featureExpr, ExprType exprType, String defaultValue, + FeatureTypeConfig featureTypeConfig) { + _featureExpr = featureExpr; + _exprType = exprType; + _defaultValue = Optional.ofNullable(defaultValue); + _featureTypeConfig = Optional.ofNullable(featureTypeConfig); + + constructConfigStr(); + } + + /** + * Constructor + * @deprecated use {@link #ComplexFeatureConfig(String, ExprType, String, FeatureTypeConfig)} instead + * @param featureExpr An MVEL expression for the feature + * @param featureType The type of the feature + * @param defaultValue A default value for the feature + */ + @Deprecated + public ComplexFeatureConfig(String featureExpr, String featureType, String defaultValue) { + this(featureExpr, defaultValue, new FeatureTypeConfig(FeatureType.valueOf(featureType))); + } + + /** + * Constructor + * @deprecated use {@link #ComplexFeatureConfig(String, ExprType, String, FeatureTypeConfig)} instead + * @param featureExpr An MVEL expression for the feature + * @param featureTypeConfig A detailed feature type information for the feature + */ + @Deprecated + public ComplexFeatureConfig(String featureExpr, FeatureTypeConfig featureTypeConfig) { + this(featureExpr, null, featureTypeConfig); + } + + /** + * Constructor + * @deprecated use {@link #ComplexFeatureConfig(String, ExprType, String, FeatureTypeConfig)} instead + * @param featureExpr An MVEL expression for the feature + * @param defaultValue A default value for the feature + * @param featureTypeConfig A detailed feature type information for the feature + */ + @Deprecated + public ComplexFeatureConfig(String featureExpr, String defaultValue, FeatureTypeConfig featureTypeConfig) { + this(featureExpr, ExprType.MVEL, defaultValue, featureTypeConfig); + } + + /** + * Constructor + * @deprecated use {@link #ComplexFeatureConfig(String, ExprType, String, FeatureTypeConfig)} instead + * @param featureExpr An MVEL expression for the feature + * @param exprType expression type of {@link ExprType} + * @param featureType The type of the feature + * @param defaultValue A default value for the feature + */ + @Deprecated + public ComplexFeatureConfig(String featureExpr, ExprType exprType, FeatureType featureType, String defaultValue) { + this(featureExpr, exprType, defaultValue, featureType == null ? null : new FeatureTypeConfig(featureType)); + } + + public String getFeatureExpr() { + return _featureExpr; + } + + public ExprType getExprType() { + return _exprType; + } + + /** + * @deprecated Please use {@link #getFeatureTypeConfig()} + */ + // TODO - 10369) Remove getFeatureType API in favor of getFeatureTypeConfig() + @Deprecated + public Optional getFeatureType() { + return getFeatureTypeConfig().map(featureTypeConfig -> featureTypeConfig.getFeatureType().name()); + } + + @Override + public Optional getDefaultValue() { + return _defaultValue; + } + + @Override + public Optional getFeatureTypeConfig() { + return _featureTypeConfig; + } + + private void constructConfigStr() { + StringBuilder sb = new StringBuilder(); + sb.append(DEF).append(": ").append(_featureExpr).append("\n"); + _defaultValue.ifPresent(v -> sb.append(DEFAULT).append(": ").append(v).append("\n")); + _configStr = sb.toString(); + } + + @Override + public String toString() { + return _configStr; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + ComplexFeatureConfig that = (ComplexFeatureConfig) o; + return Objects.equals(_featureExpr, that._featureExpr) && _exprType == that._exprType && Objects.equals( + _defaultValue, that._defaultValue) && Objects.equals(_featureTypeConfig, that._featureTypeConfig); + } + + @Override + public int hashCode() { + return Objects.hash(_featureExpr, _exprType, _defaultValue, _featureTypeConfig); + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/anchors/ExpressionBasedFeatureConfig.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/anchors/ExpressionBasedFeatureConfig.java new file mode 100644 index 000000000..46bbff542 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/anchors/ExpressionBasedFeatureConfig.java @@ -0,0 +1,162 @@ +package com.linkedin.feathr.core.config.producer.anchors; + +import com.linkedin.feathr.core.config.producer.ExprType; +import com.linkedin.feathr.core.config.producer.common.FeatureTypeConfig; +import com.linkedin.feathr.core.config.producer.definitions.FeatureType; +import java.util.Objects; +import java.util.Optional; + + +/** + * + * Represents an expression based feature configuration by specifying the object part in the following fragment: + * 
+ * {@code
+ *   : {
+ *     def: 
+ *     type: 
+ *     default: 
+ *   }
+ * }
+ *
+ * + * 
+ * {@code
+ *   : {
+ *     def.sqlExpr: 
+ *     type: 
+ *     default: 
+ *   }
+ * }
+ *
+ */ +public final class ExpressionBasedFeatureConfig extends FeatureConfig { + private final String _featureExpr; + private final ExprType _exprType; + private final Optional _defaultValue; + private final Optional _featureTypeConfig; + + private String _configStr; + + /** + * Constructor with full parameters + * @param featureExpr An expression for the feature + * @param exprType expression type of {@link ExprType} + * @param defaultValue A default value for the feature + * @param featureTypeConfig A detailed feature type information for the feature + */ + public ExpressionBasedFeatureConfig(String featureExpr, ExprType exprType, String defaultValue, + FeatureTypeConfig featureTypeConfig) { + _featureExpr = featureExpr; + _exprType = exprType; + _defaultValue = Optional.ofNullable(defaultValue); + _featureTypeConfig = Optional.ofNullable(featureTypeConfig); + + constructConfigStr(); + } + + /** + * Constructor + * @deprecated use {@link #ExpressionBasedFeatureConfig(String, ExprType, String, FeatureTypeConfig)} instead + * @param featureExpr An MVEL expression for the feature + * @param featureType The type of the feature + * @param defaultValue A default value for the feature + */ + @Deprecated + public ExpressionBasedFeatureConfig(String featureExpr, String featureType, String defaultValue) { + this(featureExpr, defaultValue, new FeatureTypeConfig(FeatureType.valueOf(featureType))); + } + + /** + * Constructor + * @deprecated use {@link #ExpressionBasedFeatureConfig(String, ExprType, String, FeatureTypeConfig)} instead + * @param featureExpr An MVEL expression for the feature + * @param featureTypeConfig A detailed feature type information for the feature + */ + @Deprecated + public ExpressionBasedFeatureConfig(String featureExpr, FeatureTypeConfig featureTypeConfig) { + this(featureExpr, null, featureTypeConfig); + } + + /** + * Constructor + * @deprecated use {@link #ExpressionBasedFeatureConfig(String, ExprType, String, FeatureTypeConfig)} instead + * @param featureExpr An MVEL expression for the feature + * @param defaultValue A default value for the feature + * @param featureTypeConfig A detailed feature type information for the feature + */ + @Deprecated + public ExpressionBasedFeatureConfig(String featureExpr, String defaultValue, FeatureTypeConfig featureTypeConfig) { + this(featureExpr, ExprType.MVEL, defaultValue, featureTypeConfig); + } + + /** + * Constructor + * @deprecated use {@link #ExpressionBasedFeatureConfig(String, ExprType, String, FeatureTypeConfig)} instead + * @param featureExpr An MVEL expression for the feature + * @param exprType expression type of {@link ExprType} + * @param featureType The type of the feature + * @param defaultValue A default value for the feature + */ + @Deprecated + public ExpressionBasedFeatureConfig(String featureExpr, ExprType exprType, FeatureType featureType, String defaultValue) { + this(featureExpr, exprType, defaultValue, featureType == null ? null : new FeatureTypeConfig(featureType)); + } + + public String getFeatureExpr() { + return _featureExpr; + } + + public ExprType getExprType() { + return _exprType; + } + + /** + * @deprecated Please use {@link #getFeatureTypeConfig()} + */ + // TODO - 10369) Remove getFeatureType API in favor of getFeatureTypeConfig() + @Deprecated + public Optional getFeatureType() { + return getFeatureTypeConfig().map(featureTypeConfig -> featureTypeConfig.getFeatureType().name()); + } + + @Override + public Optional getDefaultValue() { + return _defaultValue; + } + + @Override + public Optional getFeatureTypeConfig() { + return _featureTypeConfig; + } + + private void constructConfigStr() { + StringBuilder sb = new StringBuilder(); + sb.append(FeatureConfig.DEF).append(": ").append(_featureExpr).append("\n"); + _defaultValue.ifPresent(v -> sb.append(FeatureConfig.DEFAULT).append(": ").append(v).append("\n")); + _configStr = sb.toString(); + } + + @Override + public String toString() { + return _configStr; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + ExpressionBasedFeatureConfig that = (ExpressionBasedFeatureConfig) o; + return Objects.equals(_featureExpr, that._featureExpr) && _exprType == that._exprType && Objects.equals( + _defaultValue, that._defaultValue) && Objects.equals(_featureTypeConfig, that._featureTypeConfig); + } + + @Override + public int hashCode() { + return Objects.hash(_featureExpr, _exprType, _defaultValue, _featureTypeConfig); + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/anchors/ExtractorBasedFeatureConfig.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/anchors/ExtractorBasedFeatureConfig.java new file mode 100644 index 000000000..dd1289357 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/anchors/ExtractorBasedFeatureConfig.java @@ -0,0 +1,117 @@ +package com.linkedin.feathr.core.config.producer.anchors; + +import com.linkedin.feathr.core.config.producer.common.FeatureTypeConfig; +import java.util.Collections; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import org.apache.commons.collections.MapUtils; + + +/** + * Represents a feature config based on extractor by specifying the value part in the following fragment: + * {@code : + * { + * type: type of the feature // optional + * parameters: parameters for the extractor to configure different extractor behavior per feature // optional + * defaultValue: default value of the feature // optional + * } + */ +public final class ExtractorBasedFeatureConfig extends FeatureConfig { + /** + * Legacy field. Feature name. + */ + private final String _featureName; + /** + * Optional parameters for the extractor, to configure the extractor behavior for each feature. By default it's empty. + */ + private final Map _parameters; + private final Optional _featureTypeConfig; + private final Optional _defaultValue; + + private String _configStr; + /** + * Constructor + * @param featureName A user-defined MVEL expression specifying the feature + */ + public ExtractorBasedFeatureConfig(String featureName) { + this(featureName, null, null, Collections.emptyMap()); + } + + /** + * Constructor + */ + public ExtractorBasedFeatureConfig(String featureName, FeatureTypeConfig featureTypeConfig) { + this(featureName, featureTypeConfig, null, Collections.emptyMap()); + } + + /** + * Constructor + */ + public ExtractorBasedFeatureConfig(String featureName, FeatureTypeConfig featureTypeConfig, String defaultValue, + Map parameters) { + _featureName = featureName; + _featureTypeConfig = Optional.ofNullable(featureTypeConfig); + _defaultValue = Optional.ofNullable(defaultValue); + _parameters = parameters; + constructConfigStr(); + } + + private void constructConfigStr() { + StringBuilder sb = new StringBuilder(); + sb.append(FeatureConfig.DEF).append(": ").append(_featureName).append("\n"); + _featureTypeConfig.ifPresent(t -> sb.append(FeatureConfig.TYPE).append(": ").append(t).append("\n")); + _defaultValue.ifPresent(v -> sb.append(FeatureConfig.DEFAULT).append(": ").append(v).append("\n")); + if (MapUtils.isNotEmpty(_parameters)) { + sb.append(FeatureConfig.PARAMETERS).append(": {\n"); + _parameters.entrySet().stream().map(entry -> sb.append(String.format("%s = %s\n", entry.getKey(), entry.getValue()))); + sb.append("}\n"); + } + _configStr = sb.toString(); + } + + /* + * Returns string representation of ExtractorBasedFeatureConfig (featureName, type, defaultValue, parameters) + */ + @Override + public String toString() { + return _configStr; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + ExtractorBasedFeatureConfig that = (ExtractorBasedFeatureConfig) o; + return Objects.equals(_featureName, that._featureName) && Objects.equals(_featureTypeConfig, + that._featureTypeConfig) && Objects.equals(_defaultValue, that._defaultValue) && Objects.equals(_parameters, that._parameters); + } + + @Override + public int hashCode() { + return Objects.hash(_featureName, _featureTypeConfig, _defaultValue, _parameters); + } + + public String getFeatureName() { + return _featureName; + } + + @Override + public Optional getFeatureTypeConfig() { + return _featureTypeConfig; + } + + @Override + public Optional getDefaultValue() { + return _defaultValue; + } + + @Override + public Map getParameters() { + return _parameters; + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/anchors/FeatureConfig.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/anchors/FeatureConfig.java new file mode 100644 index 000000000..dea669483 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/anchors/FeatureConfig.java @@ -0,0 +1,46 @@ +package com.linkedin.feathr.core.config.producer.anchors; + +import com.linkedin.feathr.core.config.ConfigObj; +import com.linkedin.feathr.core.config.producer.common.FeatureTypeConfig; +import java.util.Collections; +import java.util.Map; +import java.util.Optional; + + +/** + * Abstract class for the configuration of a feature in an anchor + */ +public abstract class FeatureConfig implements ConfigObj { + public static final String DEF = "def"; + public static final String DEF_MVEL = "def.mvel"; + public static final String DEF_SQL_EXPR = "def.sqlExpr"; + public static final String TYPE = "type"; + public static final String DEFAULT = "default"; + public static final String AGGREGATION = "aggregation"; + public static final String WINDOW = "window"; + public static final String SLIDING_INTERVAL = "slidingInterval"; + public static final String FILTER = "filter"; + public static final String FILTER_MVEL = "filter.mvel"; + public static final String GROUPBY = "groupBy"; + public static final String LIMIT = "limit"; + public static final String DECAY = "decay"; + public static final String WEIGHT = "weight"; + public static final String WINDOW_PARAMETERS = "windowParameters"; + public static final String SIZE = "size"; + public static final String EMBEDDING_SIZE = "embeddingSize"; + /** + * Parameters for the extractor + */ + public static final String PARAMETERS = "parameters"; + + public abstract Optional getDefaultValue(); + public abstract Optional getFeatureTypeConfig(); + + /** + * Return parameters for the extractor. + */ + public Map getParameters() { + return Collections.emptyMap(); + } + // Note: feature definition and feature config must be "linked" together in the model layer, not here. +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/anchors/LateralViewParams.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/anchors/LateralViewParams.java new file mode 100644 index 000000000..05e857d06 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/anchors/LateralViewParams.java @@ -0,0 +1,100 @@ +package com.linkedin.feathr.core.config.producer.anchors; + +import java.util.Objects; +import java.util.Optional; + + +/** + * Some feature datasets may contain feature values as an array of tuples. These are + * typically the result of some aggregation operation. To perform further aggregation on these tuples, for + * example, rollups from say, daily to weekly, the individual tuples have to be extracted, joined with + * observation data, and aggregated. + *

+ * The extraction can be performed by using Spark's lateral view in the FROM clause. The lateral view + * can be used to generate zero or more output rows from a single input row which is exactly what we need. + * This class specifies the parameters needed to construct the lateral view. A LateralViewParams is an + * optional parameter, and if specified it's applicable only for Sliding-window aggregation features. + * Further, it's specified once in the enclosing anchor. + *

+ */ +/* + * Design doc: https://docs.google.com/document/d/1B_ahJC5AQ4lgZIIFkG6gZnzTvp4Ori7WwWj9yv7XTe0/edit?usp=sharing + * RB: https://rb.corp.linkedin.com/r/1460513/ + */ +public final class LateralViewParams { + /* + * Fields used in anchor config fragment + */ + public static final String LATERAL_VIEW_DEF = "lateralViewDef"; + public static final String LATERAL_VIEW_ITEM_ALIAS = "lateralViewItemAlias"; + public static final String LATERAL_VIEW_FILTER = "lateralViewFilter"; + + private final String _def; + private final String _itemAlias; + private final Optional _filter; + private String _configStr; + + /** + * Constructor + * @param def A table-generating function. Typically it's explode(...) + * @param itemAlias User-defined alias for the generated table + * @param filter A filter expression applied to the elements/tuples in the input row. Optional parameter. + */ + public LateralViewParams(String def, String itemAlias, String filter) { + _def = def; + _itemAlias = itemAlias; + _filter = Optional.ofNullable(filter); + } + + /** + * Constructor + * @param def A table-generating function. Typically it's explode(...) + * @param itemAlias User-defined alias for the generated table + */ + public LateralViewParams(String def, String itemAlias) { + this(def, itemAlias, null); + } + + public String getDef() { + return _def; + } + + public String getItemAlias() { + return _itemAlias; + } + + public Optional getFilter() { + return _filter; + } + + @Override + public String toString() { + if (_configStr == null) { + _configStr = String.join("\n", + LATERAL_VIEW_DEF + ": " + _def, + LATERAL_VIEW_ITEM_ALIAS + ": " + _itemAlias); + + _filter.ifPresent(filter -> _configStr = String.join("\n", _configStr, LATERAL_VIEW_FILTER + ": " + filter)); + } + + return _configStr; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + LateralViewParams that = (LateralViewParams) o; + return Objects.equals(_def, that._def) && Objects.equals(_itemAlias, that._itemAlias) && Objects.equals(_filter, + that._filter); + } + + @Override + public int hashCode() { + return Objects.hash(_def, _itemAlias, _filter); + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/anchors/SimpleFeatureConfig.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/anchors/SimpleFeatureConfig.java new file mode 100644 index 000000000..e811813bc --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/anchors/SimpleFeatureConfig.java @@ -0,0 +1,128 @@ +package com.linkedin.feathr.core.config.producer.anchors; + +import com.linkedin.feathr.core.config.producer.common.FeatureTypeConfig; +import java.util.Collections; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import org.apache.commons.collections.MapUtils; + + +/** + * Represents a feature config based on extractor by specifying the value part in the following fragment: + * {@code : + * { + * type: type of the feature // optional + * parameters: parameters for the extractor to configure different extractor behavior per feature // optional + * defaultValue: default value of the feature // optional + * } + */ +// TODO - 17615): Rename this to ExtractorBasedFeatureConfig +// This class is still used by Galene. We should renamed it in next major version bump. +public final class SimpleFeatureConfig extends FeatureConfig { + /** + * Legacy field. Feature name. + */ + private final String _featureName; + /** + * Optional parameters for the extractor, to configure the extractor behavior for each feature. By default it's empty. + */ + private final Map _parameters; + private final Optional _featureTypeConfig; + private final Optional _defaultValue; + + private String _configStr; + /** + * Constructor + * @param featureName A user-defined MVEL expression specifying the feature + */ + public SimpleFeatureConfig(String featureName) { + this(featureName, null, null, Collections.emptyMap()); + } + + /** + * Constructor + */ + public SimpleFeatureConfig(String featureName, FeatureTypeConfig featureTypeConfig) { + this(featureName, featureTypeConfig, null, Collections.emptyMap()); + } + + /** + * Constructor + */ + public SimpleFeatureConfig(String featureName, FeatureTypeConfig featureTypeConfig, String defaultValue, + Map parameters) { + _featureName = featureName; + _featureTypeConfig = Optional.ofNullable(featureTypeConfig); + _defaultValue = Optional.ofNullable(defaultValue); + _parameters = parameters; + constructConfigStr(); + } + + private void constructConfigStr() { + StringBuilder sb = new StringBuilder(); + sb.append(FeatureConfig.DEF).append(": ").append(_featureName).append("\n"); + _featureTypeConfig.ifPresent(t -> sb.append(FeatureConfig.TYPE).append(": ").append(t).append("\n")); + _defaultValue.ifPresent(v -> sb.append(FeatureConfig.DEFAULT).append(": ").append(v).append("\n")); + if (MapUtils.isNotEmpty(_parameters)) { + sb.append(FeatureConfig.PARAMETERS).append(": {\n"); + _parameters.entrySet().stream().map(entry -> sb.append(String.format("%s = %s\n", entry.getKey(), entry.getValue()))); + sb.append("}\n"); + } + _configStr = sb.toString(); + } + + /** + * @Deprecated Use {@link #getFeatureName()} instead. + */ + // TODO - 17615): Remove this API in next major release + // This method is still used by Galene. + @Deprecated + public String getFeatureExpr() { + return _featureName; + } + + public String getFeatureName() { + return _featureName; + } + + @Override + public Optional getFeatureTypeConfig() { + return _featureTypeConfig; + } + + @Override + public Optional getDefaultValue() { + return _defaultValue; + } + + @Override + public Map getParameters() { + return _parameters; + } + + // TODO - 10384) Galene is using this function in their processing code so we can not update now. We can fix this + // in next major version bump. + @Override + public String toString() { + return _featureName; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + SimpleFeatureConfig that = (SimpleFeatureConfig) o; + return Objects.equals(_featureName, that._featureName) && Objects.equals(_featureTypeConfig, + that._featureTypeConfig) && Objects.equals(_defaultValue, that._defaultValue) && Objects.equals(_parameters, that._parameters); + } + + @Override + public int hashCode() { + return Objects.hash(_featureName, _featureTypeConfig, _defaultValue, _parameters); + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/anchors/TimeWindowFeatureConfig.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/anchors/TimeWindowFeatureConfig.java new file mode 100644 index 000000000..9c215e28b --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/anchors/TimeWindowFeatureConfig.java @@ -0,0 +1,265 @@ +package com.linkedin.feathr.core.config.producer.anchors; + +import com.linkedin.feathr.core.config.TimeWindowAggregationType; +import com.linkedin.feathr.core.config.producer.ExprType; +import com.linkedin.feathr.core.config.producer.TypedExpr; +import com.linkedin.feathr.core.config.producer.common.FeatureTypeConfig; +import java.time.Duration; +import java.util.Objects; +import java.util.Optional; + + +/** + * + * This represents 2 types of configs:- + * 1. a time-window (sliding window) feature offline config. + * 
+ * {@code
+ *   : {
+ *    def: 
+ *    aggregation: 
+ *    window: 
+ *    filter: 
+ *    groupBy: 
+ *    limit: 
+ *    decay: 
+ *    weight: 
+ *    embeddingSize: 
+ *  }
+ * }
+ *
+ * 2. a nearline feature config + * : { + * def/def.mvel: // the field on which the aggregation will be computed OR an MVEL expression (use def.mvel) + * aggregation: //aggregation types: SUM, COUNT, MAX, AVG + * windowParameters: + * { + * type: //The window type: SlidingWindow (MVP), FixedWindow (MVP), SessionWindow + * size: length of window time + * slidingInterval: // (Optional) Used only by sliding windowin nearline features. Specifies the interval of sliding window starts + * } + * groupBy: // (Optional) comma separated columns/fields on which the data will be ‘grouped by’ before aggregation + * filter/filter.mvel: // (Optional) An expression for filtering the fact data before aggregation. For mvel expression, use filter.mvel). + * } + * Details can be referenced in the FeatureDefConfigSchema + * In the offline world, it is always a sliding window and window in offline is equivalent to size in nearline. + * So, we convert the offline config to the nearline config, with the only difference being window used in offline, windowParameters used in + * nearline. + * + */ +public final class TimeWindowFeatureConfig extends FeatureConfig { + private final TypedExpr _typedColumnExpr; + private final TimeWindowAggregationType _aggregation; + private final WindowParametersConfig _windowParameters; + private final Optional _typedFilter; + private final Optional _groupBy; + private final Optional _limit; + private final Optional _decay; + private final Optional _weight; + private final Optional _embeddingSize; + private final Optional _featureTypeConfig; + private final Optional _defaultValue; + + + private String _configStr; + + /** + * Constructor with all parameters + * @param typedColumnExpr The column/field on which the aggregation will be computed, with the expr type + * @param aggregation Aggregation type as specified in [[TimeWindowAggregationType]] + * @param windowParameters windowParameters as specified in [[WindowParametersConfig]] + * @param typedFilter Spark SQL / MVEL expression for filtering the fact data before aggregation, with expr type + * @param groupBy column/field on which the data will be grouped by before aggregation + * @param limit positive integer to limit the number of records for each group + * @param decay not supported currently + * @param weight not supported currently + * @param embeddingSize embedding size + * @param featureTypeConfig featureTypeConfig for this faeture + */ + public TimeWindowFeatureConfig(TypedExpr typedColumnExpr, TimeWindowAggregationType aggregation, + WindowParametersConfig windowParameters, TypedExpr typedFilter, String groupBy, Integer limit, + String decay, String weight, Integer embeddingSize, FeatureTypeConfig featureTypeConfig, String defaultValue) { + _typedColumnExpr = typedColumnExpr; + _aggregation = aggregation; + _windowParameters = windowParameters; + _typedFilter = Optional.ofNullable(typedFilter); + _groupBy = Optional.ofNullable(groupBy); + _limit = Optional.ofNullable(limit); + _decay = Optional.ofNullable(decay); + _weight = Optional.ofNullable(weight); + _embeddingSize = Optional.ofNullable(embeddingSize); + _featureTypeConfig = Optional.ofNullable(featureTypeConfig); + _defaultValue = Optional.ofNullable(defaultValue); + + constructConfigStr(); + } + + /** + * Constructor with all parameters + * @param typedColumnExpr The column/field on which the aggregation will be computed, with the expr type + * @param aggregation Aggregation type as specified in [[TimeWindowAggregationType]] + * @param windowParameters windowParameters as specified in [[WindowParametersConfig]] + * @param typedFilter Spark SQL / MVEL expression for filtering the fact data before aggregation, with expr type + * @param groupBy column/field on which the data will be grouped by before aggregation + * @param limit positive integer to limit the number of records for each group + * @param decay not supported currently + * @param weight not supported currently + * @param embeddingSize embedding size + */ + public TimeWindowFeatureConfig(TypedExpr typedColumnExpr, TimeWindowAggregationType aggregation, + WindowParametersConfig windowParameters, TypedExpr typedFilter, String groupBy, Integer limit, String decay, + String weight, Integer embeddingSize) { + this(typedColumnExpr, aggregation, windowParameters, typedFilter, groupBy, limit, decay, weight, embeddingSize, + null, null); + } + + /** + * @param columnExpr The column/field on which the aggregation will be computed + * @param columnExprType The column/field expr type + * @param aggregation Aggregation type as specified in [[TimeWindowAggregationType]] + * @param windowParameters windowParameters as specified in [[WindowParametersConfig]] + * @param filter Spark SQL / MVEL expression for filtering the fact data before aggregation + * @param filterExprType the filter expression type + * @param groupBy column/field on which the data will be grouped by before aggregation + * @param limit positive integer to limit the number of records for each group + * @param decay not supported currently + * @param weight not supported currently + * @deprecated please use the constructor with all parameters + */ + public TimeWindowFeatureConfig(String columnExpr, ExprType columnExprType, TimeWindowAggregationType aggregation, + WindowParametersConfig windowParameters, String filter, ExprType filterExprType, String groupBy, Integer limit, + String decay, String weight) { + this(new TypedExpr(columnExpr, columnExprType), aggregation, windowParameters, + filter == null ? null : new TypedExpr(filter, filterExprType), + groupBy, limit, decay, weight, null); + } + + /** + * Constructor + * @param columnExpr The column/field on which the aggregation will be computed + * @param aggregation Aggregation type as specified in [[TimeWindowAggregationType]] + * @param windowParameters windowParameters as specified in [[WindowParametersConfig]] + * @param filter Spark SQL expression for filtering the fact data before aggregation + * @param groupBy column/field on which the data will be grouped by before aggregation + * @param limit positive integer to limit the number of records for each group + * @param decay not supported currently + * @param weight not supported currently + * @deprecated please use the constructor with all parameters + */ + @Deprecated + public TimeWindowFeatureConfig(String columnExpr, TimeWindowAggregationType aggregation, WindowParametersConfig windowParameters, + String filter, String groupBy, Integer limit, + String decay, String weight) { + this(new TypedExpr(columnExpr, ExprType.SQL), aggregation, windowParameters, + filter == null ? null : new TypedExpr(filter, ExprType.SQL), groupBy, limit, decay, weight, null); + } + + private void constructConfigStr() { + StringBuilder sb = new StringBuilder(); + + sb.append(FeatureConfig.DEF).append(": ").append(_typedColumnExpr.getExpr()).append("\n"); + sb.append("def expr type").append(": ").append(_typedColumnExpr.getExprType()).append("\n"); + sb.append(FeatureConfig.AGGREGATION).append(": ").append(_aggregation).append("\n"); + sb.append(FeatureConfig.WINDOW_PARAMETERS).append(": ").append(_windowParameters).append("\n"); + _typedFilter.ifPresent(v -> sb.append(FeatureConfig.FILTER).append(": ").append(v.getExpr()).append("\n"). + append("filter expr type").append(": ").append(v.getExprType()).append("\n")); + _groupBy.ifPresent(v -> sb.append(FeatureConfig.GROUPBY).append(": ").append(v).append("\n")); + _limit.ifPresent(v -> sb.append(FeatureConfig.LIMIT).append(": ").append(v).append("\n")); + _decay.ifPresent(v -> sb.append(FeatureConfig.DECAY).append(": ").append(v).append("\n")); + _weight.ifPresent(v -> sb.append(FeatureConfig.WEIGHT).append(": ").append(v).append("\n")); + _embeddingSize.ifPresent(v -> sb.append(FeatureConfig.EMBEDDING_SIZE).append(": ").append(v).append("\n")); + + _configStr = sb.toString(); + } + + @Override + public String toString() { + return _configStr; + } + + public String getColumnExpr() { + return _typedColumnExpr.getExpr(); + } + + public TimeWindowAggregationType getAggregation() { + return _aggregation; } + + public Duration getWindow() { + return _windowParameters.getSize(); + } + + public WindowParametersConfig getWindowParameters() { + return _windowParameters; } + + public Optional getFilter() { + return _typedFilter.map(TypedExpr::getExpr); + } + + public Optional getGroupBy() { + return _groupBy; + } + + public Optional getLimit() { + return _limit; + } + + public Optional getDecay() { + return _decay; + } + + public Optional getWeight() { + return _weight; + } + + public ExprType getColumnExprType() { + return _typedColumnExpr.getExprType(); + } + + public Optional getFilterExprType() { + return _typedFilter.map(TypedExpr::getExprType); + } + + public TypedExpr getTypedColumnExpr() { + return _typedColumnExpr; + } + + public Optional getTypedFilter() { + return _typedFilter; + } + + public Optional getEmbeddingSize() { + return _embeddingSize; + } + + @Override + public Optional getDefaultValue() { + return _defaultValue; + } + + @Override + public Optional getFeatureTypeConfig() { + return _featureTypeConfig; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + TimeWindowFeatureConfig that = (TimeWindowFeatureConfig) o; + return Objects.equals(_typedColumnExpr, that._typedColumnExpr) && _aggregation == that._aggregation + && Objects.equals(_windowParameters, that._windowParameters) && Objects.equals(_typedFilter, that._typedFilter) + && Objects.equals(_groupBy, that._groupBy) && Objects.equals(_limit, that._limit) && Objects.equals(_decay, + that._decay) && Objects.equals(_weight, that._weight) && Objects.equals(_embeddingSize, that._embeddingSize) + && Objects.equals(_featureTypeConfig, that._featureTypeConfig) && Objects.equals(_defaultValue, that._defaultValue); + } + + @Override + public int hashCode() { + return Objects.hash(_typedColumnExpr, _aggregation, _windowParameters, _typedFilter, _groupBy, _limit, _decay, + _weight, _embeddingSize, _featureTypeConfig, _defaultValue); + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/anchors/TypedKey.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/anchors/TypedKey.java new file mode 100644 index 000000000..6a0cf54fa --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/anchors/TypedKey.java @@ -0,0 +1,94 @@ +package com.linkedin.feathr.core.config.producer.anchors; + +import com.linkedin.feathr.core.config.producer.ExprType; +import com.linkedin.feathr.core.config.producer.common.KeyListExtractor; +import com.linkedin.feathr.core.utils.Utils; +import java.util.List; +import java.util.Objects; + + +/** + * Key expressions with the corresponding {@link ExprType} + */ +public class TypedKey { + private final String _rawKeyExpr; + private final List _key; + private final ExprType _keyExprType; + private String _configStr; + + /** + * Constructor + * @param rawKeyExpr the raw key expression + * @param keyExprType key type + */ + public TypedKey(String rawKeyExpr, ExprType keyExprType) { + _rawKeyExpr = rawKeyExpr; + // For now, we only support HOCON String format as the raw key expression + _key = KeyListExtractor.getInstance().extractFromHocon(rawKeyExpr); + _keyExprType = keyExprType; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof TypedKey)) { + return false; + } + TypedKey typedKey = (TypedKey) o; + /* + * Using the HOCON expression is too strict to check equality. For instance + * The following three key expressions: + * + * key1: [ + * # String: 3 + * "key1", + * # String: 3 + * "key2" + * ] + * + * key2: [key1, key2] + * + * key3: ["key1", "key2"] + * + * All have the same meaning, it is misleading, + * and sometimes impossible (e.g. in unit tests) to distinguish between these. + * And we should not distinguish them given that we've already parsed them using HOCON API in frame-core. + * + * Instead, we use the parsed key list to check the equality. + */ + return Objects.equals(_key, typedKey._key) && _keyExprType == typedKey._keyExprType; + } + + @Override + public int hashCode() { + return Objects.hash(_rawKeyExpr, _key, _keyExprType); + } + + @Override + public String toString() { + if (_configStr == null) { + _configStr = String.join("\n", + String.join(": ", "raw key expression", _rawKeyExpr), + String.join(": ", "key", (_key.size() == 1 ? _key.get(0) : Utils.string(_key))), + String.join(": ", "key expression type", _keyExprType.toString())); + } + return _configStr; + } + + /** + * Get the list of key String extracted from raw key expression + */ + public List getKey() { + return _key; + } + + public ExprType getKeyExprType() { + return _keyExprType; + } + + public String getRawKeyExpr() { + return _rawKeyExpr; + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/anchors/WindowParametersConfig.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/anchors/WindowParametersConfig.java new file mode 100644 index 000000000..730e06a29 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/anchors/WindowParametersConfig.java @@ -0,0 +1,83 @@ +package com.linkedin.feathr.core.config.producer.anchors; + +import com.linkedin.feathr.core.config.WindowType; +import java.time.Duration; +import java.util.Objects; +import java.util.Optional; + +/** + * Represents a windowparameters object config which is used in + * @see TimeWindowFeatureConfig + * windowParameters: + * { + * type: //The window type: SlidingWindow (MVP), FixedWindow (MVP), SessionWindow + * size: length of window time + * slidingInterval: // (Optional) Used only by sliding window. Specifies the interval of sliding window starts + * } + * } + * Details can be referenced in the FeatureDefConfigSchema + */ +public class WindowParametersConfig { + private final WindowType _windowType; + private final Duration _size; + private final Optional _slidingInterval; + private String _configStr; + + /** + * Constructor with all parameters + * @param windowType //The window type: SlidingWindow (MVP), FixedWindow (MVP), SessionWindow + * @param size length of window time + * @param slidingInterval (Optional) Used only by sliding window. Specifies the interval of sliding window starts + */ + public WindowParametersConfig(WindowType windowType, Duration size, Duration slidingInterval) { + _windowType = windowType; + _size = size; + _slidingInterval = Optional.ofNullable(slidingInterval); + constructConfigStr(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof WindowParametersConfig)) { + return false; + } + WindowParametersConfig that = (WindowParametersConfig) o; + return Objects.equals(_windowType, that._windowType) && Objects.equals(_size, that._size) + && Objects.equals(_slidingInterval, that._slidingInterval); + } + + private void constructConfigStr() { + StringBuilder sb = new StringBuilder(); + + sb.append(FeatureConfig.TYPE).append(": ").append(_windowType).append("\n") + .append(FeatureConfig.SIZE).append(": ").append(_size).append("\n"); + _slidingInterval.ifPresent(d -> sb.append(FeatureConfig.SLIDING_INTERVAL).append(": ").append(d).append("\n")); + + _configStr = sb.toString(); + } + + @Override + public String toString() { + return _configStr; + } + + @Override + public int hashCode() { + return Objects.hash(_windowType, _size, _slidingInterval); + } + + public WindowType getWindowType() { + return _windowType; + } + + public Duration getSize() { + return _size; + } + + public Optional getSlidingInterval() { + return _slidingInterval; + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/common/FeatureTypeConfig.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/common/FeatureTypeConfig.java new file mode 100644 index 000000000..7fb47b8e3 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/common/FeatureTypeConfig.java @@ -0,0 +1,178 @@ +package com.linkedin.feathr.core.config.producer.common; + +import com.linkedin.feathr.core.config.ConfigObj; +import com.linkedin.feathr.core.config.producer.definitions.FeatureType; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import org.checkerframework.checker.nullness.qual.NonNull; + + +/** + * Represents a type configuration for a feature by specifying the object part in the following fragment: + * 1. For a simple feature type + * 
+ * {@code
+ *   : {
+ *     type: 
+ *   }
+ * }
+ *
+ * 2. For a complex feature type + * 
+ * {@code
+ *   : {
+ *     type: {
+ *       type: 
+ *       tensorCategory: 
+ *       shape: 
+ *       dimensionType: 
+ *       valType: 
+ *     }
+ *   }
+ * }
+ *
+ */ +public class FeatureTypeConfig implements ConfigObj { + public static final String TYPE = "type"; + public static final String TENSOR_CATEGORY = "tensorCategory"; + public static final String SHAPE = "shape"; + public static final String DIMENSION_TYPE = "dimensionType"; + public static final String VAL_TYPE = "valType"; + private final FeatureType _featureType; + private final Optional> _shapes; + private final Optional> _dimensionTypes; + private final Optional _valType; + + private String _configStr; + + /** + * Creates a FeatureTypeConfig. + * @param shapes Shapes of the tensor(only applicable to tensor) + * @param dimensionTypes Dimension types of the tensor(only applicable to tensor) + * @param valType Value type of the tensor(only applicable to tensor) + */ + private FeatureTypeConfig(@NonNull FeatureType featureType, List shapes, List dimensionTypes, String valType) { + // Since VECTOR is deprecated, we always represent VECTOR with DENSE_VECTOR in Frame + if (featureType == FeatureType.VECTOR) { + _featureType = FeatureType.DENSE_VECTOR; + } else { + _featureType = featureType; + } + _shapes = Optional.ofNullable(shapes); + _dimensionTypes = Optional.ofNullable(dimensionTypes); + _valType = Optional.ofNullable(valType); + + constructConfigStr(); + } + + public FeatureTypeConfig(@NonNull FeatureType featureType) { + this(featureType, null, null, null); + } + + public FeatureType getFeatureType() { + return _featureType; + } + + private void constructConfigStr() { + StringBuilder sb = new StringBuilder(); + sb.append(FeatureTypeConfig.TYPE).append(": ").append(_featureType).append("\n"); + _shapes.ifPresent(t -> sb.append(FeatureTypeConfig.SHAPE).append(": ").append(t).append("\n")); + _dimensionTypes.ifPresent(v -> sb.append(FeatureTypeConfig.DIMENSION_TYPE).append(": ").append(v).append("\n")); + _valType.ifPresent(v -> sb.append(FeatureTypeConfig.VAL_TYPE).append(": ").append(v).append("\n")); + _configStr = sb.toString(); + } + + /** + * The shape (sometimes called the “size” or “dense shape”) of the tensor. Given as an array of integers. The first + * element gives the size of the first dimension in the tensor, the second element gives the size of the second + * dimension, and so on. The length of the tensorShape array is the number of dimensions in the tensor, also called + * the tensor's rank. For scalar (rank-0) features, tensorShape should appear as an empty array. + */ + public Optional> getShapes() { + return _shapes; + } + + /** + * Array of the types for each dimension. Allowable values are "int", "long", or "string". Length must be equal to + * length of tensorShape. + */ + public Optional> getDimensionTypes() { + return _dimensionTypes; + } + + /** + * The value type. Must be "int", "long", "float", "double", "boolean", or "string". + */ + public Optional getValType() { + return _valType; + } + + /** + * The string of the serialized config object. + */ + public String getConfigStr() { + return _configStr; + } + + @Override + public String toString() { + return _configStr; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + FeatureTypeConfig that = (FeatureTypeConfig) o; + return _featureType == that._featureType && Objects.equals(_shapes, that._shapes) && Objects.equals(_dimensionTypes, + that._dimensionTypes) && Objects.equals(_valType, that._valType); + } + + @Override + public int hashCode() { + return Objects.hash(_featureType, _shapes, _dimensionTypes, _valType); + } + + /** + * The builder for {@link FeatureTypeConfig} + */ + public static class Builder { + private FeatureType _featureType; + private List _shapes; + private List _dimensionTypes; + private String _valType; + + public Builder setFeatureType(FeatureType featureType) { + _featureType = featureType; + return this; + } + + public Builder setShapes(List shapes) { + _shapes = shapes; + return this; + } + + public Builder setDimensionTypes(List dimensionTypes) { + _dimensionTypes = dimensionTypes; + return this; + } + + public Builder setValType(String valType) { + _valType = valType; + return this; + } + + /** + * Builds a new {@link FeatureTypeConfig} with existing parameters + * @return {@link FeatureTypeConfig} object + */ + public FeatureTypeConfig build() { + return new FeatureTypeConfig(this._featureType, this._shapes, this._dimensionTypes, this._valType); + } + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/common/KeyListExtractor.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/common/KeyListExtractor.java new file mode 100644 index 000000000..eeedfafdc --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/common/KeyListExtractor.java @@ -0,0 +1,38 @@ +package com.linkedin.feathr.core.config.producer.common; + +import com.linkedin.feathr.core.utils.ConfigUtils; +import com.typesafe.config.Config; +import com.typesafe.config.ConfigFactory; +import java.util.List; + +/** + * The util class to extract key list. + */ +public class KeyListExtractor { + private static final KeyListExtractor INSTANCE = new KeyListExtractor(); + private static final String KEY_PATH = "MOCK_KEY_EXPR_PATH"; + private static final String HOCON_PREFIX = "{ "; + private static final String HOCON_SUFFIX = " }"; + private static final String HOCON_DELIM = " : "; + + public static KeyListExtractor getInstance() { + return INSTANCE; + } + + private KeyListExtractor() { + // singleton constructor + } + + /** + * This function extract a List of key String from HOCON representation of key field in Frame config. + * @param keyExpression key expression in HOCON format + */ + public List extractFromHocon(String keyExpression) { + // keyExpression is in HOCON ConfigValue format, which is not yet a valid HOCON Config string that can be parsed + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append(HOCON_PREFIX).append(KEY_PATH).append(HOCON_DELIM).append(keyExpression).append(HOCON_SUFFIX); + String hoconFullString = stringBuilder.toString(); + Config config = ConfigFactory.parseString(hoconFullString); + return ConfigUtils.getStringList(config, KEY_PATH); + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/definitions/FeatureType.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/definitions/FeatureType.java new file mode 100644 index 000000000..c5860b7e7 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/definitions/FeatureType.java @@ -0,0 +1,20 @@ +package com.linkedin.feathr.core.config.producer.definitions; + +/** + * Specifies the feature type of a feature. + * This is the same as the FeatureTypes in frame-common. + */ +public enum FeatureType { + BOOLEAN, + NUMERIC, + CATEGORICAL, + CATEGORICAL_SET, + TERM_VECTOR, + VECTOR, + DENSE_VECTOR, + TENSOR, + UNSPECIFIED, + DENSE_TENSOR, + SPARSE_TENSOR, + RAGGED_TENSOR +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/definitions/TensorCategory.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/definitions/TensorCategory.java new file mode 100644 index 000000000..9963ff67f --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/definitions/TensorCategory.java @@ -0,0 +1,21 @@ +package com.linkedin.feathr.core.config.producer.definitions; + +/** + * Specifies the tensor category. + * This is the same as com.linkedin.quince.relational.types.TensorCategory + */ +public enum TensorCategory { + /** + * Tensors of this category map some subset of the dimension space to values. + */ + SPARSE, + /** + * Tensors of this category map the entire dimension space to values. + * This includes scalar values (which are modeled as dense tensors with 0 dimensions). + */ + DENSE, + /** + * More general than DENSE, this category relaxes the constraint that shape of every dimension is constant within a single data instance. + */ + RAGGED +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/derivations/BaseFeatureConfig.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/derivations/BaseFeatureConfig.java new file mode 100644 index 000000000..2c413f6eb --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/derivations/BaseFeatureConfig.java @@ -0,0 +1,83 @@ +package com.linkedin.feathr.core.config.producer.derivations; + +import com.linkedin.feathr.core.utils.Utils; +import java.util.List; +import java.util.Objects; +import java.util.Optional; + + +/** + * Represents the definition of a base feature for sequential join config + */ +public final class BaseFeatureConfig extends KeyedFeature { + private final Optional> _outputKeys; // output keys after transformation + private final Optional _transformation; // logic to transform the keys of base feature to output keys + private final Optional _transformationClass; // custom base feature to output keys transformation. + + private String _configStr; + + /** + * Constructor + * @param rawkeyExpr the raw Key expression of the base feature + * @param feature The feature name of the base feature + * @param outputKeys the output keys of base feature + * @param transformation the logic to generate outputKeys values + */ + public BaseFeatureConfig(String rawkeyExpr, String feature, List outputKeys, String transformation, String transformationClass) { + super(rawkeyExpr, feature); + _outputKeys = Optional.ofNullable(outputKeys); + _transformation = Optional.ofNullable(transformation); + _transformationClass = Optional.ofNullable(transformationClass); + } + + @Override + public String toString() { + if (_configStr == null) { + _configStr = super.toString(); + + _outputKeys.ifPresent(k -> _configStr = String.join("\n", + _configStr, String.join(": ", DerivationConfig.OUTPUT_KEY, Utils.string(k)))); + + _transformation.ifPresent(t -> _configStr = String.join("\n", + _configStr, String.join(": ", DerivationConfig.TRANSFORMATION, t))); + + _transformationClass.ifPresent(t -> _configStr = String.join("\n", + _configStr, String.join(": ", DerivationConfig.TRANSFORMATION_CLASS, t))); + } + + return _configStr; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + BaseFeatureConfig that = (BaseFeatureConfig) o; + return Objects.equals(_outputKeys, that._outputKeys) && Objects.equals(_transformation, that._transformation) + && Objects.equals(_transformationClass, that._transformationClass); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), _outputKeys, _transformation, _transformationClass); + } + + public Optional> getOutputKeys() { + return _outputKeys; + } + + public Optional getTransformation() { + return _transformation; + } + + public Optional getTransformationClass() { + return _transformationClass; + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/derivations/DerivationConfig.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/derivations/DerivationConfig.java new file mode 100644 index 000000000..241fbbb68 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/derivations/DerivationConfig.java @@ -0,0 +1,31 @@ +package com.linkedin.feathr.core.config.producer.derivations; + +import com.linkedin.feathr.core.config.ConfigObj; +import com.linkedin.feathr.core.config.producer.common.FeatureTypeConfig; + +import java.util.Optional; + + +/** + * Represents the fields used for specifying the configuration parameters for feature derivations in the derivations + * section of the FeatureDef config file. + */ +public interface DerivationConfig extends ConfigObj { + String KEY = "key"; + String INPUTS = "inputs"; + String FEATURE = "feature"; + String DEFINITION = "definition"; + String CLASS = "class"; + String JOIN = "join"; // join field for sequential join config + String BASE = "base"; // base feature for sequential join config + String EXPANSION = "expansion"; // expansion feature for sequential join config + String AGGREGATION = "aggregation"; // aggregation field for sequential join config + String OUTPUT_KEY = "outputKey"; // outputKey field for base feature in sequential join config + String TRANSFORMATION = "transformation"; // transformation field for base feature in sequential join config + String TRANSFORMATION_CLASS = "transformationClass"; // transformationClass field for base feature in sequential join config + String SQL_EXPR = "sqlExpr"; // sqlExpr field for simple derivation config with SQL expression + String SQL_DEFINITION = "definition.sqlExpr"; // sqlExpr field for derivation config with SQL definition\ + String TYPE = "type"; + + Optional getFeatureTypeConfig(); +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/derivations/DerivationConfigWithExpr.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/derivations/DerivationConfigWithExpr.java new file mode 100644 index 000000000..0e17505b7 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/derivations/DerivationConfigWithExpr.java @@ -0,0 +1,134 @@ +package com.linkedin.feathr.core.config.producer.derivations; + +import com.linkedin.feathr.core.config.producer.ExprType; +import com.linkedin.feathr.core.config.producer.TypedExpr; +import com.linkedin.feathr.core.config.producer.common.FeatureTypeConfig; +import com.linkedin.feathr.core.utils.Utils; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; + + +/** + * Represents the definition of a derived feature using keys and MVEL/SQL expression. + * + * @author djaising + * @author cesun + */ +public final class DerivationConfigWithExpr implements DerivationConfig { + private final List _keys; + private final Map _inputs; + private final TypedExpr _typedDefinition; + private final Optional _featureTypeConfig; + + private String _configStr; + + /** + * Constructor + * @param keys The key of the derived feature; can be single or composite key. + * @param inputs The parent feature(s) from whom this feature is derived. It is expressed as a java.util.Map of + * argument name to {@link KeyedFeature} + * @param typedDefinition A user-defined expression which defines the derived feature using the argument names from the + * inputs, together with the {@link ExprType} + */ + public DerivationConfigWithExpr(List keys, Map inputs, TypedExpr typedDefinition) { + _keys = keys; + _inputs = inputs; + _typedDefinition = typedDefinition; + _featureTypeConfig = Optional.empty(); + } + + /** + * Constructor + * @param keys The key of the derived feature; can be single or composite key. + * @param inputs The parent feature(s) from whom this feature is derived. It is expressed as a java.util.Map of + * argument name to {@link KeyedFeature} + * @param typedDefinition A user-defined expression which defines the derived feature using the argument names from the + * inputs, together with the {@link ExprType} + */ + public DerivationConfigWithExpr(List keys, Map inputs, TypedExpr typedDefinition, + FeatureTypeConfig featureTypeConfig) { + _keys = keys; + _inputs = inputs; + _typedDefinition = typedDefinition; + _featureTypeConfig = Optional.ofNullable(featureTypeConfig); + } + + /** + * Constructor + * @param keys The key of the derived feature; can be single or composite key. + * @param inputs The parent feature(s) from whom this feature is derived. It is expressed as a java.util.Map of + * argument name to {@link KeyedFeature} + * @param definition A user-defined MVEL expression which defines the derived feature using the argument names from the + * inputs + * @deprecated please use {@link #DerivationConfigWithExpr(List, Map, TypedExpr)} + */ + @Deprecated + public DerivationConfigWithExpr(List keys, Map inputs, String definition) { + _keys = keys; + _inputs = inputs; + _typedDefinition = new TypedExpr(definition, ExprType.MVEL); + _featureTypeConfig = Optional.empty(); + } + + public List getKeys() { + return _keys; + } + + public Map getInputs() { + return _inputs; + } + + @Deprecated + public String getDefinition() { + return _typedDefinition.getExpr(); + } + + public TypedExpr getTypedDefinition() { + return _typedDefinition; + } + + public Optional getFeatureTypeConfig() { + return _featureTypeConfig; + } + + @Override + public String toString() { + if (_configStr == null) { + StringBuilder sb = new StringBuilder(); + sb.append(KEY) + .append(": ") + .append(Utils.string(_keys)) + .append("\n") + .append(INPUTS) + .append(": ") + .append(Utils.string(_inputs, "\n")) + .append("\n") + .append(DEFINITION) + .append(": \n") + .append(_typedDefinition.toString()) + .append("\n"); + _configStr = sb.toString(); + } + return _configStr; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + DerivationConfigWithExpr that = (DerivationConfigWithExpr) o; + return Objects.equals(_keys, that._keys) && Objects.equals(_inputs, that._inputs) && Objects.equals( + _typedDefinition, that._typedDefinition) && Objects.equals(_featureTypeConfig, that._featureTypeConfig); + } + + @Override + public int hashCode() { + return Objects.hash(_keys, _inputs, _typedDefinition, _featureTypeConfig); + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/derivations/DerivationConfigWithExtractor.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/derivations/DerivationConfigWithExtractor.java new file mode 100644 index 000000000..68ca9c2de --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/derivations/DerivationConfigWithExtractor.java @@ -0,0 +1,121 @@ +package com.linkedin.feathr.core.config.producer.derivations; + +import com.linkedin.feathr.core.config.producer.common.FeatureTypeConfig; +import com.linkedin.feathr.core.utils.Utils; +import java.util.List; +import java.util.Objects; +import java.util.Optional; + + +/** + * Represents the definition of a derived feature using a user-defined class. + * + * @author djaising + * @author cesun + */ +public final class DerivationConfigWithExtractor implements DerivationConfig { + private final List _keys; + private final List _inputs; + private final String _className; + private final Optional _featureTypeConfig; + + private String _configStr; + + /** + * Constructor + * @param keys The key of the derived feature; can be single or composite key. + * @param inputs The parent feature(s) from whom this feature is derived. It is expressed as a list of {@link KeyedFeature} + * @param className The user-defined class which implements the feature derivation logic. + * + */ + public DerivationConfigWithExtractor(List keys, List inputs, String className) { + _keys = keys; + _inputs = inputs; + _className = className; + _featureTypeConfig = Optional.empty(); + + StringBuilder sb = new StringBuilder(); + sb.append(KEY) + .append(": ") + .append(Utils.string(keys)) + .append("\n") + .append(INPUTS) + .append(": ") + .append(Utils.string(inputs)) + .append("\n") + .append(CLASS) + .append(": ") + .append(className) + .append("\n"); + _configStr = sb.toString(); + } + + /** + * Constructor + * @param keys The key of the derived feature; can be single or composite key. + * @param inputs The parent feature(s) from whom this feature is derived. It is expressed as a list of {@link KeyedFeature} + * @param className The user-defined class which implements the feature derivation logic. + * + */ + public DerivationConfigWithExtractor(List keys, List inputs, String className, + FeatureTypeConfig featureTypeConfig) { + _keys = keys; + _inputs = inputs; + _className = className; + _featureTypeConfig = Optional.ofNullable(featureTypeConfig); + + StringBuilder sb = new StringBuilder(); + sb.append(KEY) + .append(": ") + .append(Utils.string(keys)) + .append("\n") + .append(INPUTS) + .append(": ") + .append(Utils.string(inputs)) + .append("\n") + .append(CLASS) + .append(": ") + .append(className) + .append("\n"); + _configStr = sb.toString(); + } + + public List getKeys() { + return _keys; + } + + public List getInputs() { + return _inputs; + } + + public String getClassName() { + return _className; + } + + public Optional getFeatureTypeConfig() { + return _featureTypeConfig; + } + + @Override + public String toString() { + return _configStr; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + DerivationConfigWithExtractor that = (DerivationConfigWithExtractor) o; + return Objects.equals(_keys, that._keys) && Objects.equals(_inputs, that._inputs) && Objects.equals(_className, + that._className) && Objects.equals(_featureTypeConfig, that._featureTypeConfig); + } + + @Override + public int hashCode() { + return Objects.hash(_keys, _inputs, _className, _featureTypeConfig); + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/derivations/DerivationsConfig.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/derivations/DerivationsConfig.java new file mode 100644 index 000000000..7ed19b730 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/derivations/DerivationsConfig.java @@ -0,0 +1,55 @@ +package com.linkedin.feathr.core.config.producer.derivations; + +import com.linkedin.feathr.core.config.ConfigObj; +import com.linkedin.feathr.core.utils.Utils; +import java.util.Map; +import java.util.Objects; + + +/** + * Container class for all derived feature configurations. + * + * @author djaising + * @author cesun + */ +public final class DerivationsConfig implements ConfigObj { + + private final Map _derivations; + + private String _configStr; + + /** + * Constructor + * @param derivations map of derivation name to {@link DerivationConfig} + */ + public DerivationsConfig(Map derivations) { + _derivations = derivations; + _configStr = Utils.string(derivations, "\n"); + } + + public Map getDerivations() { + return _derivations; + } + + @Override + public String toString() { + return _configStr; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof DerivationsConfig)) { + return false; + } + DerivationsConfig that = (DerivationsConfig) o; + return Objects.equals(_derivations, that._derivations); + } + + @Override + public int hashCode() { + return Objects.hash(_derivations); + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/derivations/KeyedFeature.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/derivations/KeyedFeature.java new file mode 100644 index 000000000..fd191bf01 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/derivations/KeyedFeature.java @@ -0,0 +1,103 @@ +package com.linkedin.feathr.core.config.producer.derivations; + +import com.linkedin.feathr.core.config.producer.common.KeyListExtractor; +import java.util.List; +import java.util.Objects; + +import static com.linkedin.feathr.core.config.producer.derivations.DerivationConfig.*; + + +/** + * A tuple that specifies the key (single or composite) associated with a feature + * + * @author djaising + * @author cesun + */ +public class KeyedFeature { + private final String _rawKeyExpr; + private final List _key; + private final String _feature; + + private String _configStr; + + /** + * Constructor. + * During construction, the input raw key expression will be extracted to a list of key String. + * For instance: + * - "x" will be converted to list ["x"]. + * - "[\"key1\", \"key2\"]" will be converted to list ["key1", "key2"] + * - "[key1, key2]" will be converted to ["key1", "key2"] also + * + * @param rawKeyExpr the raw key expression + * @param feature The name of the feature + */ + public KeyedFeature(String rawKeyExpr, String feature) { + _rawKeyExpr = rawKeyExpr; + // For now, we only support HOCON String format as the raw key expression + _key = KeyListExtractor.getInstance().extractFromHocon(rawKeyExpr); + _feature = feature; + + StringBuilder sb = new StringBuilder(); + sb.append(KEY).append(": ").append(rawKeyExpr).append(", ") + .append(FEATURE).append(": ").append(feature); + _configStr = sb.toString(); + } + + @Override + public String toString() { + return _configStr; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof KeyedFeature)) { + return false; + } + KeyedFeature that = (KeyedFeature) o; + /* + * Using the HOCON expression is too strict to check equality. For instance + * The following three key expressions: + * + * key1: [ + * # String: 3 + * "key1", + * # String: 3 + * "key2" + * ] + * + * key2: [key1, key2] + * + * key3: ["key1", "key2"] + * + * All have the same meaning, it is misleading, + * and sometimes impossible (e.g. in unit tests) to distinguish between these. + * And we should not distinguish them given that we've already parsed them using HOCON API in frame-core. + * + * Instead, we use the parsed key list to check the equality. + */ + return Objects.equals(_key, that._key) && Objects.equals(_feature, that._feature); + } + + @Override + public int hashCode() { + return Objects.hash(_rawKeyExpr, _key, _feature); + } + + public String getRawKeyExpr() { + return _rawKeyExpr; + } + + /** + * Get the list of key String extracted from raw key expression + */ + public List getKey() { + return _key; + } + + public String getFeature() { + return _feature; + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/derivations/SequentialJoinConfig.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/derivations/SequentialJoinConfig.java new file mode 100644 index 000000000..b83bd986a --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/derivations/SequentialJoinConfig.java @@ -0,0 +1,103 @@ +package com.linkedin.feathr.core.config.producer.derivations; + +import com.linkedin.feathr.core.config.producer.common.FeatureTypeConfig; +import com.linkedin.feathr.core.utils.Utils; +import java.util.List; +import java.util.Objects; +import java.util.Optional; + + +/** + * Represents the definition of a sequential join config as derivation feature + */ +public final class SequentialJoinConfig implements DerivationConfig { + private final List _keys; + private final BaseFeatureConfig _base; + private final KeyedFeature _expansion; + private final String _aggregation; + private final Optional _featureTypeConfig; + + private String _configStr; + + /** + * Constructor + * @param keys The key of the derived feature; can be single or composite key. + * @param base The base feature for sequential join + * @param expansion The expansion feature for sequential join + * @param aggregation The aggregation type + * @param featureTypeConfig The {@link FeatureTypeConfig} for this feature config + */ + public SequentialJoinConfig(List keys, BaseFeatureConfig base, KeyedFeature expansion, String aggregation, + FeatureTypeConfig featureTypeConfig) { + _keys = keys; + _base = base; + _expansion = expansion; + _aggregation = aggregation; + _featureTypeConfig = Optional.ofNullable(featureTypeConfig); + } + + /** + * Constructor + * @param keys The key of the derived feature; can be single or composite key. + * @param base The base feature for sequential join + * @param expansion The expansion feature for sequential join + * @param aggregation The aggregation type + */ + public SequentialJoinConfig(List keys, BaseFeatureConfig base, KeyedFeature expansion, String aggregation) { + _keys = keys; + _base = base; + _expansion = expansion; + _aggregation = aggregation; + _featureTypeConfig = Optional.empty(); + } + + @Override + public String toString() { + if (_configStr == null) { + _configStr = + String.join("\n", String.join(": ", KEY, Utils.string(_keys)), String.join(":\n", BASE, _base.toString()), + String.join(":\n", EXPANSION, _expansion.toString()), String.join(": ", AGGREGATION, _aggregation)); + } + + return _configStr; + } + + public List getKeys() { + return _keys; + } + + public BaseFeatureConfig getBase() { + return _base; + } + + public KeyedFeature getExpansion() { + return _expansion; + } + + public String getAggregation() { + return _aggregation; + } + + public Optional getFeatureTypeConfig() { + return _featureTypeConfig; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + SequentialJoinConfig that = (SequentialJoinConfig) o; + return Objects.equals(_keys, that._keys) && Objects.equals(_base, that._base) && Objects.equals(_expansion, + that._expansion) && Objects.equals(_aggregation, that._aggregation) && Objects.equals(_featureTypeConfig, + that._featureTypeConfig); + } + + @Override + public int hashCode() { + return Objects.hash(_keys, _base, _expansion, _aggregation, _featureTypeConfig); + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/derivations/SimpleDerivationConfig.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/derivations/SimpleDerivationConfig.java new file mode 100644 index 000000000..4d04cbd65 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/derivations/SimpleDerivationConfig.java @@ -0,0 +1,89 @@ +package com.linkedin.feathr.core.config.producer.derivations; + +import com.linkedin.feathr.core.config.producer.ExprType; +import com.linkedin.feathr.core.config.producer.TypedExpr; +import com.linkedin.feathr.core.config.producer.common.FeatureTypeConfig; +import java.util.Objects; +import java.util.Optional; + + +/** + * Represents a derived feature whose derivation can be expressed as a user-defined expression with type + * + * @author djaising + * @author cesun + */ +public final class SimpleDerivationConfig implements DerivationConfig { + private final TypedExpr _featureTypedExpr; + private final Optional _featureTypeConfig; + + /** + * Constructor + * @param featureExpr A user-defined MVEL expression + * @deprecated please use {@link #SimpleDerivationConfig(TypedExpr)} + */ + @Deprecated + public SimpleDerivationConfig(String featureExpr) { + _featureTypedExpr = new TypedExpr(featureExpr, ExprType.MVEL); + _featureTypeConfig = Optional.empty(); + } + + /** + * Constructor + * @param typedExpr A user-defined expression with type + */ + public SimpleDerivationConfig(TypedExpr typedExpr) { + _featureTypedExpr = typedExpr; + _featureTypeConfig = Optional.empty(); + } + + + /** + * Constructor + * @param typedExpr A user-defined expression with type + */ + public SimpleDerivationConfig(TypedExpr typedExpr, FeatureTypeConfig featureTypeConfig) { + _featureTypedExpr = typedExpr; + _featureTypeConfig = Optional.ofNullable(featureTypeConfig); + } + + /** + * get the expression string + * @deprecated please use {@link #getFeatureTypedExpr()} + */ + @Deprecated + public String getFeatureExpr() { + return _featureTypedExpr.getExpr(); + } + + public TypedExpr getFeatureTypedExpr() { + return _featureTypedExpr; + } + + public Optional getFeatureTypeConfig() { + return _featureTypeConfig; + } + + @Override + public String toString() { + return _featureTypedExpr.toString(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + SimpleDerivationConfig that = (SimpleDerivationConfig) o; + return Objects.equals(_featureTypedExpr, that._featureTypedExpr) && Objects.equals(_featureTypeConfig, + that._featureTypeConfig); + } + + @Override + public int hashCode() { + return Objects.hash(_featureTypedExpr, _featureTypeConfig); + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/features/Availability.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/features/Availability.java new file mode 100644 index 000000000..4e2b3d2e6 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/features/Availability.java @@ -0,0 +1,25 @@ +package com.linkedin.feathr.core.config.producer.features; + +import java.util.Optional; + +/** + * Denotes availability of a feature in a particular environment. + */ +public enum Availability { + OFFLINE, + ONLINE, + OFFLINE_ONLINE; + + public static Optional fromName(String name) { + Availability res = null; + + for (Availability a : values()) { + if (a.name().equalsIgnoreCase(name)) { + res = a; + break; + } + } + + return Optional.ofNullable(res); + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/features/ValueType.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/features/ValueType.java new file mode 100644 index 000000000..1572d15a3 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/features/ValueType.java @@ -0,0 +1,33 @@ +package com.linkedin.feathr.core.config.producer.features; + +import java.util.Optional; +import org.apache.log4j.Logger; + + +/** + * Specifies the value type of a feature. It includes all primitive types and string. + */ +public enum ValueType { + STRING, + INT, + LONG, + DOUBLE, + FLOAT, + BOOLEAN, + BYTE; + + private static final Logger logger = Logger.getLogger(ValueType.class); + + public static Optional fromName(String name) { + ValueType res = null; + + for (ValueType vt : values()) { + if (vt.name().equalsIgnoreCase(name)) { + res = vt; + break; + } + } + + return Optional.ofNullable(res); + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/sources/CouchbaseConfig.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/sources/CouchbaseConfig.java new file mode 100644 index 000000000..255898de5 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/sources/CouchbaseConfig.java @@ -0,0 +1,90 @@ +package com.linkedin.feathr.core.config.producer.sources; + +import java.util.Objects; + + +/** + * Represents the source config params for a Couchbase store + */ +public final class CouchbaseConfig extends SourceConfig { + // Couchbase bucket name + private final String _bucketName; + + // Expression used to produce Couchbase key input + private final String _keyExpr; + + // Fully qualified class name of the stored document in bucket + private final String _documentModel; + + /* + * Fields used to specify the Couchbase source configuration + */ + public static final String BUCKET_NAME = "bucketName"; + public static final String KEY_EXPR = "keyExpr"; + public static final String BOOTSTRAP_URIS = "bootstrapUris"; + public static final String DOCUMENT_MODEL = "documentModel"; + + /** + * Constructor + * @param sourceName the name of the source and it is referenced by the anchor in the feature definition + * @param bucketName Name of the Couchbase bucket + * @param keyExpr Key expression + * @param documentModel Document model stored in bucket + */ + public CouchbaseConfig(String sourceName, String bucketName, String keyExpr, String documentModel) { + super(sourceName); + _bucketName = bucketName; + _keyExpr = keyExpr; + _documentModel = documentModel; + } + + @Override + public SourceType getSourceType() { + return SourceType.COUCHBASE; + } + + public String getBucketName() { + return _bucketName; + } + + public String getKeyExpr() { + return _keyExpr; + } + + public String getDocumentModel() { + return _documentModel; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + CouchbaseConfig that = (CouchbaseConfig) o; + return Objects.equals(_bucketName, that._bucketName) && Objects.equals(_keyExpr, that._keyExpr) + && Objects.equals(_documentModel, that._documentModel); + } + + @Override + public int hashCode() { + int result = Objects.hash(super.hashCode(), _bucketName, _keyExpr, _documentModel); + return result; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("CouchbaseConfig{"); + sb.append("_bucketName='").append(_bucketName).append('\''); + sb.append(", _keyExpr='").append(_keyExpr).append('\''); + sb.append(", _documentModel='").append(_documentModel).append('\''); + sb.append(", _sourceName='").append(_sourceName).append('\''); + sb.append('}'); + return sb.toString(); + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/sources/CustomSourceConfig.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/sources/CustomSourceConfig.java new file mode 100644 index 000000000..95ed6efd2 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/sources/CustomSourceConfig.java @@ -0,0 +1,75 @@ +package com.linkedin.feathr.core.config.producer.sources; + +import java.util.Objects; + +/** + * Represents Custom source config + */ +public final class CustomSourceConfig extends SourceConfig { + + private final String _keyExpr; + + // the model of the data being fetched from the custom source + private final String _dataModel; + + /** + * Field used in CUSTOM source config fragment + */ + public static final String DATA_MODEL = "dataModel"; + public static final String KEY_EXPR = "keyExpr"; + + /** + * Constructor with parameters + * @param sourceName the name of the source and it is referenced by the anchor in the feature definition + * @param keyExpr the key expression used to compute the key against the custom source + * @param dataModel Class name of the data returned from the custom source + */ + public CustomSourceConfig(String sourceName, String keyExpr, String dataModel) { + super(sourceName); + _keyExpr = keyExpr; + _dataModel = dataModel; + } + + public String getDataModel() { + return _dataModel; + } + + public String getKeyExpr() { + return _keyExpr; + } + + @Override + public SourceType getSourceType() { + return SourceType.CUSTOM; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + CustomSourceConfig that = (CustomSourceConfig) o; + return Objects.equals(_keyExpr, that._keyExpr) && Objects.equals(_dataModel, that._dataModel); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), _keyExpr, _dataModel); + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("CustomSourceConfig{"); + sb.append("_keyExpr='").append(_keyExpr).append('\''); + sb.append(", _dataModel='").append(_dataModel).append('\''); + sb.append(", _sourceName='").append(_sourceName).append('\''); + sb.append('}'); + return sb.toString(); + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/sources/EspressoConfig.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/sources/EspressoConfig.java new file mode 100644 index 000000000..16c1c64e3 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/sources/EspressoConfig.java @@ -0,0 +1,92 @@ +package com.linkedin.feathr.core.config.producer.sources; + +import java.util.Objects; + + +/** + * Represents the configuration for an Espresso source + */ +public final class EspressoConfig extends SourceConfig { + private final String _database; + private final String _table; + private final String _d2Uri; + private final String _keyExpr; + private final String _name; + + public static final String DATABASE = "database"; + public static final String TABLE = "table"; + public static final String D2_URI = "d2Uri"; + public static final String KEY_EXPR = "keyExpr"; + + /** + * Constructor with full parameters + * @param sourceName the name of the source and it is referenced by the anchor in the feature definition + * @param database Name of the database + * @param table Name of the table + * @param d2Uri D2 URI + * @param keyExpr key expression + */ + public EspressoConfig(String sourceName, String database, String table, String d2Uri, String keyExpr) { + super(sourceName); + _database = database; + _table = table; + _d2Uri = d2Uri; + _keyExpr = keyExpr; + _name = database + "/" + table; + } + + public String getDatabase() { + return _database; + } + + public String getTable() { + return _table; + } + + public String getD2Uri() { + return _d2Uri; + } + + public String getKeyExpr() { + return _keyExpr; + } + + @Override + public SourceType getSourceType() { + return SourceType.ESPRESSO; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + EspressoConfig that = (EspressoConfig) o; + return Objects.equals(_database, that._database) && Objects.equals(_table, that._table) && Objects.equals(_d2Uri, + that._d2Uri) && Objects.equals(_keyExpr, that._keyExpr) && Objects.equals(_name, that._name); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), _database, _table, _d2Uri, _keyExpr, _name); + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("EspressoConfig{"); + sb.append("_database='").append(_database).append('\''); + sb.append(", _table='").append(_table).append('\''); + sb.append(", _d2Uri='").append(_d2Uri).append('\''); + sb.append(", _keyExpr=").append(_keyExpr); + sb.append(", _name='").append(_name).append('\''); + sb.append(", _sourceName='").append(_sourceName).append('\''); + sb.append('}'); + return sb.toString(); + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/sources/HdfsConfig.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/sources/HdfsConfig.java new file mode 100644 index 000000000..78d3ebc86 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/sources/HdfsConfig.java @@ -0,0 +1,81 @@ +package com.linkedin.feathr.core.config.producer.sources; + +import java.util.Objects; +import java.util.Optional; + + +/** + * Abstract class for all HDFS config classes + */ +public abstract class HdfsConfig extends SourceConfig { + private final String _path; + private final Optional _timePartitionPattern; + + /* Represents the fields in a HDFS source config */ + public static final String PATH = "location.path"; + public static final String HAS_TIME_SNAPSHOT = "hasTimeSnapshot"; + public static final String TIME_PARTITION_PATTERN = "timePartitionPattern"; + + /** + * Constructor + * @param sourceName the name of the source and it is referenced by the anchor in the feature definition + * @param path HDFS path or Dali URI used to access HDFS + * @param timePartitionPattern format of the time partitioned feature + */ + protected HdfsConfig(String sourceName, String path, String timePartitionPattern) { + super(sourceName); + _path = path; + _timePartitionPattern = Optional.ofNullable(timePartitionPattern); + } + + /** + * Constructor + * @param path HDFS path or Dali URI used to access HDFS + */ + protected HdfsConfig(String sourceName, String path) { + this(sourceName, path, null); + } + + public String getPath() { + return _path; + } + + public Optional getTimePartitionPattern() { + return _timePartitionPattern; + } + + @Override + public SourceType getSourceType() { + return SourceType.HDFS; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + HdfsConfig that = (HdfsConfig) o; + return Objects.equals(_path, that._path) && Objects.equals(_timePartitionPattern, that._timePartitionPattern); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), _path, _timePartitionPattern); + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("HdfsConfig{"); + sb.append("_path='").append(_path).append('\''); + sb.append(", _timePartitionPattern=").append(_timePartitionPattern); + sb.append(", _sourceName='").append(_sourceName).append('\''); + sb.append('}'); + return sb.toString(); + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/sources/HdfsConfigWithRegularData.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/sources/HdfsConfigWithRegularData.java new file mode 100644 index 000000000..017102375 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/sources/HdfsConfigWithRegularData.java @@ -0,0 +1,68 @@ +package com.linkedin.feathr.core.config.producer.sources; + +import java.util.Objects; + + +/** + * Represents HDFS config for non-time-series, that is, regular data + */ +public final class HdfsConfigWithRegularData extends HdfsConfig { + // this is a deprecated field. It is replaced by timePartitionPattern. We keep it for backward compatibility. + private final Boolean _hasTimeSnapshot; + + /** + * Constructor with full parameters + * + * @param sourceName the name of the source and it is referenced by the anchor in the feature definition + * @param path HDFS path or Dali URI used to access HDFS + * @param timePartitionPattern format of the time partitioned feature + * @param hasTimeSnapshot True if the HDFS source supports time-based access + */ + public HdfsConfigWithRegularData(String sourceName, String path, String timePartitionPattern, Boolean hasTimeSnapshot) { + super(sourceName, path, timePartitionPattern); + _hasTimeSnapshot = hasTimeSnapshot; + } + + /** + * Constructor + * @param sourceName the name of the source and it is referenced by the anchor in the feature definition + * @param path HDFS path or Dali URI used to access HDFS + * @param hasTimeSnapshot True if the HDFS source supports time-based access + */ + public HdfsConfigWithRegularData(String sourceName, String path, Boolean hasTimeSnapshot) { + this(sourceName, path, null, hasTimeSnapshot); + } + + public Boolean getHasTimeSnapshot() { + return _hasTimeSnapshot; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + HdfsConfigWithRegularData that = (HdfsConfigWithRegularData) o; + return Objects.equals(_hasTimeSnapshot, that._hasTimeSnapshot); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), _hasTimeSnapshot); + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("HdfsConfigWithRegularData{"); + sb.append("_hasTimeSnapshot=").append(_hasTimeSnapshot); + sb.append(", _sourceName='").append(_sourceName).append('\''); + sb.append('}'); + return sb.toString(); + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/sources/HdfsConfigWithSlidingWindow.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/sources/HdfsConfigWithSlidingWindow.java new file mode 100644 index 000000000..282f04985 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/sources/HdfsConfigWithSlidingWindow.java @@ -0,0 +1,66 @@ +package com.linkedin.feathr.core.config.producer.sources; + +import java.util.Objects; + + +/** + * Represents HDFS config with sliding window parameters + */ +public final class HdfsConfigWithSlidingWindow extends HdfsConfig { + private final SlidingWindowAggrConfig _swaConfig; + + /** + * Constructor + * @param sourceName the name of the source and it is referenced by the anchor in the feature definition + * @param path HDFS path + * @param timePartitionPattern format of the time partitioned feature + * @param swaConfig sliding window config + */ + public HdfsConfigWithSlidingWindow(String sourceName, String path, String timePartitionPattern, SlidingWindowAggrConfig swaConfig) { + super(sourceName, path, timePartitionPattern); + _swaConfig = swaConfig; + } + + /** + * Constructor + * @param sourceName the name of the source and it is referenced by the anchor in the feature definition + * @param path HDFS path + * @param swaConfig sliding window config + */ + public HdfsConfigWithSlidingWindow(String sourceName, String path, SlidingWindowAggrConfig swaConfig) { + this(sourceName, path, null, swaConfig); + } + + public SlidingWindowAggrConfig getSwaConfig() { + return _swaConfig; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + HdfsConfigWithSlidingWindow that = (HdfsConfigWithSlidingWindow) o; + return Objects.equals(_swaConfig, that._swaConfig); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), _swaConfig); + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("HdfsConfigWithSlidingWindow{"); + sb.append("_swaConfig=").append(_swaConfig); + sb.append(", _sourceName='").append(_sourceName).append('\''); + sb.append('}'); + return sb.toString(); + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/sources/KafkaConfig.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/sources/KafkaConfig.java new file mode 100644 index 000000000..4b8e78006 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/sources/KafkaConfig.java @@ -0,0 +1,73 @@ +package com.linkedin.feathr.core.config.producer.sources; + +import java.util.Objects; +import java.util.Optional; + + +/** + * Represents Kafka source config + */ +public final class KafkaConfig extends SourceConfig { + private final String _stream; + private final Optional _swaConfig; + + /* + * Field used in Kafka source config fragment + */ + public static final String STREAM = "stream"; + + /** + * Constructor with full parameters + * @param sourceName the name of the source and it is referenced by the anchor in the feature definition + * @param stream Name of Kafka stream + * @param swaConfig {@link SlidingWindowAggrConfig} object + */ + public KafkaConfig(String sourceName, String stream, SlidingWindowAggrConfig swaConfig) { + super(sourceName); + _stream = stream; + _swaConfig = Optional.ofNullable(swaConfig); + } + + public String getStream() { + return _stream; + } + + public Optional getSwaConfig() { + return _swaConfig; + } + + @Override + public SourceType getSourceType() { + return SourceType.KAFKA; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + KafkaConfig that = (KafkaConfig) o; + return Objects.equals(_stream, that._stream) && Objects.equals(_swaConfig, that._swaConfig); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), _stream, _swaConfig); + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("KafkaConfig{"); + sb.append("_stream='").append(_stream).append('\''); + sb.append(", _swaConfig=").append(_swaConfig); + sb.append(", _sourceName='").append(_sourceName).append('\''); + sb.append('}'); + return sb.toString(); + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/sources/PassThroughConfig.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/sources/PassThroughConfig.java new file mode 100644 index 000000000..c96595db5 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/sources/PassThroughConfig.java @@ -0,0 +1,65 @@ +package com.linkedin.feathr.core.config.producer.sources; + +import java.util.Objects; +import java.util.Optional; + + +/** + * Represents PassThrough source config + */ +public final class PassThroughConfig extends SourceConfig { + private final String _dataModel; + + /** + * Field used in PassThrough source config fragment + */ + public static final String DATA_MODEL = "dataModel"; + + /** + * Constructor + * @param sourceName the name of the source and it is referenced by the anchor in the feature definition + * @param dataModel Class name for pass-through object + */ + public PassThroughConfig(String sourceName, String dataModel) { + super(sourceName); + _dataModel = dataModel; + } + + @Override + public SourceType getSourceType() { + return SourceType.PASSTHROUGH; + } + + public Optional getDataModel() { + return Optional.ofNullable(_dataModel); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + PassThroughConfig that = (PassThroughConfig) o; + return Objects.equals(_dataModel, that._dataModel); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), _dataModel); + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("PassThroughConfig{"); + sb.append("_dataModel='").append(_dataModel).append('\''); + sb.append(", _sourceName='").append(_sourceName).append('\''); + sb.append('}'); + return sb.toString(); + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/sources/PinotConfig.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/sources/PinotConfig.java new file mode 100644 index 000000000..6e6626e37 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/sources/PinotConfig.java @@ -0,0 +1,110 @@ +package com.linkedin.feathr.core.config.producer.sources; + +import java.util.Arrays; +import java.util.Objects; +import javax.annotation.Nonnull; + +/** + * Represents the Pinot source config. For example + * + * "recentPageViewsSource": { + * type: "PINOT" + * resourceName: "recentMemberActionsPinotQuery" + * queryTemplate: "SELECT objectAttributes, timeStampSec + * FROM RecentMemberActions + * WHERE actorId IN (?) AND timeStampSec > ? + * ORDER BY timeStampSec DESC + * LIMIT 1000" + * queryArguments: ["key[0]", "System.currentTimeMillis()/1000 - 2 * 24 * 60 * 60"] + * queryKeyColumns: ["actorId"] + * } + */ +public class PinotConfig extends SourceConfig { + private final String _resourceName; + private final String _queryTemplate; + private final String[] _queryArguments; + private final String[] _queryKeyColumns; + + /* + * Fields to specify the Pinot source configuration + */ + public static final String RESOURCE_NAME = "resourceName"; + public static final String QUERY_TEMPLATE = "queryTemplate"; + public static final String QUERY_ARGUMENTS = "queryArguments"; + public static final String QUERY_KEY_COLUMNS = "queryKeyColumns"; + + /** + * Constructor + * @param sourceName the name of the source referenced by anchors in the feature definition + * @param resourceName the service name in the Pinot D2 config for the queried Pinot table + * @param queryTemplate the sql query template to fetch data from Pinot table, with “?” as placeholders for queryArguments replacement at runtime + * @param queryArguments the array of key expression, whose element is used to replace the "?" in queryTemplate in the same order + * @param queryKeyColumns the array of String for Pinot table column names that correspond to key argument defined queryArguments in the same order + */ + public PinotConfig(@Nonnull String sourceName, @Nonnull String resourceName, @Nonnull String queryTemplate, + @Nonnull String[] queryArguments, @Nonnull String[] queryKeyColumns) { + super(sourceName); + _resourceName = resourceName; + _queryTemplate = queryTemplate; + _queryArguments = queryArguments; + _queryKeyColumns = queryKeyColumns; + } + + public String getResourceName() { + return _resourceName; + } + + public String getQueryTemplate() { + return _queryTemplate; + } + + public String[] getQueryArguments() { + return _queryArguments; + } + + public String[] getQueryKeyColumns() { + return _queryKeyColumns; + } + + @Override + public SourceType getSourceType() { + return SourceType.PINOT; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + PinotConfig that = (PinotConfig) o; + return Objects.equals(_resourceName, that._resourceName) + && Objects.equals(_queryTemplate, that._queryTemplate) + && Arrays.equals(_queryArguments, that._queryArguments) + && Arrays.equals(_queryKeyColumns, that._queryKeyColumns); + } + + @Override + public int hashCode() { + int result = Objects.hash(super.hashCode(), _resourceName, _queryTemplate); + result = 31 * result + Arrays.hashCode(_queryArguments) + Arrays.hashCode(_queryKeyColumns); + return result; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("PinotConfig{"); + sb.append("_resourceName='").append(_resourceName).append('\''); + sb.append(", _queryTemplate='").append(_queryTemplate).append('\''); + sb.append(", _queryArguments='").append(Arrays.toString(_queryArguments)).append('\''); + sb.append(", _queryKeyColumns='").append(Arrays.toString(_queryKeyColumns)).append('\''); + sb.append(", _sourceName='").append(_sourceName).append('\''); + sb.append('}'); + return sb.toString(); + } +} \ No newline at end of file diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/sources/RestliConfig.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/sources/RestliConfig.java new file mode 100644 index 000000000..b8ec9d54b --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/sources/RestliConfig.java @@ -0,0 +1,161 @@ +package com.linkedin.feathr.core.config.producer.sources; + +import com.google.common.base.Preconditions; +import com.linkedin.data.schema.PathSpec; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import javax.annotation.Nonnull; + + +/** + * Represents the Rest.Li source config + */ +public final class RestliConfig extends SourceConfig { + public static final String RESOURCE_NAME = "restResourceName"; + + /** + * @deprecated As of beta, the field name is a typo and will be removed + */ + @Deprecated + public static final String RESOUCE_NAME = "restResouceName"; + // Note: typo but still being supported. Ought to be removed. + + public static final String KEY_EXPR = "keyExpr"; + + /** + * @deprecated As of beta, this field is deprecated in favor of KEY_EXPR(keyExpr) + */ + @Deprecated + public static final String ENTITY_TYPE = "restEntityType"; // Note: this field is deprecated in favor of 'keyExpr' + + public static final String REQ_PARAMS = "restReqParams"; + public static final String PATH_SPEC = "pathSpec"; + public static final String FINDER = "finder"; + + // Keys used in REQ_PARAMS + public static final String JSON = "json"; + public static final String JSON_ARRAY = "jsonArray"; + public static final String JSON_ARRAY_ARRAY = "array"; + public static final String MVEL_KEY = "mvel"; + public static final String FILE = "file"; + + private final String _resourceName; + private final Optional _keyExpr; + private final Optional> _reqParams; + private final Optional _pathSpec; + private final Optional _finder; + + /** + * Constructor with keyExpr only + * @param sourceName the name of the source and it is referenced by the anchor in the feature definition + * @param resourceName Name of the Rest.Li resource + * @param keyExpr Key expression + * @param reqParams request parameters specified as a Map + * @param pathSpec PathSpec + */ + public RestliConfig(@Nonnull String sourceName, @Nonnull String resourceName, @Nonnull String keyExpr, + Map reqParams, PathSpec pathSpec) { + this(sourceName, resourceName, keyExpr, reqParams, pathSpec, null); + } + + /** + * Construct a finder based {@link RestliConfig} for non-association resources where there is no association key required + * @param sourceName the name of the source and it is referenced by the anchor in the feature definition + * @param resourceName Name of the Rest.Li resource + * @param reqParams request parameters specified as a Map + * @param pathSpec PathSpec + * @param finder the finder method name of the resource. + */ + public RestliConfig(@Nonnull String sourceName, @Nonnull String resourceName, Map reqParams, + PathSpec pathSpec, @Nonnull String finder) { + this(sourceName, resourceName, null, reqParams, pathSpec, finder); + } + + /** + * Constructor for creating a new instance of {@link RestliConfig} with both keyExpr + * @param sourceName the name of the source and it is referenced by the anchor in the feature definition + * @param keyExpr Key expression for the resource. + * @param resourceName Name of the Rest.Li resource + * @param reqParams request parameters specified as a Map + * @param pathSpec PathSpec + * @param finder the finder method name of the resource. + */ + public RestliConfig(String sourceName, String resourceName, String keyExpr, Map reqParams, PathSpec pathSpec, String finder) { + super(sourceName); + Preconditions.checkArgument(keyExpr != null || finder != null, "Either keyExpr or finder must be present for a RestLi source"); + _resourceName = resourceName; + _keyExpr = Optional.ofNullable(keyExpr); + _reqParams = Optional.ofNullable(reqParams); + _pathSpec = Optional.ofNullable(pathSpec); + _finder = Optional.ofNullable(finder); + } + + public String getResourceName() { + return _resourceName; + } + + /** + * @deprecated this might return null, please use {@link #getOptionalKeyExpr()} instead + */ + @Deprecated + public String getKeyExpr() { + return _keyExpr.orElse(null); + } + + public Optional getOptionalKeyExpr() { + return _keyExpr; + } + + public Optional> getReqParams() { + return _reqParams; + } + + public Optional getPathSpec() { + return _pathSpec; + } + + public Optional getFinder() { + return _finder; + } + + @Override + public SourceType getSourceType() { + return SourceType.RESTLI; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + RestliConfig that = (RestliConfig) o; + return Objects.equals(_resourceName, that._resourceName) && Objects.equals(_keyExpr, that._keyExpr) + && Objects.equals(_reqParams, that._reqParams) && Objects.equals(_pathSpec, that._pathSpec) && Objects.equals( + _finder, that._finder); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), _resourceName, _keyExpr, _reqParams, _pathSpec, _finder); + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("RestliConfig{"); + sb.append("_resourceName='").append(_resourceName).append('\''); + sb.append(", _keyExpr=").append(_keyExpr); + sb.append(", _reqParams=").append(_reqParams); + sb.append(", _pathSpec=").append(_pathSpec); + sb.append(", _finder=").append(_finder); + sb.append(", _sourceName='").append(_sourceName).append('\''); + sb.append('}'); + return sb.toString(); + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/sources/RocksDbConfig.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/sources/RocksDbConfig.java new file mode 100644 index 000000000..5c7025f0d --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/sources/RocksDbConfig.java @@ -0,0 +1,120 @@ +package com.linkedin.feathr.core.config.producer.sources; + +import java.util.Objects; +import java.util.Optional; + + +/** + * Represents the RocksDB source config + */ +// TODO: verify if both encoder and decoder are required. Frame will support 'Use Mode 3' where both of these are required. +public final class RocksDbConfig extends SourceConfig { + + /* + * Fields used to specify config params in RocksDB source config + */ + public static final String REFERENCE_SOURCE = "referenceSource"; + public static final String EXTRACT_FEATURES = "extractFeatures"; + public static final String ENCODER = "encoder"; + public static final String DECODER = "decoder"; + public static final String KEYEXPR = "keyExpr"; + + private final String _referenceSource; + private final Boolean _extractFeatures; + private final Optional _encoder; + private final Optional _decoder; + private final Optional _keyExpr; + + /** + * Constructor with full parameters + * @param sourceName the name of the source and it is referenced by the anchor in the feature definition + */ + public RocksDbConfig(String sourceName, String referenceSource, Boolean extractFeatures, String encoder, String decoder, + String keyExpr) { + super(sourceName); + + _referenceSource = referenceSource; + _extractFeatures = extractFeatures; + _encoder = Optional.ofNullable(encoder); + _decoder = Optional.ofNullable(decoder); + _keyExpr = Optional.ofNullable(keyExpr); + } + + @Deprecated + /** + * Deprecated Constructor without full parameters for backwards compatibility + * @param referenceSource + * @param extractFeatures + * @param encoder encoder + * @param decoder decoder + */ + public RocksDbConfig(String sourceName, String referenceSource, Boolean extractFeatures, String encoder, String decoder) { + super(sourceName); + + _referenceSource = referenceSource; + _extractFeatures = extractFeatures; + _encoder = Optional.ofNullable(encoder); + _decoder = Optional.ofNullable(decoder); + _keyExpr = Optional.empty(); + } + + public String getReferenceSource() { + return _referenceSource; + } + + public Boolean getExtractFeatures() { + return _extractFeatures; + } + + public Optional getEncoder() { + return _encoder; + } + + public Optional getDecoder() { + return _decoder; + } + + public Optional getKeyExpr() { + return _keyExpr; + } + + @Override + public SourceType getSourceType() { + return SourceType.ROCKSDB; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + RocksDbConfig that = (RocksDbConfig) o; + return Objects.equals(_referenceSource, that._referenceSource) && Objects.equals(_extractFeatures, + that._extractFeatures) && Objects.equals(_encoder, that._encoder) && Objects.equals(_decoder, that._decoder) + && Objects.equals(_keyExpr, that._keyExpr); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), _referenceSource, _extractFeatures, _encoder, _decoder, _keyExpr); + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("RocksDbConfig{"); + sb.append("_referenceSource='").append(_referenceSource).append('\''); + sb.append(", _extractFeatures=").append(_extractFeatures); + sb.append(", _encoder=").append(_encoder); + sb.append(", _decoder=").append(_decoder); + sb.append(", _keyExpr=").append(_keyExpr); + sb.append(", _sourceName='").append(_sourceName).append('\''); + sb.append('}'); + return sb.toString(); + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/sources/SlidingWindowAggrConfig.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/sources/SlidingWindowAggrConfig.java new file mode 100644 index 000000000..15acaa289 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/sources/SlidingWindowAggrConfig.java @@ -0,0 +1,63 @@ +package com.linkedin.feathr.core.config.producer.sources; + +import java.util.Objects; + +/** + * Represents sliding time-window aggregation config + */ +public final class SlidingWindowAggrConfig { + public static final String IS_TIME_SERIES = "isTimeSeries"; + public static final String TIMEWINDOW_PARAMS = "timeWindowParameters"; + + // this is a deprecated field. It is replaced by timePartitionPattern. We keep it for backward compatibility. + private final Boolean _isTimeSeries; + + private final TimeWindowParams _timeWindowParams; + + private String _configStr; + + /** + * Constructor + * @param isTimeSeries Always true + * @param timeWindowParams Sliding time-window parameters + */ + public SlidingWindowAggrConfig(Boolean isTimeSeries, TimeWindowParams timeWindowParams) { + _isTimeSeries = isTimeSeries; + _timeWindowParams = timeWindowParams; + + StringBuilder sb = new StringBuilder(); + sb.append(IS_TIME_SERIES).append(": ").append(isTimeSeries).append("\n") + .append(TIMEWINDOW_PARAMS).append(": ").append(timeWindowParams).append("\n"); + _configStr = sb.toString(); + } + + @Override + public String toString() { + return _configStr; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof SlidingWindowAggrConfig)) { + return false; + } + SlidingWindowAggrConfig that = (SlidingWindowAggrConfig) o; + return Objects.equals(_isTimeSeries, that._isTimeSeries) && Objects.equals(_timeWindowParams, that._timeWindowParams); + } + + @Override + public int hashCode() { + return Objects.hash(_isTimeSeries, _timeWindowParams); + } + + public Boolean getTimeSeries() { + return _isTimeSeries; + } + + public TimeWindowParams getTimeWindowParams() { + return _timeWindowParams; + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/sources/SourceConfig.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/sources/SourceConfig.java new file mode 100644 index 000000000..f7662793e --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/sources/SourceConfig.java @@ -0,0 +1,50 @@ +package com.linkedin.feathr.core.config.producer.sources; + +import com.linkedin.feathr.core.config.ConfigObj; +import java.util.Objects; +import javax.annotation.Nonnull; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.Validate; + + +/** + * Base class to represent source configuration + */ +public abstract class SourceConfig implements ConfigObj { + + protected final String _sourceName; + + public static final String TYPE = "type"; + + protected SourceConfig(@Nonnull String sourceName) { + Validate.isTrue(StringUtils.isNotBlank(sourceName), "source name must not be blank!"); + _sourceName = sourceName; + } + + public abstract SourceType getSourceType(); + + /** + * Returns the name associated with the source. + * This is typically the name of the source as defined in the sources section of the feature definition file + */ + public String getSourceName() { + return _sourceName; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + SourceConfig that = (SourceConfig) o; + return Objects.equals(_sourceName, that._sourceName); + } + + @Override + public int hashCode() { + return Objects.hash(_sourceName); + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/sources/SourceType.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/sources/SourceType.java new file mode 100644 index 000000000..0a9192632 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/sources/SourceType.java @@ -0,0 +1,28 @@ +package com.linkedin.feathr.core.config.producer.sources; + + +/** + * Represents the supported source types by Frame. + */ +public enum SourceType { + HDFS("HDFS"), + ESPRESSO("Espresso"), + RESTLI("RestLi"), + VENICE("Venice"), + KAFKA("Kafka"), + ROCKSDB("RocksDB"), + PASSTHROUGH("PASSTHROUGH"), + COUCHBASE("Couchbase"), + CUSTOM("Custom"), + PINOT("Pinot"), + VECTOR("Vector"); + + private final String _sourceType; + SourceType(String sourceType) { + _sourceType = sourceType; + } + + public String getSourceType() { + return _sourceType; + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/sources/SourcesConfig.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/sources/SourcesConfig.java new file mode 100644 index 000000000..9ebb2ea66 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/sources/SourcesConfig.java @@ -0,0 +1,48 @@ +package com.linkedin.feathr.core.config.producer.sources; + +import com.linkedin.feathr.core.config.ConfigObj; +import com.linkedin.feathr.core.utils.Utils; +import java.util.Map; +import java.util.Objects; + + +/** + * Container class for the source configurations specified in the sources section of the FeatureDef config file. + */ +public final class SourcesConfig implements ConfigObj { + private final Map _sources; + + private String _configStr; + + public SourcesConfig(Map sources) { + _sources = sources; + _configStr = Utils.string(sources); + } + + @Override + public String toString() { + return _configStr; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof SourcesConfig)) { + return false; + } + SourcesConfig that = (SourcesConfig) o; + return Objects.equals(_sources, that._sources); + } + + @Override + public int hashCode() { + return Objects.hash(_sources); + } + + public Map getSources() { + return _sources; + } +} + diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/sources/TimeWindowParams.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/sources/TimeWindowParams.java new file mode 100644 index 000000000..a4f80ae63 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/sources/TimeWindowParams.java @@ -0,0 +1,63 @@ +package com.linkedin.feathr.core.config.producer.sources; + +import java.util.Objects; + + +/** + * Time-window parameters used in {@link SlidingWindowAggrConfig} + */ +public final class TimeWindowParams { + public static final String TIMESTAMP_FIELD = "timestampColumn"; + public static final String TIMESTAMP_FORMAT = "timestampColumnFormat"; + public static final String TIMESTAMP_EPOCH_SECOND_FORMAT = "epoch"; + public static final String TIMESTAMP_EPOCH_MILLISECOND_FORMAT = "epoch_millis"; + private final String _timestampField; + private final String _timestampFormat; + + private String _configStr; + + /** + * Constructor + * @param timestampField Name of the timestamp column/field in fact data + * @param timestampFormat Format pattern of the timestamp value, specified in {@link java.time.format.DateTimeFormatter} pattern + */ + public TimeWindowParams(String timestampField, String timestampFormat) { + _timestampField = timestampField; + _timestampFormat = timestampFormat; + + StringBuilder sb = new StringBuilder(); + sb.append(TIMESTAMP_FIELD).append(": ").append(timestampField).append("\n") + .append(TIMESTAMP_FORMAT).append(": ").append(timestampFormat).append("\n"); + _configStr = sb.toString(); + } + + @Override + public String toString() { + return _configStr; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof TimeWindowParams)) { + return false; + } + TimeWindowParams that = (TimeWindowParams) o; + return Objects.equals(_timestampField, that._timestampField) && Objects.equals(_timestampFormat, that._timestampFormat); + } + + @Override + public int hashCode() { + return Objects.hash(_timestampField, _timestampFormat); + } + + public String getTimestampField() { + return _timestampField; + } + + public String getTimestampFormat() { + return _timestampFormat; + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/sources/VectorConfig.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/sources/VectorConfig.java new file mode 100644 index 000000000..917b59e86 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/sources/VectorConfig.java @@ -0,0 +1,79 @@ +package com.linkedin.feathr.core.config.producer.sources; + +import java.util.Objects; +import javax.annotation.Nonnull; + +/** + * Represents the Vector source config. For example + * + * "vectorImageStoreForPNG": { + * type: "VECTOR" + * keyExpr: "key[0]" + * featureSourceName: "png_200_200" + * } + * + * Note here that the featureSourceName is a Vector query parameter which is decided between the team that will use the + * media data and Vector. This is a string but will be created via a process detailed by the Vector team. + */ +public class VectorConfig extends SourceConfig { + private final String _keyExpr; + private final String _featureSourceName; + + /* + * Fields to specify the Vector source configuration + */ + public static final String KEY_EXPR = "keyExpr"; + public static final String FEATURE_SOURCE_NAME = "featureSourceName"; + + /** + * Constructor + * @param sourceName the name of the source referenced by anchors in the feature definition + * @param keyExpr the key expression used to extract assetUrn to access asset from Vector endpoint + * @param featureSourceName the vector query parameter needed in addition the assetUrn to fetch the asset + */ + public VectorConfig(@Nonnull String sourceName, @Nonnull String keyExpr, @Nonnull String featureSourceName) { + super(sourceName); + _keyExpr = keyExpr; + _featureSourceName = featureSourceName; + } + + public String getKeyExpr() { + return _keyExpr; } + + public String getFeatureSourceName() { + return _featureSourceName; } + + @Override + public SourceType getSourceType() { + return SourceType.VECTOR; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + VectorConfig that = (VectorConfig) o; + return Objects.equals(_keyExpr, that._keyExpr) && Objects.equals(_featureSourceName, that._featureSourceName); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), _keyExpr, _featureSourceName); + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("VectorConfig{"); + sb.append("_keyExpr=").append(_keyExpr); + sb.append(", _featureSourceName=").append(_featureSourceName).append('\''); + sb.append('}'); + return sb.toString(); + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/sources/VeniceConfig.java b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/sources/VeniceConfig.java new file mode 100644 index 000000000..c036a3e6b --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/config/producer/sources/VeniceConfig.java @@ -0,0 +1,74 @@ +package com.linkedin.feathr.core.config.producer.sources; + +import java.util.Objects; + + +/** + * Represents the source config params for a Venice store + */ +public final class VeniceConfig extends SourceConfig { + private final String _storeName; + private final String _keyExpr; + + /* + * Fields used to specify the Venice source configuration + */ + public static final String STORE_NAME = "storeName"; + public static final String KEY_EXPR = "keyExpr"; + + /** + * Constructor + * + * @param sourceName the name of the source and it is referenced by the anchor in the feature definition + * @param storeName Name of the Venice store + * @param keyExpr Key expression + */ + public VeniceConfig(String sourceName, String storeName, String keyExpr) { + super(sourceName); + _storeName = storeName; + _keyExpr = keyExpr; + } + + public String getStoreName() { + return _storeName; + } + + public String getKeyExpr() { + return _keyExpr; + } + + @Override + public SourceType getSourceType() { + return SourceType.VENICE; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + VeniceConfig that = (VeniceConfig) o; + return Objects.equals(_storeName, that._storeName) && Objects.equals(_keyExpr, that._keyExpr); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), _storeName, _keyExpr); + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("VeniceConfig{"); + sb.append("_storeName='").append(_storeName).append('\''); + sb.append(", _keyExpr='").append(_keyExpr).append('\''); + sb.append(", _sourceName='").append(_sourceName).append('\''); + sb.append('}'); + return sb.toString(); + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/ConfigBuilder.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/ConfigBuilder.java new file mode 100644 index 000000000..50a467362 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/ConfigBuilder.java @@ -0,0 +1,174 @@ +package com.linkedin.feathr.core.configbuilder; + +import com.linkedin.feathr.core.config.consumer.JoinConfig; +import com.linkedin.feathr.core.config.producer.FeatureDefConfig; +import com.linkedin.feathr.core.configbuilder.typesafe.TypesafeConfigBuilder; +import com.linkedin.feathr.core.configdataprovider.ConfigDataProvider; +import java.io.Reader; +import java.net.URL; +import java.util.List; + + +/** + * Interface for building {@link com.linkedin.feathr.core.config.producer.FeatureDefConfig FeatureDefConfig} and + * {@link com.linkedin.feathr.core.config.consumer.JoinConfig JoinConfig}. Instance of a class implementing this + * interface can be obtained from the static factory method. + * + * @author djaising + */ +public interface ConfigBuilder { + + /** + * Factory method for getting an instance of ConfigBuilder + * @return ConfigBuilder object + */ + static ConfigBuilder get() { + return new TypesafeConfigBuilder(); + } + + /** + * Builds a {@link FeatureDefConfig} by specifying a {@link ConfigDataProvider} that provides FeatureDef config data + * @param provider ConfigDataProvider + * @return FeatureDefConfig + * @throws ConfigBuilderException + */ + FeatureDefConfig buildFeatureDefConfig(ConfigDataProvider provider); + + /** + * Builds several {@link FeatureDefConfig}s by specifying a {@link ConfigDataProvider} that provides FeatureDef config + * data. This method will not merge {@link FeatureDefConfig}s shared across different configs. Instead, it will construct + * individual configs for each resource provided within the {@link ConfigDataProvider}. + * @param provider ConfigDataProvider + * @return {@link List} + * @throws ConfigBuilderException + */ + List buildFeatureDefConfigList(ConfigDataProvider provider); + + /** + * Builds a {@link JoinConfig} by specifying a {@link ConfigDataProvider} that provides Join config data + * @param provider ConfigDataProvider + * @return JoinConfig + * @throws ConfigBuilderException + */ + JoinConfig buildJoinConfig(ConfigDataProvider provider); + + /* + * Deprecated methods for building Frame FeatureDef Config + */ + + /** + * Builds a single Frame FeatureDef Config from a list of configuration files referenced by URLs. + * + * @param urls List of {@link java.net.URL URLs} for configuration files + * @return {@link com.linkedin.feathr.core.config.producer.FeatureDefConfig FeatureDefConfig} config object + * @throws ConfigBuilderException + * @deprecated Use {@link #buildFeatureDefConfig(ConfigDataProvider)} where + * {@link com.linkedin.feathr.core.configdataprovider.UrlConfigDataProvider UrlConfigDataProvider} can be used as a + * {@link ConfigDataProvider} + */ + @Deprecated + FeatureDefConfig buildFeatureDefConfigFromUrls(List urls); + + /** + * Builds a Frame FeatureDef Config from a configuration file referenced by URL. + * + * @param url {@link java.net.URL URL} for the config file + * @return {@link com.linkedin.feathr.core.config.producer.FeatureDefConfig FeatureDefConfig} config object + * @throws ConfigBuilderException + * @deprecated Use {@link #buildFeatureDefConfig(ConfigDataProvider)} where + * {@link com.linkedin.feathr.core.configdataprovider.UrlConfigDataProvider UrlConfigDataProvider} can be used as a + * {@link ConfigDataProvider} + */ + @Deprecated + FeatureDefConfig buildFeatureDefConfig(URL url); + + /** + * Builds a single Frame FeatureDef Config from a list of configuration files on the classpath. + * @param resourceNames Names of the config files + * @return {@link com.linkedin.feathr.core.config.producer.FeatureDefConfig FeatureDefConfig} config object + * @throws ConfigBuilderException + * @deprecated Use {@link #buildFeatureDefConfig(ConfigDataProvider)} where + * {@link com.linkedin.feathr.core.configdataprovider.ResourceConfigDataProvider ResourceConfigDataProvider} can be + * used as a {@link ConfigDataProvider} + */ + @Deprecated + FeatureDefConfig buildFeatureDefConfig(List resourceNames); + + /** + * Builds a Frame FeatureDef Config from a configuration file on the classpath + * @param resourceName Name of the config file on the classpath + * @return {@link com.linkedin.feathr.core.config.producer.FeatureDefConfig FeatureDefConfig} config object + * @throws ConfigBuilderException + * @deprecated Use {@link #buildFeatureDefConfig(ConfigDataProvider)} where + * {@link com.linkedin.feathr.core.configdataprovider.ResourceConfigDataProvider ResourceConfigDataProvider} can be + * used as a {@link ConfigDataProvider} + */ + @Deprecated + FeatureDefConfig buildFeatureDefConfig(String resourceName); + + /** + * Builds a Frame FeatureDef Config from a configuration string + * @param configStr configuration expressed in a string + * @return {@link com.linkedin.feathr.core.config.producer.FeatureDefConfig FeatureDefConfig} config object + * @throws ConfigBuilderException + * @deprecated Use {@link #buildFeatureDefConfig(ConfigDataProvider)} where + * {@link com.linkedin.feathr.core.configdataprovider.StringConfigDataProvider StringConfigDataProvider} + * can be used as a {@link ConfigDataProvider} + */ + @Deprecated + FeatureDefConfig buildFeatureDefConfigFromString(String configStr); + + /** + * Builds a Frame FeatureDef Config from a java.io.Reader + * @param in A java.io.Reader instance + * @return {@link com.linkedin.feathr.core.config.producer.FeatureDefConfig FeatureDefConfig} config object + * @throws ConfigBuilderException + * @deprecated Use {@link #buildFeatureDefConfig(ConfigDataProvider)} where + * {@link com.linkedin.feathr.core.configdataprovider.ReaderConfigDataProvider ReaderConfigDataProvider} + * can be used as a {@link ConfigDataProvider} + */ + @Deprecated + FeatureDefConfig buildFeatureDefConfig(Reader in); + + /** + * Builds a Frame FeatureDef Config from a config manifest specified as a resource + * @param manifestResourceName + * @return {@link com.linkedin.feathr.core.config.producer.FeatureDefConfig FeatureDefConfig} config object + * @throws ConfigBuilderException + * @deprecated Use {@link #buildFeatureDefConfig(ConfigDataProvider)} where + * {@link com.linkedin.feathr.core.configdataprovider.ManifestConfigDataProvider ManifestConfigDataProvider} + * can be used as a {@link ConfigDataProvider} + */ + @Deprecated + FeatureDefConfig buildFeatureDefConfigFromManifest(String manifestResourceName); + + + /* + * Deprecated methods for building Frame Join Config + */ + + /** + * Build a Join Config from a configuration accessed via a URL + * @param url A java.net.URL + * @return {@link com.linkedin.feathr.core.config.consumer.JoinConfig JoinConfig} config object + * @throws ConfigBuilderException + * @deprecated Use {@link #buildJoinConfig(ConfigDataProvider)} where + * {@link com.linkedin.feathr.core.configdataprovider.UrlConfigDataProvider UrlConfigDataProvider} can be used as + * a {@link ConfigDataProvider} + */ + @Deprecated + JoinConfig buildJoinConfig(URL url); + + /** + * Build a Join Config from a configuration file on the classpath + * @param resourceName Name of the configuration file expressed as a resource + * @return {@link com.linkedin.feathr.core.config.consumer.JoinConfig JoinConfig} config object + * @throws ConfigBuilderException + * @deprecated Use {@link #buildJoinConfig(ConfigDataProvider)} where + * {@link com.linkedin.feathr.core.configdataprovider.ResourceConfigDataProvider ResourceConfigDataProvider} can be + * used as a {@link ConfigDataProvider} + */ + @Deprecated + JoinConfig buildJoinConfig(String resourceName); +} + diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/ConfigBuilderException.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/ConfigBuilderException.java new file mode 100644 index 000000000..f27fad15a --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/ConfigBuilderException.java @@ -0,0 +1,14 @@ +package com.linkedin.feathr.core.configbuilder; + +/** + * When an error is encountered during config processing, this exception is thrown + */ +public class ConfigBuilderException extends RuntimeException { + public ConfigBuilderException(String message) { + super(message); + } + + public ConfigBuilderException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/FrameConfigFileChecker.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/FrameConfigFileChecker.java new file mode 100644 index 000000000..a8f3e10b7 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/FrameConfigFileChecker.java @@ -0,0 +1,40 @@ +package com.linkedin.feathr.core.configbuilder.typesafe; + +import com.linkedin.feathr.core.configdataprovider.ConfigDataProvider; +import com.linkedin.feathr.core.configdataprovider.UrlConfigDataProvider; +import com.linkedin.feathr.core.config.ConfigType; +import com.linkedin.feathr.core.config.producer.FeatureDefConfig; +import com.linkedin.feathr.core.configbuilder.ConfigBuilderException; +import com.typesafe.config.Config; +import java.net.URL; +import java.util.Objects; + + +/** + * Utility class to check if a config file is a Frame config file. + */ +public class FrameConfigFileChecker { + private FrameConfigFileChecker() { + } + + /** + * Checks if a config file(file with conf extension) is a Frame config file or not. + * A config file is a Frame feature config file if anchors, sources or derivations are present in the config + * section. Metadata config files are not Frame feature config file. + * A Frame config file can still contain invalid syntax. This is mainly used to collect all the Frame configs. + */ + public static boolean isConfigFile(URL url) { + try (ConfigDataProvider cdp = new UrlConfigDataProvider(url)) { + Objects.requireNonNull(cdp, "ConfigDataProvider object can't be null"); + + TypesafeConfigBuilder builder = new TypesafeConfigBuilder(); + + Config config = builder.buildTypesafeConfig(ConfigType.FeatureDef, cdp); + + return config.hasPath(FeatureDefConfig.ANCHORS) || config.hasPath(FeatureDefConfig.DERIVATIONS) || config.hasPath( + FeatureDefConfig.SOURCES); + } catch (Exception e) { + throw new ConfigBuilderException("Error in building config object", e); + } + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/TypesafeConfigBuilder.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/TypesafeConfigBuilder.java new file mode 100644 index 000000000..61023e149 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/TypesafeConfigBuilder.java @@ -0,0 +1,345 @@ +package com.linkedin.feathr.core.configbuilder.typesafe; + +import com.linkedin.feathr.core.config.ConfigType; +import com.linkedin.feathr.core.config.consumer.JoinConfig; +import com.linkedin.feathr.core.config.producer.FeatureDefConfig; +import com.linkedin.feathr.core.configbuilder.ConfigBuilder; +import com.linkedin.feathr.core.configbuilder.ConfigBuilderException; +import com.linkedin.feathr.core.configbuilder.typesafe.consumer.JoinConfigBuilder; +import com.linkedin.feathr.core.configbuilder.typesafe.producer.FeatureDefConfigBuilder; +import com.linkedin.feathr.core.configdataprovider.ConfigDataProvider; +import com.linkedin.feathr.core.configdataprovider.ConfigDataProviderException; +import com.linkedin.feathr.core.configdataprovider.ManifestConfigDataProvider; +import com.linkedin.feathr.core.configdataprovider.ReaderConfigDataProvider; +import com.linkedin.feathr.core.configdataprovider.ResourceConfigDataProvider; +import com.linkedin.feathr.core.configdataprovider.StringConfigDataProvider; +import com.linkedin.feathr.core.configdataprovider.UrlConfigDataProvider; +import com.linkedin.feathr.core.configvalidator.ValidationResult; +import com.linkedin.feathr.core.configvalidator.typesafe.TypesafeConfigValidator; +import com.typesafe.config.Config; +import com.typesafe.config.ConfigFactory; +import com.typesafe.config.ConfigParseOptions; +import com.typesafe.config.ConfigRenderOptions; +import com.typesafe.config.ConfigSyntax; +import java.io.Reader; +import java.net.URL; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static com.linkedin.feathr.core.config.ConfigType.*; +import static com.linkedin.feathr.core.configvalidator.ValidationStatus.*; + + +/** + * Builds Frame Feature Config and Frame Join Config using the Typesafe (Lightbend) Config library. + * + * @author djaising + */ +public class TypesafeConfigBuilder implements ConfigBuilder { + + private final static Logger logger = LoggerFactory.getLogger(TypesafeConfigBuilder.class); + + // Used while parsing a config string in HOCON format + private ConfigParseOptions _parseOptions; + + // Used when rendering the parsed config to JSON string (which is then used in validation) + private ConfigRenderOptions _renderOptions; + + + /** + * Default constructor. Builds parsing and rendering options. + */ + public TypesafeConfigBuilder() { + _parseOptions = ConfigParseOptions.defaults() + .setSyntax(ConfigSyntax.CONF) // HOCON document + .setAllowMissing(false); + + _renderOptions = ConfigRenderOptions.defaults() + .setComments(false) + .setOriginComments(false) + .setFormatted(true) + .setJson(true); + } + + /* + * Methods for building FeatureDef Config + */ + + + @Override + public FeatureDefConfig buildFeatureDefConfig(ConfigDataProvider configDataProvider) { + Objects.requireNonNull(configDataProvider, "ConfigDataProvider object can't be null"); + + FeatureDefConfig configObj; + + try { + List readers = configDataProvider.getConfigDataReaders(); + configObj = doBuildFeatureDefConfig(readers); + } catch (Exception e) { + throw new ConfigBuilderException("Error in building FeatureDefConfig object", e); + } + logger.info("Built FeatureDefConfig from " + configDataProvider.getConfigDataInfo()); + + return configObj; + } + + @Override + public List buildFeatureDefConfigList(ConfigDataProvider configDataProvider) { + Objects.requireNonNull(configDataProvider, "ConfigDataProvider object can't be null"); + List featureDefConfigList = new ArrayList<>(); + + try { + List readers = configDataProvider.getConfigDataReaders(); + for (Reader reader : readers) { + List singletonReaderList = Collections.singletonList(reader); + FeatureDefConfig configObj = doBuildFeatureDefConfig(singletonReaderList); + featureDefConfigList.add(configObj); + } + } catch (ConfigBuilderException e) { + throw new ConfigBuilderException("Error in building FeatureDefConfig object", e); + } + if (featureDefConfigList.isEmpty()) { + logger.warn("No FeatureDefConfigs were built after entering buildFeatureDefConfigList(). ConfigDataProvider Info:" + + configDataProvider.getConfigDataInfo()); + } else { + logger.info("Built FeatureDefConfig from " + configDataProvider.getConfigDataInfo()); + } + return featureDefConfigList; + } + + + @Deprecated + @Override + public FeatureDefConfig buildFeatureDefConfigFromUrls(List urls) { + /* + * Delegate the config building to buildFeatureDefConfig(ConfigDataProvider configDataProvider) method + */ + try (ConfigDataProvider cdp = new UrlConfigDataProvider(urls)) { + return buildFeatureDefConfig(cdp); + } catch (Exception e) { + throw new ConfigBuilderException("Error in building FeatureDefConfig object", e); + } + } + + @Deprecated + @Override + public FeatureDefConfig buildFeatureDefConfig(URL url) { + return buildFeatureDefConfigFromUrls(Collections.singletonList(url)); + } + + @Deprecated + @Override + public FeatureDefConfig buildFeatureDefConfig(List resourceNames) { + /* + * Delegate the config building to buildFeatureDefConfig(ConfigDataProvider configDataProvider) method + */ + try (ConfigDataProvider cdp = new ResourceConfigDataProvider(resourceNames)) { + return buildFeatureDefConfig(cdp); + } catch (Exception e) { + throw new ConfigBuilderException("Error in building FeatureDefConfig object", e); + } + } + + @Deprecated + @Override + public FeatureDefConfig buildFeatureDefConfig(String resourceName) { + return buildFeatureDefConfig(Collections.singletonList(resourceName)); + } + + @Deprecated + @Override + public FeatureDefConfig buildFeatureDefConfigFromString(String configStr) { + /* + * Delegate the config building to buildFeatureDefConfig(ConfigDataProvider configDataProvider) method + */ + try (ConfigDataProvider cdp = new StringConfigDataProvider(configStr)) { + return buildFeatureDefConfig(cdp); + } catch (Exception e) { + throw new ConfigBuilderException("Error in building FeatureDefConfig object", e); + } + } + + @Deprecated + @Override + public FeatureDefConfig buildFeatureDefConfig(Reader reader) { + /* + * Delegate the config building to buildFeatureDefConfig(ConfigDataProvider configDataProvider) method + */ + try (ConfigDataProvider cdp = new ReaderConfigDataProvider(reader)) { + return buildFeatureDefConfig(cdp); + } catch (Exception e) { + throw new ConfigBuilderException("Error in building FeatureDefConfig object", e); + } + } + + /* + * Builds the FeatureDefConfig object from a manifest file that is specified as a resource. + * An example file is shown below: + * + * manifest: [ + * { + * jar: local + * conf: [config/online/feature-prod.conf] + * }, + * { + * jar: frame-feature-waterloo-online-1.1.4.jar + * conf: [config/online/prod/feature-prod.conf] + * } + * ] + */ + @Deprecated + @Override + public FeatureDefConfig buildFeatureDefConfigFromManifest(String manifestResourceName) { + /* + * Delegate the config building to buildFeatureDefConfig(ConfigDataProvider configDataProvider) method + */ + try (ConfigDataProvider cdp = new ManifestConfigDataProvider(manifestResourceName)) { + return buildFeatureDefConfig(cdp); + } catch (Exception e) { + throw new ConfigBuilderException("Error in building FeatureDefConfig object from manifest resource " + + manifestResourceName, e); + } + } + + /* + * Methods for building Frame Join Config + */ + + @Override + public JoinConfig buildJoinConfig(ConfigDataProvider configDataProvider) { + Objects.requireNonNull(configDataProvider, "ConfigDataProvider object can't be null"); + + JoinConfig configObj; + + try { + List readers = configDataProvider.getConfigDataReaders(); + if (readers.size() != 1) { + throw new ConfigDataProviderException("Expected number of Join configs = 1, found " + readers.size()); + } + configObj = doBuildJoinConfig(readers.get(0)); + } catch (Exception e) { + throw new ConfigBuilderException("Error in building JoinConfig object", e); + } + logger.info("Built JoinConfig from " + configDataProvider.getConfigDataInfo()); + + return configObj; + } + + @Deprecated + @Override + public JoinConfig buildJoinConfig(URL url) { + /* + * Delegate the config building to buildJoinConfig(ConfigDataProvider configDataProvider) method + */ + try (ConfigDataProvider cdp = new UrlConfigDataProvider(url)) { + return buildJoinConfig(cdp); + } catch (Exception e) { + throw new ConfigBuilderException("Error in building JoinConfig object from URL " + url, e); + } + } + + @Deprecated + @Override + public JoinConfig buildJoinConfig(String resourceName) { + /* + * Delegate the config building to buildJoinConfig(ConfigDataProvider configDataProvider) method + */ + try (ConfigDataProvider cdp = new ResourceConfigDataProvider(resourceName)) { + return buildJoinConfig(cdp); + } catch (Exception e) { + throw new ConfigBuilderException("Error in building JoinConfig object from resource " + resourceName, e); + } + } + + /* + * This method is intended to be used internally by other packages, for example, by TypesafeConfigValidator in + * configvalidator package. + */ + public Config buildTypesafeConfig(ConfigType configType, ConfigDataProvider configDataProvider) { + List readers = configDataProvider.getConfigDataReaders(); + + Config config; + + switch (configType) { + case FeatureDef: + config = buildMergedConfig(readers); + break; + + case Join: + case Presentation: + if (readers.size() != 1) { + throw new ConfigDataProviderException("Expected number of " + configType + " configs = 1, found " + readers.size()); + } + config = ConfigFactory.parseReader(readers.get(0), _parseOptions); + break; + + default: + throw new ConfigBuilderException("Unsupported config type " + configType); + } + logger.debug(configType + " config: \n" + config.root().render(_renderOptions.setJson(false))); + + return config; + } + + private FeatureDefConfig doBuildFeatureDefConfig(List readers) { + Config mergedConfig = buildMergedConfig(readers); + logger.debug("FeatureDef config: \n" + mergedConfig.root().render(_renderOptions.setJson(false))); + + validate(mergedConfig, FeatureDef); + + return FeatureDefConfigBuilder.build(mergedConfig); + } + + private Config buildMergedConfig(List readers) { + /* + * Merge configs into a single config. Objects with the same key are merged to form a single object, duplicate + * values are merged according to the 'left' config value overriding 'the right' config value. If the keys don't + * overlap, they are retained in the merged config with their respective values. + * For more details and examples, see the relevant sections in HOCON spec: + * Duplicate keys and object merging: + * https://github.com/lightbend/config/blob/master/HOCON.md#duplicate-keys-and-object-merging + * Config object merging and file merging: + * https://github.com/lightbend/config/blob/master/HOCON.md#config-object-merging-and-file-merging + */ + Config emptyConfig = ConfigFactory.empty(); + + // TODO: Need to decide when to do substitution resolution. After each file parse, or after the merge. + return readers.stream() + .map(r -> ConfigFactory.parseReader(r, _parseOptions)) + .map(Config::resolve) + .reduce(emptyConfig, Config::withFallback); + } + + private JoinConfig doBuildJoinConfig(Reader reader) { + Config config = ConfigFactory.parseReader(reader, _parseOptions); + logger.debug("Join config: \n" + config.root().render(_renderOptions.setJson(false))); + + validate(config, Join); + + return JoinConfigBuilder.build(config); + } + + /* + * Validates the syntax of the config. Delegates the task to a validator. + */ + private void validate(Config config, ConfigType configType) { + TypesafeConfigValidator validator = new TypesafeConfigValidator(); + + ValidationResult validationResult = validator.validateSyntax(configType, config); + logger.debug("Performed syntax validation for " + configType + " config. Result: " + validationResult); + + if (validationResult.getValidationStatus() == INVALID) { + String errMsg = validationResult.getDetails().orElse(configType + " config syntax validation failed"); + + if (validationResult.getCause().isPresent()) { + throw new ConfigBuilderException(errMsg, validationResult.getCause().get()); + } else { + throw new ConfigBuilderException(errMsg); + } + } + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/consumer/AbsoluteTimeRangeConfigBuilder.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/consumer/AbsoluteTimeRangeConfigBuilder.java new file mode 100644 index 000000000..3570f2887 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/consumer/AbsoluteTimeRangeConfigBuilder.java @@ -0,0 +1,56 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.consumer; + +import com.linkedin.feathr.core.config.consumer.AbsoluteTimeRangeConfig; +import com.linkedin.feathr.core.configbuilder.ConfigBuilderException; +import com.linkedin.feathr.core.utils.ConfigUtils; +import com.typesafe.config.Config; +import org.apache.log4j.Logger; + +import static com.linkedin.feathr.core.config.consumer.AbsoluteTimeRangeConfig.*; + + +/** + * Build the [[AbsoluteTimeRangeConfig]] class object. + * absoluteTimeRange: { + * startTime: 20200809 + * endTime: 20200811 + * timeFormat: yyyyMMdd + * } + * @author rkashyap + */ +public class AbsoluteTimeRangeConfigBuilder { + private final static Logger logger = Logger.getLogger(AbsoluteTimeRangeConfigBuilder.class); + + private AbsoluteTimeRangeConfigBuilder() { + } + + public static AbsoluteTimeRangeConfig build(Config absoluteTimeRangeConfig) { + String startTime = absoluteTimeRangeConfig.hasPath(START_TIME) ? absoluteTimeRangeConfig.getString(START_TIME) : null; + + if (startTime == null) { + throw new ConfigBuilderException(String.format("startTime is a required parameter in absoluteTimeRage config object %s", absoluteTimeRangeConfig)); + } + + String endTime = absoluteTimeRangeConfig.hasPath(END_TIME) ? absoluteTimeRangeConfig.getString(END_TIME) : null; + + if (endTime == null) { + throw new ConfigBuilderException(String.format("endTime is a required parameter in absoluteTimeRage config object %s", absoluteTimeRangeConfig)); + } + + String timeFormat = absoluteTimeRangeConfig.hasPath(TIME_FORMAT) ? absoluteTimeRangeConfig.getString(TIME_FORMAT) : null; + + if (timeFormat == null) { + throw new ConfigBuilderException(String.format("timeFormat is a required parameter in absoluteTimeRage config object %s", absoluteTimeRangeConfig)); + } + + // We only need to validate if the startTime/endTime corresponds to the given format, the actual conversion is done if frame offline. + ConfigUtils.validateTimestampPatternWithEpoch(START_TIME, startTime, timeFormat); + ConfigUtils.validateTimestampPatternWithEpoch(END_TIME, endTime, timeFormat); + + AbsoluteTimeRangeConfig configObj = new AbsoluteTimeRangeConfig(startTime, endTime, timeFormat); + + logger.debug("Built AbsoluteTimeRangeConfig object"); + + return configObj; + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/consumer/FeatureBagConfigBuilder.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/consumer/FeatureBagConfigBuilder.java new file mode 100644 index 000000000..6011c5a73 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/consumer/FeatureBagConfigBuilder.java @@ -0,0 +1,29 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.consumer; + +import com.linkedin.feathr.core.config.consumer.FeatureBagConfig; +import com.linkedin.feathr.core.config.consumer.KeyedFeatures; +import com.typesafe.config.Config; +import java.util.List; +import java.util.stream.Collectors; +import org.apache.log4j.Logger; + + +/** + * Builds FeatureBagConfig objects. These objects specify the features to be fetched. + */ +class FeatureBagConfigBuilder { + private final static Logger logger = Logger.getLogger(FeatureBagConfigBuilder.class); + + private FeatureBagConfigBuilder() { + } + + public static FeatureBagConfig build(List featuresConfigList) { + List keyedFeatures = featuresConfigList.stream(). + map(KeyedFeaturesConfigBuilder::build).collect(Collectors.toList()); + + FeatureBagConfig configObj = new FeatureBagConfig(keyedFeatures); + logger.debug("Built FeatureBagConfig object"); + + return configObj; + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/consumer/JoinConfigBuilder.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/consumer/JoinConfigBuilder.java new file mode 100644 index 000000000..5085edf25 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/consumer/JoinConfigBuilder.java @@ -0,0 +1,59 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.consumer; + +import com.linkedin.feathr.core.config.consumer.FeatureBagConfig; +import com.linkedin.feathr.core.config.consumer.JoinConfig; +import com.linkedin.feathr.core.config.consumer.SettingsConfig; +import com.typesafe.config.Config; +import com.typesafe.config.ConfigObject; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.log4j.Logger; + +import static com.linkedin.feathr.core.config.consumer.JoinConfig.*; +import static com.linkedin.feathr.core.utils.Utils.*; + + +/** + * Builds a JoinConfig object. It does so by delegating to child builders. + */ +public class JoinConfigBuilder { + private final static Logger logger = Logger.getLogger(JoinConfigBuilder.class); + + private JoinConfigBuilder() { + } + + public static JoinConfig build(Config fullConfig) { + SettingsConfig settings = null; + if (fullConfig.hasPath(SETTINGS)) { + Config config = fullConfig.getConfig(SETTINGS); + settings = SettingsConfigBuilder.build(config); + } + + Map featureBags = new HashMap<>(); + ConfigObject rootConfigObj = fullConfig.root(); + + // Extract all feature bag names by excluding the 'settings' field name + Set featureBagNameSet = rootConfigObj.keySet().stream().filter(fbn -> !fbn.equals(SETTINGS)).collect( + Collectors.toSet()); + + // Iterate over each feature bag name to build feature bag config objects, and insert them into a map + for (String featureBagName : featureBagNameSet) { + List featuresConfigList = fullConfig.getConfigList(quote(featureBagName)); + FeatureBagConfig featureBagConfig = FeatureBagConfigBuilder.build(featuresConfigList); + featureBags.put(featureBagName, featureBagConfig); + } + + /* + * TODO: Semantic validation + * validate that the feature names refer to valid feature names in the FeatureDef config. + */ + + JoinConfig configObj = new JoinConfig(settings, featureBags); + logger.debug("Built JoinConfig object"); + + return configObj; + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/consumer/JoinTimeSettingsConfigBuilder.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/consumer/JoinTimeSettingsConfigBuilder.java new file mode 100644 index 000000000..11c81d705 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/consumer/JoinTimeSettingsConfigBuilder.java @@ -0,0 +1,75 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.consumer; + +import com.linkedin.feathr.core.config.consumer.JoinTimeSettingsConfig; +import com.linkedin.feathr.core.config.consumer.TimestampColumnConfig; +import com.linkedin.feathr.core.configbuilder.ConfigBuilderException; +import com.typesafe.config.Config; +import java.time.Duration; +import org.apache.log4j.Logger; + +import static com.linkedin.feathr.core.config.consumer.JoinTimeSettingsConfig.*; + + +/** + * Builds the [[JoinTimeSettingsConfig]] class + * joinTimeSettings: { + * timestampColumn: { + * def: timestamp + * format: yyyyMMdd + * } + * simulateTimeDelay: 2d + * } + * + * (or) + * + * joinTimeSettings: { + * useLatestFeatureData: true + * } + * @author rkashyap + */ +class JoinTimeSettingsConfigBuilder { + private final static Logger logger = Logger.getLogger(JoinTimeSettingsConfigBuilder.class); + + private JoinTimeSettingsConfigBuilder() { + } + + public static JoinTimeSettingsConfig build(Config joinTimSettingsConfig) { + TimestampColumnConfig timestampColumn = joinTimSettingsConfig.hasPath(TIMESTAMP_COLUMN) + ? TimestampColumnConfigBuilder.build(joinTimSettingsConfig.getConfig(TIMESTAMP_COLUMN)) + : null; + + Duration simulateTimeDelay = joinTimSettingsConfig.hasPath(SIMULATE_TIME_DELAY) + ? joinTimSettingsConfig.getDuration(SIMULATE_TIME_DELAY) + : null; + + Boolean useLatestFeatureData = joinTimSettingsConfig.hasPath(USE_LATEST_FEATURE_DATA) + ? joinTimSettingsConfig.getBoolean(USE_LATEST_FEATURE_DATA) + : null; + + if (timestampColumn == null && useLatestFeatureData == null) { + StringBuilder messageBuilder = new StringBuilder(); + messageBuilder.append("One of the fields: ").append(TIMESTAMP_COLUMN).append(" or ") + .append(USE_LATEST_FEATURE_DATA).append("is required but both are missing"); + throw new ConfigBuilderException(messageBuilder.toString()); + } + + if (useLatestFeatureData != null && useLatestFeatureData) { + if (timestampColumn != null || simulateTimeDelay != null) { + StringBuilder messageBuilder = new StringBuilder(); + messageBuilder.append("When ").append(USE_LATEST_FEATURE_DATA).append(" is set to true, ") + .append("None of the following fields can exist: ").append(TIMESTAMP_COLUMN) + .append(", ").append(SIMULATE_TIME_DELAY).append("."); + throw new ConfigBuilderException(messageBuilder.toString()); + } + } + + JoinTimeSettingsConfig configObj = + new JoinTimeSettingsConfig(timestampColumn, simulateTimeDelay, useLatestFeatureData); + + + + logger.debug("Built TimeWindowJoinConfig object"); + + return configObj; + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/consumer/KeyedFeaturesConfigBuilder.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/consumer/KeyedFeaturesConfigBuilder.java new file mode 100644 index 000000000..ba266174d --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/consumer/KeyedFeaturesConfigBuilder.java @@ -0,0 +1,88 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.consumer; + +import com.linkedin.feathr.core.config.consumer.DateTimeRange; +import com.linkedin.feathr.core.config.consumer.KeyedFeatures; +import com.linkedin.feathr.core.configbuilder.ConfigBuilderException; +import com.linkedin.feathr.core.utils.Utils; +import com.typesafe.config.Config; +import com.typesafe.config.ConfigValueType; +import java.time.Duration; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.Collections; +import java.util.List; +import org.apache.log4j.Logger; + +import static com.linkedin.feathr.core.config.consumer.KeyedFeatures.*; + + +/** + * Builds the KeyedFeatures config object + */ +class KeyedFeaturesConfigBuilder { + private final static Logger logger = Logger.getLogger(KeyedFeaturesConfigBuilder.class); + + private static DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern(TIMESTAMP_FORMAT); + + private KeyedFeaturesConfigBuilder() { + } + + public static KeyedFeatures build(Config featuresConfig) { + List key = getKey(featuresConfig); + + List features = featuresConfig.getStringList(FEATURE_LIST); + + DateTimeRange dates = getDates(featuresConfig); + + Duration overrideTimeDelay = featuresConfig.hasPath(OVERRIDE_TIME_DELAY) + ? featuresConfig.getDuration(OVERRIDE_TIME_DELAY) + : null; + + return new KeyedFeatures(key, features, dates, overrideTimeDelay); + } + + private static List getKey(Config config) { + ConfigValueType keyValueType = config.getValue(KEY).valueType(); + switch (keyValueType) { + case STRING: + return Collections.singletonList(config.getString(KEY)); + + case LIST: + return config.getStringList(KEY); + + default: + throw new ConfigBuilderException("Expected key type String or List[String], got " + keyValueType); + } + } + + private static DateTimeRange getDates(Config config) { + DateTimeRange dateTimeParams; + + if (config.hasPath(START_DATE)) { + String startDateStr = config.getString(START_DATE); + String endDateStr = config.getString(END_DATE); + + LocalDateTime startDate = LocalDate.parse(startDateStr, dateTimeFormatter).atStartOfDay(); + LocalDateTime endDate = LocalDate.parse(endDateStr, dateTimeFormatter).atStartOfDay(); + + dateTimeParams = new DateTimeRange(startDate, endDate); + } else if (config.hasPath(DATE_OFFSET)) { + int dateOffset = config.getInt(DATE_OFFSET); + int numDays = config.getInt(NUM_DAYS); + + // TODO: This will be checked during validation phase; we can remove it when implemented + String messageStr = String.format("Expected %s > 0 && %s > 0 && %s < %s; got %s = %d, %s = %d", + DATE_OFFSET, NUM_DAYS, NUM_DAYS, DATE_OFFSET, DATE_OFFSET, dateOffset, NUM_DAYS, numDays); + Utils.require(numDays > 0 && numDays < dateOffset, messageStr); + + LocalDateTime startDate = LocalDate.now().minusDays(dateOffset).atStartOfDay(); + LocalDateTime endDate = startDate.plusDays(numDays); + + dateTimeParams = new DateTimeRange(startDate, endDate); + } else { + dateTimeParams = null; + } + return dateTimeParams; + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/consumer/ObservationDataTimeSettingsConfigBuilder.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/consumer/ObservationDataTimeSettingsConfigBuilder.java new file mode 100644 index 000000000..97aaae4e9 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/consumer/ObservationDataTimeSettingsConfigBuilder.java @@ -0,0 +1,64 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.consumer; + +import com.linkedin.feathr.core.config.consumer.AbsoluteTimeRangeConfig; +import com.linkedin.feathr.core.config.consumer.ObservationDataTimeSettingsConfig; +import com.linkedin.feathr.core.config.consumer.RelativeTimeRangeConfig; +import com.linkedin.feathr.core.configbuilder.ConfigBuilderException; +import com.typesafe.config.Config; +import org.apache.log4j.Logger; + +import static com.linkedin.feathr.core.config.consumer.ObservationDataTimeSettingsConfig.*; + + +/** + * Builds the [[ObservationDataTimeSettingsConfig]] object + * + * observationDataTimeSettings: { + * absoluteTimeRange: { + * startTime: 20200809 + * endTime: 20200810 + * timeFormat: yyyyMMdd + * } + * (or) + * relativeTimeRange: { + * offset: 1d + * window: 1d + * } + * } + * @author rkashyap + */ +public class ObservationDataTimeSettingsConfigBuilder { + private final static Logger logger = Logger.getLogger(ObservationDataTimeSettingsConfigBuilder.class); + + private ObservationDataTimeSettingsConfigBuilder() { + } + + public static ObservationDataTimeSettingsConfig build(Config observationDataTimeSettings) { + + AbsoluteTimeRangeConfig absoluteTimeRangeConfig = observationDataTimeSettings.hasPath(ABSOLUTE_TIME_RANGE) + ? AbsoluteTimeRangeConfigBuilder.build(observationDataTimeSettings.getConfig(ABSOLUTE_TIME_RANGE)) + : null; + + RelativeTimeRangeConfig relativeTimeRangeConfig = observationDataTimeSettings.hasPath(RELATIVE_TIME_RANGE) + ? RelativeTimeRangeConfigBuilder.build(observationDataTimeSettings.getConfig(RELATIVE_TIME_RANGE)) + : null; + + if (absoluteTimeRangeConfig != null && relativeTimeRangeConfig != null) { + throw new ConfigBuilderException(String.format("Please provide only one of the absoluteTimeRange or RelativeTimeRange. Currently, you" + + "have provided both the configs:- AbsoluteTimeRange: %s , RelativeTimeRange: %s", absoluteTimeRangeConfig.toString(), + relativeTimeRangeConfig.toString())); + } + + if (absoluteTimeRangeConfig == null && relativeTimeRangeConfig == null) { + throw new ConfigBuilderException(String.format("Please provide atleast one of absoluteTimeRange or RelativeTimeRange. If you do not" + + "intend to filter the observation data, please remove the section observationDataTimeSettings from the settings section.", + absoluteTimeRangeConfig.toString(), relativeTimeRangeConfig.toString())); + } + + ObservationDataTimeSettingsConfig configObj = + new ObservationDataTimeSettingsConfig(absoluteTimeRangeConfig, relativeTimeRangeConfig); + logger.debug("Built Observation data time settings object"); + + return configObj; + } +} \ No newline at end of file diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/consumer/RelativeTimeRangeConfigBuilder.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/consumer/RelativeTimeRangeConfigBuilder.java new file mode 100644 index 000000000..3a3909eca --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/consumer/RelativeTimeRangeConfigBuilder.java @@ -0,0 +1,40 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.consumer; + +import com.linkedin.feathr.core.config.consumer.RelativeTimeRangeConfig; +import com.linkedin.feathr.core.configbuilder.ConfigBuilderException; +import com.typesafe.config.Config; +import java.time.Duration; +import org.apache.log4j.Logger; + +import static com.linkedin.feathr.core.config.consumer.RelativeTimeRangeConfig.*; + + +/** + * Build the [[RelativeTimeRangeConfig]] class. + * relativeTimeRange: { + * offset: 2d + * window: 3d + * } + */ +public class RelativeTimeRangeConfigBuilder { + private final static Logger logger = Logger.getLogger(RelativeTimeRangeConfigBuilder.class); + + private RelativeTimeRangeConfigBuilder() { + } + + public static RelativeTimeRangeConfig build(Config relativeTimeRangeConfig) { + Duration window = relativeTimeRangeConfig.hasPath(WINDOW) ? relativeTimeRangeConfig.getDuration(WINDOW) : null; + + if (window == null) { + throw new ConfigBuilderException("window is a required parameter in relativeTimeRange config object"); + } + + Duration offset = relativeTimeRangeConfig.hasPath(OFFSET) ? relativeTimeRangeConfig.getDuration(OFFSET) : null; + + RelativeTimeRangeConfig configObj = new RelativeTimeRangeConfig(window, offset); + + logger.debug("Built AbsoluteTimeRangeConfig object"); + + return configObj; + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/consumer/SettingsConfigBuilder.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/consumer/SettingsConfigBuilder.java new file mode 100644 index 000000000..794ca64b0 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/consumer/SettingsConfigBuilder.java @@ -0,0 +1,35 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.consumer; + +import com.linkedin.feathr.core.config.consumer.JoinTimeSettingsConfig; +import com.linkedin.feathr.core.config.consumer.ObservationDataTimeSettingsConfig; +import com.linkedin.feathr.core.config.consumer.SettingsConfig; +import com.typesafe.config.Config; +import org.apache.log4j.Logger; + +import static com.linkedin.feathr.core.config.consumer.SettingsConfig.*; + + +/** + * Builds a {@link SettingsConfig} object + */ +class SettingsConfigBuilder { + private final static Logger logger = Logger.getLogger(SettingsConfigBuilder.class); + + private SettingsConfigBuilder() { + } + + public static SettingsConfig build(Config settingsConfig) { + SettingsConfig configObj; + ObservationDataTimeSettingsConfig observationDataTimeSettingsConfig = settingsConfig.hasPath(OBSERVATION_DATA_TIME_SETTINGS) + ? ObservationDataTimeSettingsConfigBuilder.build(settingsConfig.getConfig(OBSERVATION_DATA_TIME_SETTINGS)) + : null; + + JoinTimeSettingsConfig joinTimeSettingsConfig = settingsConfig.hasPath(JOIN_TIME_SETTINGS) + ? JoinTimeSettingsConfigBuilder.build(settingsConfig.getConfig(JOIN_TIME_SETTINGS)) + : null; + + configObj = new SettingsConfig(observationDataTimeSettingsConfig, joinTimeSettingsConfig); + + return configObj; + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/consumer/TimestampColumnConfigBuilder.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/consumer/TimestampColumnConfigBuilder.java new file mode 100644 index 000000000..31aec05ee --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/consumer/TimestampColumnConfigBuilder.java @@ -0,0 +1,43 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.consumer; + +import com.linkedin.feathr.core.config.consumer.TimestampColumnConfig; +import com.linkedin.feathr.core.configbuilder.ConfigBuilderException; +import com.typesafe.config.Config; +import org.apache.log4j.Logger; + +import static com.linkedin.feathr.core.config.consumer.TimestampColumnConfig.*; + +/** + * Build the TimestampColumn config object. + * timestampColumn: { + * def: timestamp + * format: yyyyMMdd + * } + * @author rkashyap + */ +public class TimestampColumnConfigBuilder { + private final static Logger logger = Logger.getLogger(TimestampColumnConfigBuilder.class); + + private TimestampColumnConfigBuilder() { + } + + public static TimestampColumnConfig build(Config timestampColumnConfig) { + String name = timestampColumnConfig.hasPath(NAME) ? timestampColumnConfig.getString(NAME) : null; + + if (name == null) { + throw new ConfigBuilderException(String.format("name is a required parameter in timestamp config object %s", timestampColumnConfig.toString())); + } + + String format = timestampColumnConfig.hasPath(FORMAT) ? timestampColumnConfig.getString(FORMAT) : null; + + if (format == null) { + throw new ConfigBuilderException(String.format("format is a required parameter in absoluteTimeRage config object %s", timestampColumnConfig.toString())); + } + + TimestampColumnConfig configObj = new TimestampColumnConfig(name, format); + + logger.debug("Built Timestamp object"); + + return configObj; + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/generation/DateTimeConfigBuilder.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/generation/DateTimeConfigBuilder.java new file mode 100644 index 000000000..d37ba8da2 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/generation/DateTimeConfigBuilder.java @@ -0,0 +1,46 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.generation; + +import com.linkedin.feathr.core.config.common.DateTimeConfig; +import com.linkedin.feathr.core.utils.ConfigUtils; +import com.typesafe.config.Config; +import java.time.Duration; +import java.time.temporal.ChronoUnit; +import java.util.TimeZone; +import org.apache.log4j.Logger; + + +/** + * Build DateTimeConfig from config + */ +public class DateTimeConfigBuilder { + + private final static Logger logger = Logger.getLogger(DateTimeConfigBuilder.class); + private static final String DEFAULT_TIME_ZONE = "America/Los_Angeles"; + private static final String END_TIME = "endTime"; + private static final String END_TIME_FORMAT = "endTimeFormat"; + private static final String TIME_RESOLUTION = "resolution"; + private static final String OFFSET = "offset"; + private static final String LENGTH = "length"; + private static final String TIME_ZONE = "timeZone"; + + private DateTimeConfigBuilder() { + } + + /** + * build time information object + * default values are: length = 0 and offset = 0 and timeZone = PDT/PST + */ + public static DateTimeConfig build(Config config) { + String endTIme = config.getString(END_TIME); + String endTimeFormat = config.getString(END_TIME_FORMAT); + String timeResolutionStr = config.getString(TIME_RESOLUTION); + ChronoUnit timeResolution = ConfigUtils.getChronoUnit(timeResolutionStr); + long length = ConfigUtils.getLongWithDefault(config, LENGTH, 0); + Duration offset = ConfigUtils.getDurationWithDefault(config, OFFSET, Duration.ofSeconds(0)); + String timeZoneStr = ConfigUtils.getStringWithDefault(config, TIME_ZONE, DEFAULT_TIME_ZONE); + TimeZone timeZone = TimeZone.getTimeZone(timeZoneStr); + DateTimeConfig dateTimeConfig = new DateTimeConfig(endTIme, endTimeFormat, timeResolution, length, offset, timeZone); + logger.trace("Built DateTimeConfig object"); + return dateTimeConfig; + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/generation/FeatureGenConfigBuilder.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/generation/FeatureGenConfigBuilder.java new file mode 100644 index 000000000..83bce81fc --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/generation/FeatureGenConfigBuilder.java @@ -0,0 +1,32 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.generation; + +import com.linkedin.feathr.core.config.generation.FeatureGenConfig; +import com.linkedin.feathr.core.config.generation.OperationalConfig; +import com.typesafe.config.Config; +import java.util.List; +import org.apache.log4j.Logger; + + +/** + * Feature generation config builder + */ +public class FeatureGenConfigBuilder { + private final static Logger logger = Logger.getLogger(FeatureGenConfigBuilder.class); + private final static String OPERATIONAL = "operational"; + private final static String FEATURES = "features"; + + private FeatureGenConfigBuilder() { + } + + /** + * config represents the object part in: + * {@code operational : { ... } } + */ + public static FeatureGenConfig build(Config config) { + OperationalConfig operationalConfig = OperationalConfigBuilder.build(config.getConfig(OPERATIONAL)); + List features = config.getStringList(FEATURES); + FeatureGenConfig featureGenConfig = new FeatureGenConfig(operationalConfig, features); + logger.trace("Built FeatureGenConfig object"); + return featureGenConfig; + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/generation/OperationEnvironment.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/generation/OperationEnvironment.java new file mode 100644 index 000000000..b148121fb --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/generation/OperationEnvironment.java @@ -0,0 +1,5 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.generation; + +public enum OperationEnvironment { + OFFLINE, NEARLINE +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/generation/OperationalConfigBuilder.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/generation/OperationalConfigBuilder.java new file mode 100644 index 000000000..6865a88e3 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/generation/OperationalConfigBuilder.java @@ -0,0 +1,63 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.generation; + +import com.linkedin.feathr.core.config.common.DateTimeConfig; +import com.linkedin.feathr.core.config.generation.NearlineOperationalConfig; +import com.linkedin.feathr.core.config.generation.OperationalConfig; +import com.linkedin.feathr.core.config.generation.OfflineOperationalConfig; +import com.linkedin.feathr.core.config.generation.OutputProcessorConfig; +import com.linkedin.feathr.core.utils.ConfigUtils; +import com.typesafe.config.Config; +import java.time.Duration; +import java.util.List; +import java.util.stream.Collectors; +import org.apache.log4j.Logger; + + +/** + * Operation config object builder + */ + +public class OperationalConfigBuilder { + + private final static Logger logger = Logger.getLogger(OperationalConfigBuilder.class); + private static final String NAME = "name"; + private static final String RETENTION = "retention"; + private static final String OUTPUT = "output"; + private static final String SIMULATE_TIME_DELAY = "timeDelay"; + private static final String ENABLE_INCREMENTAL = "enableIncremental"; + private static final String ENV = "env"; + + private OperationalConfigBuilder() { + } + + /** + * Build operational config object in feature generation config file + * default values: retention = 1 unit of time resolution, and simulate delay = 0 + */ + public static OperationalConfig build(Config config) { + String name = config.getString(NAME); + List outputConfigs = config.getConfigList(OUTPUT); + List + outputProcessorConfigs = outputConfigs.stream().map(cfg -> OutputProcessorBuilder.build(cfg)).collect(Collectors.toList()); + OperationalConfig operationalConfig = null; + + // represents a nearline feature gen config, it should not have retention or any of the other time fields. + if (config.hasPath(ENV) && config.getString(ENV).equals(OperationEnvironment.NEARLINE.toString())) { + operationalConfig = new NearlineOperationalConfig(outputProcessorConfigs, name); + logger.trace("Built OperationalConfig object for nearline feature"); + } else { // represents offline config. If env is not specified, it is offline by default. Env can be specified as offline also. + // However, we do not need to check that case for now. + DateTimeConfig dateTimeConfig = DateTimeConfigBuilder.build(config); + Duration timeResolution = dateTimeConfig.get_timeResolution().getDuration(); + Duration retention = ConfigUtils.getDurationWithDefault(config, RETENTION, timeResolution); + Duration simulateTimeDelay = ConfigUtils.getDurationWithDefault(config, SIMULATE_TIME_DELAY, Duration.ofSeconds(0)); + Boolean enableIncremental = ConfigUtils.getBooleanWithDefault(config, ENABLE_INCREMENTAL, false); + + operationalConfig = + new OfflineOperationalConfig(outputProcessorConfigs, name, dateTimeConfig, retention, simulateTimeDelay, + enableIncremental); + logger.trace("Built OperationalConfig object for offline feature"); + } + return operationalConfig; + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/generation/OutputProcessorBuilder.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/generation/OutputProcessorBuilder.java new file mode 100644 index 000000000..1a999fc97 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/generation/OutputProcessorBuilder.java @@ -0,0 +1,40 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.generation; + +import com.linkedin.feathr.core.config.common.OutputFormat; +import com.linkedin.feathr.core.config.generation.OutputProcessorConfig; +import com.typesafe.config.Config; +import org.apache.log4j.Logger; + + +/** + * Output processor config object builder, e.g., HDFS, VENICE processor + */ +public class OutputProcessorBuilder { + private final static Logger logger = Logger.getLogger(OutputProcessorBuilder.class); + private static final String OUTPUT_FORMAT = "outputFormat"; + private static final String PARAMS = "params"; + private static final String NAME = "name"; + + private OutputProcessorBuilder() { + } + + /** + * build output processor from config object + */ + public static OutputProcessorConfig build(Config config) { + String name = config.getString(NAME); + OutputFormat outputFormat = OutputFormat.valueOf(config.getString(OUTPUT_FORMAT)); + Config params = config.getConfig(PARAMS); + logger.trace("Built OperationalConfig object"); + return new OutputProcessorConfig(name, outputFormat, params); + } + + /** + * build output processor from all the class members + * This is typically used to rebuild a new config object from the existing one when there's + * need to modify/pass in extra parameters + */ + public static OutputProcessorConfig build(String name, OutputFormat outputFormat, Config params) { + return new OutputProcessorConfig(name, outputFormat, params); + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/FeatureDefConfigBuilder.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/FeatureDefConfigBuilder.java new file mode 100644 index 000000000..7f929b82c --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/FeatureDefConfigBuilder.java @@ -0,0 +1,58 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.producer; + +import com.linkedin.feathr.core.config.producer.FeatureDefConfig; +import com.linkedin.feathr.core.config.producer.anchors.AnchorsConfig; +import com.linkedin.feathr.core.config.producer.derivations.DerivationsConfig; +import com.linkedin.feathr.core.config.producer.sources.SourcesConfig; +import com.linkedin.feathr.core.configbuilder.typesafe.producer.anchors.AnchorsConfigBuilder; +import com.linkedin.feathr.core.configbuilder.typesafe.producer.derivations.DerivationsConfigBuilder; +import com.linkedin.feathr.core.configbuilder.typesafe.producer.sources.SourcesConfigBuilder; +import com.typesafe.config.Config; +import org.apache.log4j.Logger; + +import static com.linkedin.feathr.core.config.producer.FeatureDefConfig.*; + + +/** + * Builds the complete FeatureDefConfig object by delegating to its children, one per config section. + */ +public class FeatureDefConfigBuilder { + private final static Logger logger = Logger.getLogger(FeatureDefConfigBuilder.class); + + public static FeatureDefConfig build(Config config) { + SourcesConfig sources = null; + if (config.hasPath(SOURCES)) { + Config sourcesCfg = config.getConfig(SOURCES); + sources = SourcesConfigBuilder.build(sourcesCfg); + } + + AnchorsConfig anchors = null; + if (config.hasPath(ANCHORS)) { + Config anchorsCfg = config.getConfig(ANCHORS); + anchors = AnchorsConfigBuilder.build(anchorsCfg); + } + + DerivationsConfig derivations = null; + if (config.hasPath(DERIVATIONS)) { + Config derivationCfg = config.getConfig(DERIVATIONS); + derivations = DerivationsConfigBuilder.build(derivationCfg); + } + + FeatureDefConfig configObj = new FeatureDefConfig(sources, anchors, derivations); + //validateSemantics(configObj) // TODO Semantic validation + logger.debug("Built FeatureDefConfig object"); + + return configObj; + } + + /* + * TODO: Semantic validation + * Validate: + * extractor class name refers to a valid class on the classpath + * source names, if any, in the anchors are resolved to those in the sources section + * date-time values are valid, i.e. not in the future and not too-far in the past + */ + private Boolean validateSemantics(FeatureDefConfig configObj) { + return true; + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/anchors/AnchorConfigBuilder.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/anchors/AnchorConfigBuilder.java new file mode 100644 index 000000000..3e5c61764 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/anchors/AnchorConfigBuilder.java @@ -0,0 +1,54 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.producer.anchors; + +import com.linkedin.feathr.core.config.producer.anchors.AnchorConfig; +import com.typesafe.config.Config; +import org.apache.log4j.Logger; + +import static com.linkedin.feathr.core.config.producer.anchors.AnchorConfig.*; + + +/** + * Build a {@link AnchorConfig} object + */ +class AnchorConfigBuilder { + private final static Logger logger = Logger.getLogger(AnchorConfigBuilder.class); + + private AnchorConfigBuilder() { + } + + /* + * config represents the object part in: + * : { ... } + */ + public static AnchorConfig build(String name, Config config) { + logger.debug("Building AnchorConfig object for anchor " + name); + + + AnchorConfig anchorConfig; + // Delegates the actual build to a child config builder + if (config.hasPath(EXTRACTOR) || config.hasPath(TRANSFORMER)) { + /* + * This check should always go before config.hasPath(KEY_EXTRACTOR), or config.hasPath(KEY), + * as the config might contain keyExtractor field or key field + */ + anchorConfig = AnchorConfigWithExtractorBuilder.build(name, config); + } else if (config.hasPath(KEY_EXTRACTOR)) { + /* + * AnchorConfigWithKeyExtractor contains ONLY keyExtractor, without extractor, + * it is mutually exclusive with AnchorConfigWithExtractor + */ + anchorConfig = AnchorConfigWithKeyExtractorBuilder.build(name, config); + } else if (config.hasPath(KEY)) { + /* + * AnchorConfigWithKey can not contain extractor field, + * it is mutually exclusive with AnchorConfigWithExtractor + */ + anchorConfig = AnchorConfigWithKeyBuilder.build(name, config); + } else { + anchorConfig = AnchorConfigWithOnlyMvelBuilder.build(name, config); + } + + logger.debug("Built AnchorConfig object for anchor " + name); + return anchorConfig; + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/anchors/AnchorConfigWithExtractorBuilder.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/anchors/AnchorConfigWithExtractorBuilder.java new file mode 100644 index 000000000..c50bc8c7e --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/anchors/AnchorConfigWithExtractorBuilder.java @@ -0,0 +1,84 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.producer.anchors; + +import com.linkedin.feathr.core.config.producer.anchors.AnchorConfigWithExtractor; +import com.linkedin.feathr.core.config.producer.anchors.FeatureConfig; +import com.linkedin.feathr.core.config.producer.anchors.TypedKey; +import com.linkedin.feathr.core.configbuilder.ConfigBuilderException; +import com.linkedin.feathr.core.utils.ConfigUtils; +import com.typesafe.config.Config; +import com.typesafe.config.ConfigValueType; +import java.util.List; +import java.util.Map; +import javax.lang.model.SourceVersion; +import org.apache.log4j.Logger; + +import static com.linkedin.feathr.core.config.producer.anchors.AnchorConfig.*; + + +/** + * Builds AnchorConfig objects that have features that are extracted via a udf class (an extractor) + */ +class AnchorConfigWithExtractorBuilder extends BaseAnchorConfigBuilder { + private final static Logger logger = Logger.getLogger(AnchorConfigWithExtractorBuilder.class); + + private AnchorConfigWithExtractorBuilder() { + } + + public static AnchorConfigWithExtractor build(String name, Config config) { + String source = config.getString(SOURCE); + + String extractor; + String extractorClassName = config.hasPath(EXTRACTOR) + ? getExtractorClassName(config) + : getTransformerClassName(config); + if (SourceVersion.isName(extractorClassName)) { + extractor = extractorClassName; + } else { + throw new ConfigBuilderException("Invalid class name for extractor: " + extractorClassName); + } + + String keyExtractor = config.hasPath(KEY_EXTRACTOR) ? config.getString(KEY_EXTRACTOR) : null; + + TypedKey typedKey = TypedKeyBuilder.getInstance().build(config); + + List keyAlias = ConfigUtils.getStringList(config, KEY_ALIAS); + + if ((keyAlias != null || typedKey != null) && keyExtractor != null) { + throw new ConfigBuilderException("The keyExtractor field and keyAlias field can not coexist."); + } + + Map features = getFeatures(config); + AnchorConfigWithExtractor anchorConfig = + new AnchorConfigWithExtractor(source, keyExtractor, typedKey, keyAlias, extractor, features); + logger.trace("Built AnchorConfigWithExtractor object for anchor " + name); + + return anchorConfig; + } + + private static String getExtractorClassName(Config config) { + ConfigValueType valueType = config.getValue(EXTRACTOR).valueType(); + + String extractorClassName; + switch (valueType) { + case STRING: + extractorClassName = config.getString(EXTRACTOR); + break; + + /* + * Support for legacy/deprecated extractor: {class: "..."}. Ought to be removed. + */ + case OBJECT: + extractorClassName = config.getString(EXTRACTOR + ".class"); + break; + + default: + throw new ConfigBuilderException("Unknown value type " + valueType + " for key " + EXTRACTOR); + } + return extractorClassName; + } + + // Support for legacy/deprecated "transformer" field. Ought to be removed. + private static String getTransformerClassName(Config config) { + return config.getString(TRANSFORMER); + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/anchors/AnchorConfigWithKeyBuilder.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/anchors/AnchorConfigWithKeyBuilder.java new file mode 100644 index 000000000..74497bb9a --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/anchors/AnchorConfigWithKeyBuilder.java @@ -0,0 +1,51 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.producer.anchors; + +import com.linkedin.feathr.core.config.producer.anchors.AnchorConfigWithKey; +import com.linkedin.feathr.core.config.producer.anchors.FeatureConfig; +import com.linkedin.feathr.core.config.producer.anchors.LateralViewParams; +import com.linkedin.feathr.core.config.producer.anchors.TypedKey; +import com.linkedin.feathr.core.configbuilder.ConfigBuilderException; +import com.linkedin.feathr.core.utils.ConfigUtils; +import com.typesafe.config.Config; +import java.util.List; +import java.util.Map; +import org.apache.log4j.Logger; + +import static com.linkedin.feathr.core.config.producer.anchors.AnchorConfig.*; + + +/** + * Builds AnchorConfig objects that have features with keys + */ +class AnchorConfigWithKeyBuilder extends BaseAnchorConfigBuilder { + private final static Logger logger = Logger.getLogger(BaseAnchorConfigBuilder.class); + + private AnchorConfigWithKeyBuilder() { + } + + public static AnchorConfigWithKey build(String name, Config config) { + String source = config.getString(SOURCE); + + // key field is guaranteed to exist for AnchorConfigWithKeyBuilder + TypedKey typedKey = TypedKeyBuilder.getInstance().build(config); + + Map features = getFeatures(config); + + List keyAlias = ConfigUtils.getStringList(config, KEY_ALIAS); + if (keyAlias != null && keyAlias.size() != typedKey.getKey().size()) { + throw new ConfigBuilderException("The size of key and keyAlias does not match"); + } + /* + * Build LateralViewParams if the anchor contains time-window features (aka sliding-window features) + * and if the lateral view parameters have been specified in the anchor config. + */ + LateralViewParams lateralViewParams = (hasTimeWindowFeatureConfig(features) && config.hasPath(LATERAL_VIEW_PARAMS)) + ? LateralViewParamsBuilder.build(name, config.getConfig(LATERAL_VIEW_PARAMS)) : null; + + AnchorConfigWithKey anchorConfig = + new AnchorConfigWithKey(source, typedKey, keyAlias, lateralViewParams, features); + logger.trace("Built AnchorConfigWithKey object for anchor " + name); + + return anchorConfig; + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/anchors/AnchorConfigWithKeyExtractorBuilder.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/anchors/AnchorConfigWithKeyExtractorBuilder.java new file mode 100644 index 000000000..2660b9cb9 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/anchors/AnchorConfigWithKeyExtractorBuilder.java @@ -0,0 +1,53 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.producer.anchors; + +import com.linkedin.feathr.core.config.producer.anchors.AnchorConfigWithKeyExtractor; +import com.linkedin.feathr.core.config.producer.anchors.FeatureConfig; +import com.linkedin.feathr.core.config.producer.anchors.LateralViewParams; +import com.linkedin.feathr.core.configbuilder.ConfigBuilderException; +import com.typesafe.config.Config; +import java.util.Map; +import javax.lang.model.SourceVersion; +import org.apache.log4j.Logger; + +import static com.linkedin.feathr.core.config.producer.anchors.AnchorConfig.*; + + +/** + * Builds AnchorConfig objects that have features that are extracted via a udf class (an extractor) + */ +class AnchorConfigWithKeyExtractorBuilder extends BaseAnchorConfigBuilder { + private final static Logger logger = Logger.getLogger(AnchorConfigWithKeyExtractorBuilder.class); + + private AnchorConfigWithKeyExtractorBuilder() { + } + + public static AnchorConfigWithKeyExtractor build(String name, Config config) { + String source = config.getString(SOURCE); + + String keyExtractor; + String className = config.getString(KEY_EXTRACTOR); + if (SourceVersion.isName(className)) { + keyExtractor = className; + } else { + throw new ConfigBuilderException("Invalid class name for keyExtractor: " + className); + } + + if (config.hasPath(KEY_ALIAS)) { + throw new ConfigBuilderException("keyAlias and keyExtractor are mutually exclusive fields"); + } + + Map features = getFeatures(config); + + /* + * Build LateralViewParams if the anchor contains time-window features (aka sliding-window features) + * and if the lateral view parameters have been specified in the anchor config. + */ + LateralViewParams lateralViewParams = (hasTimeWindowFeatureConfig(features) && config.hasPath(LATERAL_VIEW_PARAMS)) + ? LateralViewParamsBuilder.build(name, config.getConfig(LATERAL_VIEW_PARAMS)) : null; + + AnchorConfigWithKeyExtractor anchorConfig = new AnchorConfigWithKeyExtractor(source, keyExtractor, features, lateralViewParams); + logger.trace("Built AnchorConfigWithExtractor object for anchor " + name); + + return anchorConfig; + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/anchors/AnchorConfigWithOnlyMvelBuilder.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/anchors/AnchorConfigWithOnlyMvelBuilder.java new file mode 100644 index 000000000..71cb51f10 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/anchors/AnchorConfigWithOnlyMvelBuilder.java @@ -0,0 +1,32 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.producer.anchors; + +import com.linkedin.feathr.core.config.producer.anchors.AnchorConfigWithOnlyMvel; +import com.linkedin.feathr.core.config.producer.anchors.FeatureConfig; +import com.typesafe.config.Config; +import java.util.Map; +import org.apache.log4j.Logger; + +import static com.linkedin.feathr.core.config.producer.anchors.AnchorConfig.*; + + +/** + * Builds AnchorConfig objects that have features directly expressed as an MVEL expression without any + * key or extractor + */ +class AnchorConfigWithOnlyMvelBuilder extends BaseAnchorConfigBuilder { + private final static Logger logger = Logger.getLogger(AnchorConfigWithOnlyMvelBuilder.class); + + private AnchorConfigWithOnlyMvelBuilder() { + } + + public static AnchorConfigWithOnlyMvel build(String name, Config config) { + String source = config.getString(SOURCE); + + Map features = getFeatures(config); + + AnchorConfigWithOnlyMvel anchorConfig = new AnchorConfigWithOnlyMvel(source, features); + logger.trace("Build AnchorConfigWithOnlyMvel object for anchor " + name); + + return anchorConfig; + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/anchors/AnchorsConfigBuilder.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/anchors/AnchorsConfigBuilder.java new file mode 100644 index 000000000..ce4a63ff9 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/anchors/AnchorsConfigBuilder.java @@ -0,0 +1,43 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.producer.anchors; + +import com.linkedin.feathr.core.config.producer.anchors.AnchorConfig; +import com.linkedin.feathr.core.config.producer.anchors.AnchorsConfig; +import com.typesafe.config.Config; +import com.typesafe.config.ConfigObject; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.log4j.Logger; + +import static com.linkedin.feathr.core.utils.Utils.*; + + +/** + * Builds a map of anchor name to its config by delegating the building of each anchor config object + * to its child + */ +public class AnchorsConfigBuilder { + private final static Logger logger = Logger.getLogger(AnchorsConfigBuilder.class); + + private AnchorsConfigBuilder() { + } + + /** + * config represents the object part in: + * {@code anchors : { ... } } + */ + public static AnchorsConfig build(Config config) { + ConfigObject configObj = config.root(); + + Stream anchorNames = configObj.keySet().stream(); + + Map nameConfigMap = anchorNames.collect( + Collectors.toMap(Function.identity(), aName -> AnchorConfigBuilder.build(aName, config.getConfig(quote(aName))))); + + AnchorsConfig anchorsConfig = new AnchorsConfig(nameConfigMap); + logger.debug("Built all AnchorConfig objects"); + + return anchorsConfig; + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/anchors/BaseAnchorConfigBuilder.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/anchors/BaseAnchorConfigBuilder.java new file mode 100644 index 000000000..464ab449c --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/anchors/BaseAnchorConfigBuilder.java @@ -0,0 +1,53 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.producer.anchors; + +import com.linkedin.feathr.core.config.producer.anchors.FeatureConfig; +import com.linkedin.feathr.core.config.producer.anchors.TimeWindowFeatureConfig; +import com.linkedin.feathr.core.configbuilder.ConfigBuilderException; +import com.typesafe.config.Config; +import com.typesafe.config.ConfigValue; +import com.typesafe.config.ConfigValueType; +import java.util.List; +import java.util.Map; +import org.apache.log4j.Logger; + +import static com.linkedin.feathr.core.config.producer.anchors.AnchorConfig.*; + + +abstract class BaseAnchorConfigBuilder { + private final static Logger logger = Logger.getLogger(BaseAnchorConfigBuilder.class); + + // Gets feature config objects by invoking the FeatureConfigBuilder appropriately + public static Map getFeatures(Config anchorConfig) { + logger.debug("Building FeatureConfig objects in anchor " + anchorConfig); + + ConfigValue value = anchorConfig.getValue(FEATURES); + ConfigValueType valueType = value.valueType(); + + Map features; + switch (valueType) { // Note that features can be expressed as a list or as an object + case LIST: + List featureNames = anchorConfig.getStringList(FEATURES); + features = FeatureConfigBuilder.build(featureNames); + break; + + case OBJECT: + Config featuresConfig = anchorConfig.getConfig(FEATURES); + features = FeatureConfigBuilder.build(featuresConfig); + break; + + default: + throw new ConfigBuilderException("Expected " + FEATURES + " value type List or Object, got " + valueType); + } + + return features; + } + + /* + * Check if the feature configs have TimeWindowFeatureConfig objects. An anchor can contain + * time-window features or regular features but never a mix of both. + */ + static boolean hasTimeWindowFeatureConfig(Map featureConfigMap) { + FeatureConfig featureConfig = featureConfigMap.values().iterator().next(); + return featureConfig instanceof TimeWindowFeatureConfig; + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/anchors/ExpressionBasedFeatureConfigBuilder.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/anchors/ExpressionBasedFeatureConfigBuilder.java new file mode 100644 index 000000000..497798f3e --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/anchors/ExpressionBasedFeatureConfigBuilder.java @@ -0,0 +1,49 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.producer.anchors; + +import com.linkedin.feathr.core.config.producer.anchors.ExpressionBasedFeatureConfig; +import com.linkedin.feathr.core.configbuilder.typesafe.producer.common.FeatureTypeConfigBuilder; +import com.linkedin.feathr.core.config.producer.ExprType; +import com.linkedin.feathr.core.config.producer.anchors.ComplexFeatureConfig; +import com.linkedin.feathr.core.config.producer.common.FeatureTypeConfig; +import com.typesafe.config.Config; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static com.linkedin.feathr.core.config.producer.anchors.FeatureConfig.*; + + +/** + * Builds an ExpressionBasedFeatureConfig object + */ +class ExpressionBasedFeatureConfigBuilder { + private final static Logger logger = LoggerFactory.getLogger(ExpressionBasedFeatureConfigBuilder.class); + + private ExpressionBasedFeatureConfigBuilder() { + } + + public static ExpressionBasedFeatureConfig build(String featureName, Config featureConfig) { + String expr; + ExprType exprType; + if (featureConfig.hasPath(DEF_SQL_EXPR)) { + expr = featureConfig.getString(DEF_SQL_EXPR); + exprType = ExprType.SQL; + } else if (featureConfig.hasPath(DEF)) { + expr = featureConfig.getString(DEF); + exprType = ExprType.MVEL; + } else { + throw new RuntimeException( + "ExpressionBasedFeatureConfig should have " + DEF_SQL_EXPR + " field or " + DEF + " field but found none in : " + + featureConfig); + } + + FeatureTypeConfig featureTypeConfig = FeatureTypeConfigBuilder.build(featureConfig); + + String defaultValue = featureConfig.hasPath(DEFAULT) ? featureConfig.getValue(DEFAULT).render() : null; + + ExpressionBasedFeatureConfig configObj = + new ExpressionBasedFeatureConfig(expr, exprType, defaultValue, featureTypeConfig); + logger.trace("Built ExpressionBasedFeatureConfig for feature" + featureName); + + return configObj; + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/anchors/ExtractorBasedFeatureConfigBuilder.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/anchors/ExtractorBasedFeatureConfigBuilder.java new file mode 100644 index 000000000..11c1e4e1a --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/anchors/ExtractorBasedFeatureConfigBuilder.java @@ -0,0 +1,47 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.producer.anchors; + +import com.linkedin.feathr.core.config.producer.anchors.ExtractorBasedFeatureConfig; +import com.linkedin.feathr.core.config.producer.common.FeatureTypeConfig; +import com.linkedin.feathr.core.configbuilder.typesafe.producer.common.FeatureTypeConfigBuilder; +import com.typesafe.config.Config; +import com.typesafe.config.ConfigObject; +import com.typesafe.config.ConfigRenderOptions; +import java.util.Collections; +import java.util.Map; +import java.util.stream.Collectors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static com.linkedin.feathr.core.config.producer.anchors.FeatureConfig.*; + + +/** + * Builds an ExtractorBasedFeatureConfig object + */ +class ExtractorBasedFeatureConfigBuilder { + private final static Logger logger = LoggerFactory.getLogger(ExtractorBasedFeatureConfigBuilder.class); + + private ExtractorBasedFeatureConfigBuilder() { + } + + public static ExtractorBasedFeatureConfig build(String featureName, Config featureConfig) { + + FeatureTypeConfig featureTypeConfig = FeatureTypeConfigBuilder.build(featureConfig); + + String defaultValue = featureConfig.hasPath(DEFAULT) ? featureConfig.getValue(DEFAULT).render() : null; + Map parameters = + featureConfig.hasPath(PARAMETERS) ? getParameters(featureConfig) : Collections.emptyMap(); + logger.trace("Built ExtractorBasedFeatureConfig for feature" + featureName); + return new ExtractorBasedFeatureConfig(featureName, featureTypeConfig, defaultValue, parameters); + } + + public static Map getParameters(Config anchorConfig) { + logger.debug("Building Parameters objects in anchor " + anchorConfig); + + Config config = anchorConfig.getConfig(PARAMETERS); + ConfigObject featuresConfigObj = config.root(); + return featuresConfigObj.entrySet() + .stream() + .collect(Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().render(ConfigRenderOptions.concise()))); + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/anchors/FeatureConfigBuilder.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/anchors/FeatureConfigBuilder.java new file mode 100644 index 000000000..35e4c810b --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/anchors/FeatureConfigBuilder.java @@ -0,0 +1,137 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.producer.anchors; + +import com.linkedin.feathr.core.config.producer.anchors.ExtractorBasedFeatureConfig; +import com.linkedin.feathr.core.config.producer.anchors.FeatureConfig; +import com.linkedin.feathr.core.config.producer.anchors.SimpleFeatureConfig; +import com.linkedin.feathr.core.configbuilder.ConfigBuilderException; +import com.linkedin.feathr.core.utils.Utils; +import com.typesafe.config.Config; +import com.typesafe.config.ConfigObject; +import com.typesafe.config.ConfigValue; +import com.typesafe.config.ConfigValueType; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static com.linkedin.feathr.core.config.producer.anchors.FeatureConfig.*; +import static com.linkedin.feathr.core.utils.Utils.*; + + +/** + * Builds FeatureConfig objects, specifically a Map of feature names to FeatureConfig objects in a + * single anchor + */ +class FeatureConfigBuilder { + private final static Logger logger = LoggerFactory.getLogger(FeatureConfigBuilder.class); + + private FeatureConfigBuilder() { + } + + public static Map build(Config featuresConfig) { + logger.debug("Building FeatureConfig object for featuresConfig " + featuresConfig); + + ConfigObject featuresConfigObj = featuresConfig.root(); + Set featureNames = featuresConfigObj.keySet(); + logger.trace("Found feature names:" + Utils.string(featureNames)); + + Map configObjMap = featureNames.stream() + .collect(Collectors.toMap(Function.identity(), fName -> FeatureConfigBuilder.build(featuresConfig, fName))); + + logger.debug("Built all FeatureConfig objects"); + + return configObjMap; + } + + public static Map build(List featureNames) { + logger.debug("Building FeatureConfig objects for features " + Utils.string(featureNames)); + + Map configObjMap = featureNames.stream(). + collect(Collectors.toMap(Function.identity(), ExtractorBasedFeatureConfig::new)); + + logger.debug("Built all FeatureConfig objects"); + + return configObjMap; + } + + /** + * Builds a single FeatureConfig object from the enclosing featuresConfig object. The actual build is delegated + * to a child builder depending on the type of the feature - simple (built in this method), complex, or + * time-window feature. + * + * featuresConfig refers to the object part of: + * + * {@code features : { ...} } + * + * The features may be specified in three ways as shown below: + * 
+   * {@code
+   *   features: {
+   *     : {
+   *       def: 
+   *       type: 
+   *       default: 
+   *     }
+   *     ...
+   *   }
+   *
+   *   features: {
+   *     : ,
+   *     ...
+   *   }
+   *
+   *   features: {
+   *     : {
+   *       def:                 // the column/field on which the aggregation will be computed.
+   *                                         // Could be specified as a Spark column expression.
+   *                                         // for TIMESINCE feature, it should be left as an empty string.
+   *       aggregation:    // one of 5 aggregation types: SUM, COUNT, MAX, TIMESINCE, AVG
+   *       window:    // support 4 type of units: d(day), h(hour), m(minute), s(second).
+   *                                         // The example value are "7d' or "5h" or "3m" or "1s"
+   *       filter:                   // (Optional) a Spark SQL expression for filtering the fact data before aggregation.
+   *       groupBy:             // (Optional) the column/field on which the data will be grouped by before aggregation.
+   *       limit:                       // (Optional) a number specifying for each group, taking the records with the TOP k aggregation value.
+   *     }
+   *     ...
+   *   }
+   * }
+   *
+ */ + + private static FeatureConfig build(Config featuresConfig, String featureName) { + String quotedFeatureName = quote(featureName); + ConfigValue configValue = featuresConfig.getValue(quotedFeatureName); + ConfigValueType configValueType = configValue.valueType(); + FeatureConfig configObj; + + switch (configValueType) { + case STRING: + String featureExpr = featuresConfig.getString(quotedFeatureName); + configObj = new ExtractorBasedFeatureConfig(featureExpr); + logger.trace("Built ExtractorBasedFeatureConfig object for feature " + featureName); + break; + + case OBJECT: + Config featureCfg = featuresConfig.getConfig(quotedFeatureName); + if (featuresConfig.hasPath(quotedFeatureName + "." + WINDOW) || featuresConfig.hasPath(quotedFeatureName + "." + WINDOW_PARAMETERS)) { + configObj = TimeWindowFeatureConfigBuilder.build(featureName, featureCfg); + } else if (featureCfg.hasPath(DEF_SQL_EXPR) || featureCfg.hasPath(DEF)) { + configObj = ExpressionBasedFeatureConfigBuilder.build(featureName, featureCfg); + } else { + // An ExtractorBased feature config with type, default value information, and optional parameters + configObj = ExtractorBasedFeatureConfigBuilder.build(featureName, featureCfg); + } + break; + + default: + throw new ConfigBuilderException("Expected " + featureName + " value type String or Object, got " + configValueType); + } + + logger.debug("Built FeatureConfig object for feature " + featureName); + + return configObj; + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/anchors/LateralViewParamsBuilder.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/anchors/LateralViewParamsBuilder.java new file mode 100644 index 000000000..0e08d3e90 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/anchors/LateralViewParamsBuilder.java @@ -0,0 +1,34 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.producer.anchors; + +import com.linkedin.feathr.core.config.producer.anchors.TimeWindowFeatureConfig; +import com.linkedin.feathr.core.config.producer.anchors.LateralViewParams; +import com.typesafe.config.Config; +import org.apache.log4j.Logger; + +import static com.linkedin.feathr.core.config.producer.anchors.LateralViewParams.LATERAL_VIEW_DEF; +import static com.linkedin.feathr.core.config.producer.anchors.LateralViewParams.LATERAL_VIEW_ITEM_ALIAS; +import static com.linkedin.feathr.core.config.producer.anchors.LateralViewParams.LATERAL_VIEW_FILTER; + + +/** + * Builds {@link LateralViewParams} object that are (optionally) used with + * {@link TimeWindowFeatureConfig} (aka sliding-window features) + */ +class LateralViewParamsBuilder { + private final static Logger logger = Logger.getLogger(LateralViewParamsBuilder.class); + + private LateralViewParamsBuilder() { + } + + public static LateralViewParams build(String anchorName, Config lateralViewParamsConfig) { + String def = lateralViewParamsConfig.getString(LATERAL_VIEW_DEF); + String itemAlias = lateralViewParamsConfig.getString(LATERAL_VIEW_ITEM_ALIAS); + String filter = lateralViewParamsConfig.hasPath(LATERAL_VIEW_FILTER) + ? lateralViewParamsConfig.getString(LATERAL_VIEW_FILTER) : null; + + LateralViewParams lateralViewParams = new LateralViewParams(def, itemAlias, filter); + logger.trace("Built LateralViewParams config object for anchor " + anchorName); + + return lateralViewParams; + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/anchors/TimeWindowFeatureConfigBuilder.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/anchors/TimeWindowFeatureConfigBuilder.java new file mode 100644 index 000000000..d6005a7d1 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/anchors/TimeWindowFeatureConfigBuilder.java @@ -0,0 +1,96 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.producer.anchors; + +import com.linkedin.feathr.core.config.TimeWindowAggregationType; +import com.linkedin.feathr.core.config.WindowType; +import com.linkedin.feathr.core.config.producer.ExprType; +import com.linkedin.feathr.core.config.producer.TypedExpr; +import com.linkedin.feathr.core.config.producer.anchors.TimeWindowFeatureConfig; +import com.linkedin.feathr.core.config.producer.anchors.WindowParametersConfig; +import com.linkedin.feathr.core.config.producer.common.FeatureTypeConfig; +import com.linkedin.feathr.core.configbuilder.ConfigBuilderException; +import com.linkedin.feathr.core.configbuilder.typesafe.producer.common.FeatureTypeConfigBuilder; +import com.typesafe.config.Config; +import com.typesafe.config.ConfigValueType; +import java.time.Duration; +import org.apache.log4j.Logger; + +import static com.linkedin.feathr.core.config.producer.anchors.FeatureConfig.*; + + +/** + * Build {@link TimeWindowFeatureConfig} object + */ +class TimeWindowFeatureConfigBuilder { + private final static Logger logger = Logger.getLogger(FeatureConfigBuilder.class); + + private TimeWindowFeatureConfigBuilder() { + } + + public static TimeWindowFeatureConfig build(String featureName, Config featureConfig) { + + // nearline features can use DEF_MVEL to denote def mvel expression + String defType = featureConfig.hasPath(DEF_MVEL) ? DEF_MVEL : DEF; + ExprType defExprType = featureConfig.hasPath(DEF_MVEL) ? ExprType.MVEL : ExprType.SQL; + String columnExpr = featureConfig.getString(defType); + + String aggregationStr = featureConfig.getString(AGGREGATION); + TimeWindowAggregationType aggregation = TimeWindowAggregationType.valueOf(aggregationStr); + + // if window_parameters exists it represents a nearline feature, else if window exists it is an offline feature. + WindowParametersConfig windowParameters = null; + if (featureConfig.hasPath(WINDOW_PARAMETERS)) { + Config windowsParametersConfig = featureConfig.getConfig(WINDOW_PARAMETERS); + windowParameters = WindowParametersConfigBuilder.build(windowsParametersConfig); + } else if (featureConfig.hasPath(WINDOW)) { + WindowType type = WindowType.SLIDING; + Duration window = featureConfig.getDuration(WINDOW); + if (window.getSeconds() <= 0) { + String errMsg = WINDOW + " field must be in units of seconds, minutes, hours or days, and must be > 0. Refer to " + + "https://github.com/lightbend/config/blob/master/HOCON.md#duration-format for supported unit strings."; + throw new ConfigBuilderException(errMsg); + } + + // Offline case - We take the window and slidingInterval values and convert it to represent a sliding window parameters config. + // slidingInterval is null for offline. + windowParameters = new WindowParametersConfig(type, window, null); + + } + + // nearline features can use FILTER_MVEL to denote mvel filter expression + TypedExpr typedFilter = null; + if (featureConfig.hasPath(FILTER_MVEL) || featureConfig.hasPath(FILTER)) { + ExprType filterExprType = featureConfig.hasPath(FILTER_MVEL) ? ExprType.MVEL : ExprType.SQL; + String filterType = featureConfig.getValue(FILTER).valueType() == ConfigValueType.OBJECT ? FILTER_MVEL : FILTER; + String filter = featureConfig.getString(filterType); + typedFilter = new TypedExpr(filter, filterExprType); + } + + String groupBy = getString(featureConfig, GROUPBY); + + Integer limit = getInt(featureConfig, LIMIT); + + String decay = getString(featureConfig, DECAY); + + String weight = getString(featureConfig, WEIGHT); + + Integer embeddingSize = getInt(featureConfig, EMBEDDING_SIZE); + + FeatureTypeConfig featureTypeConfig = FeatureTypeConfigBuilder.build(featureConfig); + + String defaultValue = featureConfig.hasPath(DEFAULT) ? featureConfig.getValue(DEFAULT).unwrapped().toString() : null; + + TimeWindowFeatureConfig configObj = new TimeWindowFeatureConfig(new TypedExpr(columnExpr, defExprType), aggregation, + windowParameters, typedFilter, groupBy, limit, decay, weight, embeddingSize, featureTypeConfig, defaultValue); + logger.trace("Built TimeWindowFeatureConfig object for feature: " + featureName); + + return configObj; + } + + private static String getString(Config featureConfig, String key) { + return featureConfig.hasPath(key) ? featureConfig.getString(key) : null; + } + + private static Integer getInt(Config featureConfig, String key) { + return featureConfig.hasPath(key) ? featureConfig.getInt(key) : null; + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/anchors/TypedKeyBuilder.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/anchors/TypedKeyBuilder.java new file mode 100644 index 000000000..2a32a9dec --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/anchors/TypedKeyBuilder.java @@ -0,0 +1,61 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.producer.anchors; + +import com.linkedin.feathr.core.config.producer.ExprType; +import com.linkedin.feathr.core.config.producer.anchors.TypedKey; +import com.linkedin.feathr.core.utils.ConfigUtils; +import com.typesafe.config.Config; + +import static com.linkedin.feathr.core.config.producer.anchors.AnchorConfig.*; + +/** + * Package private class to build {@link TypedKey} from the following config syntax: + * 
+ *{@code
+ * key: [key1, key2]
+ * }
+ *
+ * + * or + * + * 
+ *{@code
+ * key.sqlExpr: [key1, key2]
+ * }
+ *
+ * + * or + * + * 
+ *{@code
+ * key.mvel: [key1, key2]
+ * }
+ *
+ */ +class TypedKeyBuilder { + // instance initialized when loading the class + private static final TypedKeyBuilder INSTANCE = new TypedKeyBuilder(); + + private TypedKeyBuilder() { } + + public static TypedKeyBuilder getInstance() { + return INSTANCE; + } + + TypedKey build(Config config) { + String keyExprTypeStr; + ExprType keyExprType; + if (config.hasPath(KEY_MVEL)) { + keyExprTypeStr = KEY_MVEL; + keyExprType = ExprType.MVEL; + } else if (config.hasPath(KEY_SQL_EXPR)) { + keyExprTypeStr = KEY_SQL_EXPR; + keyExprType = ExprType.SQL; + } else { + keyExprTypeStr = KEY; + keyExprType = ExprType.MVEL; + } + // get the raw key expr which is in HOCON format + String rawKeyExpr = ConfigUtils.getHoconString(config, keyExprTypeStr); + return rawKeyExpr == null ? null : new TypedKey(rawKeyExpr, keyExprType); + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/anchors/WindowParametersConfigBuilder.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/anchors/WindowParametersConfigBuilder.java new file mode 100644 index 000000000..1638b37a7 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/anchors/WindowParametersConfigBuilder.java @@ -0,0 +1,51 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.producer.anchors; + +import com.linkedin.feathr.core.config.WindowType; +import com.linkedin.feathr.core.config.producer.anchors.WindowParametersConfig; +import com.linkedin.feathr.core.configbuilder.ConfigBuilderException; +import com.typesafe.config.Config; +import java.time.Duration; +import java.util.Arrays; +import org.apache.log4j.Logger; + +import static com.linkedin.feathr.core.config.producer.anchors.FeatureConfig.*; + +/** + * Build {@link WindowParametersConfig} object + */ +public class WindowParametersConfigBuilder { + private final static Logger logger = Logger.getLogger(FeatureConfigBuilder.class); + + /* + * Prevent instantiation of class from outside + */ + private WindowParametersConfigBuilder() { + } + + /* + * Build a [[WindowParametersConfig]] object. + * @param windowParametersConfig Config of windowParameters object mentioned in a feature. + * @return WindowParametersConfig object + */ + public static WindowParametersConfig build(Config windowParametersConfig) { + String type = windowParametersConfig.getString(TYPE); + WindowType windowType; + try { + windowType = WindowType.valueOf(type); + } catch (IllegalArgumentException e) { + throw new ConfigBuilderException("Unsupported window type " + type + "; expected one of " + + Arrays.toString(WindowType.values())); + } + + Duration size = windowParametersConfig.getDuration(SIZE); + + Duration slidingInterval = null; + if (windowParametersConfig.hasPath(SLIDING_INTERVAL)) { + slidingInterval = windowParametersConfig.getDuration(SLIDING_INTERVAL); + } + + WindowParametersConfig configObj = new WindowParametersConfig(windowType, size, slidingInterval); + + return configObj; + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/common/FeatureTypeConfigBuilder.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/common/FeatureTypeConfigBuilder.java new file mode 100644 index 000000000..eb04c1283 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/common/FeatureTypeConfigBuilder.java @@ -0,0 +1,111 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.producer.common; + +import com.google.common.base.Preconditions; +import com.linkedin.feathr.core.config.producer.common.FeatureTypeConfig; +import com.linkedin.feathr.core.config.producer.definitions.FeatureType; +import com.linkedin.feathr.core.config.producer.definitions.TensorCategory; +import com.linkedin.feathr.core.configbuilder.ConfigBuilderException; +import com.typesafe.config.Config; +import com.typesafe.config.ConfigValue; +import com.typesafe.config.ConfigValueType; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import static com.linkedin.feathr.core.config.producer.common.FeatureTypeConfig.*; +import static com.linkedin.feathr.core.config.producer.derivations.DerivationConfig.TYPE; + + +/** + * Builds a {@link FeatureTypeConfig} object + */ +public class FeatureTypeConfigBuilder { + private static final Set SUPPORTED_TENSOR_TYPES = + new HashSet<>(Arrays.asList(FeatureType.DENSE_TENSOR, FeatureType.SPARSE_TENSOR, FeatureType.RAGGED_TENSOR)); + + private FeatureTypeConfigBuilder() { + } + + public static FeatureTypeConfig build(Config config) { + FeatureTypeConfig featureTypeConfig = null; + if (config.hasPath(TYPE)) { + ConfigValue configValue = config.getValue(TYPE); + ConfigValueType configValueType = configValue.valueType(); + + switch (configValueType) { + case STRING: + featureTypeConfig = new FeatureTypeConfig(FeatureType.valueOf(config.getString(TYPE))); + break; + case OBJECT: + featureTypeConfig = FeatureTypeConfigBuilder.buildComplexTypeConfig(config.getConfig(TYPE)); + break; + default: + throw new ConfigBuilderException( + "Expected " + TYPE + " config value type should be String or Object, got " + configValueType); + } + } + return featureTypeConfig; + } + + private static FeatureTypeConfig buildComplexTypeConfig(Config config) { + Preconditions.checkArgument(config.hasPath(TYPE), "The config should contain \"type\" child node."); + FeatureType featureType = FeatureType.valueOf(config.getString(TYPE)); + + // If config has `tensorCategory` field, the TENSOR featureType will be refined with tensorCategory: + // e.g. DENSE tensorCategory + TENSOR featureType -> DENSE_TENSOR featureType. + // The same for SPARSE and RAGGED category. + // If the featureType is not TENSOR, will throw exception. + if (config.hasPath(TENSOR_CATEGORY)) { + if (featureType != FeatureType.TENSOR) { + throw new ConfigBuilderException("tensorCategory field is specified but the feature type is not TENSOR: \n" + + config.root().render()); + } + TensorCategory tensorCategory = TensorCategory.valueOf(config.getString(TENSOR_CATEGORY)); + switch (tensorCategory) { + case DENSE: + featureType = FeatureType.DENSE_TENSOR; + break; + case SPARSE: + featureType = FeatureType.SPARSE_TENSOR; + break; + case RAGGED: + featureType = FeatureType.RAGGED_TENSOR; + break; + default: + throw new ConfigBuilderException("The feature type tensorCategory is not supported: " + tensorCategory); + } + } + + List shapes = null; + if (config.hasPath(SHAPE)) { + shapes = config.getIntList(SHAPE); + } + + List dimensionTypes = null; + if (config.hasPath(DIMENSION_TYPE)) { + dimensionTypes = config.getStringList(DIMENSION_TYPE); + } + + if (shapes != null && dimensionTypes != null && shapes.size() != dimensionTypes.size()) { + throw new RuntimeException( + "Sizes of dimensionType and shape should match but got: " + dimensionTypes + " and " + shapes); + } + + String valType = null; + if (config.hasPath(VAL_TYPE)) { + valType = config.getString(VAL_TYPE); + } else { + // For tensor, valType is required. + if (SUPPORTED_TENSOR_TYPES.contains(featureType)) { + throw new RuntimeException("valType field is required for tensor types but is missing in the config: " + config); + } + } + + return new FeatureTypeConfig.Builder().setFeatureType(featureType) + .setShapes(shapes) + .setDimensionTypes(dimensionTypes) + .setValType(valType) + .build(); + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/derivations/DerivationConfigBuilder.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/derivations/DerivationConfigBuilder.java new file mode 100644 index 000000000..eb0903d06 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/derivations/DerivationConfigBuilder.java @@ -0,0 +1,227 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.producer.derivations; + +import com.linkedin.feathr.core.configbuilder.typesafe.producer.common.FeatureTypeConfigBuilder; +import com.linkedin.feathr.core.config.producer.ExprType; +import com.linkedin.feathr.core.config.producer.TypedExpr; +import com.linkedin.feathr.core.config.producer.common.FeatureTypeConfig; +import com.linkedin.feathr.core.config.producer.derivations.BaseFeatureConfig; +import com.linkedin.feathr.core.config.producer.derivations.DerivationConfig; +import com.linkedin.feathr.core.config.producer.derivations.DerivationConfigWithExpr; +import com.linkedin.feathr.core.config.producer.derivations.DerivationConfigWithExtractor; +import com.linkedin.feathr.core.config.producer.derivations.KeyedFeature; +import com.linkedin.feathr.core.config.producer.derivations.SequentialJoinConfig; +import com.linkedin.feathr.core.config.producer.derivations.SimpleDerivationConfig; +import com.linkedin.feathr.core.configbuilder.ConfigBuilderException; +import com.linkedin.feathr.core.utils.ConfigUtils; +import com.linkedin.feathr.core.utils.Utils; +import com.typesafe.config.Config; +import com.typesafe.config.ConfigObject; +import com.typesafe.config.ConfigValue; +import com.typesafe.config.ConfigValueType; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import javax.lang.model.SourceVersion; +import org.apache.log4j.Logger; + +import static com.linkedin.feathr.core.config.producer.derivations.DerivationConfig.*; +import static com.linkedin.feathr.core.utils.Utils.*; + + +/** + * Builds a feature derivation config object. It delegates the actual build task to its children + * depending on the type of the feature derivation. + */ +class DerivationConfigBuilder { + private final static Logger logger = Logger.getLogger(DerivationConfigBuilder.class); + + private DerivationConfigBuilder() { + } + + public static DerivationConfig build(String derivedFeatureName, Config derivationsConfig) { + String quotedDerivedFeatureName = quote(derivedFeatureName); + DerivationConfig configObj; + ConfigValue value = derivationsConfig.getValue(quotedDerivedFeatureName); + + switch (value.valueType()) { + case STRING: + String expr = derivationsConfig.getString(quotedDerivedFeatureName); + configObj = new SimpleDerivationConfig(new TypedExpr(expr, ExprType.MVEL)); + break; + + case OBJECT: + Config derivCfg = derivationsConfig.getConfig(quotedDerivedFeatureName); + + if (derivCfg.hasPath(JOIN)) { + configObj = buildWithJoin(derivedFeatureName, derivCfg); + } else if (derivCfg.hasPath(CLASS)) { + configObj = buildWithExtractor(derivedFeatureName, derivCfg); + } else if (derivCfg.hasPath(INPUTS)) { + configObj = buildWithExpr(derivedFeatureName, derivCfg); + } else if (derivCfg.hasPath(SQL_EXPR)) { + String sqlExpr = derivCfg.getString(SQL_EXPR); + FeatureTypeConfig featureTypeConfig = FeatureTypeConfigBuilder.build(derivCfg); + return new SimpleDerivationConfig(new TypedExpr(sqlExpr, ExprType.SQL), featureTypeConfig); + } else if (derivCfg.hasPath(DEFINITION)) { + String mvelExpr = derivCfg.getString(DEFINITION); + FeatureTypeConfig featureTypeConfig = FeatureTypeConfigBuilder.build(derivCfg); + return new SimpleDerivationConfig(new TypedExpr(mvelExpr, ExprType.MVEL), featureTypeConfig); + } else { + throw new ConfigBuilderException("Expected one of 'definition' or 'class' field in: " + value.render()); + } + break; + + default: + throw new ConfigBuilderException("Expected " + derivedFeatureName + " value type String or Object, got " + + value.valueType()); + } + + logger.debug("Built DerivationConfig object for derived feature " + derivedFeatureName); + + return configObj; + } + + /** + * Builds a derived feature config object for derivations expressed with key and MVEL expression + */ + private static DerivationConfigWithExpr buildWithExpr(String derivedFeatureName, Config derivationConfig) { + List key = getKey(derivationConfig); + + Config inputsConfig = derivationConfig.getConfig(INPUTS); + ConfigObject inputsConfigObj = inputsConfig.root(); + Set inputArgs = inputsConfigObj.keySet(); + + Map inputs = inputArgs.stream().collect(HashMap::new, + (map, arg) -> { + Config cfg = inputsConfig.getConfig(arg); + String keyExprOfCfg = getKeyExpr(cfg); + String inputFeature = cfg.getString(FEATURE); + KeyedFeature keyedFeature = new KeyedFeature(keyExprOfCfg, inputFeature); + map.put(arg, keyedFeature); + }, HashMap::putAll); + + String defType = derivationConfig.hasPath(SQL_DEFINITION) ? SQL_DEFINITION : DEFINITION; + ExprType defExprType = derivationConfig.hasPath(SQL_DEFINITION) ? ExprType.SQL : ExprType.MVEL; + + String definition = derivationConfig.getString(defType); + + FeatureTypeConfig featureTypeConfig = FeatureTypeConfigBuilder.build(derivationConfig); + + DerivationConfigWithExpr configObj = new DerivationConfigWithExpr(key, inputs, new TypedExpr(definition, defExprType), featureTypeConfig); + logger.trace("Built DerivationConfigWithExpr object for derived feature " + derivedFeatureName); + + return configObj; + } + + /** + * Builds a derived feature config object for derivations expressed with a udf (extractor class) + */ + private static DerivationConfigWithExtractor buildWithExtractor(String derivedFeatureName, Config derivationConfig) { + List key = getKey(derivationConfig); + + List inputsConfigList = derivationConfig.getConfigList(INPUTS); + + List inputs = inputsConfigList.stream().map(c -> new KeyedFeature(getKeyExpr(c), c.getString(FEATURE))) + .collect(Collectors.toList()); + + String name = derivationConfig.getString(CLASS); + String className; + if (SourceVersion.isName(name)) { + className = name; + } else { + throw new ConfigBuilderException("Invalid name for extractor class: " + name); + } + + FeatureTypeConfig featureTypeConfig = FeatureTypeConfigBuilder.build(derivationConfig); + + DerivationConfigWithExtractor configObj = new DerivationConfigWithExtractor(key, inputs, className, featureTypeConfig); + logger.trace("Built DerivationConfigWithExtractor object for derived feature" + derivedFeatureName); + + return configObj; + } + + /** + * Builds a sequential join config, which is a special form of derived feature config + */ + private static SequentialJoinConfig buildWithJoin(String sequentialJoinFeatureName, Config derivationConfig) { + List key = getKey(derivationConfig); + + Config joinConfig = derivationConfig.getConfig(JOIN); + // there is only two configs in joinConfigList, one is base, the other is expansion + ConfigObject joinConfigObj = joinConfig.root(); + Set joinArgs = joinConfigObj.keySet(); + + if (!joinArgs.contains(BASE) || !joinArgs.contains(EXPANSION) || joinArgs.size() != 2) { + throw new ConfigBuilderException("Sequential join config should contains both base and expansion feature config, got" + + Utils.string(joinArgs)); + } + + BaseFeatureConfig base = buildBaseFeatureConfig(joinConfig.getConfig(BASE)); + + Config expansionCfg = joinConfig.getConfig(EXPANSION); + String keyExprOfCfg = getKeyExpr(expansionCfg); + String inputFeature = expansionCfg.getString(FEATURE); + KeyedFeature expansion = new KeyedFeature(keyExprOfCfg, inputFeature); + + String aggregation = derivationConfig.getString(AGGREGATION); + + FeatureTypeConfig featureTypeConfig = FeatureTypeConfigBuilder.build(derivationConfig); + + SequentialJoinConfig configObj = new SequentialJoinConfig(key, base, expansion, aggregation, featureTypeConfig); + logger.trace("Built SequentialJoinConfig object for sequential join feature" + sequentialJoinFeatureName); + + return configObj; + } + + /** + * Build the base feature config for sequential join feature + */ + private static BaseFeatureConfig buildBaseFeatureConfig(Config baseConfig) { + String keyExpr = getKeyExpr(baseConfig); + String feature = baseConfig.getString(FEATURE); + List outputKey = baseConfig.hasPath(OUTPUT_KEY) ? getKey(baseConfig, OUTPUT_KEY) : null; + String transformation = baseConfig.hasPath(TRANSFORMATION) ? baseConfig.getString(TRANSFORMATION) : null; + String transformationClass = baseConfig.hasPath(TRANSFORMATION_CLASS) ? baseConfig.getString(TRANSFORMATION_CLASS) : null; + if (transformation != null && transformationClass != null) { + throw new ConfigBuilderException("Sequential join base feature config cannot have both transformation \"" + + transformation + "\" and transformationClass \"" + transformationClass + "\"."); + } + return new BaseFeatureConfig(keyExpr, feature, outputKey, transformation, transformationClass); + } + + /** + * get list of keys from Config object + * @param config the config + * @param keyField the key field name, in derivation config, it can be either "key" or "outputKey" + * @return the list of keys + */ + private static List getKey(Config config, String keyField) { + ConfigValueType keyValueType = config.getValue(keyField).valueType(); + List key; + switch (keyValueType) { + case STRING: + key = Collections.singletonList(config.getString(keyField)); + break; + case LIST: + key = config.getStringList(keyField); + break; + default: + throw new ConfigBuilderException("Expected key type String or List[String], got " + keyValueType); + } + return key; + } + + /** + * Get list of keys from Config object, by default(in most cases), the key field name is "key" + */ + private static List getKey(Config config) { + return getKey(config, KEY); + } + + private static String getKeyExpr(Config config) { + return ConfigUtils.getHoconString(config, KEY); + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/derivations/DerivationsConfigBuilder.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/derivations/DerivationsConfigBuilder.java new file mode 100644 index 000000000..a2ef3005c --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/derivations/DerivationsConfigBuilder.java @@ -0,0 +1,44 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.producer.derivations; + +import com.linkedin.feathr.core.config.producer.derivations.DerivationConfig; +import com.linkedin.feathr.core.config.producer.derivations.DerivationsConfig; +import com.typesafe.config.Config; +import com.typesafe.config.ConfigObject; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.log4j.Logger; + + +/** + * Builds a map of anchor name to its config by delegating the building of each anchor config object + * to its child + */ +public class DerivationsConfigBuilder { + private final static Logger logger = Logger.getLogger(DerivationsConfigBuilder.class); + + private DerivationsConfigBuilder() { + } + + /** + * config represents the object part in: + * {@code derivations : { ... }} + */ + public static DerivationsConfig build(Config config) { + logger.debug("Building DerivationConfig objects"); + ConfigObject configObj = config.root(); + + Stream derivedFeatureNames = configObj.keySet().stream(); + + Map nameConfigMap = derivedFeatureNames.collect( + Collectors.toMap(Function.identity(), + derivedFeatureName -> DerivationConfigBuilder.build(derivedFeatureName, config)) + ); + + DerivationsConfig derivationsConfig = new DerivationsConfig(nameConfigMap); + logger.debug("Built all DerivationConfig objects"); + + return derivationsConfig; + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/sources/CouchbaseConfigBuilder.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/sources/CouchbaseConfigBuilder.java new file mode 100644 index 000000000..c05e57179 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/sources/CouchbaseConfigBuilder.java @@ -0,0 +1,29 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.producer.sources; + +import com.linkedin.feathr.core.config.producer.sources.CouchbaseConfig; +import com.typesafe.config.Config; +import org.apache.log4j.Logger; + +import static com.linkedin.feathr.core.config.producer.sources.CouchbaseConfig.*; + + +/** + * Builds {@link CouchbaseConfig} objects + */ +class CouchbaseConfigBuilder { + private final static Logger logger = Logger.getLogger(CouchbaseConfigBuilder.class); + + private CouchbaseConfigBuilder() { + } + + public static CouchbaseConfig build(String sourceName, Config sourceConfig) { + String bucketName = sourceConfig.getString(BUCKET_NAME); + String keyExpr = sourceConfig.getString(KEY_EXPR); + String documentModel = sourceConfig.getString(DOCUMENT_MODEL); + + CouchbaseConfig configObj = new CouchbaseConfig(sourceName, bucketName, keyExpr, documentModel); + logger.debug("Built CouchbaseConfig object for source " + sourceName); + + return configObj; + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/sources/CustomSourceConfigBuilder.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/sources/CustomSourceConfigBuilder.java new file mode 100644 index 000000000..d19aa1a27 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/sources/CustomSourceConfigBuilder.java @@ -0,0 +1,27 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.producer.sources; + +import com.linkedin.feathr.core.config.producer.sources.CustomSourceConfig; +import com.typesafe.config.Config; +import org.apache.log4j.Logger; + +import static com.linkedin.feathr.core.config.producer.sources.CustomSourceConfig.*; + +/** + * Builds {@link CustomSourceConfig} objects + */ +class CustomSourceConfigBuilder { + private final static Logger logger = Logger.getLogger(CustomSourceConfigBuilder.class); + + private CustomSourceConfigBuilder() { + } + + public static CustomSourceConfig build(String sourceName, Config sourceConfig) { + String keyExpr = sourceConfig.getString(KEY_EXPR); + String dataModel = sourceConfig.getString(DATA_MODEL); + + CustomSourceConfig configObj = new CustomSourceConfig(sourceName, keyExpr, dataModel); + logger.debug("Built CustomSourceConfig object for source " + sourceName); + + return configObj; + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/sources/EspressoConfigBuilder.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/sources/EspressoConfigBuilder.java new file mode 100644 index 000000000..db643cf1f --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/sources/EspressoConfigBuilder.java @@ -0,0 +1,30 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.producer.sources; + +import com.linkedin.feathr.core.config.producer.sources.EspressoConfig; +import com.typesafe.config.Config; +import org.apache.log4j.Logger; + +import static com.linkedin.feathr.core.config.producer.sources.EspressoConfig.*; + + +/** + * Builds EspressoConfig objects + */ +class EspressoConfigBuilder { + private final static Logger logger = Logger.getLogger(EspressoConfigBuilder.class); + + private EspressoConfigBuilder() { + } + + public static EspressoConfig build(String sourceName, Config sourceConfig) { + String database = sourceConfig.getString(DATABASE); + String table = sourceConfig.getString(TABLE); + String d2Uri = sourceConfig.getString(D2_URI); + String keyExpr = sourceConfig.getString(KEY_EXPR); + + EspressoConfig configObj = new EspressoConfig(sourceName, database, table, d2Uri, keyExpr); + logger.debug("Built EspressoConfig object for source " + sourceName); + + return configObj; + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/sources/HdfsConfigBuilder.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/sources/HdfsConfigBuilder.java new file mode 100644 index 000000000..30432bb75 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/sources/HdfsConfigBuilder.java @@ -0,0 +1,47 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.producer.sources; + +import com.linkedin.feathr.core.config.producer.sources.HdfsConfig; +import com.linkedin.feathr.core.configbuilder.ConfigBuilderException; +import com.typesafe.config.Config; +import org.apache.log4j.Logger; + +import static com.linkedin.feathr.core.config.producer.sources.HdfsConfig.*; +import static com.linkedin.feathr.core.config.producer.sources.SlidingWindowAggrConfig.*; + + +/** + * Builds HdfsConfig objects by delegating to child builders + */ +class HdfsConfigBuilder { + private final static Logger logger = Logger.getLogger(HdfsConfigBuilder.class); + + private HdfsConfigBuilder() { + } + + public static HdfsConfig build(String sourceName, Config sourceConfig) { + boolean hasTimePartitionPattern = sourceConfig.hasPath(TIME_PARTITION_PATTERN); + boolean hasTimeSnapshot = sourceConfig.hasPath(HAS_TIME_SNAPSHOT); + boolean hasIsTimeSeries = sourceConfig.hasPath(IS_TIME_SERIES); + + // hasTimeSnapshot and isTimeSeries were used to indicate a time-partitioned source. + // isTimeSeries is used by sliding window aggregation and hasTimeSnapshot is used by time-aware join and time-based join. + // In the unification effort(https://docs.google.com/document/d/1C6u2CKWSmOmHDQEL8Ovm5V5ZZFKhC_HdxVxU9D1F9lg/edit#), + // they are replaced by the new field hasTimePartitionPattern. We only keep hasTimeSnapshot and isTimeSeries for backward-compatibility. + // TODO - 12604) we should remove the legacy fields after the users migrate to new syntax + if (hasTimePartitionPattern && (hasTimeSnapshot || hasIsTimeSeries)) { + throw new ConfigBuilderException("hasTimeSnapshot and isTimeSeries are legacy fields. They cannot coexist with timePartitionPattern. " + + "Please remove them from the source " + sourceName); + } + if (hasTimeSnapshot && hasIsTimeSeries) { + throw new ConfigBuilderException("hasTimeSnapshot and isTimeSeries cannot coexist in source " + sourceName); + } + + boolean hasSlidingWindowConfig = sourceConfig.hasPath(TIMEWINDOW_PARAMS); + + HdfsConfig configObj = hasSlidingWindowConfig ? HdfsConfigWithSlidingWindowBuilder.build(sourceName, sourceConfig) + : HdfsConfigWithRegularDataBuilder.build(sourceName, sourceConfig); + logger.debug("Built HdfsConfig object for source " + sourceName); + + return configObj; + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/sources/HdfsConfigWithRegularDataBuilder.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/sources/HdfsConfigWithRegularDataBuilder.java new file mode 100644 index 000000000..0be70002c --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/sources/HdfsConfigWithRegularDataBuilder.java @@ -0,0 +1,53 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.producer.sources; + +import com.linkedin.feathr.core.config.producer.sources.HdfsConfigWithRegularData; +import com.linkedin.feathr.core.configbuilder.ConfigBuilderException; +import com.typesafe.config.Config; +import com.typesafe.config.ConfigValueType; +import java.util.Collections; +import java.util.List; +import org.apache.log4j.Logger; + +import static com.linkedin.feathr.core.config.producer.sources.HdfsConfig.*; + + +/** + * Builds HdfsConfigWithRegularData objects. + */ +class HdfsConfigWithRegularDataBuilder { + private final static Logger logger = Logger.getLogger(HdfsConfigWithRegularDataBuilder.class); + + private HdfsConfigWithRegularDataBuilder() { + } + + public static HdfsConfigWithRegularData build(String sourceName, Config sourceConfig) { + + String path = sourceConfig.getString(PATH); + String timePartitionPattern = sourceConfig.hasPath(TIME_PARTITION_PATTERN) + ? sourceConfig.getString(TIME_PARTITION_PATTERN) : null; + boolean hasTimeSnapshot = sourceConfig.hasPath(HAS_TIME_SNAPSHOT) && sourceConfig.getBoolean(HAS_TIME_SNAPSHOT); + + HdfsConfigWithRegularData configObj = new HdfsConfigWithRegularData(sourceName, path, timePartitionPattern, hasTimeSnapshot); + logger.trace("Built HdfsConfigWithRegularData object for source" + sourceName); + + return configObj; + } + + private static List getStringList(Config sourceConfig, String field) { + ConfigValueType valueType = sourceConfig.getValue(field).valueType(); + List stringList; + switch (valueType) { + case STRING: + stringList = Collections.singletonList(sourceConfig.getString(field)); + break; + + case LIST: + stringList = sourceConfig.getStringList(field); + break; + + default: + throw new ConfigBuilderException("Expected " + field + " value type String or List, got " + valueType); + } + return stringList; + }; +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/sources/HdfsConfigWithSlidingWindowBuilder.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/sources/HdfsConfigWithSlidingWindowBuilder.java new file mode 100644 index 000000000..6c8815f75 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/sources/HdfsConfigWithSlidingWindowBuilder.java @@ -0,0 +1,33 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.producer.sources; + +import com.linkedin.feathr.core.config.producer.sources.HdfsConfigWithSlidingWindow; +import com.linkedin.feathr.core.config.producer.sources.SlidingWindowAggrConfig; +import com.typesafe.config.Config; +import org.apache.log4j.Logger; + +import static com.linkedin.feathr.core.config.producer.sources.HdfsConfig.*; + + +/** + * Build {@link HdfsConfigWithSlidingWindow} objects + */ +class HdfsConfigWithSlidingWindowBuilder { + private final static Logger logger = Logger.getLogger(HdfsConfigWithSlidingWindowBuilder.class); + + private HdfsConfigWithSlidingWindowBuilder() { + } + + public static HdfsConfigWithSlidingWindow build(String sourceName, Config sourceConfig) { + String path = sourceConfig.getString(PATH); + String timePartitionPattern = sourceConfig.hasPath(TIME_PARTITION_PATTERN) + ? sourceConfig.getString(TIME_PARTITION_PATTERN) : null; + + SlidingWindowAggrConfig swaConfigObj = SlidingWindowAggrConfigBuilder.build(sourceConfig); + + HdfsConfigWithSlidingWindow configObj = new HdfsConfigWithSlidingWindow(sourceName, path, timePartitionPattern, swaConfigObj); + + logger.trace("Built HdfsConfigWithSlidingWindow object for source " + sourceName); + + return configObj; + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/sources/KafkaConfigBuilder.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/sources/KafkaConfigBuilder.java new file mode 100644 index 000000000..45c3a314c --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/sources/KafkaConfigBuilder.java @@ -0,0 +1,32 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.producer.sources; + +import com.linkedin.feathr.core.config.producer.sources.KafkaConfig; +import com.linkedin.feathr.core.config.producer.sources.SlidingWindowAggrConfig; +import com.typesafe.config.Config; +import org.apache.log4j.Logger; + +import static com.linkedin.feathr.core.config.producer.sources.KafkaConfig.*; +import static com.linkedin.feathr.core.config.producer.sources.SlidingWindowAggrConfig.IS_TIME_SERIES; + +/** + * Builds {@link KafkaConfig} objects + */ +class KafkaConfigBuilder { + private final static Logger logger = Logger.getLogger(KafkaConfigBuilder.class); + + private KafkaConfigBuilder() { + } + + public static KafkaConfig build(String sourceName, Config sourceConfig) { + String stream = sourceConfig.getString(STREAM); + + // Sliding window aggregation config + boolean isTimeSeries = sourceConfig.hasPath(IS_TIME_SERIES) && sourceConfig.getBoolean(IS_TIME_SERIES); + SlidingWindowAggrConfig swaConfig = isTimeSeries ? SlidingWindowAggrConfigBuilder.build(sourceConfig) : null; + + KafkaConfig configObj = new KafkaConfig(sourceName, stream, swaConfig); + logger.debug("Built KafkaConfig object for source " + sourceName); + + return configObj; + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/sources/PassThroughConfigBuilder.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/sources/PassThroughConfigBuilder.java new file mode 100644 index 000000000..09436a539 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/sources/PassThroughConfigBuilder.java @@ -0,0 +1,33 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.producer.sources; + +import com.linkedin.feathr.core.config.producer.sources.PassThroughConfig; +import com.linkedin.feathr.core.configbuilder.ConfigBuilderException; +import com.typesafe.config.Config; +import javax.lang.model.SourceVersion; +import org.apache.log4j.Logger; + + +/** + * Builds {@link PassThroughConfig} objects by delegating to child builders + */ +class PassThroughConfigBuilder { + private final static Logger logger = Logger.getLogger(PassThroughConfigBuilder.class); + + private PassThroughConfigBuilder() { + } + + public static PassThroughConfig build(String sourceName, Config sourceConfig) { + String dataModel = sourceConfig.hasPath(PassThroughConfig.DATA_MODEL) + ? sourceConfig.getString(PassThroughConfig.DATA_MODEL) + : null; + + if (dataModel != null && !SourceVersion.isName(dataModel)) { + throw new ConfigBuilderException("Invalid class name for dataModel: " + dataModel); + } + + PassThroughConfig configObj = new PassThroughConfig(sourceName, dataModel); + logger.debug("Built PassThroughConfig object for source " + sourceName); + + return configObj; + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/sources/PinotConfigBuilder.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/sources/PinotConfigBuilder.java new file mode 100644 index 000000000..c5b85f984 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/sources/PinotConfigBuilder.java @@ -0,0 +1,100 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.producer.sources; + +import com.linkedin.feathr.core.config.producer.sources.PinotConfig; +import com.linkedin.feathr.core.configbuilder.ConfigBuilderException; +import com.typesafe.config.Config; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.apache.commons.lang3.StringUtils; +import org.apache.log4j.Logger; + +import static com.linkedin.feathr.core.config.producer.sources.PinotConfig.*; + +/** + * Builds {@link PinotConfig} objects + */ +public class PinotConfigBuilder { + private final static Logger logger = Logger.getLogger(PinotConfigBuilder.class); + private final static String QUERY_ARGUMENT_PLACEHOLDER = "?"; + + private PinotConfigBuilder() { + } + + public static PinotConfig build(String sourceName, Config sourceConfig) { + // first validate the sourceConfig + validate(sourceConfig); + + // construct the PinotConfig object + String resourceName = sourceConfig.getString(RESOURCE_NAME); + String queryTemplate = sourceConfig.getString(QUERY_TEMPLATE); + String[] queryArguments = sourceConfig.getStringList(QUERY_ARGUMENTS).toArray(new String[]{}); + String[] queryKeyColumns = sourceConfig.getStringList(QUERY_KEY_COLUMNS).toArray(new String[]{}); + PinotConfig configObj = new PinotConfig(sourceName, resourceName, queryTemplate, queryArguments, queryKeyColumns); + logger.debug("Built PinotConfig object for source " + sourceName); + return configObj; + } + + /** + * Validate the following: + * 1. the column names specified in queryKeyColumns need to be unique + * 2. the count of argument placeholder("?") in queryTemplate needs to match the size of queryArguments + * 3. the count of key based queryArguments needs to match the size of queryKeyColumns + * 4. "?" in queryTemplate needs to be always wrapped inside an IN clause if the argument is key based + * If validation failed, throw ConfigBuilderException. + * + * @param sourceConfig {@link Config} + */ + private static void validate(Config sourceConfig) { + List queryKeyColumnList = sourceConfig.getStringList(QUERY_KEY_COLUMNS); + if (new HashSet(queryKeyColumnList).size() != queryKeyColumnList.size()) { + throw new ConfigBuilderException( + String.format("Column name in queryKeyColumns [%s] need to be unique", queryKeyColumnList)); + } + String[] queryKeyColumns = queryKeyColumnList.toArray(new String[]{}); + + String queryTemplate = sourceConfig.getString(QUERY_TEMPLATE); + String[] queryArguments = sourceConfig.getStringList(QUERY_ARGUMENTS).toArray(new String[]{}); + // the count of argument placeholder ("?") in queryTemplate needs to match the size of queryArguments + int placeHolderCnt = StringUtils.countMatches(queryTemplate, QUERY_ARGUMENT_PLACEHOLDER); + if (placeHolderCnt != queryArguments.length) { + throw new ConfigBuilderException( + String.format("Arguments count does not match between [%s] and [%s]", queryTemplate, queryArguments)); + } + + //the count of key based queryArguments needs to match the size of queryKeyColumns + int keyBasedArgCnt = Arrays.stream(queryArguments).filter(arg -> isArgValFromKey(arg)).toArray().length; + if (keyBasedArgCnt != queryKeyColumns.length) { + throw new ConfigBuilderException( + String.format("Key based arguments count does not match between [%s] and [%s]", queryArguments, + queryKeyColumns)); + } + + // iterate through individual key based argument, and make sure the corresponding "?" in the query template is + // wrapped inside an IN clause. + Pattern p = Pattern.compile("\\b(?i)(in\\s*\\(\\s*\\?\\s*\\))"); + Matcher matcher = p.matcher(queryTemplate); + int keyColumnPlaceHolderCnt = 0; + while (matcher.find()) { + keyColumnPlaceHolderCnt++; + } + + //"?" in queryTemplate needs to be always wrapped inside an IN clause if the argument is key based + if (keyColumnPlaceHolderCnt != queryKeyColumns.length) { + throw new ConfigBuilderException( + String.format("Please make sure the key based placeholders are always wrapped inside an IN clause [%s] [%s]", queryArguments, + queryKeyColumns)); + } + } + + /** + * Check if the argument expression is key based + * @param argExpr the argument expression + * @return if the argument expression is key based + */ + private static boolean isArgValFromKey(String argExpr) { + return Pattern.compile(".*key\\[\\d.*\\].*").matcher(argExpr).find(); + } +} \ No newline at end of file diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/sources/RestliConfigBuilder.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/sources/RestliConfigBuilder.java new file mode 100644 index 000000000..c79ec759d --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/sources/RestliConfigBuilder.java @@ -0,0 +1,209 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.producer.sources; + +import com.linkedin.data.DataList; +import com.linkedin.data.DataMap; +import com.linkedin.data.schema.PathSpec; +import com.linkedin.feathr.core.configbuilder.ConfigBuilderException; +import com.linkedin.feathr.core.utils.Utils; +import com.linkedin.feathr.core.config.producer.sources.RestliConfig; +import com.typesafe.config.Config; +import com.typesafe.config.ConfigFactory; +import com.typesafe.config.ConfigObject; +import com.typesafe.config.ConfigValueType; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.BiConsumer; +import java.util.function.Function; +import java.util.stream.Collectors; +import org.apache.commons.lang3.StringUtils; +import org.apache.log4j.Logger; + +import static com.linkedin.feathr.core.config.producer.sources.RestliConfig.*; + +/** + * Builds {@link RestliConfig} objects + */ +class RestliConfigBuilder { + private final static Logger logger = Logger.getLogger(RestliConfigBuilder.class); + + private RestliConfigBuilder() { + } + + public static RestliConfig build(String sourceName, Config sourceConfig) { + String resourceName = sourceConfig.hasPath(RESOURCE_NAME) ? sourceConfig.getString(RESOURCE_NAME) + : sourceConfig.getString(RESOUCE_NAME); // TODO: we'll fix this. + + Map reqParams = sourceConfig.hasPath(REQ_PARAMS) ? buildReqParams(sourceConfig) : null; + + PathSpec pathSpec = sourceConfig.hasPath(PATH_SPEC) ? buildPathSpec(sourceConfig) : null; + + String keyExpr = null; + String finder = null; + + if (sourceConfig.hasPath(KEY_EXPR)) { + keyExpr = sourceConfig.getString(KEY_EXPR); + } else if (sourceConfig.hasPath(ENTITY_TYPE)) { + /* + * TODO: We'll remove entity type + * "restEntityType" is deprecated. Until we remove it, a restEntityType can be converted to a keyExpr + * (which is a MVEL expression). For example, if restEntityType: member, the resulting key expression + * will be: "toUrn(\"member\", key[0])" + */ + String entityType = sourceConfig.getString(ENTITY_TYPE); + keyExpr = String.format("toUrn(\"%s\", key[0])", entityType); + } + + if (sourceConfig.hasPath(FINDER)) { + finder = sourceConfig.getString(FINDER); + } + + if (StringUtils.isAllBlank(finder, keyExpr)) { + throw new ConfigBuilderException("Rest.li config cannot have both blank \"keyExpr\" and \"finder\" fields"); + } + + RestliConfig configObj = new RestliConfig(sourceName, resourceName, keyExpr, reqParams, pathSpec, finder); + + logger.debug("Built RestliConfig object for source " + sourceName); + + return configObj; + } + + private static Map buildReqParams(Config sourceConfig) { + Config reqParamsConfig = sourceConfig.getConfig(REQ_PARAMS); + ConfigObject reqParamsConfigObj = reqParamsConfig.root(); + Set reqParamsKeys = reqParamsConfigObj.keySet(); + logger.debug("reqParamsKeys: " + Utils.string(reqParamsKeys)); + + BiConsumer, String> accumulator = (acc, key) -> { + ConfigValueType configValueType = reqParamsConfig.getValue(key).valueType(); + + switch (configValueType) { + case STRING: + acc.put(key, reqParamsConfig.getString(key)); + break; + + case OBJECT: + Config paramConfig = reqParamsConfig.getConfig(key); + String keyWord = paramConfig.root().keySet().iterator().next(); + + switch (keyWord) { + case JSON: + ConfigValueType valueType = paramConfig.getValue(JSON).valueType(); + Config config; + if (valueType == ConfigValueType.OBJECT) { + config = paramConfig.getConfig(JSON); + } else { + /* + * Assumed to be string which contains a config, so parse it + * Note: this notation should not be allowed, HOCON notation should be used to specify the object. + * Due to this, the code has become bloated. + */ + config = ConfigFactory.parseString(paramConfig.getString(JSON)); + } + DataMap dataMap = buildDataMap(config); + acc.put(key, dataMap); + break; + + case JSON_ARRAY: + ConfigValueType jsonArrayValueType = paramConfig.getValue(JSON_ARRAY).valueType(); + Config jsonArrayConfig; + if (jsonArrayValueType == ConfigValueType.OBJECT) { + jsonArrayConfig = paramConfig.getConfig(JSON_ARRAY); + } else { + /* + * Assumed to be string which contains a config, so parse it + * Note: this notation should not be allowed, HOCON notation should be used to specify the object. + * Due to this, the code has become bloated. + */ + jsonArrayConfig = ConfigFactory.parseString(paramConfig.getString(JSON_ARRAY)); + } + DataList dataList = buildDataList(jsonArrayConfig); + acc.put(key, dataList); + break; + + case MVEL_KEY: + String mvelExpr = paramConfig.getString(MVEL_KEY); + // when the param is an MVEL expression, store it as a DataMap={"mvel"-> EXPR} instead of just a raw string + // to differentiate it from the case where it is truly just a static String + DataMap mvelDataMap = new DataMap(); + mvelDataMap.put(MVEL_KEY, mvelExpr); + acc.put(key, mvelDataMap); + break; + + case FILE: + StringBuilder warnSb = new StringBuilder(); + warnSb.append("Handling of keyword ").append(FILE).append(" in ").append(REQ_PARAMS) + .append(" is not yet implemented"); + logger.warn(warnSb.toString()); + break; + + default: + StringBuilder errSb = new StringBuilder(); + errSb.append("Unsupported key ").append(keyWord).append(". Keys in ").append(REQ_PARAMS) + .append(" object must be one of ").append(JSON).append(", ").append(JSON_ARRAY).append(", ") + .append(MVEL_KEY).append(", or ").append(FILE); + throw new ConfigBuilderException(errSb.toString()); + } + break; + + default: + throw new ConfigBuilderException("Expected value type 'String' or 'Object'; found " + configValueType); + + } + }; + + return reqParamsKeys.stream().collect(HashMap::new, accumulator, Map::putAll); + } + + /* + * jsonConfig refers to the value part of key 'json': + * json: { // } + */ + private static DataMap buildDataMap(Config jsonConfig) { + Set keys = jsonConfig.root().keySet(); + Map map = keys.stream().collect(Collectors.toMap(Function.identity(), jsonConfig::getString)); + return new DataMap(map); + } + + /* + * jsonArrayConfig refers to the value part of key 'jsonArray': + * jsonArray: { array: [ // ] } + */ + private static DataList buildDataList(Config jsonArrayConfig) { + List listOfConfigs = jsonArrayConfig.getConfigList(JSON_ARRAY_ARRAY); + List listOfDataMaps = listOfConfigs.stream().map(config -> { + Set keys = config.root().keySet(); + // TODO simplify converting from DataList to DataMap + Map dm = keys.stream().collect(Collectors.toMap(Function.identity(), k -> config.getString(k))); + return new DataMap(dm); + }).collect(Collectors.toList()); + + return new DataList(listOfDataMaps); + } + + private static PathSpec buildPathSpec(Config sourceConfig) { + PathSpec pathSpec; + ConfigValueType configValueType = sourceConfig.getValue(PATH_SPEC).valueType(); + switch (configValueType) { + case STRING: + String pathSpecStr = sourceConfig.getString(PATH_SPEC); + pathSpec = new PathSpec(pathSpecStr); + break; + + case LIST: + List pathSpecList = sourceConfig.getStringList(PATH_SPEC); + String[] pathSpecArray = new String[pathSpecList.size()]; + pathSpecArray = pathSpecList.toArray(pathSpecArray); + pathSpec = new PathSpec(pathSpecArray); + break; + + default: + throw new ConfigBuilderException(PATH_SPEC + " must be of 'String' or 'List', got " + configValueType); + } + + return pathSpec; + } + +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/sources/RocksDbConfigBuilder.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/sources/RocksDbConfigBuilder.java new file mode 100644 index 000000000..464ecc990 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/sources/RocksDbConfigBuilder.java @@ -0,0 +1,48 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.producer.sources; + +import com.linkedin.feathr.core.config.producer.sources.RocksDbConfig; +import com.linkedin.feathr.core.configbuilder.ConfigBuilderException; +import com.typesafe.config.Config; +import javax.lang.model.SourceVersion; +import org.apache.log4j.Logger; + +import static com.linkedin.feathr.core.config.producer.sources.RocksDbConfig.*; + +/** + * Builds {@link RocksDbConfig} objects + */ +class RocksDbConfigBuilder { + private final static Logger logger = Logger.getLogger(RocksDbConfigBuilder.class); + + private RocksDbConfigBuilder() { + } + + public static RocksDbConfig build(String sourceName, Config sourceConfig) { + String referenceSource = sourceConfig.getString(REFERENCE_SOURCE); + Boolean extractFeatures = sourceConfig.getBoolean(EXTRACT_FEATURES); + + String encoder = getCodec(sourceConfig, ENCODER); + + String decoder = getCodec(sourceConfig, DECODER); + + String keyExpr = getCodec(sourceConfig, KEYEXPR); + + RocksDbConfig configObj = new RocksDbConfig(sourceName, referenceSource, extractFeatures, encoder, decoder, keyExpr); + logger.debug("Built RocksDbConfig object for source" + sourceName); + + return configObj; + } + + private static String getCodec(Config sourceConfig, String codec) { + if (sourceConfig.hasPath(codec)) { + String name = sourceConfig.getString(codec); + if (SourceVersion.isName(name)) { + return name; + } else { + throw new ConfigBuilderException("Invalid name for " + codec + " : " + name); + } + } else { + return null; + } + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/sources/SlidingWindowAggrConfigBuilder.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/sources/SlidingWindowAggrConfigBuilder.java new file mode 100644 index 000000000..e9e5dd875 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/sources/SlidingWindowAggrConfigBuilder.java @@ -0,0 +1,45 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.producer.sources; + +import com.linkedin.feathr.core.config.producer.sources.SlidingWindowAggrConfig; +import com.linkedin.feathr.core.config.producer.sources.TimeWindowParams; +import com.typesafe.config.Config; +import org.apache.log4j.Logger; + +import static com.linkedin.feathr.core.config.producer.sources.SlidingWindowAggrConfig.*; +import static com.linkedin.feathr.core.config.producer.sources.TimeWindowParams.*; + + +/** + * Build {@link SlidingWindowAggrConfig} object + */ +class SlidingWindowAggrConfigBuilder { + private final static Logger logger = Logger.getLogger(SlidingWindowAggrConfigBuilder.class); + + private final static String LEGACY_TIMESTAMP_FIELD = "timestamp"; + private final static String LEGACY_TIMESTAMP_FORMAT = "timestamp_format"; + + private SlidingWindowAggrConfigBuilder() { + } + + public static SlidingWindowAggrConfig build(Config sourceConfig) { + Boolean isTimeSeries = sourceConfig.hasPath(IS_TIME_SERIES) && sourceConfig.getBoolean(IS_TIME_SERIES); + Config timeWindowConfig = sourceConfig.getConfig(TIMEWINDOW_PARAMS); + String timestampField; + String timestampFormat; + if (timeWindowConfig.hasPath(LEGACY_TIMESTAMP_FIELD)) { + // TODO - 12604) we should remove the legacy fields after the users migrate to new syntax + timestampField = timeWindowConfig.getString(LEGACY_TIMESTAMP_FIELD); + timestampFormat = timeWindowConfig.getString(LEGACY_TIMESTAMP_FORMAT); + } else { + timestampField = timeWindowConfig.getString(TIMESTAMP_FIELD); + timestampFormat = timeWindowConfig.getString(TIMESTAMP_FORMAT); + } + + TimeWindowParams timeWindowParams = new TimeWindowParams(timestampField, timestampFormat); + + SlidingWindowAggrConfig configObj = new SlidingWindowAggrConfig(isTimeSeries, timeWindowParams); + logger.trace("Built SlidingWindowAggrConfig object"); + + return configObj; + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/sources/SourceConfigBuilder.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/sources/SourceConfigBuilder.java new file mode 100644 index 000000000..b0fa8f8c6 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/sources/SourceConfigBuilder.java @@ -0,0 +1,84 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.producer.sources; + +import com.linkedin.feathr.core.config.producer.sources.HdfsConfig; +import com.linkedin.feathr.core.config.producer.sources.SourceConfig; +import com.linkedin.feathr.core.config.producer.sources.SourceType; +import com.linkedin.feathr.core.configbuilder.ConfigBuilderException; +import com.typesafe.config.Config; +import org.apache.log4j.Logger; + +import static com.linkedin.feathr.core.config.producer.sources.SourceConfig.*; + + +/** + * Build {@link SourceConfig} object + */ +class SourceConfigBuilder { + private final static Logger logger = Logger.getLogger(SourceConfigBuilder.class); + + private SourceConfigBuilder() { + } + + public static SourceConfig build(String sourceName, Config sourceConfig) { + SourceConfig configObj; + if (sourceConfig.hasPath(TYPE)) { + String sourceTypeStr = sourceConfig.getString(TYPE); + + SourceType sourceType = SourceType.valueOf(sourceTypeStr); + switch (sourceType) { + case HDFS: + configObj = HdfsConfigBuilder.build(sourceName, sourceConfig); + break; + + case ESPRESSO: + configObj = EspressoConfigBuilder.build(sourceName, sourceConfig); + break; + + case RESTLI: + configObj = RestliConfigBuilder.build(sourceName, sourceConfig); + break; + + case VENICE: + configObj = VeniceConfigBuilder.build(sourceName, sourceConfig); + break; + + case KAFKA: + configObj = KafkaConfigBuilder.build(sourceName, sourceConfig); + break; + + case ROCKSDB: + configObj = RocksDbConfigBuilder.build(sourceName, sourceConfig); + break; + + case PASSTHROUGH: + configObj = PassThroughConfigBuilder.build(sourceName, sourceConfig); + break; + + case COUCHBASE: + configObj = CouchbaseConfigBuilder.build(sourceName, sourceConfig); + break; + + case CUSTOM: + configObj = CustomSourceConfigBuilder.build(sourceName, sourceConfig); + break; + + case PINOT: + configObj = PinotConfigBuilder.build(sourceName, sourceConfig); + break; + + default: + throw new ConfigBuilderException("Unknown source type " + sourceTypeStr); + } + + } else { + // TODO: Remove. We'll make 'type' mandatory field. + // default handling: it's assumed to be HDFS + if (sourceConfig.hasPath(HdfsConfig.PATH)) { + configObj = HdfsConfigBuilder.build(sourceName, sourceConfig); + } else { + throw new ConfigBuilderException("Unsupported source type for source " + sourceName); + } + } + return configObj; + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/sources/SourcesConfigBuilder.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/sources/SourcesConfigBuilder.java new file mode 100644 index 000000000..0349bc378 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/sources/SourcesConfigBuilder.java @@ -0,0 +1,44 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.producer.sources; + +import com.linkedin.feathr.core.config.producer.sources.SourceConfig; +import com.linkedin.feathr.core.config.producer.sources.SourcesConfig; +import com.typesafe.config.Config; +import com.typesafe.config.ConfigObject; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.log4j.Logger; + +import static com.linkedin.feathr.core.utils.Utils.*; + + +/** + * Builds a map of source name to {@link SourceConfig} object. Each SourceConfig object is built by a child builder, + * specific to the type of the source. + */ +public class SourcesConfigBuilder { + private final static Logger logger = Logger.getLogger(SourcesConfigBuilder.class); + + private SourcesConfigBuilder() { + } + + /** + * config represents the object part in: + * {@code sources : { ... } } + */ + public static SourcesConfig build(Config config) { + ConfigObject configObj = config.root(); + Stream sourceNames = configObj.keySet().stream(); + + Map nameConfigMap = sourceNames.collect( + Collectors.toMap(Function.identity(), + sourceName -> SourceConfigBuilder.build(sourceName, config.getConfig(quote(sourceName)))) + ); + + SourcesConfig sourcesConfig = new SourcesConfig(nameConfigMap); + logger.debug("Built all SourceConfig objects"); + + return sourcesConfig; + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/sources/VeniceConfigBuilder.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/sources/VeniceConfigBuilder.java new file mode 100644 index 000000000..699cd50f6 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/sources/VeniceConfigBuilder.java @@ -0,0 +1,27 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.producer.sources; + +import com.linkedin.feathr.core.config.producer.sources.VeniceConfig; +import com.typesafe.config.Config; +import org.apache.log4j.Logger; + +import static com.linkedin.feathr.core.config.producer.sources.VeniceConfig.*; + +/** + * Builds {@link VeniceConfig} objects + */ +class VeniceConfigBuilder { + private final static Logger logger = Logger.getLogger(VeniceConfigBuilder.class); + + private VeniceConfigBuilder() { + } + + public static VeniceConfig build(String sourceName, Config sourceConfig) { + String storeName = sourceConfig.getString(STORE_NAME); + String keyExpr = sourceConfig.getString(KEY_EXPR); + + VeniceConfig configObj = new VeniceConfig(sourceName, storeName, keyExpr); + logger.debug("Built VeniceConfig object for source " + sourceName); + + return configObj; + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configdataprovider/BaseConfigDataProvider.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configdataprovider/BaseConfigDataProvider.java new file mode 100644 index 000000000..f1b3f633b --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configdataprovider/BaseConfigDataProvider.java @@ -0,0 +1,37 @@ +package com.linkedin.feathr.core.configdataprovider; + +import java.io.IOException; +import java.io.Reader; +import java.util.ArrayList; +import java.util.List; +import org.apache.log4j.Logger; + + +/** + * A base class for {@link ConfigDataProvider} that concrete classes should extend rather than implementing + * ConfigDataProvider directly. It implements the {@link java.io.Closeable#close()} method that concrete classes typically + * shouldn't have to worry about. + */ +public abstract class BaseConfigDataProvider implements ConfigDataProvider { + private static final Logger logger = Logger.getLogger(BaseConfigDataProvider.class); + + protected List _readers; + + public BaseConfigDataProvider() { + _readers = new ArrayList<>(); + } + + @Override + public void close() { + try { + for (Reader reader : _readers) { + reader.close(); + } + } catch (IOException e) { + logger.warn("Unable to close a reader"); + } + logger.debug("Closed " + _readers.size() + " readers"); + + _readers.clear(); + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configdataprovider/ConfigDataProvider.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configdataprovider/ConfigDataProvider.java new file mode 100644 index 000000000..4a78e0d31 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configdataprovider/ConfigDataProvider.java @@ -0,0 +1,39 @@ +package com.linkedin.feathr.core.configdataprovider; + +import com.linkedin.feathr.core.configbuilder.ConfigBuilder; +import java.io.Closeable; +import java.io.Reader; +import java.util.List; + + +/** + * ConfigDataProvider abstracts aways the source of config data which may come from, for example, a resource, or a URL, + * or as a String. Doing so allows {@link ConfigBuilder ConfigBuilder} API to + * have a narrow surface area. Further, it also allows clients to plug in their own custom ConfigDataProviders. + * + * Example usage: + * 
{@code
+ * ConfigBuilder configBuilder = ConfigBuilder.get();
+ *
+ * try (ConfigDataProvider cdp = new ResourceConfigDataProvider("config/offline/myFeatures.conf")) {
+ *  FeatureDef configObj = configBuilder.buildFeatureDefConfig(cdp);
+ * } catch (Exception e) {
+ *   // process exception
+ * }
+ * }
+ */ +public interface ConfigDataProvider extends Closeable { + /** + * Return the config data as a list of {@link Reader} objects. Clients should ideally provide + * {@link java.io.BufferedReader BufferedReader} objects. + * @return List of Readers + */ + List getConfigDataReaders(); + + /** + * Provides some information about config data. This information is used in logging and debugging. For example, a + * {@link UrlConfigDataProvider} will provide a list of URLs from which the config data is obtained. + * @return A String representing config data + */ + String getConfigDataInfo(); +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configdataprovider/ConfigDataProviderException.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configdataprovider/ConfigDataProviderException.java new file mode 100644 index 000000000..ea9b7ff6a --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configdataprovider/ConfigDataProviderException.java @@ -0,0 +1,14 @@ +package com.linkedin.feathr.core.configdataprovider; + +/** + * Runtime Exception thrown by a {@link ConfigDataProvider} object when an error is encountered in fetching config data. + */ +public class ConfigDataProviderException extends RuntimeException { + public ConfigDataProviderException(String message) { + super(message); + } + + public ConfigDataProviderException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configdataprovider/ManifestConfigDataProvider.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configdataprovider/ManifestConfigDataProvider.java new file mode 100644 index 000000000..1071647c5 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configdataprovider/ManifestConfigDataProvider.java @@ -0,0 +1,176 @@ +package com.linkedin.feathr.core.configdataprovider; + +import com.typesafe.config.Config; +import com.typesafe.config.ConfigFactory; +import com.typesafe.config.ConfigRenderOptions; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.Reader; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; +import org.apache.log4j.Logger; + + +/** + * A Config Data Provider that reads a manifest file, and provides Reader objects for each config file listed in the + * said manifest. + *

+ * An example manifest file is shown below: It'll contain at most FeatureDef and Metadata config file locations, + * never Join config file locations. + * + * 

{@code
+ * manifest: [
+ *   {
+ *     jar: local
+ *     conf: [config/online/feature-prod.conf]
+ *   },
+ *   {
+ *     jar: frame-feature-waterloo-online-1.1.4.jar
+ *     conf: [config/online/prod/feature-prod.conf]
+ *   }
+ * ]
+ * }
+ * + */ +/* + * TODO: The manifest file currently lumps all config files in the "conf" field. It should be modified to list + * FeatureDef and Metadata config files in "featureDefConf" and "metadataConf" fields respectively. This will also + * necessitate changes in ConfigDataProvider interface. + */ +public class ManifestConfigDataProvider extends BaseConfigDataProvider { + private static final Logger logger = Logger.getLogger(ManifestConfigDataProvider.class); + + /* + * The various config keys and value in the manifest file + */ + private static final String MANIFEST_KEY = "manifest"; + private static final String JAR_KEY = "jar"; + private static final String CONF_KEY = "conf"; + private static final String LOCAL_VALUE = "local"; + + private String _manifestResourceName; + + private Config _manifestConfig; + + private List _jarFiles; + + public ManifestConfigDataProvider(String manifestResourceName) { + Objects.requireNonNull(manifestResourceName, "Manifest resource name can't be null"); + + _manifestResourceName = manifestResourceName; + + _jarFiles = new ArrayList<>(); + + ConfigRenderOptions renderOptions = ConfigRenderOptions.defaults() + .setComments(false) + .setOriginComments(false) + .setFormatted(true) + .setJson(true); + + _manifestConfig = ConfigFactory.parseResources(manifestResourceName); + logger.debug("Manifest config: \n" + _manifestConfig.root().render(renderOptions.setJson(false))); + } + + @Override + public List getConfigDataReaders() { + List jarConfConfigList = _manifestConfig.getConfigList(MANIFEST_KEY); + + ClassLoader loader = Thread.currentThread().getContextClassLoader(); + + /* + * Iterate over all jar-conf pairs. If the jar file is 'local', that is, it's the current library + * then read the conf files as resources else read them from the specified jar file. In both cases, + * build a Reader object for each conf file. + */ + for (Config jarConfConfig : jarConfConfigList) { + String jarFileName = jarConfConfig.getString(JAR_KEY); + + List confFileNames = jarConfConfig.getStringList(CONF_KEY); + + if (jarFileName.equalsIgnoreCase(LOCAL_VALUE)) { + createReaders(loader, confFileNames, _readers); + } else { + createReaders(loader, jarFileName, confFileNames, _readers); + } + } + + return _readers; + } + + @Override + public String getConfigDataInfo() { + return "Manifest: " + _manifestResourceName; + } + + /* + * This method is provided here so that JarFile objects, if any, can be closed. + */ + @Override + public void close() { + super.close(); + + try { + for (JarFile jf : _jarFiles) { + jf.close(); + } + } catch (IOException e) { + logger.warn("Unable to close a jar file"); + } + logger.debug("Closed " + _jarFiles.size() + " jar files"); + + _jarFiles.clear(); + } + + private void createReaders(ClassLoader loader, List confFileNames, List readers) { + for (String resName : confFileNames) { + InputStream in = loader.getResourceAsStream(resName); + if (in == null) { + throw new ConfigDataProviderException("Config file " + resName + " can't be obtained as an input stream"); + } + + Reader reader = new BufferedReader(new InputStreamReader(in)); + // Since the conf files are local, they may be overrides. As such add them to the head of the list. + readers.add(0, reader); + } + } + + private void createReaders(ClassLoader loader, String jarFileName, List confFileNames, + List readers) { + // load the jar file as a URL, and check for validity + URL jarFileUrl = loader.getResource(jarFileName); + if (jarFileUrl == null) { + throw new ConfigDataProviderException("Unable to load jar file " + jarFileName); + } + + /* + * Create JarFile -> InputStream -> InputStreamReader -> wrap in BufferedReader + */ + String jarFilePath = jarFileUrl.getPath(); + + /* + * Create a JarFile object that is used to get a JarEntry for each conf file. Each JarEntry + * is used to get an InputStream which is then wrapped by InputStreamReader and BufferedReader. + */ + try { + JarFile jarFile = new JarFile(jarFilePath); + _jarFiles.add(jarFile); // Hold on to these JarFile objects, they'll be closed during close() invocation + + for (String confFileName : confFileNames) { + JarEntry entry = jarFile.getJarEntry(confFileName); + + InputStream inStream = jarFile.getInputStream(entry); + InputStreamReader inStreamReader = new InputStreamReader(inStream); + BufferedReader reader = new BufferedReader(inStreamReader); + readers.add(reader); + } + } catch (Exception e) { + throw new ConfigDataProviderException("Error in creating config file readers from jar " + jarFileName, e); + } + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configdataprovider/ReaderConfigDataProvider.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configdataprovider/ReaderConfigDataProvider.java new file mode 100644 index 000000000..3db3e3ff9 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configdataprovider/ReaderConfigDataProvider.java @@ -0,0 +1,38 @@ +package com.linkedin.feathr.core.configdataprovider; + +import com.linkedin.feathr.core.configbuilder.ConfigBuilder; +import java.io.Reader; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + + +/** + * A Config Data Provider that obtains config data from Reader objects. It merely exposes the same Reader objects + * to its clients, and is provided for consistent usage of + * {@link ConfigBuilder ConfigBuilder} API. + */ +public class ReaderConfigDataProvider extends BaseConfigDataProvider { + + public ReaderConfigDataProvider(Reader reader) { + this(Collections.singletonList(reader)); + } + + public ReaderConfigDataProvider(List readers) { + Objects.requireNonNull(readers, "List of Readers can't be null"); + for (Reader r : readers) { + Objects.requireNonNull(r, "A Reader object can't be null"); + } + _readers = readers; + } + + @Override + public List getConfigDataReaders() { + return _readers; + } + + @Override + public String getConfigDataInfo() { + return "Reader object(s)"; + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configdataprovider/ResourceConfigDataProvider.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configdataprovider/ResourceConfigDataProvider.java new file mode 100644 index 000000000..be0f1400a --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configdataprovider/ResourceConfigDataProvider.java @@ -0,0 +1,86 @@ +package com.linkedin.feathr.core.configdataprovider; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.Reader; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import org.apache.log4j.Logger; + +import static com.linkedin.feathr.core.utils.Utils.*; + + +/** + * A Config Data Provider that obtains config data from resource files, that is, config files that are on the + * classpath. The config data from each resource is obtained via a {@link Reader} object. Optionally we can pass + * in a custom {@link ClassLoader} object when resources need to be loaded from a specific or isolated namespace. + */ +public class ResourceConfigDataProvider extends BaseConfigDataProvider { + private static final Logger logger = Logger.getLogger(ResourceConfigDataProvider.class); + + private final List _resourceNames; + private final ClassLoader _classLoader; + + public ResourceConfigDataProvider(String resourceName) { + this(Collections.singletonList(resourceName), null); + } + + public ResourceConfigDataProvider(String resourceName, ClassLoader classLoader) { + this(Collections.singletonList(resourceName), classLoader); + } + + public ResourceConfigDataProvider(List resourceNames) { + this(resourceNames, null); + } + + public ResourceConfigDataProvider(List resourceNames, ClassLoader classLoader) { + Objects.requireNonNull(resourceNames, "List of resource names can't be null"); + for (String resName : resourceNames) { + Objects.requireNonNull(resName, "Resource name can't be null"); + } + _resourceNames = resourceNames; + // Use the invoking thread's context class loader when custom class loader is not provided + _classLoader = classLoader != null ? classLoader : Thread.currentThread().getContextClassLoader(); + } + + @Override + public List getConfigDataReaders() { + for (String resName : _resourceNames) { + InputStream in = _classLoader.getResourceAsStream(resName); + if (in == null) { + throw new ConfigDataProviderException("Resource " + resName + " can't be obtained as an input stream"); + } + + Reader reader = new BufferedReader(new InputStreamReader(in)); + logger.debug("Created Reader object for resource " + resName); + + _readers.add(reader); + } + + return _readers; + } + + @Override + public String getConfigDataInfo() { + return "Resources: " + string(_resourceNames) + " Classloader: " + _classLoader; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + ResourceConfigDataProvider that = (ResourceConfigDataProvider) o; + return _resourceNames.equals(that._resourceNames) && _classLoader.equals(that._classLoader); + } + + @Override + public int hashCode() { + return Objects.hash(_resourceNames, _classLoader); + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configdataprovider/StringConfigDataProvider.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configdataprovider/StringConfigDataProvider.java new file mode 100644 index 000000000..e82b6df65 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configdataprovider/StringConfigDataProvider.java @@ -0,0 +1,50 @@ +package com.linkedin.feathr.core.configdataprovider; + +import java.io.BufferedReader; +import java.io.Reader; +import java.io.StringReader; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; +import org.apache.log4j.Logger; + + +/** + * A Config Data Provider that obtains config data from config string. The config data from each string is obtained + * via a {@link Reader} object. + */ +public class StringConfigDataProvider extends BaseConfigDataProvider { + private static final Logger logger = Logger.getLogger(StringConfigDataProvider.class); + + private final List _configStringList; + + public StringConfigDataProvider(String configString) { + this(Collections.singletonList(configString)); + } + + public StringConfigDataProvider(List configStringList) { + Objects.requireNonNull(configStringList, "List of config strings can't be null"); + for (String configString : configStringList) { + Objects.requireNonNull(configString, "Config string can't be null"); + } + _configStringList = configStringList; + } + + @Override + public List getConfigDataReaders() { + _readers = _configStringList.stream().map(StringReader::new).map(BufferedReader::new).collect(Collectors.toList()); + logger.debug("Created Reader object(s) for config string(s)"); + + return _readers; + } + + @Override + public String getConfigDataInfo() { + String firstConfigString = _configStringList.get(0); + int endIdx = Math.min(256, firstConfigString.length()); + String substring = firstConfigString.substring(0, endIdx).trim().replace("\n", " "); + + return "Config strings: \"" + substring + "...\""; + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configdataprovider/UrlConfigDataProvider.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configdataprovider/UrlConfigDataProvider.java new file mode 100644 index 000000000..f09d0b899 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configdataprovider/UrlConfigDataProvider.java @@ -0,0 +1,65 @@ +package com.linkedin.feathr.core.configdataprovider; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.Reader; +import java.net.URL; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import org.apache.log4j.Logger; + +import static com.linkedin.feathr.core.utils.Utils.*; + + +/** + * A Config Data Provider that obtains config data from URLs. The config data from each URL is obtained via a + * {@link Reader} object. + */ +public class UrlConfigDataProvider extends BaseConfigDataProvider { + private static final Logger logger = Logger.getLogger(UrlConfigDataProvider.class); + + private final List _urls; + + public UrlConfigDataProvider(URL url) { + this(Collections.singletonList(url)); + } + + public UrlConfigDataProvider(List urls) { + Objects.requireNonNull(urls, "url list can't be null"); + for (URL url : urls) { + Objects.requireNonNull(url, "url can't be null"); + } + + _urls = urls; + } + + @Override + public List getConfigDataReaders() { + for (URL url : _urls) { + try { + InputStream in = url.openStream(); + + Reader reader = new BufferedReader(new InputStreamReader(in)); + logger.debug("Created Reader object for URL " + url); + + _readers.add(reader); + } catch (IOException e) { + throw new ConfigDataProviderException("Error creating a Reader from URL " + url, e); + } + } + + return _readers; + } + + @Override + public String getConfigDataInfo() { + return "URLs: " + string(_urls); + } + + public List getUrls() { + return Collections.unmodifiableList(_urls); + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configvalidator/ClientType.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configvalidator/ClientType.java new file mode 100644 index 000000000..80beb792e --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configvalidator/ClientType.java @@ -0,0 +1,10 @@ +package com.linkedin.feathr.core.configvalidator; + +/** + * Enum for the type of Frame client. + * Different validations might be performed to different Frame client types + */ +public enum ClientType { + FEATURE_PRODUCER, + FEATURE_CONSUMER +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configvalidator/ConfigValidationException.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configvalidator/ConfigValidationException.java new file mode 100644 index 000000000..7c17c9ed9 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configvalidator/ConfigValidationException.java @@ -0,0 +1,15 @@ +package com.linkedin.feathr.core.configvalidator; + +/** + * Runtime exception thrown if the config validation couldn't be performed. Any exceptions encountered during validation + * itself will be provided in {@link ValidationResult} + */ +public class ConfigValidationException extends RuntimeException { + public ConfigValidationException(String message) { + super(message); + } + + public ConfigValidationException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configvalidator/ConfigValidator.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configvalidator/ConfigValidator.java new file mode 100644 index 000000000..c5e985b58 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configvalidator/ConfigValidator.java @@ -0,0 +1,66 @@ +package com.linkedin.feathr.core.configvalidator; + +import com.linkedin.feathr.core.configvalidator.typesafe.FeatureConsumerConfValidator; +import com.linkedin.feathr.core.configvalidator.typesafe.FeatureProducerConfValidator; +import com.linkedin.feathr.core.configvalidator.typesafe.TypesafeConfigValidator; +import com.linkedin.feathr.core.config.ConfigType; +import com.linkedin.feathr.core.configdataprovider.ConfigDataProvider; +import java.util.Map; + + +/** + * Validates Frame configuration such as FeatureDef config, Join config, etc. Provides capability to perform both + * syntactic and semantic validations. + */ +public interface ConfigValidator { + + /** + * Validates the configuration. Configuration type is provided by {@link ConfigType}, the validation to be performed + * (for example, syntactic) is provided by {@link ValidationType}, and the configuration to be validated is provided + * by {@link ConfigDataProvider}. Note that the client is responsible for closing the ConfigDataProvider resource. + * @param configType ConfigType + * @param validationType ValidationType + * @param configDataProvider ConfigDataProvider + * @return {@link ValidationResult} + * @throws ConfigValidationException if validation can't be performed + */ + ValidationResult validate(ConfigType configType, ValidationType validationType, + ConfigDataProvider configDataProvider); + + /** + * Validates multiple Frame configuration types individually. Note that the client is responsible for closing the + * ConfigDataProvider resources. + * @param configTypeWithDataProvider Provides a K-V pair of {@link ConfigType} and {@link ConfigDataProvider} + * @param validationType The validation to be performed {@link ValidationType} + * @return Map of ConfigType and the {@link ValidationResult} + * @throws ConfigValidationException if validation can't be performed + */ + Map validate(Map configTypeWithDataProvider, + ValidationType validationType); + + /** + * Factory method to get an instance of ConfigValidator + * @return an instance of ConfigValidator + * @deprecated please use {{@link #getInstance(ClientType)}} instead + */ + @Deprecated + static ConfigValidator getInstance() { + return new TypesafeConfigValidator(); + } + + /** + * Factory method to get an instance of ConfigValidator + * @param clientType the Frame client type {@link ClientType} + * @return an instance of ConfigValidator + */ + static ConfigValidator getInstance(ClientType clientType) { + switch (clientType) { + case FEATURE_PRODUCER: + return new FeatureProducerConfValidator(); + case FEATURE_CONSUMER: + return new FeatureConsumerConfValidator(); + default: + throw new UnsupportedOperationException("Frame client type not support: " + clientType.toString()); + } + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configvalidator/ConfigValidatorFactory.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configvalidator/ConfigValidatorFactory.java new file mode 100644 index 000000000..36da95508 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configvalidator/ConfigValidatorFactory.java @@ -0,0 +1,46 @@ +package com.linkedin.feathr.core.configvalidator; + +import com.linkedin.feathr.core.configvalidator.typesafe.FeatureConsumerConfValidator; +import com.linkedin.feathr.core.configvalidator.typesafe.FeatureProducerConfValidator; + + +/** + * Factory class for {@link ConfigValidator} to replace the usage of the static method of + * {@link ConfigValidator#getInstance(ClientType clientType)} + * Since the above getInstance method is used in li-frame-plugin, which is written in Groovy. + * And Groovy has a known bug to not fully support calling static method with parameters (introduced in Java 8). + * One discussion can be found here: + * https://community.smartbear.com/t5/SoapUI-Pro/ERROR-groovy-lang-MissingMethodException-No-signature-of-method/td-p/187960 + */ +public class ConfigValidatorFactory { + + private static ConfigValidatorFactory _instance = new ConfigValidatorFactory(); + + // Singleton with static factory + private ConfigValidatorFactory() { + + } + + /** + * get singleton instance + */ + public static ConfigValidatorFactory getFactoryInstance() { + return _instance; + } + + /** + * to get an instance of ConfigValidator + * @param clientType the Frame client type {@link ClientType} + * @return an instance of ConfigValidator + */ + public ConfigValidator getValidatorInstance(ClientType clientType) { + switch (clientType) { + case FEATURE_PRODUCER: + return new FeatureProducerConfValidator(); + case FEATURE_CONSUMER: + return new FeatureConsumerConfValidator(); + default: + throw new UnsupportedOperationException("Frame client type not support: " + clientType.toString()); + } + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configvalidator/ValidationResult.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configvalidator/ValidationResult.java new file mode 100644 index 000000000..f1bdcac68 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configvalidator/ValidationResult.java @@ -0,0 +1,81 @@ +package com.linkedin.feathr.core.configvalidator; + +import java.util.Objects; +import java.util.Optional; +import java.util.StringJoiner; + + +/** + * Class to hold the configuration validation results + */ +public class ValidationResult { + private ValidationType _type; + private ValidationStatus _status; + private String _details; + private final Throwable _cause; + + // default valid results for different validation types + public static final ValidationResult VALID_SYNTAX = new ValidationResult(ValidationType.SYNTACTIC, ValidationStatus.VALID); + public static final ValidationResult VALID_SEMANTICS = new ValidationResult(ValidationType.SEMANTIC, ValidationStatus.VALID); + + public ValidationResult(ValidationType type, ValidationStatus status) { + this(type, status, null, null); + } + + public ValidationResult(ValidationType type, ValidationStatus status, String details) { + this(type, status, details, null); + } + + public ValidationResult(ValidationType type, ValidationStatus status, String details, Throwable cause) { + Objects.requireNonNull(type, "ValidationType can't be null"); + Objects.requireNonNull(status, "ValidationStatus can't be null"); + + _type = type; + _status = status; + _details = details; + _cause = cause; + } + + public ValidationType getValidationType() { + return _type; + } + + public ValidationStatus getValidationStatus() { + return _status; + } + + public Optional getDetails() { + return Optional.ofNullable(_details); + } + + public Optional getCause() { + return Optional.ofNullable(_cause); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + ValidationResult result = (ValidationResult) o; + return _type == result._type && _status == result._status && Objects.equals(_details, result._details) + && Objects.equals(_cause, result._cause); + } + + @Override + public int hashCode() { + return Objects.hash(_type, _status, _details, _cause); + } + + @Override + public String toString() { + return new StringJoiner(", ", ValidationResult.class.getSimpleName() + "[", "]").add("type = " + _type) + .add("status = " + _status) + .add("details = '" + _details + "'") + .add("cause = " + _cause) + .toString(); + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configvalidator/ValidationStatus.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configvalidator/ValidationStatus.java new file mode 100644 index 000000000..d7b89753c --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configvalidator/ValidationStatus.java @@ -0,0 +1,22 @@ +package com.linkedin.feathr.core.configvalidator; + +/** + * Enum for config validation status. + */ +public enum ValidationStatus { + VALID("valid"), + WARN("warn"), // Config is valid but has warnings + INVALID("invalid"), + PROCESSING_ERROR("processingError"); // error when processing Frame configs + + private final String _value; + + ValidationStatus(String value) { + _value = value; + } + + @Override + public String toString() { + return _value; + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configvalidator/ValidationType.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configvalidator/ValidationType.java new file mode 100644 index 000000000..7c88816c5 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configvalidator/ValidationType.java @@ -0,0 +1,20 @@ +package com.linkedin.feathr.core.configvalidator; + +/** + * Enum for the type of config validation to be performed + */ +public enum ValidationType { + SYNTACTIC("syntactic"), + SEMANTIC("semantic"); + + private final String _value; + + ValidationType(String value) { + _value = value; + } + + @Override + public String toString() { + return _value; + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configvalidator/typesafe/ExtractorClassValidationUtils.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configvalidator/typesafe/ExtractorClassValidationUtils.java new file mode 100644 index 000000000..2f9d71d2c --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configvalidator/typesafe/ExtractorClassValidationUtils.java @@ -0,0 +1,188 @@ +package com.linkedin.feathr.core.configvalidator.typesafe; + +import com.linkedin.feathr.core.config.ConfigType; +import com.linkedin.feathr.core.config.consumer.JoinConfig; +import com.linkedin.feathr.core.config.producer.FeatureDefConfig; +import com.linkedin.feathr.core.config.producer.anchors.AnchorConfig; +import com.linkedin.feathr.core.config.producer.anchors.AnchorConfigWithExtractor; +import com.linkedin.feathr.core.config.producer.anchors.AnchorConfigWithKeyExtractor; +import com.linkedin.feathr.core.config.producer.anchors.AnchorsConfig; +import com.linkedin.feathr.core.config.producer.derivations.DerivationConfig; +import com.linkedin.feathr.core.config.producer.derivations.DerivationConfigWithExtractor; +import com.linkedin.feathr.core.config.producer.derivations.DerivationsConfig; +import com.linkedin.feathr.core.configbuilder.ConfigBuilder; +import com.linkedin.feathr.core.configdataprovider.ConfigDataProvider; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + + +/** + * Utils to validate extractor classes in FeatureDef config, to check if extractor classes are defined in jars + * + * This is designed for independent usage of FeatureConsumerConfValidator or FeatureProducerConfValidator, + * as extractor class validation has different Gradle task dependency from general Frame config validation (performed by + * FeatureConsumerConfValidator or FeatureProducerConfValidator). + * + * For general Frame config validation, the validation need to be performed before jar task. + * For extractor class validation, the validation need to wait for all jars built, to search if the depended jars + * contain the definition of the extractor class + * + * Since Gradle has more powerful APIs to process jar file. The validation logic (including jar searching) + * will be placed in Gradle plugins which perform the validation. + * And instead of building a ExtractorClassValidator class, here we only build some public utils that can be used + * for extractor class validation. + */ +public class ExtractorClassValidationUtils { + + // Util class + private ExtractorClassValidationUtils() { + + } + + /** + * Get a list of full class names of extractors in FeatureDef config for anchors and derivations. + * If the join config is specified, then only get extractors associated with required features. + * If the join config is not specified, then get all extractors defined in FeatureDef config. + * + * Note classes in MVELs are skipped. + */ + public static Set getExtractorClasses(Map configDataProviderMap) { + Set allClasses = new HashSet<>(); + + ConfigBuilder configBuilder = ConfigBuilder.get(); + if (configDataProviderMap.containsKey(ConfigType.FeatureDef)) { + FeatureDefConfig featureDefConfig = + configBuilder.buildFeatureDefConfig(configDataProviderMap.get(ConfigType.FeatureDef)); + + // mapping from anchor name to feature name set + Map> anchorFeaturesMap = new HashMap<>(); + + /* + * mapping from anchor name to extractor name list, + * one anchor can have at most two extractors (extractor and key extractor) + */ + Map> anchorExtractorsMap = getExtractorClassesInAnchors(featureDefConfig, anchorFeaturesMap); + // mapping from derived feature name to extractor name + Map derivedExtractorMap = getExtractorClassesInDerivations(featureDefConfig); + + /* + * If the join config is specified, then only get extractors associated with required features. + * else get all extractors defined in FeatureDef config. + */ + if (configDataProviderMap.containsKey(ConfigType.Join)) { + JoinConfig joinConfig = configBuilder.buildJoinConfig(configDataProviderMap.get(ConfigType.Join)); + Set requiredFeatureNames = FeatureDefConfigSemanticValidator.getRequiredFeatureNames(featureDefConfig, + JoinConfSemanticValidator.getRequestedFeatureNames(joinConfig)); + + return filterClassesWithRequiredFeatures(requiredFeatureNames, anchorExtractorsMap, anchorFeaturesMap, + derivedExtractorMap); + } else { + allClasses.addAll(anchorExtractorsMap.values().stream().flatMap(List::stream).collect(Collectors.toSet())); + allClasses.addAll(derivedExtractorMap.values()); + } + } // else no op if there is no FeatureDef config, and empty set will be returned + + return allClasses; + } + + /** + * Given a {@link FeatureDefConfig} object, get mapping from anchor name to extractor name list, + * one anchor can have at most two extractors (extractor and key extractor) + * @param featureDefConfig the {@link FeatureDefConfig} object + * @param anchorFeaturesMap the container map, that maps anchor name to the set of features. The information can + * lately be used to have a mapping from anchored feature name to extractor name. + * The mapping from feature name to extractor name contains a lot of + * redundant information as multiple features with the same + * anchor can share the same extractor. Also, this information is optional for later + * processing. + * @return mapping from anchor name to extractor name list. + */ + private static Map> getExtractorClassesInAnchors(FeatureDefConfig featureDefConfig, + Map> anchorFeaturesMap) { + Map> anchorExtractorsMap = new HashMap<>(); + + Map anchors = featureDefConfig.getAnchorsConfig() + .orElse(new AnchorsConfig(new HashMap<>())).getAnchors(); + + for (Map.Entry entry: anchors.entrySet()) { + String anchorName = entry.getKey(); + AnchorConfig anchor = entry.getValue(); + if (anchor instanceof AnchorConfigWithExtractor) { + AnchorConfigWithExtractor anchorWithExtractor = (AnchorConfigWithExtractor) anchor; + // collect extractors, might be two (extractor and keyExtractor) + anchorExtractorsMap.put(anchorName, new ArrayList<>(Arrays.asList(anchorWithExtractor.getExtractor()))); + anchorWithExtractor.getKeyExtractor().map(e -> anchorExtractorsMap.get(anchorName).add(e)); + // collect features + anchorFeaturesMap.put(anchorName, anchorWithExtractor.getFeatures().keySet()); + } else if (anchor instanceof AnchorConfigWithKeyExtractor) { + AnchorConfigWithKeyExtractor anchorWithKeyExtractor = (AnchorConfigWithKeyExtractor) anchor; + anchorExtractorsMap.put(anchorName, Collections.singletonList(anchorWithKeyExtractor.getKeyExtractor())); + anchorFeaturesMap.put(anchorName, anchorWithKeyExtractor.getFeatures().keySet()); + } + } + return anchorExtractorsMap; + } + + /** + * Given a {@link FeatureDefConfig} object, get mapping from derived feature name to extractor class name + */ + private static Map getExtractorClassesInDerivations(FeatureDefConfig featureDefConfig) { + Map derivations = featureDefConfig.getDerivationsConfig() + .orElse(new DerivationsConfig(new HashMap<>())).getDerivations(); + // mapping from derived feature to the extractor used + Map derivedExtractorMap = new HashMap<>(); + + for (Map.Entry entry: derivations.entrySet()) { + String derivedFeature = entry.getKey(); + DerivationConfig derivation = entry.getValue(); + if (derivation instanceof DerivationConfigWithExtractor) { + DerivationConfigWithExtractor derivationWithExtractor = (DerivationConfigWithExtractor) derivation; + derivedExtractorMap.put(derivedFeature, derivationWithExtractor.getClassName()); + } + /* + * Here skip classes in MVEL expressions. In some derivations, such as online derivations sometime the MVEL + * expression can import some classes with "import", or the optional transformation expression used in + * sequential join. + */ + } + return derivedExtractorMap; + } + + /** + * Get all extractor classes associated with required features + * @param requiredFeatureNames required feature names + * @param anchorExtractorsMap mapping from anchor name to extractor class names + * @param anchorFeaturesMap mapping from anchor name to feature name + * @param derivedExtractorMap mapping from derived feature name to extractor class name + * @return all extractor classes associated with required features + */ + private static Set filterClassesWithRequiredFeatures(Set requiredFeatureNames, + Map> anchorExtractorsMap, Map> anchorFeaturesMap, + Map derivedExtractorMap) { + Set allClasses = new HashSet<>(); + + // get required anchors, whose features are required + Set requiredAnchors = anchorFeaturesMap.entrySet().stream() + .filter(e -> e.getValue().removeAll(requiredFeatureNames)) // check if at least one feature in anchor is required + .map(Map.Entry::getKey).collect(Collectors.toSet()); + + // collect extractor classes whose anchors are required + anchorExtractorsMap.entrySet().stream() + .filter(e -> requiredAnchors.contains(e.getKey())).map(Map.Entry::getValue) + .forEach(allClasses::addAll); + + // collect extractor class of derived features that are required + derivedExtractorMap.entrySet().stream().filter(e -> requiredFeatureNames.contains(e.getKey())) + .map(Map.Entry::getValue) + .forEach(allClasses::add); + + return allClasses; + } +} \ No newline at end of file diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configvalidator/typesafe/FeatureConsumerConfValidator.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configvalidator/typesafe/FeatureConsumerConfValidator.java new file mode 100644 index 000000000..0829c4474 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configvalidator/typesafe/FeatureConsumerConfValidator.java @@ -0,0 +1,183 @@ +package com.linkedin.feathr.core.configvalidator.typesafe; + +import com.linkedin.feathr.core.config.ConfigType; +import com.linkedin.feathr.core.config.consumer.JoinConfig; +import com.linkedin.feathr.core.config.producer.FeatureDefConfig; +import com.linkedin.feathr.core.config.producer.sources.SourceConfig; +import com.linkedin.feathr.core.config.producer.sources.SourcesConfig; +import com.linkedin.feathr.core.configbuilder.ConfigBuilder; +import com.linkedin.feathr.core.configdataprovider.ConfigDataProvider; +import com.linkedin.feathr.core.configvalidator.ConfigValidationException; +import com.linkedin.feathr.core.configvalidator.ConfigValidator; +import com.linkedin.feathr.core.configvalidator.ValidationResult; +import com.linkedin.feathr.core.configvalidator.ValidationStatus; +import com.linkedin.feathr.core.configvalidator.ValidationType; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.StringJoiner; + + +/** + * Validator specific for Frame feature consumer clients. + * + * The validator provides syntax and semantic validation for Frame configs in the Frame feature consumer clients. + * For instance, it checks the syntax restrictions from Frame libraries. Some examples of semantic validation will + * be checking if requested features are reachable (feature is said reachable if the feature is defined in anchors + * section in FeatureDef config, or if it is a derived feature, then the depended features are reachable), + * and checking if the source used in feature definition is defined. + * + */ +public class FeatureConsumerConfValidator extends TypesafeConfigValidator { + + /** + * validate configs for Frame feature consumer + * + * @see ConfigValidator#validate(Map, ValidationType) + */ + @Override + public Map validate(Map configTypeWithDataProvider, + ValidationType validationType) { + + switch (validationType) { + case SYNTACTIC: + // reuse default implementation in super class to perform syntax validation + return super.validate(configTypeWithDataProvider, ValidationType.SYNTACTIC); + case SEMANTIC: + return validateSemantics(configTypeWithDataProvider); + default: + throw new ConfigValidationException("Unsupported validation type: " + validationType.name()); + } + } + + /** + * Perform semantic validations for provided configs: + * 1. if no FeatureDef config provided, then return empty result, as all semantic validation requires at least + * FeatureDef config provided + * 2. if only FeatureDef config provided, then perform semantic validation for FeatureDef config + * 3. if Join config provided, then perform semantic validation for Join config, together with the information provided + * in FeatureDef config. For instance, check if features requested in Join config are reachable features in + * FeatureDef config + * 4. if FeatureGeneration config provided, then perform semantic validation for FeatureGeneration config, together + * with the information provided in FeatureDef config + */ + private Map validateSemantics(Map configTypeWithDataProvider) { + Map result = new HashMap<>(); + + // edge cases when the input is not valid or is empty + if (configTypeWithDataProvider == null || configTypeWithDataProvider.isEmpty()) { + return result; + } + + ConfigBuilder configBuilder = ConfigBuilder.get(); + Optional optionalFeatureDefConfig; + Optional sourceNameValidationWarnStr; + + if (configTypeWithDataProvider.containsKey(ConfigType.FeatureDef)) { + // Populate ValidationResult warning string when source name duplicates exist in different feature def configs + sourceNameValidationWarnStr = validateFeatureDefConfigSourceNames(configTypeWithDataProvider.get(ConfigType.FeatureDef)); + ConfigDataProvider featureDefConfigDataProvider = configTypeWithDataProvider.get(ConfigType.FeatureDef); + optionalFeatureDefConfig = Optional.of(configBuilder.buildFeatureDefConfig(featureDefConfigDataProvider)); + } else { + optionalFeatureDefConfig = Optional.empty(); + sourceNameValidationWarnStr = Optional.empty(); + } + + if (configTypeWithDataProvider.containsKey(ConfigType.Join)) { + ConfigDataProvider joinConfigDataProvider = configTypeWithDataProvider.get(ConfigType.Join); + JoinConfig joinConfig = configBuilder.buildJoinConfig(joinConfigDataProvider); + String errMsg = String.join("", "Can not perform semantic validation as the Join config is", + "provided but the FeatureDef config is missing."); + FeatureDefConfig featureDefConfig = optionalFeatureDefConfig.orElseThrow(() -> new ConfigValidationException(errMsg)); + result = validateConsumerConfigSemantics(joinConfig, featureDefConfig); + + } else { + // TODO add feature generation config semantic validation support + // only perform semantic check for FeatureDef config + FeatureDefConfig featureDefConfig = optionalFeatureDefConfig.orElseThrow(() -> new ConfigValidationException( + "Can not perform semantic validation as the FeatureDef config is missing.")); + result.put(ConfigType.FeatureDef, validateSemantics(featureDefConfig)); + } + + if (sourceNameValidationWarnStr.isPresent() && result.containsKey(ConfigType.FeatureDef)) { + result.put(ConfigType.FeatureDef, + new ValidationResult(ValidationType.SEMANTIC, ValidationStatus.WARN, sourceNameValidationWarnStr.get())); + } + return result; + } + + /** + * Validates feature consumer configs semantically. Requires both {@link JoinConfig} and {@link FeatureDefConfig} to be passed in. + * @param joinConfig {@link JoinConfig} + * @param featureDefConfig {@link FeatureDefConfig} + * @return Map of ConfigType and the {@link ValidationResult} + */ + private Map validateConsumerConfigSemantics(JoinConfig joinConfig, FeatureDefConfig featureDefConfig) { + Map validationResultMap = new HashMap<>(); + FeatureDefConfigSemanticValidator featureDefConfSemanticValidator = new FeatureDefConfigSemanticValidator(true, true); + validationResultMap.put(ConfigType.FeatureDef, featureDefConfSemanticValidator.validate(featureDefConfig)); + + JoinConfSemanticValidator joinConfSemanticValidator = new JoinConfSemanticValidator(); + validationResultMap.put(ConfigType.Join, joinConfSemanticValidator.validate(joinConfig, + featureDefConfSemanticValidator.getFeatureAccessInfo(featureDefConfig))); + return validationResultMap; + } + + /** + * Check that source names are not duplicated across different feature definition configs. + * If duplicates exist then the optional string will have a value present, if not, then the optional string will be empty. + * + * @param configDataProvider a {@link ConfigDataProvider} with the FeatureDefConfig + * @return {@link Optional} + */ + private static Optional validateFeatureDefConfigSourceNames(ConfigDataProvider configDataProvider) { + StringJoiner warnMsgSj = new StringJoiner("\n"); + Set sourcesSet = new HashSet<>(); + Set duplicateSourceNames = new HashSet<>(); + // for each resource, construct a FeatureDefConfig + ConfigBuilder configBuilder = ConfigBuilder.get(); + List builtFeatureDefConfigList = configBuilder.buildFeatureDefConfigList(configDataProvider); + + for (FeatureDefConfig featureDefConfig : builtFeatureDefConfigList) { + + if (featureDefConfig.getSourcesConfig().isPresent()) { + SourcesConfig source = featureDefConfig.getSourcesConfig().get(); + Map sources = source.getSources(); + + for (String sourceName : sources.keySet()) { + if (sourcesSet.contains(sourceName)) { + duplicateSourceNames.add(sourceName); + } else { + sourcesSet.add(sourceName); + } + } + } + } + + if (duplicateSourceNames.size() > 0) { + warnMsgSj.add("The following source name(s) are duplicates between two or more feature definition configs: "); + for (String entry : duplicateSourceNames) { + warnMsgSj.add("source name: " + entry); + } + warnMsgSj.add("File paths of two or more files that have duplicate source names: \n" + configDataProvider.getConfigDataInfo()); + } + + String warnMsg = warnMsgSj.toString(); + Optional returnString = warnMsg.isEmpty() ? Optional.empty() : Optional.of(warnMsg); + + return returnString; + } + + /** + * Validates FeatureDef config semantically + * @param featureDefConfig {@link FeatureDefConfig} + * @return {@link ValidationResult} + */ + @Override + public ValidationResult validateSemantics(FeatureDefConfig featureDefConfig) { + return new FeatureDefConfigSemanticValidator(true, true).validate(featureDefConfig); + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configvalidator/typesafe/FeatureDefConfigSemanticValidator.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configvalidator/typesafe/FeatureDefConfigSemanticValidator.java new file mode 100644 index 000000000..0e300330b --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configvalidator/typesafe/FeatureDefConfigSemanticValidator.java @@ -0,0 +1,462 @@ +package com.linkedin.feathr.core.configvalidator.typesafe; + +import com.google.common.collect.ImmutableSet; +import com.linkedin.feathr.core.config.producer.FeatureDefConfig; +import com.linkedin.feathr.core.config.producer.anchors.AnchorConfig; +import com.linkedin.feathr.core.config.producer.anchors.AnchorConfigWithExtractor; +import com.linkedin.feathr.core.config.producer.anchors.ExtractorBasedFeatureConfig; +import com.linkedin.feathr.core.config.producer.anchors.FeatureConfig; +import com.linkedin.feathr.core.config.producer.derivations.DerivationConfig; +import com.linkedin.feathr.core.config.producer.derivations.DerivationConfigWithExpr; +import com.linkedin.feathr.core.config.producer.derivations.DerivationConfigWithExtractor; +import com.linkedin.feathr.core.config.producer.derivations.DerivationsConfig; +import com.linkedin.feathr.core.config.producer.derivations.KeyedFeature; +import com.linkedin.feathr.core.config.producer.derivations.SequentialJoinConfig; +import com.linkedin.feathr.core.config.producer.derivations.SimpleDerivationConfig; +import com.linkedin.feathr.core.configvalidator.ValidationResult; +import com.linkedin.feathr.core.configvalidator.ValidationStatus; +import com.linkedin.feathr.core.configvalidator.ValidationType; +import com.linkedin.feathr.exception.ErrorLabel; +import com.linkedin.feathr.exception.FeathrConfigException; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.Map; +import java.util.Queue; +import java.util.Set; +import java.util.StringJoiner; +import java.util.function.BiConsumer; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static com.linkedin.feathr.core.configvalidator.typesafe.FeatureReachType.*; + + +/** + * validator specific for FeatureDef config validation + */ +class FeatureDefConfigSemanticValidator { + + // Represents the regex for only feature name + private static final String FEATURE_NAME_REGEX = "([a-zA-Z][.:\\w]*)"; + public static final Pattern FEATURE_NAME_PATTERN = Pattern.compile(FEATURE_NAME_REGEX); + + private boolean _withFeatureReachableValidation; + private boolean _withUndefinedSourceValidation; + // Anchors with parameters can only be used with approval. The following set is the allowed extractors. + // Adding a first allowed dummy extractor for testing. + // TODO - 17349): Add Galene's parameterized extractors. + private static final Set ALLOWED_EXTRACTOR_WITH_PARAMETERS = ImmutableSet.of( + "com.linkedin.feathr.SampleExtractorWithParams", + // For feed use cases, key tags themselves are also used as features, such as actorUrn, objectUrn etc. This + // extractor is to extract features from key tags. + "com.linkedin.followfeed.feathr.extractor.KeyTagFeatureExtractor"); + + /** + * constructor + * @param withFeatureReachableValidation flag to perform feature reachable validation + * @param withUndefinedSourceValidation flag to perform undefined source validation + */ + FeatureDefConfigSemanticValidator(boolean withFeatureReachableValidation, boolean withUndefinedSourceValidation) { + _withFeatureReachableValidation = withFeatureReachableValidation; + _withUndefinedSourceValidation = withUndefinedSourceValidation; + } + + /** + * constructor + */ + FeatureDefConfigSemanticValidator() { + _withFeatureReachableValidation = false; + _withUndefinedSourceValidation = false; + } + + /** + * the entry for FeatureDef config semantic validation + */ + ValidationResult validate(FeatureDefConfig featureDefConfig) { + validateApprovedExtractorWithParameters(featureDefConfig); + + StringJoiner warnMsgSj = new StringJoiner("\n"); // concat all warning messages together and output + int warnMsgSjInitLength = warnMsgSj.length(); // get the init length of the warning message, + + try { + // check duplicate feature names + Set duplicateFeatures = getDuplicateFeatureNames(featureDefConfig); + if (!duplicateFeatures.isEmpty()) { + String warnMsg = String.join("\n", "The following features' definitions are duplicate: ", + String.join("\n", duplicateFeatures)); + warnMsgSj.add(warnMsg); + } + + // check if all sources used in anchors are defined + if (_withUndefinedSourceValidation) { + Map undefinedAnchorSources = getUndefinedAnchorSources(featureDefConfig); + if (!undefinedAnchorSources.isEmpty()) { + StringJoiner sj = new StringJoiner("\n"); + for (Map.Entry entry : undefinedAnchorSources.entrySet()) { + sj.add(String.join(" ", "Source", entry.getValue(), "used in anchor", entry.getKey(), "is not defined.")); + } + return new ValidationResult(ValidationType.SEMANTIC, ValidationStatus.INVALID, sj.toString()); + } + } + + /* + * check if all input features for derived features are reachable + * This can only be a warning here as the features might not be required + */ + if (_withFeatureReachableValidation) { + Map> featureAccessInfo = getFeatureAccessInfo(featureDefConfig); + Set unreachableFeatures = featureAccessInfo.getOrDefault(UNREACHABLE, Collections.emptySet()); + if (!unreachableFeatures.isEmpty()) { + String warnMsg = String.join("", "The following derived features cannot be computed as ", + "one or more of their ancestor features cannot be found:\n", String.join("\n", unreachableFeatures)); + warnMsgSj.add(warnMsg); + } + } + + /* + * dedicate to MvelValidator for MVEL expression validation + */ + MvelValidator mvelValidator = MvelValidator.getInstance(); + ValidationResult mvelValidationResult = mvelValidator.validate(featureDefConfig); + if (mvelValidationResult.getValidationStatus() == ValidationStatus.WARN) { + warnMsgSj.add(mvelValidationResult.getDetails().orElse("")); + } + + /* + * validate HDFS sources + */ + HdfsSourceValidator hdfsSourceValidator = HdfsSourceValidator.getInstance(); + ValidationResult hdfsSourceValidationResult = hdfsSourceValidator.validate(featureDefConfig); + if (hdfsSourceValidationResult.getValidationStatus() == ValidationStatus.WARN) { + warnMsgSj.add(hdfsSourceValidationResult.getDetails().orElse("")); + } else if (hdfsSourceValidationResult.getValidationStatus() == ValidationStatus.INVALID) { + return hdfsSourceValidationResult; + } + + } catch (Throwable e) { + return new ValidationResult(ValidationType.SEMANTIC, ValidationStatus.PROCESSING_ERROR, e.getMessage(), e); + } + + /* + * If new warning message is added, return a warning validation result, + * else, return a valid validation result + */ + return warnMsgSj.length() > warnMsgSjInitLength + ? new ValidationResult(ValidationType.SEMANTIC, ValidationStatus.WARN, warnMsgSj.toString()) + : new ValidationResult(ValidationType.SEMANTIC, ValidationStatus.VALID); + } + + /** + * Validate that feature params is only allowed to be used by approved use cases. Here we use extractor name to target + * the approved use cases. + */ + void validateApprovedExtractorWithParameters(FeatureDefConfig featureDefConfig) { + for (Map.Entry entry : featureDefConfig.getAnchorsConfig().get().getAnchors().entrySet()) { + AnchorConfig anchorConfig = entry.getValue(); + for (Map.Entry featureEntry : anchorConfig.getFeatures().entrySet()) { + FeatureConfig featureConfig = featureEntry.getValue(); + if (featureConfig instanceof ExtractorBasedFeatureConfig && !featureConfig.getParameters().isEmpty()) { + if (anchorConfig instanceof AnchorConfigWithExtractor) { + String extractor = ((AnchorConfigWithExtractor) anchorConfig).getExtractor(); + if (!ALLOWED_EXTRACTOR_WITH_PARAMETERS.contains(extractor)) { + throw new FeathrConfigException(ErrorLabel.FEATHR_USER_ERROR, "anchorConfig: " + anchorConfig + + " has parameters. Parameters are only approved to be used by the following extractors: " + + ALLOWED_EXTRACTOR_WITH_PARAMETERS); + } + } else { + // If it's not AnchorConfigWithExtractor but it has parameters, it's not allowed. + throw new FeathrConfigException(ErrorLabel.FEATHR_USER_ERROR, + "Parameters are only to be used by AnchorConfigWithExtractor. The anchor config is: " + + anchorConfig); + } + } + } + } + } + + /** + * Semantic check, get all the anchors whose source is not defined + * @param featureDefConfig {@link FeatureDefConfig} object + * @return mapping of anchor name to the undefined source name + */ + Map getUndefinedAnchorSources(FeatureDefConfig featureDefConfig) { + Map undefinedAnchorSource = new HashMap<>(); + Set definedSourceNames = getDefinedSourceNames(featureDefConfig); + // if an anchor's source is not defined, then return the mapping from anchor name to source name + BiConsumer consumeAnchor = (anchorName, anchorConfig) -> { + String sourceName = anchorConfig.getSource(); + /* + * Here sourceName can be file path in Frame offline, in which case it is not defined in sources section. + * The source defined in sources section can not contain special char / and ., which can be used to distinguish + * source definition from file path. + */ + if (!(sourceName.contains("/") || sourceName.contains("."))) { + if (!definedSourceNames.contains(sourceName)) { + undefinedAnchorSource.put(anchorName, sourceName); + } + } + }; + + featureDefConfig.getAnchorsConfig().ifPresent(anchorsConfig -> + anchorsConfig.getAnchors().forEach(consumeAnchor) + ); + return undefinedAnchorSource; + } + + /** + * get all defined source names + * @param featureDefConfig {@link FeatureDefConfig} object + * @return set of all defined source names + */ + private Set getDefinedSourceNames(FeatureDefConfig featureDefConfig) { + Set definedSourceNames = new HashSet<>(); + featureDefConfig.getSourcesConfig().ifPresent(sourcesConfig -> + definedSourceNames.addAll(sourcesConfig.getSources().keySet())); + return definedSourceNames; + } + + /** + * get duplicate features defined in FeatureDefConfig + * @param featureDefConfig {@link FeatureDefConfig} object, the object should be built from single config file + */ + Set getDuplicateFeatureNames(FeatureDefConfig featureDefConfig) { + Set definedFeatures = new HashSet<>(); + Set duplicateFeatures = new HashSet<>(); + + // check if there is duplicate features in multiple anchors + BiConsumer checkAnchor = (anchorName, anchorConfig) -> { + Set features = anchorConfig.getFeatures().keySet(); + for (String feature: features) { + if (definedFeatures.contains(feature)) { + duplicateFeatures.add(feature); + } + definedFeatures.add(feature); + } + }; + + featureDefConfig.getAnchorsConfig().ifPresent(anchorsConfig -> { + anchorsConfig.getAnchors().forEach(checkAnchor); + }); + + // check if there is duplicate features defined in both derivations and above anchors + BiConsumer checkDerivation = (featureName, derivationConfig) -> { + if (definedFeatures.contains(featureName)) { + duplicateFeatures.add(featureName); + } + definedFeatures.add(featureName); + }; + + featureDefConfig.getDerivationsConfig().ifPresent(derivationsConfig -> { + derivationsConfig.getDerivations().forEach(checkDerivation); + }); + + return duplicateFeatures; + } + + + /** + * Get all required features from a set of requested features. + * Definition: + * A feature is a required feature if it is a requested feature, or it is a depended feature of a required derive feature. + * + * Note, this can also be achieved with the dependency graph built with frame-common library. However, + * frame-core can not depend on frame-common to avoid a circular dependency. Here we implement a lighter version + * of dependency graph with only feature names to get required feature names. + * + * @param featureDefConfig {@link FeatureDefConfig} object + * @param requestedFeatureNames set of requested feature names + * @return set of required feature names + */ + static Set getRequiredFeatureNames(FeatureDefConfig featureDefConfig, Set requestedFeatureNames) { + Set requiredFeatureNames = new HashSet<>(); + // put requested feature names into a queue, and resolve its dependency with BFS + Queue featuresToResolve = new LinkedList<>(requestedFeatureNames); + + Map> dependencyGraph = getDependencyGraph(featureDefConfig); + // BFS to find all required feature names in the dependency graph + while (!featuresToResolve.isEmpty()) { + String feature = featuresToResolve.poll(); + requiredFeatureNames.add(feature); + dependencyGraph.getOrDefault(feature, Collections.emptySet()).forEach(featuresToResolve::offer); + } + + return requiredFeatureNames; + } + + /** + * Get all anchored feature names, which are considered reachable directly. + * See the definition of "reachable" in {@link #getFeatureAccessInfo(FeatureDefConfig)}. + * @param featureDefConfig {@link FeatureDefConfig} object + * @return set of anchored feature names + */ + private static Set getAnchoredFeatureNames(FeatureDefConfig featureDefConfig) { + Set anchoredFeatures = new HashSet<>(); + + featureDefConfig.getAnchorsConfig().ifPresent(anchorsConfig -> { + Set features = anchorsConfig.getAnchors().entrySet().stream() + .flatMap(x -> x.getValue().getFeatures().keySet().stream()).collect(Collectors.toSet()); + anchoredFeatures.addAll(features); + }); + + return anchoredFeatures; + } + + /** + * Get all reachable and unreachable feature names in the input FeatureDef config. + * Here a feature is reachable if and only if the feature is defined in anchors section, or + * its depend features (a.k.a input features or base features) are all reachable. + * @param featureDefConfig {@link FeatureDefConfig} object + * @return all reachable and unreachable feature names + */ + Map> getFeatureAccessInfo(FeatureDefConfig featureDefConfig) { + Set reachableFeatures = getAnchoredFeatureNames(featureDefConfig); + + Map derivations = featureDefConfig.getDerivationsConfig(). + orElse(new DerivationsConfig(Collections.emptyMap())).getDerivations(); + Set allDerivedFeatures = derivations.keySet(); + + // get all defined features in "anchors" section, and "derivations" section. + Set allDefinedFeatures = new HashSet<>(reachableFeatures); + allDefinedFeatures.addAll(allDerivedFeatures); + + Set unreachableFeatures = new HashSet<>(); + // recursively find all reachable and unreachable features + for (String derivedFeature: derivations.keySet()) { + checkFeatureReachable(reachableFeatures, unreachableFeatures, derivations, allDefinedFeatures, derivedFeature); + } + + Map> features = new HashMap<>(); + features.put(REACHABLE, reachableFeatures); + features.put(UNREACHABLE, unreachableFeatures); + return features; + } + + /** + * Recursive call to check if a query feature is reachable, collect all reachable and unreachable features during the + * recursive processes(side effect). + * See the definition of "reachable" in {@link #getFeatureAccessInfo(FeatureDefConfig)}. + * @param reachableFeatures all known reachable features + * @param unreachableFeatures all features that are not reachable + * @param derivations derived feature name mapping to its definition as {@link DerivationConfig} obj + * @param allDefinedFeatures all defined feature names in "anchors" and "derivations" section + * @param queryFeature the query feature + * @return if the query feature is reachable (boolean) + */ + private boolean checkFeatureReachable(Set reachableFeatures, + Set unreachableFeatures, + Map derivations, + Set allDefinedFeatures, + String queryFeature) { + + boolean featureReachable = true; + // base case, we've already known if the query feature is reachable or not + if (reachableFeatures.contains(queryFeature)) { + return true; + } else if (unreachableFeatures.contains(queryFeature)) { + return false; + } else if (!derivations.containsKey(queryFeature)) { + /* + * Since all anchored features are considered as reachable features, + * if the feature is not a known reachable feature, then it is not a anchored feature. + * It is also not defined in derivation, then it is a undefined feature, and should be considered as + * unreachable. + */ + featureReachable = false; + } else { + /* + * If the feature is not directly reachable, check if all the dependencies are reachable + * Do not stop the recursive call when finding the first unreachable feature, instead collect all the features + * that are not reachable in one shot. + */ + for (String baseFeature: getInputFeatures(queryFeature, derivations.get(queryFeature), allDefinedFeatures)) { + if (!checkFeatureReachable(reachableFeatures, unreachableFeatures, derivations, allDefinedFeatures, baseFeature)) { + featureReachable = false; + } + } + } + + //collect reachable and unreachable features + if (featureReachable) { + reachableFeatures.add(queryFeature); + } else { + unreachableFeatures.add(queryFeature); + } + + return featureReachable; + } + + /** + * a light version feature name dependency graph represented by adjacent list(set), + * where the key is a feature name, and the value is the set of features the keyed-feature depends on. + * If the feature is a anchored feature, then the depended feature set is EMPTY. + */ + private static Map> getDependencyGraph(FeatureDefConfig featureDefConfig) { + Map> dependencyGraph = new HashMap<>(); + Set anchoredFeatures = getAnchoredFeatureNames(featureDefConfig); + anchoredFeatures.forEach(f -> dependencyGraph.put(f, Collections.emptySet())); + + Map derivations = featureDefConfig.getDerivationsConfig(). + orElse(new DerivationsConfig(Collections.emptyMap())).getDerivations(); + Set allDerivedFeatures = derivations.keySet(); + + Set allDefinedFeatures = new HashSet<>(anchoredFeatures); + allDefinedFeatures.addAll(allDerivedFeatures); + + derivations.forEach((k, v) -> dependencyGraph.put(k, getInputFeatures(k, v, allDefinedFeatures))); + + return dependencyGraph; + } + + /** + * get input features of a derived feature from {@link DerivationConfig} obj + * @param derivedFeature derived feature name + * @param derivationConfig derived feature {@link DerivationConfig} obj + * @param allDefinedFeatureNames all defined feature names, this is considered as reference to extract input features + * if input features are defined in MVEL expression + * @return set of input feature names + */ + private static Set getInputFeatures(String derivedFeature, + DerivationConfig derivationConfig, + Set allDefinedFeatureNames) { + + Set inputs; // all the base/input keyed features + if (derivationConfig instanceof DerivationConfigWithExpr) { + DerivationConfigWithExpr derivationConfigWithExpr = (DerivationConfigWithExpr) derivationConfig; + inputs = derivationConfigWithExpr.getInputs().values().stream().map(KeyedFeature::getFeature). + collect(Collectors.toSet()); + } else if (derivationConfig instanceof DerivationConfigWithExtractor) { + DerivationConfigWithExtractor derivationConfigWithExtractor = (DerivationConfigWithExtractor) derivationConfig; + inputs = derivationConfigWithExtractor.getInputs().stream().map(KeyedFeature::getFeature). + collect(Collectors.toSet()); + } else if (derivationConfig instanceof SimpleDerivationConfig) { + SimpleDerivationConfig simpleDerivationConfig = (SimpleDerivationConfig) derivationConfig; + /* + * For derived feature defined as SimpleDerivationConfig, we only have the feature expression. + * The base features in feature expression should be in the set of defined features. + */ + String featureExpr = simpleDerivationConfig.getFeatureExpr(); + Matcher matcher = FEATURE_NAME_PATTERN.matcher(featureExpr); + + inputs = new HashSet<>(); + while (matcher.find()) { + String word = matcher.group(1); + if (allDefinedFeatureNames.contains(word)) { + inputs.add(word); + } + } + } else if (derivationConfig instanceof SequentialJoinConfig) { + // for sequential join feature, the input is the base feature and expansion feature + SequentialJoinConfig sequentialJoinConfig = (SequentialJoinConfig) derivationConfig; + inputs = Stream.of(sequentialJoinConfig.getBase().getFeature(), sequentialJoinConfig.getExpansion().getFeature()) + .collect(Collectors.toSet()); + } else { + throw new RuntimeException("The DerivationConfig type of " + derivedFeature + " is not supported."); + } + + return inputs; + } +} \ No newline at end of file diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configvalidator/typesafe/FeatureProducerConfValidator.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configvalidator/typesafe/FeatureProducerConfValidator.java new file mode 100644 index 000000000..86df3b812 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configvalidator/typesafe/FeatureProducerConfValidator.java @@ -0,0 +1,44 @@ +package com.linkedin.feathr.core.configvalidator.typesafe; + +import com.linkedin.feathr.core.config.ConfigType; +import com.linkedin.feathr.core.config.producer.FeatureDefConfig; +import com.linkedin.feathr.core.configdataprovider.ConfigDataProvider; +import com.linkedin.feathr.core.configvalidator.ValidationResult; +import com.linkedin.feathr.core.configvalidator.ValidationType; +import java.util.Map; + + +/** + * validator specific for Frame feature producer clients + */ +public class FeatureProducerConfValidator extends TypesafeConfigValidator { + + /** + * validate each config in Frame feature producer MPs + * + * @see ConfigValidator#validate(Map, ValidationType) + */ + @Override + public Map validate(Map configTypeWithDataProvider, + ValidationType validationType) { + + // feature producer MP should not have join config + if (configTypeWithDataProvider.containsKey(ConfigType.Join)) { + String errMsg = "Found Join config provided for config validation in feature producer MP."; + throw new RuntimeException(errMsg); + } + + return super.validate(configTypeWithDataProvider, validationType); + } + + /** + * Validates FeatureDef config semantically + * @param featureDefConfig {@link FeatureDefConfig} + * @return {@link ValidationResult} + */ + @Override + public ValidationResult validateSemantics(FeatureDefConfig featureDefConfig) { + return new FeatureDefConfigSemanticValidator().validate(featureDefConfig); + } + +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configvalidator/typesafe/FeatureReachType.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configvalidator/typesafe/FeatureReachType.java new file mode 100644 index 000000000..aadc192af --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configvalidator/typesafe/FeatureReachType.java @@ -0,0 +1,11 @@ +package com.linkedin.feathr.core.configvalidator.typesafe; + +/** + * Enum for feature reachable. + * A feature is reachable if and only if the feature is defined in anchors section, or + * its depend features (a.k.a input features or base features) are all reachable. + */ +enum FeatureReachType { + UNREACHABLE, + REACHABLE +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configvalidator/typesafe/HdfsSourceValidator.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configvalidator/typesafe/HdfsSourceValidator.java new file mode 100644 index 000000000..ee1aea4b4 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configvalidator/typesafe/HdfsSourceValidator.java @@ -0,0 +1,97 @@ +package com.linkedin.feathr.core.configvalidator.typesafe; + +import com.linkedin.feathr.core.config.producer.FeatureDefConfig; +import com.linkedin.feathr.core.config.producer.anchors.AnchorsConfig; +import com.linkedin.feathr.core.config.producer.sources.HdfsConfig; +import com.linkedin.feathr.core.config.producer.sources.SourceType; +import com.linkedin.feathr.core.config.producer.sources.SourcesConfig; +import com.linkedin.feathr.core.configvalidator.ValidationResult; +import com.linkedin.feathr.core.configvalidator.ValidationStatus; +import com.linkedin.feathr.core.configvalidator.ValidationType; +import java.util.AbstractMap; +import java.util.Collections; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + + +/** + * class to validate HDFS resource + */ +class HdfsSourceValidator { + + private static final HdfsSourceValidator HDFS_SOURCE_VALIDATOR = new HdfsSourceValidator(); + private HdfsSourceValidator() { + + } + + static HdfsSourceValidator getInstance() { + return HDFS_SOURCE_VALIDATOR; + } + /* + * Based on go/dalipolicy, All datasets located under the following directories are managed datasets and should use DALI + * + * Note, the policy might be changed, and there is no way to keep it sync. + * So here we only generate warnings if the user is using managed datasets directly. + */ + static Set gridManagedDataSets = Stream.of( + "/data/tracking", + "/data/tracking_column", + "/data/databases", + "/data/service", + "/data/service_column", + "/jobs/metrics/ump_v2/metrics", + "/jobs/metrics/ump_v2/metrics_union", + "/jobs/metrics/ump_v2/metrics_union_column", + "/jobs/metrics/udp/snapshot", + "/jobs/metrics/udp/datafiles").collect(Collectors.toSet()); + + /** + * validate HDFS source in FeatureDef config + * @param featureDefConfig the {@link FeatureDefConfig} object + * @return validation result in the format of {@link ValidationResult} + */ + ValidationResult validate(FeatureDefConfig featureDefConfig) { + + Map invalidPaths = getInvalidManagedDataSets(featureDefConfig); + if (!invalidPaths.isEmpty()) { + Set invalidSourceInfoSet = invalidPaths.entrySet().stream() + .map(e -> String.join(": ", e.getKey(), e.getValue())) + .collect(Collectors.toSet()); + String warnMsg = String.join("", "Based on go/dalipolicy, the following HDFS sources are invalid. ", + "For managed datasets, you need to use DALI path instead of directly using HDFS path: \n", + String.join("\n", invalidSourceInfoSet), + "\nFor detailed information, please refer to go/dalipolicy"); + return new ValidationResult(ValidationType.SEMANTIC, ValidationStatus.WARN, warnMsg); + } + return new ValidationResult(ValidationType.SEMANTIC, ValidationStatus.VALID); + } + + Map getInvalidManagedDataSets(FeatureDefConfig featureDefConfig) { + // first search all source definitions + Map invalidDataSets = featureDefConfig.getSourcesConfig() + .orElse(new SourcesConfig(Collections.emptyMap())) // return empty map if no sources section + .getSources().entrySet().stream() + .filter(e -> e.getValue().getSourceType().equals(SourceType.HDFS)) // get all sources with HDFS + // get mapping from source name to HDFS path string + .map(e -> new AbstractMap.SimpleEntry<>(e.getKey(), ((HdfsConfig) e.getValue()).getPath())) + // get all HDFS path with prefix in gridManagedDataSets + .filter(e -> gridManagedDataSets.stream().anyMatch(prefix -> e.getValue().startsWith(prefix))) // filter invalid + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + + // then search anchor definitions + featureDefConfig.getAnchorsConfig() + .orElse(new AnchorsConfig(Collections.emptyMap())) + .getAnchors().entrySet().stream() + .filter(e -> e.getValue().getSource().startsWith("/")) // get all sources with simple HDFS + // get mapping from anchor name to source path + .map(e -> new AbstractMap.SimpleEntry<>(e.getKey(), e.getValue().getSource())) + .filter(e -> gridManagedDataSets.stream().anyMatch(prefix -> e.getValue().startsWith(prefix))) // filter invalid + .forEach(e -> invalidDataSets.put(e.getKey(), e.getValue())); // add to result + + return invalidDataSets; + } +} + + diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configvalidator/typesafe/JoinConfSemanticValidator.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configvalidator/typesafe/JoinConfSemanticValidator.java new file mode 100644 index 000000000..e7277173f --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configvalidator/typesafe/JoinConfSemanticValidator.java @@ -0,0 +1,90 @@ +package com.linkedin.feathr.core.configvalidator.typesafe; + +import com.linkedin.feathr.core.configvalidator.ValidationResult; +import com.linkedin.feathr.core.config.consumer.JoinConfig; +import com.linkedin.feathr.core.configvalidator.ValidationStatus; +import com.linkedin.feathr.core.configvalidator.ValidationType; +import java.util.Collections; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.StringJoiner; +import java.util.stream.Collectors; + + +/** + * package private validator class specific for Join config semantic validation + */ +class JoinConfSemanticValidator { + + /** + * semantic validation for Join config + * @param joinConfig the {@link JoinConfig} + * @param featureReachableInfo feature reachable information extracted from FeatureDef config + */ + ValidationResult validate(JoinConfig joinConfig, Map> featureReachableInfo) { + + Set requestedFeatureNames = getRequestedFeatureNames(joinConfig); + + // get reachable features defined in FeatureDef config + Set reachableFeatureNames = featureReachableInfo.getOrDefault(FeatureReachType.REACHABLE, + Collections.emptySet()); + // get unreachable features defined in FeatureDef config + Set unreachableFeatureNames = featureReachableInfo.getOrDefault(FeatureReachType.UNREACHABLE, + Collections.emptySet()); + + // requested features that are not defined + Set undefinedRequestedFeatures = new HashSet<>(); + + /* + * requested features that are defined in FeatureDef config, but these features are in fact not reachable + * For instance, the requested features can be defined in "derivations" section, but the derived feature might + * not be reachable because its depended features might not be reachable + */ + Set unreachableRequestedFeatures = new HashSet<>(); + + requestedFeatureNames.stream().filter(f -> !reachableFeatureNames.contains(f)).forEach(f -> { + if (unreachableFeatureNames.contains(f)) { + unreachableRequestedFeatures.add(f); + } else { + undefinedRequestedFeatures.add(f); + } + }); + + return constructRequestedFeaturesValidationResult(undefinedRequestedFeatures, unreachableRequestedFeatures); + } + + /** + * construct final ValidationResult based on the found undefined requested features, and unreachable requested features + */ + private ValidationResult constructRequestedFeaturesValidationResult(Set undefinedRequestedFeatures, + Set unreachableRequestedFeatures) { + if (undefinedRequestedFeatures.isEmpty() && unreachableRequestedFeatures.isEmpty()) { + return ValidationResult.VALID_SEMANTICS; + } + + StringJoiner errMsgJoiner = new StringJoiner("\n"); + if (!undefinedRequestedFeatures.isEmpty()) { + String tipMsg = String.join("", "The following requested features are not defined.", + " It could be possible that 1) typos in feature name, 2) feature definition is not included: "); + errMsgJoiner.add(tipMsg); + undefinedRequestedFeatures.forEach(errMsgJoiner::add); + } + + if (!unreachableRequestedFeatures.isEmpty()) { + String tipMsg = String.join("", "The following requested features are unreachable", + " features defined in FeatureDef. This is usually due to incorrect feature definition: "); + errMsgJoiner.add(tipMsg); + unreachableRequestedFeatures.forEach(errMsgJoiner::add); + } + + return new ValidationResult(ValidationType.SEMANTIC, ValidationStatus.INVALID, errMsgJoiner.toString()); + } + + // static method get all requested features in the Join config, by merging requested features in each FeatureBag + static Set getRequestedFeatureNames(JoinConfig joinConfig) { + return joinConfig.getFeatureBagConfigs().entrySet().stream() + .flatMap(entry -> entry.getValue().getKeyedFeatures().stream().flatMap(f -> f.getFeatures().stream())) + .collect(Collectors.toSet()); + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configvalidator/typesafe/MvelValidator.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configvalidator/typesafe/MvelValidator.java new file mode 100644 index 000000000..294338e43 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configvalidator/typesafe/MvelValidator.java @@ -0,0 +1,247 @@ +package com.linkedin.feathr.core.configvalidator.typesafe; + +import com.google.common.annotations.VisibleForTesting; +import com.linkedin.feathr.core.config.producer.ExprType; +import com.linkedin.feathr.core.config.producer.FeatureDefConfig; +import com.linkedin.feathr.core.config.producer.anchors.AnchorConfig; +import com.linkedin.feathr.core.config.producer.anchors.AnchorConfigWithKey; +import com.linkedin.feathr.core.config.producer.anchors.ExpressionBasedFeatureConfig; +import com.linkedin.feathr.core.config.producer.anchors.ExtractorBasedFeatureConfig; +import com.linkedin.feathr.core.config.producer.anchors.FeatureConfig; +import com.linkedin.feathr.core.config.producer.anchors.TimeWindowFeatureConfig; +import com.linkedin.feathr.core.config.producer.derivations.DerivationConfig; +import com.linkedin.feathr.core.config.producer.derivations.DerivationConfigWithExpr; +import com.linkedin.feathr.core.config.producer.derivations.SimpleDerivationConfig; +import com.linkedin.feathr.core.configvalidator.ValidationResult; +import com.linkedin.feathr.core.configvalidator.ValidationStatus; +import com.linkedin.feathr.core.configvalidator.ValidationType; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.Stack; +import java.util.function.BiConsumer; +import java.util.stream.Collectors; +import java.util.stream.Stream; + + +/** + * package private class to validate MVEL expression + */ +class MvelValidator { + + private static final MvelValidator MVEL_VALIDATOR = new MvelValidator(); + private MvelValidator() { + + } + + static MvelValidator getInstance() { + return MVEL_VALIDATOR; + } + + /** + * validate MVEL expressions in FeatureDef config + * @param featureDefConfig the {@link FeatureDefConfig} object + * @return validation result in the format of {@link ValidationResult} + */ + ValidationResult validate(FeatureDefConfig featureDefConfig) { + // mapping from feature/anchor name to its MVEL expression + Map> invalidMvels = getPossibleInvalidMvelsUsingIn(featureDefConfig); + if (!invalidMvels.isEmpty()) { + Set invalidMvelInfoSet = invalidMvels.entrySet().stream() + .map(e -> String.join(": ", e.getKey(), "[", String.join(", ", e.getValue()), "]")) + .collect(Collectors.toSet()); + String warnMsg = String.join("", "For MVEL expression, if you are using `in` expression, ", + "there should be parenthesis around it. Based on a heuristic check, the following anchors/features have invalid MVEL ", + "definitions containing `in` keyword: \n", String.join("\n", invalidMvelInfoSet)); + return new ValidationResult(ValidationType.SEMANTIC, ValidationStatus.WARN, warnMsg); + } + return new ValidationResult(ValidationType.SEMANTIC, ValidationStatus.VALID); + } + + /** + * heuristic check to find all invalid MVEL expression using "in" + * @param featureDefConfig the {@link FeatureDefConfig} object + * @return mapping of feature name to its invalid MVEL expression + */ + Map> getPossibleInvalidMvelsUsingIn(FeatureDefConfig featureDefConfig) { + Map> invalidFeatureMvels = getFeatureMvels(featureDefConfig).entrySet().stream() + .filter(e -> !heuristicProjectionExprCheck(e.getValue())) // get all heuristically invalid MVEL expressions + .collect(Collectors.toMap(Map.Entry::getKey, entry -> Collections.singletonList(entry.getValue()))); + + Map> invalidAnchorKeyMvels = getAnchorKeyMvels(featureDefConfig).entrySet().stream() + .filter(e -> !e.getValue().stream().allMatch(this::heuristicProjectionExprCheck)) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + + return Stream.concat(invalidFeatureMvels.entrySet().stream(), invalidAnchorKeyMvels.entrySet().stream()) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + + /** + * collect all features whose definition is based on MVEL + * @return mapping of feature name to its MVEL expression + */ + @VisibleForTesting + Map getFeatureMvels(FeatureDefConfig featureDefConfig) { + Map featureNameToMvel = new HashMap<>(); + + // get MVEL expression from each anchor + BiConsumer consumeAnchor = (anchorName, anchorConfig) -> { + for (Map.Entry entry : anchorConfig.getFeatures().entrySet()) { + FeatureConfig featureConfig = entry.getValue(); + String featureName = entry.getKey(); + if (featureConfig instanceof ExtractorBasedFeatureConfig) { + featureNameToMvel.put(featureName, ((ExtractorBasedFeatureConfig) featureConfig).getFeatureName()); + } else if (featureConfig instanceof ExpressionBasedFeatureConfig) { + ExpressionBasedFeatureConfig expressionBasedFeatureConfig = (ExpressionBasedFeatureConfig) featureConfig; + if (expressionBasedFeatureConfig.getExprType() == ExprType.MVEL) { + featureNameToMvel.put(featureName, expressionBasedFeatureConfig.getFeatureExpr()); + } + } else if (featureConfig instanceof TimeWindowFeatureConfig) { + TimeWindowFeatureConfig timeWindowFeatureConfig = (TimeWindowFeatureConfig) featureConfig; + if (timeWindowFeatureConfig.getColumnExprType() == ExprType.MVEL) { + featureNameToMvel.put(featureName, timeWindowFeatureConfig.getColumnExpr()); + } + } // for the rest FeatureConfig types, do nothing + } + }; + + featureDefConfig.getAnchorsConfig().ifPresent(anchorsConfig -> + anchorsConfig.getAnchors().forEach(consumeAnchor) + ); + + // get MVEL expression from each derivation + BiConsumer consumeDerivation = (featureName, derivationConfig) -> { + // SimpleDerivationConfig can have MVEL and SQL expr type + if (derivationConfig instanceof SimpleDerivationConfig) { + SimpleDerivationConfig simpleDerivationConfig = ((SimpleDerivationConfig) derivationConfig); + if (simpleDerivationConfig.getFeatureTypedExpr().getExprType() == ExprType.MVEL) { + featureNameToMvel.put(featureName, simpleDerivationConfig.getFeatureTypedExpr().getExpr()); + } + } else if (derivationConfig instanceof DerivationConfigWithExpr) { + DerivationConfigWithExpr derivationConfigWithExpr = (DerivationConfigWithExpr) derivationConfig; + if (derivationConfigWithExpr.getTypedDefinition().getExprType() == ExprType.MVEL) { + featureNameToMvel.put(featureName, derivationConfigWithExpr.getTypedDefinition().getExpr()); + } + } // for the rest DerivationConfig types, do nothing + }; + + featureDefConfig.getDerivationsConfig().ifPresent(derivationsConfig -> + derivationsConfig.getDerivations().forEach(consumeDerivation) + ); + return featureNameToMvel; + } + + /** + * get MVEL expressions used in anchor level + * for now, just key definition in type {@link AnchorConfigWithKey} + * @param featureDefConfig + * @return + */ + Map> getAnchorKeyMvels(FeatureDefConfig featureDefConfig) { + Map> anchorNameToMvel = new HashMap<>(); + + // get MVEL expression from each anchor + BiConsumer consumeAnchor = (anchorName, anchorConfig) -> { + // if anchor keys are MVEL expressions, + if (anchorConfig instanceof AnchorConfigWithKey) { + AnchorConfigWithKey anchorConfigWithKey = (AnchorConfigWithKey) anchorConfig; + if (anchorConfigWithKey.getTypedKey().getKeyExprType() == ExprType.MVEL) { + anchorNameToMvel.put(anchorName, anchorConfigWithKey.getKey()); + } + } + }; + + featureDefConfig.getAnchorsConfig().ifPresent(anchorsConfig -> + anchorsConfig.getAnchors().forEach(consumeAnchor) + ); + + return anchorNameToMvel; + } + + /** + * heuristic check if a given MVEL projection expression(http://mvel.documentnode.com/#projections-and-folds) is valid + * + * When inspecting very complex object models inside collections, MVEL requires parentheses around the + * projection expression. If missing the parentheses, sometimes it + * won't throw exception. Instead, it will only return wrong results. + * + * Without a fully-built MVEL syntax and semantic analyzer, we can only perform some heuristic check here. + * The heuristic strategy is to first search for the “in” keyword, + * and then try to locate the parentheses around the keyword. + * The check is based on the observation that if there are multiple `in`, then these `in` are nested + * Specifically, the following checks are performed: + * 1. check if parenthesis are balanced + * 2. for each `in`, check if there is a parentheses pair around it, and there can not be other `in` within the pair + * If the pair is used to match a `in`, it can not be used to match other `in` + * + * Some valid examples are: + * - "(parent.name in users)" + * - "(name in (familyMembers in users))" + * + * Some invalid examples are: + * - "parent.name in users" + * - "(name in familyMembers in users)" + * - "(some expression) familyMembers in users" + * @param mvelExpr the MVEL expression + * @return heuristic result of whether the MVEL projection expression is valid + */ + boolean heuristicProjectionExprCheck(String mvelExpr) { + String inKeyword = " in "; // make sure it is a single word + + // find all "in" occurrences backward + List reversedInPosList = new ArrayList<>(); + int index = mvelExpr.lastIndexOf(inKeyword); + while (index >= 0) { + reversedInPosList.add(index); + index = mvelExpr.lastIndexOf(inKeyword, index - 1); + } + + // if no "in" keyword, return true + if (reversedInPosList.isEmpty()) { + return true; + } + + /* + * check if parentheses is balanced + */ + List sortedLeftParenthesis = new LinkedList<>(); + Stack stack = new Stack<>(); // use stack to make sure the parenthesis is balanced + for (int pos = 0; pos < mvelExpr.length(); pos++) { + if (mvelExpr.charAt(pos) == '(') { + stack.push(pos); // record the left parenthesis position + } else if (mvelExpr.charAt(pos) == ')') { + if (stack.isEmpty()) { + return false; // unbalanced parenthesis + } + int leftPos = stack.pop(); + /* record the parenthesis pair positions + * do not record if it is pair on the left side of the first "in", or on the right side of the last "in" + */ + if (pos < reversedInPosList.get(reversedInPosList.size() - 1) || leftPos > reversedInPosList.get(0)) { + continue; + } + sortedLeftParenthesis.add(leftPos); + } + } + + // quick check if there are enough parenthesis pairs + return reversedInPosList.size() <= sortedLeftParenthesis.size(); + + /* TODO As heuristic check, the current one above is enough for existing cases. But we can add more strict check, + * to cover more extreme case, if we discover any in the future. Here just document the idea, as it is expensive + * to perform the check, but we might be dealing with non-existing use cases. + * + * Based on the observation that for projection with nested "in", the inner "in" expression is always on the right side, + * we check all "in" keywords from right to left. + * For each "in", find the right most "(" on its left. There should be no other "in" keyword between the pair of parentheses, + * and the "in" should be within the parentheses pair. + * If yes, remove the pair of parentheses as it is matched for the specific "in" keyword, and can not be used for + * other "in" keyword. + * If no, or if there are not enough pair of parentheses, return invalid + */ + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/configvalidator/typesafe/TypesafeConfigValidator.java b/feathr-config/src/main/java/com/linkedin/feathr/core/configvalidator/typesafe/TypesafeConfigValidator.java new file mode 100644 index 000000000..76cf6b2e6 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/configvalidator/typesafe/TypesafeConfigValidator.java @@ -0,0 +1,449 @@ +package com.linkedin.feathr.core.configvalidator.typesafe; + +import com.linkedin.feathr.core.configvalidator.ValidationResult; +import com.linkedin.feathr.core.config.ConfigType; +import com.linkedin.feathr.core.config.consumer.JoinConfig; +import com.linkedin.feathr.core.config.producer.FeatureDefConfig; +import com.linkedin.feathr.core.configbuilder.typesafe.TypesafeConfigBuilder; +import com.linkedin.feathr.core.configdataprovider.ConfigDataProvider; +import com.linkedin.feathr.core.configvalidator.ConfigValidationException; +import com.linkedin.feathr.core.configvalidator.ConfigValidator; +import com.linkedin.feathr.core.configvalidator.ValidationType; +import com.linkedin.feathr.core.utils.Utils; +import com.typesafe.config.Config; +import com.typesafe.config.ConfigException; +import com.typesafe.config.ConfigRenderOptions; +import com.typesafe.config.ConfigValue; +import com.typesafe.config.ConfigValueType; +import java.io.InputStream; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.StringJoiner; +import java.util.regex.Pattern; +import org.apache.log4j.Logger; +import org.everit.json.schema.Schema; +import org.everit.json.schema.ValidationException; +import org.everit.json.schema.loader.SchemaLoader; +import org.json.JSONObject; +import org.json.JSONTokener; + +import static com.linkedin.feathr.core.config.producer.FeatureDefConfig.*; +import static com.linkedin.feathr.core.config.producer.anchors.AnchorConfig.FEATURES; +import static com.linkedin.feathr.core.configvalidator.ValidationStatus.*; +import static com.linkedin.feathr.core.configvalidator.ValidationType.*; + + +/** + * @deprecated package private use only, please use {@link FeatureConsumerConfValidator} or + * {@link FeatureProducerConfValidator} as needed + * + * This class implements {@link ConfigValidator} using the Lightbend (aka Typesafe) Config Library. + * Also provides config validation methods that operate on Typesafe Config objects instead of a + * {@link ConfigDataProvider}. These methods will be used by {@link TypesafeConfigBuilder} during + * config building. + */ +@Deprecated +public class TypesafeConfigValidator implements ConfigValidator { + private static final Logger logger = Logger.getLogger(TypesafeConfigValidator.class); + + // Used when rendering the parsed config to JSON string (which is then used in validation) + private ConfigRenderOptions _renderOptions; + + // Schema for FeatureDef config + private Schema _featureDefSchema; + + // Schema for Join config + private Schema _joinConfigSchema; + + private Schema _presentationConfigSchema; + + private final static String FEATUREDEF_CONFIG_SCHEMA = "/FeatureDefConfigSchema.json"; + + private final static String JOIN_CONFIG_SCHEMA = "/JoinConfigSchema.json"; + + private final static String PRESENTATION_CONFIG_SCHEMA = "/PresentationsConfigSchema.json"; + + private static final String ANCHOR_SOURCE_NAME_REGEX = "(^[a-zA-Z][-\\w]*$)"; + private static final Pattern ANCHOR_SOURCE_NAME_PATTERN = Pattern.compile(ANCHOR_SOURCE_NAME_REGEX); + + /* + * We use the following four fields to name the capturing groups, for ease of use + */ + private static final String NAMESPACE = "namespace"; + private static final String NAME = "name"; + private static final String MAJOR = "major"; + private static final String MINOR = "minor"; + + /* + * The delimiter used to separate namespace, name and version fields. It must be chosen such that it doesn't + * conflict with the restricted characters used in HOCON, Pegasus's PathSpec and the characters used in Java + * variable names. + */ + public static final String DELIM = "-"; + + // BNF of the typed ref is: (namespace-)?name(-major-minor)? + public static final String TYPED_REF_BNF = String .join(DELIM, "(namespace", ")?name(", "major", "minor)?"); + + /* + * For all of the regex's below, the outer group where applicable, is made non-capturing by using "?:" construct. + * This is done since we want to extract only "foo" in "foo-". Also, we use named-capturing groups by using "?" + * construct. This is done for ease of reference when getting the matched value of the group. + */ + + // Represents the regex for (namespace-)? + private static final String NAMESPACE_REGEX = "(?:(?<" + NAMESPACE + ">[a-zA-Z][\\w]+)" + DELIM + ")?"; + + // Represents the regex for name + // Note: We shouldn't allow '.' or ':' in name, but in some legacy feature names, "." or ":" are being used. + // Build validation project will gradually migrate these legacy feature names off from using special characters, + // when a clean state is reached, we should remove these special characters from the regex. + private static final String NAME_REGEX = "(?<" + NAME + ">[a-zA-Z][.:\\w]*)"; + private static final String STRICT_NAME_REGEX = "(?<" + NAME + ">[a-zA-Z][\\w]*)"; + + // Represents the regex for only feature name + private static final String FEATURE_NAME_REGEX = "([a-zA-Z][.:\\w]*)"; + + // Represents regex for (-major-minor)? + private static final String VERSION_REGEX = "((?:" + DELIM + "(?<" + MAJOR + ">[\\d]+))(?:" + DELIM + "(?<" + MINOR + ">[\\d]+)))?"; + + private static final String TYPED_REF_REGEX = NAMESPACE_REGEX + NAME_REGEX + VERSION_REGEX; + + private static final String STRICT_TYPED_REF_REGEX = "^" + NAMESPACE_REGEX + STRICT_NAME_REGEX + VERSION_REGEX + "$"; + public static final Pattern STRICT_TYPED_REF_PATTERN = Pattern.compile(STRICT_TYPED_REF_REGEX); + + public TypesafeConfigValidator() { + _renderOptions = ConfigRenderOptions.defaults() + .setComments(false) + .setOriginComments(false) + .setFormatted(true) + .setJson(true); + } + + /** + * @see ConfigValidator#validate(ConfigType, ValidationType, ConfigDataProvider) + */ + @Override + public ValidationResult validate(ConfigType configType, ValidationType validationType, + ConfigDataProvider configDataProvider) { + ValidationResult result; + + switch (validationType) { + case SYNTACTIC: + // First build a Typesafe Config object representation + Config config; + try { + config = buildTypesafeConfig(configType, configDataProvider); + } catch (ConfigException e) { + String details = "Config parsing failed due to invalid HOCON syntax"; + result = new ValidationResult(SYNTACTIC, INVALID, details, e); + break; + } + + // Delegate syntax validation to another method + result = validateSyntax(configType, config); + break; + + case SEMANTIC: + result = validateSemantics(configType, configDataProvider); + break; + + default: + throw new ConfigValidationException("Unsupported validation type " + validationType); + } + logger.info("Performed " + validationType + " validation for " + configType + " config from " + + configDataProvider.getConfigDataInfo()); + + return result; + + } + + /** + * @see ConfigValidator#validate(Map, ValidationType) + */ + @Override + public Map validate(Map configTypeWithDataProvider, + ValidationType validationType) { + Map resultMap = new HashMap<>(); + + for (Map.Entry entry : configTypeWithDataProvider.entrySet()) { + ConfigType configType = entry.getKey(); + ConfigDataProvider configDataProvider = entry.getValue(); + ValidationResult result = validate(configType, validationType, configDataProvider); + resultMap.put(configType, result); + } + + return resultMap; + } + + /** + * Validates the configuration syntax. Configuration type is provided by {@link ConfigType}, and the configuration + * to be validated is provided by {@link Config} object + * @param configType ConfigType + * @param config Config object + * @return {@link ValidationResult} + * @throws ConfigValidationException if validation can't be performed + */ + public ValidationResult validateSyntax(ConfigType configType, Config config) { + ValidationResult result; + + /* + * Creates a JSON string from the HOCON config object, and validates the syntax of the config string as a valid + * Frame config (FeatureDef or Join). + */ + try { + String jsonStr = config.root().render(_renderOptions); + + JSONTokener tokener = new JSONTokener(jsonStr); + JSONObject root = new JSONObject(tokener); + + switch (configType) { + case FeatureDef: + if (_featureDefSchema == null) { + _featureDefSchema = loadFeatureDefSchema(); + logger.info("FeatureDef config schema loaded"); + } + _featureDefSchema.validate(root); + + // validate naming convention + result = validateFeatureDefNames(config); + break; + + case Join: + if (_joinConfigSchema == null) { + _joinConfigSchema = loadJoinConfigSchema(); + logger.info("Join config schema loaded"); + } + _joinConfigSchema.validate(root); + result = new ValidationResult(SYNTACTIC, VALID); + break; + + case Presentation: + if (_presentationConfigSchema == null) { + _presentationConfigSchema = loadPresentationConfigSchema(); + logger.info("Presentation config schema loaded"); + } + _presentationConfigSchema.validate(root); + result = new ValidationResult(SYNTACTIC, VALID); + break; + default: + throw new ConfigValidationException("Unknown config type: " + configType); + } + } catch (ConfigValidationException e) { + throw e; + } catch (ValidationException e) { + String header = configType + " config syntax is invalid. Details:"; + String details = String.join("\n", header, String.join("\n", e.getAllMessages())); + result = new ValidationResult(SYNTACTIC, INVALID, details, e); + } catch (Exception e) { + throw new ConfigValidationException("Config validation error", e); + } + logger.debug("Validated " + configType + " config syntax"); + + return result; + } + + /** + * Validates FeatureDef config semantically. Intended to be used by TypesafeConfigBuilder. + * @param featureDefConfig {@link FeatureDefConfig} + * @return {@link ValidationResult} + */ + public ValidationResult validateSemantics(FeatureDefConfig featureDefConfig) { + return new FeatureDefConfigSemanticValidator().validate(featureDefConfig); + } + + /** + * Validates Join config semantically. Requires both {@link JoinConfig} and {@link FeatureDefConfig} to be passed in. + * @param joinConfig {@link JoinConfig} + * @param featureDefConfig {@link FeatureDefConfig} + * @return {@link ValidationResult} + */ + public ValidationResult validateSemantics(JoinConfig joinConfig, FeatureDefConfig featureDefConfig) { + throw new ConfigValidationException("Join config semantic validation not yet implemented!"); + } + + private ValidationResult validateSemantics(ConfigType configType, ConfigDataProvider configDataProvider) { + ValidationResult result; + + switch (configType) { + case FeatureDef: + result = validateFeatureDefConfigSemantics(configDataProvider); + break; + + case Join: + result = validateJoinConfigSemantics(configDataProvider); + break; + + default: + throw new ConfigValidationException("Unsupported config type " + configType); + } + + return result; + } + + private ValidationResult validateFeatureDefConfigSemantics(ConfigDataProvider configDataProvider) { + try { + TypesafeConfigBuilder typesafeConfigBuilder = new TypesafeConfigBuilder(); + FeatureDefConfig featureDefConfig = typesafeConfigBuilder.buildFeatureDefConfig(configDataProvider); + return validateSemantics(featureDefConfig); + } catch (Throwable e) { + throw new ConfigValidationException("Fail to perform semantic validation for FeatureDef config with" + + configDataProvider.getConfigDataInfo(), e); + } + } + + private ValidationResult validateJoinConfigSemantics(ConfigDataProvider configDataProvider) { + /* + * TODO: To semantically validate a Join Config, we'll need both Join and FeatureDef configs. This will + * require changes to ConfigDataProvider interface which should have methods for getting config data + * separately for FeatureDef config, Join config, etc. + * Once obtained as above, build Frame's FeatureDefConfig and JoinConfig objects, and perform semantic + * validation. So, + * 1. Invoke TypesafeConfigBuilder to build FeatureDefConfig object. + * 2. Invoke TypesafeConfigBuilder to build JoinConfig object. + * 3. Invoke #validateSemantics(JoinConfig joinConfig, FeatureDefConfig featureDefConfig) + */ + throw new ConfigValidationException("Join config semantic validation not yet implemented!"); + } + + /** + * validate defined source name, anchor name, feature name in typesafe FeatureDef config + */ + private ValidationResult validateFeatureDefNames(Config config) { + Set definedSourceAnchorNames = new HashSet<>(); + Set definedFeatureNames = new HashSet<>(); + + if (config.hasPath(SOURCES)) { // add all source names + definedSourceAnchorNames.addAll(config.getConfig(SOURCES).root().keySet()); + } + + if (config.hasPath(ANCHORS)) { + Config anchorsCfg = config.getConfig(ANCHORS); + Set anchorNames = anchorsCfg.root().keySet(); + definedSourceAnchorNames.addAll(anchorNames); // add all anchor names + + // add all anchor defined feature names + anchorNames.stream().map(Utils::quote).forEach(quotedName -> + definedFeatureNames.addAll(getFeatureNamesFromAnchorDef(anchorsCfg.getConfig(quotedName))) + ); + } + + if (config.hasPath(DERIVATIONS)) { // add all derived feature names + definedFeatureNames.addAll(config.getConfig(DERIVATIONS).root().keySet()); + } + + definedSourceAnchorNames.removeIf(name -> ANCHOR_SOURCE_NAME_PATTERN.matcher(name).find()); + definedFeatureNames.removeIf(name -> STRICT_TYPED_REF_PATTERN.matcher(name).find()); + + return constructNamingValidationResult(definedSourceAnchorNames, definedFeatureNames); + } + + /** + * construct naming convention check validation result for invalid names + */ + private ValidationResult constructNamingValidationResult(Set invalidSourceAnchorNames, + Set invalidFeatureNames) { + + if (invalidFeatureNames.isEmpty() && invalidSourceAnchorNames.isEmpty()) { + return new ValidationResult(SYNTACTIC, VALID); + } + + StringJoiner sj = new StringJoiner("\n", "", "\n"); + + if (!invalidFeatureNames.isEmpty()) { + String msg = String.join("\n", + "The feature references/names in Frame configs must conform to the pattern (shown in BNF syntax): " + + TYPED_REF_BNF + + ", where the 'name' must conform to the pattern (shown as regex) [a-zA-Z][\\w]+", + "The following names violate Frame's feature naming convention: ", + String.join("\n", invalidFeatureNames) + ); + sj.add(msg); + } + + if (!invalidSourceAnchorNames.isEmpty()) { + String msg = String.join("\n", + "The source and anchor names in Frame configs follow the pattern (shown as regex) " + + ANCHOR_SOURCE_NAME_REGEX, + "The following names violate Frame's source and anchor naming convention: ", + String.join("\n", invalidSourceAnchorNames) + ); + sj.add(msg); + } + + return new ValidationResult(SYNTACTIC, WARN, sj.toString()); + } + + /** + * get feature names from typesafe config with anchor definition + */ + private Set getFeatureNamesFromAnchorDef(Config anchorConfig) { + + ConfigValue value = anchorConfig.getValue(FEATURES); + ConfigValueType valueType = value.valueType(); + + Set featureNames; + switch (valueType) { // Note that features can be expressed as a list or as an object + case LIST: + featureNames = new HashSet<>(anchorConfig.getStringList(FEATURES)); + break; + + case OBJECT: + featureNames = anchorConfig.getConfig(FEATURES).root().keySet(); + break; + + default: + StringBuilder sb = new StringBuilder(); + sb.append("Fail to extract feature names from anchor config. ").append("Expected ") + .append(FEATURES).append(" value type List or Object, got ").append(valueType.toString()); + throw new RuntimeException(sb.toString()); + } + + return featureNames; + } + + private Config buildTypesafeConfig(ConfigType configType, ConfigDataProvider configDataProvider) { + TypesafeConfigBuilder builder = new TypesafeConfigBuilder(); + return builder.buildTypesafeConfig(configType, configDataProvider); + } + + /* + * Loads schema for FeatureDef config using Everit JSON Schema Validator + * (https://github.com/everit-org/json-schema) + */ + private Schema loadFeatureDefSchema() { + try (InputStream inputStream = getClass().getResourceAsStream(FEATUREDEF_CONFIG_SCHEMA)) { + JSONObject rawSchema = new JSONObject(new JSONTokener(inputStream)); + return SchemaLoader.load(rawSchema); + } catch (Exception e) { + throw new ConfigValidationException("Error in loading FeatureDef schema", e); + } + } + + /* + * Loads schema for Join config using Everit JSON Schema Validator + * (https://github.com/everit-org/json-schema) + */ + private Schema loadJoinConfigSchema() { + try (InputStream inputStream = getClass().getResourceAsStream(JOIN_CONFIG_SCHEMA)) { + JSONObject rawSchema = new JSONObject(new JSONTokener(inputStream)); + return SchemaLoader.load(rawSchema); + } catch (Exception e) { + throw new ConfigValidationException("Error in loading FeatureDef schema", e); + } + } + + /* + * Loads schema for Presentation config using Everit JSON Schema Validator + * (https://github.com/everit-org/json-schema) + */ + private Schema loadPresentationConfigSchema() { + try (InputStream inputStream = getClass().getResourceAsStream(PRESENTATION_CONFIG_SCHEMA)) { + JSONObject rawSchema = new JSONObject(new JSONTokener(inputStream)); + return SchemaLoader.load(rawSchema); + } catch (Exception e) { + throw new ConfigValidationException("Error in loading PresentationConfig schema", e); + } + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/utils/ConfigUtils.java b/feathr-config/src/main/java/com/linkedin/feathr/core/utils/ConfigUtils.java new file mode 100644 index 000000000..1e3977a16 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/utils/ConfigUtils.java @@ -0,0 +1,194 @@ +package com.linkedin.feathr.core.utils; + +import com.linkedin.feathr.core.configbuilder.ConfigBuilderException; +import com.typesafe.config.ConfigRenderOptions; +import com.typesafe.config.Config; +import com.typesafe.config.ConfigValue; +import com.typesafe.config.ConfigValueType; + +import java.time.Duration; +import java.time.LocalDate; +import java.time.format.DateTimeFormatter; +import java.time.temporal.ChronoUnit; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * Utils to read typesafe configs + */ +public class ConfigUtils { + public static final String TIMESTAMP_FORMAT_EPOCH = "epoch"; + public static final String TIMESTAMP_FORMAT_EPOCH_MILLIS = "epoch_millis"; + + private ConfigUtils() { + + } + + /** + * return string config value with default + * @param config typesafe config to read value from + * @param path path of the config value + * @return config value + */ + public static String getStringWithDefault(Config config, String path, String defaultValue) { + return config.hasPath(path) ? config.getString(path) : defaultValue; + } + + /** + * return int config value with default + * @param config typesafe config to read value from + * @param path path of the config value + * @return config value + */ + public static int getIntWithDefault(Config config, String path, int defaultValue) { + return config.hasPath(path) ? config.getInt(path) : defaultValue; + } + + /** + * return numeric config value with default + * @param config typesafe config to read value from + * @param path path of the config value + * @return config value + */ + public static Number getNumberWithDefault(Config config, String path, Number defaultValue) { + return config.hasPath(path) ? config.getNumber(path) : defaultValue; + } + + /** + * return numeric config value with default + * @param config typesafe config to read value from + * @param path path of the config value + * @return config value + */ + public static Duration getDurationWithDefault(Config config, String path, Duration defaultValue) { + return config.hasPath(path) ? config.getDuration(path) : defaultValue; + } + + + /** + * return long config value with default + * @param config typesafe config to read value from + * @param path path of the config value + * @return config value + */ + public static long getLongWithDefault(Config config, String path, long defaultValue) { + return config.hasPath(path) ? config.getLong(path) : defaultValue; + } + + /** + * return boolean config value with default + * @param config typesafe config to read value from + * @param path path of the config value + * @return config value + */ + public static boolean getBooleanWithDefault(Config config, String path, Boolean defaultValue) { + return config.hasPath(path) ? config.getBoolean(path) : defaultValue; + } + + /** + * return a String map config value where the key and value are both simple {@link String} + * @param config the typesafe config containing the String map + * @return the map value + */ + public static Map getStringMap(Config config) { + return config.root().keySet().stream().collect(Collectors.toMap(k -> k, config::getString)); + } + + /** + * convert ChronoUnit String to ChronoUnit enum + * @param timeResolutionStr the timeResolution String + * @return + */ + public static ChronoUnit getChronoUnit(String timeResolutionStr) { + ChronoUnit timeResolution; + switch (timeResolutionStr.toUpperCase()) { + case "DAILY": + timeResolution = ChronoUnit.DAYS; + break; + case "HOURLY": + timeResolution = ChronoUnit.HOURS; + break; + default: + throw new RuntimeException("Unsupported time resolution unit " + timeResolutionStr); + } + return timeResolution; + } + + /** + * Check if the input timestamp pattern is valid by checking for epoch/epoch_millis and then invoking the DateTimeFormatter. + * @param fieldName Field name where present to throw a meaningful error message + * @param timestampPattern The timestamp pattern string + * @return true if valid string, else will throw an exception + */ + public static void validateTimestampPatternWithEpoch(String fieldName, String fieldValue, String timestampPattern) { + if (timestampPattern.equalsIgnoreCase(TIMESTAMP_FORMAT_EPOCH) || timestampPattern.equalsIgnoreCase(TIMESTAMP_FORMAT_EPOCH_MILLIS)) { + return; + } else { // try + validateTimestampPattern(fieldName, fieldValue, timestampPattern); + } + } + + /** + * Check if the input timestamp pattern is valid by invoking the DateTimeFormatter. + * @param fieldName Field name where present to throw a meaningful error message + * @param timestampPattern The timestamp pattern string + * @return true if valid string, else will throw an exception + */ + public static void validateTimestampPattern(String fieldName, String fieldValue, String timestampPattern) { + try { + DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern(timestampPattern); + LocalDate.parse(fieldValue, dateTimeFormatter); + } catch (Throwable e) { + throw new ConfigBuilderException(String.format("Parsing settings configuration failed for " + + "timestamp_format=%s for field name %s.", timestampPattern, fieldName), e); + } + } + + /** + * return a String list config value where the value can be either a single String or String list + * @param config the typesafe config to read value from + * @param path path of the config value + * @return config value + */ + public static List getStringList(Config config, String path) { + if (!config.hasPath(path)) { + return null; + } + + ConfigValueType valueType = config.getValue(path).valueType(); + List valueList; + switch (valueType) { + case STRING: + valueList = Collections.singletonList(config.getString(path)); + break; + + case LIST: + valueList = config.getStringList(path); + break; + + default: + throw new ConfigBuilderException("Expected value type String or List, got " + valueType); + } + return valueList; + } + + /** + * Get the typesafe {@link ConfigValue#render()} with given path + * @param config the typesafe {@Config} object to read value from + * @param path the path + * @return {@link String} representation for the {@link ConfigValue}, and null if the path does not exist + */ + public static String getHoconString(Config config, String path) { + ConfigRenderOptions renderOptions = ConfigRenderOptions.concise(); + if (!config.hasPath(path)) { + return null; + } + ConfigValue configValue = config.getValue(path); + + // Warning: HOCON might automatically add comments or quote, which won't influence HOCON parser + return configValue.render(renderOptions); + } + +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/utils/MvelInputsResolver.java b/feathr-config/src/main/java/com/linkedin/feathr/core/utils/MvelInputsResolver.java new file mode 100644 index 000000000..b64323399 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/utils/MvelInputsResolver.java @@ -0,0 +1,79 @@ +package com.linkedin.feathr.core.utils; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import org.mvel2.MVEL; +import org.mvel2.ParserContext; + + +/** + * The class is used to figure out the input features in a mvel expresion. + */ +public class MvelInputsResolver { + private static final MvelInputsResolver INSTANCE = new MvelInputsResolver(); + + public static MvelInputsResolver getInstance() { + return INSTANCE; + } + + private MvelInputsResolver() { + } + + /** + * Gets the input features in the mvel expression. + * It leverages Mvel compiler to compute the input variables. However, Mvel needs to resolve the imports via the + * classloader. To make this functionality light, we don't want to rely on the class loaders as sometimes we only + * have a simple config file. Instead, we use a heuristic approach to replace the import with some dummy class that + * we have and the input variables will still be correctly computed by Mvel. + * TODO - 7784): Migrate this inline mvel expression to a more structured derived syntax + * Part of the reason we need to do this is we are not using the more explicit derived syntax where input features + * are explicitly specified. We should explore if we can migrate the implicit inline derived features to the explicit + * ones. + */ + public List getInputFeatures(String mvelExpr) { + List expressions = Arrays.stream(mvelExpr.split(";")) + .map(String::trim) + // normalize spaces + .map(expression -> expression.replaceAll("\\s{2,}", " ")) + .collect(Collectors.toList()); + Set imports = + expressions.stream().map(String::trim).filter(x -> x.startsWith("import ")).collect(Collectors.toSet()); + + // Use the cleaned expressions for further processing + String rewrittenExpr = String.join(";", expressions); + for (String mvelImport : imports) { + List importSplit = Arrays.asList(mvelImport.split("\\.")); + String className = importSplit.get(importSplit.size() - 1); + // Use java.lang.Object as the dummy class to replace other classes to get over Mvel's import check. + // Mvel compiler will check if a class exist in the classpath. In some scenarios, we don't have the classes in + // the classpath but only the config file but we still want to run the mvel compiler. The approach here is to + // replace those imported classes with a dummy class and then the mvel compiler will continue to run(Mvel compiler + // doesn't check if the class has that function). This is a hack as mvel compiler doesn't provide other ways to + // achieve this. + // For example: "import come.linkedin.MyClass; MyClass.apply(featureA);" will be converted into + // "import java.lang.Object; Object.apply(featureA);" + rewrittenExpr = rewrittenExpr.replace(mvelImport + ";", "import java.lang.Object;"); + rewrittenExpr = rewrittenExpr.replaceAll(className + ".", "Object."); + } + // Use MVEL "analysis compiler" to figure out what the inputs are + ParserContext parserContext = new ParserContext(); + MVEL.analysisCompile(rewrittenExpr, parserContext); + + // MVEL Hack: remove '$' from the inputs, since it's a "special" input used for fold/projection statements + // For example, typeAndPermissionList = ($.type + ", " + getPermission($) in users). Here $ sign will be considered + // as an input. + // Refer to https://iwww.corp.linkedin.com/wiki/cf/pages/viewpage.action?pageId=272932479#FrameMVELUserGuide(go/framemvel)-Dollar($)SignSyntax + // for more deltails. + List list = new ArrayList<>(); + for (String featureName : parserContext.getInputs().keySet()) { + // Filter out com and org since they are imports + if (!"$".equals(featureName) && !featureName.equals("com") && !featureName.equals("org")) { + list.add(featureName); + } + } + return list; + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/core/utils/Utils.java b/feathr-config/src/main/java/com/linkedin/feathr/core/utils/Utils.java new file mode 100644 index 000000000..9a74da897 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/core/utils/Utils.java @@ -0,0 +1,115 @@ +package com.linkedin.feathr.core.utils; + +import com.typesafe.config.ConfigUtil; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + + +/** + * Utility class with methods to pretty-print different Java collections + */ +public final class Utils { + + private Utils() { + } + + /* + * For List + */ + public static String string(List list, String start, String sep, String end) { + String mid = list.stream().map(T::toString).collect(Collectors.joining(sep)); + //String mid = String.join(sep, list); + return start + mid + end; + } + + public static String string(List list) { + return string(list, "[", ", ", "]"); + } + + public static String string(List list, String sep) { + return string(list, "[", sep, "]"); + } + + /* + * For Set + */ + public static String string(Set set, String start, String sep, String end) { + String mid = set.stream().map(T::toString).collect(Collectors.joining(sep)); + return start + mid + end; + } + + public static String string(Set set) { + return string(set, "{", ", ", "}"); + } + + public static String string(Set set, String sep) { + return string(set, "{", sep, "}"); + } + + /* + * For Map + */ + public static String string(Map map, String start, String sep, String end) { + StringBuilder sb = new StringBuilder(); + sb.append(start); + map.forEach((k, v) -> sb.append(k.toString()).append(":").append(v.toString()).append(sep)); + sb.append(end); + return sb.toString(); + } + + public static String string(Map map) { + return string(map, "{", ", ", "}"); + } + + public static String string(Map map, String sep) { + return string(map, "{", sep, "}"); + } + + /* + * For Array + */ + public static String string(T[] array, String start, String sep, String end) { + String mid = Arrays.stream(array).map(T::toString).collect(Collectors.joining(sep)); + return start + mid + end; + } + + public static String string(T[] array) { + return string(array, "[", ", ", "]"); + } + + public static String string(T[] array, String sep) { + return string(array, "[", sep, "]"); + } + + /* + * for test, similar to require function in Scala + */ + public static void require(boolean expression, String message) { + if (!expression) { + throw new IllegalArgumentException(message); + } + } + + public static void require(boolean expression) { + if (!expression) { + throw new IllegalArgumentException(); + } + } + + /* + * Quotes a key if + * it contains "." or ":" + * and it's not already quoted + * so that the key is not interpreted as a path expression by HOCON/Lightbend + * Config library. Examples of such keys are names such as anchor names and feature names. + * @param key the string to be quoted if needed + * @return quoted string as per JSON specification + */ + public static String quote(String key) { + return ((key.contains(".") || key.contains(":")) && !key.startsWith("\"") && !key.endsWith("\"")) + ? ConfigUtil.quoteString(key) : key; + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/exception/ErrorLabel.java b/feathr-config/src/main/java/com/linkedin/feathr/exception/ErrorLabel.java new file mode 100644 index 000000000..7312b09fc --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/exception/ErrorLabel.java @@ -0,0 +1,9 @@ +package com.linkedin.feathr.exception; + +/** + * Error label that is used in exception message. See ExceptionMessageUtil. + */ +public enum ErrorLabel { + FEATHR_USER_ERROR, + FEATHR_ERROR +} \ No newline at end of file diff --git a/feathr-config/src/main/java/com/linkedin/feathr/exception/ExceptionMessageUtil.java b/feathr-config/src/main/java/com/linkedin/feathr/exception/ExceptionMessageUtil.java new file mode 100644 index 000000000..9ff167500 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/exception/ExceptionMessageUtil.java @@ -0,0 +1,12 @@ +package com.linkedin.feathr.exception; + +/** + * A util for creating exception message. + */ +public class ExceptionMessageUtil { + public static final String NO_SOLUTION_TEMPLATE = "This is likely a Frame issue. Contact Frame team via ask_frame@linkedin.com."; + + private ExceptionMessageUtil() { + + } +} diff --git a/feathr-config/src/main/java/com/linkedin/feathr/exception/FeathrConfigException.java b/feathr-config/src/main/java/com/linkedin/feathr/exception/FeathrConfigException.java new file mode 100644 index 000000000..19d58ede4 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/exception/FeathrConfigException.java @@ -0,0 +1,15 @@ +package com.linkedin.feathr.exception; + +/** + * This exception is thrown when the feature definition is incorrect. + */ +public class FeathrConfigException extends FeathrException { + + public FeathrConfigException(ErrorLabel errorLabel, String msg, Throwable cause) { + super(errorLabel, msg, cause); + } + + public FeathrConfigException(ErrorLabel errorLabel, String msg) { + super(errorLabel, msg); + } +} \ No newline at end of file diff --git a/feathr-config/src/main/java/com/linkedin/feathr/exception/FeathrException.java b/feathr-config/src/main/java/com/linkedin/feathr/exception/FeathrException.java new file mode 100644 index 000000000..c74c40fb5 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/exception/FeathrException.java @@ -0,0 +1,22 @@ +package com.linkedin.feathr.exception; + +/** + * Base exception for Frame + */ +public class FeathrException extends RuntimeException { + public FeathrException(String msg) { + super(msg); + } + + public FeathrException(String msg, Throwable cause) { + super(msg, cause); + } + + public FeathrException(ErrorLabel errorLabel, String msg, Throwable cause) { + super(String.format("[%s]", errorLabel) + " " + msg, cause); + } + + public FeathrException(ErrorLabel errorLabel, String msg) { + super(String.format("[%s]", errorLabel) + " " + msg); + } +} \ No newline at end of file diff --git a/feathr-config/src/main/java/com/linkedin/feathr/exception/FrameDataOutputException.java b/feathr-config/src/main/java/com/linkedin/feathr/exception/FrameDataOutputException.java new file mode 100644 index 000000000..9c0a1eae7 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/exception/FrameDataOutputException.java @@ -0,0 +1,15 @@ +package com.linkedin.feathr.exception; + +/** + * This exception is thrown when the data output is not not successful. + */ +public class FrameDataOutputException extends FeathrException { + + public FrameDataOutputException(ErrorLabel errorLabel, String msg, Throwable cause) { + super(errorLabel, msg, cause); + } + + public FrameDataOutputException(ErrorLabel errorLabel, String msg) { + super(errorLabel, msg); + } +} \ No newline at end of file diff --git a/feathr-config/src/main/java/com/linkedin/feathr/exception/FrameFeatureJoinException.java b/feathr-config/src/main/java/com/linkedin/feathr/exception/FrameFeatureJoinException.java new file mode 100644 index 000000000..dd5b3c507 --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/exception/FrameFeatureJoinException.java @@ -0,0 +1,15 @@ +package com.linkedin.feathr.exception; + +/** + * This exception is thrown when the feature join is incorrect. + */ +public class FrameFeatureJoinException extends FeathrException { + + public FrameFeatureJoinException(ErrorLabel errorLabel, String msg, Throwable cause) { + super(errorLabel, msg, cause); + } + + public FrameFeatureJoinException(ErrorLabel errorLabel, String msg) { + super(errorLabel, msg); + } +} \ No newline at end of file diff --git a/feathr-config/src/main/java/com/linkedin/feathr/exception/FrameFeatureTransformationException.java b/feathr-config/src/main/java/com/linkedin/feathr/exception/FrameFeatureTransformationException.java new file mode 100644 index 000000000..9f1e4f61b --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/exception/FrameFeatureTransformationException.java @@ -0,0 +1,15 @@ +package com.linkedin.feathr.exception; + +/** + * This exception is thrown when something wrong happened during feature transformation. + */ +public class FrameFeatureTransformationException extends FeathrException { + + public FrameFeatureTransformationException(ErrorLabel errorLabel, String msg, Throwable cause) { + super(errorLabel, msg, cause); + } + + public FrameFeatureTransformationException(ErrorLabel errorLabel, String msg) { + super(errorLabel, msg); + } +} \ No newline at end of file diff --git a/feathr-config/src/main/java/com/linkedin/feathr/exception/FrameInputDataException.java b/feathr-config/src/main/java/com/linkedin/feathr/exception/FrameInputDataException.java new file mode 100644 index 000000000..2e1058ade --- /dev/null +++ b/feathr-config/src/main/java/com/linkedin/feathr/exception/FrameInputDataException.java @@ -0,0 +1,15 @@ +package com.linkedin.feathr.exception; + +/** + * This exception is thrown when the data input is incorrect. + */ +public class FrameInputDataException extends FeathrException { + + public FrameInputDataException(ErrorLabel errorLabel, String msg, Throwable cause) { + super(errorLabel, msg, cause); + } + + public FrameInputDataException(ErrorLabel errorLabel, String msg) { + super(errorLabel, msg); + } +} \ No newline at end of file diff --git a/feathr-config/src/main/resources/FeatureDefConfigSchema.json b/feathr-config/src/main/resources/FeatureDefConfigSchema.json new file mode 100644 index 000000000..35efa07ea --- /dev/null +++ b/feathr-config/src/main/resources/FeatureDefConfigSchema.json @@ -0,0 +1,1120 @@ +{ + "$id": "FeatureDefConfigSchema.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "sources": { "$ref": "#/sectionDefinitions/sourcesSection" }, + "anchors": { "$ref": "#/sectionDefinitions/anchorsSection" }, + "derivations": { "$ref": "#/sectionDefinitions/derivationsSection" }, + "advancedDerivations": { "$ref": "#/sectionDefinitions/advancedDerivations" }, + "features": { "$ref": "#/sectionDefinitions/featuresSection" }, + "dimensions": { "$ref": "#/sectionDefinitions/dimensionsSection" } + }, + "additionalProperties": false, + "basic": { + "boolean": { + "$comment": "define our own boolean type, which accepts json boolean or json string 'true/false'", + "oneOf": [ + { + "type": "boolean" + }, + { + "enum": ["true", "True", "TRUE", "false", "False", "FALSE"] + } + ] + }, + "stringOrStringList": { + "oneOf": [ + { + "type": "string" + }, + { + "$ref":"#/basic/stringList" + } + ] + }, + "stringList": { + "type": "array", + "items": { + "type": "string" + } + }, + "stringMap": { + "type": "object" + }, + "fullyQualifiedClassName": { + "type": "string" + }, + "featureTypeEnum": { + "enum": [ + "BOOLEAN", + "NUMERIC", + "CATEGORICAL", + "CATEGORICAL_SET", + "TERM_VECTOR", + "VECTOR", + "DENSE_VECTOR", + "TENSOR" + ] + }, + "tensorCategoryEnum": { + "enum": [ + "DENSE", + "SPARSE", + "RAGGED" + ] + }, + "featureType": { + "oneOf": [ + { + "$ref":"#/basic/featureTypeEnum" + }, + { + "$ref":"#/basic/complexFeatureType" + } + ] + }, + "complexFeatureType": { + "type": "object", + "additionalProperties": false, + "required": ["type"], + "properties": { + "type": { + "$ref":"#/basic/featureTypeEnum" + }, + "tensorCategory": { + "$ref":"#/basic/tensorCategoryEnum" + }, + "shape": { + "type": "array", + "items": { + "type": "integer" + } + }, + "dimensionType": { + "type": "array", + "items": { + "type": "string" + } + }, + "valType": { + "type": "string" + } + } + } + }, + + "source": { + "type": "object", + "sourceName": { + "type": "string" + }, + "HdfsPath": { + "type": "string" + }, + "slidingWindowAggregationConfig": { + "oneOf" : [ + { + "additionalProperties": false, + "required": [ + "timestampColumn", + "timestampColumnFormat" + ], + "properties": { + "timestampColumn": { + "type": "string" + }, + "timestampColumnFormat": { + "type": "string" + } + } + }, + { + "additionalProperties": false, + "required": [ + "timestamp", + "timestamp_format" + ], + "properties": { + "timestamp": { + "type": "string" + }, + "timestamp_format": { + "type": "string" + } + } + } + ] + }, + + "HdfsConfig": { + "type": "object", + "required": ["location"], + "properties": { + "type": { + "enum": [ "HDFS"] + }, + "location": { + "type": "object", + "properties": { + "path": { + "type": "string" + } + }, + "additionalProperties": false + }, + "timePartitionPattern" : { + "type" : "string" + }, + "hasTimeSnapshot": { + "$ref": "#/basic/boolean" + }, + "isTimeSeries": { + "$ref": "#/basic/boolean" + }, + "timeWindowParameters": { "$ref": "#/source/slidingWindowAggregationConfig" } + }, + "additionalProperties": false + }, + + "EspressoConfig": { + "type": "object", + "required": ["type", "database", "table", "d2Uri", "keyExpr"], + "additionalProperties": false, + "properties": { + "type": { + "enum": [ + "ESPRESSO" + ] + }, + "database": { + "type": "string" + }, + "table": { + "type": "string" + }, + "d2Uri": { + "$ref": "#/source/D2URL" + }, + "keyExpr": {"$ref":"#/anchor/MVELExpr"} + } + }, + + "D2URL": { + "type": "string", + "pattern": "^d2://.*" + }, + + "VeniceConfig": { + "type": "object", + "required": ["type", "storeName", "keyExpr"], + "additionalProperties": false, + "properties": { + "type": { + "enum": [ + "VENICE" + ] + }, + "storeName": { + "type": "string" + }, + "keyExpr": {"$ref":"#/anchor/MVELExpr"} + } + }, + + "RocksDBConfig": { + "type": "object", + "additionalProperties": false, + "required": ["type", "referenceSource", "extractFeatures", "encoder", "decoder"], + "properties": { + "type": { + "enum": [ + "ROCKSDB" + ] + }, + "referenceSource": { + "type": "string" + }, + "extractFeatures": { + "$ref": "#/basic/boolean" + }, + "encoder": { + "type": "string" + }, + "decoder": { + "type": "string" + }, + "keyExpr": { + "type": "string" + } + } + }, + + "KafkaConfig": { + "type": "object", + "additionalProperties": false, + "required": ["type", "stream"], + "properties": { + "type": { + "enum": [ + "KAFKA" + ] + }, + "stream": { + "type": "string" + }, + "isTimeSeries": { + "$ref": "#/basic/boolean" + }, + "timeWindowParameters": { "$ref": "#/source/slidingWindowAggregationConfig" } + } + }, + + "PassThroughConfig": { + "type": "object", + "additionalProperties": false, + "required": ["type"], + "properties": { + "type": { + "enum": [ + "PASSTHROUGH" + ] + }, + "dataModel": { + "type": "string" + } + } + }, + + "CouchbaseConfig": { + "type": "object", + "required": ["type", "bucketName", "keyExpr", "documentModel"], + "additionalProperties": false, + "properties": { + "type": { + "enum": [ + "COUCHBASE" + ] + }, + "bucketName": { + "type": "string" + }, + "keyExpr": {"$ref":"#/anchor/MVELExpr"}, + "bootstrapUris": { + "type": "array", + "items": { + "type": "string" + } + }, + "documentModel": { + "type": "string" + } + } + }, + "CustomSourceConfig": { + "type": "object", + "required": ["type", "keyExpr", "dataModel"], + "additionalProperties": false, + "properties": { + "type": { + "enum": [ + "CUSTOM" + ] + }, + "keyExpr": {"$ref":"#/anchor/MVELExpr"}, + "dataModel": { + "type": "string" + } + } + }, + + "RestLiConfig": { + "type": "object", + "required": ["type", "restResourceName"], + "propertyNames": {"enum": ["finder", "keyExpr", "pathSpec", "restReqParams", "restResourceName", "restEntityType", "type"]}, + "allOf": [ + { + "properties": { + "type": { + "enum": [ "RESTLI" ] + }, + "restResourceName": { + "type": "string" + }, + "restReqParams": { + "$ref": "#/source/RestLiConfig/RestReqParams" + }, + "pathSpec": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + } + } + }, + { + "oneOf": [ + { + "$ref": "#/source/RestLiConfig/RestLiEntityType" + }, + { + "anyOf": [ + { + "$ref": "#/source/RestLiConfig/RestLiKeyExpr" + }, + { + "$ref": "#/source/RestLiConfig/RestLiFinder" + } + ] + } + + ] + } + ], + "RestLiFinder": { + "required": ["finder"], + "properties": { + "finder": { + "type": "string" + } + } + }, + "RestLiKeyExpr": { + "required": ["keyExpr"], + "properties": { + "keyExpr": { + "$ref": "#/anchor/MVELExpr" + } + } + }, + "RestLiEntityType": { + "required": ["restEntityType"], + "properties": { + "restEntityType": { + "type": "string" + } + } + }, + "RestReqParams": { + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^([a-zA-Z].*)$": { + "$ref": "#/source/RestLiConfig/RestReqParams/reqParam" + + } + }, + "reqParam": { + "$comment": "cannot declare this as type = object, otherwise will introduce extra layer of object when ref it and cause error", + "oneOf": [ + { + "type": "string" + }, + { + "type": "object", + "additionalProperties": false, + "properties": { + "json": { + "$ref": "#/source/JSONObject" + } + } + }, + { + "type": "object", + "additionalProperties": false, + "properties": { + "jsonArray": { + "type": "string" + } + } + }, + { + "type": "object", + "additionalProperties": false, + "properties": { + "mvel": { + "type": "string" + } + } + }, + { + "type": "object", + "additionalProperties": false, + "properties": { + "file": { + "type": "string" + } + } + } + ] + } + } + }, + + "PinotConfig": { + "type": "object", + "required": ["type", "resourceName", "queryTemplate", "queryArguments", "queryKeyColumns"], + "additionalProperties": false, + "properties": { + "type": { + "enum": [ + "PINOT" + ] + }, + "resourceName": { + "type": "string" + }, + "queryTemplate": { + "type": "string" + }, + "queryArguments": { + "type": "array", + "items": { + "$ref": "#/anchor/MVELExpr" + } + }, + "queryKeyColumns": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + + "JSONObject": { + "type": "string" + }, + "JSONArray": { + "type": "string" + } + }, + + "anchor": { + "anchorConfig": { + "type": "object", + "$comment":"use allOf and properties achieve combination/inheritance, since we use allOf, we can not use additionalProperties = false, instead, we use propertyNames, see https://github.com/json-schema-org/json-schema-org.github.io/issues/77", + "propertyNames": {"enum": ["source", "features", "keyExtractor", "extractor", "key", "keyAlias", "transformer", "extract", "lateralViewParameters"]}, + "allOf": [ + { + "properties": { + "source": { + "$ref": "#/source/sourceName" + } + }, + "required": ["source"] + }, + { + "oneOf": [ + { + "$ref": "#/anchor/featuresWithKey" + }, + { + "$ref": "#/anchor/featuresWithExtractor" + } + ] + } + ] + }, + "featuresWithKey": { + "type": "object", + "required": ["features"], + "$comment": "featuresWithKey does not allow transformer or extractor", + "properties": { + "transformer": { "not" : {} }, + "extractor": { "not": {} }, + "key": { + "$ref": "#/anchor/defExpr" + }, + "keyAlias": { + "$ref": "#/basic/stringOrStringList" + }, + "keyExtractor": { + "type": "string" + }, + "lateralViewParameters": { + "type": "object", + "additionalProperties": false, + "required": [ + "lateralViewItemAlias", + "lateralViewDef" + ], + "properties": { + "lateralViewDef": { + "type": "string" + }, + "lateralViewItemAlias": { + "type": "string" + } + } + }, + "features": { + "type": "object", + "patternProperties": { + "^([a-zA-Z].*)$": { + "$ref": "#/anchor/featureKConfig" + } + } + } + } + }, + + "featuresWithExtractor": { + "type": "object", + "required": ["features"], + "$comment": "need to include 'souce' as well, although this belongs to upper level", + "propertyNames": {"enum": ["extractor", "extract", "features", "key", "keyAlias", "keyExtractor", "source", "transformer"]}, + "allOf": [ + { + "oneOf": [ + { + "required": ["transformer"], + "properties": { + "transformer": { + "type": "string" + } + } + }, + { + "required": ["extractor"], + "properties": { + "extractor": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "object", + "required": ["class"], + "propertyNames": {"enum": ["class", "params"]}, + "properties": { + "class": { + "type": "string" + }, + "params": { + "type": "object" + } + } + } + ] + } + } + } + ] + }, + { + "properties": { + "key": { + "$ref": "#/anchor/defExpr" + }, + "keyAlias": { + "$ref": "#/basic/stringOrStringList" + }, + "keyExtractor": { + "type": "string" + }, + "features": { + "oneOf": [ + { + "type": "array", + "items": { + "type": "string" + } + }, + { + "type": "object", + "patternProperties": { + "^([a-zA-Z].*)$": { + "type": "object", + "additionalProperties": false, + "properties": { + "def": { + "$ref": "#/anchor/defExpr" + }, + "default": { + "$ref":"#/anchor/defaultValue" + }, + "type": { + "$ref": "#/basic/featureType" + }, + "parameters": { + "$ref": "#/basic/stringMap" + } + } + } + } + }, + { + "type": "object", + "patternProperties": { + "^([a-zA-Z].*)$": { + "$ref":"#/anchor/simpleFeatureKConfig" + } + } + } + ] + } + } + } + ] + }, + "defExpr": { + "oneOf": [ + { + "$ref": "#/anchor/validExpr" + }, + { + "type": "object", + "additionalProperties": false, + "properties": { + "sqlExpr": { + "$ref": "#/anchor/validExpr" + }, + "mvel": { + "$ref": "#/anchor/MVELExpr" + } + } + } + ] + }, + "validExpr" : { + "oneOf": [ + { + "$ref": "#/basic/stringOrStringList" + }, + { + "type":"number" + }, + { + "type":"boolean" + } + ] + }, + "featureKConfig": { + "$comment":" Don't declare this as type = object, otherwise, it will fail because of having this extra 'level' of object", + "oneOf": [ + { + "$ref":"#/anchor/simpleFeatureKConfig" + }, + { + "$ref":"#/anchor/complexFeatureKConfig" + }, + { + "$ref":"#/anchor/nearLineFeatureKConfig" + } + ] + }, + "simpleFeatureKConfig": { + "$ref":"#/anchor/MVELExpr" + }, + "complexFeatureKConfig": { + "type": "object", + "additionalProperties": false, + "properties": { + "def": { + "$ref": "#/anchor/defExpr" + }, + "type": { + "$ref": "#/basic/featureType" + }, + "default": { + "$ref":"#/anchor/defaultValue" + }, + "aggregation": { + "enum": ["SUM", "COUNT", "MAX", "MIN", "AVG", "LATEST", "AVG_POOLING", "MAX_POOLING", "MIN_POOLING"] + }, + "window": { + "$ref":"#/anchor/durationPattern" + }, + "filter": { + "type":"string" + }, + "groupBy": { + "type":"string" + }, + "limit": { + "type":"integer" + }, + "embeddingSize": { + "type": "integer" + } + } + }, + "nearLineFeatureKConfig": { + "type": "object", + "required": ["windowParameters"], + "additionalProperties": false, + "properties": { + "def": { + "$ref": "#/anchor/defExpr" + }, + "aggregation": { + "enum": ["SUM", "COUNT", "MAX", "AVG", "AVG_POOLING", "MAX_POOLING", "MIN_POOLING"] + }, + "windowParameters": { + "type": "object", + "additionalProperties": false, + "properties": { + "type": { + "enum": ["SLIDING", "FIXED", "SESSION"] + }, + "size": { + "$ref":"#/anchor/durationPattern" + }, + "slidingInterval": { + "$ref":"#/anchor/durationPattern" + } + } + }, + "groupBy": { + "oneOf": [ + { + "type": "string" + }, + { + "$ref": "#/basic/stringList" + } + ] + }, + "filter": { + "$ref": "#/anchor/defExpr" + } + } + }, + "MVELExpr": { + "type": "string" + }, + "durationPattern": { + "type": "string", + "pattern": "^(\\s*)(\\d)+(d|day|days|h|hour|hours|m|minute|minutes|s|second|seconds)(\\s*)$" + }, + "defaultValue": { + "$comment": "intentionally left empty" + } + }, + "derivation": { + "type": "object", + "properties": { + }, + "advancedDerivedFeature": { + "type": "object", + "required": ["features", "class", "key", "inputs"], + "additionalProperties": false, + "properties": { + "features": { + "$ref": "#/basic/stringOrStringList" + }, + "class": { + "oneOf": [ + { + "$ref":"#/derivation/advancedDerivedFunction" + }, + { + "type": "string" + } + ] + }, + "key": { + "$ref": "#/basic/stringOrStringList" + }, + "inputs": { + "oneOf": [ + { + "enum": ["PROVIDED_BY_CLASS"] + }, + { + "$ref": "#/derivation/inputsObj" + }] + } + } + }, + "derivationConfig": { + "oneOf": [ + { + "$ref": "#/anchor/MVELExpr" + }, + { + "$ref": "#/derivation/derivationConfigWithSqlExpr" + }, + { + "$ref": "#/derivation/derivationConfigWithExtractor" + }, + { + "$ref": "#/derivation/derivationConfigWithExpr" + }, + { + "$ref": "#/derivation/derivationConfigForSequentialJoin" + } + ] + }, + "derivationConfigWithSqlExpr": { + "type": "object", + "required": ["sqlExpr"], + "additionalProperties": false, + "properties": { + "sqlExpr": { + "type": "string" + }, + "type": { + "$ref": "#/basic/featureType" + } + } + }, + "derivationConfigWithExpr": { + "type": "object", + "required": ["definition"], + "additionalProperties": false, + "properties": { + "definition": { + "$ref": "#/anchor/defExpr" + }, + "key": { + "$ref": "#/basic/stringOrStringList" + }, + "inputs": { + "$ref":"#/derivation/inputsObj" + }, + "type": { + "$ref": "#/basic/featureType" + } + } + }, + "inputsObj": { + "type": "object", + "patternProperties": { + "^([a-zA-Z].*)$": { "$ref": "#/derivation/keyedFeature" } + } + }, + "inputsList": { + "type":"array", + "items": { + "$ref":"#/derivation/keyedFeature" + } + }, + "advancedDerivedFunction" : { + "type": "object", + "required": ["name"], + "properties": { + "name": { + "type": "string" + } + } + }, + "UDF": { + "$ref":"#/anchor/MVELExpr" + }, + "derivationConfigWithExtractor": { + "type": "object", + "additionalProperties": false, + "required": ["key", "inputs", "class"], + "properties": { + "key": { + "oneOf": [ + { + "$ref":"#/anchor/MVELExpr" + }, + { + "$ref":"#/basic/stringList" + } + ] + }, + "inputs": { + "oneOf": [ + { + "$ref": "#/derivation/inputsList" + }, + { + "$ref": "#/derivation/inputsObj" + } + ] + }, + "class": { + "$ref":"#/basic/fullyQualifiedClassName" + }, + "type": { + "$ref": "#/basic/featureType" + } + } + }, + "derivationConfigForSequentialJoin": { + "type": "object", + "required": ["key", "join", "aggregation"], + "additionalProperties": false, + "properties": { + "key": { + "$ref": "#/basic/stringOrStringList" + }, + "join": { + "$ref": "#/derivation/sequentialJoinObj" + }, + "aggregation": { + "$comment": "need to support empty string, as the aggregation is not supported in frame-offline, as the aggregation is not supported in frame-offline, and empty string is used as a placeholder", + "enum": ["UNION", "SUM", "AVG", "MAX", "MIN", "ELEMENTWISE_MAX", "ELEMENTWISE_MIN", "ELEMENTWISE_AVG", "", "ELEMENTWISE_SUM"] + }, + "type": { + "$ref": "#/basic/featureType" + } + } + }, + "sequentialJoinObj": { + "type": "object", + "required": ["base", "expansion"], + "additionalProperties": false, + "properties": { + "base": { + "$ref": "#/derivation/baseFeature" + }, + "expansion": { + "$ref": "#/derivation/keyedFeature" + } + } + }, + "baseFeature": { + "type": "object", + "required": ["key", "feature"], + "additionalProperties": false, + "properties": { + "key": { + "$ref": "#/basic/stringOrStringList" + }, + "feature": { + "type": "string" + }, + "outputKey": { + "$ref": "#/basic/stringOrStringList" + }, + "transformation": { + "$ref": "#/anchor/validExpr" + }, + "transformationClass": { + "$ref":"#/basic/fullyQualifiedClassName" + } + }, + "oneOf": [ + { + "$comment": "if transformation is present, outputKey should also be present", + "required": ["outputKey", "transformation"] + }, + { + "$comment": "if transformationClass is present, outputKey should also be present", + "required": ["outputKey", "transformationClass"] + }, + { + "$comment": "Otherwise, neither transformation or transformationClass should be present", + "allOf": [ + {"not": { "required" :["transformation"]}}, + {"not": { "required" :["transformationClass"]}} + ] + } + ] + }, + "keyedFeature": { + "type": "object", + "required": ["key", "feature"], + "additionalProperties": false, + "properties": { + "key": { + "oneOf": [ + { + "type": "string" + }, + { + "$ref": "#/basic/stringList" + } + ] + }, + "feature": { + "type":"string" + } + } + } + }, + "sectionDefinitions": { + "sourcesSection": { + "type": "object", + "properties": { + }, + "patternProperties": { + "^([a-zA-Z].*)$": { + "type": "object", + "oneOf": [ + { + "$ref": "#/source/HdfsConfig" + }, + { + "$ref": "#/source/EspressoConfig" + }, + { + "$ref": "#/source/RestLiConfig" + }, + { + "$ref": "#/source/VeniceConfig" + }, + { + "$ref": "#/source/RocksDBConfig" + }, + { + "$ref": "#/source/KafkaConfig" + }, + { + "$ref": "#/source/PassThroughConfig" + }, + { + "$ref": "#/source/CouchbaseConfig" + }, + { + "$ref": "#/source/CustomSourceConfig" + }, + { + "$ref": "#/source/PinotConfig" + } + ] + } + }, + "additionalProperties": false + }, + + "anchorsSection": { + "type": "object", + "patternProperties": { + "^([a-zA-Z].*)$": { + "$ref": "#/anchor/anchorConfig" + } + }, + "additionalProperties": false + }, + "derivationsSection": { + "type": "object", + "patternProperties": { + "^(.*)": { + "$ref": "#/derivation/derivationConfig" + } + }, + "additionalProperties": false + }, + + "advancedDerivations": { + "type": "array", + "items": { + "$ref":"#/derivation/advancedDerivedFeature" + } + }, + + "featuresSection": { + "$comment": "TO BE DONE", + "type": "object" + }, + + "dimensionsSection": { + "$comment": "TO BE DONE", + "type": "object" + } + } +} \ No newline at end of file diff --git a/feathr-config/src/main/resources/JoinConfigSchema.json b/feathr-config/src/main/resources/JoinConfigSchema.json new file mode 100644 index 000000000..0df46b325 --- /dev/null +++ b/feathr-config/src/main/resources/JoinConfigSchema.json @@ -0,0 +1,162 @@ +{ + "$id": "JoinConfigSchema.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "basic": { + "stringList":{ + "type": "array", + "items": { + "type": "string" + } + }, + "stringOrStringList": { + "oneOf": [ + { + "type": "string" + }, + { + "$ref": "#/basic/stringList" + } + ] + }, + "durationPattern": { + "type": "string", + "pattern": "^(\\s*)(-?)(\\d)+(d|day|days|h|hour|hours|m|minute|minutes|s|second|seconds)(\\s*)$" + }, + "boolean": { + "$comment": "define our own boolean type", + "oneOf": [ + { + "type": "boolean" + }, + { + "enum": ["true", "false"] + } + ] + } + }, + "definitions": { + "joinTimeSettingsConfig": { + "type": "object", + "properties": { + "timestampColumn": { + "type": "object", + "properties": { + "def": { + "type": "string" + }, + "format": { + "type": "string" + } + }, + "required": ["def", "format"] + }, + "simulateTimeDelay": { + "$ref": "#/basic/durationPattern" + }, + "useLatestFeatureData": { + "$ref": "#/basic/boolean" + } + }, + "additionalProperties": false + }, + "observationDataTimeSettingsConfig": { + "type": "object", + "properties": { + "absoluteTimeRange": { + "type": "object", + "properties": { + "startTime": { + "type": "string" + }, + "endTime": { + "type": "string" + }, + "timeFormat": { + "type": "string" + } + }, + "required": ["startTime", "endTime", "timeFormat"] + }, + "relativeTimeRange": { + "type": "object", + "properties": { + "window": { + "type": "string" + }, + "offset": { + "type": "string" + } + }, + "required": ["window"] + } + }, + "additionalProperties": false + }, + "absoluteTimeRange": { + "type": "object", + "properties": { + "startTime": { + "type": "string" + }, + "endTime": { + "type": "string" + }, + "timeFormat": { + "type": "string" + } + }, + "required": ["startTime", "endTime", "timeFormat"] + }, + "relativeTimeRange": { + "type": "object", + "properties": { + "window": { + "type": "string" + }, + "offset": { + "type": "string" + } + }, + "required": ["window"] + }, + "featuresWithSameKey":{ + "type": "object", + "required": ["key", "featureList"], + "properties": { + "key": { + "$ref": "#/basic/stringOrStringList" + }, + "featureList": { + "$ref": "#/basic/stringOrStringList" + }, + "overrideTimeDelay": { + "$ref": "#/basic/durationPattern" + } + } + } + }, + "patternProperties": { + "^(?!settings).*$": { + "type": "array", + "items": { + "$ref": "#/definitions/featuresWithSameKey" + } + }, + "settings": { + "type": "object", + "$comment": "settings can have observationDataTimeSettings, joinTimeSettings", + "properties": { + "observationDataTimeSettings": { + "type": "object", + "$ref": "#/definitions/observationDataTimeSettingsConfig" + }, + "joinTimeSettings": { + "type": "object", + "$ref": "#/definitions/joinTimeSettingsConfig" + } + }, + "additionalProperties": false + } + } + } diff --git a/feathr-config/src/main/resources/PresentationsConfigSchema.json b/feathr-config/src/main/resources/PresentationsConfigSchema.json new file mode 100644 index 000000000..ecb3dae66 --- /dev/null +++ b/feathr-config/src/main/resources/PresentationsConfigSchema.json @@ -0,0 +1,49 @@ +{ + "$id": "PresentationsConfigSchema.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "basic": { + "stringList": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "properties": { + "presentations": { "$ref": "#/presentationsSection" } + }, + "presentationsSection": { + "type": "object", + "patternProperties": { + "^([a-zA-Z][.:\\w]*)$": { + "$ref": "#/presentationConfig" + } + }, + "additionalProperties": false + }, + "presentationConfig": { + "type": "object", + "properties": { + "memberViewFeatureName": { + "type": "string" + }, + "linkedInViewFeatureName": { + "type": "string" + }, + "featureDescription": { + "type": "string" + }, + "valueTranslation": { + "type": "string" + }, + "exportModes": { + "$ref":"#/basic/stringList" + }, + "isValueExportable": { + "type": "boolean" + } + }, + "additionalProperties": false + } +} \ No newline at end of file diff --git a/feathr-config/src/main/resources/log4j.properties b/feathr-config/src/main/resources/log4j.properties new file mode 100644 index 000000000..ef6b061a8 --- /dev/null +++ b/feathr-config/src/main/resources/log4j.properties @@ -0,0 +1,9 @@ +# Set root logger level to INFO and its only appender to A1. +log4j.rootLogger=INFO, A1 + +# A1 is set to be a ConsoleAppender. +log4j.appender.A1=org.apache.log4j.ConsoleAppender + +# A1 uses PatternLayout. +log4j.appender.A1.layout=org.apache.log4j.PatternLayout +log4j.appender.A1.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} [%t] %-5p %c %x - %m%n diff --git a/feathr-config/src/test/java/com/linkedin/feathr/core/config/producer/sources/PinotConfigTest.java b/feathr-config/src/test/java/com/linkedin/feathr/core/config/producer/sources/PinotConfigTest.java new file mode 100644 index 000000000..c5190850f --- /dev/null +++ b/feathr-config/src/test/java/com/linkedin/feathr/core/config/producer/sources/PinotConfigTest.java @@ -0,0 +1,14 @@ +package com.linkedin.feathr.core.config.producer.sources; + +import nl.jqno.equalsverifier.EqualsVerifier; +import org.testng.annotations.Test; + +/** + * Test class for {@link PinotConfig} + */ +public class PinotConfigTest { + @Test(description = "test equals and hashcode") + public void testEqualsHashcode() { + EqualsVerifier.forClass(PinotConfig.class).usingGetClass().verify(); + } +} diff --git a/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/ConfigBuilderTest.java b/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/ConfigBuilderTest.java new file mode 100644 index 000000000..fb5e072e0 --- /dev/null +++ b/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/ConfigBuilderTest.java @@ -0,0 +1,34 @@ +package com.linkedin.feathr.core.configbuilder; + +import com.linkedin.feathr.core.configbuilder.typesafe.producer.FeatureDefFixture; +import com.linkedin.feathr.core.config.producer.FeatureDefConfig; +import org.testng.annotations.Test; + +import static org.testng.Assert.*; + + +public class ConfigBuilderTest { + + @Test(description = "Tests build of FeatureDefConfig object for a syntactically valid config") + public void testFeatureDefConfig() { + ConfigBuilder configBuilder = ConfigBuilder.get(); + try { + FeatureDefConfig obsFeatureDefConfigObj = configBuilder.buildFeatureDefConfigFromString( + FeatureDefFixture.featureDefConfigStr1); + assertEquals(obsFeatureDefConfigObj, FeatureDefFixture.expFeatureDefConfigObj1); + } catch (ConfigBuilderException e) { + fail("Test failed", e); + } + } + + @Test + public void testFeatureCareers() { + ConfigBuilder configBuilder = ConfigBuilder.get(); + try { + FeatureDefConfig obsFeatureDefConfigObj + = configBuilder.buildFeatureDefConfig("frame-feature-careers-featureDef-offline.conf"); + } catch (ConfigBuilderException e) { + fail("Test failed", e); + } + } +} diff --git a/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/AbstractConfigBuilderTest.java b/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/AbstractConfigBuilderTest.java new file mode 100644 index 000000000..daa48fc28 --- /dev/null +++ b/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/AbstractConfigBuilderTest.java @@ -0,0 +1,70 @@ +package com.linkedin.feathr.core.configbuilder.typesafe; + +import com.linkedin.feathr.core.config.ConfigObj; +import com.typesafe.config.Config; +import com.typesafe.config.ConfigFactory; +import java.util.List; +import java.util.function.BiFunction; +import java.util.function.Function; +import nl.jqno.equalsverifier.EqualsVerifier; + +import static com.linkedin.feathr.core.utils.Utils.*; +import static org.testng.Assert.*; + + +public abstract class AbstractConfigBuilderTest { + + public void testConfigBuilder(String configStr, BiFunction configBuilder, + ConfigObj expConfigObj) { + ConfigInfo configInfo = getKeyAndConfig(configStr); + ConfigObj obsConfigObj = configBuilder.apply(configInfo.configName, configInfo.config); + assertEquals(obsConfigObj, expConfigObj); + } + + public void testConfigBuilder(String configStr, Function configBuilder, ConfigObj expConfigObj) { + ConfigInfo configInfo = getKeyAndConfig(configStr); + ConfigObj obsConfigObj = configBuilder.apply(configInfo.config); + assertEquals(obsConfigObj, expConfigObj); + } + + @FunctionalInterface + public interface ConfigListToConfigObjBuilder extends Function, ConfigObj> {} + + public void testConfigBuilder(String configStr, ConfigListToConfigObjBuilder configBuilder, ConfigObj expConfigObj) { + Config fullConfig = ConfigFactory.parseString(configStr); + String configName = fullConfig.root().keySet().iterator().next(); + List configList = fullConfig.getConfigList(quote(configName)); + + ConfigObj obsConfigObj = configBuilder.apply(configList); + assertEquals(obsConfigObj, expConfigObj); + } + + public ConfigObj buildConfig(String configStr, BiFunction configBuilder) { + ConfigInfo configInfo = getKeyAndConfig(configStr); + return configBuilder.apply(configInfo.configName, configInfo.config); + } + + public void testEqualsAndHashCode(Class clazz, String... ignoredFields) { + EqualsVerifier.forClass(clazz) + .usingGetClass() + .withIgnoredFields(ignoredFields) + .verify(); + } + + private class ConfigInfo{ + final String configName; + final Config config; + + ConfigInfo(String configName, Config config) { + this.configName = configName; + this.config = config; + } + } + + private ConfigInfo getKeyAndConfig(String configStr) { + Config fullConfig = ConfigFactory.parseString(configStr); + String configName = fullConfig.root().keySet().iterator().next(); + Config config = fullConfig.getConfig(quote(configName)); + return new ConfigInfo(configName, config); + } +} diff --git a/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/TriFunction.java b/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/TriFunction.java new file mode 100644 index 000000000..cfba96429 --- /dev/null +++ b/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/TriFunction.java @@ -0,0 +1,6 @@ +package com.linkedin.feathr.core.configbuilder.typesafe; + +@FunctionalInterface +public interface TriFunction { + R apply(T t, U u, V v); +} diff --git a/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/TypesafeConfigBuilderTest.java b/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/TypesafeConfigBuilderTest.java new file mode 100644 index 000000000..8ae5e884d --- /dev/null +++ b/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/TypesafeConfigBuilderTest.java @@ -0,0 +1,189 @@ +package com.linkedin.feathr.core.configbuilder.typesafe; + +import com.linkedin.feathr.core.config.producer.FeatureDefConfig; +import com.linkedin.feathr.core.config.producer.sources.EspressoConfig; +import com.linkedin.feathr.core.config.producer.sources.HdfsConfigWithRegularData; +import com.linkedin.feathr.core.config.producer.sources.SourceConfig; +import com.linkedin.feathr.core.configbuilder.ConfigBuilderException; +import com.linkedin.feathr.core.configbuilder.typesafe.producer.FeatureDefFixture; +import java.io.File; +import java.net.URL; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import org.testng.annotations.Test; + +import static com.linkedin.feathr.core.configbuilder.typesafe.TypesafeFixture.*; +import static com.linkedin.feathr.core.configbuilder.typesafe.producer.FeatureDefFixture.*; +import static org.testng.Assert.*; + + +public class TypesafeConfigBuilderTest { + + private TypesafeConfigBuilder configBuilder = new TypesafeConfigBuilder(); + + @Test(description = "Tests build of FeatureDefConfig object for a syntactically valid config") + public void testFeatureDefConfig() { + try { + FeatureDefConfig obsFeatureDefConfigObj = configBuilder.buildFeatureDefConfigFromString(featureDefConfigStr1); + assertEquals(obsFeatureDefConfigObj, FeatureDefFixture.expFeatureDefConfigObj1); + } catch (ConfigBuilderException e) { + fail("Test failed", e); + } + } + + @Test(expectedExceptions = ConfigBuilderException.class, description = "Tests build of invalid FeatureDef config") + public void testFeatureDefConfig2() { + String featureDefConfigStr = "{invalidSectionName: {}}"; + FeatureDefConfig obsFeatureDefConfigObj = configBuilder.buildFeatureDefConfigFromString(featureDefConfigStr); + fail("Test shouldn't pass for invalid config"); + } + + @Test(description = "Include of another config and selective overrides") + public void includeTest() { + String expEspressoConfigName = "MemberPreferenceData"; + String expHdfsConfigName = "member_derived_data"; + + EspressoConfig expEspressoConfigObj = new EspressoConfig(expEspressoConfigName, "CareersPreferenceDB", + "MemberPreference", "d2://EI_ESPRESSO_MT2", "key[0]"); + + + String path = "/eidata/derived/standardization/waterloo/members_std_data/#LATEST"; + HdfsConfigWithRegularData expHdfsConfigObj = new HdfsConfigWithRegularData(expHdfsConfigName, path, false); + + TypesafeConfigBuilder configBuilder = new TypesafeConfigBuilder(); + try { + FeatureDefConfig config = configBuilder.buildFeatureDefConfig("dir2/features-1-ei.conf"); + + assertTrue(config.getSourcesConfig().isPresent()); + + Map sourcesConfig = config.getSourcesConfig().get().getSources(); + + assertTrue(sourcesConfig.containsKey(expEspressoConfigName)); + SourceConfig obsEspressoConfigObj = sourcesConfig.get(expEspressoConfigName); + assertEquals(obsEspressoConfigObj, expEspressoConfigObj); + + assertTrue(sourcesConfig.containsKey(expHdfsConfigName)); + SourceConfig obsHdfsConfigObj = sourcesConfig.get(expHdfsConfigName); + assertEquals(obsHdfsConfigObj, expHdfsConfigObj); + } catch (ConfigBuilderException e) { + fail("Error in building config", e); + } + } + + @Test(description = "Tests build of FeatureDefConfig object from single resource file") + public void testFeatureDefConfigFromResource1() { + try { + FeatureDefConfig obsFeatureDef1ConfigObj = configBuilder.buildFeatureDefConfig("dir1/features-2-prod.conf"); + + assertEquals(obsFeatureDef1ConfigObj, expFeatureDef1ConfigObj); + + } catch (ConfigBuilderException e) { + fail("Error in building config", e); + } + } + + @Test(description = "Tests build of FeatureDefConfig object from multiple resource files") + public void testFeatureDefConfigFromResource2() { + try { + List sources = Arrays.asList("dir1/features-3-prod.conf", "dir1/features-2-prod.conf"); + FeatureDefConfig obsFeatureDef2ConfigObj = configBuilder.buildFeatureDefConfig(sources); + + assertEquals(obsFeatureDef2ConfigObj, expFeatureDef2ConfigObj); + + } catch (ConfigBuilderException e) { + fail("Error in building config", e); + } + } + + @Test(description = "Tests build of FeatureDefConfig object with single configuration file specified by URL") + public void testFeatureDefConfigFromUrl1() { + try { + URL url = new File("src/test/resources/dir1/features-2-prod.conf").toURI().toURL(); + FeatureDefConfig obsFeatureDef1ConfigObj = configBuilder.buildFeatureDefConfig(url); + + assertEquals(obsFeatureDef1ConfigObj, expFeatureDef1ConfigObj); + + } catch (Throwable e) { + fail("Error in building config", e); + } + } + + @Test(description = "Tests build of FeatureDefConfig object with multiple configuration files specified by list of URLs") + public void testFeatureDefConfigFromUrl2() { + try { + URL url1 = new File("src/test/resources/dir1/features-3-prod.conf").toURI().toURL(); + URL url2 = new File("src/test/resources/dir1/features-2-prod.conf").toURI().toURL(); + List urls = Arrays.asList(url1, url2); + FeatureDefConfig obsFeatureDef2ConfigObj = configBuilder.buildFeatureDefConfigFromUrls(urls); + + assertEquals(obsFeatureDef2ConfigObj, expFeatureDef2ConfigObj); + + } catch (Throwable e) { + fail("Error in building config", e); + } + } + + @Test(description = "Tests build of FeatureDefConfig object from a local config file specified in a manifest") + public void testFeatureDefConfigFromManifest1() { + try { + FeatureDefConfig obsFeatureDef1ConfigObj = configBuilder.buildFeatureDefConfigFromManifest("config/manifest1.conf"); + + assertEquals(obsFeatureDef1ConfigObj, expFeatureDef1ConfigObj); + } catch (ConfigBuilderException e) { + fail("Error in building config", e); + } + } + + @Test(description = "Tests build of FeatureDefConfig object from a config file in external jar specified in a manifest") + public void testFeatureDefConfigFromManifest2() { + try { + FeatureDefConfig obsFeatureDefConfigObj = configBuilder.buildFeatureDefConfigFromManifest("config/manifest2.conf"); + + assertTrue(obsFeatureDefConfigObj.getAnchorsConfig().isPresent()); + assertTrue(obsFeatureDefConfigObj.getSourcesConfig().isPresent()); + assertTrue(obsFeatureDefConfigObj.getDerivationsConfig().isPresent()); + } catch (ConfigBuilderException e) { + fail("Error in building config", e); + } + } + + @Test(description = "Tests build of FeatureDefConfig object from local and external config files specified in a manifest") + public void testFeatureDefConfigFromManifest3() { + try { + FeatureDefConfig obsFeatureDefConfigObj = configBuilder.buildFeatureDefConfigFromManifest("config/manifest3.conf"); + + assertTrue(obsFeatureDefConfigObj.getAnchorsConfig().isPresent()); + assertTrue(obsFeatureDefConfigObj.getSourcesConfig().isPresent()); + assertTrue(obsFeatureDefConfigObj.getDerivationsConfig().isPresent()); + } catch (ConfigBuilderException e) { + fail("Error in building config", e); + } + } + + /* + @Test(description = "Tests build of JoinConfig object from single resource file") + public void testJoinConfigFromResource1() { + try { + JoinConfig obsJoinConfigObj1 = configBuilder.buildJoinConfig("dir1/join.conf"); + + assertEquals(obsJoinConfigObj1, expJoinConfigObj1); + + } catch (ConfigBuilderException e) { + fail("Error in building config", e); + } + } + + @Test(description = "Tests build of JoinConfig object with single configuration file specified by URL") + public void testJoinConfigFromUrl1() { + try { + URL url = new File("src/test/resources/dir1/join.conf").toURI().toURL(); + JoinConfig obsJoinConfigObj1 = configBuilder.buildJoinConfig(url); + + assertEquals(obsJoinConfigObj1, expJoinConfigObj1); + + } catch (Throwable e) { + fail("Error in building config", e); + } + }*/ +} diff --git a/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/TypesafeFixture.java b/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/TypesafeFixture.java new file mode 100644 index 000000000..82d9636d0 --- /dev/null +++ b/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/TypesafeFixture.java @@ -0,0 +1,37 @@ +package com.linkedin.feathr.core.configbuilder.typesafe; + +import com.linkedin.feathr.core.config.producer.FeatureDefConfig; +import com.linkedin.feathr.core.config.producer.anchors.AnchorConfig; +import com.linkedin.feathr.core.config.producer.anchors.AnchorsConfig; +import com.linkedin.feathr.core.config.producer.sources.SourceConfig; +import com.linkedin.feathr.core.config.producer.sources.SourcesConfig; +import java.util.HashMap; +import java.util.Map; + +import static com.linkedin.feathr.core.configbuilder.typesafe.producer.anchors.AnchorsFixture.*; +import static com.linkedin.feathr.core.configbuilder.typesafe.producer.sources.SourcesFixture.*; + + +class TypesafeFixture { + + static final FeatureDefConfig expFeatureDef1ConfigObj; + static { + Map anchors = new HashMap<>(); + anchors.put("member-lix-segment", expAnchor1ConfigObj); + AnchorsConfig anchorsConfigObj = new AnchorsConfig(anchors); + expFeatureDef1ConfigObj = new FeatureDefConfig(null, anchorsConfigObj, null); + } + + static final FeatureDefConfig expFeatureDef2ConfigObj; + static { + Map sources = new HashMap<>(); + sources.put("MemberPreferenceData", expEspressoSource1ConfigObj); + sources.put("member_derived_data", expHdfsSource1ConfigObj); + SourcesConfig sourcesConfigObj = new SourcesConfig(sources); + + Map anchors = new HashMap<>(); + anchors.put("member-lix-segment", expAnchor1ConfigObj); + AnchorsConfig anchorsConfigObj = new AnchorsConfig(anchors); + expFeatureDef2ConfigObj = new FeatureDefConfig(sourcesConfigObj, anchorsConfigObj, null); + } +} diff --git a/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/consumer/FeatureBagConfigBuilderTest.java b/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/consumer/FeatureBagConfigBuilderTest.java new file mode 100644 index 000000000..44e0fe654 --- /dev/null +++ b/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/consumer/FeatureBagConfigBuilderTest.java @@ -0,0 +1,21 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.consumer; + +import com.linkedin.feathr.core.configbuilder.typesafe.AbstractConfigBuilderTest; +import org.testng.annotations.Test; + +import static com.linkedin.feathr.core.configbuilder.typesafe.consumer.JoinFixture.*; + + +public class FeatureBagConfigBuilderTest extends AbstractConfigBuilderTest { + + + @Test(description = "Tests build of FeatureBag config objects") + public void testFeatureBagConfigBuilder() { + testConfigBuilder(featureBagConfigStr, FeatureBagConfigBuilder::build, expFeatureBagConfigObj); + } + + @Test(description = "Tests build of FeatureBag config objects with special chars") + public void testFeatureBagConfigBuilderWithSpecialChars() { + testConfigBuilder(featureBagConfigStrWithSpecialChars, FeatureBagConfigBuilder::build, expFeatureBagConfigObjWithSpecialChars); + } +} diff --git a/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/consumer/JoinConfigBuilderTest.java b/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/consumer/JoinConfigBuilderTest.java new file mode 100644 index 000000000..b11811534 --- /dev/null +++ b/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/consumer/JoinConfigBuilderTest.java @@ -0,0 +1,45 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.consumer; + +import com.linkedin.feathr.core.config.consumer.JoinConfig; +import com.linkedin.feathr.core.configbuilder.typesafe.AbstractConfigBuilderTest; +import com.typesafe.config.Config; +import com.typesafe.config.ConfigFactory; +import org.testng.annotations.Test; + +import static com.linkedin.feathr.core.configbuilder.typesafe.consumer.JoinFixture.*; +import static org.testng.Assert.*; + + +public class JoinConfigBuilderTest extends AbstractConfigBuilderTest { + + @Test(description = "Tests build of JoinConfig config object with single feature bag but no settings") + public void testWithNoSettings() { + testJoinConfigBuilder(joinConfigStr1, expJoinConfigObj1); + } + + @Test(description = "Tests build of JoinConfig config object with single feature bag which has special characters but no settings") + public void testWithNoSettingsAndWithSpecialChars() { + testJoinConfigBuilder(joinConfigStr1WithSpecialChars, expJoinConfigObj1WithSpecialChars); + } + + @Test(description = "Tests build of JoinConfig config object with single feature bag but empty settings") + public void testWithEmptySettings() { + testJoinConfigBuilder(joinConfigStr2, expJoinConfigObj2); + } + + @Test(description = "Tests build of JoinConfig config object with single feature bag and time-window settings") + public void testWithTimeWindowSettings() { + testJoinConfigBuilder(joinConfigStr3, expJoinConfigObj3); + } + + @Test(description = "Tests build of JoinConfig config object with multiple feature bags") + public void testWithMultiFeatureBags() { + testJoinConfigBuilder(joinConfigStr4, expJoinConfigObj4); + } + + private void testJoinConfigBuilder(String configStr, JoinConfig expJoinConfigObj) { + Config fullConfig = ConfigFactory.parseString(configStr); + JoinConfig obsJoinConfigObj = JoinConfigBuilder.build(fullConfig); + assertEquals(obsJoinConfigObj, expJoinConfigObj); + } +} diff --git a/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/consumer/JoinFixture.java b/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/consumer/JoinFixture.java new file mode 100644 index 000000000..9a1b7bc85 --- /dev/null +++ b/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/consumer/JoinFixture.java @@ -0,0 +1,379 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.consumer; + +import com.linkedin.feathr.core.config.consumer.AbsoluteTimeRangeConfig; +import com.linkedin.feathr.core.config.consumer.DateTimeRange; +import com.linkedin.feathr.core.config.consumer.FeatureBagConfig; +import com.linkedin.feathr.core.config.consumer.JoinConfig; +import com.linkedin.feathr.core.config.consumer.JoinTimeSettingsConfig; +import com.linkedin.feathr.core.config.consumer.KeyedFeatures; +import com.linkedin.feathr.core.config.consumer.ObservationDataTimeSettingsConfig; +import com.linkedin.feathr.core.config.consumer.RelativeTimeRangeConfig; +import com.linkedin.feathr.core.config.consumer.SettingsConfig; +import com.linkedin.feathr.core.config.consumer.TimestampColumnConfig; +import java.time.Duration; +import java.time.LocalDateTime; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + + +public class JoinFixture { + static final String emptySettingsConfigStr = "settings: {\n}"; + + static final SettingsConfig expEmptySettingsConfigObj = new SettingsConfig(null, null); + + public static final String settingsWithAbsoluteTimeRange = String.join("\n", + "settings: {", + " observationDataTimeSettings: {", + " absoluteTimeRange: {", + " startTime: \"2018/05/01/00/00/00\"", + " endTime:\"2018/05/05/23/59/59\"", + " timeFormat: \"yyyy/MM/dd/HH/mm/ss\"", + " }", + " }", + " joinTimeSettings: {", + " timestampColumn: {", + " def: timestamp", + " format: \"yyyy/MM/dd/HH/mm/ss\"", + " }", + " simulateTimeDelay: 1d", + " }", + "}"); + + static final SettingsConfig expSettingsWithAbsoluteTimeRange; + static { + String timestampField = "timestamp"; + String timestampFormat = "yyyy/MM/dd/HH/mm/ss"; + + String startTime = "2018/05/01/00/00/00"; + String endTime = "2018/05/05/23/59/59"; + Duration simulateTimeDelay = Duration.ofDays(1); + AbsoluteTimeRangeConfig absoluteTimeRangeConfig = new AbsoluteTimeRangeConfig(startTime, endTime, timestampFormat); + ObservationDataTimeSettingsConfig observationDataTimeSettingsConfig = new ObservationDataTimeSettingsConfig( + absoluteTimeRangeConfig, null); + TimestampColumnConfig timestampColumnConfig = new TimestampColumnConfig(timestampField, timestampFormat); + JoinTimeSettingsConfig joinTimeSettingsConfig = new JoinTimeSettingsConfig(timestampColumnConfig, simulateTimeDelay, null); + + expSettingsWithAbsoluteTimeRange = new SettingsConfig(observationDataTimeSettingsConfig, joinTimeSettingsConfig); + } + + public static final String settingsWithLatestFeatureData = String.join("\n", + "settings: {", + " joinTimeSettings: {", + " useLatestFeatureData: true", + " }", + "}"); + + static final SettingsConfig expSettingsWithLatestFeatureData; + static { + JoinTimeSettingsConfig joinTimeSettingsConfig = new JoinTimeSettingsConfig( null, null,true); + + expSettingsWithLatestFeatureData = new SettingsConfig(null, joinTimeSettingsConfig); + } + + public static final String settingsWithRelativeTimeRange = String.join("\n", + "settings: {", + " observationDataTimeSettings: {", + " relativeTimeRange: {", + " window: 1d", + " offset: 1d", + " }", + " }", + " joinTimeSettings: {", + " useLatestFeatureData: true", + " }", + "}"); + + static final SettingsConfig expSettingsWithRelativeTimeRange; + static { + Duration window = Duration.ofDays(1); + Duration offset = Duration.ofDays(1); + Duration simulateTimeDelay = Duration.ofDays(1); + RelativeTimeRangeConfig relativeTimeRangeConfig = new RelativeTimeRangeConfig(window, offset); + ObservationDataTimeSettingsConfig observationDataTimeSettingsConfig = new ObservationDataTimeSettingsConfig( + null, relativeTimeRangeConfig); + JoinTimeSettingsConfig joinTimeSettingsConfig = new JoinTimeSettingsConfig(null, null, true); + + expSettingsWithRelativeTimeRange = new SettingsConfig(observationDataTimeSettingsConfig, joinTimeSettingsConfig); + } + + public static final String settingsWithOnlyWindow = String.join("\n", + "settings: {", + " observationDataTimeSettings: {", + " relativeTimeRange: {", + " window: 1d", + " }", + " }", + " joinTimeSettings: {", + " timestampColumn: {", + " def: timestamp", + " format: yyyy/MM/dd", + " }", + " simulateTimeDelay: 1d", + " }", + "}"); + + static final SettingsConfig expSettingsWithOnlyWindow; + static { + Duration window = Duration.ofDays(1); + Duration simulateTimeDelay = Duration.ofDays(1); + String timestampField = "timestamp"; + String timestampFormat = "yyyy/MM/dd"; + TimestampColumnConfig timestampColumnConfig = new TimestampColumnConfig(timestampField, timestampFormat); + RelativeTimeRangeConfig relativeTimeRangeConfig = new RelativeTimeRangeConfig(window, null); + ObservationDataTimeSettingsConfig observationDataTimeSettingsConfig = new ObservationDataTimeSettingsConfig( + null, relativeTimeRangeConfig); + JoinTimeSettingsConfig joinTimeSettingsConfig = new JoinTimeSettingsConfig(timestampColumnConfig, simulateTimeDelay, null); + + expSettingsWithOnlyWindow = new SettingsConfig(observationDataTimeSettingsConfig, joinTimeSettingsConfig); + } + public static final String invalidWithOnlyStartTime = String.join("\n", + "settings: {", + " observationDataTimeSettings: {", + " absoluteTimeRange: {", + " startTime: 2020/09/20", + " }", + " }", + "}"); + + public static final String invalidWithNoTimestampFormat = String.join("\n", + "settings: {", + " joinTimeSettings: {", + " timestampColumn: {", + " def: timestamp", + " }", + " }", + "}"); + + public static final String invalidWithBothAbsoluteTimeRangeAndRelativeTimeRange = String.join("\n", + "settings: {", + " observationDataTimeSettings: {", + " absoluteTimeRange: {", + " startTime: 2020/09/20", + " endTime: 2020/09/25", + " timeFormat: yyyy/MM/dd", + " }", + " relativeTimeRange: {", + " window: 1d", + " offset: 1d", + " }", + " }", + "}"); + + public static final String invalidWithUseLatestFeatureDataAndTimestampCol = String.join("\n", + "settings: {", + " joinTimeSettings: {", + " timestampColumn: {", + " def: timestamp", + " format: \"yyyy/MM/dd/HH/mm/ss\"", + " }", + " useLatestFeatureData: true", + " }", + "}"); + + public static final String invalidWithUseLatestFeatureDataAndTimeDelay = String.join("\n", + "settings: {", + " joinTimeSettings: {", + " simulateTimeDelay: 1d", + " useLatestFeatureData: true", + " }", + "}"); + + public static final String settingsWithTimeWindowConfigAndNegativeTimeDelay = String.join("\n", + "settings: {", + " joinTimeSettings: {", + " timestampColumn: {", + " def: timestamp", + " format: yyyy/MM/dd", + " }", + " simulateTimeDelay: -1d", + " }", + "}"); + + public static final String invalidSettingsWithTimeWindowConfigNegativeTimeDelay = String.join("\n", + "settings: {", + " joinTimeSettings: {", + " timestampColumn: {", + " def: timestamp", + " format: yyyy/MM/dd", + " }", + " simulateTimeDelay: ---1d", + " }", + "}"); + + + static final String featureBagConfigStr = String.join("\n", + "features: [", + " {", + " key: \"targetId\"", + " featureList: [\"waterloo_job_location\", ", + "\"waterloo_job_jobTitle\", \"waterloo_job_jobSeniority\"]", + " },", + " {", + " key: \"sourceId\"", + " featureList: [\"TimeBasedFeatureA\"]", + " startDate: \"20170522\"", + " endDate: \"20170522\"", + " },", + " {", + " key: \"sourceId\"", + " featureList: [\"jfu_resolvedPreference_seniority\", ", + "\"jfu_resolvedPreference_country\", \"waterloo_member_currentTitle\"]", + " },", + " {", + " key: [\"sourceId\",\"targetId\"]", + " featureList: [\"memberJobFeature1\",\"memberJobFeature2\"]", + " },", + " {", + " key: [x],", + " featureList: [\"sumPageView1d\", \"waterloo-member-title\"]", + " }", + " {", + " key: [x],", + " featureList: [\"pageId\", \"memberJobFeature6\"]", + " overrideTimeDelay: 3d", + " }", + "]"); + + static final String featureBagConfigStrWithSpecialChars = String.join("\n", + "\"features.dot:colon\": [", + " {", + " key: \"targetId\"", + " featureList: [\"waterloo:job.location\", ", + "\"waterloo_job_jobTitle\", \"waterloo_job_jobSeniority\"]", + " },", + " {", + " key: \"sourceId\"", + " featureList: [\"TimeBased.Feature:A\"]", + " startDate: \"20170522\"", + " endDate: \"20170522\"", + " },", + "]"); + + + static FeatureBagConfig expFeatureBagConfigObj; + static final Map expFeatureBagConfigs; + static { + List key1 = Collections.singletonList("targetId"); + List features1 = + Arrays.asList("waterloo_job_location", "waterloo_job_jobTitle", "waterloo_job_jobSeniority"); + KeyedFeatures keyedFeature1 = new KeyedFeatures(key1, features1, null, null); + + List key2 = Collections.singletonList("sourceId"); + List features2 = Collections.singletonList("TimeBasedFeatureA"); + LocalDateTime start = LocalDateTime.of(2017, 5, 22, 0, 0); + LocalDateTime end = LocalDateTime.of(2017, 5, 22, 0, 0); + DateTimeRange dates = new DateTimeRange(start, end); + KeyedFeatures keyedFeature2 = new KeyedFeatures(key2, features2, dates, null); + + List key3 = Collections.singletonList("sourceId"); + List features3 = Arrays.asList("jfu_resolvedPreference_seniority", + "jfu_resolvedPreference_country", "waterloo_member_currentTitle"); + KeyedFeatures keyedFeature3 = new KeyedFeatures(key3, features3, null, null); + + List key4 = Arrays.asList("sourceId","targetId"); + List features4 = Arrays.asList("memberJobFeature1","memberJobFeature2"); + KeyedFeatures keyedFeature4 = new KeyedFeatures(key4, features4, null, null); + + List key = Collections.singletonList("x"); + List features = Arrays.asList("sumPageView1d", "waterloo-member-title"); + KeyedFeatures keyedFeatures5 = new KeyedFeatures(key, features, null, null); + + List key5 = Collections.singletonList("x"); + List features5 = Arrays.asList("pageId", "memberJobFeature6"); + Duration overrideTimeDelay = Duration.ofDays(3); + KeyedFeatures keyedFeatures6 = new KeyedFeatures(key5, features5, null, overrideTimeDelay); + + expFeatureBagConfigObj = + new FeatureBagConfig(Arrays.asList(keyedFeature1, keyedFeature2, keyedFeature3, keyedFeature4, keyedFeatures5, keyedFeatures6)); + + expFeatureBagConfigs = new HashMap<>(); + expFeatureBagConfigs.put("features", expFeatureBagConfigObj); + } + + static FeatureBagConfig expFeatureBagConfigObjWithSpecialChars; + static final Map expFeatureBagConfigsWithSpecialChars; + static { + List key1 = Collections.singletonList("targetId"); + List features1 = + Arrays.asList("waterloo:job.location", "waterloo_job_jobTitle", "waterloo_job_jobSeniority"); + KeyedFeatures keyedFeature1 = new KeyedFeatures(key1, features1, null, null); + + List key2 = Collections.singletonList("sourceId"); + List features2 = Collections.singletonList("TimeBased.Feature:A"); + LocalDateTime start = LocalDateTime.of(2017, 5, 22, 0, 0); + LocalDateTime end = LocalDateTime.of(2017, 5, 22, 0, 0); + DateTimeRange dates = new DateTimeRange(start, end); + KeyedFeatures keyedFeature2 = new KeyedFeatures(key2, features2, dates, null); + + expFeatureBagConfigObjWithSpecialChars = + new FeatureBagConfig(Arrays.asList(keyedFeature1, keyedFeature2)); + + expFeatureBagConfigsWithSpecialChars = new HashMap<>(); + expFeatureBagConfigsWithSpecialChars.put("features.dot:colon", expFeatureBagConfigObjWithSpecialChars); + } + + static final String joinConfigStr1 = featureBagConfigStr; + + static final String joinConfigStr1WithSpecialChars = featureBagConfigStrWithSpecialChars; + + public static final JoinConfig expJoinConfigObj1 = new JoinConfig(null, expFeatureBagConfigs); + + public static final JoinConfig expJoinConfigObj1WithSpecialChars = new JoinConfig(null, expFeatureBagConfigsWithSpecialChars); + + static final String joinConfigStr2 = String.join("\n", emptySettingsConfigStr, featureBagConfigStr); + + static final JoinConfig expJoinConfigObj2 = + new JoinConfig(expEmptySettingsConfigObj, expFeatureBagConfigs); + + static final String joinConfigStr3 = String.join("\n", settingsWithAbsoluteTimeRange, featureBagConfigStr); + + static final JoinConfig expJoinConfigObj3 = + new JoinConfig(expSettingsWithAbsoluteTimeRange, expFeatureBagConfigs); + + static final String multiFeatureBagsStr = String.join("\n", + "featuresGroupA: [", + " {", + " key: \"viewerId\"", + " featureList: [", + " waterloo_member_currentCompany,", + " waterloo_job_jobTitle,", + " ]", + " }", + "]", + "featuresGroupB: [", + " {", + " key: \"viewerId\"", + " featureList: [", + " waterloo_member_location,", + " waterloo_job_jobSeniority", + " ]", + " }", + "]"); + + static final Map expMultiFeatureBagConfigs; + static { + String featureBag1Name = "featuresGroupA"; + List key1 = Collections.singletonList("viewerId"); + List featuresList1 = Arrays.asList("waterloo_member_currentCompany", "waterloo_job_jobTitle"); + KeyedFeatures keyedFeatures1 = new KeyedFeatures(key1, featuresList1, null, null); + FeatureBagConfig featureBag1Config = new FeatureBagConfig(Collections.singletonList(keyedFeatures1)); + + String featureBag2Name = "featuresGroupB"; + List key2 = Collections.singletonList("viewerId"); + List featuresList2 = Arrays.asList("waterloo_member_location", "waterloo_job_jobSeniority"); + KeyedFeatures keyedFeatures2 = new KeyedFeatures(key2, featuresList2, null, null); + FeatureBagConfig featureBag2Config = new FeatureBagConfig(Collections.singletonList(keyedFeatures2)); + + expMultiFeatureBagConfigs = new HashMap<>(); + expMultiFeatureBagConfigs.put(featureBag1Name, featureBag1Config); + expMultiFeatureBagConfigs.put(featureBag2Name, featureBag2Config); + } + + static final String joinConfigStr4 = multiFeatureBagsStr; + + static final JoinConfig expJoinConfigObj4 = + new JoinConfig(null, expMultiFeatureBagConfigs); +} diff --git a/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/consumer/SettingsConfigBuilderTest.java b/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/consumer/SettingsConfigBuilderTest.java new file mode 100644 index 000000000..6bd0c8174 --- /dev/null +++ b/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/consumer/SettingsConfigBuilderTest.java @@ -0,0 +1,68 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.consumer; + +import com.linkedin.feathr.core.configbuilder.ConfigBuilderException; +import com.linkedin.feathr.core.configbuilder.typesafe.AbstractConfigBuilderTest; +import org.testng.annotations.Test; + +import static com.linkedin.feathr.core.configbuilder.typesafe.consumer.JoinFixture.*; + + +public class SettingsConfigBuilderTest extends AbstractConfigBuilderTest { + + @Test(description = "Tests an empty settings config") + public void testEmptySettings() { + testConfigBuilder(emptySettingsConfigStr, SettingsConfigBuilder::build, expEmptySettingsConfigObj); + } + + @Test(description = "Tests a settings config with absoluteTimeRange set, normal case") + public void testSettingsWithAbsoluteTimeRange() { + testConfigBuilder(settingsWithAbsoluteTimeRange, + SettingsConfigBuilder::build, expSettingsWithAbsoluteTimeRange); + } + + @Test(description = "Tests a settings config with only useLatestFeatureData set to true") + public void testSettingsWithOnlyLatestFeatureData() { + testConfigBuilder(settingsWithLatestFeatureData, + SettingsConfigBuilder::build, expSettingsWithLatestFeatureData); + } + + @Test(description = "Tests a settings config with relativeTimeRange set") + public void testSettingsWithRelativeTimeRange() { + testConfigBuilder(settingsWithRelativeTimeRange, + SettingsConfigBuilder::build, expSettingsWithRelativeTimeRange); + } + + @Test(description = "Tests a settings config with only window field set") + public void testSettingsWithOnlyWindow() { + testConfigBuilder(settingsWithOnlyWindow, + SettingsConfigBuilder::build, expSettingsWithOnlyWindow); + } + + @Test(description = "Tests a settings config with only start time", + expectedExceptions = ConfigBuilderException.class) + public void testSettingsWithOnlyStartTime() { + testConfigBuilder(invalidWithOnlyStartTime, + SettingsConfigBuilder::build, expEmptySettingsConfigObj); + } + + @Test(description = "Tests a settings config with both absolute time range and relative time range", + expectedExceptions = ConfigBuilderException.class) + public void testSettingsWithAbsTimeRangeAndRelTimeRange() { + testConfigBuilder(invalidWithBothAbsoluteTimeRangeAndRelativeTimeRange, + SettingsConfigBuilder::build, expEmptySettingsConfigObj); + } + + @Test(description = "Tests a settings config with both use latest feature data set to true and timestamp column field defined", + expectedExceptions = ConfigBuilderException.class) + public void testSettingsWithUseLatestFeatureDataAndTimestampCol() { + testConfigBuilder(invalidWithUseLatestFeatureDataAndTimestampCol, + SettingsConfigBuilder::build, expEmptySettingsConfigObj); + } + + @Test(description = "Tests a settings config with both use latest feature data set to true and time delay field defined", + expectedExceptions = ConfigBuilderException.class) + public void testSettingsWithUseLatestFeatureDataAndTimeDelay() { + testConfigBuilder(invalidWithUseLatestFeatureDataAndTimeDelay, + SettingsConfigBuilder::build, expEmptySettingsConfigObj); + } +} diff --git a/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/generation/FeatureGenConfigBuilderTest.java b/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/generation/FeatureGenConfigBuilderTest.java new file mode 100644 index 000000000..8c5beed53 --- /dev/null +++ b/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/generation/FeatureGenConfigBuilderTest.java @@ -0,0 +1,37 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.generation; + +import com.linkedin.feathr.core.config.generation.FeatureGenConfig; +import com.typesafe.config.Config; +import com.typesafe.config.ConfigFactory; +import org.testng.annotations.Test; + +import static org.testng.Assert.*; + + +/** + * test of Frame feature generation config object + */ +public class FeatureGenConfigBuilderTest { + + @Test(description = "Tests building of generation config for the case with all supported fields") + public void testWithFullFieldsCase() { + testFeatureGenConfigBuilder(GenerationFixture.generationConfigStr1, GenerationFixture.expGenerationConfigObj1); + } + + @Test(description = "Tests building of generation config for cases with minimal supported fields") + public void testWithDefaultFieldsCase() { + testFeatureGenConfigBuilder(GenerationFixture.generationConfigStr2, GenerationFixture.expGenerationConfigObj2); + } + + @Test(description = "Tests building of nearline generation config for all possible cases") + public void testWithNealineFieldsCase() { + testFeatureGenConfigBuilder( + GenerationFixture.nearlineGenerationConfigStr, GenerationFixture.nearlineGenerationConfigObj); + } + + private void testFeatureGenConfigBuilder(String configStr, FeatureGenConfig expFeatureGenConfigObj) { + Config withDefaultConfig = ConfigFactory.parseString(configStr); + FeatureGenConfig generationConfigObj = FeatureGenConfigBuilder.build(withDefaultConfig); + assertEquals(generationConfigObj, expFeatureGenConfigObj); + } +} diff --git a/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/generation/GenerationFixture.java b/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/generation/GenerationFixture.java new file mode 100644 index 000000000..b08eae4c7 --- /dev/null +++ b/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/generation/GenerationFixture.java @@ -0,0 +1,190 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.generation; + +import com.linkedin.feathr.core.config.common.DateTimeConfig; +import com.linkedin.feathr.core.config.common.OutputFormat; +import com.linkedin.feathr.core.config.generation.FeatureGenConfig; +import com.linkedin.feathr.core.config.generation.NearlineOperationalConfig; +import com.linkedin.feathr.core.config.generation.OfflineOperationalConfig; +import com.linkedin.feathr.core.config.generation.OperationalConfig; +import com.linkedin.feathr.core.config.generation.OutputProcessorConfig; +import com.typesafe.config.ConfigFactory; +import java.time.Duration; +import java.time.temporal.ChronoUnit; +import java.util.Arrays; +import java.util.List; +import java.util.TimeZone; + + +public class GenerationFixture { + + static final String generationConfigStr1 = + String.join("// operational section\n", + "operational: {\n", + " name: XAffinity\n", + " endTime: \"2018-05-08\" // specify a date/time, or ‘NOW’\n", + " endTimeFormat: \"yyyy-MM-dd\"\n", + " resolution: DAILY // DAILY/HOURLY\n", + " timeDelay: 2 days // default value is 1, which means generate yesterday’ data\n", + " retention: 3 days // only keep one snapshot for frame access and incremental aggregation\n", + " offset: 4 days \n", + " enableIncremental: true\n", + " timeZone: \"America/Los_Angeles\" \n", + " output: [ // accept a list of output processors\n", + " { name: HDFS \n", + " outputFormat: RAW_DATA // output format can be customized when user changed the feature \n", + " // schema in the processor, or just keep the input format to pass to next\n", + " // processor \n", + " params: { \n", + " path: \"/jobs/frame/df\" // processor can take arbitrary parameters\n", + " } \n", + " }\n", + " {\n", + " name: VENICE \n", + " outputFormat: NAME_TERM_VALUE \n", + " params: { \n", + " path: \"/jobs/frame/NAME_TERM_VALUE/daily\" // this will be extended according to time set in each\n", + " // operational section, e.g, /jobs/frame/daily/2019/02/02”\n", + " } \n", + " } \n", + " ]\n", + "}\n ", + "// features section, specify list of features to generate\n", + "features: [F1, F2]"); + + static final FeatureGenConfig expGenerationConfigObj1; + static { + Duration offset = Duration.ofDays(4); + TimeZone timeZone = TimeZone.getTimeZone("America/Los_Angeles"); + DateTimeConfig timeSettings = new DateTimeConfig("2018-05-08", "yyyy-MM-dd", + ChronoUnit.DAYS, 0, offset, timeZone) ; + OutputProcessorConfig hdfsProcessor = new OutputProcessorConfig("HDFS", OutputFormat.RAW_DATA, + ConfigFactory.parseString("{path:/jobs/frame/df}")); + OutputProcessorConfig veniceProcessor = new OutputProcessorConfig("VENICE", + OutputFormat.NAME_TERM_VALUE, ConfigFactory.parseString("{path: /jobs/frame/NAME_TERM_VALUE/daily}")); + + List outputProcessorConfigList = Arrays.asList(hdfsProcessor, veniceProcessor); + Duration retention = Duration.ofDays(3); + String name = "XAffinity"; + Duration simulateTImeDelay = Duration.ofDays(2); + Boolean enableIncremental = Boolean.TRUE; + OperationalConfig operationalConfig = + new OfflineOperationalConfig(outputProcessorConfigList, name, timeSettings, retention, simulateTImeDelay, enableIncremental); + List features = Arrays.asList("F1", "F2"); + expGenerationConfigObj1 = new FeatureGenConfig(operationalConfig, features); + } + + static final String generationConfigStr2 = + String.join("// operational section\n", + "operational: {\n", + " name: XAffinity\n", + " endTime: \"2018-05-08 17:00:00\" // specify a date/time, or ‘NOW’\n", + " endTimeFormat: \"yyyy-MM-dd hh:mm:ss\"\n", + " resolution: HOURLY // DAILY/HOURLY\n", + " enableIncremental: true\n", + " output: [ // accept a list of output processors\n", + " { \n", + " name: HDFS \n", + " outputFormat: NAME_TERM_VALUE // output format can be customized when user changed the feature \n", + " // schema in the processor, or just keep the input format to pass to next\n", + " // processor \n", + " params: { \n", + " path: \"/jobs/frame/df\" // processor can take arbitrary parameters\n", + " } \n", + " }\n", + " ]\n", + "}\n ", + "// features section, specify list of features to generate\n", + "features: [F1, F2]"); + + static final FeatureGenConfig expGenerationConfigObj2; + static { + Duration offset = Duration.ofHours(0); + TimeZone timeZone = TimeZone.getTimeZone("America/Los_Angeles"); + DateTimeConfig timeSettings = new DateTimeConfig("2018-05-08 17:00:00", "yyyy-MM-dd hh:mm:ss", + ChronoUnit.HOURS, 0, offset, timeZone); + OutputProcessorConfig hdfsProcessor = new OutputProcessorConfig("HDFS", OutputFormat.NAME_TERM_VALUE, + ConfigFactory.parseString("{path:/jobs/frame/df}")); + List + outputProcessorConfigList = Arrays.asList(hdfsProcessor); + Duration retention = Duration.ofHours(1); + String name = "XAffinity"; + Duration simulateTImeDelay = Duration.ofHours(0); + Boolean enableIncremental = Boolean.TRUE; + OperationalConfig operationalConfig = + new OfflineOperationalConfig(outputProcessorConfigList, name, timeSettings, retention, simulateTImeDelay, enableIncremental); + List features = Arrays.asList("F1", "F2"); + expGenerationConfigObj2 = new FeatureGenConfig(operationalConfig, features); + } + + static final String nearlineGenerationConfigStr = + String.join("// operational section\n", + "operational: {\n", + " name: XAffinity\n", + " output: [ // accept a list of output processors\n", + " { \n", + " name: KAFKA \n", + " outputFormat: NAME_TERM_VALUE // output format can be customized when user changed the feature \n", + " // schema in the processor, or just keep the input format to pass to next\n", + " // processor \n", + " params: { \n", + " type: KAFKA", + " topic: kafkaTopic", + " path: \"/jobs/frame/df\" // processor can take arbitrary parameters\n", + " } \n", + " }\n", + " { \n", + " name: VENICE \n", + " outputFormat: NAME_TERM_VALUE // output format can be customized when user changed the feature \n", + " // schema in the processor, or just keep the input format to pass to next\n", + " // processor \n", + " params: { \n", + " type: VENICE", + " store: veniceStore", + " } \n", + " }\n", + " { \n", + " name: ESPRESSO \n", + " outputFormat: NAME_TERM_VALUE // output format can be customized when user changed the feature \n", + " // schema in the processor, or just keep the input format to pass to next\n", + " // processor \n", + " params: { \n", + " type: ESPRESSO", + " store: espressoStore", + " table: tableName", + " d2uri: d2uri", + " } \n", + " }\n", + " { \n", + " name: LOG \n", + " outputFormat: NAME_TERM_VALUE // output format can be customized when user changed the feature \n", + " // schema in the processor, or just keep the input format to pass to next\n", + " // processor \n", + " params: { \n", + " type: CONSOLE", + " } \n", + " }\n", + " ]\n", + " env: NEARLINE\n", + "}\n ", + "// features section, specify list of features to generate\n", + "features: [F1, F2]"); + + static final FeatureGenConfig nearlineGenerationConfigObj; + static { + OutputProcessorConfig kafkaProcessor = new OutputProcessorConfig("KAFKA", OutputFormat.NAME_TERM_VALUE, + ConfigFactory.parseString("{type: KAFKA\n topic: kafkaTopic\n path:/jobs/frame/df}")); + OutputProcessorConfig veniceProcessor = new OutputProcessorConfig("VENICE", OutputFormat.NAME_TERM_VALUE, + ConfigFactory.parseString("{type: VENICE\n store: veniceStore\n}")); + OutputProcessorConfig espressoProcessor = new OutputProcessorConfig("ESPRESSO", OutputFormat.NAME_TERM_VALUE, + ConfigFactory.parseString("{type: ESPRESSO\n store: espressoStore\n table: tableName\n d2uri: d2uri\n}")); + OutputProcessorConfig logProcessor = new OutputProcessorConfig("LOG", OutputFormat.NAME_TERM_VALUE, + ConfigFactory.parseString("{type: CONSOLE\n}")); + List + outputProcessorConfigList = Arrays.asList(kafkaProcessor, veniceProcessor, espressoProcessor, logProcessor); + String name = "XAffinity"; + OperationalConfig operationalConfig = + new NearlineOperationalConfig(outputProcessorConfigList, name); + List features = Arrays.asList("F1", "F2"); + nearlineGenerationConfigObj = new FeatureGenConfig(operationalConfig, features); + } +} diff --git a/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/FeatureDefConfigBuilderTest.java b/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/FeatureDefConfigBuilderTest.java new file mode 100644 index 000000000..2c5263f78 --- /dev/null +++ b/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/FeatureDefConfigBuilderTest.java @@ -0,0 +1,37 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.producer; + +import com.linkedin.feathr.core.config.producer.FeatureDefConfig; +import com.typesafe.config.Config; +import com.typesafe.config.ConfigFactory; +import org.testng.annotations.Test; + +import static com.linkedin.feathr.core.configbuilder.typesafe.producer.FeatureDefFixture.*; +import static org.testng.Assert.*; + + +public class FeatureDefConfigBuilderTest { + + @Test(description = "Tests building of FeatureDef config object") + public void test() { + Config fullConfig = ConfigFactory.parseString(featureDefConfigStr1); + FeatureDefConfig obsFeatureDefConfigObj = FeatureDefConfigBuilder.build(fullConfig); + + assertEquals(obsFeatureDefConfigObj, expFeatureDefConfigObj1); + } + + @Test(description = "Tests building of FeatureDef config object with only AnchorConfig") + public void testWithOnlyAnchorConfig() { + Config fullConfig = ConfigFactory.parseString(featureDefConfigStr2); + FeatureDefConfig obsFeatureDefConfigObj = FeatureDefConfigBuilder.build(fullConfig); + + assertEquals(obsFeatureDefConfigObj, expFeatureDefConfigObj2); + } + + @Test(description = "Tests building of FeatureDef config object with feature and dimension sections") + public void testWithFeatureAndDimensionSections() { + Config fullConfig = ConfigFactory.parseString(featureDefConfigStr3); + FeatureDefConfig obsFeatureDefConfigObj = FeatureDefConfigBuilder.build(fullConfig); + + assertEquals(obsFeatureDefConfigObj, expFeatureDefConfigObj3); + } +} diff --git a/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/FeatureDefFixture.java b/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/FeatureDefFixture.java new file mode 100644 index 000000000..db1217cc9 --- /dev/null +++ b/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/FeatureDefFixture.java @@ -0,0 +1,233 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.producer; + +import com.google.common.collect.ImmutableMap; +import com.linkedin.data.DataMap; +import com.linkedin.data.schema.PathSpec; +import com.linkedin.feathr.core.config.producer.FeatureDefConfig; +import com.linkedin.feathr.core.config.producer.anchors.AnchorConfig; +import com.linkedin.feathr.core.config.producer.anchors.AnchorConfigWithExtractor; +import com.linkedin.feathr.core.config.producer.anchors.AnchorsConfig; +import com.linkedin.feathr.core.config.producer.anchors.ExtractorBasedFeatureConfig; +import com.linkedin.feathr.core.config.producer.anchors.FeatureConfig; +import com.linkedin.feathr.core.config.producer.derivations.DerivationConfig; +import com.linkedin.feathr.core.config.producer.derivations.DerivationsConfig; +import com.linkedin.feathr.core.config.producer.derivations.SimpleDerivationConfig; +import com.linkedin.feathr.core.config.producer.sources.RestliConfig; +import com.linkedin.feathr.core.config.producer.sources.SourceConfig; +import com.linkedin.feathr.core.config.producer.sources.SourcesConfig; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.function.Function; + + +public class FeatureDefFixture { + /* + * The following config strings have been extracted and culled from feature-prod.conf in frame-feature-careers MP. + * https://jarvis.corp.linkedin.com/codesearch/result/?name=feature-prod.conf&path=frame-feature-careers%2Fframe-feature-careers-online%2Fsrc%2Fmain%2Fresources%2Fconfig%2Fonline%2Fprod&reponame=multiproducts%2Fframe-feature-careers + */ + static final String sourcesConfigStr = String.join("\n", + "sources: {", + " JobsTargetingSegments: {", + " type: RESTLI", + " restResourceName: jobsTargetingSegments", + " restEntityType: jobPosting", + " pathSpec: targetingFacetsSet", + " },", + " Profile: {", + " type: RESTLI", + " restResourceName: profiles", + " keyExpr: \"toComplexResourceKey({\\\"id\\\": key[0]},{:})\"", + " restReqParams: {", + " viewerId: {mvel: \"key[0]\"}", + " }", + " pathSpec: positions", + " },", + " MemberPreferenceData: {", + " type: RESTLI", + " restResourceName: jobSeekers", + " restEntityType: member", + " }", + "}"); + + + static final SourcesConfig expSourcesConfigObj; + static { + Function toKeyExpr = entityType -> "toUrn(\"" + entityType + "\", key[0])"; + + String resourceName1 = "jobsTargetingSegments"; + String keyExpr1 = toKeyExpr.apply("jobPosting"); + Map reqParams1 = null; + PathSpec pathSpec1 = new PathSpec("targetingFacetsSet"); + RestliConfig expSource1ConfigObj = new RestliConfig("JobsTargetingSegments", resourceName1, keyExpr1, reqParams1, pathSpec1); + + String resourceName2 = "profiles"; + String keyExpr2 = "toComplexResourceKey({\"id\": key[0]},{:})"; + Map paramsMap = new HashMap<>(); + paramsMap.put("viewerId", new DataMap(ImmutableMap.of(RestliConfig.MVEL_KEY, "key[0]"))); + Map reqParams2 = paramsMap; + PathSpec pathSpec2 = new PathSpec("positions"); + RestliConfig expSource2ConfigObj = new RestliConfig("Profile", resourceName2, keyExpr2, reqParams2, pathSpec2); + + String resourceName3 = "jobSeekers"; + String keyExpr3 = toKeyExpr.apply("member"); + Map reqParams3 = null; + PathSpec pathSpec3 = null; + RestliConfig expSource3ConfigObj = new RestliConfig("MemberPreferenceData", resourceName3, keyExpr3, reqParams3, pathSpec3); + + Map sources = new HashMap<>(); + sources.put("JobsTargetingSegments", expSource1ConfigObj); + sources.put("Profile", expSource2ConfigObj); + sources.put("MemberPreferenceData", expSource3ConfigObj); + + expSourcesConfigObj = new SourcesConfig(sources); + } + + static final String anchorsConfigStr = String.join("\n", + "anchors: {", + " jobs-targeting-term-vectors: {", + " source: JobsTargetingSegments", + " extractor: com.linkedin.jobs.relevance.feathr.online.extractor.JobsTargetingSegmentTermVectorExtractor", + " keyAlias: [y] ", + " features: [", + " careers_targeting_companies,", + " careers_targeting_functions", + " ]", + " },", + " member-profile-yoe: {", + " source: Profile", + " extractor: com.linkedin.jobs.relevance.feathr.online.extractor.ISBYoeTermVectorExtractor", + " features: [", + " careers_member_positionsYoE", + " ]", + " },", + " jfu-member-preferences: {", + " source: MemberPreferenceData", + " extractor: com.linkedin.jobs.relevance.feathr.online.extractor.MemberPreferenceExtractor", + " features: [", + " careers_preference_companySize,", + " careers_preference_industry,", + " careers_preference_location", + " ]", + " }", + "}"); + + static final AnchorsConfig expAnchorsConfigObj; + static { + + String source1 = "JobsTargetingSegments"; + String extractor1 = "com.linkedin.jobs.relevance.feathr.online.extractor.JobsTargetingSegmentTermVectorExtractor"; + Map features1 = new HashMap<>(); + features1.put("careers_targeting_companies", new ExtractorBasedFeatureConfig("careers_targeting_companies")); + features1.put("careers_targeting_functions", new ExtractorBasedFeatureConfig("careers_targeting_functions")); + AnchorConfigWithExtractor expAnchor1ConfigObj = + new AnchorConfigWithExtractor(source1, null, null, + Collections.singletonList("y"), extractor1, features1); + + String source2 = "Profile"; + String extractor2 = "com.linkedin.jobs.relevance.feathr.online.extractor.ISBYoeTermVectorExtractor"; + Map features2 = new HashMap<>(); + features2.put("careers_member_positionsYoE", new ExtractorBasedFeatureConfig("careers_member_positionsYoE")); + AnchorConfigWithExtractor expAnchor2ConfigObj = + new AnchorConfigWithExtractor(source2, extractor2, features2); + + String source3 = "MemberPreferenceData"; + String extractor3 = "com.linkedin.jobs.relevance.feathr.online.extractor.MemberPreferenceExtractor"; + Map features3 = new HashMap<>(); + features3.put("careers_preference_companySize", new ExtractorBasedFeatureConfig("careers_preference_companySize")); + features3.put("careers_preference_industry", new ExtractorBasedFeatureConfig("careers_preference_industry")); + features3.put("careers_preference_location", new ExtractorBasedFeatureConfig("careers_preference_location")); + AnchorConfigWithExtractor expAnchor3ConfigObj = + new AnchorConfigWithExtractor(source3, extractor3, features3); + + Map anchors = new HashMap<>(); + + anchors.put("jobs-targeting-term-vectors", expAnchor1ConfigObj); + anchors.put("member-profile-yoe", expAnchor2ConfigObj); + anchors.put("jfu-member-preferences", expAnchor3ConfigObj); + + expAnchorsConfigObj = new AnchorsConfig(anchors); + } + + static final String derivationsConfigStr = String.join("\n", + "derivations: {", + " waterloo_job_regionCode: \"import com.linkedin.jobs.relevance.feathr.common.StandardizedLocationGeoRegionExtractor; StandardizedLocationGeoRegionExtractor.extractRegionCode(waterloo_job_location)\"", + " waterloo_member_regionCode: \"import com.linkedin.jobs.relevance.feathr.common.StandardizedLocationGeoRegionExtractor; StandardizedLocationGeoRegionExtractor.extractRegionCode(waterloo_member_location)\"", + " CustomPlusLatentPreferences_LOCATION: \"isNonZero(careers_preference_location) ? careers_preference_location : careers_latentPreference_location\"", + "}"); + + static final DerivationsConfig expDerivationsConfigObj; + static { + SimpleDerivationConfig expDerivation1ConfigObj = new SimpleDerivationConfig("import com.linkedin.jobs.relevance.feathr.common.StandardizedLocationGeoRegionExtractor; StandardizedLocationGeoRegionExtractor.extractRegionCode(waterloo_job_location)"); + SimpleDerivationConfig expDerivation2ConfigObj = new SimpleDerivationConfig("import com.linkedin.jobs.relevance.feathr.common.StandardizedLocationGeoRegionExtractor; StandardizedLocationGeoRegionExtractor.extractRegionCode(waterloo_member_location)"); + SimpleDerivationConfig expDerivation3ConfigObj = new SimpleDerivationConfig("isNonZero(careers_preference_location) ? careers_preference_location : careers_latentPreference_location"); + + Map derivations = new HashMap<>(); + + derivations.put("waterloo_job_regionCode", expDerivation1ConfigObj); + derivations.put("waterloo_member_regionCode", expDerivation2ConfigObj); + derivations.put("CustomPlusLatentPreferences_LOCATION", expDerivation3ConfigObj); + + expDerivationsConfigObj = new DerivationsConfig(derivations); + } + + /* + * Note: We didn't add all the features referenced above in anchors. This fragment is only for testing that the + * feature section is built + */ + static final String featureSectionStr = String.join("\n", + "features: {", + " careers: {", + " careers_preference_companySize: {", + " versions: {", + " \"1.0\": {", + " dims: []", + " }", + " }", + " valType: INT", + " availability: ONLINE", + " }", + " }", + "}"); + + /* + * Note: We didn't add any known dimensions. This fragment is only for testing that the dimension section is built + */ + static final String dimensionSectionStr = String.join("\n", + "dimensions: {", + " careers: {", + " dim1: {", + " versions: {", + " \"4.2\": {", + " type: DISCRETE", + " }", + " }", + " }", + " }", + "}"); + + public static final String featureDefConfigStr1 = String.join("\n", + sourcesConfigStr, + anchorsConfigStr, + derivationsConfigStr); + + public static final FeatureDefConfig expFeatureDefConfigObj1 = + new FeatureDefConfig(expSourcesConfigObj, + expAnchorsConfigObj, expDerivationsConfigObj); + + static final String featureDefConfigStr2 = anchorsConfigStr; + + static final FeatureDefConfig expFeatureDefConfigObj2 = + new FeatureDefConfig(null, expAnchorsConfigObj, null); + + public static final String featureDefConfigStr3 = String.join("\n", + sourcesConfigStr, + anchorsConfigStr, + derivationsConfigStr, + featureSectionStr, + dimensionSectionStr); + + public static final FeatureDefConfig expFeatureDefConfigObj3 = + new FeatureDefConfig(expSourcesConfigObj, + expAnchorsConfigObj, expDerivationsConfigObj); +} diff --git a/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/anchors/AnchorConfigBuilderTest.java b/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/anchors/AnchorConfigBuilderTest.java new file mode 100644 index 000000000..c87b38f3d --- /dev/null +++ b/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/anchors/AnchorConfigBuilderTest.java @@ -0,0 +1,148 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.producer.anchors; + +import com.linkedin.feathr.core.configbuilder.typesafe.AbstractConfigBuilderTest; +import com.linkedin.feathr.core.config.ConfigObj; +import com.linkedin.feathr.core.config.producer.anchors.ComplexFeatureConfig; +import com.linkedin.feathr.core.config.producer.anchors.LateralViewParams; +import com.linkedin.feathr.core.config.producer.anchors.SimpleFeatureConfig; +import com.linkedin.feathr.core.config.producer.anchors.TimeWindowFeatureConfig; +import com.linkedin.feathr.core.config.producer.common.FeatureTypeConfig; +import com.linkedin.feathr.core.configbuilder.ConfigBuilderException; +import com.typesafe.config.Config; +import java.util.function.BiFunction; +import org.testng.annotations.Test; + +import static com.linkedin.feathr.core.configbuilder.typesafe.producer.anchors.AnchorsFixture.*; + + +public class AnchorConfigBuilderTest extends AbstractConfigBuilderTest { + + BiFunction configBuilder = AnchorConfigBuilder::build; + + @Test(description = "Tests build of anchor config object with key and Simple Feature") + public void testWithSimpleFeature() { + testConfigBuilder(anchor1ConfigStr, configBuilder, expAnchor1ConfigObj); + } + + @Test(description = "Tests build of anchor config object with key and Complex Feature") + public void testWithComplexFeature() { + testConfigBuilder(anchor2ConfigStr, configBuilder, expAnchor2ConfigObj); + } + + @Test(description = "Tests build of anchor config object with key and Time-Window Feature") + public void testWithTimeWindowFeature() { + testConfigBuilder(anchor3ConfigStr, configBuilder, expAnchor3ConfigObj); + } + + @Test(description = "Tests build of anchor config object that contains a feature name with forbidden char '.'") + public void testWithSpecialCharacter1() { + testConfigBuilder(anchor6ConfigStr, configBuilder, expAnchor6ConfigObj); + } + + @Test(description = "Tests build of anchor config object that contains a feature name with forbidden char ':'") + public void testWithSpecialCharacter2() { + testConfigBuilder(anchor7ConfigStr, configBuilder, expAnchor7ConfigObj); + } + + @Test(description = "Tests build of anchor config object with key and Time-Window Feature with optional slidingInterval") + public void testWithTimeWindowFeature2() { + testConfigBuilder(anchor8ConfigStr, configBuilder, expAnchor8ConfigObj); + } + + @Test(description = "Tests build of anchor config object with key and Time-Window Feature with lateral view params") + public void testWithLateralViewParams() { + testConfigBuilder(anchor9ConfigStr, configBuilder, expAnchor9ConfigObj); + } + + @Test(description = "Tests build of anchor config object with key and Time-Window Feature with lateral view params with filter") + public void testWithLateralViewParamsWithFilter() { + testConfigBuilder(anchor10ConfigStr, configBuilder, expAnchor10ConfigObj); + } + + @Test(description = "Tests build of anchor config object with key and feature def defined in SQL expression") + public void testWithSqlExpr() { + testConfigBuilder(anchor12ConfigStr, configBuilder, expAnchor12ConfigObj); + } + + @Test(description = "Tests build of anchor config object with keyExtractor only ") + public void testWithKeyExtractor() { + testConfigBuilder(anchor13ConfigStr, configBuilder, expAnchor13ConfigObj); + } + + @Test(description = "Tests build of anchor config object with keyExtractor and extractor ") + public void testWithKeyExtractorAndExtractor() { + testConfigBuilder(anchor14ConfigStr, configBuilder, expAnchor14ConfigObj); + } + + @Test(description = "Tests build of anchor config object with extractor") + public void testWithExtractor() { + testConfigBuilder(anchor4ConfigStr, configBuilder, expAnchor4ConfigObj); + } + + @Test(description = "Tests build of anchor config object with extractor and keyAlias fields") + public void testExtractorWithKeyAlias() { + testConfigBuilder(anchor15ConfigStr, configBuilder, expAnchor15ConfigObj); + } + + @Test(description = "Tests build of anchor config object with key and keyAlias fields") + public void testKeyWithKeyAlias() { + testConfigBuilder(anchor16ConfigStr, configBuilder, expAnchor16ConfigObj); + } + + @Test(description = "Tests build of anchor config object with extractor, key, and keyAlias fields") + public void testExtractorWithKeyAndKeyAlias() { + testConfigBuilder(anchor19ConfigStr, configBuilder, expAnchor19ConfigObj); + } + + @Test(description = "Tests build of anchor config object with extractor, keyExtractor, and lateralView fields") + public void testExtractorWithKeyExtractorAndLateralView() { + testConfigBuilder(anchor21ConfigStr, configBuilder, expAnchor21ConfigObj); + } + + @Test(description = "Tests build of anchor config object with mismatched key and keyAlias", + expectedExceptions = ConfigBuilderException.class) + public void testKeyWithKeyAliasSizeMismatch() { + testConfigBuilder(anchor17ConfigStr, configBuilder, null); + } + + @Test(description = "Tests build of anchor config object with both keyExtractor and keyAlias", + expectedExceptions = ConfigBuilderException.class) + public void testKeyExtractorWithKeyAlias() { + testConfigBuilder(anchor18ConfigStr, configBuilder, null); + } + + @Test(description = "Tests build of anchor config object with extractor, keyExtractor, and key fields", + expectedExceptions = ConfigBuilderException.class) + public void testExtractorWithKeyAndKeyExtractor() { + testConfigBuilder(anchor20ConfigStr, configBuilder, null); + } + + @Test(description = "Tests build of anchor config object with (deprecated) transformer") + public void testWithTransformer() { + testConfigBuilder(anchor5ConfigStr, configBuilder, expAnchor5ConfigObj); + } + + @Test(description = "Tests build of anchor config object with key and NearLine Feature with Window parameters") + public void testWithNearlineFeature() { + testConfigBuilder(anchor11ConfigStr, configBuilder, expAnchor11ConfigObj); + } + + @Test(description = "Tests build of anchor config object with parameterized extractor") + public void testParameterizedExtractor() { + testConfigBuilder(anchor22ConfigStr, configBuilder, expAnchor22ConfigObj); + } + + @Test(description = "Tests build of anchor config object with parameterized extractor with other fields") + public void testParameterizedExtractorWithOtherFields() { + testConfigBuilder(anchor23ConfigStr, configBuilder, expAnchor23ConfigObj); + } + + @Test(description = "Tests equals and hashCode of various config classes") + public void testEqualsAndHashCode() { + super.testEqualsAndHashCode(SimpleFeatureConfig.class, "_configStr"); + super.testEqualsAndHashCode(ComplexFeatureConfig.class, "_configStr"); + super.testEqualsAndHashCode(TimeWindowFeatureConfig.class, "_configStr"); + super.testEqualsAndHashCode(LateralViewParams.class, "_configStr"); + super.testEqualsAndHashCode(FeatureTypeConfig.class, "_configStr"); + } +} diff --git a/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/anchors/AnchorsConfigBuilderTest.java b/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/anchors/AnchorsConfigBuilderTest.java new file mode 100644 index 000000000..faef9b6d5 --- /dev/null +++ b/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/anchors/AnchorsConfigBuilderTest.java @@ -0,0 +1,15 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.producer.anchors; + +import com.linkedin.feathr.core.configbuilder.typesafe.AbstractConfigBuilderTest; +import org.testng.annotations.Test; + +import static com.linkedin.feathr.core.configbuilder.typesafe.producer.anchors.AnchorsFixture.*; + + +public class AnchorsConfigBuilderTest extends AbstractConfigBuilderTest { + + @Test(description = "Tests build of all anchor config objects that may contain key or extractor") + public void anchorsTest() { + testConfigBuilder(anchorsConfigStr, AnchorsConfigBuilder::build, expAnchorsConfig); + } +} diff --git a/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/anchors/AnchorsFixture.java b/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/anchors/AnchorsFixture.java new file mode 100644 index 000000000..0beed1bca --- /dev/null +++ b/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/anchors/AnchorsFixture.java @@ -0,0 +1,742 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.producer.anchors; + +import com.google.common.collect.ImmutableMap; +import com.linkedin.feathr.core.config.TimeWindowAggregationType; +import com.linkedin.feathr.core.config.WindowType; +import com.linkedin.feathr.core.config.producer.ExprType; +import com.linkedin.feathr.core.config.producer.TypedExpr; +import com.linkedin.feathr.core.config.producer.anchors.AnchorConfig; +import com.linkedin.feathr.core.config.producer.anchors.AnchorConfigWithExtractor; +import com.linkedin.feathr.core.config.producer.anchors.AnchorConfigWithKey; +import com.linkedin.feathr.core.config.producer.anchors.AnchorConfigWithKeyExtractor; +import com.linkedin.feathr.core.config.producer.anchors.AnchorsConfig; +import com.linkedin.feathr.core.config.producer.anchors.ExpressionBasedFeatureConfig; +import com.linkedin.feathr.core.config.producer.anchors.ExtractorBasedFeatureConfig; +import com.linkedin.feathr.core.config.producer.anchors.FeatureConfig; +import com.linkedin.feathr.core.config.producer.anchors.LateralViewParams; +import com.linkedin.feathr.core.config.producer.anchors.TimeWindowFeatureConfig; +import com.linkedin.feathr.core.config.producer.anchors.TypedKey; +import com.linkedin.feathr.core.config.producer.anchors.WindowParametersConfig; +import com.linkedin.feathr.core.config.producer.common.FeatureTypeConfig; +import com.linkedin.feathr.core.config.producer.definitions.FeatureType; +import java.time.Duration; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + + +public class AnchorsFixture { + static final FeatureTypeConfig expectedFeatureTypeConfig = + new FeatureTypeConfig.Builder().setFeatureType(FeatureType.DENSE_TENSOR) + .setShapes(Collections.singletonList(10)) + .setDimensionTypes(Collections.singletonList("INT")) + .setValType("FLOAT") + .build(); + + static final String anchor1ConfigStr = String.join("\n", + "member-lix-segment: {", + " source: \"/data/derived/lix/euc/member/#LATEST\"", + " key: \"id\"", + " features: {", + " member_lixSegment_isStudent: \"is_student\"", + " member_lixSegment_isJobSeeker: \"job_seeker_class == 'active'\"", + " }", + "}"); + + public static final AnchorConfigWithKey expAnchor1ConfigObj; + static { + String source = "/data/derived/lix/euc/member/#LATEST"; + TypedKey TypedKey = new TypedKey("\"id\"", ExprType.MVEL); + Map features = new HashMap<>(); + features.put("member_lixSegment_isStudent", new ExtractorBasedFeatureConfig("is_student")); + features.put("member_lixSegment_isJobSeeker", new ExtractorBasedFeatureConfig("job_seeker_class == 'active'")); + expAnchor1ConfigObj = new AnchorConfigWithKey(source, TypedKey, null, features); + } + + static final String anchor2ConfigStr = String.join("\n", + "member-sent-invitations: {", + " source: \"/jobs/frame/inlab/data/features/InvitationStats\"", + " key: \"x\"", + " features: {", + " member_sentInvitations_numIgnoredRejectedInvites: {", + " def: \"toNumeric(numIgnoredRejectedInvites)\"", + " default: 0", + " type: {", + " type: \"DENSE_TENSOR\"", + " shape: [10]", + " dimensionType: [\"INT\"]", + " valType: \"FLOAT\"", + " }", + " }", + " member_sentInvitations_numGuestInvites: {", + " def: \"toNumeric(numGuestInvites)\"", + " type: {", + " type: \"DENSE_TENSOR\"", + " shape: [10]", + " dimensionType: [\"INT\"]", + " valType: \"FLOAT\"", + " }", + " default: 0", + " }", + " }", + "}"); + + static final AnchorConfigWithKey expAnchor2ConfigObj; + static{ + String source = "/jobs/frame/inlab/data/features/InvitationStats"; + TypedKey TypedKey = new TypedKey("\"x\"", ExprType.MVEL); + String defaultValue = "0"; + ExpressionBasedFeatureConfig feature1 = new ExpressionBasedFeatureConfig("toNumeric(numIgnoredRejectedInvites)", + ExprType.MVEL, defaultValue, expectedFeatureTypeConfig); + ExpressionBasedFeatureConfig feature2= new ExpressionBasedFeatureConfig("toNumeric(numGuestInvites)", + ExprType.MVEL, defaultValue, expectedFeatureTypeConfig); + Map features = new HashMap<>(); + features.put("member_sentInvitations_numIgnoredRejectedInvites", feature1); + features.put("member_sentInvitations_numGuestInvites", feature2); + expAnchor2ConfigObj = new AnchorConfigWithKey(source, TypedKey, null, features); + } + + static final String anchor3ConfigStr = String.join("\n", + "swaAnchor: {", + " source: \"swaSource\"", + " key: \"mid\"", + " features: {", + " simplePageViewCount: {", + " def: \"pageView\"", + " aggregation: COUNT", + " window: 1d", + " type: {", + " type: \"DENSE_TENSOR\"", + " shape: [10]", + " dimensionType: [\"INT\"]", + " valType: \"FLOAT\"", + " doc: \"this is doc\"", + " }", + " }", + " maxPV12h: {", + " def: \"pageView\"", + " aggregation: MAX", + " window: 12h", + " groupBy: \"pageKey\"", + " limit: 2", + " type: {", + " type: \"DENSE_TENSOR\"", + " shape: [10]", + " dimensionType: [\"INT\"]", + " valType: \"FLOAT\"", + " doc: \"this is doc\"", + " }", + " }", + " }", + "}"); + + static final AnchorConfigWithKey expAnchor3ConfigObj; + static{ + String source = "swaSource"; + TypedKey TypedKey = new TypedKey("\"mid\"", ExprType.MVEL); + TypedExpr typedExpr = new TypedExpr("pageView", ExprType.SQL); + + WindowParametersConfig windowParameters1 = new WindowParametersConfig(WindowType.SLIDING, Duration.ofDays(1), null); + TimeWindowFeatureConfig feature1 = new TimeWindowFeatureConfig(typedExpr, + TimeWindowAggregationType.COUNT, windowParameters1, null, null, null, null, null, null, expectedFeatureTypeConfig, null); + WindowParametersConfig windowParameters2 = new WindowParametersConfig(WindowType.SLIDING, Duration.ofHours(12), null); + TimeWindowFeatureConfig feature2 = new TimeWindowFeatureConfig(typedExpr, + TimeWindowAggregationType.MAX, windowParameters2, null, "pageKey",2, null, null, null, expectedFeatureTypeConfig, null); + Map features = new HashMap<>(); + features.put("simplePageViewCount", feature1); + features.put("maxPV12h", feature2); + expAnchor3ConfigObj = new AnchorConfigWithKey(source, TypedKey, null, features); + } + + static final String anchor4ConfigStr = String.join("\n", + "waterloo-job-term-vectors: {", + " source: \"/data/derived/standardization/waterloo/jobs_std_data/test/#LATEST\"", + " extractor: \"com.linkedin.frameproto.foundation.anchor.NiceJobFeatures\"", + " features: {", + " waterloo_job_jobTitle: {", + " type: BOOLEAN", + " }", + " waterloo_job_companyId: {},", + " waterloo_job_companySize: {}", + " }", + "}"); + + static final AnchorConfigWithExtractor expAnchor4ConfigObj; + static{ + FeatureTypeConfig featureTypeConfig = new FeatureTypeConfig(FeatureType.BOOLEAN); + + String source = "/data/derived/standardization/waterloo/jobs_std_data/test/#LATEST"; + String extractor = "com.linkedin.frameproto.foundation.anchor.NiceJobFeatures"; + Map features = new HashMap<>(); + features.put("waterloo_job_jobTitle", new ExtractorBasedFeatureConfig("waterloo_job_jobTitle", featureTypeConfig)); + features.put("waterloo_job_companyId", new ExtractorBasedFeatureConfig("waterloo_job_companyId")); + features.put("waterloo_job_companySize", new ExtractorBasedFeatureConfig("waterloo_job_companySize")); + expAnchor4ConfigObj = new AnchorConfigWithExtractor(source, extractor, features); + } + + static final String anchor5ConfigStr = String.join("\n", + "careers-member-education: {", + " source: \"/jobs/liar/jymbii-features-engineering/production/memberFeatures/education/#LATEST\"", + " transformer: \"com.linkedin.careers.relevance.feathr.offline.anchor.LegacyFeastFormattedFeatures\"", + " features: [", + " \"careers_member_degree\",", + " \"careers_member_rolledUpDegree\",", + " \"careers_member_fieldOfStudy\",", + " ]", + "}"); + + static final AnchorConfigWithExtractor expAnchor5ConfigObj; + static{ + String source = "/jobs/liar/jymbii-features-engineering/production/memberFeatures/education/#LATEST"; + String extractor = "com.linkedin.careers.relevance.feathr.offline.anchor.LegacyFeastFormattedFeatures"; + Map features = new HashMap<>(); + features.put("careers_member_degree", new ExtractorBasedFeatureConfig("careers_member_degree")); + features.put("careers_member_rolledUpDegree", new ExtractorBasedFeatureConfig("careers_member_rolledUpDegree")); + features.put("careers_member_fieldOfStudy", new ExtractorBasedFeatureConfig("careers_member_fieldOfStudy")); + expAnchor5ConfigObj = new AnchorConfigWithExtractor(source, extractor, features); + } + + static final String anchor6ConfigStr = String.join("\n", + "\"careers-job-embedding-0.0.2\": {", + " source: \"/jobs/jobrel/careers-embedding-serving/job-embeddings-versions/0.0.2/#LATEST\"", + " key: \"getIdFromRawUrn(key.entityUrn)\"", + " features: {", + " \"careers_job_embedding_0.0.2\": {", + " def: \"value.embedding\"", + " type: VECTOR", + " }", + " }", + "}"); + + static final AnchorConfigWithKey expAnchor6ConfigObj; + static{ + FeatureTypeConfig featureTypeConfig = new FeatureTypeConfig(FeatureType.VECTOR); + String source = "/jobs/jobrel/careers-embedding-serving/job-embeddings-versions/0.0.2/#LATEST"; + TypedKey TypedKey = new TypedKey("\"getIdFromRawUrn(key.entityUrn)\"", ExprType.MVEL); + String featureName = "careers_job_embedding_0.0.2"; + String featureExpr = "value.embedding"; + ExpressionBasedFeatureConfig feature = new ExpressionBasedFeatureConfig(featureExpr, featureTypeConfig); + Map features = new HashMap<>(); + features.put(featureName, feature); + expAnchor6ConfigObj = new AnchorConfigWithKey(source, TypedKey, null, features); + } + + static final String anchor7ConfigStr = String.join("\n", + "\"careers-job-embedding-0.0.2\": {", + " source: \"/jobs/jobrel/careers-embedding-serving/job-embeddings-versions/0.0.2/#LATEST\"", + " key: \"getIdFromRawUrn(key.entityUrn)\"", + " features: {", + " \"foo:bar\": {", + " def: \"value.embedding\"", + " type: VECTOR", + " }", + " }", + "}"); + + static final AnchorConfigWithKey expAnchor7ConfigObj; + static{ + FeatureTypeConfig featureTypeConfig = new FeatureTypeConfig(FeatureType.VECTOR); + String source = "/jobs/jobrel/careers-embedding-serving/job-embeddings-versions/0.0.2/#LATEST"; + TypedKey TypedKey = new TypedKey("\"getIdFromRawUrn(key.entityUrn)\"", ExprType.MVEL); + String featureName = "foo:bar"; + String featureExpr = "value.embedding"; + String featureType = "VECTOR"; + ExpressionBasedFeatureConfig feature = new ExpressionBasedFeatureConfig(featureExpr, featureTypeConfig); + Map features = new HashMap<>(); + features.put(featureName, feature); + expAnchor7ConfigObj = new AnchorConfigWithKey(source, TypedKey, null, features); + } + + static final String anchor8ConfigStr = String.join("\n", + "swaAnchor: {", + " source: \"kafkaTestSource\"", + " key: \"mid\"", + " features: {", + " simplePageViewCount: {", + " def: \"pageView\"", + " aggregation: COUNT", + " window: 1d", + " }", + " maxPV12h: {", + " def: \"pageView\"", + " aggregation: MAX", + " window: 12h", + " groupBy: \"pageKey\"", + " limit: 2", + " }", + " }", + "}"); + + static final AnchorConfigWithKey expAnchor8ConfigObj; + static { + String source = "kafkaTestSource"; + TypedKey TypedKey = new TypedKey("\"mid\"", ExprType.MVEL); + WindowParametersConfig windowParameters1 = new WindowParametersConfig(WindowType.SLIDING, Duration.ofDays(1), null); + TimeWindowFeatureConfig feature1 = new TimeWindowFeatureConfig("pageView", + TimeWindowAggregationType.COUNT, windowParameters1, null, null, null, null, null); + WindowParametersConfig windowParameters2 = new WindowParametersConfig(WindowType.SLIDING, Duration.ofHours(12), null); + TimeWindowFeatureConfig feature2 = new TimeWindowFeatureConfig("pageView", + TimeWindowAggregationType.MAX, windowParameters2, + null, "pageKey", 2, null, null); + + Map features = new HashMap<>(); + features.put("simplePageViewCount", feature1); + features.put("maxPV12h", feature2); + expAnchor8ConfigObj = new AnchorConfigWithKey(source, TypedKey, null, features); + } + + static final String anchor9ConfigStr = String.join("\n", + "swaAnchor2: {", + " source: windowAgg1dSource", + " key: \"substring(x, 15)\"", + " lateralViewParameters: {", + " lateralViewDef: \"explode(features)\"", + " lateralViewItemAlias: feature", + " }", + " features: {", + " articleCount_sum_1d: {", + " def: \"feature.col.value\"", + " filter: \"feature.col.name = 'articleCount'\"", + " aggregation: LATEST", + " window: 2 days", + " }", + " }", + "}"); + + static final AnchorConfigWithKey expAnchor9ConfigObj; + static { + String source = "windowAgg1dSource"; + TypedKey TypedKey = new TypedKey("\"substring(x, 15)\"", ExprType.MVEL); + + LateralViewParams lateralViewParams = new LateralViewParams("explode(features)", "feature"); + + WindowParametersConfig windowParameters = new WindowParametersConfig(WindowType.SLIDING, Duration.ofDays(2), null); + + TimeWindowFeatureConfig feature1 = new TimeWindowFeatureConfig("feature.col.value", + TimeWindowAggregationType.LATEST, windowParameters, "feature.col.name = 'articleCount'", null, null, null, + null); + + Map features = new HashMap<>(); + features.put("articleCount_sum_1d", feature1); + expAnchor9ConfigObj = new AnchorConfigWithKey(source, TypedKey, lateralViewParams, features); + } + + static final String anchor10ConfigStr = String.join("\n", + "swaAnchor2: {", + " source: windowAgg1dSource", + " key: \"substring(x, 15)\"", + " lateralViewParameters: {", + " lateralViewDef: \"explode(features)\"", + " lateralViewItemAlias: feature", + " }", + " features: {", + " facetTitles_sum_30d: {", + " def: \"feature.col.value\"", + " aggregation: SUM", + " groupBy: \"feature.col.term\"", + " window: 30 days", + " }", + " }", + "}"); + + static final AnchorConfigWithKey expAnchor10ConfigObj; + static { + String source = "windowAgg1dSource"; + TypedKey TypedKey = new TypedKey("\"substring(x, 15)\"", ExprType.MVEL); + + LateralViewParams lateralViewParams = new LateralViewParams("explode(features)", "feature"); + + WindowParametersConfig windowParameters = new WindowParametersConfig(WindowType.SLIDING, Duration.ofDays(30), null); + TimeWindowFeatureConfig feature1 = new TimeWindowFeatureConfig("feature.col.value", + TimeWindowAggregationType.SUM, windowParameters, null, "feature.col.term", null, null, null); + + Map features = new HashMap<>(); + features.put("facetTitles_sum_30d", feature1); + expAnchor10ConfigObj = new AnchorConfigWithKey(source, TypedKey, lateralViewParams, features); + } + + static final String anchor11ConfigStr = String.join("\n", + "nearLineFeatureAnchor: {", + " source: kafkaTestSource", + " key.mvel: mid", + " features: {", + " feature1: {", + " def.mvel: pageView", + " aggregation: MAX", + " windowParameters: {", + " type: SLIDING", + " size: 1h", + " slidingInterval: 10m", + " }", + " groupBy: pageKey", + " }", + " feature2: {", + " def.mvel: pageView", + " aggregation: MAX", + " windowParameters: {", + " type: SLIDING", + " size: 1h", + " slidingInterval: 10m", + " }", + " groupBy: pageKey", + " filter.mvel: \"$.getAsTermVector().keySet()\"", + " }", + " }", + "}"); + + static final AnchorConfigWithKey expAnchor11ConfigObj; + static { + String source = "kafkaTestSource"; + TypedKey TypedKey = new TypedKey("\"mid\"", ExprType.MVEL); + WindowParametersConfig windowParametersConfig = new WindowParametersConfig(WindowType.SLIDING, Duration.ofHours(1), Duration.ofMinutes(10)); + TimeWindowFeatureConfig feature1 = new TimeWindowFeatureConfig("pageView", ExprType.MVEL, + TimeWindowAggregationType.MAX, windowParametersConfig, null, null, "pageKey", null, null, null); + TimeWindowFeatureConfig feature2 = new TimeWindowFeatureConfig("pageView", ExprType.MVEL, + TimeWindowAggregationType.MAX, windowParametersConfig, "$.getAsTermVector().keySet()", ExprType.MVEL, "pageKey", null, null, null); + Map features = new HashMap<>(); + features.put("feature1", feature1); + features.put("feature2", feature2); + expAnchor11ConfigObj = new AnchorConfigWithKey(source, TypedKey, null, features); + } + + static final String anchor12ConfigStr = String.join("\n", + "member-sent-invitations: {", + " source: \"/jobs/frame/inlab/data/features/InvitationStats\"", + " key.sqlExpr: \"x\"", + " features: {", + " member_sentInvitations_numIgnoredRejectedInvitesV2: {", + " def.sqlExpr: \"numIgnoredRejectedInvites\"", + " default: 0", + " }", + " member_sentInvitations_numGuestInvitesV2: {", + " def.sqlExpr: \"numGuestInvites\"", + " default: 0", + " }", + " }", + "}"); + + static final AnchorConfigWithKey expAnchor12ConfigObj; + static{ + String source = "/jobs/frame/inlab/data/features/InvitationStats"; + String defaultValue = "0"; + ExpressionBasedFeatureConfig feature1 = new ExpressionBasedFeatureConfig("numIgnoredRejectedInvites", + ExprType.SQL, null, defaultValue); + ExpressionBasedFeatureConfig feature2= new ExpressionBasedFeatureConfig("numGuestInvites", + ExprType.SQL,null, defaultValue); + Map features = new HashMap<>(); + features.put("member_sentInvitations_numIgnoredRejectedInvitesV2", feature1); + features.put("member_sentInvitations_numGuestInvitesV2", feature2); + expAnchor12ConfigObj = new AnchorConfigWithKey(source, new TypedKey("\"x\"", ExprType.SQL), null, features); + } + + static final String anchor13ConfigStr = String.join("\n", + "member-sent-invitationsV3: {", + " source: \"/jobs/frame/inlab/data/features/InvitationStats\"", + " keyExtractor: \"com.linkedin.frameproto.foundation.anchor.NiceJobFeaturesKeyExtractor\"", + " features: {", + " member_sentInvitations_numIgnoredRejectedInvitesV3: {", + " def.sqlExpr: \"numIgnoredRejectedInvites\"", + " default: 0", + " }", + " member_sentInvitations_numGuestInvitesV3: {", + " def.sqlExpr: \"numGuestInvites\"", + " default: 0", + " }", + " }", + "}"); + + static final AnchorConfigWithKeyExtractor expAnchor13ConfigObj; + static{ + String source = "/jobs/frame/inlab/data/features/InvitationStats"; + String keyExtractor = "com.linkedin.frameproto.foundation.anchor.NiceJobFeaturesKeyExtractor"; + String defaultValue = "0"; + ExpressionBasedFeatureConfig feature1 = new ExpressionBasedFeatureConfig("numIgnoredRejectedInvites", + ExprType.SQL, null, defaultValue); + ExpressionBasedFeatureConfig feature2= new ExpressionBasedFeatureConfig("numGuestInvites", + ExprType.SQL,null, defaultValue); + Map features = new HashMap<>(); + features.put("member_sentInvitations_numIgnoredRejectedInvitesV3", feature1); + features.put("member_sentInvitations_numGuestInvitesV3", feature2); + expAnchor13ConfigObj = new AnchorConfigWithKeyExtractor(source, keyExtractor, features); + } + + static final String anchor14ConfigStr = String.join("\n", + "waterloo-job-term-vectors: {", + " source: \"/data/derived/standardization/waterloo/jobs_std_data/test/#LATEST\"", + " keyExtractor: \"com.linkedin.frameproto.foundation.anchor.NiceJobFeaturesKeyExtractor\"", + " extractor: \"com.linkedin.frameproto.foundation.anchor.NiceJobFeatures\"", + " features: [", + " waterloo_job_jobTitleV2,", + " waterloo_job_companyIdV2,", + " waterloo_job_companySizeV2", + " ]", + "}"); + + static final AnchorConfigWithExtractor expAnchor14ConfigObj; + static{ + String source = "/data/derived/standardization/waterloo/jobs_std_data/test/#LATEST"; + String keyExtractor = "com.linkedin.frameproto.foundation.anchor.NiceJobFeaturesKeyExtractor"; + String extractor = "com.linkedin.frameproto.foundation.anchor.NiceJobFeatures"; + Map features = new HashMap<>(); + features.put("waterloo_job_jobTitleV2", new ExtractorBasedFeatureConfig("waterloo_job_jobTitleV2")); + features.put("waterloo_job_companyIdV2", new ExtractorBasedFeatureConfig("waterloo_job_companyIdV2")); + features.put("waterloo_job_companySizeV2", new ExtractorBasedFeatureConfig("waterloo_job_companySizeV2")); + expAnchor14ConfigObj = new AnchorConfigWithExtractor(source, keyExtractor, extractor, features); + } + + // extractor with keyAlias + static final String anchor15ConfigStr = String.join("\n", + "waterloo-job-term-vectors: {", + " source: \"/data/derived/standardization/waterloo/jobs_std_data/test/#LATEST\"", + " keyAlias: [key1, key2]", + " extractor: \"com.linkedin.frameproto.foundation.anchor.NiceJobFeatures\"", + " features: {", + " waterloo_job_jobTitle: {", + " type: BOOLEAN", + " }", + " waterloo_job_companyId: {},", + " waterloo_job_companySize: {}", + " }", + "}"); + + static final AnchorConfigWithExtractor expAnchor15ConfigObj; + static{ + FeatureTypeConfig featureTypeConfig = new FeatureTypeConfig(FeatureType.BOOLEAN); + + String source = "/data/derived/standardization/waterloo/jobs_std_data/test/#LATEST"; + String extractor = "com.linkedin.frameproto.foundation.anchor.NiceJobFeatures"; + Map features = new HashMap<>(); + features.put("waterloo_job_jobTitle", new ExtractorBasedFeatureConfig("waterloo_job_jobTitle", featureTypeConfig)); + features.put("waterloo_job_companyId", new ExtractorBasedFeatureConfig("waterloo_job_companyId")); + features.put("waterloo_job_companySize", new ExtractorBasedFeatureConfig("waterloo_job_companySize")); + expAnchor15ConfigObj = new AnchorConfigWithExtractor(source, null, null, + Arrays.asList("key1", "key2"), extractor, features); + } + + // key and keyAlias co-exist + static final String anchor16ConfigStr = String.join("\n", + "\"careers-job-embedding-0.0.2\": {", + " source: \"/jobs/jobrel/careers-embedding-serving/job-embeddings-versions/0.0.2/#LATEST\"", + " key: \"getIdFromRawUrn(key.entityUrn, key.someProperty)\"", + " keyAlias: \"keyAlias1\"", + " features: {", + " \"foo:bar\": {", + " def: \"value.embedding\"", + " type: VECTOR", + " }", + " }", + "}"); + + static final AnchorConfigWithKey expAnchor16ConfigObj; + static{ + FeatureTypeConfig featureTypeConfig = new FeatureTypeConfig(FeatureType.VECTOR); + String source = "/jobs/jobrel/careers-embedding-serving/job-embeddings-versions/0.0.2/#LATEST"; + TypedKey TypedKey = + new TypedKey( "\"getIdFromRawUrn(key.entityUrn, key.someProperty)\"", ExprType.MVEL); + List keyAlias = Collections.singletonList("keyAlias1"); + String featureName = "foo:bar"; + String featureExpr = "value.embedding"; + String featureType = "VECTOR"; + ExpressionBasedFeatureConfig feature = new ExpressionBasedFeatureConfig(featureExpr, featureTypeConfig); + Map features = new HashMap<>(); + features.put(featureName, feature); + expAnchor16ConfigObj = new AnchorConfigWithKey(source, TypedKey, keyAlias, null, features); + } + + // key size and keyAlias size do not match + static final String anchor17ConfigStr = String.join("\n", + "\"careers-job-embedding-0.0.2\": {", + " source: \"/jobs/jobrel/careers-embedding-serving/job-embeddings-versions/0.0.2/#LATEST\"", + " key: \"getIdFromRawUrn(key.entityUrn)\"", + " keyAlias: [keyAlias1, keyAlias2]", + " features: {", + " \"foo:bar\": {", + " def: \"value.embedding\"", + " type: VECTOR", + " }", + " }", + "}"); + + // invalid case where keyExtractor and keyAlias coexist + static final String anchor18ConfigStr = String.join("\n", + "member-sent-invitationsV3: {", + " source: \"/jobs/frame/inlab/data/features/InvitationStats\"", + " keyExtractor: \"com.linkedin.frameproto.foundation.anchor.NiceJobFeaturesKeyExtractor\"", + " keyAlias: [key1, key2]", + " features: {", + " member_sentInvitations_numIgnoredRejectedInvitesV3: {", + " def.sqlExpr: \"numIgnoredRejectedInvites\"", + " default: 0", + " }", + " member_sentInvitations_numGuestInvitesV3: {", + " def.sqlExpr: \"numGuestInvites\"", + " default: 0", + " }", + " }", + "}"); + + // extractor with keyAlias and key + static final String anchor19ConfigStr = String.join("\n", + "waterloo-job-term-vectors: {", + " source: \"/data/derived/standardization/waterloo/jobs_std_data/test/#LATEST\"", + " key.sqlExpr: [key1, key2]", + " keyAlias: [keyAlias1, keyAlias2]", + " extractor: \"com.linkedin.frameproto.foundation.anchor.NiceJobFeatures\"", + " features: {", + " waterloo_job_jobTitle: {", + " type: BOOLEAN", + " }", + " waterloo_job_companyId: {},", + " waterloo_job_companySize: {}", + " }", + "}"); + + static final AnchorConfigWithExtractor expAnchor19ConfigObj; + static{ + FeatureTypeConfig featureTypeConfig = new FeatureTypeConfig(FeatureType.BOOLEAN); + + String source = "/data/derived/standardization/waterloo/jobs_std_data/test/#LATEST"; + String extractor = "com.linkedin.frameproto.foundation.anchor.NiceJobFeatures"; + TypedKey TypedKey = new TypedKey("[key1, key2]", ExprType.SQL); + Map features = new HashMap<>(); + features.put("waterloo_job_jobTitle", new ExtractorBasedFeatureConfig("waterloo_job_jobTitle", featureTypeConfig)); + features.put("waterloo_job_companyId", new ExtractorBasedFeatureConfig("waterloo_job_companyId")); + features.put("waterloo_job_companySize", new ExtractorBasedFeatureConfig("waterloo_job_companySize")); + expAnchor19ConfigObj = new AnchorConfigWithExtractor(source, null, TypedKey, + Arrays.asList("keyAlias1", "keyAlias2"), extractor, features); + } + + // extractor with keyExtractor and key + static final String anchor20ConfigStr = String.join("\n", + "waterloo-job-term-vectors: {", + " source: \"/data/derived/standardization/waterloo/jobs_std_data/test/#LATEST\"", + " key.sqlExpr: [key1, key2]", + " keyExtractor: \"com.linkedin.frameproto.foundation.anchor.NiceJobFeaturesKeyExtractor\"", + " extractor: \"com.linkedin.frameproto.foundation.anchor.NiceJobFeatures\"", + " features: {", + " waterloo_job_jobTitle: {", + " type: BOOLEAN", + " }", + " waterloo_job_companyId: {},", + " waterloo_job_companySize: {}", + " }", + "}"); + + // extractor with keyExtractor and lateralViewParameters + static final String anchor21ConfigStr = String.join("\n", + "swaAnchor2: {", + " source: windowAgg1dSource", + " keyExtractor: \"com.linkedin.frameproto.foundation.anchor.NiceJobFeaturesKeyExtractor\"", + " lateralViewParameters: {", + " lateralViewDef: \"explode(features)\"", + " lateralViewItemAlias: feature", + " }", + " features: {", + " facetTitles_sum_30d: {", + " def: \"feature.col.value\"", + " aggregation: SUM", + " groupBy: \"feature.col.term\"", + " window: 30 days", + " }", + " }", + "}"); + + static final AnchorConfigWithKeyExtractor expAnchor21ConfigObj; + static { + String source = "windowAgg1dSource"; + + String keyExtractor = "com.linkedin.frameproto.foundation.anchor.NiceJobFeaturesKeyExtractor"; + LateralViewParams lateralViewParams = new LateralViewParams("explode(features)", "feature"); + + WindowParametersConfig windowParameters = new WindowParametersConfig(WindowType.SLIDING, Duration.ofDays(30), null); + TimeWindowFeatureConfig feature1 = new TimeWindowFeatureConfig("feature.col.value", + TimeWindowAggregationType.SUM, windowParameters, null, "feature.col.term", null, null, null); + + Map features = new HashMap<>(); + features.put("facetTitles_sum_30d", feature1); + expAnchor21ConfigObj = new AnchorConfigWithKeyExtractor(source, keyExtractor, features, lateralViewParams); + } + + static final String anchor22ConfigStr = String.join("\n", + "waterloo-job-term-vectors: {", + " source: \"/data/derived/standardization/waterloo/jobs_std_data/test/#LATEST\"", + " keyExtractor: \"com.linkedin.frameproto.foundation.anchor.NiceJobFeaturesKeyExtractor\"", + " extractor: \"com.linkedin.frameproto.foundation.anchor.NiceJobFeatures\"", + " features: {", + " waterloo_job_jobTitleV2 : {", + " parameters: {", + " param1 : [waterlooCompany_terms_hashed, waterlooCompany_values]", + " param2 : [waterlooCompany_terms_hashed, waterlooCompany_values]", + " }", + " }", + " }", + "}"); + + static final AnchorConfigWithExtractor expAnchor22ConfigObj; + static{ + String source = "/data/derived/standardization/waterloo/jobs_std_data/test/#LATEST"; + String keyExtractor = "com.linkedin.frameproto.foundation.anchor.NiceJobFeaturesKeyExtractor"; + String extractor = "com.linkedin.frameproto.foundation.anchor.NiceJobFeatures"; + Map features = new HashMap<>(); + features.put("waterloo_job_jobTitleV2", new ExtractorBasedFeatureConfig( + "waterloo_job_jobTitleV2", null, null, + ImmutableMap.of("param1", "[\"waterlooCompany_terms_hashed\",\"waterlooCompany_values\"]", + "param2", "[\"waterlooCompany_terms_hashed\",\"waterlooCompany_values\"]"))); + expAnchor22ConfigObj = new AnchorConfigWithExtractor( + source, keyExtractor, null, null, extractor, features); + } + + static final String anchor23ConfigStr = String.join("\n", + "waterloo-job-term-vectors: {", + " source: \"/data/derived/standardization/waterloo/jobs_std_data/test/#LATEST\"", + " keyExtractor: \"com.linkedin.frameproto.foundation.anchor.NiceJobFeaturesKeyExtractor\"", + " extractor: \"com.linkedin.frameproto.foundation.anchor.NiceJobFeatures\"", + " features: {", + " waterloo_job_jobTitleV2 : {", + " parameters: {", + " param1 : [waterlooCompany_terms_hashed, waterlooCompany_values]", + " param2 : [waterlooCompany_terms_hashed, waterlooCompany_values]", + " }", + " default: true", + " type: BOOLEAN", + " }", + " }", + "}"); + + static final AnchorConfigWithExtractor expAnchor23ConfigObj; + static{ + String source = "/data/derived/standardization/waterloo/jobs_std_data/test/#LATEST"; + String keyExtractor = "com.linkedin.frameproto.foundation.anchor.NiceJobFeaturesKeyExtractor"; + String extractor = "com.linkedin.frameproto.foundation.anchor.NiceJobFeatures"; + Map parameters = new HashMap<>(); + parameters.put("param1", "[\"waterlooCompany_terms_hashed\", \"waterlooCompany_values\"]"); + Map features = new HashMap<>(); + features.put("waterloo_job_jobTitleV2", new ExtractorBasedFeatureConfig( + "waterloo_job_jobTitleV2", new FeatureTypeConfig(FeatureType.BOOLEAN), "true", + ImmutableMap.of("param1", "[\"waterlooCompany_terms_hashed\",\"waterlooCompany_values\"]", + "param2", "[\"waterlooCompany_terms_hashed\",\"waterlooCompany_values\"]"))); + expAnchor23ConfigObj = new AnchorConfigWithExtractor( + source, keyExtractor, null, null, extractor, features); + } + + static final String anchorsConfigStr = String.join("\n", + "anchors: {", + anchor1ConfigStr, + anchor2ConfigStr, + anchor3ConfigStr, + anchor4ConfigStr, + "}"); + + static final AnchorsConfig expAnchorsConfig; + static{ + Map anchors = new HashMap<>(); + anchors.put("member-lix-segment", expAnchor1ConfigObj); + anchors.put("member-sent-invitations", expAnchor2ConfigObj); + anchors.put("swaAnchor", expAnchor3ConfigObj); + anchors.put("waterloo-job-term-vectors", expAnchor4ConfigObj); + expAnchorsConfig = new AnchorsConfig(anchors); + } + +} diff --git a/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/anchors/FeatureConfigBuilderTest.java b/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/anchors/FeatureConfigBuilderTest.java new file mode 100644 index 000000000..57dcf0b81 --- /dev/null +++ b/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/anchors/FeatureConfigBuilderTest.java @@ -0,0 +1,75 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.producer.anchors; + +import com.linkedin.feathr.core.config.producer.anchors.AnchorConfig; +import com.linkedin.feathr.core.config.producer.anchors.FeatureConfig; +import com.typesafe.config.Config; +import com.typesafe.config.ConfigFactory; +import com.typesafe.config.ConfigValue; +import java.util.List; +import java.util.Map; +import org.testng.annotations.Test; + +import static com.linkedin.feathr.core.configbuilder.typesafe.producer.anchors.FeatureFixture.*; +import static org.testng.Assert.*; + + +public class FeatureConfigBuilderTest { + @Test(description = "Parsing and building of extractor based feature config") + public void extractorBasedFeatureConfigs() { + testFeatureConfigBuilder(feature1ConfigStr, expFeature1ConfigObj); + } + + @Test(description = "Parsing and building of extractor based feature config with special characters . and :") + public void extractorBasedFeatureConfigsWithSpecialCharacters() { + testFeatureConfigBuilder(feature1ConfigStr, expFeature1ConfigObj); + } + + @Test(description = "Parsing and building of extractor based feature config") + public void extractorBasedFeatureConfigsWithExtractor() { + testFeatureConfigBuilder(feature2ConfigStr, expFeature2ConfigObj); + } + + @Test(description = "Parsing and building of extractor based feature config with type config") + public void extractorBasedFeatureConfigsWithExtractorWithType() { + testFeatureConfigBuilder(feature2ConfigWithTypeStr, expFeature2WithTypeConfigObj); + } + + @Test(description = "Parsing and building of extractor based feature config with type config and parameters") + public void extractorBasedFeatureConfigsWithParameterizedExtractor() { + testFeatureConfigBuilder(feature5ConfigWithTypeStr, expFeature5WithTypeConfigObj); + } + + @Test(description = "Parsing and building of expression based feature config") + public void expressionBasedFeatureConfigs() { + testFeatureConfigBuilder(feature3ConfigStr, expFeature3ConfigObj); + } + + @Test(description = "Parsing and building of time-window feature config") + public void timeWindowFeatureConfigs() { + testFeatureConfigBuilder(feature4ConfigStr, expFeature4ConfigObj); + } + + private Map buildFeatureConfig(String featureConfigStr) { + Config fullConfig = ConfigFactory.parseString(featureConfigStr); + ConfigValue configValue = fullConfig.getValue(AnchorConfig.FEATURES); + + switch (configValue.valueType()) { + case OBJECT: + Config featuresConfig = fullConfig.getConfig(AnchorConfig.FEATURES); + return FeatureConfigBuilder.build(featuresConfig); + + case LIST: + List featureNames = fullConfig.getStringList(AnchorConfig.FEATURES); + return FeatureConfigBuilder.build(featureNames); + + default: + throw new RuntimeException("Unexpected value type " + configValue.valueType() + + " for " + AnchorConfig.FEATURES); + } + } + + private void testFeatureConfigBuilder(String featureConfigStr, Map expFeatureConfigObj) { + Map obsFeatureConfigObj = buildFeatureConfig(featureConfigStr); + assertEquals(obsFeatureConfigObj, expFeatureConfigObj); + } +} diff --git a/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/anchors/FeatureFixture.java b/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/anchors/FeatureFixture.java new file mode 100644 index 000000000..590eea31a --- /dev/null +++ b/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/anchors/FeatureFixture.java @@ -0,0 +1,254 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.producer.anchors; + +import com.google.common.collect.ImmutableMap; +import com.linkedin.feathr.core.config.TimeWindowAggregationType; +import com.linkedin.feathr.core.config.WindowType; +import com.linkedin.feathr.core.config.producer.ExprType; +import com.linkedin.feathr.core.config.producer.TypedExpr; +import com.linkedin.feathr.core.config.producer.anchors.ExpressionBasedFeatureConfig; +import com.linkedin.feathr.core.config.producer.anchors.ExtractorBasedFeatureConfig; +import com.linkedin.feathr.core.config.producer.anchors.FeatureConfig; +import com.linkedin.feathr.core.config.producer.anchors.TimeWindowFeatureConfig; +import com.linkedin.feathr.core.config.producer.anchors.WindowParametersConfig; +import com.linkedin.feathr.core.config.producer.common.FeatureTypeConfig; +import com.linkedin.feathr.core.config.producer.definitions.FeatureType; +import java.time.Duration; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + + +class FeatureFixture { + + static final String feature1ConfigStr = String.join("\n", + "features: {", + " member_lixSegment_isStudent: \"is_student\"", + " member_lixSegment_isJobSeeker: \"job_seeker_class == 'active'\"", + "}"); + + static final Map expFeature1ConfigObj; + static { + expFeature1ConfigObj = new HashMap<>(); + expFeature1ConfigObj.put("member_lixSegment_isStudent", new ExtractorBasedFeatureConfig("is_student")); + expFeature1ConfigObj.put( + "member_lixSegment_isJobSeeker", new ExtractorBasedFeatureConfig("job_seeker_class == 'active'")); + } + + static final String feature1ConfigStrWithSpecialChars = String.join("\n", + "features: {", + " \"member:lixSegment.isStudent\": \"is_student\"", + " \"member:lixSegment.isJobSeeker\": \"job_seeker_class == 'active'\"", + "}"); + + static final Map expFeature1ConfigObjWithSpecialChars; + static { + expFeature1ConfigObjWithSpecialChars = new HashMap<>(); + expFeature1ConfigObjWithSpecialChars.put("member:lixSegment.isStudent", new ExtractorBasedFeatureConfig("is_student")); + expFeature1ConfigObjWithSpecialChars.put( + "member:lixSegment.isJobSeeker", new ExtractorBasedFeatureConfig("job_seeker_class == 'active'")); + } + + static final String feature2ConfigStr = String.join("\n", + "features: [", + " waterloo_job_jobTitle,", + " waterloo_job_companyId,", + " waterloo_job_companySize,", + " waterloo_job_companyDesc", + "]"); + + + + static final Map expFeature2ConfigObj; + + + static { + expFeature2ConfigObj = new HashMap<>(); + expFeature2ConfigObj.put("waterloo_job_jobTitle", new ExtractorBasedFeatureConfig("waterloo_job_jobTitle")); + expFeature2ConfigObj.put("waterloo_job_companyId", new ExtractorBasedFeatureConfig("waterloo_job_companyId")); + expFeature2ConfigObj.put("waterloo_job_companySize", new ExtractorBasedFeatureConfig("waterloo_job_companySize")); + expFeature2ConfigObj.put("waterloo_job_companyDesc", new ExtractorBasedFeatureConfig("waterloo_job_companyDesc")); + } + + static final String feature2ConfigWithTypeStr = String.join("\n", + "features: {", + " waterloo_job_jobTitle : {", + " type: BOOLEAN", + " },", + " waterloo_job_companyId : {", + " type: BOOLEAN", + " default: true", + " },", + " waterloo_job_companySize : {},", + " waterloo_job_companyDesc: {}", + "}"); + + static final Map expFeature2WithTypeConfigObj; + + static { + expFeature2WithTypeConfigObj = new HashMap<>(); + FeatureTypeConfig featureTypeConfig = new FeatureTypeConfig(FeatureType.BOOLEAN); + expFeature2WithTypeConfigObj.put("waterloo_job_jobTitle", + new ExtractorBasedFeatureConfig("waterloo_job_jobTitle", featureTypeConfig)); + expFeature2WithTypeConfigObj.put("waterloo_job_companyId", + new ExtractorBasedFeatureConfig("waterloo_job_companyId", featureTypeConfig, "true", Collections.emptyMap())); + expFeature2WithTypeConfigObj.put("waterloo_job_companySize", new ExtractorBasedFeatureConfig("waterloo_job_companySize")); + expFeature2WithTypeConfigObj.put("waterloo_job_companyDesc", new ExtractorBasedFeatureConfig("waterloo_job_companyDesc")); + } + + static final String feature3ConfigStr = String.join("\n", + "features: {", + " member_sentInvitations_numIgnoredRejectedInvites: {", + " def: \"toNumeric(numIgnoredRejectedInvites)\"", + " type: \"BOOLEAN\"", + " default: 0", + " }", + " member_sentInvitations_numGuestInvites: {", + " def: \"toNumeric(numGuestInvites)\"", + " default: 0", + " }", + " member_sentInvitations_numMemberInvites: {", + " def: \"toNumeric(numMemberInvites)\"", + " }", + "}"); + + static final Map expFeature3ConfigObj; + static { + expFeature3ConfigObj = new HashMap<>(); + String defaultValue = "0"; + FeatureTypeConfig featureTypeConfig = new FeatureTypeConfig(FeatureType.BOOLEAN); + ExpressionBasedFeatureConfig feature1 = new ExpressionBasedFeatureConfig("toNumeric(numIgnoredRejectedInvites)", + defaultValue, featureTypeConfig); + ExpressionBasedFeatureConfig feature2= new ExpressionBasedFeatureConfig("toNumeric(numGuestInvites)", + defaultValue, (FeatureTypeConfig) null); + ExpressionBasedFeatureConfig feature3= new ExpressionBasedFeatureConfig("toNumeric(numMemberInvites)", null); + + expFeature3ConfigObj.put("member_sentInvitations_numIgnoredRejectedInvites", feature1); + expFeature3ConfigObj.put("member_sentInvitations_numGuestInvites", feature2); + expFeature3ConfigObj.put("member_sentInvitations_numMemberInvites", feature3); + } + + static final String feature4ConfigStr = String.join("\n", + "features: {", + " simplePageViewCount: {", + " def: \"pageView\"", + " aggregation: COUNT", + " window: 1d", + " default: 0", + " type: \"BOOLEAN\"", + " }", + " sumPageView1d: {", + " def: \"pageView\"", + " aggregation: COUNT", + " window: 1d", + " filter: \"pageKey = 5\"", + " }", + " maxPV12h: {", + " def: \"pageView\"", + " aggregation: MAX", + " window: 12h", + " groupBy: \"pageKey\"", + " limit: 2", + " }", + " minPV12h: {", + " def: \"pageView\"", + " aggregation: MIN", + " window: 12h", + " groupBy: \"pageKey\"", + " limit: 2", + " }", + " timeSincePV: {", + " def: \"\"", + " aggregation: TIMESINCE", + " window: 5d", + " }", + " nearLine: {", + " def.mvel: \"pageView\"", + " aggregation: MAX", + " windowParameters: {", + " type: FIXED", + " size: 12h", + " }", + " }", + " latestPV: {", + " def: \"pageView\"", + " aggregation: LATEST", + " window: 5d", + " }", + " testMinPoolingAndEmbeddingSize: {", + " def: \"careersJobEmbedding\"", + " filter: \"action IN ('APPLY_OFFSITE', 'APPLY_ONSITE')\"", + " aggregation: MIN_POOLING", + " window: 4d", + " embeddingSize: 200", + " }", + "}"); + + static final Map expFeature4ConfigObj; + static { + expFeature4ConfigObj = new HashMap<>(); + FeatureTypeConfig featureTypeConfig = new FeatureTypeConfig(FeatureType.BOOLEAN); + WindowParametersConfig windowParameters1 = new WindowParametersConfig(WindowType.SLIDING, Duration.ofDays(1), null); + TimeWindowFeatureConfig feature1 = new TimeWindowFeatureConfig(new TypedExpr("pageView", ExprType.SQL), + TimeWindowAggregationType.COUNT, windowParameters1, null, null, null, null, null, null, featureTypeConfig, "0"); + + WindowParametersConfig windowParameters2 = new WindowParametersConfig(WindowType.SLIDING, Duration.ofDays(1), null); + TimeWindowFeatureConfig feature2 = new TimeWindowFeatureConfig("pageView", + TimeWindowAggregationType.COUNT, windowParameters2, "pageKey = 5",null, null, null, null); + + WindowParametersConfig windowParameters3 = new WindowParametersConfig(WindowType.SLIDING, Duration.ofHours(12), null); + TimeWindowFeatureConfig feature3 = new TimeWindowFeatureConfig("pageView", + TimeWindowAggregationType.MAX, windowParameters3, null, "pageKey", 2, null,null); + + WindowParametersConfig windowParameters4 = new WindowParametersConfig(WindowType.SLIDING, Duration.ofHours(12), null); + TimeWindowFeatureConfig feature4 = new TimeWindowFeatureConfig("pageView", + TimeWindowAggregationType.MIN, windowParameters4, null, "pageKey", 2, null,null); + + WindowParametersConfig windowParameters5 = new WindowParametersConfig(WindowType.SLIDING, Duration.ofDays(5), null); + TimeWindowFeatureConfig feature5 = new TimeWindowFeatureConfig("", + TimeWindowAggregationType.TIMESINCE, windowParameters5, null, null, null, null, null); + + WindowParametersConfig windowParameters6 = new WindowParametersConfig(WindowType.FIXED, Duration.ofHours(12), null); + TimeWindowFeatureConfig feature6 = new TimeWindowFeatureConfig("pageView", ExprType.MVEL, + TimeWindowAggregationType.MAX, windowParameters6, null, null, null, null, null, null); + + WindowParametersConfig windowParameters7 = new WindowParametersConfig(WindowType.SLIDING, Duration.ofDays(5), null); + TimeWindowFeatureConfig feature7 = new TimeWindowFeatureConfig("pageView", + TimeWindowAggregationType.LATEST, windowParameters7, null, null, null, null, null); + + WindowParametersConfig windowParameters8 = new WindowParametersConfig(WindowType.SLIDING, Duration.ofDays(4), null); + TimeWindowFeatureConfig feature8 = new TimeWindowFeatureConfig( + new TypedExpr("careersJobEmbedding", ExprType.SQL), + TimeWindowAggregationType.MIN_POOLING, windowParameters8, + new TypedExpr("action IN ('APPLY_OFFSITE', 'APPLY_ONSITE')", ExprType.SQL), + null, null, null, null, 200); + + expFeature4ConfigObj.put("simplePageViewCount", feature1); + expFeature4ConfigObj.put("sumPageView1d", feature2); + expFeature4ConfigObj.put("maxPV12h", feature3); + expFeature4ConfigObj.put("minPV12h", feature4); + expFeature4ConfigObj.put("timeSincePV", feature5); + expFeature4ConfigObj.put("nearLine", feature6); + expFeature4ConfigObj.put("latestPV", feature7); + expFeature4ConfigObj.put("testMinPoolingAndEmbeddingSize", feature8); + } + + static final String feature5ConfigWithTypeStr = String.join("\n", + "features: {", + " waterloo_job_jobTitleV2 : {", + " parameters: {", + " param1 : [waterlooCompany_terms_hashed, waterlooCompany_values]", + " }", + " default: true", + " type: BOOLEAN", + " }", + " }"); + + static final Map expFeature5WithTypeConfigObj; + + static { + expFeature5WithTypeConfigObj = new HashMap<>(); + Map parameters = ImmutableMap.of("param1", "[\"waterlooCompany_terms_hashed\",\"waterlooCompany_values\"]"); + expFeature5WithTypeConfigObj.put("waterloo_job_jobTitleV2", + new ExtractorBasedFeatureConfig("waterloo_job_jobTitleV2", new FeatureTypeConfig(FeatureType.BOOLEAN), "true", parameters)); + } +} diff --git a/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/common/FeatureTypeConfigBuilderTest.java b/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/common/FeatureTypeConfigBuilderTest.java new file mode 100644 index 000000000..78ab9f883 --- /dev/null +++ b/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/common/FeatureTypeConfigBuilderTest.java @@ -0,0 +1,77 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.producer.common; + +import com.linkedin.feathr.core.config.producer.common.FeatureTypeConfig; +import com.typesafe.config.Config; +import com.typesafe.config.ConfigFactory; +import org.testng.annotations.Test; + +import static com.linkedin.feathr.core.configbuilder.typesafe.producer.common.FeatureTypeFixture.*; +import static org.testng.Assert.*; + + +/** + * Tests for {@link FeatureTypeConfigBuilder} + */ +public class FeatureTypeConfigBuilderTest { + + @Test + public void testOnlyType() { + testFeatureTypeConfig(simpleTypeConfigStr, expSimpleTypeConfigObj); + } + + @Test + public void testTypeWithDocumentation() { + testFeatureTypeConfig(simpleTypeWithDocConfigStr, expSimpleTypeWithDocConfigObj); + } + + @Test + public void testTensorTypeWithUnknownShape() { + testFeatureTypeConfig(tensorTypeWithUnknownShapeConfigStr, expTensorTypeWithUnknownShapeConfig); + } + + @Test + public void test0DSparseTensorType() { + testFeatureTypeConfig(zeroDimSparseTensorConfigStr, expZeroDimSparseTensorConfig); + } + + @Test(expectedExceptions = RuntimeException.class) + public void testInvalidType() { + createFeatureTypeConfig(invalidTypeConfigStr); + } + + @Test(expectedExceptions = RuntimeException.class) + public void testInvalidTensorCategory() { + createFeatureTypeConfig(invalidTensorTypeConfigStr); + } + + @Test(expectedExceptions = RuntimeException.class) + public void testMissingType() { + createFeatureTypeConfig(missingTypeConfigStr); + } + + @Test(expectedExceptions = RuntimeException.class) + public void testMissingValType() { + createFeatureTypeConfig(missingValType); + } + + @Test(expectedExceptions = RuntimeException.class) + public void testTensorTypeSizeMismatchException() { + createFeatureTypeConfig(shapeAndDimSizeMismatchTypeConfigStr); + } + + @Test(expectedExceptions = RuntimeException.class) + public void tesNonIntShapeValType() { + createFeatureTypeConfig(nonIntShapeConfigStr); + } + + + private FeatureTypeConfig createFeatureTypeConfig(String configStr) { + Config fullConfig = ConfigFactory.parseString(configStr); + return FeatureTypeConfigBuilder.build(fullConfig); + } + + private void testFeatureTypeConfig(String configStr, FeatureTypeConfig expFeatureTypeConfig) { + FeatureTypeConfig featureTypeConfig = createFeatureTypeConfig(configStr); + assertEquals(featureTypeConfig, expFeatureTypeConfig); + } +} diff --git a/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/common/FeatureTypeFixture.java b/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/common/FeatureTypeFixture.java new file mode 100644 index 000000000..c49f3c95f --- /dev/null +++ b/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/common/FeatureTypeFixture.java @@ -0,0 +1,81 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.producer.common; + +import com.linkedin.feathr.core.config.producer.common.FeatureTypeConfig; +import com.linkedin.feathr.core.config.producer.definitions.FeatureType; +import java.util.Arrays; + + +class FeatureTypeFixture { + + static final String simpleTypeConfigStr = "type: {type: VECTOR}"; + static final FeatureTypeConfig expSimpleTypeConfigObj = new FeatureTypeConfig(FeatureType.DENSE_VECTOR); + + static final String simpleTypeWithDocConfigStr = "type: {type: BOOLEAN}"; + static final FeatureTypeConfig expSimpleTypeWithDocConfigObj = + new FeatureTypeConfig.Builder().setFeatureType(FeatureType.BOOLEAN) + .build(); + + static final String tensorTypeWithUnknownShapeConfigStr = String.join("\n", + " type: {", + " type: \"TENSOR\"", + " tensorCategory: \"DENSE\"", + " dimensionType: [\"INT\", \"INT\"]", + " valType:FLOAT", + " }"); + static final FeatureTypeConfig expTensorTypeWithUnknownShapeConfig = + new FeatureTypeConfig.Builder().setFeatureType(FeatureType.DENSE_TENSOR) + .setDimensionTypes(Arrays.asList("INT", "INT")) + .setValType("FLOAT") + .build(); + + static final String zeroDimSparseTensorConfigStr = String.join("\n", + " type: {", + " type: \"TENSOR\"", + " tensorCategory: \"SPARSE\"", + " valType:FLOAT", + " }"); + static final FeatureTypeConfig expZeroDimSparseTensorConfig = + new FeatureTypeConfig.Builder().setFeatureType(FeatureType.SPARSE_TENSOR) + .setValType("FLOAT") + .build(); + + + static final String invalidTypeConfigStr = "type: {type: UNKOWN_TYPE, doc: \"this is doc\"}"; + + // if tensorCategory is specified, the type should be TENSOR only + static final String invalidTensorTypeConfigStr = String.join("\n", + " type: {", + " type: \"VECTOR\"", + " tensorCategory: \"DENSE\"", + " shape: [10]", + " dimensionType: [\"INT\"]", + " }"); + + static final String missingTypeConfigStr = "type: {shape:[10], doc: \"this is doc\"}"; + + static final String missingValType = String.join("\n", + " type: {", + " type: \"TENSOR\"", + " tensorCategory: \"DENSE\"", + " shape: [10]", + " dimensionType: [\"INT\"]", + " }"); + + static final String shapeAndDimSizeMismatchTypeConfigStr = String.join("\n", + " type: {", + " type: \"TENSOR\"", + " tensorCategory: \"DENSE\"", + " shape: [10]", + " dimensionType: [\"INT\", \"INT\"]", + " valType:FLOAT", + " }"); + + static final String nonIntShapeConfigStr = String.join("\n", + " type: {", + " type: \"TENSOR\"", + " tensorCategory: \"DENSE\"", + " shape: [FLOAT]", + " dimensionType: [\"INT\", \"INT\"]", + " valType:FLOAT", + " }"); +} diff --git a/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/common/KeyListExtractorTest.java b/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/common/KeyListExtractorTest.java new file mode 100644 index 000000000..3160b5599 --- /dev/null +++ b/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/common/KeyListExtractorTest.java @@ -0,0 +1,52 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.producer.common; + +import com.typesafe.config.ConfigException; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import com.linkedin.feathr.core.config.producer.common.KeyListExtractor; +import org.testng.annotations.Test; + +import static org.testng.Assert.*; + +public class KeyListExtractorTest { + private KeyListExtractor _keyListConverter = KeyListExtractor.getInstance(); + + @Test(description = "test get single key from HOCON expression, and verify that the quote does not influence the parsing") + public void testSingleKeyInHocon() { + String keyExpression1 = "key1"; + String keyExpression2 = "\"key1\""; + List keysFromExpression1 = _keyListConverter.extractFromHocon(keyExpression1); + assertEquals(keysFromExpression1, Collections.singletonList(keyExpression1)); + assertEquals(keysFromExpression1, _keyListConverter.extractFromHocon(keyExpression2)); + } + + @Test(description = "test get single key from HOCON expression with complex quote notation") + public void testSingleKeyInHocon2() { + String keyExpression = "\"toCompoundKey({\\\"jobPosting\\\" : toUrn(\\\"jobPosting\\\", key[0]), \\\"member\\\" : toUrn(\\\"member\\\", key[1])})\""; + String expectedResult = "toCompoundKey({\"jobPosting\" : toUrn(\"jobPosting\", key[0]), \"member\" : toUrn(\"member\", key[1])})"; + List keys = _keyListConverter.extractFromHocon(keyExpression); + assertEquals(keys, Collections.singletonList(expectedResult)); + } + + @Test(description = "test get single key from invalid HOCON expression", expectedExceptions = ConfigException.class) + public void testSingleKeyInHocon3() { + String keyExpression = "toCompoundKey({\"jobPosting\" : toUrn(\"jobPosting\", key[0]), \"member\" : toUrn(\"member\", key[1])})"; + List keys = _keyListConverter.extractFromHocon(keyExpression); + assertEquals(keys, Collections.singletonList(keyExpression)); + } + + @Test(description = "test get multiple key from HOCON expression") + public void testMultipleKeyInHocon() { + String keyExpression = "[\"key1\", \"key2\"]"; + List keys = _keyListConverter.extractFromHocon(keyExpression); + assertEquals(keys, Arrays.asList("key1", "key2")); + } + + @Test(description = "test get multiple key from HOCON expression") + public void testMultipleKeyInHocon2() { + String keyExpression = "[key1, key2]"; + List keys = _keyListConverter.extractFromHocon(keyExpression); + assertEquals(keys, Arrays.asList("key1", "key2")); + } +} diff --git a/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/derivations/DerivationConfigBuilderTest.java b/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/derivations/DerivationConfigBuilderTest.java new file mode 100644 index 000000000..9ff500210 --- /dev/null +++ b/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/derivations/DerivationConfigBuilderTest.java @@ -0,0 +1,81 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.producer.derivations; + +import com.linkedin.feathr.core.config.producer.derivations.DerivationConfig; +import com.linkedin.feathr.core.configbuilder.ConfigBuilderException; +import com.typesafe.config.Config; +import com.typesafe.config.ConfigFactory; +import org.testng.annotations.Test; + +import static org.testng.Assert.assertEquals; + + +public class DerivationConfigBuilderTest { + + @Test + public void testSimpleDerivation() { + testDerivation(DerivationsFixture.derivation1ConfigStr, DerivationsFixture.expDerivation1ConfigObj); + } + + @Test + public void testSimpleDerivationWithSpecialCharacters() { + testDerivation( + DerivationsFixture.derivation1ConfigStrWithSpecialChars, DerivationsFixture.expDerivation1ConfigObjWithSpecialChars); + } + + @Test + public void testSimpleDerivationWithSqlExpr() { + testDerivation( + DerivationsFixture.derivationConfigStrWithSqlExpr, DerivationsFixture.expDerivationConfigObjWithSqlExpr); + } + + @Test + public void testSimpleDerivationWithType() { + testDerivation(DerivationsFixture.derivationConfigStrWithType, DerivationsFixture.expDerivationConfigObjWithDef); + } + + @Test + public void testDerivationWithMvelExpr() { + testDerivation(DerivationsFixture.derivation2ConfigStr, DerivationsFixture.expDerivation2ConfigObj); + } + + @Test + public void testDerivationWithExtractor() { + testDerivation(DerivationsFixture.derivation3ConfigStr, DerivationsFixture.expDerivation3ConfigObj); + } + + @Test + public void testDerivationWithSqlExpr() { + testDerivation(DerivationsFixture.derivation4ConfigStr, DerivationsFixture.expDerivation4ConfigObj); + } + + @Test + public void testSequentialJoinConfig() { + testDerivation(DerivationsFixture.sequentialJoin1ConfigStr, DerivationsFixture.expSequentialJoin1ConfigObj); + } + + @Test(description = "test sequential join config where base feature has outputKey and transformation field") + public void testSequentialJoinConfig2() { + testDerivation(DerivationsFixture.sequentialJoin2ConfigStr, DerivationsFixture.expSequentialJoin2ConfigObj); + } + + @Test(description = "test sequential join config with transformation class") + public void testSequentialJoinWithTransformationClass() { + testDerivation( + DerivationsFixture.sequentialJoinWithTransformationClassConfigStr, DerivationsFixture.expSequentialJoinWithTransformationClassConfigObj); + } + + @Test(description = "test sequential join config with both transformation and transformationClass", expectedExceptions = ConfigBuilderException.class) + public void testSequentialJoinWithInvalidTransformation() { + Config fullConfig = ConfigFactory.parseString(DerivationsFixture.sequentialJoinWithInvalidTransformationConfigStr); + DerivationConfigBuilder.build("seq_join_feature", fullConfig); + } + + private void testDerivation(String configStr, DerivationConfig expDerivationConfig) { + Config fullConfig = ConfigFactory.parseString(configStr); + String derivedFeatureName = fullConfig.root().keySet().iterator().next(); + + DerivationConfig obsDerivationConfigObj = DerivationConfigBuilder.build(derivedFeatureName, fullConfig); + + assertEquals(obsDerivationConfigObj, expDerivationConfig); + } +} diff --git a/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/derivations/DerivationsConfigBuilderTest.java b/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/derivations/DerivationsConfigBuilderTest.java new file mode 100644 index 000000000..e01c542dc --- /dev/null +++ b/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/derivations/DerivationsConfigBuilderTest.java @@ -0,0 +1,14 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.producer.derivations; + +import com.linkedin.feathr.core.configbuilder.typesafe.AbstractConfigBuilderTest; +import org.testng.annotations.Test; + + +public class DerivationsConfigBuilderTest extends AbstractConfigBuilderTest { + + @Test + public void derivationsTest() { + testConfigBuilder( + DerivationsFixture.derivationsConfigStr, DerivationsConfigBuilder::build, DerivationsFixture.expDerivationsConfigObj); + } +} diff --git a/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/derivations/DerivationsFixture.java b/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/derivations/DerivationsFixture.java new file mode 100644 index 000000000..27a2d9bc7 --- /dev/null +++ b/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/derivations/DerivationsFixture.java @@ -0,0 +1,252 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.producer.derivations; + +import com.linkedin.feathr.core.config.producer.ExprType; +import com.linkedin.feathr.core.config.producer.TypedExpr; +import com.linkedin.feathr.core.config.producer.common.FeatureTypeConfig; +import com.linkedin.feathr.core.config.producer.definitions.FeatureType; +import com.linkedin.feathr.core.config.producer.derivations.BaseFeatureConfig; +import com.linkedin.feathr.core.config.producer.derivations.DerivationConfig; +import com.linkedin.feathr.core.config.producer.derivations.DerivationConfigWithExpr; +import com.linkedin.feathr.core.config.producer.derivations.DerivationConfigWithExtractor; +import com.linkedin.feathr.core.config.producer.derivations.DerivationsConfig; +import com.linkedin.feathr.core.config.producer.derivations.KeyedFeature; +import com.linkedin.feathr.core.config.producer.derivations.SequentialJoinConfig; +import com.linkedin.feathr.core.config.producer.derivations.SimpleDerivationConfig; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + + +class DerivationsFixture { + + static final String derivation1ConfigStr = "featureX: \"featureA + featureB\""; + + static final String derivation1ConfigStrWithSpecialChars = "\"fea:ture.X\": \"\"fe.atureA\" + featureB\""; + + static final SimpleDerivationConfig expDerivation1ConfigObj = + new SimpleDerivationConfig(new TypedExpr("featureA + featureB", ExprType.MVEL)); + + static final SimpleDerivationConfig expDerivation1ConfigObjWithSpecialChars = + new SimpleDerivationConfig("fe.atureA + featureB"); + + static final FeatureTypeConfig expectedFeatureTypeConfig = + new FeatureTypeConfig.Builder().setFeatureType(FeatureType.DENSE_TENSOR) + .setShapes(Collections.singletonList(10)) + .setDimensionTypes(Collections.singletonList("INT")) + .setValType("FLOAT") + .build(); + + static final String derivationConfigStrWithType = String.join("\n", + "abuse_member_invitation_inboundOutboundSkew:{", + " definition: \"case when abuse_member_invitation_numInviters = 0 then -1 else abuse_member_invitation_numInvites/abuse_member_invitation_numInviters end\"", + " type: {", + " type: \"DENSE_TENSOR\"", + " shape: [10]", + " dimensionType: [\"INT\"]", + " valType: \"FLOAT\"", + " }", + "}"); + + static final String derivationConfigStrWithSqlExpr = String.join("\n", + "abuse_member_invitation_inboundOutboundSkew:{", + " sqlExpr: \"case when abuse_member_invitation_numInviters = 0 then -1 else abuse_member_invitation_numInvites/abuse_member_invitation_numInviters end\"", + " type: {", + " type: \"DENSE_TENSOR\"", + " shape: [10]", + " dimensionType: [\"INT\"]", + " valType: \"FLOAT\"", + " }", + "}"); + + static final SimpleDerivationConfig expDerivationConfigObjWithSqlExpr = + new SimpleDerivationConfig(new TypedExpr("case when abuse_member_invitation_numInviters = 0 then -1 else " + + "abuse_member_invitation_numInvites/abuse_member_invitation_numInviters end", + ExprType.SQL), expectedFeatureTypeConfig); + + static final SimpleDerivationConfig expDerivationConfigObjWithDef = + new SimpleDerivationConfig(new TypedExpr("case when abuse_member_invitation_numInviters = 0 then -1 else " + + "abuse_member_invitation_numInvites/abuse_member_invitation_numInviters end", + ExprType.MVEL), expectedFeatureTypeConfig); + + static final String derivation2ConfigStr = String.join("\n", + "featureZ: {", + " key: [m, j]", + " inputs: {", + " foo: {key: m, feature: featureA},", + " bar: {key: j, feature: featureC}", + " }", + " definition: \"cosineSimilarity(foo, bar)\"", + " type: {", + " type: \"DENSE_TENSOR\"", + " shape: [10]", + " dimensionType: [\"INT\"]", + " valType: \"FLOAT\"", + " }", + "}"); + + static final DerivationConfigWithExpr expDerivation2ConfigObj; + static { + List keys = Arrays.asList("m", "j"); + Map inputs = new HashMap<>(); + inputs.put("foo", new KeyedFeature("m", "featureA")); + inputs.put("bar", new KeyedFeature("j", "featureC")); + + String definition = "cosineSimilarity(foo, bar)"; + expDerivation2ConfigObj = new DerivationConfigWithExpr(keys, inputs, new TypedExpr(definition, ExprType.MVEL), expectedFeatureTypeConfig); + } + + static final String derivation3ConfigStr = String.join("\n", + "jfu_member_placeSimTopK: {", + " key: [member]", + " inputs: [{key: member, feature: jfu_resolvedPreference_location}]", + " class: \"com.linkedin.jymbii.nice.derived.MemberPlaceSimTopK\"", + " type: {", + " type: \"DENSE_TENSOR\"", + " shape: [10]", + " dimensionType: [\"INT\"]", + " valType: \"FLOAT\"", + " }", + "}"); + + static final DerivationConfigWithExtractor expDerivation3ConfigObj; + static { + List keys = Collections.singletonList("member"); + List inputs = Collections.singletonList( + new KeyedFeature("member", "jfu_resolvedPreference_location")); + String className = "com.linkedin.jymbii.nice.derived.MemberPlaceSimTopK"; + expDerivation3ConfigObj = new DerivationConfigWithExtractor(keys, inputs, className, expectedFeatureTypeConfig); + } + + static final String derivation4ConfigStr = String.join("\n", + "sessions_v2_macrosessions_sum_sqrt_7d: {", + " key: id", + " inputs: {", + " sessions_v2_macrosessions_sum_7d: {key: id, feature: sessions_v2_macrosessions_sum_7d},", + " }\n", + " definition.sqlExpr: \"sqrt(sessions_v2_macrosessions_sum_7d)\"", + " type: {", + " type: \"DENSE_TENSOR\"", + " shape: [10]", + " dimensionType: [\"INT\"]", + " valType: \"FLOAT\"", + " }", + "}"); + + static final DerivationConfigWithExpr expDerivation4ConfigObj; + static { + List keys = Collections.singletonList("id"); + Map inputs = new HashMap<>(); + inputs.put("sessions_v2_macrosessions_sum_7d", + new KeyedFeature("id", "sessions_v2_macrosessions_sum_7d")); + + String definition = "sqrt(sessions_v2_macrosessions_sum_7d)"; + expDerivation4ConfigObj = new DerivationConfigWithExpr(keys, inputs, new TypedExpr(definition, ExprType.SQL), expectedFeatureTypeConfig); + } + + static final String sequentialJoin1ConfigStr = String.join("\n", + "seq_join_feature1: { ", + " key: \"x\" ", + " join: { ", + " base: { key: x, feature: MemberIndustryId } ", + " expansion: { key: skillId, feature: MemberIndustryName } ", + " } ", + " aggregation:\"\"", + "}"); + + static final SequentialJoinConfig expSequentialJoin1ConfigObj; + static { + List keys = Collections.singletonList("x"); + String baseKeyExpr = "\"x\""; + BaseFeatureConfig base = new BaseFeatureConfig(baseKeyExpr, "MemberIndustryId", null, null, null); + KeyedFeature expansion = new KeyedFeature("skillId", "MemberIndustryName"); + expSequentialJoin1ConfigObj = new SequentialJoinConfig(keys, base, expansion, ""); + } + + static final String sequentialJoin2ConfigStr = String.join("\n", + "seq_join_feature2: { ", + " key: \"x\"", + " join: { ", + " base: { key: x,", + " feature: MemberIndustryId,", + " outputKey: x,", + " transformation: \"import com.linkedin.frame.MyFeatureUtils; MyFeatureUtils.dotProduct(MemberIndustryId);\"} ", + " expansion: { key: key.entityUrn, feature: MemberIndustryName }", + " } ", + " aggregation:\"ELEMENTWISE_MAX\"", + " type: {", + " type: \"DENSE_TENSOR\"", + " shape: [10]", + " dimensionType: [\"INT\"]", + " valType: \"FLOAT\"", + " }", + "}"); + + static final SequentialJoinConfig expSequentialJoin2ConfigObj; + static { + List keys = Collections.singletonList("x"); + String baseKeyExpr = "\"x\""; + List baseOutputKeys = Collections.singletonList("x"); + BaseFeatureConfig base = new BaseFeatureConfig(baseKeyExpr, "MemberIndustryId", baseOutputKeys, + "import com.linkedin.frame.MyFeatureUtils; MyFeatureUtils.dotProduct(MemberIndustryId);", null); + KeyedFeature expansion = new KeyedFeature("\"key.entityUrn\"", "MemberIndustryName"); + expSequentialJoin2ConfigObj = new SequentialJoinConfig(keys, base, expansion, "ELEMENTWISE_MAX", expectedFeatureTypeConfig); + } + + static final String sequentialJoinWithTransformationClassConfigStr = String.join("\n", + "seq_join_feature: { ", + " key: \"x\"", + " join: { ", + " base: { key: x,", + " feature: MemberIndustryId,", + " outputKey: x,", + " transformationClass: \"com.linkedin.frame.MyFeatureTransformer\"} ", + " expansion: { key: key.entityUrn, feature: MemberIndustryName }", + " } ", + " aggregation:\"ELEMENTWISE_MAX\"", + "}"); + + static final SequentialJoinConfig expSequentialJoinWithTransformationClassConfigObj; + static { + List keys = Collections.singletonList("x"); + String baseKeyExpr = "\"x\""; + List baseOutputKeys = Collections.singletonList("x"); + BaseFeatureConfig base = new BaseFeatureConfig(baseKeyExpr, "MemberIndustryId", baseOutputKeys, null, + "com.linkedin.frame.MyFeatureTransformer"); + KeyedFeature expansion = new KeyedFeature("\"key.entityUrn\"", "MemberIndustryName"); + expSequentialJoinWithTransformationClassConfigObj = new SequentialJoinConfig(keys, base, expansion, "ELEMENTWISE_MAX"); + } + + static final String sequentialJoinWithInvalidTransformationConfigStr = String.join("\n", + "seq_join_feature: { ", + " key: \"x\"", + " join: { ", + " base: { key: x,", + " feature: MemberIndustryId,", + " outputKey: x,", + " transformation: \"import com.linkedin.frame.MyFeatureUtils; MyFeatureUtils.dotProduct(MemberIndustryId);\"", + " transformationClass: \"com.linkedin.frame.MyFeatureTransformer\"} ", + " expansion: { key: key.entityUrn, feature: MemberIndustryName }", + " } ", + " aggregation:\"ELEMENTWISE_MAX\"", + "}"); + + static final String derivationsConfigStr = String.join("\n", + "derivations: {", + derivation1ConfigStr, + derivation2ConfigStr, + derivation3ConfigStr, + "}"); + + static final DerivationsConfig expDerivationsConfigObj; + static { + Map derivations = new HashMap<>(); + + derivations.put("featureX", expDerivation1ConfigObj); + derivations.put("featureZ", expDerivation2ConfigObj); + derivations.put("jfu_member_placeSimTopK", expDerivation3ConfigObj); + + expDerivationsConfigObj = new DerivationsConfig(derivations); + } +} diff --git a/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/sources/PinotConfigBuilderTest.java b/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/sources/PinotConfigBuilderTest.java new file mode 100644 index 000000000..87b0bbfa7 --- /dev/null +++ b/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/sources/PinotConfigBuilderTest.java @@ -0,0 +1,88 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.producer.sources; + +import com.linkedin.feathr.core.config.producer.sources.PinotConfig; +import com.linkedin.feathr.core.configbuilder.ConfigBuilderException; +import com.typesafe.config.Config; +import com.typesafe.config.ConfigFactory; +import org.testng.Assert; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import static com.linkedin.feathr.core.utils.Utils.*; + + +public class PinotConfigBuilderTest { + static final String pinotSourceName = "pinotTestSource"; + static final String resourceName = "recentMemberActionsPinotQuery"; + static final String queryTemplate = "SELECT verb, object, verbAttributes, timeStampSec FROM RecentMemberActions WHERE actorId IN (?)"; + static final String[] queryArguments = new String[]{"key[0]"}; + static final String[] queryKeyColumns = new String[]{"actorId"}; + + static final PinotConfig expectedPinotConfig = new PinotConfig(pinotSourceName, resourceName, queryTemplate, queryArguments, queryKeyColumns); + + static final String goodPinotSourceConfigStr = + String.join("\n", "pinotTestSource {", + " type: PINOT", + " resourceName : \"recentMemberActionsPinotQuery\"", + " queryTemplate : \"SELECT verb, object, verbAttributes, timeStampSec FROM RecentMemberActions WHERE actorId IN (?)\"", + " queryArguments : [\"key[0]\"]", + " queryKeyColumns: [\"actorId\"]", + "}"); + + // placeholder for key expression is not wrapped inside IN clause + static final String badPinotSourceConfigStr1 = + String.join("\n", "pinotTestSource {", + " type: PINOT", + " resourceName : \"recentMemberActionsPinotQuery\"", + " queryTemplate : \"SELECT verb, object, verbAttributes, timeStampSec FROM RecentMemberActions WHERE actorId = ?\"", + " queryArguments : [\"key[0]\"]", + " queryKeyColumns: [\"actorId\"]", + "}"); + + // queryArgument count does not match the place holder count in queryTemplate + static final String badPinotSourceConfigStr2 = + String.join("\n", "pinotTestSource {", + " type: PINOT", + " resourceName : \"recentMemberActionsPinotQuery\"", + " queryTemplate : \"SELECT verb, object, verbAttributes, timeStampSec FROM RecentMemberActions WHERE actorId IN (?)\"", + " queryArguments : [\"key[0]\", \"key[1]\"]", + " queryKeyColumns: [\"actorId\"]", + "}"); + + // column names in queryKeyColumns are not unique + static final String badPinotSourceConfigStr3 = + String.join("\n", "pinotTestSource {", + " type: PINOT", + " resourceName : \"recentMemberActionsPinotQuery\"", + " queryTemplate : \"SELECT verb, object, verbAttributes, timeStampSec FROM RecentMemberActions WHERE actorId IN (?) AND object IN (?)\"", + " queryArguments : [\"key[0]\", \"key[1]\"]", + " queryKeyColumns: [\"actorId\", \"actorId\"]", + "}"); + + @DataProvider() + public Object[][] dataProviderPinotConfigStr() { + return new Object[][]{ + {badPinotSourceConfigStr1}, + {badPinotSourceConfigStr2}, + {badPinotSourceConfigStr3} + }; + } + + @Test + public void pinotGoodConfigTest() { + Config fullConfig = ConfigFactory.parseString(goodPinotSourceConfigStr); + String configName = fullConfig.root().keySet().iterator().next(); + Config config = fullConfig.getConfig(quote(configName)); + + Assert.assertEquals(PinotConfigBuilder.build("pinotTestSource", config), expectedPinotConfig); + } + + @Test(description = "Tests Pinot config validation", dataProvider = "dataProviderPinotConfigStr", + expectedExceptions = ConfigBuilderException.class) + public void pinotConfigTest(String sourceConfigStr) { + Config fullConfig = ConfigFactory.parseString(sourceConfigStr); + String configName = fullConfig.root().keySet().iterator().next(); + Config config = fullConfig.getConfig(quote(configName)); + PinotConfigBuilder.build("pinotTestSource", config); + } +} \ No newline at end of file diff --git a/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/sources/SourceConfigBuilderTest.java b/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/sources/SourceConfigBuilderTest.java new file mode 100644 index 000000000..f12cadf61 --- /dev/null +++ b/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/sources/SourceConfigBuilderTest.java @@ -0,0 +1,168 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.producer.sources; + +import com.linkedin.feathr.core.configbuilder.typesafe.AbstractConfigBuilderTest; +import com.linkedin.feathr.core.config.ConfigObj; +import com.linkedin.feathr.core.configbuilder.ConfigBuilderException; +import com.typesafe.config.Config; +import java.util.function.BiFunction; +import org.testng.annotations.Test; + + +public class SourceConfigBuilderTest extends AbstractConfigBuilderTest { + + BiFunction configBuilder = SourceConfigBuilder::build; + + @Test(description = "Tests HDFS config without 'type' field") + public void hdfsConfigTest1() { + testConfigBuilder(SourcesFixture.hdfsSource1ConfigStr, configBuilder, SourcesFixture.expHdfsSource1ConfigObj); + } + + @Test(description = "Tests HDFS config with 'type' field") + public void hdfsConfigTest2() { + testConfigBuilder(SourcesFixture.hdfsSource2ConfigStr, configBuilder, SourcesFixture.expHdfsSource2ConfigObj); + } + + @Test(description = "Tests HDFS config with Dali URI") + public void hdfsConfigTest3() { + testConfigBuilder(SourcesFixture.hdfsSource3ConfigStr, configBuilder, SourcesFixture.expHdfsSource3ConfigObj); + } + + @Test(description = "Tests HDFS config with sliding time window") + public void hdfsConfigTest4() { + testConfigBuilder(SourcesFixture.hdfsSource4ConfigStr, configBuilder, SourcesFixture.expHdfsSource4ConfigObj); + } + + @Test(description = "Tests HDFS config with timePartitionPattern") + public void hdfsConfigTest5WithTimePartitionPattern() { + testConfigBuilder( + SourcesFixture.hdfsSource5ConfigStrWithTimePartitionPattern, configBuilder, SourcesFixture.expHdfsSource5ConfigObjWithTimePartitionPattern); + } + + @Test(description = "Tests HDFS config with sliding time window") + public void hdfsConfigTest6WithLegacyTimeWindowParameters() { + testConfigBuilder( + SourcesFixture.hdfsSource6ConfigStrWithLegacyTimeWindowParameters, configBuilder, SourcesFixture.expHdfsSource6ConfigObjWithLegacyTimeWindowParameters); + } + + @Test(description = "It should fail if both timePartitionPattern and isTimeSeries is set.", expectedExceptions = ConfigBuilderException.class) + public void hdfsConfigTestWithTimePartitionPatternAndIsTimeSeries() { + buildConfig(SourcesFixture.invalidHdfsSourceconfigStrWithTimePartitionPatternAndIsTimeSeries, configBuilder); + } + + @Test(description = "It should fail if both hasTimeSnapshot and isTimeSeries is set.", expectedExceptions = ConfigBuilderException.class) + public void hdfsConfigTestWithHasTimeSnapshotAndIsTimeSeries() { + buildConfig(SourcesFixture.invalidHdfsSourceconfigStrWithHasTimeSnapshotAndIsTimeSeries, configBuilder); + } + + @Test(description = "Tests Espresso config") + public void espressoConfigTest1() { + testConfigBuilder(SourcesFixture.espressoSource1ConfigStr, configBuilder, SourcesFixture.expEspressoSource1ConfigObj); + } + + @Test(description = "Tests Venice config with Avro key") + public void veniceConfigTest1() { + testConfigBuilder(SourcesFixture.veniceSource1ConfigStr, configBuilder, SourcesFixture.expVeniceSource1ConfigObj); + } + + @Test(description = "Tests Venice config with integer key") + public void veniceConfigTest2() { + testConfigBuilder(SourcesFixture.veniceSource2ConfigStr, configBuilder, SourcesFixture.expVeniceSource2ConfigObj); + } + + @Test(description = "Tests RestLi config with entity type and path spec") + public void restliConfigTest1() { + testConfigBuilder(SourcesFixture.restliSource1ConfigStr, configBuilder, SourcesFixture.expRestliSource1ConfigObj); + } + + @Test(description = "Tests RestLi config with entity type and REST request params containing 'json' object") + public void restliConfigTest2() { + testConfigBuilder(SourcesFixture.restliSource2ConfigStr, configBuilder, SourcesFixture.expRestliSource2ConfigObj); + } + + @Test(description = "Tests RestLi config with entity type and REST request params containing 'jsonArray' array") + public void restliConfigTest3() { + testConfigBuilder(SourcesFixture.restliSource3ConfigStr, configBuilder, SourcesFixture.expRestliSource3ConfigObj); + } + + @Test(description = "Tests RestLi config with key expression, REST request params containing 'mvel' expression") + public void restliConfigTest4() { + testConfigBuilder(SourcesFixture.restliSource4ConfigStr, configBuilder, SourcesFixture.expRestliSource4ConfigObj); + } + + @Test(description = "Tests RestLi config with entity type and " + + "REST request params containg 'json' whose value is a string enclosing an object") + public void restliConfigTest5() { + testConfigBuilder(SourcesFixture.restliSource5ConfigStr, configBuilder, SourcesFixture.expRestliSource5ConfigObj); + } + + @Test(description = "Tests RestLi config with entity type and REST request params containg 'json' object" + + "but the 'json' object is empty.") + public void restliConfigTest6() { + testConfigBuilder(SourcesFixture.restliSource6ConfigStr, configBuilder, SourcesFixture.expRestliSource6ConfigObj); + } + + @Test(description = "Tests RestLi config with entity type and REST request params containing 'jsonArray' array," + + " but the 'json' array is empty") + public void restliConfigTest7() { + testConfigBuilder(SourcesFixture.restliSource7ConfigStr, configBuilder, SourcesFixture.expRestliSource7ConfigObj); + } + + @Test(description = "Tests RestLi config with finder field") + public void restliConfigTest8() { + testConfigBuilder(SourcesFixture.restliSource8ConfigStr, configBuilder, SourcesFixture.expRestliSource8ConfigObj); + } + + @Test(description = "Tests RestLi config with both keyExpr and finder field") + public void restliConfigTest9() { + testConfigBuilder(SourcesFixture.restliSource9ConfigStr, configBuilder, SourcesFixture.expRestliSource9ConfigObj); + } + + @Test(description = "Tests RestLi config missing both keyExpr and finder fields results in an error", expectedExceptions = ConfigBuilderException.class) + public void restliConfigTest10() { + testConfigBuilder(SourcesFixture.restliSource10ConfigStr, configBuilder, null); + } + + @Test(description = "Tests Kafka config") + public void kafkaConfigTest1() { + testConfigBuilder(SourcesFixture.kafkaSource1ConfigStr, configBuilder, SourcesFixture.expKafkaSource1ConfigObj); + } + + @Test(description = "Tests Kafka config with sliding window aggregation") + public void kafkaConfigTest2() { + testConfigBuilder(SourcesFixture.kafkaSource2ConfigStr, configBuilder, SourcesFixture.expKafkaSource2ConfigObj); + } + + @Test(description = "Tests RocksDB config with keyExpr field") + public void rocksDbConfigTest1() { + testConfigBuilder(SourcesFixture.rocksDbSource1ConfigStr, configBuilder, SourcesFixture.expRocksDbSource1ConfigObj); + } + + @Test(description = "Tests RocksDB config without keyExpr field") + public void rocksDbConfigTest2() { + testConfigBuilder(SourcesFixture.rocksDbSource2ConfigStr, configBuilder, SourcesFixture.expRocksDbSource2ConfigObj); + } + + @Test(description = "Tests PassThrough config") + public void passThroughConfigTest1() { + testConfigBuilder( + SourcesFixture.passThroughSource1ConfigStr, configBuilder, SourcesFixture.expPassThroughSource1ConfigObj); + } + + @Test(description = "Tests Couchbase config") + public void couchbaseConfigTest1() { + testConfigBuilder( + SourcesFixture.couchbaseSource1ConfigStr, configBuilder, SourcesFixture.expCouchbaseSource1ConfigObj); + } + + @Test(description = "Tests Couchbase config name with special characters") + public void couchbaseConfigTest1WithSpecialCharacters() { + testConfigBuilder( + SourcesFixture.couchbaseSource1ConfigStrWithSpecialChars, configBuilder, SourcesFixture.expCouchbaseSourceWithSpecialCharsConfigObj); + } + + @Test(description = "Tests Pinot config") + public void pinotConfigTest() { + testConfigBuilder(SourcesFixture.pinotSource1ConfigStr, configBuilder, SourcesFixture.expPinotSource1ConfigObj); + } +} + diff --git a/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/sources/SourcesConfigBuilderTest.java b/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/sources/SourcesConfigBuilderTest.java new file mode 100644 index 000000000..ddb74398c --- /dev/null +++ b/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/sources/SourcesConfigBuilderTest.java @@ -0,0 +1,20 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.producer.sources; + +import com.linkedin.feathr.core.configbuilder.typesafe.AbstractConfigBuilderTest; +import org.testng.annotations.Test; + + +public class SourcesConfigBuilderTest extends AbstractConfigBuilderTest { + + @Test(description = "Tests build of all offline source configs") + public void offlineSourcesConfigTest() { + testConfigBuilder( + SourcesFixture.offlineSourcesConfigStr, SourcesConfigBuilder::build, SourcesFixture.expOfflineSourcesConfigObj); + } + + @Test(description = "Tests build of all online source configs") + public void onlineSourcesConfigTest() { + testConfigBuilder( + SourcesFixture.onlineSourcesConfigStr, SourcesConfigBuilder::build, SourcesFixture.expOnlineSourcesConfigObj); + } +} diff --git a/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/sources/SourcesFixture.java b/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/sources/SourcesFixture.java new file mode 100644 index 000000000..f2d2bbebd --- /dev/null +++ b/feathr-config/src/test/java/com/linkedin/feathr/core/configbuilder/typesafe/producer/sources/SourcesFixture.java @@ -0,0 +1,667 @@ +package com.linkedin.feathr.core.configbuilder.typesafe.producer.sources; + +import com.google.common.collect.ImmutableMap; +import com.linkedin.data.DataList; +import com.linkedin.data.DataMap; +import com.linkedin.data.schema.PathSpec; +import com.linkedin.feathr.core.config.producer.sources.CouchbaseConfig; +import com.linkedin.feathr.core.config.producer.sources.EspressoConfig; +import com.linkedin.feathr.core.config.producer.sources.HdfsConfigWithRegularData; +import com.linkedin.feathr.core.config.producer.sources.HdfsConfigWithSlidingWindow; +import com.linkedin.feathr.core.config.producer.sources.KafkaConfig; +import com.linkedin.feathr.core.config.producer.sources.PassThroughConfig; +import com.linkedin.feathr.core.config.producer.sources.PinotConfig; +import com.linkedin.feathr.core.config.producer.sources.RestliConfig; +import com.linkedin.feathr.core.config.producer.sources.RocksDbConfig; +import com.linkedin.feathr.core.config.producer.sources.SlidingWindowAggrConfig; +import com.linkedin.feathr.core.config.producer.sources.SourceConfig; +import com.linkedin.feathr.core.config.producer.sources.SourcesConfig; +import com.linkedin.feathr.core.config.producer.sources.TimeWindowParams; +import com.linkedin.feathr.core.config.producer.sources.VeniceConfig; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + + +public class SourcesFixture { + /* + * HDFS sources + */ + // Source with just HDFS location path + static final String hdfsSource1ConfigStr = String.join("\n", + "member_derived_data: {", + " location: {path: \"/data/test/#LATEST\"}", + "}"); + + public static final HdfsConfigWithRegularData expHdfsSource1ConfigObj; + static { + String path = "/data/test/#LATEST"; + expHdfsSource1ConfigObj = new HdfsConfigWithRegularData("member_derived_data", path, false); + } + + // Source with type HDFS and location + static final String hdfsSource2ConfigStr = String.join("\n", + "member_derived_data2: {", + " type: \"HDFS\"", + " location: {path: \"/data/test/#LATEST\"}", + "}"); + + static final HdfsConfigWithRegularData expHdfsSource2ConfigObj; + static { + String path = "/data/test/#LATEST"; + expHdfsSource2ConfigObj = new HdfsConfigWithRegularData("member_derived_data2", path, false); + } + + // hdfsSource1ConfigStr and hdfsSource2ConfigStr have been removed + static final String hdfsSource3ConfigStr = String.join("\n", + "member_derived_data_dali: {", + " location: {path: ", + "\"dalids:///standardizationwaterloomembersstddata_mp.standardization_waterloo_members_std_data\"}", + "}"); + + static final HdfsConfigWithRegularData expHdfsSource3ConfigObj; + static { + String path = "dalids:///standardizationwaterloomembersstddata_mp.standardization_waterloo_members_std_data"; + expHdfsSource3ConfigObj = new HdfsConfigWithRegularData("member_derived_data_dali", path, false); + } + + static final String hdfsSource4ConfigStr = String.join("\n", + "swaSource: {", + " type: \"HDFS\"", + " location: { path: \"dalids://sample_database.fact_data_table\" }", + " timeWindowParameters: {", + " timestampColumn: \"timestamp\"", + " timestampColumnFormat: \"yyyy/MM/dd/HH/mm/ss\"", + " }", + "}"); + + static final HdfsConfigWithSlidingWindow expHdfsSource4ConfigObj; + static { + String path = "dalids://sample_database.fact_data_table"; + TimeWindowParams timeWindowParams = + new TimeWindowParams("timestamp", "yyyy/MM/dd/HH/mm/ss"); + SlidingWindowAggrConfig swaConfig = new SlidingWindowAggrConfig(false, timeWindowParams); + expHdfsSource4ConfigObj = new HdfsConfigWithSlidingWindow("swaSource", path, swaConfig); + } + + static final String hdfsSource5ConfigStrWithTimePartitionPattern = String.join("\n", + "source: {", + " type: \"HDFS\"", + " location: { path: \"dalids://sample_database.fact_data_table\" }", + " timePartitionPattern: \"yyyy-MM-dd\"", + "}"); + + + static final HdfsConfigWithRegularData expHdfsSource5ConfigObjWithTimePartitionPattern; + static { + String path = "dalids://sample_database.fact_data_table"; + expHdfsSource5ConfigObjWithTimePartitionPattern = new HdfsConfigWithRegularData("source", path, "yyyy-MM-dd",false); + } + + static final String hdfsSource6ConfigStrWithLegacyTimeWindowParameters = String.join("\n", + "swaSource: {", + " type: \"HDFS\"", + " location: { path: \"dalids://sample_database.fact_data_table\" }", + " isTimeSeries: true", + " timeWindowParameters: {", + " timestamp: \"timestamp\"", + " timestamp_format: \"yyyy/MM/dd/HH/mm/ss\"", + " }", + "}"); + + static final HdfsConfigWithSlidingWindow expHdfsSource6ConfigObjWithLegacyTimeWindowParameters; + static { + String path = "dalids://sample_database.fact_data_table"; + TimeWindowParams timeWindowParams = + new TimeWindowParams("timestamp", "yyyy/MM/dd/HH/mm/ss"); + SlidingWindowAggrConfig swaConfig = new SlidingWindowAggrConfig(true, timeWindowParams); + expHdfsSource6ConfigObjWithLegacyTimeWindowParameters = new HdfsConfigWithSlidingWindow("swaSource", path, swaConfig); + } + + static final String invalidHdfsSourceconfigStrWithTimePartitionPatternAndIsTimeSeries = String.join("\n", + "swaSource: {", + " type: \"HDFS\"", + " location: { path: \"dalids://sample_database.fact_data_table\" }", + " timePartitionPattern: \"yyyy-MM-dd\"", + " isTimeSeries: true", + " timeWindowParameters: {", + " timestamp: \"timestamp\"", + " timestamp_format: \"yyyy/MM/dd/HH/mm/ss\"", + " }", + "}"); + + static final String invalidHdfsSourceconfigStrWithHasTimeSnapshotAndIsTimeSeries = String.join("\n", + "swaSource: {", + " type: \"HDFS\"", + " location: { path: \"dalids://sample_database.fact_data_table\" }", + " hasTimeSnapshot: true", + " isTimeSeries: true", + " timeWindowParameters: {", + " timestamp: \"timestamp\"", + " timestamp_format: \"yyyy/MM/dd/HH/mm/ss\"", + " }", + "}"); + + /* + * Espresso + */ + static final String espressoSource1ConfigStr = String.join("\n", + "MemberPreferenceData: {", + " type: ESPRESSO", + " database: \"CareersPreferenceDB\"", + " table: \"MemberPreference\"", + " d2Uri: \"d2://ESPRESSO_MT2\"", + " keyExpr: \"key[0]\"", + "}"); + + public static final EspressoConfig expEspressoSource1ConfigObj = new EspressoConfig("MemberPreferenceData", "CareersPreferenceDB", + "MemberPreference", "d2://ESPRESSO_MT2", "key[0]"); + + /* + * Venice sources + */ + static final String veniceSource1ConfigStr = String.join("\n", + "veniceTestSourceWithAvroKey {", + " type: VENICE", + " keyExpr : \"{\\\"x\\\" : (Integer)key[0], \\\"version\\\" : \\\"v2\\\"}\"", + " storeName: \"vtstore\"", + "}"); + + static final VeniceConfig expVeniceSource1ConfigObj; + static { + String storeName = "vtstore"; + String keyExpr = "{\"x\" : (Integer)key[0], \"version\" : \"v2\"}"; + expVeniceSource1ConfigObj = new VeniceConfig("veniceTestSourceWithAvroKey", storeName, keyExpr); + } + + static final String veniceSource2ConfigStr = String.join("\n", + "veniceTestSourceWithIntegerKey {", + " type: VENICE", + " keyExpr : \"(Integer)key[0]\"", + " storeName: \"vtstore2\"", + "}"); + + static final VeniceConfig expVeniceSource2ConfigObj; + static { + String storeName = "vtstore2"; + String keyExpr = "(Integer)key[0]"; + expVeniceSource2ConfigObj = new VeniceConfig("veniceTestSourceWithIntegerKey", storeName, keyExpr); + } + + /* + * Rest.Li sources + */ + static final String restliSource1ConfigStr = String.join("\n", + "JobsTargetingSegments: {", + " type: RESTLI", + " restResourceName: \"jobsTargetingSegments\"", + " restEntityType: \"jobPosting\"", + " pathSpec: \"targetingFacetsSet\"", + "}"); + + static final RestliConfig expRestliSource1ConfigObj; + static { + String resourceName = "jobsTargetingSegments"; + String keyExpr = "toUrn(\"jobPosting\", key[0])"; + PathSpec pathSpec = new PathSpec("targetingFacetsSet"); + expRestliSource1ConfigObj = new RestliConfig("JobsTargetingSegments", resourceName, keyExpr, null, pathSpec); + } + + static final String restliSource2ConfigStr = String.join("\n", + "MemberConnectionIntersection: {", + " type: RESTLI", + " restResourceName: setOperations", + " restEntityType: member", + " restReqParams: {", + " operator : INTERSECT", + " edgeSetSpecifications : {", + " json: {", + " firstEdgeType: MemberToMember", + " secondEdgeType: MemberToMember", + " }", + " }", + " second: {", + " mvel: \"key[1]\"", // key[0] is by default used as the request key + " }", + " }", + "}"); + + static final RestliConfig expRestliSource2ConfigObj; + static { + String resourceName = "setOperations"; + + String keyExpr = "toUrn(\"member\", key[0])"; + + Map map = new HashMap<>(); + map.put("firstEdgeType", "MemberToMember"); + map.put("secondEdgeType", "MemberToMember"); + DataMap dataMap = new DataMap(map); + + String mvelExpr = "key[1]"; //MVEL.compileExpression("key[1]"); + + Map paramsMap = new HashMap<>(); + paramsMap.put("operator", "INTERSECT"); + paramsMap.put("edgeSetSpecifications", dataMap); + paramsMap.put("second", new DataMap(ImmutableMap.of(RestliConfig.MVEL_KEY, mvelExpr))); + + expRestliSource2ConfigObj = new RestliConfig("MemberConnectionIntersection", resourceName, keyExpr, paramsMap, null); + } + + static final String restliSource3ConfigStr = String.join("\n", + "MemberConnectionIntersection2: {", + " type: RESTLI", + " restResourceName: setOperations", + " restEntityType: member", + " restReqParams: {", + " operator : INTERSECT", + " edgeSetSpecifications : {", + " jsonArray: {", + " array: [", + " {firstEdgeType: MemberToMember, secondEdgeType : MemberToMember}", + " ]", + " }", + " }", + " second: {", + " mvel: \"key[1]\"", + " }", + " }", + "}"); + + static final RestliConfig expRestliSource3ConfigObj; + static { + String resourceName = "setOperations"; + + String keyExpr = "toUrn(\"member\", key[0])"; + + Map map = new HashMap<>(); + map.put("firstEdgeType", "MemberToMember"); + map.put("secondEdgeType", "MemberToMember"); + DataMap dataMap = new DataMap(map); + List list = new ArrayList<>(); + list.add(dataMap); + DataList dataList = new DataList(list); + + String mvelExpr = "key[1]"; //MVEL.compileExpression("key[1]"); + + Map paramsMap = new HashMap<>(); + paramsMap.put("operator", "INTERSECT"); + paramsMap.put("edgeSetSpecifications", dataList); + paramsMap.put("second", new DataMap(ImmutableMap.of(RestliConfig.MVEL_KEY, mvelExpr))); + + expRestliSource3ConfigObj = new RestliConfig("MemberConnectionIntersection2", resourceName, keyExpr, paramsMap, null); + } + + + static final String restliSource4ConfigStr = String.join("\n", + "Profile: {", + " type: RESTLI", + " restResouceName: \"profiles\"", + " keyExpr: \"toComplexResourceKey({\\\"id\\\": key[0]},{:})\"", + " restReqParams: {", + " viewerId: {mvel: \"key[0]\"}", + " }", + " pathSpec: \"positions\"", + "}"); + + static final RestliConfig expRestliSource4ConfigObj; + static { + String resourceName = "profiles"; + + String keyExpr = "toComplexResourceKey({\"id\": key[0]},{:})"; + + String mvelExpr = "key[0]"; //MVEL.compileExpression("key[0]") + Map map = new HashMap<>(); + map.put("viewerId", new DataMap(ImmutableMap.of(RestliConfig.MVEL_KEY, mvelExpr))); + + PathSpec pathSpec = new PathSpec("positions"); + + expRestliSource4ConfigObj = new RestliConfig("Profile", resourceName, keyExpr, map, pathSpec); + } + + static final String restliSource5ConfigStr = String.join("\n", + "MemberConnectionIntersection: {", + " type: RESTLI", + " restResourceName: setOperations", + " restEntityType: member", + " restReqParams: {", + " operator : INTERSECT", + " edgeSetSpecifications : {", + " json: \"{firstEdgeType: MemberToMember, secondEdgeType: MemberToMember}\"", + " }", + " second: {", + " mvel: \"key[1]\"", // key[0] is by default used as the request key + " }", + " }", + "}"); + + static final RestliConfig expRestliSource5ConfigObj = expRestliSource2ConfigObj; + + static final String restliSource6ConfigStr = String.join("\n", + "MemberConnectionIntersection: {", + " type: RESTLI", + " restResourceName: setOperations", + " restEntityType: member", + " restReqParams: {", + " operator : INTERSECT", + " edgeSetSpecifications : {", + " json: {", + " }", + " }", + " second: {", + " mvel: \"key[1]\"", // key[0] is by default used as the request key + " }", + " }", + "}"); + + static final RestliConfig expRestliSource6ConfigObj; + static { + String resourceName = "setOperations"; + + String keyExpr = "toUrn(\"member\", key[0])"; + + Map map = new HashMap<>(); + DataMap dataMap = new DataMap(map); + + String mvelExpr = "key[1]"; //MVEL.compileExpression("key[1]"); + + Map paramsMap = new HashMap<>(); + paramsMap.put("operator", "INTERSECT"); + paramsMap.put("edgeSetSpecifications", dataMap); + paramsMap.put("second", new DataMap(ImmutableMap.of(RestliConfig.MVEL_KEY, mvelExpr))); + + expRestliSource6ConfigObj = new RestliConfig("MemberConnectionIntersection", resourceName, keyExpr, paramsMap, null); + } + + static final String restliSource7ConfigStr = String.join("\n", + "MemberConnectionIntersection2: {", + " type: RESTLI", + " restResourceName: setOperations", + " restEntityType: member", + " restReqParams: {", + " operator : INTERSECT", + " edgeSetSpecifications : {", + " jsonArray: {", + " array: [", + " ]", + " }", + " }", + " second: {", + " mvel: \"key[1]\"", + " }", + " }", + "}"); + + static final RestliConfig expRestliSource7ConfigObj; + static { + String resourceName = "setOperations"; + + String keyExpr = "toUrn(\"member\", key[0])"; + + List list = new ArrayList<>(); + DataList dataList = new DataList(list); + + String mvelExpr = "key[1]"; //MVEL.compileExpression("key[1]"); + + Map paramsMap = new HashMap<>(); + paramsMap.put("operator", "INTERSECT"); + paramsMap.put("edgeSetSpecifications", dataList); + paramsMap.put("second", new DataMap(ImmutableMap.of(RestliConfig.MVEL_KEY, mvelExpr))); + + expRestliSource7ConfigObj = new RestliConfig("MemberConnectionIntersection2", resourceName, keyExpr, paramsMap, null); + } + + static final String restliSource8ConfigStr = String.join("\n", + "Profile: {", + " type: RESTLI", + " restResouceName: \"profiles\"", + " finder: \"rule\"", + " restReqParams: {", + " ruleName: \"search/CurrentCompaniesOfConnections\"", + " ruleArguments: {mvel: \"[\\\"names\\\" : [\\\"member\\\", \\\"company\\\"], \\\"arguments\\\" : [[[\\\"value\\\" : key[0]], [:]]]]\"}", + " }", + " pathSpec: \"positions\"", + "}"); + + static final RestliConfig expRestliSource8ConfigObj; + static { + String resourceName = "profiles"; + String finder = "rule"; + String mvelExpr = "[\"names\" : [\"member\", \"company\"], \"arguments\" : [[[\"value\" : key[0]], [:]]]]"; + Map map = new HashMap<>(); + map.put("ruleName", "search/CurrentCompaniesOfConnections"); + map.put("ruleArguments", new DataMap(ImmutableMap.of(RestliConfig.MVEL_KEY, mvelExpr))); + + PathSpec pathSpec = new PathSpec("positions"); + + expRestliSource8ConfigObj = new RestliConfig("Profile", resourceName, map, pathSpec, finder); + } + + // Case where both keyExpr and finder are present. + static final String restliSource9ConfigStr = String.join("\n", + "Profile: {", + " type: RESTLI", + " restResourceName: \"profiles\"", + " finder: \"rule\"", + " keyExpr: \"toCompoundKey(\\\"member\\\", 123)\"", + "}"); + + static final RestliConfig expRestliSource9ConfigObj; + static { + String resourceName = "profiles"; + String finder = "rule"; + String mvelExpr = "toCompoundKey(\"member\", 123)"; + expRestliSource9ConfigObj = new RestliConfig("Profile", resourceName, mvelExpr, null, null, finder); + } + + // Case where both keyExpr and finder are missing. + static final String restliSource10ConfigStr = String.join("\n", + "Profile: {", + " type: RESTLI", + " restResourceName: \"profiles\"", + "}"); + + /* + * Kafka sources + */ + static final String kafkaSource1ConfigStr = String.join("\n", + "kafkaTestSource1: {", + " type: KAFKA", + " stream: \"kafka.testCluster.testTopic\"", + "}"); + + static final KafkaConfig expKafkaSource1ConfigObj = + new KafkaConfig("kafkaTestSource1", "kafka.testCluster.testTopic", null); + + static final String kafkaSource2ConfigStr = String.join("\n", + "kafkaTestSource2: {", + " type: KAFKA", + " stream: \"kafka.testCluster.testTopic\"", + " isTimeSeries: true", + " timeWindowParameters: {", + " timestamp: \"timestamp\"", + " timestamp_format: \"yyyy/MM/dd/HH/mm/ss\"", + " }", + "}"); + + static final KafkaConfig expKafkaSource2ConfigObj; + static { + String stream = "kafka.testCluster.testTopic"; + TimeWindowParams timeWindowParams = + new TimeWindowParams("timestamp", "yyyy/MM/dd/HH/mm/ss"); + SlidingWindowAggrConfig swaConfig = new SlidingWindowAggrConfig(true, timeWindowParams); + expKafkaSource2ConfigObj = new KafkaConfig("kafkaTestSource2", stream, swaConfig); + } + + /* + * RocksDB sources + */ + static final String rocksDbSource1ConfigStr = String.join("\n", + "rocksDBTestSource1: {", + " type: ROCKSDB", + " referenceSource: \"kafkaTestSource\"", + " extractFeatures: true", + " encoder: \"com.linkedin.frame.online.config.FoobarExtractor\"", + " decoder: \"com.linkedin.frame.online.config.FoobarExtractor\"", + " keyExpr: \"keyExprName\"", + "}"); + + static final RocksDbConfig expRocksDbSource1ConfigObj; + static { + String referenceSource = "kafkaTestSource"; + String encoder = "com.linkedin.frame.online.config.FoobarExtractor"; + String decoder = "com.linkedin.frame.online.config.FoobarExtractor"; + String keyExpr = "keyExprName"; + expRocksDbSource1ConfigObj = new RocksDbConfig("rocksDBTestSource1", referenceSource, true, encoder, decoder, keyExpr); + } + + static final String rocksDbSource2ConfigStr = String.join("\n", + "rocksDBTestSource2: {", + " type: ROCKSDB", + " referenceSource: \"kafkaTestSource\"", + " extractFeatures: true", + " encoder: \"com.linkedin.frame.online.config.FoobarExtractor\"", + " decoder: \"com.linkedin.frame.online.config.FoobarExtractor\"", + "}"); + + static final RocksDbConfig expRocksDbSource2ConfigObj; + static { + String referenceSource = "kafkaTestSource"; + String encoder = "com.linkedin.frame.online.config.FoobarExtractor"; + String decoder = "com.linkedin.frame.online.config.FoobarExtractor"; + expRocksDbSource2ConfigObj = new RocksDbConfig("rocksDBTestSource2", referenceSource, true, encoder, decoder, null); + } + /* + * PassThrough sources + */ + static final String passThroughSource1ConfigStr = String.join("\n", + "passThroughTestSource: {", + " type: PASSTHROUGH", + " dataModel: \"com.linkedin.some.service.SomeEntity\"", + "}"); + + static final PassThroughConfig expPassThroughSource1ConfigObj = + new PassThroughConfig("passThroughTestSource", "com.linkedin.some.service.SomeEntity"); + + /* + * Couchbase sources + */ + static final String couchbaseSource1ConfigStr = String.join("\n", + "couchbaseTestSource {", + " type: COUCHBASE", + " keyExpr : \"key[0]\"", + " bucketName: \"testBucket\"", + " bootstrapUris: [\"some-app.linkedin.com:8091\", \"other-app.linkedin.com:8091\"]", + " documentModel: \"com.linkedin.some.Document\"", + "}"); + + static final CouchbaseConfig expCouchbaseSource1ConfigObj; + static { + String bucketName = "testBucket"; + String keyExpr = "key[0]"; + String[] bootstrapUris = new String[] {"some-app.linkedin.com:8091", "other-app.linkedin.com:8091"}; + String documentModel = "com.linkedin.some.Document"; + expCouchbaseSource1ConfigObj = new CouchbaseConfig("couchbaseTestSource", bucketName, keyExpr, documentModel); + } + + /* + * Couchbase sources with special characters + */ + static final String couchbaseSource1ConfigStrWithSpecialChars = String.join("\n", + "\"couchbase:Test.Source\" {", + " type: COUCHBASE", + " keyExpr : \"key[0]\"", + " bucketName: \"testBucket\"", + " bootstrapUris: [\"some-app.linkedin.com:8091\", \"other-app.linkedin.com:8091\"]", + " documentModel: \"com.linkedin.some.Document\"", + "}"); + static final CouchbaseConfig expCouchbaseSourceWithSpecialCharsConfigObj; + static { + String bucketName = "testBucket"; + String keyExpr = "key[0]"; + String[] bootstrapUris = new String[] {"some-app.linkedin.com:8091", "other-app.linkedin.com:8091"}; + String documentModel = "com.linkedin.some.Document"; + expCouchbaseSourceWithSpecialCharsConfigObj = new CouchbaseConfig("couchbase:Test.Source", bucketName, keyExpr, documentModel); + } + + static final CouchbaseConfig expCouchbaseSource1ConfigObjWithSpecialChars; + static { + String bucketName = "testBucket"; + String keyExpr = "key[0]"; + String[] bootstrapUris = new String[]{"some-app.linkedin.com:8091", "other-app.linkedin.com:8091"}; + String documentModel = "com.linkedin.some.Document"; + expCouchbaseSource1ConfigObjWithSpecialChars = new CouchbaseConfig("couchbase:Test.Source", bucketName, keyExpr, documentModel); + } + + /* + * Pinot sources + */ + static final String pinotSource1ConfigStr = + String.join("\n", "pinotTestSource {", + " type: PINOT", + " resourceName : \"recentMemberActionsPinotQuery\"", + " queryTemplate : \"SELECT verb, object, verbAttributes, timeStampSec FROM RecentMemberActions WHERE actorId IN (?)\"", + " queryArguments : [\"key[0]\"]", + " queryKeyColumns: [\"actorId\"]", + "}"); + + static final PinotConfig expPinotSource1ConfigObj; + + static { + String resourceName = "recentMemberActionsPinotQuery"; + String queryTemplate = "SELECT verb, object, verbAttributes, timeStampSec FROM RecentMemberActions WHERE actorId IN (?)"; + String[] queryArguments = new String[]{"key[0]"}; + String[] queryKeyColumns = new String[]{"actorId"}; + + expPinotSource1ConfigObj = new PinotConfig("pinotTestSource", resourceName, queryTemplate, queryArguments, queryKeyColumns); + } + + static final String offlineSourcesConfigStr = String.join("\n", + "sources: {", + hdfsSource1ConfigStr, + hdfsSource2ConfigStr, + hdfsSource3ConfigStr, + hdfsSource4ConfigStr, + "}"); + + static final SourcesConfig expOfflineSourcesConfigObj; + static { + Map sources = new HashMap<>(); + sources.put("member_derived_data", expHdfsSource1ConfigObj); + sources.put("member_derived_data2", expHdfsSource2ConfigObj); + sources.put("member_derived_data_dali", expHdfsSource3ConfigObj); + sources.put("swaSource", expHdfsSource4ConfigObj); + expOfflineSourcesConfigObj = new SourcesConfig(sources); + } + + + static final String onlineSourcesConfigStr = String.join("\n", + "sources: {", + espressoSource1ConfigStr, + veniceSource1ConfigStr, + veniceSource2ConfigStr, + kafkaSource1ConfigStr, + kafkaSource2ConfigStr, + rocksDbSource1ConfigStr, + rocksDbSource2ConfigStr, + passThroughSource1ConfigStr, + couchbaseSource1ConfigStr, + pinotSource1ConfigStr, + "}"); + + static final SourcesConfig expOnlineSourcesConfigObj; + static { + Map sources = new HashMap<>(); + sources.put("MemberPreferenceData", expEspressoSource1ConfigObj); + sources.put("veniceTestSourceWithAvroKey", expVeniceSource1ConfigObj); + sources.put("veniceTestSourceWithIntegerKey", expVeniceSource2ConfigObj); + sources.put("kafkaTestSource1", expKafkaSource1ConfigObj); + sources.put("kafkaTestSource2", expKafkaSource2ConfigObj); + sources.put("rocksDBTestSource1", expRocksDbSource1ConfigObj); + sources.put("rocksDBTestSource2", expRocksDbSource2ConfigObj); + sources.put("passThroughTestSource", expPassThroughSource1ConfigObj); + sources.put("couchbaseTestSource", expCouchbaseSource1ConfigObj); + sources.put("pinotTestSource", expPinotSource1ConfigObj); + expOnlineSourcesConfigObj = new SourcesConfig(sources); + } +} \ No newline at end of file diff --git a/feathr-config/src/test/java/com/linkedin/feathr/core/configdataprovider/FrameConfigFileCheckerTest.java b/feathr-config/src/test/java/com/linkedin/feathr/core/configdataprovider/FrameConfigFileCheckerTest.java new file mode 100644 index 000000000..177f3b61d --- /dev/null +++ b/feathr-config/src/test/java/com/linkedin/feathr/core/configdataprovider/FrameConfigFileCheckerTest.java @@ -0,0 +1,54 @@ +package com.linkedin.feathr.core.configdataprovider; + +import com.linkedin.feathr.core.configbuilder.ConfigBuilderException; +import com.linkedin.feathr.core.configbuilder.typesafe.FrameConfigFileChecker; +import java.net.URL; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import static org.testng.Assert.*; + + +/** + * Unit tests for {@link FrameConfigFileChecker} + */ +public class FrameConfigFileCheckerTest { + private static ClassLoader _classLoader; + + @BeforeClass + public static void init() { + _classLoader = Thread.currentThread().getContextClassLoader(); + } + + @Test(description = "A valid Frame config file with valid syntax should return true.") + public void testValidFrameConfigFile() { + URL url = _classLoader.getResource("frame-feature-careers-featureDef-offline.conf"); + + boolean configFile = FrameConfigFileChecker.isConfigFile(url); + assertTrue(configFile); + } + + @Test(description = "Test that a txt file should throw exception.", expectedExceptions = ConfigBuilderException.class) + public void testTxtFile() { + URL url = _classLoader.getResource("Foo.txt"); + + boolean configFile = FrameConfigFileChecker.isConfigFile(url); + assertTrue(configFile); + } + + @Test(description = "An invalid Frame feature config file should return false.") + public void testInvalidConfigFile() { + URL url = _classLoader.getResource("PresentationsSchemaTestCases.conf"); + + boolean configFile = FrameConfigFileChecker.isConfigFile(url); + assertFalse(configFile); + } + + @Test(description = "An valid Frame config file with invalid syntax should return true.") + public void testValidConfigFileWithInvalidSyntax() { + URL url = _classLoader.getResource("validFrameConfigWithInvalidSyntax.conf"); + + boolean configFile = FrameConfigFileChecker.isConfigFile(url); + assertTrue(configFile); + } +} diff --git a/feathr-config/src/test/java/com/linkedin/feathr/core/configdataprovider/ManifestConfigDataProviderTest.java b/feathr-config/src/test/java/com/linkedin/feathr/core/configdataprovider/ManifestConfigDataProviderTest.java new file mode 100644 index 000000000..49e703bbc --- /dev/null +++ b/feathr-config/src/test/java/com/linkedin/feathr/core/configdataprovider/ManifestConfigDataProviderTest.java @@ -0,0 +1,38 @@ +package com.linkedin.feathr.core.configdataprovider; + +import java.io.BufferedReader; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.testng.annotations.Test; + +import static org.testng.Assert.*; + + +/** + * Unit tests for {@link ManifestConfigDataProvider} + */ +public class ManifestConfigDataProviderTest { + + @Test(description = "Tests getting Readers for files listed in a manifest file") + public void test() { + String manifest = "config/manifest3.conf"; + + try (ManifestConfigDataProvider cdp = new ManifestConfigDataProvider(manifest)) { + List readers = cdp.getConfigDataReaders() + .stream() + .map(BufferedReader::new) + .collect(Collectors.toList()); + + assertEquals(readers.size(), 2); + + for (BufferedReader r : readers) { + Stream stringStream = r.lines(); + long lineCount = stringStream.count(); + assertTrue(lineCount > 0, "Expected line count > 0, found " + lineCount); + } + } catch (Exception e) { + fail("Caught exception", e); + } + } +} diff --git a/feathr-config/src/test/java/com/linkedin/feathr/core/configdataprovider/ResourceConfigDataProviderTest.java b/feathr-config/src/test/java/com/linkedin/feathr/core/configdataprovider/ResourceConfigDataProviderTest.java new file mode 100644 index 000000000..e14b94a65 --- /dev/null +++ b/feathr-config/src/test/java/com/linkedin/feathr/core/configdataprovider/ResourceConfigDataProviderTest.java @@ -0,0 +1,74 @@ +package com.linkedin.feathr.core.configdataprovider; + +import java.io.BufferedReader; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.testng.annotations.Test; + +import static org.testng.Assert.*; + + +/** + * Unit tests for {@link ResourceConfigDataProvider} + */ +public class ResourceConfigDataProviderTest { + + @Test(description = "Tests with a single resource file") + public void testWithSingleResource() { + String resource = "Foo.txt"; + + try (ConfigDataProvider cdp = new ResourceConfigDataProvider(resource)) { + List readers = cdp.getConfigDataReaders() + .stream() + .map(BufferedReader::new) + .collect(Collectors.toList()); + + assertEquals(readers.size(), 1); + Stream stringStream = readers.get(0).lines(); + assertEquals(stringStream.count(), 3L); + } catch (Exception e) { + fail("Test failed", e); + } + } + + @Test(description = "Tests with 2 resource files") + public void testWithMultipleResources() { + List resources = Arrays.asList("Foo.txt", "Bar.txt"); + + try (ConfigDataProvider cdp = new ResourceConfigDataProvider(resources)) { + List readers = cdp.getConfigDataReaders() + .stream() + .map(BufferedReader::new) + .collect(Collectors.toList()); + + assertEquals(readers.size(), resources.size()); + + Stream stringStream1 = readers.get(0).lines(); + assertEquals(stringStream1.count(), 3L); + + Stream stringStream2 = readers.get(1).lines(); + assertEquals(stringStream2.count(), 2L); + } catch (Exception e) { + fail("Test failed", e); + } + } + + @Test(description = "Tests custom class loader") + public void testCustomClassLoader() { + String resource = "Foo.txt"; + + try (ConfigDataProvider cdp = + new ResourceConfigDataProvider(resource, Thread.currentThread().getContextClassLoader())) { + List readers = + cdp.getConfigDataReaders().stream().map(BufferedReader::new).collect(Collectors.toList()); + + assertEquals(readers.size(), 1); + Stream stringStream = readers.get(0).lines(); + assertEquals(stringStream.count(), 3L); + } catch (Exception e) { + fail("Test failed", e); + } + } +} diff --git a/feathr-config/src/test/java/com/linkedin/feathr/core/configdataprovider/StringConfigDataProviderTest.java b/feathr-config/src/test/java/com/linkedin/feathr/core/configdataprovider/StringConfigDataProviderTest.java new file mode 100644 index 000000000..c92973a81 --- /dev/null +++ b/feathr-config/src/test/java/com/linkedin/feathr/core/configdataprovider/StringConfigDataProviderTest.java @@ -0,0 +1,78 @@ +package com.linkedin.feathr.core.configdataprovider; + +import java.io.BufferedReader; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; +import org.testng.annotations.Test; + +import static org.testng.Assert.*; + + +/** + * Unit tests for {@link StringConfigDataProvider} + */ +public class StringConfigDataProviderTest { + + @Test(description = "Tests with single string") + public void testWithSingleString() { + String line1 = "This is line 1"; + String line2 = "This is line two"; + String line3 = "This is line number 3"; + String lines = String.join("\n", line1, line2, line3); + + try (ConfigDataProvider cdp = new StringConfigDataProvider(lines)) { + List stringReaders = cdp.getConfigDataReaders() + .stream() + .map(BufferedReader::new) + .collect(Collectors.toList()); + + assertEquals(stringReaders.size(), 1); + + BufferedReader strReader = stringReaders.get(0); + assertEquals(strReader.readLine(), line1); + assertEquals(strReader.readLine(), line2); + assertEquals(strReader.readLine(), line3); + assertNull(strReader.readLine()); + } catch (Exception e) { + fail("Caught exception", e); + } + } + + @Test(description = "Tests with 2 strings") + public void testWithMultipleStrings() { + String line11 = "This is line 1"; + String line12 = "This is line two"; + String line13 = "This is line number 3"; + String str1 = String.join("\n", line11, line12, line13); + + String line21 = "There is no greatness where there is not simplicity, goodness, and truth."; + String line22 = "The strongest of all warriors are these two — Time and Patience."; + String str2 = String.join("\n", line21, line22); + + List strings = Arrays.asList(str1, str2); + + try (ConfigDataProvider cdp = new StringConfigDataProvider(strings)) { + List stringReaders = cdp.getConfigDataReaders() + .stream() + .map(BufferedReader::new) + .collect(Collectors.toList()); + + assertEquals(stringReaders.size(), strings.size()); + + BufferedReader strReader1 = stringReaders.get(0); + assertEquals(strReader1.readLine(), line11); + assertEquals(strReader1.readLine(), line12); + assertEquals(strReader1.readLine(), line13); + assertNull(strReader1.readLine()); + + BufferedReader strReader2 = stringReaders.get(1); + assertEquals(strReader2.readLine(), line21); + assertEquals(strReader2.readLine(), line22); + assertNull(strReader2.readLine()); + + } catch (Exception e) { + fail("Caught exception", e); + } + } +} diff --git a/feathr-config/src/test/java/com/linkedin/feathr/core/configdataprovider/UrlConfigDataProviderTest.java b/feathr-config/src/test/java/com/linkedin/feathr/core/configdataprovider/UrlConfigDataProviderTest.java new file mode 100644 index 000000000..27751436b --- /dev/null +++ b/feathr-config/src/test/java/com/linkedin/feathr/core/configdataprovider/UrlConfigDataProviderTest.java @@ -0,0 +1,68 @@ +package com.linkedin.feathr.core.configdataprovider; + +import java.io.BufferedReader; +import java.net.URL; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import static org.testng.Assert.*; + + +/** + * Unit tests for {@link UrlConfigDataProvider} + */ +public class UrlConfigDataProviderTest { + private static ClassLoader _classLoader; + + @BeforeClass + public static void init() { + _classLoader = Thread.currentThread().getContextClassLoader(); + } + + @Test(description = "Tests with a single URL") + public void testWithSingleUrl() { + String resource = "Foo.txt"; + URL url = _classLoader.getResource(resource); + + try (ConfigDataProvider cdp = new UrlConfigDataProvider(url)) { + List readers = cdp.getConfigDataReaders() + .stream() + .map(BufferedReader::new) + .collect(Collectors.toList()); + + assertEquals(readers.size(), 1); + Stream stringStream = readers.get(0).lines(); + assertEquals(stringStream.count(), 3L); + } catch (Exception e) { + fail("Caught exception", e); + } + } + + @Test(description = "Tests with two URLs") + public void testWithMultipleUrls() { + List resources = Arrays.asList("Foo.txt", "Bar.txt"); + List urls = resources.stream().map(r -> _classLoader.getResource(r)).collect(Collectors.toList()); + + try (ConfigDataProvider cdp = new UrlConfigDataProvider(urls)) { + List readers = cdp.getConfigDataReaders() + .stream() + .map(BufferedReader::new) + .collect(Collectors.toList()); + + assertEquals(readers.size(), urls.size()); + + Stream stringStream1 = readers.get(0).lines(); + assertEquals(stringStream1.count(), 3L); + + Stream stringStream2 = readers.get(1).lines(); + assertEquals(stringStream2.count(), 2L); + } catch (Exception e) { + fail("Caught exception", e); + } + + } +} diff --git a/feathr-config/src/test/java/com/linkedin/feathr/core/configvalidator/ConfigValidatorFixture.java b/feathr-config/src/test/java/com/linkedin/feathr/core/configvalidator/ConfigValidatorFixture.java new file mode 100644 index 000000000..5123c7bf4 --- /dev/null +++ b/feathr-config/src/test/java/com/linkedin/feathr/core/configvalidator/ConfigValidatorFixture.java @@ -0,0 +1,215 @@ +package com.linkedin.feathr.core.configvalidator; + +/** + * Fixture used during validation testing + */ +public class ConfigValidatorFixture { + public static final String invalidHoconStr1 = String.join("\n", + "sources: {", + " // Source name is incorrect since ':' isn't permitted in the key name if the key name isn't quoted.", + " invalid:source: {", + " type: VENICE", + " storeName: \"someStore\"", + " keyExpr: \"some key expression\"", + " }", + "}"); + + public static final String invalidHoconStr2 = String.join("\n", + "anchors: {", + " a1: {", + " source: \"some/source\"", + " key: \"someKey\"", + " features: {", + " // Character '$' is forbidden if present in unquoted string", + " $feature_name_is_invalid: \"some feature expr\"", + " }", + " }", + "}"); + + public static final String validFeatureDefConfig = String.join("\n", + "anchors: {", + " A1: {", + " source: \"/data/databases/CareersPreferenceDB/MemberPreference/#LATEST\"", + " extractor: \"com.linkedin.jymbii.frame.anchor.PreferencesFeatures\"", + " keyAlias: \"x\"", + " features: [", + " jfu_preference_companySize", + " ]", + " }", + "}" + ); + + public static final String validFeatureDefConfigWithParameters = String.join("\n", + "anchors: {", + " A1: {", + " source: \"/data/databases/CareersPreferenceDB/MemberPreference/#LATEST\"", + " extractor: \"com.linkedin.jymbii.frame.anchor.PreferencesFeatures\"", + " keyAlias: \"x\"", + " features: {", + " jfu_preference_companySize : {", + " parameters : {", + " param0 : \" some param 1\"", + " param1 : some_param", + " param2 : true", + " param3 : [p1, p2]", + " param4 : {java : 3}", + " param5 : {\"key1\":[\"v1\",\"v2\"]}", + " param6 : [{\"key1\":[\"v1\",\"v2\"]}, {\"key2\":[\"v1\",\"v2\"]}]", + " }", + " }", + " }", + " }", + "}" + ); + + /** + * The parameters are invalid because param1 and param2 are not of string type. + */ + public static final String invalidFeatureDefConfigWithParameters = String.join("\n", + "anchors: {", + " A1: {", + " source: \"/data/databases/CareersPreferenceDB/MemberPreference/#LATEST\"", + " extractor: \"com.linkedin.jymbii.frame.anchor.PreferencesFeatures\"", + " keyAlias: \"x\"", + " features: {", + " jfu_preference_companySize : {", + " parameters : param", + " }", + " }", + " }", + "}" + ); + + public static final String legacyFeatureDefConfigWithGlobals = String.join("\n", + "globals: {", + "}", + "anchors: {", + "}", + "sources: {", + "}" + ); + + public static final String invalidFeatureDefConfig = String.join("\n", + "anchors: {", + " A1: {", + " source: \"some/path/in/HDFS/#LATEST\"", + " key: \"x\"", + " features: {", + " f1: 4.2", + " default: 123.0", + " }", + " }", + + " A2: {", + " key: \"x\"", + " features: [\"f2\", \"f3\"]", + " }", + + " // This anchor contains valid features, there shouldn't be any error flagged here", + " A3: {", + " source: \"/data/databases/CareersPreferenceDB/MemberPreference/#LATEST\"", + " extractor: \"com.linkedin.jymbii.frame.anchor.PreferencesFeatures\"", + " keyAlias: \"x\"", + " features: [", + " jfu_preference_companySize", + " ]", + " }", + "}"); + + public static final String invalidFeatureDefConfig2 = String.join("\n", + "anchors: {", + " A1: {", + " source: \"/data/databases/CareersPreferenceDB/MemberPreference/#LATEST\"", + " extractor: \"com.linkedin.jymbii.frame.anchor.PreferencesFeatures\"", + " keyAlias: \"x\"", + " features: [", + " jfu_preference_companySize.0.0.1", + " ]", + " }", + "}" + ); + + public static final String validJoinConfigWithSingleFeatureBag = String.join("\n", + "myFeatureBag: [", + " {", + " key: \"targetId\"", + " featureList: [waterloo_job_location, waterloo_job_jobTitle, waterloo_job_jobSeniority]", + " }", + " {", + " key: sourceId", + " featureList: [jfu_resolvedPreference_seniority]", + " }", + " {", + " key: [sourceId, targetId]", + " featureList: [memberJobFeature1, memberJobFeature2]", + " }", + "]"); + + public static final String validJoinConfigWithMultFeatureBags = String.join("\n", + "featuresGroupA: [", + " {", + " key: \"viewerId\"", + " featureList: [", + " waterloo_member_currentCompany,", + " waterloo_job_jobTitle,", + " ]", + " }", + "]", + "featuresGroupB: [", + " {", + " key: \"viewerId\"", + " featureList: [", + " waterloo_member_location,", + " waterloo_job_jobSeniority", + " ]", + " }", + "]"); + + public static final String invalidJoinConfig = String.join("\n", + "features: [", + " {", + " // Missing key", + " featureList: [", + " jfu_resolvedPreference_seniority, ", + " jfu_resolvedPreference_country", + " ]", + " }", + "]"); + + public static final String validPresentationConfig = String.join("\n", + "presentations: {", + " my_ccpa_feature: {", + " linkedInViewFeatureName: decision_makers_score", + " featureDescription: \"feature description that shows to the users\"", + " valueTranslation: \"translateLikelihood(this)\"", + " }", + "}"); + + /* + * Join config request features that are defined in FeatureDef config, but not reachable + */ + public static final String joinConfig1 = String.join("\n", + "features: [", + " {", + " key: \"viewerId\"", + " featureList: [", + " feature_not_defined_1,", + " feature_not_defined_2,", + " ]", + " }", + "]"); + + /* + * Join config request features that are not defined in FeatureDef config + * "resources/invalidSemanticsConfig/feature-not-reachable-def.conf" + */ + public static final String joinConfig2 = String.join("\n", + "features: [", + " {", + " key: [\"m\", \"j\"]", + " featureList: [", + " derived_feature_3", + " ]", + " }", + "]"); +} diff --git a/feathr-config/src/test/java/com/linkedin/feathr/core/configvalidator/ConfigValidatorTest.java b/feathr-config/src/test/java/com/linkedin/feathr/core/configvalidator/ConfigValidatorTest.java new file mode 100644 index 000000000..d5b02db2e --- /dev/null +++ b/feathr-config/src/test/java/com/linkedin/feathr/core/configvalidator/ConfigValidatorTest.java @@ -0,0 +1,192 @@ +package com.linkedin.feathr.core.configvalidator; + +import com.linkedin.feathr.core.config.ConfigType; +import com.linkedin.feathr.core.config.consumer.JoinConfig; +import com.linkedin.feathr.core.configdataprovider.ConfigDataProvider; +import com.linkedin.feathr.core.configdataprovider.ResourceConfigDataProvider; +import com.linkedin.feathr.core.configdataprovider.StringConfigDataProvider; +import com.typesafe.config.Config; +import com.typesafe.config.ConfigException; +import com.typesafe.config.ConfigFactory; +import com.typesafe.config.ConfigParseOptions; +import com.typesafe.config.ConfigRenderOptions; +import com.typesafe.config.ConfigSyntax; +import java.io.InputStream; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.everit.json.schema.Schema; +import org.everit.json.schema.ValidationException; +import org.everit.json.schema.loader.SchemaLoader; +import org.json.JSONObject; +import org.json.JSONTokener; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import static com.linkedin.feathr.core.config.ConfigType.*; +import static com.linkedin.feathr.core.configvalidator.ValidationStatus.*; +import static com.linkedin.feathr.core.configvalidator.ValidationType.*; +import static org.testng.Assert.*; + + +/** + * Unit tests for {@link ConfigValidator} + */ +/* + * Note: These tests exercise the validation API and aren't intended to test syntax validation itself. + * Such (exhaustive) syntax tests should be added in typesafe/ConfigSchemaTest. + */ +public class ConfigValidatorTest { + private ConfigValidator _validator; + + @BeforeClass + public void init() { + _validator = ConfigValidator.getInstance(); + } + + @Test(description = "Attempts to validate syntax of config with invalid HOCON syntax") + public void testConfigWithInvalidHocon() { + List configStrings = Arrays.asList( + ConfigValidatorFixture.invalidHoconStr1, ConfigValidatorFixture.invalidHoconStr2); + + for (String cfgStr : configStrings) { + try (ConfigDataProvider cdp = new StringConfigDataProvider(cfgStr)) { + ValidationResult obsResult = _validator.validate(FeatureDef, SYNTACTIC, cdp); + + assertEquals(obsResult.getValidationStatus(), INVALID); + assertTrue(obsResult.getDetails().isPresent()); + assertTrue(obsResult.getCause().isPresent()); + assertEquals(obsResult.getCause().get().getClass(), ConfigException.Parse.class); + } catch (Exception e) { + fail("Caught exception: " + e.getMessage(), e); + } + } + } + + @Test(description = "Tests syntax validation of a valid FeatureDef config") + public void testFeatureDefConfigWithValidSyntax() { + ValidationResult expResult = new ValidationResult(SYNTACTIC, VALID); + + try (ConfigDataProvider cdp = new StringConfigDataProvider(ConfigValidatorFixture.validFeatureDefConfig)) { + ValidationResult obsResult = _validator.validate(FeatureDef, SYNTACTIC, cdp); + + assertEquals(obsResult, expResult); + } catch (Exception e) { + fail("Caught exception: " + e.getMessage(), e); + } + } + + @Test(description = "Tests syntax validation of an invalid FeatureDef config") + public void testFeatureDefConfigWithInvalidSyntax() { + try (ConfigDataProvider cdp = new StringConfigDataProvider(ConfigValidatorFixture.invalidFeatureDefConfig)) { + ValidationResult obsResult = _validator.validate(FeatureDef, SYNTACTIC, cdp); + + assertEquals(obsResult.getValidationStatus(), INVALID); + assertTrue(obsResult.getDetails().isPresent()); + assertTrue(obsResult.getCause().isPresent()); + + // Get details and verify that there are no error messages related to (syntactially valid) anchor A3 + String details = obsResult.getDetails().get(); + assertFalse(details.contains("#/anchors/A3")); + } catch (Exception e) { + fail("Caught exception: " + e.getMessage(), e); + } + } + + @Test(description = "Tests syntax validation of a valid Join config") + public void testJoinConfigWithValidSyntax() { + List configStrings = Arrays.asList(ConfigValidatorFixture.validJoinConfigWithSingleFeatureBag, ConfigValidatorFixture.validJoinConfigWithMultFeatureBags); + + ValidationResult expResult = new ValidationResult(SYNTACTIC, VALID); + + for (String cfgStr : configStrings) { + try (ConfigDataProvider cdp = new StringConfigDataProvider(cfgStr)) { + ValidationResult obsResult = _validator.validate(Join, SYNTACTIC, cdp); + + assertEquals(obsResult, expResult); + } catch (Exception e) { + fail("Caught exception: " + e.getMessage(), e); + } + } + } + + @Test(description = "Tests syntax validation of an invalid Join config") + public void testJoinConfigWithInvalidSyntax() { + try (ConfigDataProvider cdp = new StringConfigDataProvider(ConfigValidatorFixture.invalidJoinConfig)) { + ValidationResult obsResult = _validator.validate(Join, SYNTACTIC, cdp); + + assertEquals(obsResult.getValidationStatus(), INVALID); + assertTrue(obsResult.getDetails().isPresent()); + assertTrue(obsResult.getCause().isPresent()); + } catch (Exception e) { + fail("Caught exception: " + e.getMessage(), e); + } + } + + @Test(description = "Tests syntax validation of both FeatureDef and Join config together") + public void testFeatureDefAndJoinConfigSyntax() { + Map configTypeWithDataProvider = new HashMap<>(); + + try (ConfigDataProvider featureDefCdp = new StringConfigDataProvider(ConfigValidatorFixture.validFeatureDefConfig); + ConfigDataProvider joinCdp = new StringConfigDataProvider( + ConfigValidatorFixture.validJoinConfigWithSingleFeatureBag) + ) { + configTypeWithDataProvider.put(FeatureDef, featureDefCdp); + configTypeWithDataProvider.put(Join, joinCdp); + + ValidationResult expResult = new ValidationResult(SYNTACTIC, VALID); + + Map obsResult = _validator.validate(configTypeWithDataProvider, SYNTACTIC); + assertEquals(obsResult.get(FeatureDef), expResult); + assertEquals(obsResult.get(Join), expResult); + } catch (Exception e) { + fail("Caught exception: " + e.getMessage(), e); + } + } + + /** + * In galene library, Frame-Galene online scoring uses frame-core to read frame-galene.conf as FeatureDef conf. + * For now, we need to make sure the syntax used in frame-galene.conf is supported in validation + */ + @Test(description = "Tests syntax validation of an valid Frame-Galene scoring config") + public void testFrameGaleneScoringConfigWithValidSyntax() { + try (ConfigDataProvider cdp = new ResourceConfigDataProvider("frame-galene.conf")) { + ValidationResult obsResult = _validator.validate(FeatureDef, SYNTACTIC, cdp); + if (obsResult.getValidationStatus() != VALID) { + String details = obsResult.getDetails().orElse(""); + } + + assertEquals(obsResult.getValidationStatus(), VALID); + + } catch (Exception e) { + fail("Caught exception: " + e.getMessage(), e); + } + } + + @Test(description = "Tests build of identifying valid FrameGalene configs") + public void testFrameGaleneConfigValidCases() { + ConfigRenderOptions _renderOptions = ConfigRenderOptions.defaults() + .setComments(false) + .setOriginComments(false) + .setFormatted(true) + .setJson(true); + ConfigParseOptions _parseOptions = ConfigParseOptions.defaults() + .setSyntax(ConfigSyntax.CONF) // HOCON document + .setAllowMissing(false); + InputStream inputStream = JoinConfig.class.getClassLoader() + .getResourceAsStream("FeatureDefConfigSchema.json"); + JSONObject rawSchema = new JSONObject(new JSONTokener(inputStream)); + Schema schema = SchemaLoader.load(rawSchema); + Config myCfg = ConfigFactory.parseResources("frame-feature-careers-featureDef-offline.conf", _parseOptions); + String jsonStr = myCfg.root().render(_renderOptions); + JSONTokener tokener = new JSONTokener(jsonStr); + JSONObject root = new JSONObject(tokener); + try { + schema.validate(root); + } catch (ValidationException e) { + System.out.println(e.toJSON()); + throw e; + } + } +} diff --git a/feathr-config/src/test/java/com/linkedin/feathr/core/configvalidator/typesafe/ConfigSchemaTest.java b/feathr-config/src/test/java/com/linkedin/feathr/core/configvalidator/typesafe/ConfigSchemaTest.java new file mode 100644 index 000000000..0651db850 --- /dev/null +++ b/feathr-config/src/test/java/com/linkedin/feathr/core/configvalidator/typesafe/ConfigSchemaTest.java @@ -0,0 +1,171 @@ +package com.linkedin.feathr.core.configvalidator.typesafe; + +import com.linkedin.feathr.core.configbuilder.typesafe.consumer.JoinFixture; +import com.linkedin.feathr.core.config.consumer.JoinConfig; +import com.typesafe.config.Config; +import com.typesafe.config.ConfigFactory; +import com.typesafe.config.ConfigParseOptions; +import com.typesafe.config.ConfigRenderOptions; +import com.typesafe.config.ConfigSyntax; +import java.io.IOException; +import java.io.InputStream; +import org.everit.json.schema.Schema; +import org.everit.json.schema.ValidationException; +import org.json.JSONObject; +import org.json.JSONTokener; +import org.testng.annotations.Test; +import org.everit.json.schema.loader.SchemaLoader; + +import static org.testng.Assert.assertEquals; + + +public class ConfigSchemaTest { + + ConfigRenderOptions _renderOptions = ConfigRenderOptions.defaults() + .setComments(false) + .setOriginComments(false) + .setFormatted(true) + .setJson(true); + ConfigParseOptions _parseOptions = ConfigParseOptions.defaults() + .setSyntax(ConfigSyntax.CONF) // HOCON document + .setAllowMissing(false); + + @Test(description = "Tests build of identifying invalid Frame configs") + public void testFrameConfigInvalidCases() { + int invalidCount = 0; + // initialize to different numbers and overwrite by test code below + int totalCount = -999; + try (InputStream inputStream = JoinConfig.class.getClassLoader() + .getResourceAsStream("FeatureDefConfigSchema.json")) { + try { + JSONObject rawSchema = new JSONObject(new JSONTokener(inputStream)); + Schema schema = SchemaLoader.load(rawSchema); + + Config myCfg = ConfigFactory.parseResources("FeatureDefSchemaTestInvalidCases.conf", _parseOptions); + String jsonStr = myCfg.root().render(_renderOptions); + JSONTokener tokener = new JSONTokener(jsonStr); + JSONObject root = new JSONObject(tokener); + + JSONObject anchors = root.getJSONObject("anchors"); + JSONObject sources = root.getJSONObject("sources"); + JSONObject derivations = root.getJSONObject("derivations"); + totalCount = anchors.keySet().size() + sources.keySet().size() + derivations.keySet().size(); + JSONObject newConfig = new JSONObject(); + newConfig.put("anchors", new JSONObject()); + newConfig.put("sources", new JSONObject()); + newConfig.put("derivations", new JSONObject()); + // construct a case for each one of the anchors/sources/derived features to test + for (String key : anchors.keySet()) { + newConfig.getJSONObject("anchors").put(key, anchors.getJSONObject(key)); + try { + schema.validate(newConfig); + } catch (ValidationException ex) { + invalidCount += 1; + } + newConfig.getJSONObject("anchors").remove(key); + } + for (String key : sources.keySet()) { + newConfig.getJSONObject("sources").put(key, sources.getJSONObject(key)); + try { + schema.validate(newConfig); + } catch (ValidationException ex) { + invalidCount += 1; + } + newConfig.getJSONObject("sources").remove(key); + } + for (String key : derivations.keySet()) { + if (derivations.get(key) instanceof JSONObject) { + newConfig.getJSONObject("derivations").put(key, derivations.getJSONObject(key)); + } else { + newConfig.getJSONObject("derivations").put(key, derivations.get(key)); + } + try { + schema.validate(newConfig); + } catch (ValidationException ex) { + invalidCount += 1; + } + newConfig.getJSONObject("derivations").remove(key); + } + } catch (Exception e) { + e.printStackTrace(); + } + } catch (IOException e) { + e.printStackTrace(); + } + assertEquals(invalidCount, totalCount); + } + + @Test(description = "Tests build of identifying valid Frame configs") + public void testFrameConfigValidCases() { + InputStream inputStream = JoinConfig.class.getClassLoader() + .getResourceAsStream("FeatureDefConfigSchema.json"); + JSONObject rawSchema = new JSONObject(new JSONTokener(inputStream)); + Schema schema = SchemaLoader.load(rawSchema); + Config myCfg = ConfigFactory.parseResources("FeatureDefSchemaTestCases.conf", _parseOptions); + String jsonStr = myCfg.root().render(_renderOptions); + JSONTokener tokener = new JSONTokener(jsonStr); + JSONObject root = new JSONObject(tokener); + try { + schema.validate(root); + } catch (ValidationException e) { + System.out.println(e.toJSON()); + throw e; + } + } + + + @Test(description = "Tests build of identifying valid join configs") + public void testJoinConfigValidCases() { + Config myCfg = ConfigFactory.parseResources("JoinSchemaTestCases.conf", _parseOptions); + validateJoinConfig(myCfg); + } + + + @Test(description = "Tests build of valid join config with absolute time range") + public void testJoinConfigWithAbsTimeRange() { + Config myCfg = ConfigFactory.parseString(JoinFixture.settingsWithAbsoluteTimeRange, _parseOptions); + validateJoinConfig(myCfg); + } + + @Test(description = "Tests build of valid join config with useLatestFeatureData") + public void testJoinConfigWithUseLatestFeatureData() { + Config myCfg = ConfigFactory.parseString(JoinFixture.settingsWithLatestFeatureData, _parseOptions); + validateJoinConfig(myCfg); + } + + + @Test(description = "Tests valid join config with time_window_join and negative value for simulate_time_delay") + public void testSettingWithNegativeSimulateTimeDelay() { + Config myCfg = ConfigFactory.parseString(JoinFixture.settingsWithTimeWindowConfigAndNegativeTimeDelay, _parseOptions); + validateJoinConfig(myCfg); + } + + @Test(expectedExceptions = ValidationException.class, + description = "Tests invalid join config invalid pattern for simulate_time_delay") + public void testTimeWindowJoinSettingWithInvalidNegativeSimulateTimeDelay() { + Config myCfg = ConfigFactory.parseString(JoinFixture.invalidSettingsWithTimeWindowConfigNegativeTimeDelay, _parseOptions); + validateJoinConfig(myCfg); + } + + @Test(expectedExceptions = ValidationException.class, description = "Tests invalid join config with only start time") + public void testTimeWindowJoinSettingWithNoEndTime() { + Config myCfg = ConfigFactory.parseString(JoinFixture.invalidWithOnlyStartTime, _parseOptions); + validateJoinConfig(myCfg); + } + + @Test(expectedExceptions = ValidationException.class, description = "Tests invalid join config with no timestamp format") + public void testTimeWindowJoinSettingWithNoTimestampFormat() { + Config myCfg = ConfigFactory.parseString(JoinFixture.invalidWithNoTimestampFormat, _parseOptions); + validateJoinConfig(myCfg); + } + + private void validateJoinConfig(Config cfg) { + InputStream inputStream = JoinConfig.class.getClassLoader().getResourceAsStream("JoinConfigSchema.json"); + JSONObject rawSchema = new JSONObject(new JSONTokener(inputStream)); + Schema schema = SchemaLoader.load(rawSchema); + String jsonStr = cfg.root().render(_renderOptions); + JSONTokener tokener = new JSONTokener(jsonStr); + JSONObject root = new JSONObject(tokener); + schema.validate(root); + } +} diff --git a/feathr-config/src/test/java/com/linkedin/feathr/core/configvalidator/typesafe/ExtractorClassValidationUtilsTest.java b/feathr-config/src/test/java/com/linkedin/feathr/core/configvalidator/typesafe/ExtractorClassValidationUtilsTest.java new file mode 100644 index 000000000..c7929202c --- /dev/null +++ b/feathr-config/src/test/java/com/linkedin/feathr/core/configvalidator/typesafe/ExtractorClassValidationUtilsTest.java @@ -0,0 +1,60 @@ +package com.linkedin.feathr.core.configvalidator.typesafe; + +import com.linkedin.feathr.core.config.ConfigType; +import com.linkedin.feathr.core.configdataprovider.ConfigDataProvider; +import com.linkedin.feathr.core.configdataprovider.StringConfigDataProvider; +import java.io.IOException; +import java.util.Collections; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.testng.Assert; +import org.testng.annotations.Test; + +import static org.testng.Assert.*; + + +/** + * Test class for {@link ExtractorClassValidationUtils} + */ +public class ExtractorClassValidationUtilsTest { + @Test(description = "Test getting classes from FeatureDef conf with Join conf") + public void testGetClassesWithJoinConf() { + try ( + ConfigDataProvider featureDefProvider + = new StringConfigDataProvider(FeatureDefConfFixture.featureDefWithExtractors); + ConfigDataProvider joinProvider + = new StringConfigDataProvider(JoinConfFixture.joinConf1) + ) { + Map map = Stream.of(new Object[][] { + {ConfigType.FeatureDef, featureDefProvider}, + {ConfigType.Join, joinProvider}, + }).collect(Collectors.toMap(d -> (ConfigType) d[0], d -> (ConfigDataProvider) d[1])); + + Set extractors = ExtractorClassValidationUtils.getExtractorClasses(map); + Set expectedExtractors = new HashSet<>(FeatureDefConfFixture.expectedExtractors); + // if Join config provided, won't return extractors that are not used + expectedExtractors.remove("com.linkedin.frame.online.anchor.test.ExtractorNotUsed"); + + Assert.assertEquals(extractors, expectedExtractors); + + } catch (IOException e) { + fail("Error in building config", e); + } + } + + @Test(description = "Test getting classes from FeatureDef conf without Join conf") + public void testGetClassesWithoutJoinConf() { + try (ConfigDataProvider featureDefProvider + = new StringConfigDataProvider(FeatureDefConfFixture.featureDefWithExtractors)) { + Map map = + Collections.singletonMap(ConfigType.FeatureDef, featureDefProvider); + Set extractors = ExtractorClassValidationUtils.getExtractorClasses(map); + Assert.assertEquals(extractors, FeatureDefConfFixture.expectedExtractors); + } catch (Throwable e) { + fail("Error in building config", e); + } + } +} diff --git a/feathr-config/src/test/java/com/linkedin/feathr/core/configvalidator/typesafe/FeatureConsumerConfValidatorTest.java b/feathr-config/src/test/java/com/linkedin/feathr/core/configvalidator/typesafe/FeatureConsumerConfValidatorTest.java new file mode 100644 index 000000000..08f71a087 --- /dev/null +++ b/feathr-config/src/test/java/com/linkedin/feathr/core/configvalidator/typesafe/FeatureConsumerConfValidatorTest.java @@ -0,0 +1,52 @@ +package com.linkedin.feathr.core.configvalidator.typesafe; + +import com.linkedin.feathr.core.config.ConfigType; +import com.linkedin.feathr.core.configbuilder.typesafe.TypesafeConfigBuilder; +import com.linkedin.feathr.core.configdataprovider.ConfigDataProvider; +import com.linkedin.feathr.core.configdataprovider.ResourceConfigDataProvider; +import com.linkedin.feathr.core.configdataprovider.StringConfigDataProvider; +import com.linkedin.feathr.core.configvalidator.ConfigValidatorFixture; +import com.linkedin.feathr.core.configvalidator.ValidationResult; +import com.linkedin.feathr.core.configvalidator.ValidationStatus; +import com.linkedin.feathr.core.configvalidator.ValidationType; +import java.util.HashMap; +import java.util.Map; +import org.testng.Assert; +import org.testng.annotations.Test; + +import static org.testng.Assert.*; + + +/** + * Test class for {@link FeatureConsumerConfValidator} + */ +public class FeatureConsumerConfValidatorTest { + private FeatureConsumerConfValidator _featureConsumerConfValidator = new FeatureConsumerConfValidator(); + private TypesafeConfigBuilder _configBuilder = new TypesafeConfigBuilder(); + + @Test(description = "test validation for Frame feature consumer") + public void testRequestUnreachableFeatures() { + try { + Map configs = new HashMap<>(); + configs.put(ConfigType.FeatureDef, new ResourceConfigDataProvider("invalidSemanticsConfig/feature-not-reachable-def.conf")); + configs.put(ConfigType.Join, new StringConfigDataProvider(ConfigValidatorFixture.joinConfig1)); + + // perform syntax validation + Map syntaxResult = _featureConsumerConfValidator.validate(configs, ValidationType.SYNTACTIC); + ValidationResult featureDefSyntaxResult = syntaxResult.get(ConfigType.FeatureDef); + Assert.assertEquals(featureDefSyntaxResult.getValidationStatus(), ValidationStatus.VALID); + ValidationResult joinSyntaxResult = syntaxResult.get(ConfigType.Join); + Assert.assertEquals(joinSyntaxResult.getValidationStatus(), ValidationStatus.VALID); + + // perform semantic validation + Map semanticResult = _featureConsumerConfValidator.validate(configs, ValidationType.SEMANTIC); + ValidationResult featureDefSemanticResult = semanticResult.get(ConfigType.FeatureDef); + Assert.assertEquals(featureDefSemanticResult.getValidationStatus(), ValidationStatus.WARN); + ValidationResult joinSemanticResult = semanticResult.get(ConfigType.Join); + Assert.assertEquals(joinSemanticResult.getValidationStatus(), ValidationStatus.INVALID); + + } catch (Throwable e) { + fail("Error in building config", e); + } + } +} diff --git a/feathr-config/src/test/java/com/linkedin/feathr/core/configvalidator/typesafe/FeatureDefConfFixture.java b/feathr-config/src/test/java/com/linkedin/feathr/core/configvalidator/typesafe/FeatureDefConfFixture.java new file mode 100644 index 000000000..a1d9bb6e6 --- /dev/null +++ b/feathr-config/src/test/java/com/linkedin/feathr/core/configvalidator/typesafe/FeatureDefConfFixture.java @@ -0,0 +1,217 @@ +package com.linkedin.feathr.core.configvalidator.typesafe; + +import java.util.HashSet; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + + +class FeatureDefConfFixture { + static final String featureDefWithMvel = String.join("\n", + "// all possible feature definitions using MVEL", + "{", + " \"anchors\": {", + + " // SimpleFeatureConfig", + " industry-local: {", + " source: \"LocalSQLAnchorTest/industry.avro.json\"", + " features: {", + " waterloo_member_geoCountry_local: \"$.countryCode in geoStdData\"", + " }", + " }", + + " // ComplexFeatureConfig", + " swaAnchorWithKeyExtractor: {", + " source: \"swaSource\"", + " keyExtractor: \"com.linkedin.frame.offline.SimpleSampleKeyExtractor\"", + " features: {", + " waterloo_job_standardizedSkillsString: {", + " def: \"aggregationWindow\"", + " aggregation: SUM", + " window: 3d", + " }", + " }", + " }", + + " // TimeWindowFeatureConfig", + " nearLineFeatureAnchor: {", + " source: kafkaTestSource,", + " key.mvel: \"a in b\",", + " features: {", + " maxPV12h: {", + " def.mvel: pageView,", + " aggregation: MAX,", + " windowParameters: {", + " type: SLIDING,", + " size: 1h,", + " slidingInterval: 10m,", + " },", + " groupBy: pageKey,", + " filter.mvel: \"$.getAsTermVector().keySet()\"", + " }", + " }", + " }", + " }", + + " \"derivations\": {", + + " // SimpleFeatureConfig", + " \"waterloo_member_geoCountry_local_alias\": \"waterloo_member_geoCountry_local\",", + + " abuse_member_invitation_inboundOutboundSkew: { ", + " sqlExpr: \"case when abuse_member_invitation_numInviters = 0 then -1 else abuse_member_invitation_numInvites/abuse_member_invitation_numInviters end\"", + " },", + + " \"waterloo_member_job_cosineSimilarity\": {", + " \"key\": [", + " \"m\",", + " \"j\"", + " ],", + " \"inputs\": {", + " \"a\": {", + " \"key\": \"m\",", + " \"feature\": \"waterloo_member_geoCountry_local\"", + " },", + " \"b\": {", + " \"key\": \"j\",", + " \"feature\": \"waterloo_job_standardizedSkillsString\"", + " }", + " },", + " \"definition\": \"cosineSimilarity(a, b)\",", + " type: \"NUMERIC\"", + " },", + " }", + "}"); + + static final String featureDefWithHdfsSource = String.join("\n", + "sources: {", + " hdfsSource1: {", + " location: { path: \"/data/tracking_column/test\" }", + " isTimeSeries: true", + " timeWindowParameters: {", + " timestamp: \"timestamp\"", + " timestamp_format: \"yyyy-MM-dd\"" + " }", + " }", + + " hdfsSource2: {", + " type: \"HDFS\"", + " location: { path: \"/jobs/metrics/ump_v2/metrics/test/test/test/test\" }", + " isTimeSeries: true", + " timeWindowParameters: {", + " timestamp: \"metadataMap.timestamp.STRING\"", + " timestamp_format: \"epoch\"", + " }", + " }", + + " hdfsSource3: {", + " location: { path: \"/jobs/metrics/udp/datafiles/test\" }", + " }", + "}", + + "anchors: {", + " testAnchor1: { ", + " source: \"/jobs/metrics/udp/snapshot/test/#LATEST\" ", + " keyAlias: \"x\" ", + " extractor: \"com.linkedin.frame.feature.anchor.TestExtractor\" ", + " features: [ ", + " test_feature_1 ", + " ] ", + " } ", + "}" + ); + + static final String featureDefWithExtractors = String.join("\n", + "anchors: { ", + " offlineAnchor1: { ", + " source: \"/test/test/test/#LATEST\" ", + " extractor: \"com.linkedin.frame.offline.anchor.test.Extractor1\" ", + " features: [ ", + " offline_feature1_1 ", + " ] ", + " } ", + + " offlineAnchor2: { ", + " source: \"/test/test/test/#LATEST\" ", + " transformer: \"com.linkedin.frame.offline.anchor.test.Transformer2\" ", + " features: [ ", + " \"offline_feature2_1\", ", + " \"offline_feature2_2\"", + " ] ", + " } ", + + " offlineAnchor3: { ", + " source: \"/test/test/test/#LATEST\" ", + " keyExtractor: \"com.linkedin.frame.offline.anchor.test.KeyExtractor3\" ", + " features: { ", + " offline_feature3_1: { ", + " def: \"count\" ", + " filter: \"name = 'queryCount14d'\" ", + " aggregation: LATEST ", + " window: 3d ", + " default: 0.0 ", + " } ", + " } ", + " } ", + + " offlineAnchor4: { ", + " source: \"/test/test/test/#LATEST\" ", + " extractor: \"com.linkedin.frame.offline.anchor.test.Extractor4\" ", + " keyExtractor: \"com.linkedin.frame.offline.anchor.test.KeyExtractor4\" ", + " features: [ ", + " \"offline_feature4_1\", ", + " \"offline_feature4_2\"", + " ] ", + " } ", + + " \"onlineAnchor1\": {", + " source: \"testSource\"", + " extractor: {class: \"com.linkedin.frame.online.anchor.test.Extractor1\"}", + " features: [", + " online_feature1_1", + " ]", + " }", + + " \"onlineAnchor2\": {", + " source: \"testSource\"", + " extractor: {class: \"com.linkedin.frame.online.anchor.test.Extractor2\"}", + " features: [", + " online_feature2_1", + " ]", + " }", + + " \"onlineAnchorNotUsed\": {", + " source: \"testSource\"", + " extractor: {class: \"com.linkedin.frame.online.anchor.test.ExtractorNotUsed\"}", + " features: [", + " online_feature_not_used", + " ]", + " }", + "}", + + "derivations: { ", + " derived_feature_1: { ", + " key: [\"member\"] ", + " inputs: [ { key: \"member\", feature: \"offline_feature3_1\"} ] ", + " class: \"com.linkedin.frame.offline.derived.DerivedExtractor1\" ", + " }", + + " derived_feature_2: \"import com.linkedin.frame.offline.derived.DerivationUtil; DerivationUtil.extractRegionCode(online_feature1_1)\"", + + " derived_feature_3: \"online_feature2_1\"", + " derived_feature_4: \"derived_feature_3\"", + "}"); + + static Set expectedExtractors; + static { + expectedExtractors = Stream.of("com.linkedin.frame.offline.anchor.test.Extractor1", + "com.linkedin.frame.offline.anchor.test.Transformer2", + "com.linkedin.frame.offline.anchor.test.KeyExtractor3", + "com.linkedin.frame.offline.anchor.test.Extractor4", + "com.linkedin.frame.offline.anchor.test.KeyExtractor4", + "com.linkedin.frame.online.anchor.test.Extractor1", + "com.linkedin.frame.online.anchor.test.Extractor2", + "com.linkedin.frame.online.anchor.test.ExtractorNotUsed", + "com.linkedin.frame.offline.derived.DerivedExtractor1") + .collect(Collectors.toSet()); + } +} diff --git a/feathr-config/src/test/java/com/linkedin/feathr/core/configvalidator/typesafe/FeatureDefConfSemanticValidatorTest.java b/feathr-config/src/test/java/com/linkedin/feathr/core/configvalidator/typesafe/FeatureDefConfSemanticValidatorTest.java new file mode 100644 index 000000000..cb784608b --- /dev/null +++ b/feathr-config/src/test/java/com/linkedin/feathr/core/configvalidator/typesafe/FeatureDefConfSemanticValidatorTest.java @@ -0,0 +1,259 @@ +package com.linkedin.feathr.core.configvalidator.typesafe; + +import com.linkedin.feathr.core.config.ConfigType; +import com.linkedin.feathr.core.config.producer.FeatureDefConfig; +import com.linkedin.feathr.core.configbuilder.typesafe.TypesafeConfigBuilder; +import com.linkedin.feathr.core.configdataprovider.ConfigDataProvider; +import com.linkedin.feathr.core.configdataprovider.ResourceConfigDataProvider; +import com.linkedin.feathr.core.configdataprovider.StringConfigDataProvider; +import com.linkedin.feathr.core.configvalidator.ValidationResult; +import com.linkedin.feathr.core.configvalidator.ValidationStatus; +import com.linkedin.feathr.core.configvalidator.ValidationType; +import com.linkedin.feathr.exception.FeathrConfigException; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.testng.Assert; +import org.testng.annotations.Test; + +import static org.testng.Assert.*; + + +/** + * Tests for {@link FeatureDefConfigSemanticValidator} + */ +public class FeatureDefConfSemanticValidatorTest { + private TypesafeConfigBuilder configBuilder = new TypesafeConfigBuilder(); + private FeatureDefConfigSemanticValidator configValidator = new FeatureDefConfigSemanticValidator(); + private MvelValidator mvelValidator = MvelValidator.getInstance(); + private HdfsSourceValidator hdfsSourceValidator = HdfsSourceValidator.getInstance(); + + + @Test(description = "Tests getting duplicate feature names in FeatureDef config") + public void testGetDuplicateFeatureNames() { + try (ConfigDataProvider provider = new ResourceConfigDataProvider("invalidSemanticsConfig/duplicate-feature.conf")) { + FeatureDefConfigSemanticValidator featureDefConfigSemanticValidator = new FeatureDefConfigSemanticValidator(); + FeatureDefConfig featureDefConfig = configBuilder.buildFeatureDefConfig(provider); + ValidationResult validationResult = featureDefConfigSemanticValidator.validate(featureDefConfig); + Assert.assertEquals(validationResult.getValidationStatus(), ValidationStatus.WARN); + Assert.assertEquals(validationResult.getDetails().toString(), "Optional[The following features' definitions are duplicate: \n" + + "member_lixSegment_isJobSeeker]"); + } catch (Throwable e) { + fail("Error in building config", e); + } + } + + + @Test(description = "Tests config failure when duplicate source names are in several FeatureDef configs") + public void testMultipleConfigDuplicateSourceNames() { + + List resources = Arrays.asList("invalidSemanticsConfig/duplicate-feature.conf", + "invalidSemanticsConfig/undefined-source.conf"); + + try (ConfigDataProvider featureDefConfigProvider = new ResourceConfigDataProvider(resources)) { + FeatureConsumerConfValidator validator = new FeatureConsumerConfValidator(); + Map configTypeWithDataProvider = new HashMap<>(); + configTypeWithDataProvider.put(ConfigType.FeatureDef, featureDefConfigProvider); + Map validationResultMap = + validator.validate(configTypeWithDataProvider, ValidationType.SEMANTIC); + + ValidationResult validationResult = validationResultMap.get(ConfigType.FeatureDef); + Assert.assertEquals(validationResult.getValidationStatus(), ValidationStatus.WARN); + String expected = "Optional[The following source name(s) are " + + "duplicates between two or more feature definition configs: \n" + + "source name: member_derived_data\n" + + "File paths of two or more files that have duplicate source names: \n" + + "Resources: [invalidSemanticsConfig/duplicate-feature.conf, invalidSemanticsConfig/undefined-source.conf] "; + Assert.assertEquals(validationResult.getDetails().toString().substring(0,307), expected); + } catch (Throwable e) { + fail("Error in building config", e); + } + } + + @Test(description = "Tests getting undefined sources in anchors from FeatureDef config") + public void testGetUndefinedAnchorSources() { + try (ConfigDataProvider provider = new ResourceConfigDataProvider("invalidSemanticsConfig/undefined-source.conf")) { + FeatureDefConfig featureDefConfig = configBuilder.buildFeatureDefConfig(provider); + + Map undefinedAnchorSources = + configValidator.getUndefinedAnchorSources(featureDefConfig); + + Assert.assertEquals(undefinedAnchorSources.size(), 1); + Assert.assertTrue(undefinedAnchorSources.containsKey("memberLixSegmentV2")); + Assert.assertEquals(undefinedAnchorSources.get("memberLixSegmentV2"), "member_derived_date"); + + } catch (Throwable e) { + fail("Error in building config", e); + } + } + + @Test(description = "Tests approved extractor with parameters won't throw exception.") + public void testApprovedExtractorWithParams() { + try (ConfigDataProvider provider = new ResourceConfigDataProvider("extractor-with-params.conf")) { + FeatureDefConfig featureDefConfig = configBuilder.buildFeatureDefConfig(provider); + + configValidator.validateApprovedExtractorWithParameters(featureDefConfig); + } catch (Throwable e) { + fail("Error in building config", e); + } + } + + @Test(description = "Tests non-approved extractor with parameters will throw exception.", expectedExceptions = FeathrConfigException.class) + public void testNonApprovedExtractorWithParams() throws Exception { + try (ConfigDataProvider provider = new ResourceConfigDataProvider( + "invalidSemanticsConfig/extractor-with-params-not-approved.conf")) { + FeatureDefConfig featureDefConfig = configBuilder.buildFeatureDefConfig(provider); + + configValidator.validateApprovedExtractorWithParameters(featureDefConfig); + } + } + + @Test(description = "Tests getting all reachable and unreachable features in FeatureDef config with an invalid config.") + public void testGetReachableFeatures() { + + try (ConfigDataProvider provider = new ResourceConfigDataProvider( + "invalidSemanticsConfig/feature-not-reachable-def.conf")) { + FeatureDefConfig featureDefConfig = configBuilder.buildFeatureDefConfig(provider); + Map> featureAccessInfo = configValidator.getFeatureAccessInfo(featureDefConfig); + + Set reachableFeatures = featureAccessInfo.get(FeatureReachType.REACHABLE); + Set expectedReachableFeatures = new HashSet<>(); + expectedReachableFeatures.add("feature1"); + expectedReachableFeatures.add("feature2"); + expectedReachableFeatures.add("derived_feature_1"); + expectedReachableFeatures.add("derived_feature_2"); + Assert.assertEquals(reachableFeatures.size(), 4); + Assert.assertEquals(reachableFeatures, expectedReachableFeatures); + + Set unreachableFeatures = featureAccessInfo.get(FeatureReachType.UNREACHABLE); + Set expectedUnreachableFeatures = new HashSet<>(); + expectedUnreachableFeatures.add("feature3"); + expectedUnreachableFeatures.add("derived_feature_3"); + Assert.assertEquals(unreachableFeatures.size(), 2); + Assert.assertEquals(unreachableFeatures, expectedUnreachableFeatures); + } catch (Throwable e) { + fail("Error in building config", e); + } + } + + @Test(description = "Test MVEL heuristic validation for single MVEL expression") + public void testSingleMvelHeuristicCheckWithIn() { + Assert.assertTrue(mvelValidator.heuristicProjectionExprCheck("(parent.name in users)")); + Assert.assertTrue(mvelValidator.heuristicProjectionExprCheck("(name in (familyMembers in users))")); + Assert.assertTrue(mvelValidator.heuristicProjectionExprCheck("myFunc(abc)")); + Assert.assertFalse(mvelValidator.heuristicProjectionExprCheck("parent.name in users")); + Assert.assertFalse(mvelValidator.heuristicProjectionExprCheck("(name in familyMembers in users)")); + Assert.assertFalse(mvelValidator.heuristicProjectionExprCheck("(some expression) familyMembers in users")); + } + + @Test(description = "Test feature MVEL extracting") + public void testExtractingMvelFromFeatureDef() { + try (ConfigDataProvider provider = new StringConfigDataProvider(FeatureDefConfFixture.featureDefWithMvel)) { + FeatureDefConfig featureDefConfig = configBuilder.buildFeatureDefConfig(provider); + Map mvelDef = mvelValidator.getFeatureMvels(featureDefConfig); + Map expectedResult = new HashMap() {{ + put("waterloo_member_geoCountry_local", "$.countryCode in geoStdData"); + put("waterloo_member_job_cosineSimilarity", "cosineSimilarity(a, b)"); + put("maxPV12h", "pageView"); + put("waterloo_member_geoCountry_local_alias", "waterloo_member_geoCountry_local"); + }}; + Assert.assertEquals(mvelDef, expectedResult); + } catch (Throwable e) { + fail("Error in building config", e); + } + } + + @Test(description = "Test anchor key MVEL extracting") + public void testExtractingMvelFromAnchor() { + try (ConfigDataProvider provider = new StringConfigDataProvider(FeatureDefConfFixture.featureDefWithMvel)) { + FeatureDefConfig featureDefConfig = configBuilder.buildFeatureDefConfig(provider); + Map> mvelDef = mvelValidator.getAnchorKeyMvels(featureDefConfig); + Map> expectedResult = new HashMap>() {{ + put("nearLineFeatureAnchor", Collections.singletonList("a in b")); // the anchor key MVEL expr + }}; + Assert.assertEquals(mvelDef, expectedResult); + } catch (Throwable e) { + fail("Error in building config", e); + } + } + + @Test(description = "Test MVEL heuristic check") + public void testMvelHeuristicCheck() { + try (ConfigDataProvider provider = new StringConfigDataProvider(FeatureDefConfFixture.featureDefWithMvel)) { + FeatureDefConfig featureDefConfig = configBuilder.buildFeatureDefConfig(provider); + Map> invalidMvels = mvelValidator.getPossibleInvalidMvelsUsingIn(featureDefConfig); + Map> expectedResult = new HashMap>() {{ + put("waterloo_member_geoCountry_local", Collections.singletonList("$.countryCode in geoStdData")); + put("nearLineFeatureAnchor", Collections.singletonList("a in b")); // the anchor key MVEL expr + }}; + Assert.assertEquals(invalidMvels, expectedResult); + } catch (Throwable e) { + fail("Error in building config", e); + } + } + + @Test(description = "Test MVEL validator") + public void testMvelValidator() { + try (ConfigDataProvider provider = new StringConfigDataProvider(FeatureDefConfFixture.featureDefWithMvel)) { + FeatureDefConfig featureDefConfig = configBuilder.buildFeatureDefConfig(provider); + ValidationResult result = mvelValidator.validate(featureDefConfig); + Assert.assertEquals(result.getValidationStatus(), ValidationStatus.WARN); + } catch (Throwable e) { + fail("Error in building config", e); + } + } + + @Test(description = "Test getting invalid Hdfs source") + public void testGetHdfsInvalidManagedDataSets() { + try (ConfigDataProvider provider = new StringConfigDataProvider(FeatureDefConfFixture.featureDefWithHdfsSource)) { + FeatureDefConfig featureDefConfig = configBuilder.buildFeatureDefConfig(provider); + Map invalidDataSets = hdfsSourceValidator.getInvalidManagedDataSets(featureDefConfig); + Map expectedResult = new HashMap() {{ + put("hdfsSource1", "/data/tracking_column/test"); + put("hdfsSource2", "/jobs/metrics/ump_v2/metrics/test/test/test/test"); + put("hdfsSource3", "/jobs/metrics/udp/datafiles/test"); + put("testAnchor1", "/jobs/metrics/udp/snapshot/test/#LATEST"); + }}; + Assert.assertEquals(invalidDataSets, expectedResult); + } catch (Throwable e) { + fail("Error in building config", e); + } + } + + @Test(description = "Test HdfsSource validator") + public void testHdfsSourceValidator() { + try (ConfigDataProvider provider = new StringConfigDataProvider(FeatureDefConfFixture.featureDefWithHdfsSource)) { + FeatureDefConfig featureDefConfig = configBuilder.buildFeatureDefConfig(provider); + ValidationResult result = hdfsSourceValidator.validate(featureDefConfig); + Assert.assertEquals(result.getValidationStatus(), ValidationStatus.WARN); + } catch (Throwable e) { + fail("Error in building config", e); + } + } + + @Test(description = "Test getting required features") + public void testGetRequiredFeatures() { + try (ConfigDataProvider provider = new StringConfigDataProvider(FeatureDefConfFixture.featureDefWithExtractors)) { + FeatureDefConfig featureDefConfig = configBuilder.buildFeatureDefConfig(provider); + Set requestedFeatures = Stream.of("offline_feature1_1", "offline_feature2_1", "offline_feature4_1", + "derived_feature_1", "derived_feature_2", "derived_feature_4").collect(Collectors.toSet()); + + Set requiredFeatures = + FeatureDefConfigSemanticValidator.getRequiredFeatureNames(featureDefConfig, requestedFeatures); + + Set expectedRequiredFeatures = Stream.of("offline_feature1_1", "offline_feature2_1", "offline_feature3_1", + "offline_feature4_1", "online_feature1_1", "online_feature2_1", "derived_feature_1", + "derived_feature_2", "derived_feature_3", "derived_feature_4").collect(Collectors.toSet()); + + Assert.assertEquals(requiredFeatures, expectedRequiredFeatures); + } catch (Throwable e) { + fail("Error in building config", e); + } + } +} diff --git a/feathr-config/src/test/java/com/linkedin/feathr/core/configvalidator/typesafe/FeatureProducerConfValidatorTest.java b/feathr-config/src/test/java/com/linkedin/feathr/core/configvalidator/typesafe/FeatureProducerConfValidatorTest.java new file mode 100644 index 000000000..149d40b2e --- /dev/null +++ b/feathr-config/src/test/java/com/linkedin/feathr/core/configvalidator/typesafe/FeatureProducerConfValidatorTest.java @@ -0,0 +1,46 @@ +package com.linkedin.feathr.core.configvalidator.typesafe; + +import com.linkedin.feathr.core.config.ConfigType; +import com.linkedin.feathr.core.configbuilder.typesafe.TypesafeConfigBuilder; +import com.linkedin.feathr.core.configdataprovider.ConfigDataProvider; +import com.linkedin.feathr.core.configdataprovider.ResourceConfigDataProvider; +import com.linkedin.feathr.core.configdataprovider.StringConfigDataProvider; +import com.linkedin.feathr.core.configvalidator.ConfigValidatorFixture; +import com.linkedin.feathr.core.configvalidator.ValidationResult; +import com.linkedin.feathr.core.configvalidator.ValidationStatus; +import com.linkedin.feathr.core.configvalidator.ValidationType; +import java.util.HashMap; +import java.util.Map; +import org.testng.Assert; +import org.testng.annotations.Test; + + +/** + * Test class for {@link FeatureProducerConfValidator} + */ +public class FeatureProducerConfValidatorTest { + private FeatureProducerConfValidator _featureProducerConfValidator = new FeatureProducerConfValidator(); + private TypesafeConfigBuilder _configBuilder = new TypesafeConfigBuilder(); + + @Test(expectedExceptions = RuntimeException.class, + description = "test unsupported Config type for Frame feature producer") + public void testUnsupportedConfigType() { + Map configs = new HashMap<>(); + configs.put(ConfigType.FeatureDef, new ResourceConfigDataProvider("invalidSemanticsConfig/feature-not-reachable-def.conf")); + configs.put(ConfigType.Join, new StringConfigDataProvider(ConfigValidatorFixture.joinConfig1)); + + // perform semantic validation + Map semanticResult = _featureProducerConfValidator.validate(configs, ValidationType.SEMANTIC); + } + + @Test(description = "For Frame feature producer, feature reachable validation won't be applied") + public void testRequestUnreachableFeatures() { + Map configs = new HashMap<>(); + configs.put(ConfigType.FeatureDef, new ResourceConfigDataProvider("invalidSemanticsConfig/feature-not-reachable-def.conf")); + + // perform semantic validation + Map semanticResult = _featureProducerConfValidator.validate(configs, ValidationType.SEMANTIC); + ValidationResult featureDefSemanticResult = semanticResult.get(ConfigType.FeatureDef); + Assert.assertEquals(featureDefSemanticResult.getValidationStatus(), ValidationStatus.VALID); + } +} diff --git a/feathr-config/src/test/java/com/linkedin/feathr/core/configvalidator/typesafe/JoinConfFixture.java b/feathr-config/src/test/java/com/linkedin/feathr/core/configvalidator/typesafe/JoinConfFixture.java new file mode 100644 index 000000000..df00c1305 --- /dev/null +++ b/feathr-config/src/test/java/com/linkedin/feathr/core/configvalidator/typesafe/JoinConfFixture.java @@ -0,0 +1,38 @@ +package com.linkedin.feathr.core.configvalidator.typesafe; + +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + + +public class JoinConfFixture { + + static final String joinConf1 = String.join("\n", + "featureBag1: [ ", + " { ", + " key: [id1] ", + " featureList: [ ", + " offline_feature1_1,", + " offline_feature2_1,", + " offline_feature4_1,", + " ] ", + " } ", + "] ", + + "featureBag2: [", + " {", + " key: [id1]", + " featureList: [", + " derived_feature_1,", + " derived_feature_2,", + " derived_feature_4", + " ]", + " }", + "]"); + + static final Set requestedFeatureNames1; + static { + requestedFeatureNames1 = Stream.of("offline_feature1_1", "offline_feature2_1", "offline_feature4_1", + "derived_feature_1", "derived_feature_2", "derived_feature_4").collect(Collectors.toSet()); + } +} diff --git a/feathr-config/src/test/java/com/linkedin/feathr/core/configvalidator/typesafe/JoinConfSemanticValidatorTest.java b/feathr-config/src/test/java/com/linkedin/feathr/core/configvalidator/typesafe/JoinConfSemanticValidatorTest.java new file mode 100644 index 000000000..697ecf69e --- /dev/null +++ b/feathr-config/src/test/java/com/linkedin/feathr/core/configvalidator/typesafe/JoinConfSemanticValidatorTest.java @@ -0,0 +1,82 @@ +package com.linkedin.feathr.core.configvalidator.typesafe; + +import com.linkedin.feathr.core.config.consumer.JoinConfig; +import com.linkedin.feathr.core.config.producer.FeatureDefConfig; +import com.linkedin.feathr.core.configbuilder.typesafe.TypesafeConfigBuilder; +import com.linkedin.feathr.core.configdataprovider.ConfigDataProvider; +import com.linkedin.feathr.core.configdataprovider.ResourceConfigDataProvider; +import com.linkedin.feathr.core.configdataprovider.StringConfigDataProvider; +import com.linkedin.feathr.core.configvalidator.ConfigValidatorFixture; +import com.linkedin.feathr.core.configvalidator.ValidationResult; +import com.linkedin.feathr.core.configvalidator.ValidationStatus; +import com.linkedin.feathr.core.configvalidator.ValidationType; +import java.util.Map; +import java.util.Set; +import org.testng.Assert; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import static org.testng.Assert.*; + + +/** + * Test class for {@link JoinConfSemanticValidator} + */ +public class JoinConfSemanticValidatorTest { + private TypesafeConfigBuilder _configBuilder = new TypesafeConfigBuilder(); + private JoinConfSemanticValidator _joinConfSemanticValidator = new JoinConfSemanticValidator(); + + private Map> _featureReachableInfo; + + @BeforeClass + public void init() { + try (ConfigDataProvider featureDefProvider = + new ResourceConfigDataProvider("invalidSemanticsConfig/feature-not-reachable-def.conf")) { + FeatureDefConfigSemanticValidator featureDefConfSemanticValidator = new FeatureDefConfigSemanticValidator(); + FeatureDefConfig featureDefConfig = _configBuilder.buildFeatureDefConfig(featureDefProvider); + + _featureReachableInfo = featureDefConfSemanticValidator.getFeatureAccessInfo(featureDefConfig); + } catch (Throwable e) { + fail("Error in building config", e); + } + } + + @Test(description = "Tests requesting unreachable features") + public void testRequestUnreachableFeatures() { + try (ConfigDataProvider joinConfProvider = new StringConfigDataProvider(ConfigValidatorFixture.joinConfig1)) { + JoinConfig joinConfig = _configBuilder.buildJoinConfig(joinConfProvider); + + ValidationResult validationResult = _joinConfSemanticValidator.validate(joinConfig, _featureReachableInfo); + Assert.assertEquals(validationResult.getValidationType(), ValidationType.SEMANTIC); + Assert.assertEquals(validationResult.getValidationStatus(), ValidationStatus.INVALID); + Assert.assertNotNull(validationResult.getDetails()); + } catch (Throwable e) { + fail("Error in building config", e); + } + } + + @Test(description = "Tests requesting undefined features") + public void testRequestUndefinedFeatures() { + try (ConfigDataProvider joinConfProvider = new StringConfigDataProvider(ConfigValidatorFixture.joinConfig2)) { + JoinConfig joinConfig = _configBuilder.buildJoinConfig(joinConfProvider); + + ValidationResult validationResult = _joinConfSemanticValidator.validate(joinConfig, _featureReachableInfo); + Assert.assertEquals(validationResult.getValidationType(), ValidationType.SEMANTIC); + Assert.assertEquals(validationResult.getValidationStatus(), ValidationStatus.INVALID); + Assert.assertNotNull(validationResult.getDetails()); + } catch (Throwable e) { + fail("Error in building config", e); + } + } + + @Test(description = "Test get requested features") + public void testGetRequestedFeatures() { + try (ConfigDataProvider joinConfProvider = new StringConfigDataProvider(JoinConfFixture.joinConf1)) { + JoinConfig joinConfig = _configBuilder.buildJoinConfig(joinConfProvider); + Set requestedFeatureNames = JoinConfSemanticValidator.getRequestedFeatureNames(joinConfig); + Assert.assertEquals(requestedFeatureNames, JoinConfFixture.requestedFeatureNames1); + } catch (Throwable e) { + fail("Error in building config", e); + } + } +} diff --git a/feathr-config/src/test/java/com/linkedin/feathr/core/configvalidator/typesafe/PresentationsConfigSchemaTest.java b/feathr-config/src/test/java/com/linkedin/feathr/core/configvalidator/typesafe/PresentationsConfigSchemaTest.java new file mode 100644 index 000000000..44b01d1ef --- /dev/null +++ b/feathr-config/src/test/java/com/linkedin/feathr/core/configvalidator/typesafe/PresentationsConfigSchemaTest.java @@ -0,0 +1,40 @@ +package com.linkedin.feathr.core.configvalidator.typesafe; + +import com.linkedin.feathr.core.config.consumer.JoinConfig; +import com.typesafe.config.Config; +import com.typesafe.config.ConfigFactory; +import com.typesafe.config.ConfigParseOptions; +import com.typesafe.config.ConfigRenderOptions; +import com.typesafe.config.ConfigSyntax; +import java.io.InputStream; +import org.everit.json.schema.Schema; +import org.everit.json.schema.loader.SchemaLoader; +import org.json.JSONObject; +import org.json.JSONTokener; +import org.testng.annotations.Test; + + +public class PresentationsConfigSchemaTest { + + ConfigRenderOptions _renderOptions = ConfigRenderOptions.defaults() + .setComments(false) + .setOriginComments(false) + .setFormatted(true) + .setJson(true); + ConfigParseOptions _parseOptions = ConfigParseOptions.defaults() + .setSyntax(ConfigSyntax.CONF) // HOCON document + .setAllowMissing(false); + + + @Test(description = "Tests build of identifying valid presentations configs") + public void testPresentationsConfigValidCases() { + InputStream inputStream = JoinConfig.class.getClassLoader().getResourceAsStream("PresentationsConfigSchema.json"); + JSONObject rawSchema = new JSONObject(new JSONTokener(inputStream)); + Schema schema = SchemaLoader.load(rawSchema); + Config myCfg = ConfigFactory.parseResources("PresentationsSchemaTestCases.conf", _parseOptions); + String jsonStr = myCfg.root().render(_renderOptions); + JSONTokener tokener = new JSONTokener(jsonStr); + JSONObject root = new JSONObject(tokener); + schema.validate(root); + } +} diff --git a/feathr-config/src/test/java/com/linkedin/feathr/core/configvalidator/typesafe/TypesafeConfigValidatorTest.java b/feathr-config/src/test/java/com/linkedin/feathr/core/configvalidator/typesafe/TypesafeConfigValidatorTest.java new file mode 100644 index 000000000..b8d902bc7 --- /dev/null +++ b/feathr-config/src/test/java/com/linkedin/feathr/core/configvalidator/typesafe/TypesafeConfigValidatorTest.java @@ -0,0 +1,101 @@ +package com.linkedin.feathr.core.configvalidator.typesafe; + +import com.linkedin.feathr.core.configvalidator.ConfigValidator; +import com.linkedin.feathr.core.config.ConfigType; +import com.linkedin.feathr.core.configbuilder.typesafe.TypesafeConfigBuilder; +import com.linkedin.feathr.core.configdataprovider.ConfigDataProvider; +import com.linkedin.feathr.core.configdataprovider.StringConfigDataProvider; +import com.linkedin.feathr.core.configvalidator.ValidationResult; +import com.linkedin.feathr.core.configvalidator.ValidationStatus; +import com.linkedin.feathr.core.configvalidator.ValidationType; +import com.typesafe.config.Config; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import static com.linkedin.feathr.core.config.ConfigType.*; +import static com.linkedin.feathr.core.configvalidator.ConfigValidatorFixture.*; +import static com.linkedin.feathr.core.configvalidator.ValidationStatus.*; +import static com.linkedin.feathr.core.configvalidator.ValidationType.*; +import static org.testng.Assert.*; + + +/** + * Unit tests for {@link TypesafeConfigValidator}. Tests are provided for only those methods that are public but not + * provided as part of {@link ConfigValidator ConfigValidator}. + */ +public class TypesafeConfigValidatorTest { + private TypesafeConfigValidator _validator; + + @BeforeClass + public void init() { + _validator = new TypesafeConfigValidator(); + } + + @Test(description = "Tests validation of FeatureDef config syntax") + public void testFeatureDefConfigSyntax() { + ValidationResult expResult = new ValidationResult(SYNTACTIC, VALID); + runAndValidate(FeatureDef, validFeatureDefConfig, expResult); + } + + @Test(description = "Legacy feature def configs with global section should fail the validation") + public void testFeatureDefConfigWithLegacyGlobalSection() { + runAndValidate(FeatureDef, legacyFeatureDefConfigWithGlobals, SYNTACTIC, INVALID); + } + + @Test(description = "Tests validation of Join config syntax") + public void testJoinConfigSyntax() { + ValidationResult expResult = new ValidationResult(SYNTACTIC, VALID); + runAndValidate(Join, validJoinConfigWithSingleFeatureBag, expResult); + } + + @Test(description = "Test validation of FeatureDef naming validation") + public void testNamingValidation() { + ConfigDataProvider cdp = new StringConfigDataProvider(invalidFeatureDefConfig2); + ValidationResult obsResult = _validator.validate(FeatureDef, SYNTACTIC, cdp); + + assertEquals(obsResult.getValidationStatus(), WARN); + assertNotNull(obsResult.getDetails().orElse(null)); + } + + @Test(description = "Tests validation of Presentation config syntax") + public void testPresentationConfigSyntax() { + ValidationResult expResult = new ValidationResult(SYNTACTIC, VALID); + runAndValidate(Presentation, validPresentationConfig, expResult); + } + + @Test(description = "Test validation of anchors with parameters") + public void testValidParameterizedAnchorConfig() { + ValidationResult expResult = new ValidationResult(SYNTACTIC, VALID); + runAndValidate(FeatureDef, validFeatureDefConfigWithParameters, expResult); + } + + @Test(description = "Test invalid anchors with parameters. The parameters are invalid because they are not of string type") + public void testInvalidParameterizedAnchorConfig() { + runAndValidate(FeatureDef, invalidFeatureDefConfigWithParameters, SYNTACTIC, INVALID); + } + + private void runAndValidate(ConfigType configType, String configStr, ValidationResult expResult) { + try (ConfigDataProvider cdp = new StringConfigDataProvider(configStr)) { + TypesafeConfigBuilder builder = new TypesafeConfigBuilder(); + Config config = builder.buildTypesafeConfig(configType, cdp); + ValidationResult obsResult = _validator.validateSyntax(configType, config); + + assertEquals(obsResult, expResult); + } catch (Exception e) { + fail("Caught exception: " + e.getMessage(), e); + } + } + + private void runAndValidate(ConfigType configType, String configStr, ValidationType validationType, ValidationStatus validationStatus) { + try (ConfigDataProvider cdp = new StringConfigDataProvider(configStr)) { + TypesafeConfigBuilder builder = new TypesafeConfigBuilder(); + Config config = builder.buildTypesafeConfig(configType, cdp); + ValidationResult obsResult = _validator.validateSyntax(configType, config); + + assertEquals(obsResult.getValidationType(), validationType); + assertEquals(obsResult.getValidationStatus(), validationStatus); + } catch (Exception e) { + fail("Caught exception: " + e.getMessage(), e); + } + } +} diff --git a/feathr-config/src/test/java/com/linkedin/feathr/core/utils/ConfigUtilsTest.java b/feathr-config/src/test/java/com/linkedin/feathr/core/utils/ConfigUtilsTest.java new file mode 100644 index 000000000..504e1720f --- /dev/null +++ b/feathr-config/src/test/java/com/linkedin/feathr/core/utils/ConfigUtilsTest.java @@ -0,0 +1,25 @@ +package com.linkedin.feathr.core.utils; + +import com.linkedin.feathr.core.configbuilder.ConfigBuilderException; +import org.testng.annotations.Test; + + +public class ConfigUtilsTest { + @Test(description = "Tests validating timestamp pattern.") + public void testTimestampPatternValidCases() { + ConfigUtils.validateTimestampPatternWithEpoch("Default", "2020/10/01", "yyyy/MM/dd"); + ConfigUtils.validateTimestampPatternWithEpoch("Default", "2020/10/01/00/00/00","yyyy/MM/dd/HH/mm/ss"); + ConfigUtils.validateTimestampPatternWithEpoch("Default", "1601279713", "epoch"); + ConfigUtils.validateTimestampPatternWithEpoch("Default", "1601279713000", "epoch_millis"); + } + + @Test(expectedExceptions = ConfigBuilderException.class, description = "Tests validating timestamp pattern.") + public void testTimestampPatternInvalidValidCase1() { + ConfigUtils.validateTimestampPatternWithEpoch("Default", "2020/10/01","yyy/mm/dd"); + } + + @Test(expectedExceptions = ConfigBuilderException.class, description = "Tests validating timestamp pattern.") + public void testTimestampPatternInvalidValidCase2() { + ConfigUtils.validateTimestampPatternWithEpoch("Default", "1601279713","epcho"); + } +} diff --git a/feathr-config/src/test/java/com/linkedin/feathr/core/utils/MvelInputsResolverTest.java b/feathr-config/src/test/java/com/linkedin/feathr/core/utils/MvelInputsResolverTest.java new file mode 100644 index 000000000..8fb4bdc49 --- /dev/null +++ b/feathr-config/src/test/java/com/linkedin/feathr/core/utils/MvelInputsResolverTest.java @@ -0,0 +1,61 @@ +package com.linkedin.feathr.core.utils; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import static org.testng.Assert.*; + + +public class MvelInputsResolverTest { + MvelInputsResolver _mvelInputsResolver = MvelInputsResolver.getInstance(); + + @DataProvider + public Object[][] testGetInputFeaturesDataProvider() { + return new Object[][]{ + // Tests simple alias syntax + {"featureA", Collections.singletonList("featureA")}, + // Tests Mvel expresion with multiple input features with no import + {"featureA + featureB", Arrays.asList("featureA", "featureB")}, + // Test fully-qualified existing class that starts with com will work + {"com.linkedin.frame.core.utils.Object.apply(featureA, featureB ) ; ", + Arrays.asList("featureA", "featureB")}, + // Test fully-qualified existing class that starts with org will work + {"org.linkedin.frame.core.utils.Object.apply(featureA, featureB ) ; ", + Arrays.asList("featureA", "featureB")}, + // Test fully-qualified existing class that starts with java will work + {"java.lang.Object.apply(featureA, featureB ) ; ", + Arrays.asList("featureA", "featureB")}, + // Tests Mvel expresion with additional whitespaces + {" import com.linkedin.frame.core.utils.MemberJobFunctionToYoeExtractor ; MemberJobFunctionToYoeExtractor.apply(featureA, featureB ) ; ", + Arrays.asList("featureA", "featureB")}, + // Test Mvel with built-in frame functions + {"getTerms(careers_job_applicants_90d).size()", Collections.singletonList("careers_job_applicants_90d")}, + // Test Mvel with complex projections + {"if (isNonZero(waterloo_member_location)) {([$.getKey.substring(11) : $.getValue] in waterloo_member_location.getValue().entrySet() if $.getKey.startsWith('geo_region='))}", + Collections.singletonList("waterloo_member_location")}, + // Test mvel with null + {"isPresent(waterloo_member_location) ? Math.abs(waterloo_member_location) : null", + Collections.singletonList("waterloo_member_location")}, + // Test mvel with numbers + {"isPresent(waterloo_member_location) ? waterloo_member_location : 0.0", + Collections.singletonList("waterloo_member_location")}, + // Tests Mvel expresion with multiple input features with multiple imports + {"import com.linkedin.frame.core.utils.MemberJobFunctionToYoeExtractor; MemberJobFunctionToYoeExtractor.apply(featureA, featureB);", + Arrays.asList("featureA", "featureB")}, + // Tests Mvel expresion with multiple input features with multiple imports + {"import com.linkedin.frame.stz.ExtractorA; import com.linkedin.frame.stz.ExtractorB; ExtractorA.test(featureA) + ExtractorB.apply(featureB, featureC);", + Arrays.asList("featureA", "featureB", "featureC")}, + // Tests Mvel expresion with multiple input features and constant, with single imports + {"import com.linkedin.frame.stz.Extractor; Extractor.test(featureA, featureB, 100L, 'a_constant_string');", + Arrays.asList("featureA", "featureB")}}; + } + + @Test(dataProvider = "testGetInputFeaturesDataProvider") + public void testGetInputFeatures(String input, List expected) { + List inputFeatures = _mvelInputsResolver.getInputFeatures(input); + assertEquals(inputFeatures, expected); + } +} diff --git a/feathr-config/src/test/resources/Bar.txt b/feathr-config/src/test/resources/Bar.txt new file mode 100644 index 000000000..f8a96e228 --- /dev/null +++ b/feathr-config/src/test/resources/Bar.txt @@ -0,0 +1,2 @@ +There is no greatness where there is not simplicity, goodness, and truth. +The strongest of all warriors are these two — Time and Patience. \ No newline at end of file diff --git a/feathr-config/src/test/resources/FeatureDefSchemaTestCases.conf b/feathr-config/src/test/resources/FeatureDefSchemaTestCases.conf new file mode 100644 index 000000000..f6e81382e --- /dev/null +++ b/feathr-config/src/test/resources/FeatureDefSchemaTestCases.conf @@ -0,0 +1,702 @@ +{ + "sources": { + "source1": { + "location": { + "path": "source-simple.json" + } + }, + "source2": { + "location": { + "path": "source-simple.json" + }, + "hasTimeSnapshot": false + }, + + "source23": { + "location": { + "path": "source-simple.json" + }, + "hasTimeSnapshot": "False" + }, + "MemberStdCmp": { + "type": "ESPRESSO", + "database": "StandardizationEI", + "table": "MemberStandardizedCompany", + "d2Uri": "d2://ESPRESSO_MT2" + "keyExpr": "key[0]" + }, + "JYMBIIMemberFeatures": { + "type": "VENICE", + "storeName": "JYMBIIMemberFeatures", + "keyExpr": "com.linkedin.jobs.relevance.frame.online.util.AvroKeyGeneratorJymbiiMemberSourceKey.getKey(key[0])", + }, + "MemberPreferenceData": { + "type": "RESTLI", + "restResourceName": "jobSeekers", + "keyExpr": "member" + }, + "MemberPreferenceData2": { + "type": "RESTLI", + "restResourceName": "jobSeekers", + "restEntityType": "member" + }, + "MemberPreferenceData3": { + "type": "RESTLI", + "restResourceName": "jobSeekers", + "finder": "rule" + }, + "memberDerivedData": { + "type": "RESTLI", + "restResourceName": "memberDerivedData", + "restEntityType": "member", + "pathSpec": "standardizedSkills,standardizedIndustries,standardizedProfileIndustries,standardizedLocation,standardizedEducations,standardizedPositions" + }, + "CareersMemberEntityEmbeddings-0.0.2": { + "type": "VENICE", + "storeName": "CareersMemberEntityEmbeddings", + "keyExpr": "{\"entityUrn\" : new com.linkedin.common.urn.Urn(\"member\", key[0]).toString(), \"version\" : \"0.0.2\"}" + }, + + "kafkaTestSource": { + "type": "KAFKA", + "stream": "kafka.testCluster.testTopic" + }, + "rocksDBTestSource": { + "type": "ROCKSDB", + "referenceSource": "kafkaTestSource", + "extractFeatures": true, + "encoder": "com.linkedin.frame.online.config.FoobarExtractor", + "decoder": "com.linkedin.frame.online.config.FoobarExtractor", + "keyExpr": "keyExprName" + }, + "rocksDBTestSourceWithoutKeyExpr": { + "type": "ROCKSDB", + "referenceSource": "kafkaTestSource", + "extractFeatures": true, + "encoder": "com.linkedin.frame.online.config.FoobarExtractor", + "decoder": "com.linkedin.frame.online.config.FoobarExtractor", + }, + "jobScoringEntity": { + "type": "PASSTHROUGH", + "dataModel": "com.linkedin.jobsprediction.JobScoringEntity" + }, + "jobScoringEntityCustomSource": { + "type": "CUSTOM", + "keyExpr": "key[0]", + "dataModel": "com.linkedin.jobsprediction.JobScoringEntity" + }, + "hiringProjectCandidates": { + type: RESTLI + restResourceName: "hiringProjectCandidates" + keyExpr: "toCompoundKey({\"hiringContext\": toUrn(\"contract\", key[0]), \"hiringProject\": toUrn(\"hiringProject\", toUrn(\"contract\", key[0]), key[1])})" + finder: "hiringProject" + restReqParams: { + CandidateHiringStates: {mvel: "[toUrn(\"candidateHiringState\", toUrn(\"contract\", key[0]), key[2])]"}, + } + }, + "MemberConnectionIntersection": { + "type": "RESTLI", + "restResourceName": "setOperations", + "restEntityType": "member", + "restReqParams": { + "operator": "INTERSECT", + "edgeSetSpecifications": { + "jsonArray": "{\"array\": [{\"firstEdgeType\":\"MemberToMember\", \"secondEdgeType\":\"MemberToMember\"}]}" + }, + "second": { + "mvel": "key[1]" + }, + "a":{ + "file":"sd" + } + } + }, + "contentShareWindowAggLegacySource": { + "type": "HDFS", + "location": { + "path": "/jobs/mlf/contentShareFeatures/daily" + }, + "isTimeSeries": "true", + "timeWindowParameters": { + "timestamp": "timestamp", + "timestamp_format": "yyyy/MM/dd" + } + }, + "contentShareWindowAggSource": { + "type": "HDFS", + "location": { + "path": "/jobs/mlf/contentShareFeatures/daily" + }, + "timePartitionPattern": "yyyy/MM/dd", + "timeWindowParameters": { + "timestampColumn": "timestamp", + "timestampColumnFormat": "yyyy/MM/dd" + } + }, + "sourceWithTimeAwarePath": { + "type": "HDFS", + "location": { + "path": "/jobs/mlf/contentShareFeatures/daily" + }, + "timePartitionPattern": "yyyy/MM/dd" + }, + + "couchbaseTestSource": { + "type": "COUCHBASE", + "bucketName": "testBucket" + "keyExpr": "key[0]", + "bootstrapUris": ["some-app.corp.linkedin.com:8091", "other-app.corp.linkedin.com:8091"], + "documentModel": "com.linkedin.frame.online.SomeDocumentClass" + }, + "couchbaseTestSource2": { + "type": "COUCHBASE", + "bucketName": "testBucket" + "keyExpr": "key[0]", + "documentModel": "com.linkedin.frame.online.SomeDocumentClass" + }, + ContentTopic: { + location: {path: "/data/databases/TopicTags/AlgorithmicTopicTagsV2/#LATEST"} + }, + "recentPageViewsSource": { + "type": "PINOT" + "resourceName": "recentMemberActionsPinotQuery" + "queryTemplate": "SELECT objectAttributes, timeStampSec FROM RecentMemberActions WHERE actorId IN (?) AND timeStampSec > ? ORDER BY timeStampSec DESC LIMIT 1000" + "queryArguments": ["key[0]", "System.currentTimeMillis()/1000 - 2 * 24 * 60 * 60"] + "queryKeyColumns": ["actorId"] + } + }, + "anchors": { + accessTimeFeatures: { + source: "/jobs/emerald/Features/LatestFeatures/accessTimeStats/#LATEST", + key.sqlExpr: "x", + keyAlias: "x", + features: { + // Using same default value as in emerald + abuse_member_accessTime_lastVisitedTime: { + def.sqlExpr: "lastVisitedTime", + default: 0.0, + type: "NUMERIC" + } + abuse_member_accessTime_daysSinceLastVisitedTime: { + def.sqlExpr: "daysSinceLastVisitedTime", + default: 0.0, + type: "NUMERIC" + } + } + } + + industry-local: { + source: "LocalSQLAnchorTest/industry.avro.json" + key.sqlExpr: industryId + features: { + waterloo_member_geoCountry_local.def.sqlExpr: "geoStdData.countryCode" + } + } + + // this is an existing in production feature definition waterloo-member-derived-data-skills-by-source-v5 + // it contains extractor, and MVEL feature definition together + "test-member-derived-data-skills-by-source-v5": { + source: "memberDerivedData-skillV5" + extractor: {class: "com.linkedin.frame.feature.online.TestMemberSkillV5TermVectorTransformer"} + features: { + test_member_standardizedSkillsV5_explicit: + """standardizedSkills == null ? [] : + ([getIdFromRawUrn($.skill.entity) : $.skill.score] in standardizedSkills if ($.skillSource == 'EXPLICIT'))""" + test_member_standardizedSkillsV5_implicit: + """standardizedSkills == null ? [] : + ([getIdFromRawUrn($.skill.entity) : $.skill.score] in standardizedSkills if ($.skillSource == 'IMPLICIT'))""" + } + } + + "test-member-derived-data-skills-by-source-v5-with-type": { + source: "memberDerivedData-skillV5" + extractor: {class: "com.linkedin.frame.feature.online.TestMemberSkillV5TermVectorTransformer"} + features: { + test_member_standardizedSkillsV5_explicit_type: { + def: "mvel", + default: 0 + type: NUMERIC + } + test_member_standardizedSkillsV5_implicit_type: { + def: "mvel", + default: 0 + type: { + type: VECTOR + } + } + } + } + + waterloo-member-geolocation-local: { + source: "LocalSQLAnchorTest/member.avro.json" + key.sqlExpr: "x" + features: { + MemberIndustryId: { + def.sqlExpr: profileIndustryId + default: 1 + type: NUMERIC + } + } + } + + swaAnchorWithKeyExtractor: { + source: "swaSource" + keyExtractor: "com.linkedin.frame.offline.SimpleSampleKeyExtractor" + features: { + f3: { + def: "aggregationWindow" + aggregation: SUM + window: 3d + type: { + type: "NUMERIC" + shape: [10, 10] + dimensionType: ["INT", "INT"] + valType: "FLOAT" + } + } + } + } + + careers-member-lix-segment: { + source: "/data/derived/lix/euc/member/#LATEST" + key: "id" + features: { + careers_member_lixSegment_isJobSeeker: { + def: "job_seeker_class == 'active'", + type: "BOOLEAN" + } + } + } + + "member-sent-invitations": { + "source": "/jobs/frame/inlab/data/features/InvitationStats", + "key": "x", + "lateralViewParameters": { + "lateralViewDef": "explode(features)", + "lateralViewItemAlias": "feature" + }, + "features": { + "member_sentInvitations_numIgnoredRejectedInvites": { + "def": "toNumeric(numIgnoredRejectedInvites)", + "default": "123", + type: "BOOLEAN" + } + } + }, + "featuresWithKey": { + "source": "/data/test/#LATEST", + "key": "x", + "keyAlias": "x", + "features": { + "waterloo_member_geoCountry": "geoStdData.countryCode" + } + }, + nearLineFeatureAnchor: { + source: kafkaTestSource, + key.mvel: mid, + features: { + maxPV12h: { + def.mvel: pageView, + aggregation: MAX, + windowParameters: { + type: SLIDING, + size: 1h, + slidingInterval: 10m, + }, + groupBy: pageKey, + filter.mvel: "$.getAsTermVector().keySet()" + } + } + }, + pageViewCountAnchor: { + source: "PageViewEvent" + key: "header.x" + features: { + "pageViewCount4h" : { + def: "pageType" + aggregation: "MAX_POOLING" + windowParameters: { + type: SLIDING + size: 1m + slidingInterval: 10s + } + } + } + }, + SWAfeatureWithMinAgg: { + source: partitionedHDFSSource + key: "x" + features: { + SWAfeatureWithMinAgg: { + def: count + aggregation: MIN + window: 2d + } + } + } + "featuresWithOnlyMVEL": { + "source": "/data/test/#LATEST", + "features": { + "waterloo_member_geoCountry": "geoStdData.countryCode", + "waterloo_member_geoRegion": "geoStdData.countryCode + ':' + geoStdData.regionCode" + } + }, + "featuresWithTransformer": { + "source": "/data/databases/CareersPreferenceDB/MemberPreference/#LATEST", + "transformer": "com.linkedin.jymbii.frame.anchor.PreferencesFeatures", + "keyAlias": "x", + "features": [ + "jfu_preference_companySize,", + "jfu_preference_seniority,", + "jfu_preference_industry,", + "jfu_preference_industryCategory,", + "jfu_preference_location" + ] + }, + "featuresWithTransformerAndExtract": { + "source": "/jobs/liar/jymbii-features-engineering/production/memberFeatures/education/#LATEST", + "transformer": "com.linkedin.jymbii.frame.anchor.LegacyFeastFormattedFeatures", + "features": [ + "jfu_member_degree" + ], + "extract": [ + { + "extract": "member_degree", + "as": "jfu_member_degree" + } + ] + }, + "flagship-viralActionAffinityWithActorFrame-1-0": { + source: "FeedViewerTensorStore" + extractor: {"class": "com.linkedin.flagship.frame.extractor.SingleTensorDataExtractor"} + features: { + flagship-viralActionAffinityWithActorFrame-1-0 : { + type: "TENSOR" + } + } + }, + "flagship-viewerFrame-1-0": { + source: "FeedViewerTensorStore" + features: { + flagship-viralActionAffinityWithActorFrame-1-0 : { + def: "viewer" + type: "TENSOR" + } + } + }, + "flagship-viewerFrame-2-0": { + source: "FeedViewerTensorStore" + features: { + flagship-viralActionAffinityWithActorFrame-2-0 : { + def: "viewer" + type: { + type: "TENSOR" + tensorCategory: "DENSE" + shape: [10] + dimensionType: ["INT"] + valType: FLOAT + } + } + } + }, + "featuresWithExtractor": { + "source": "/data/databases/CareersPreferenceDB/MemberPreference/#LATEST", + "extractor": "com.linkedin.jymbii.frame.anchor.PreferencesFeatures", + "keyAlias": "x", + "features": [ + "jfu_preference_companySize" + ] + } , + "featuresWithExtractorClass": { + "source": "/data/databases/CareersPreferenceDB/MemberPreference/#LATEST", + "key": "mockKey" + "extractor": {"class":"com.linkedin.jymbii.frame.anchor.PreferencesFeatures"}, + "features": [ + "jfu_preference_companySize," + ] + }, + "contentShareWindowAggAnchor": { + "source": "contentShareWindowAggSource", + "key": "id", + "keyAlias": "x", + "features": { + "fc_feed_7d_share_third_party_article_count": { + "def": "thirdPartyArticleCount", + "aggregation": "SUM", + "window": "7d", + type: "BOOLEAN" + } + } + } + + couchbase-features: { + source: "couchbaseTestSource" + extractor: {"class": "com.linkedin.frame.extractor.CustomFeatureExtractor"} + features: [ + couchbase-one-sample-feature, + couchbase-another-sample-feature + ] + } + + couchbase-features-with-params: { + source: "couchbaseTestSource" + extractor: { + class: "com.linkedin.frame.extractor.CustomFeatureExtractor" + params: { + abc: "test_string" + features: [comm_influenceScore, other_comm_influenceBucket, simpleSWAFeature] + columnName: "testColumn" + } + } + features: [ + couchbase-one-sample-feature-with-params, + couchbase-another-sample-feature-with-params + ] + }, + jobActivityCareersJobEmbedding100Anchor: { + source: "jobActivityCareersJobEmbedding100FactTableSource" + key: "substring(header.x,15)" + features: { + mlf_member_jobActivityCareersJobEmbedding100_jobApply_avg_4d: { + def: "careersJobEmbedding" + filter: "action IN ('APPLY_OFFSITE', 'APPLY_ONSITE')" + aggregation: AVG_POOLING + window: 4d + embeddingSize: 200 + default: 0.0, + type: "NUMERIC" + } + } + } + + offlineAnchor4: { + source: "/test/test/test/#LATEST" + extractor: "com.linkedin.frame.offline.anchor.test.Extractor4" + keyExtractor: "com.linkedin.frame.offline.anchor.test.KeyExtractor4" + features: [ + "offline_feature4_1", + "offline_feature4_2" + ] + }, + "recentPageViewsAnchor": { + source: "recentPageViewsSource" + extractor: "com.linkedin.flagship.search.PinotPageViewFeaturesExtractor" + features: [ + "recent_page_views" + ] + }, + "mostRecentJobApplyAnchor": { + source: "mostRecentJobApplySource" + extractor: "com.linkedin.flagship.search.PinotJobApplyFeaturesExtractor" + features: [ + "most_recent_job_apply" + ] + } + }, + "derivations": { + "waterloo_member_summary_alias": "waterloo_member_summary", + abuse_member_invitation_inboundOutboundSkew:{ + sqlExpr: "case when abuse_member_invitation_numInviters = 0 then -1 else abuse_member_invitation_numInvites/abuse_member_invitation_numInviters end" + }, + simpleMvelDerivedTypeCast: { + definition: simpleHDFSMvelCount + type: CATEGORICAL + }, + sessions_v2_macrosessions_sum_sqrt_7d: { + key: id + inputs: { + sessions_v2_macrosessions_sum_7d: {key: id, feature: sessions_v2_macrosessions_sum_7d}, + } + definition.sqlExpr: "sqrt(sessions_v2_macrosessions_sum_7d)" + type: "NUMERIC" + }, + "jfu_member_placeSimTopK": { + "key": [ + "member" + ], + "inputs": [ + { + "key": "member", + "feature": "jfu_resolvedPreference_location" + } + ], + "class": "com.linkedin.jymbii.nice.derived.MemberPlaceSimTopK" + type: "NUMERIC" + }, + "waterloo_member_pastTitleString:waterloo_job_standardizedSkillsString": { + "key": [ + "m", + "j" + ], + "inputs": { + "a": { + "key": "m", + "feature": "waterloo_member_pastTitleString" + }, + "b": { + "key": "j", + "feature": "waterloo_job_standardizedSkillsString" + } + }, + "definition": "cosineSimilarity(a, b)", + type: "NUMERIC" + }, + seq_join_feature1: { + key: "x" + join: { + base: { key: x, feature: MemberIndustryId } + expansion: { key: skillId, feature: MemberIndustryName } + } + aggregation:"" + type: "NUMERIC" + }, + seq_join_feature2: { + key: "x" + join: { + base: { key: x, + feature: MemberIndustryId, + outputKey: x, + transformation: "import com.linkedin.frame.MyFeatureUtils; MyFeatureUtils.dotProduct(MemberIndustryId);"} + expansion: { key: skillId, feature: MemberIndustryName } + } + aggregation:"ELEMENTWISE_MAX" + type: "NUMERIC" + }, + seq_join_feature3: { + key: "x" + join: { + base: { key: x, + feature: MemberIndustryId, + outputKey: x, + transformationClass: "com.linkedin.frame.MyFeatureTransformer"} + expansion: { key: skillId, feature: MemberIndustryName } + } + aggregation:"ELEMENTWISE_AVG" + }, + seq_join_feature4: { + key: "x" + join: { + base: { key: x, + feature: MemberIndustryId, + outputKey: x} + expansion: { key: skillId, feature: MemberIndustryName } + } + aggregation:"ELEMENTWISE_AVG" + } + seq_join_feature5: { + key: "x" + join: { + base: { key: x, + feature: MemberIndustryId, + outputKey: x} + expansion: { key: skillId, feature: MemberIndustryName } + } + aggregation:"ELEMENTWISE_SUM" + } + }, + "advancedDerivations": [ + { + "features": [ + "quasarScoreFeature" + ], + "key": [ + "mId", + "jId" + ], + "inputs": "PROVIDED_BY_CLASS", + "class": { + "name": "com.linkedin.frame.quasar.DerivationWithQuasarDSL", + "quasarModelFile": "/quasarModels/testModel2.quasar", + "modelParam": { + "a": 1, + "b": { + "c": 2 + } + } + } + }, + { + "features": [ + "M", + "N" + ], + "key": [ + "x", + "y" + ], + "inputs": { + "nc": { + "key": "x", + "feature": "C" + }, + "nd": { + "key": "y", + "feature": "D" + } + }, + "class": "com.linkedin.frame.offline.SampleAdvancedDerivationFunctionExtractor" + }, + { + "features": [ + "Q" + ], + "key": [ + "x", + "y" + ], + "inputs": { + "nc": { + "key": "x", + "feature": "C" + }, + "nd": { + "key": "y", + "feature": "D" + } + }, + "class": "com.linkedin.frame.offline.SampleAdvancedDerivationFunctionExtractor" + }, + { + "features": [ + "P" + ], + "key": [ + "x", + "y" + ], + "inputs": { + "nc": { + "key": "x", + "feature": "C" + }, + "nd": { + "key": "y", + "feature": "D" + } + }, + "class": { + "name": "com.linkedin.frame.offline.SampleAdvancedDerivationFunctionExtractor", + "onlyProduceP": true + } + } + ], + "features": { + "careers": { + "careers_preference_companySize": { + "version": "1.0", + "dims": [], + "valType": "INT", + "availability": "ONLINE" + } + } + }, + + "dimensions": { + "careers": { + "dim1": { + "version": "4.2", + "type": "DISCRETE" + } + } + } +} \ No newline at end of file diff --git a/feathr-config/src/test/resources/FeatureDefSchemaTestInvalidCases.conf b/feathr-config/src/test/resources/FeatureDefSchemaTestInvalidCases.conf new file mode 100644 index 000000000..acc65634e --- /dev/null +++ b/feathr-config/src/test/resources/FeatureDefSchemaTestInvalidCases.conf @@ -0,0 +1,365 @@ +{ + "sources": { + "source1": { + "location1": { + "path": "source-simple.json" + } + }, + + "source11": { + "location": { + "path1": "source-simple.json" + } + }, + "source12": { + "location": { + "path": "source-simple.json", + "extra":1 + } + }, + "source13": { + "location": { + "path": 132 + } + }, + "source2": { + "location": { + "path": "source-simple.json" + }, + "hasTimeSnapshot2": false + }, + + "source23": { + "location": { + "path": "source-simple.json" + }, + "hasTimeSnapshot": "fasle" + }, + "source3": { + "location": { + "path": "source-symmetric-key.json" + }, + "extraParams": { + "viewOpType": "symmetricKey", + "targetFields": [ + "viewerId", + "vieweeId" + ], + "otherFields": "affinity" + } + }, + "source4": { + "location": { + "path": "source-flatten-id.json" + }, + "extraParams": { + "viewOpType": "flattenId", + "targetFields": "vector", + "otherFields": [ + "viewerId", + "viewerTitle" + ] + } + }, + "MemberStdCmpMalformedField": { + "type": "ESPRESSO", + "database2": "StandardizationEI", + "table": "MemberStandardizedCompany", + "d2Uri": "d2://ESPRESSO_MT2" + }, + "MemberStdCmpMissingKeyExpr": { + "type": "ESPRESSO", + "database": "StandardizationEI", + "table": "MemberStandardizedCompany", + "d2Uri": "d2://ESPRESSO_MT2" + }, + "JYMBIIMemberFeatures": { + "type": "VENICE", + "storeName": "JYMBIIMemberFeatures", + "keyExpr2": "com.linkedin.jobs.relevance.frame.online.util.AvroKeyGeneratorJymbiiMemberSourceKey.getKey(key[0])", + }, + "MemberPreferenceData": { + "type": "RESTLI2", + "restResourceName": "jobSeekers", + "keyExpr": "member" + }, + "MemberPreferenceData2": { + "type": "RESTLI", + "restResourceName": "jobSeekers" + }, + "memberDerivedData": { + "type": "RESTLI", + "restResourceName": "memberDerivedData", + "restEntityType": "member", + "pathSpec2": "standardizedSkills,standardizedIndustries,standardizedProfileIndustries,standardizedLocation,standardizedEducations,standardizedPositions" + }, + "CareersMemberEntityEmbeddings-0.0.2": { + "type": "VENICE", + "storeName2": "CareersMemberEntityEmbeddings", + "keyExpr": "{\"entityUrn\" : new com.linkedin.common.urn.Urn(\"member\", key[0]).toString(), \"version\" : \"0.0.2\"}" + }, + + "kafkaTestSource": { + "type": "KAFKA", + "stream2": "kafka.testCluster.testTopic" + }, + "rocksDBTestSource": { + "type": "ROCKSDB", + "referenceSource": "kafkaTestSource", + "extractFeatures": true, + "decoder": "com.linkedin.frame.online.config.FoobarExtractor" + }, + "jobScoringEntity": { + "type": "PASSTHROUGH2", + "dataModel": "com.linkedin.jobsprediction.JobScoringEntity" + }, + "customMissingDataModel": { + "type": "CUSTOM", + "keyExpr": "key[0]" + }, + "customMissingKeyExpr": { + "type": "CUSTOM", + "dataModel": "Long" + }, + "MemberConnectionIntersection": { + "type": "RESTLI", + "restResourceName": "setOperations", + "restEntityType2": "member", + "restReqParams": { + "operator2": "INTERSECT", + "edgeSetSpecifications": { + "jsonArray": "{\"array\": [{\"firstEdgeType\":\"MemberToMember\", \"secondEdgeType\":\"MemberToMember\"}]}" + }, + "second": { + "mvel": "key[1]" + }, + "a":{ + "file":"sd" + } + } + }, + "contentShareWindowAggSource": { + "type": "HDFS2", + "location": { + "path": "/jobs/mlf/contentShareFeatures/daily" + }, + "timePartitionPattern": "yyyy/MM/dd", + "timeWindowParameters": { + "timestampColumn": "timestamp", + "timestampColumnFormat": "yyyy/MM/dd" + } + } + + "couchbaseTestSource": { + "type": "COUCHBASE", + "bucketName": "testBucket" + "keyExpr": "key[0]", + "bootstrapUris": "some-app.corp.linkedin.com:8091", + "documentModel": "com.linkedin.frame.online.SomeDocumentClass" + }, + // INVALID queryKeyColumns type + "recentPageViewsSource": { + "type": "PINOT" + "resourceName": "recentMemberActionsPinotQuery" + "queryTemplate": "SELECT objectAttributes, timeStampSec FROM RecentMemberActions WHERE actorId IN (?)" + "queryArguments": ["[key[0]"] + "queryKeyColumns": "actorId" + } + }, + "anchors": { + "member-sent-invitations": { + "source": "/jobs/frame/inlab/data/features/InvitationStats", + "key": "x", + "features": { + "member_sentInvitations_numIgnoredRejectedInvites": { + "def2": "toNumeric(numIgnoredRejectedInvites)", + "default": "123" + } + } + }, + "featuresWithKey": { + "source": "/data/test/#LATEST", + "key": "x", + "features2": { + "waterloo_member_geoCountry": "geoStdData.countryCode" + } + }, + + "featuresWithOnlyMVEL": { + "source2": "/data/test/#LATEST", + "features": { + "waterloo_member_geoCountry": "geoStdData.countryCode", + "waterloo_member_geoRegion": "geoStdData.countryCode + ':' + geoStdData.regionCode" + } + }, + "featuresWithTransformer": { + "source": "/data/databases/CareersPreferenceDB/MemberPreference/#LATEST", + "transformer": "com.linkedin.jymbii.frame.anchor.PreferencesFeatures" + }, + "featuresWithTransformerAndExtract": { + "source": "/jobs/liar/jymbii-features-engineering/production/memberFeatures/education/#LATEST", + "transformer": "com.linkedin.jymbii.frame.anchor.LegacyFeastFormattedFeatures", + "features": [ + "jfu_member_degree" + ], + "extract2": [ + { + "extract": "member_degree", + "as": "jfu_member_degree" + } + ] + }, + "featuresWithExtractor": { + "source": "/data/databases/CareersPreferenceDB/MemberPreference/#LATEST", + "features": [ + "jfu_preference_companySize" + ] + } , + "featuresWithExtractorClass": { + "source": "/data/databases/CareersPreferenceDB/MemberPreference/#LATEST", + "extractor": {"class2":"com.linkedin.jymbii.frame.anchor.PreferencesFeatures"}, + "features": [ + "jfu_preference_companySize," + ] + }, + "contentShareWindowAggAnchor": { + "source": "contentShareWindowAggSource", + "key": "id", + "features": { + "fc_feed_7d_share_third_party_article_count": { + "def2": "thirdPartyArticleCount", + "aggregation": "SUM", + "window": "7d" + } + } + } + + couchbase-features: { + source: "couchbaseTestSource" + features: [ + couchbase-one-sample-feature, + couchbase-another-sample-feature + ] + } + + // Type related tests + // INVALID type enum + "test-member-derived-data-skills-by-source-v5-with-type": { + source: "memberDerivedData-skillV5" + extractor: {class: "com.linkedin.frame.feature.online.TestMemberSkillV5TermVectorTransformer"} + features: { + test_member_standardizedSkillsV5_explicit_type: { + def: "mvel", + default: 0 + type: INVALID_TYPE + } + test_member_standardizedSkillsV5_implicit_type: { + def: "mvel", + default: 0 + type: NUMERIC + } + } + } + // Invalid filed in type config + "test-member-derived-data-skills-by-source-v5-with-type3": { + source: "memberDerivedData-skillV5" + extractor: {class: "com.linkedin.frame.feature.online.TestMemberSkillV5TermVectorTransformer"} + features: { + test_member_standardizedSkillsV5_explicit_type3: { + def: "mvel", + default: 0 + type: { + type_valid: NUMERIC + } + } + } + } + // Missing type filed in type config + "test-member-derived-data-skills-by-source-v5-with-type3": { + source: "memberDerivedData-skillV5" + extractor: {class: "com.linkedin.frame.feature.online.TestMemberSkillV5TermVectorTransformer"} + features: { + test_member_standardizedSkillsV5_explicit_type3: { + def: "mvel", + default: 0 + type: { + valType: FLOAT + } + } + } + } + }, + "derivations": { + // Invalid type + "d1": { + sqlExpr: "case when abuse_member_invitation_numInviters = 0 then -1 else abuse_member_invitation_numInvites/abuse_member_invitation_numInviters end" + type: "INVALID_TYPE" + }, + "jfu_member_placeSimTopK": { + "key": [ + "member" + ], + "inputsa": [ + { + "key": "member", + "feature": "jfu_resolvedPreference_location" + } + ], + "class": "com.linkedin.jymbii.nice.derived.MemberPlaceSimTopK" + }, + "waterloo_member_pastTitleString:waterloo_job_standardizedSkillsString": { + "key": [ + "m", + "j" + ], + "inputs": { + "a": { + "key": "m", + "feature": "waterloo_member_pastTitleString" + }, + "b": { + "key": "j", + "feature2": "waterloo_job_standardizedSkillsString" + } + }, + "definition": "cosineSimilarity(a, b)" + }, + seq_join_feature1: { + key: "x" + join: { + base: { key: x, feature: MemberIndustryId } + expansion: { key: skillId, feature: MemberIndustryName, outputKey: x } + } + aggregation:"" + }, + seq_join_feature2: { + key: "x" + join: { + base: { key: x, feature: MemberIndustryId, transformation: "import com.linkedin.frame.MyFeatureUtils; MyFeatureUtils.dotProduct(MemberIndustryId);" } + expansion: { key: skillId, feature: MemberIndustryName } + } + aggregation:"ELEMENTWISE_AVG" + }, + seq_join_feature3: { + key: "x" + join: { + base: { key: x, feature: MemberIndustryId ,transformationClass: "com.linkedin.frame.MyFeatureTransformer"} + expansion: { key: skillId, feature: MemberIndustryName } + } + aggregation:"ELEMENTWISE_AVG" + }, + seq_join_feature4: { + key: "x" + join: { + base: { + key: x, + feature: MemberIndustryId, + transformation: "import com.linkedin.frame.MyFeatureUtils; MyFeatureUtils.dotProduct(MemberIndustryId);", + transformationClass: "com.linkedin.frame.MyFeatureTransformer" + } + expansion: { key: skillId, feature: MemberIndustryName } + } + aggregation:"ELEMENTWISE_AVG" + } + } +} diff --git a/feathr-config/src/test/resources/Foo.txt b/feathr-config/src/test/resources/Foo.txt new file mode 100644 index 000000000..e97bf0c74 --- /dev/null +++ b/feathr-config/src/test/resources/Foo.txt @@ -0,0 +1,3 @@ +This is line 1 +This is line 2 +This is line 3 diff --git a/feathr-config/src/test/resources/JoinSchemaTestCases.conf b/feathr-config/src/test/resources/JoinSchemaTestCases.conf new file mode 100644 index 000000000..31b531624 --- /dev/null +++ b/feathr-config/src/test/resources/JoinSchemaTestCases.conf @@ -0,0 +1,51 @@ +{ + settings: { + observationDataTimeSettings: { + absoluteTimeRange: { + startTime: "20180809" + endTime: "20180812" + timeFormat: "yyyyMMdd" + } + } + joinTimeSettings: { + timestampColumn: { + def: "timestamp/1000" + format: "epoch" + } + simulateTimeDelay: 2d + } + }, + "features": [ + { + "key": "viewerId", + "featureList": [ + "jfu_resolvedPreference_seniority", + "jfu_resolvedPreference_country", + "waterloo_member_currentTitle" + ], + overrideTimeDelay: 1d + }, + { + "key": "vieweeId", + "featureList": [ + "jfu_resolvedPreference_seniority", + "jfu_resolvedPreference_country", + "waterloo_member_currentTitle" + ], + overrideTimeDelay: 3d + } + ], + "globalFeatures": [ + { + "key": [ + "x", + "y" + ], + "featureList": [ + "waterloo_member_pastTitleString:waterloo_job_standardizedSkillsString", + "waterloo_member_headline:waterloo_job_titleString", + "waterloo_member_pastTitleString:waterloo_job_companyDesc" + ] + } + ] +} \ No newline at end of file diff --git a/feathr-config/src/test/resources/PresentationsSchemaTestCases.conf b/feathr-config/src/test/resources/PresentationsSchemaTestCases.conf new file mode 100644 index 000000000..fbace9bd0 --- /dev/null +++ b/feathr-config/src/test/resources/PresentationsSchemaTestCases.conf @@ -0,0 +1,8 @@ +presentation { + my_ccpa_feature: { + memberViewFeatureName: "standardization job standardizedSkillsV5" + linkedInViewFeatureName: standardization_job_standardizedSkillsV5 + featureDescription: feature description that shows to the users + valueTranslation: "translateLikelihood(waterloo_member_geoRegion, [[0, 0.33, 'Low'], [0.33, 0.66, 'Medium'],[0.66, 1.0, 'High']])" + } +} \ No newline at end of file diff --git a/feathr-config/src/test/resources/config/fruits.csv b/feathr-config/src/test/resources/config/fruits.csv new file mode 100644 index 000000000..86996453e --- /dev/null +++ b/feathr-config/src/test/resources/config/fruits.csv @@ -0,0 +1,8 @@ +// First comment line +// Second comment line +0, OUT_OF_VOCAB +1, apple +2, banana +3, orange +4, pear +5, guava \ No newline at end of file diff --git a/feathr-config/src/test/resources/config/fruitsWithDupIds.csv b/feathr-config/src/test/resources/config/fruitsWithDupIds.csv new file mode 100644 index 000000000..0a9ac1e2f --- /dev/null +++ b/feathr-config/src/test/resources/config/fruitsWithDupIds.csv @@ -0,0 +1,7 @@ +// Contains duplicate IDs +0, OUT_OF_VOCAB +1, apple +2, banana +3, orange +1, pear +0, guava \ No newline at end of file diff --git a/feathr-config/src/test/resources/config/fruitsWithDupNames.csv b/feathr-config/src/test/resources/config/fruitsWithDupNames.csv new file mode 100644 index 000000000..ae35b4ef9 --- /dev/null +++ b/feathr-config/src/test/resources/config/fruitsWithDupNames.csv @@ -0,0 +1,8 @@ +// First comment line +// Second comment line +0, OUT_OF_VOCAB +1, apple +2, banana +3, apple +4, pear +5, banana \ No newline at end of file diff --git a/feathr-config/src/test/resources/config/hashedFruits.csv b/feathr-config/src/test/resources/config/hashedFruits.csv new file mode 100644 index 000000000..2c9cc9d23 --- /dev/null +++ b/feathr-config/src/test/resources/config/hashedFruits.csv @@ -0,0 +1,6 @@ +// The hashed values are arbitrarily created for testing purposes. +123456789, apple +234567890, banana +345678901, orange +456789012, pear +567890123, guava \ No newline at end of file diff --git a/feathr-config/src/test/resources/config/manifest1.conf b/feathr-config/src/test/resources/config/manifest1.conf new file mode 100644 index 000000000..22730c582 --- /dev/null +++ b/feathr-config/src/test/resources/config/manifest1.conf @@ -0,0 +1,6 @@ +manifest: [ + { + jar: local + conf: [dir1/features-2-prod.conf] // [frame-feature-careers-featureDef-offline.conf] + } +] \ No newline at end of file diff --git a/feathr-config/src/test/resources/config/manifest2.conf b/feathr-config/src/test/resources/config/manifest2.conf new file mode 100644 index 000000000..1ab24ccc7 --- /dev/null +++ b/feathr-config/src/test/resources/config/manifest2.conf @@ -0,0 +1,6 @@ +manifest: [ + { + jar: frame-feature-waterloo-online-1.1.4.jar + conf: [config/online/prod/feature-prod.conf] + } +] \ No newline at end of file diff --git a/feathr-config/src/test/resources/config/manifest3.conf b/feathr-config/src/test/resources/config/manifest3.conf new file mode 100644 index 000000000..a5df5bd93 --- /dev/null +++ b/feathr-config/src/test/resources/config/manifest3.conf @@ -0,0 +1,10 @@ +manifest: [ + { + jar: local + conf: [frame-feature-careers-featureDef-offline.conf] + }, + { + jar: frame-feature-waterloo-online-1.1.4.jar + conf: [config/online/prod/feature-prod.conf] + } +] \ No newline at end of file diff --git a/feathr-config/src/test/resources/dir1/features-1-prod.conf b/feathr-config/src/test/resources/dir1/features-1-prod.conf new file mode 100644 index 000000000..8b0f95314 --- /dev/null +++ b/feathr-config/src/test/resources/dir1/features-1-prod.conf @@ -0,0 +1,24 @@ +sources : { + MemberPreferenceData: { + type: ESPRESSO + database: "CareersPreferenceDB" + table: "MemberPreference" + d2Uri: "d2://PROD_ESPRESSO_MT2" + keyExpr: "key[0]" + } + + member_derived_data: { + location: {path: "/data/test/#LATEST"} + } +} + +anchors : { + member-lix-segment: { + source: "/data/derived/lix/euc/member/#LATEST" + key: "id" + features: { + member_lixSegment_isStudent: "is_student" + member_lixSegment_isJobSeeker: "job_seeker_class == 'active'" + } + } +} diff --git a/feathr-config/src/test/resources/dir1/features-2-prod.conf b/feathr-config/src/test/resources/dir1/features-2-prod.conf new file mode 100644 index 000000000..b93d77c1d --- /dev/null +++ b/feathr-config/src/test/resources/dir1/features-2-prod.conf @@ -0,0 +1,10 @@ +anchors : { + member-lix-segment: { + source: "/data/derived/lix/euc/member/#LATEST" + key: "id" + features: { + member_lixSegment_isStudent: "is_student" + member_lixSegment_isJobSeeker: "job_seeker_class == 'active'" + } + } +} diff --git a/feathr-config/src/test/resources/dir1/features-3-prod.conf b/feathr-config/src/test/resources/dir1/features-3-prod.conf new file mode 100644 index 000000000..cd4785ea3 --- /dev/null +++ b/feathr-config/src/test/resources/dir1/features-3-prod.conf @@ -0,0 +1,13 @@ +sources : { + MemberPreferenceData: { + type: ESPRESSO + database: "CareersPreferenceDB" + table: "MemberPreference" + d2Uri: "d2://ESPRESSO_MT2" + keyExpr: "key[0]" + } + + member_derived_data: { + location: {path: "/data/test/#LATEST"} + } +} diff --git a/feathr-config/src/test/resources/dir1/join.conf b/feathr-config/src/test/resources/dir1/join.conf new file mode 100644 index 000000000..df72130a5 --- /dev/null +++ b/feathr-config/src/test/resources/dir1/join.conf @@ -0,0 +1,24 @@ +features: [ + { + key: "targetId" + featureList: ["waterloo_job_location", "waterloo_job_jobTitle", "waterloo_job_jobSeniority"] + }, + { + key: "sourceId" + featureList: ["TimeBasedFeatureA"] + startDate: "20170522" + endDate: "20170522" + }, + { + key: "sourceId" + featureList: ["jfu_resolvedPreference_seniority", "jfu_resolvedPreference_country", "waterloo_member_currentTitle"] + }, + { + key: ["sourceId","targetId"] + featureList: ["memberJobFeature1","memberJobFeature2"] + }, + { + key: [x], + featureList: ["sumPageView1d", "waterloo-member-title"] + } +] \ No newline at end of file diff --git a/feathr-config/src/test/resources/dir2/features-1-ei.conf b/feathr-config/src/test/resources/dir2/features-1-ei.conf new file mode 100644 index 000000000..95424ee71 --- /dev/null +++ b/feathr-config/src/test/resources/dir2/features-1-ei.conf @@ -0,0 +1,15 @@ +// A resource is specified via the classpath +include classpath("dir1/features-1-prod.conf") + +// Overrides d2Uri to point to EI-specific url. Here we use a path expression +sources.MemberPreferenceData.d2Uri: "d2://EI_ESPRESSO_MT2" + +// Overrides hdfs path to point to EI-specific path. Instead of a path expression (dot-notation), we can also use the +// object notation +sources: { + member_derived_data: { + location: { + path: "/eidata/derived/standardization/waterloo/members_std_data/#LATEST" + } + } +} diff --git a/feathr-config/src/test/resources/extractor-with-params.conf b/feathr-config/src/test/resources/extractor-with-params.conf new file mode 100644 index 000000000..24f0598aa --- /dev/null +++ b/feathr-config/src/test/resources/extractor-with-params.conf @@ -0,0 +1,25 @@ +sources : { + member_derived_data: { + location: {path: "/data/test/#LATEST"} + } +} + +anchors : { + waterloo-job-term-vectors: { + source: "member_derived_data" + extractor: "com.linkedin.feathr.SampleExtractorWithParams" + features: { + feature_with_params : { + parameters: { + param0 : {type: CATEGORICAL, default: "n/a"} + param1 : "java", + param2 : [waterlooCompany_terms_hashed, waterlooCompany_values], + param3 : true, + param4 : {"java" : "3"}, + param5 : {"key1":["v1","v2"]}, + param6 : [{"key1":["v1","v2"]}, {"key2":["v1","v2"]}] + } + } + } + } +} diff --git a/feathr-config/src/test/resources/foo-2.0.1.jar b/feathr-config/src/test/resources/foo-2.0.1.jar new file mode 100644 index 000000000..8dffb3ebb Binary files /dev/null and b/feathr-config/src/test/resources/foo-2.0.1.jar differ diff --git a/feathr-config/src/test/resources/invalidSemanticsConfig/duplicate-feature.conf b/feathr-config/src/test/resources/invalidSemanticsConfig/duplicate-feature.conf new file mode 100644 index 000000000..890fa892d --- /dev/null +++ b/feathr-config/src/test/resources/invalidSemanticsConfig/duplicate-feature.conf @@ -0,0 +1,25 @@ +sources : { + member_derived_data: { + location: {path: "/data/test/#LATEST"} + } +} + +anchors : { + memberLixSegment: { + source: "/data/derived/lix/euc/member/#LATEST" + key: "id" + features: { + member_lixSegment_isStudent: "is_student" + member_lixSegment_isJobSeeker: "job_seeker_class == 'active'" + } + } + + memberLixSegmentV2: { + source: "/data/derived/lix/euc/member_v2/#LATEST" + key: "id" + features: { + member_lixSegment_isStudent_V2: "is_student" + member_lixSegment_isJobSeeker: "job_seeker_class == 'active'" + } + } +} diff --git a/feathr-config/src/test/resources/invalidSemanticsConfig/extractor-with-params-not-approved.conf b/feathr-config/src/test/resources/invalidSemanticsConfig/extractor-with-params-not-approved.conf new file mode 100644 index 000000000..ec541163d --- /dev/null +++ b/feathr-config/src/test/resources/invalidSemanticsConfig/extractor-with-params-not-approved.conf @@ -0,0 +1,20 @@ +sources : { + forwardIndex: { + type: PASSTHROUGH + dataModel: "com.linkedin.galene.buffers.BufferRecord" + }, +} + +anchors : { + waterloo-job-term-vectors: { + source: "forwardIndex" + extractor: "com.linkedin.galene.NotApprovedExtractorWithParams" + features: { + waterloo_job_jobTitleV2 : { + parameters: { + param1: "a" + } + } + } + } +} diff --git a/feathr-config/src/test/resources/invalidSemanticsConfig/feature-not-reachable-def.conf b/feathr-config/src/test/resources/invalidSemanticsConfig/feature-not-reachable-def.conf new file mode 100644 index 000000000..7e0f331de --- /dev/null +++ b/feathr-config/src/test/resources/invalidSemanticsConfig/feature-not-reachable-def.conf @@ -0,0 +1,55 @@ +// in this config, one derivation feature (derived_feature_3) has a undefined input feature (feature3) +// this is usually due to typo. For instance, the user might want to type feature2 instead +{ + "anchors": { + accessTimeFeatures: { + source: "/jobs/emerald/Features/LatestFeatures/accessTimeStats/#LATEST", + key: "x", + features: { + feature1: { + def: "lastVisitedTime", + default: 0.0, + type: "NUMERIC" + } + feature2: { + def: "daysSinceLastVisitedTime", + default: 0.0, + type: "NUMERIC" + } + } + } + }, + "derivations": { + "derived_feature_1": "feature1", + "derived_feature_2": { + "key": [ + "member" + ], + "inputs": [ + { + "key": "member", + "feature": "feature2" + } + ], + "class": "com.linkedin.jymbii.nice.derived.MemberPlaceSimTopK" + }, + // this is not reachable, as feature 3 is not defined + "derived_feature_3": { + "key": [ + "m", + "j" + ], + "inputs": { + "a": { + "key": "m", + "feature": "feature3" + }, + "b": { + "key": "j", + "feature": "derived_feature_2" + } + }, + "definition": "cosineSimilarity(a, b)" + } + } +} \ No newline at end of file diff --git a/feathr-config/src/test/resources/invalidSemanticsConfig/undefined-source.conf b/feathr-config/src/test/resources/invalidSemanticsConfig/undefined-source.conf new file mode 100644 index 000000000..5b85fedfe --- /dev/null +++ b/feathr-config/src/test/resources/invalidSemanticsConfig/undefined-source.conf @@ -0,0 +1,25 @@ +sources : { + member_derived_data: { + location: {path: "/data/test/#LATEST"} + } +} + +anchors : { + memberLixSegment: { + source: "/data/derived/lix/euc/member/#LATEST" + key: "id" + features: { + member_lixSegment_isStudent: "is_student" + member_lixSegment_isJobSeeker: "job_seeker_class == 'active'" + } + } + + memberLixSegmentV2: { + source: member_derived_date + key: "id" + features: { + member_lixSegment_isStudent_V2: "is_student" + member_lixSegment_isJobSeeker_V2: "job_seeker_class == 'active'" + } + } +} diff --git a/feathr-config/src/test/resources/validFrameConfigWithInvalidSyntax.conf b/feathr-config/src/test/resources/validFrameConfigWithInvalidSyntax.conf new file mode 100644 index 000000000..8334cb221 --- /dev/null +++ b/feathr-config/src/test/resources/validFrameConfigWithInvalidSyntax.conf @@ -0,0 +1,11 @@ +// This conf valid Frame config file but with invalid syntax. + +anchors: { + careers-member-profile-yoe: { + invalidSourceKey: "/data/databases/Identity/Profile/#LATEST" + extractor: "com.linkedin.careers.relevance.frame.offline.anchor.ISBYoeTermVectorFeatures" + features: [ + careers_member_positionsYoE + ] + } +} \ No newline at end of file diff --git a/feathr-data-models/build.gradle b/feathr-data-models/build.gradle new file mode 100644 index 000000000..437857152 --- /dev/null +++ b/feathr-data-models/build.gradle @@ -0,0 +1,51 @@ +apply plugin: 'pegasus' +apply plugin: 'maven-publish' +apply plugin: 'signing' +apply plugin: 'java' +apply plugin: "com.vanniktech.maven.publish.base" + +afterEvaluate { + dependencies { + dataTemplateCompile spec.product.pegasus.data + } +} + +java { + withSourcesJar() + withJavadocJar() +} + +tasks.withType(Javadoc) { + options.addStringOption('Xdoclint:none', '-quiet') + options.addStringOption('encoding', 'UTF-8') + options.addStringOption('charSet', 'UTF-8') +} + +repositories { + mavenCentral() + mavenLocal() + maven { + url "https://repository.mulesoft.org/nexus/content/repositories/public/" + } + maven { + url "https://linkedin.jfrog.io/artifactory/open-source/" // GMA, pegasus + } +} + +// Required for publishing to local maven +publishing { + publications { + mavenJava(MavenPublication) { + artifactId = 'feathr-data-models' + from components.java + versionMapping { + usage('java-api') { + fromResolutionOf('runtimeClasspath') + } + usage('java-runtime') { + fromResolutionResult() + } + } + } + } +} \ No newline at end of file diff --git a/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/AbstractNode.pdl b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/AbstractNode.pdl new file mode 100644 index 000000000..d9348a539 --- /dev/null +++ b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/AbstractNode.pdl @@ -0,0 +1,22 @@ +// LINT_SUPPRESS: namespace.three.parts + +namespace com.linkedin.feathr.compute + +/** + * Generic abstraction of a node. All other nodes should derive from this node. + */ +record AbstractNode { + /** + * The node would be represented by this id. + */ + id: NodeId + + /** + * The key for which this node is being requested. + * If this node is a Source node, the engine can use the key to fetch or join the feature. + * If this node is NOT a Source node, the engine should NOT use the key to determine fetch/join behavior, but + * should follow the node's inputs. (The core libraries may use the key information in order to optimize the graph, + * e.g. it can be used for identifying duplicate sections of the graph that can be pruned.) + */ + concreteKey: optional ConcreteKey +} \ No newline at end of file diff --git a/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/Aggregation.pdl b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/Aggregation.pdl new file mode 100644 index 000000000..f44500b98 --- /dev/null +++ b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/Aggregation.pdl @@ -0,0 +1,29 @@ +// LINT_SUPPRESS: namespace.three.parts + +namespace com.linkedin.feathr.compute + +/** + * A node to represent an aggregation step. The aggregation inputs like the groupBy field, agg function are delegated to [[AggregationFunction]]. + * This node can represent a feature. As of now, in this step we will be using the SWA library from Spark-algorithms. + */ +record Aggregation includes AbstractNode { + /** + * The input node on which aggregation is to be performed. As of now, we would only be supporting this node to be a data source node. + */ + input: NodeReference + + /** + * All the aggregation related parameters and functions are bundled into this. + */ + function: AggregationFunction + + /** + * If the node is representing a feature, the feature name should be associated with the node. + */ + featureName: string + + /** + * feature version of the feature + */ + featureVersion: FeatureVersion +} \ No newline at end of file diff --git a/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/AggregationFunction.pdl b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/AggregationFunction.pdl new file mode 100644 index 000000000..d5d43dccf --- /dev/null +++ b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/AggregationFunction.pdl @@ -0,0 +1,24 @@ +// LINT_SUPPRESS: namespace.three.parts + +namespace com.linkedin.feathr.compute + +/** + * All parameters related to an aggregation operation. This class should be used in conjunction with the [[Aggregation]] node. + */ +record AggregationFunction { + /** + * The aggregation function. + */ + operator: OperatorId + /** + * All the aggregation parameters should be bundled into this map. For now, the possible parameters are:- + * a. target_column - Aggregation column + * b. window_size - aggregation window size + * c. window unit - aggregation window unit (ex - day, hour) + * d. lateral_view_expression - definition of a lateral view for the feature. + * e. lateral_view_table_alias - An alias for the lateral view + * f. filter - An expression to filter out any data before aggregation. Should be a sparkSql expression. + * g. groupBy - groupBy columns. Should be a sparkSql expression. + */ + parameters: optional map[string, string] // kind of like Attributes in Onnx? +} \ No newline at end of file diff --git a/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/AnyNode.pdl b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/AnyNode.pdl new file mode 100644 index 000000000..8a36ed3d0 --- /dev/null +++ b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/AnyNode.pdl @@ -0,0 +1,14 @@ +// LINT_SUPPRESS: namespace.three.parts + +namespace com.linkedin.feathr.compute + +/** + * A typeref for all the different types of nodes. + */ +typeref AnyNode = union[ + Aggregation + DataSource + Lookup + Transformation + External +] \ No newline at end of file diff --git a/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/ComputeGraph.pdl b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/ComputeGraph.pdl new file mode 100644 index 000000000..805b82327 --- /dev/null +++ b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/ComputeGraph.pdl @@ -0,0 +1,20 @@ +// LINT_SUPPRESS: namespace.three.parts + +namespace com.linkedin.feathr.compute + +/** + * Feature computation graph. The passed in feature definition graph should get converted to this dependency graph. This graph is a + * direct translation of all the features present, and is not optimized with respect to the join config. + */ +record ComputeGraph { + + /** + * The nodes in the graph (order does not matter) + */ + nodes: array[AnyNode], + + /** + * Map from feature name to node ID, for those nodes in the graph that represent named features. + */ + featureNames: map[string, int] +} \ No newline at end of file diff --git a/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/ConcreteKey.pdl b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/ConcreteKey.pdl new file mode 100644 index 000000000..fb040b730 --- /dev/null +++ b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/ConcreteKey.pdl @@ -0,0 +1,15 @@ +// LINT_SUPPRESS: namespace.three.parts + +namespace com.linkedin.feathr.compute + +/** + * The key (node) for which the node in question is requested. + */ +record ConcreteKey { + /** + * Most of the time, this should point to a CONTEXT SOURCE node, e.g. a key in the context called x. + * The main exception would be for a Lookup feature, in which case it would point to another node where the lookup + * key gets computed. + */ + key: array[NodeId] +} \ No newline at end of file diff --git a/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/DataSource.pdl b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/DataSource.pdl new file mode 100644 index 000000000..0607fbef6 --- /dev/null +++ b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/DataSource.pdl @@ -0,0 +1,44 @@ +// LINT_SUPPRESS: namespace.three.parts + +namespace com.linkedin.feathr.compute + +/** + * Representation of the datasource node. There are 3 types of datasource nodes:- + * Context - To represent the observation data entities (like the join key or passthrough feature columns) + * Update - To represent a non-timepartitioned datasource node. + * Event - To represent a time-partitioned datasource node. + * + * TODO - Maybe, it makes sense more sense to refactor it by make this an abstract object, and deriving the three different nodes from it. + */ +record DataSource includes AbstractNode { + + /** + * Type of node, ie - Context, Update, Event + */ + sourceType: DataSourceType + + /** + * for CONTEXT type, this is the name of the context column. otherwise, it should be a path or URI. + */ + externalSourceRef: string + + /** + * Raw key expression as entered by the user. This hocon parsing happens at the execution engine side. + */ + keyExpression: string + + /** + * mvel or spark or user-defined class + */ + keyExpressionType: KeyExpressionType + + /** + * File partition format. + */ + filePartitionFormat: optional string + + /** + * Timestamp column info, to be available only for an event datasource node. + */ + timestampColumnInfo: optional TimestampCol +} \ No newline at end of file diff --git a/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/DataSourceType.pdl b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/DataSourceType.pdl new file mode 100644 index 000000000..b2299cbf7 --- /dev/null +++ b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/DataSourceType.pdl @@ -0,0 +1,24 @@ +// LINT_SUPPRESS: namespace.three.parts + +namespace com.linkedin.feathr.compute + +/** + * Type of datasource node. + */ +enum DataSourceType { + /** + * Update data sources provide keyed data about entities. A fully specified table data source contains both a snapshot view and an update log. + */ + UPDATE + + /** + * Event data sources are append-only event logs whose records need to be grouped and aggregated (e.g. counted, averaged, top-K’d) + * over a limited window of time. + */ + EVENT + + /** + * Reprent the observation data entities (like the join key or passthrough feature columns) + */ + CONTEXT +} \ No newline at end of file diff --git a/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/DateTimeInterval.pdl b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/DateTimeInterval.pdl new file mode 100644 index 000000000..baf028d4a --- /dev/null +++ b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/DateTimeInterval.pdl @@ -0,0 +1,16 @@ +namespace com.linkedin.feathr.compute + +/** + * Represent a data time interval + */ +record DateTimeInterval { + /** + * Represents the inclusive (greater than or equal to) value in which to start the range. This field is optional. An unset field here indicates an open range; for example, if end is 1455309628000 (Fri, 12 Feb 2016 20:40:28 GMT), and start is not set, it would indicate times up to, but excluding, 1455309628000. Note that this interpretation was not originally documented. New uses of this model should follow this interpretation, but older models may not, and their documentation should reflect this fact. + */ + start: optional Time + + /** + * Represents the exclusive (strictly less than) value in which to end the range. This field is optional. An unset field here indicates an open range; for example, if start is 1455309628000 (Fri, 12 Feb 2016 20:40:28 GMT), and end is not set, it would mean everything at, or after, 1455309628000. New uses of this model should follow this interpretation, but older models may not, and their documentation should reflect this fact. + */ + end: optional Time +} \ No newline at end of file diff --git a/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/Dimension.pdl b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/Dimension.pdl new file mode 100644 index 000000000..f67a1ecd2 --- /dev/null +++ b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/Dimension.pdl @@ -0,0 +1,18 @@ +// LINT_SUPPRESS: namespace.three.parts + +namespace com.linkedin.feathr.compute + +/** + * Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. In Quince Tensor specifically, the last column is designated as the value, and the rest of the columns are keys (aka dimensions). + */ +record Dimension { + /** + * Type of the dimension in the tensor. Each dimension can have a different type. + */ + type: DimensionType + + /** + * Size of the dimension in the tensor. If unset, it means the size is unknown and actual size will be determined at runtime. + */ + shape: optional int +} diff --git a/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/DimensionType.pdl b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/DimensionType.pdl new file mode 100644 index 000000000..62a975ed7 --- /dev/null +++ b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/DimensionType.pdl @@ -0,0 +1,17 @@ +// LINT_SUPPRESS: namespace.three.parts + +namespace com.linkedin.feathr.compute + +/** + * Supported dimension types for tensors in Quince and feathr. + */ +enum DimensionType { + /** Long. */ + LONG + /** Integer. */ + INT + /** String. */ + STRING + /** Boolean. */ + BOOLEAN +} diff --git a/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/External.pdl b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/External.pdl new file mode 100644 index 000000000..4a04ea142 --- /dev/null +++ b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/External.pdl @@ -0,0 +1,14 @@ +// LINT_SUPPRESS: namespace.three.parts + +namespace com.linkedin.feathr.compute + +/** + * A temporary node which would exist only while parsing the graph. For example, when parsing an object if there is a reference to a feature + * name, we will create an external node. This would get resolved later in the computation. + */ +record External includes AbstractNode { + /** + * Name of the external object it should refer to. + */ + name: string +} \ No newline at end of file diff --git a/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/FeatureValue.pdl b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/FeatureValue.pdl new file mode 100644 index 000000000..0d3810768 --- /dev/null +++ b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/FeatureValue.pdl @@ -0,0 +1,16 @@ +// LINT_SUPPRESS: namespace.three.parts + +namespace com.linkedin.feathr.compute + +/** + * Defines supported types that can be used to represent the value of a feature data. An example usage is specifying feature's default value. It currently starts with scalar types and more complex types can be added along with more use cases. + */ +typeref FeatureValue = union[ + boolean + int + long + float + double + string + bytes +] diff --git a/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/FeatureVersion.pdl b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/FeatureVersion.pdl new file mode 100644 index 000000000..cee7d786d --- /dev/null +++ b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/FeatureVersion.pdl @@ -0,0 +1,19 @@ +// LINT_SUPPRESS: namespace.three.parts + +namespace com.linkedin.feathr.compute + +record FeatureVersion { + /** + * Defines the high level semantic type of a feature. The high level semantic types are supported in early version of feathr before Tensorization and will be kept around until a full transition to Tensor types is completed + */ + type: FrameFeatureType = "UNSPECIFIED" + /** + * Defines the format of feature data. Feature data is produced by applying transformation on source, in a FeatureAnchor. feathr will make some default assumptions if FeatureFormat is not provided, but this should be considered limited support, and format should be defined for all new features. + */ + format: optional TensorFeatureFormat + + /** + * An optional default value can be provided. In case of missing data or errors occurred while applying transformation on source in FeatureAnchor, the default value will be used to populate feature data. + */ + defaultValue: optional FeatureValue +} diff --git a/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/FrameFeatureType.pdl b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/FrameFeatureType.pdl new file mode 100644 index 000000000..d20a98f48 --- /dev/null +++ b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/FrameFeatureType.pdl @@ -0,0 +1,25 @@ +// LINT_SUPPRESS: namespace.three.parts + +namespace com.linkedin.feathr.compute + +/** + * The high level types associated with a feature. In contrast with TensorFeatureFormat which contains additional metadata about the type of the tensor, this represents the high level semantic types supported by early versions of feathr. See https://iwww.corp.linkedin.com/wiki/cf/display/ENGS/Feature+Representation+and+Feature+Type+System for more detais. TODO - this is expected to be deprecated once the full transition to TensorType is completed + */ +enum FrameFeatureType { + /** Boolean valued feature */ + BOOLEAN, + /** Numerically valued feature such as INT, LONG, DOUBLE, etc */ + NUMERIC, + /** Represents a feature that consists of a single category (e.g. MOBILE, DESKSTOP) */ + CATEGORICAL, + /** Represents a feature that consists of multiple categories (e.g. MOBILE, DESKSTOP) */ + CATEGORICAL_SET, + /** Represents a feature in vector format where the the majority of the elements are non-zero */ + DENSE_VECTOR, + /** Represents features that has string terms and numeric value*/ + TERM_VECTOR, + /** Represents tensor based features. Note: this represents the high level semantic tensor type but does not include the low level tensor format such as category, shape, dimension and value types. The latter are defined as part of the new tensor annotation (via TensorFeatureFormat) or the legacy FML (go/FML).*/ + TENSOR, + /** Placeholder for when no types are specified */ + UNSPECIFIED +} diff --git a/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/KeyExpressionType.pdl b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/KeyExpressionType.pdl new file mode 100644 index 000000000..113d857e1 --- /dev/null +++ b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/KeyExpressionType.pdl @@ -0,0 +1,24 @@ +namespace com.linkedin.feathr.compute + +/** + * Different key formats supported. + * Todo - We probably do not want to generalize this as a kind of key-operator in the core compute model, + * with instances such as for MVEL or SQL being available (e.g. via an OperatorId reference). + */ +enum KeyExpressionType { + + /** + * Java-based MVEL + */ + MVEL, + + /** + * Spark-SQL + */ + SQL, + + /** + * Custom java/scala UDF + */ + UDF +} diff --git a/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/KeyReference.pdl b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/KeyReference.pdl new file mode 100644 index 000000000..ecc40a054 --- /dev/null +++ b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/KeyReference.pdl @@ -0,0 +1,14 @@ +// LINT_SUPPRESS: namespace.three.parts + +namespace com.linkedin.feathr.compute + +/** + * This represents the position of the key in the node which is being referred to. For example, if the original node has a key + * like [x, y], and the keyReference says 1, it is referring to y. + */ +record KeyReference { + /** + * Position in the original key array + */ + position: int +} \ No newline at end of file diff --git a/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/LateralView.pdl b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/LateralView.pdl new file mode 100644 index 000000000..883a89a07 --- /dev/null +++ b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/LateralView.pdl @@ -0,0 +1,20 @@ +namespace com.linkedin.feathr.compute + +/** + * Lateral view is used in conjunction with table generating functions (eg. the most commonly used explode()), which typically generates zero or more output rows for each input row. A lateral view first applies the table generating function to each row of base table, and then joins resulting output rows to the input rows to form a virtual table with the supplied table alias. For more details and examples, refer to https://cwiki.apache.org/confluence/display/Hive/LanguageManual+LateralView. + */ +record LateralView { + + /** + * A table-generating function transforms a single input row to multiple output rows. For example, explode(array('A','B','C') will produce 3 one-column rows, which are row1: 'A'; row2: 'B'; row3: 'C'. + */ + tableGeneratingFunction: union[ + // SparkSql-based expression. One of the most common lateral view operation is explode, for example, explode(features). + SqlExpression + ] + + /** + * Represents the alias for referencing the generated virtual table. It will be used in subsequent statements (eg. filter, groupBy) in the sliding window feature definition. + */ + virtualTableAlias: string +} diff --git a/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/Lookup.pdl b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/Lookup.pdl new file mode 100644 index 000000000..edb48e64a --- /dev/null +++ b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/Lookup.pdl @@ -0,0 +1,56 @@ +// LINT_SUPPRESS: namespace.three.parts + +namespace com.linkedin.feathr.compute + +/** + * A node to represent a feature which is to be computed by using an already computed feature as the key. + * https://iwww.corp.linkedin.com/wiki/cf/pages/viewpage.action?spaceKey=ENGS&title=feathr+Offline+User+Guide#FrameOfflineUserGuide-sequentialjoin + */ +record Lookup includes AbstractNode { + + /** + * An array of references to a node and keys. + * + * For now, we do not support lookup of just a key reference, but we have added that as a placeholder. + * + * A node reference consists of node id and a key reference. + * In sequential join the lookup key would be a combination of the + * feature node representing the base feature (lookup node) and the key associated with it. For example,:- + * seqJoinFeature: { + * base: {key: x, feature: baseFeature} + * expansion: {key: y, feature: expansionFeature} + * aggregation: UNION + * } + * Here, the lookupKey's node reference would point to the node which computes the base feature, and the keyReference would + * point to the index of "x" in the key array of baseFeature. + */ + lookupKey: array[union[NodeReference, KeyReference]] + + /** + * The node id of the node containing the expansion feature. + */ + lookupNode: NodeId + + /** + * Aggregation type as listed in + * https://jarvis.corp.linkedin.com/codesearch/result/ + * ?name=FeatureAggregationType.java&path=feathr-common%2Fframe-common%2Fsrc%2Fmain%2Fjava%2Fcom%2Flinkedin%2Fframe%2Fcommon&reponame=feathr%2Fframe-common#7 + * + */ + aggregation: string + + /** + * feature name of the feature which would be computed. + * we need feature name here for 2 main reasons. + * 1. For type information. There are existing APIs that create a map from feature name -> type info from FR model and + * we want to leverage that. + * 2. For default values. Similar to above, there are existing APIs which create default value map from feature name -> + * default value. + */ + featureName: string + + /** + * feature version of the feature + */ + featureVersion: FeatureVersion +} \ No newline at end of file diff --git a/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/MvelExpression.pdl b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/MvelExpression.pdl new file mode 100644 index 000000000..2eee59271 --- /dev/null +++ b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/MvelExpression.pdl @@ -0,0 +1,13 @@ +// LINT_SUPPRESS: namespace.three.parts + +namespace com.linkedin.feathr.compute + +/** + * An expression in MVEL language. For more information please refer to go/framemvel. + */ +record MvelExpression { +/** + * The MVEL expression. + */ +mvel: string +} diff --git a/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/NodeId.pdl b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/NodeId.pdl new file mode 100644 index 000000000..19f520be7 --- /dev/null +++ b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/NodeId.pdl @@ -0,0 +1,8 @@ +// LINT_SUPPRESS: namespace.three.parts + +namespace com.linkedin.feathr.compute + +/** + * A type ref to int node id + */ +typeref NodeId = int diff --git a/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/NodeReference.pdl b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/NodeReference.pdl new file mode 100644 index 000000000..0018d6e63 --- /dev/null +++ b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/NodeReference.pdl @@ -0,0 +1,33 @@ +// LINT_SUPPRESS: namespace.three.parts + +namespace com.linkedin.feathr.compute + +/** + * This is used to refer to a node from another node. It is a combination of a node id and the indices of the keys from the + * original node array. + * For example, consider:- + * anchorA: { + * key: [viewerId, vieweeId] + * feature: featureA + * } + * Let us say featureA is evaluated in node 1. + * derivation: { + * key: [vieweeId, viewerId] + * args1: {key: [vieweeId, viewerId], feature: featureA} + * definition: args1*2 + * } + * Now, the node reference (to represent args1) would be: + * nodeId: 1 + * keyReference: [1,0] - // Indicates the ordering of the key indices. + */ +record NodeReference { + /** + * node id of the referring node. + */ + id: NodeId + + /** + * The key references in the keys of the referring node. + */ + keyReference: array[KeyReference] +} \ No newline at end of file diff --git a/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/OfflineKeyFunction.pdl b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/OfflineKeyFunction.pdl new file mode 100644 index 000000000..1d87edcaf --- /dev/null +++ b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/OfflineKeyFunction.pdl @@ -0,0 +1,23 @@ +// LINT_SUPPRESS: namespace.three.parts + +namespace com.linkedin.feathr.compute + +/** + * Represents a feature's key that is extracted from each row of an offline data source and is used to join with observation data to form a training dataset. This class is expected to be included so the definitions of enclosed fields can be reused. + */ +record OfflineKeyFunction { + +/** + * Key function specifies how to extract the feature's key from each row of the offline data source. For example, an offline dataset has x field, a key function being defined as getIdFromUrn(x) means the feature key is a numeric member id, which can later be used to join with observation data that also has numeric member id column. A feature's key can have one key part or multiple key parts (compound key). This field should be required, keeping it optional for fulfilling backward compatiblity requirement during schema evolution. + */ +keyFunction: optional union[ +//MVEL-based key function. It can either be a simple reference to a field name in the offline dataset, or apply some trasformations on top of some columns. + MvelExpression + +//SparkSql-based key function. Note this is experimental and can be deprecated in near future. + SqlExpression + +//UDF-based key function. It is useful when key function can't be written easily with an expression language like MVEL. For more details, refer to SourceKeyExtractor interface in above doc link. + UserDefinedFunction +] +} diff --git a/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/OperatorId.pdl b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/OperatorId.pdl new file mode 100644 index 000000000..02d550c4e --- /dev/null +++ b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/OperatorId.pdl @@ -0,0 +1,8 @@ +// LINT_SUPPRESS: namespace.three.parts + +namespace com.linkedin.feathr.compute + +/** + * operator id to set an operator. It can be referring to an mvel expression, sql expression or a java udf. + */ +typeref OperatorId = string \ No newline at end of file diff --git a/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/SlidingWindowFeature.pdl b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/SlidingWindowFeature.pdl new file mode 100644 index 000000000..d1e39833e --- /dev/null +++ b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/SlidingWindowFeature.pdl @@ -0,0 +1,72 @@ +// LINT_SUPPRESS: namespace.three.parts + +namespace com.linkedin.feathr.compute +/** + * Sliding window aggregation produces feature data by aggregating a collection of data within a given time interval into an aggregate value. It ensures point-in-time correctness, when joining with label data, feathr looks back the configurable time window from each entry's timestamp and compute the aggregagate value. + */ +record SlidingWindowFeature { + + /** + * The target column to perform aggregation against. + */ + targetColumn: union[ + //A Spark SQL expression. It can be a simple field reference, or a complex Spark SQL statement. + SqlExpression + ] + + /** + * Represents supported types of aggregation. + */ + aggregationType: enum AggregationType { + /** Sum. */ + SUM + /** Count. */ + COUNT + /** Max. */ + MAX + /** Min. */ + MIN + /** Average. */ + AVG + /** Pooling is a sample-based discretization process. The objective is to down-sample an input representation and reduce its dimensionality. Max pooling is done by applying a max filter to (usually) non-overlapping subregions of the initial representation. */ + MAX_POOLING + /** Pooling is a sample-based discretization process. The objective is to down-sample an input representation and reduce its dimensionality. Min pooling is done by applying a min filter to (usually) non-overlapping subregions of the initial representation. */ + MIN_POOLING + /** Pooling is a sample-based discretization process. The objective is to down-sample an input representation and reduce its dimensionality. Average pooling is done by applying a average filter to (usually) non-overlapping subregions of the initial representation. */ + AVG_POOLING + /** Latest */ + LATEST + } + + /** + * Represents the time window to look back from label data's timestamp. + */ + window: Window + + /** + * Represents lateral view statements to be applied before the aggregation. Refer to LateralView for more details. + */ + lateralViews: array[LateralView] = [] + + /** + * Represents the filter statement before the aggregation. + */ + filter: optional union[ + //A Spark SQL expression, for example, "channel = 'RECRUITER_SEARCH' AND event = 'SKIP'". + SqlExpression + ] + + /** + * Represents the target to be grouped by before aggregation. If groupBy is not set, the aggregation will be performed over the entire dataset. + */ + groupBy: optional union[ + //A Spark SQL expression, it can be a simple field reference, or a complex Spark SQL statement. + SqlExpression + ] + + /** + * Represents the max number of groups (with aggregation results) to return. + */ + limit: optional int +} + diff --git a/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/SqlExpression.pdl b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/SqlExpression.pdl new file mode 100644 index 000000000..5220f46c7 --- /dev/null +++ b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/SqlExpression.pdl @@ -0,0 +1,13 @@ +// LINT_SUPPRESS: namespace.three.parts + +namespace com.linkedin.feathr.compute + +/** + * An expression in Spark SQL. + */ +record SqlExpression { + /** + * The Spark SQL expression. + */ + sql: string +} diff --git a/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/TensorCategory.pdl b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/TensorCategory.pdl new file mode 100644 index 000000000..012315899 --- /dev/null +++ b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/TensorCategory.pdl @@ -0,0 +1,23 @@ +// LINT_SUPPRESS: namespace.three.parts + +namespace com.linkedin.feathr.compute + +/** + * Supported Tensor categories in feathr and Quince. + */ +enum TensorCategory { + /** + * Dense tensors store values in a contiguous sequential block of memory where all values are represented. + */ + DENSE + + /** + * Sparse tensor represents a dataset in which most of the entries are zero. It does not store the whole values of the tensor object but stores the non-zero values and the corresponding coordinates of them. + */ + SPARSE + + /** + * Ragged tensors (also known as nested tensors) are similar to dense tensors but have variable-length dimensions. + */ + RAGGED +} \ No newline at end of file diff --git a/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/TensorFeatureFormat.pdl b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/TensorFeatureFormat.pdl new file mode 100644 index 000000000..2a30db22f --- /dev/null +++ b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/TensorFeatureFormat.pdl @@ -0,0 +1,24 @@ +// LINT_SUPPRESS: namespace.three.parts + +namespace com.linkedin.feathr.compute + +/** + * Defines the format of feature data. Feature data is produced by applying transformation on source, in a FeatureAnchor. Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. In Quince Tensor specifically, the last column is designated as the value, and the rest of the columns are keys (aka dimensions). Each row defines a single key/value pair, each column can have a different type. For more details, refer to doc: https://docs.google.com/document/d/1D3JZWBwI7sgHrNzkHZwV3YNEHn69lZcl4VfhdHVmDJo/edit#. Currently in feathr, there are two ways to specify Feature formats, one is via Name-Term-Value (NTV) types (eg. NUMERIC, TERM_VECTOR, CATEGORICAL, see go/featuretypes), the other is via FML metadata (Feature Metadata Library, go/fml). For NTV types, there is a conversion path to Quince Tensor via Auto Tensorization. Existing NTV types can be mapped to different combinations of valueType and dimensionTypes in a deterministic manner. Refer to doc: https://docs.google.com/document/d/10bJMYlCixhsghCtyD08FsQaoQdAJMcpGnRyGe64TSr4/edit#. Feature owners can choose to define FML metadata (eg. valType, dimension's type, etc, see go/fml), which will also be converted to Quince Tensor internally. The data model in this class should be able to uniformly represent both cases. + */ +record TensorFeatureFormat { + + /** + * Type of the tensor, for example, dense tensor. + */ + tensorCategory: TensorCategory + + /** + * Type of the value column. + */ + valueType: ValueType + + /** + * A feature data can have zero or more dimensions (columns that represent keys). + */ + dimensions: array[Dimension] +} diff --git a/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/Time.pdl b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/Time.pdl new file mode 100644 index 000000000..575d7ba24 --- /dev/null +++ b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/Time.pdl @@ -0,0 +1,8 @@ +namespace com.linkedin.feathr.compute + +/** + * Number of milliseconds since midnight, January 1, 1970 UTC. It must be a positive number + */ +@compliance = "NONE" +typeref Time = long + diff --git a/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/TimestampCol.pdl b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/TimestampCol.pdl new file mode 100644 index 000000000..4e066eabb --- /dev/null +++ b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/TimestampCol.pdl @@ -0,0 +1,16 @@ +namespace com.linkedin.feathr.compute + +/** + * Representation of a timestamp column field + */ +record TimestampCol { + /** + * Timestamp column expression. + */ + expression: string + + /** + * Format of the timestamp, example - yyyy/MM/dd, epoch, epoch_millis + */ + format: string +} \ No newline at end of file diff --git a/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/Transformation.pdl b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/Transformation.pdl new file mode 100644 index 000000000..10c1fd9cd --- /dev/null +++ b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/Transformation.pdl @@ -0,0 +1,29 @@ +// LINT_SUPPRESS: namespace.three.parts + +namespace com.linkedin.feathr.compute + +/** + * Representation of a transformation node. + */ +record Transformation includes AbstractNode { + /** + * An array of node references which should be considered as input to apply the transformation function. + */ + inputs: array[NodeReference] + + /** + * The transformation function. + */ + function: TransformationFunction + + /** + * Feature name here is used so we retain feature name, type, and default values even after graph is resolved. + * Feature name here is also used for feature aliasing in the case where TransformationFunction is feature_alias. + */ + featureName: string + + /** + * feature version of the feature + */ + featureVersion: FeatureVersion +} \ No newline at end of file diff --git a/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/TransformationFunction.pdl b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/TransformationFunction.pdl new file mode 100644 index 000000000..32f4c0b15 --- /dev/null +++ b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/TransformationFunction.pdl @@ -0,0 +1,20 @@ +// LINT_SUPPRESS: namespace.three.parts + +namespace com.linkedin.feathr.compute + +/** + * The transformation function + */ +record TransformationFunction { + /** + * Indicates the operator type to be used here. The various different operators supported are in [[Operators]] class. + * + */ + operator: OperatorId + + /** + * The various attributes required to represent the transformation function are captured in a map format. + * For example, mvel expression or java udf class name + */ + parameters: optional map[string, string] +} diff --git a/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/UserDefinedFunction.pdl b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/UserDefinedFunction.pdl new file mode 100644 index 000000000..279328868 --- /dev/null +++ b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/UserDefinedFunction.pdl @@ -0,0 +1,17 @@ +// LINT_SUPPRESS: namespace.three.parts + +namespace com.linkedin.feathr.compute + +/** + * User defined function that can be used in feature extraction or derivation. + */ +record UserDefinedFunction { + /** + * Reference to the class that implements the user defined function. + */ + clazz: string + /** + * Some UserDefinedFunction requires additional custom parameters. This field defines the custom parameters of the user defined function, represented as a map of string to json blob. The key is the parameter name, and the value is the parameter value represented as a json blob. For example, the parameters may look like: { param1 : ["waterlooCompany_terms_hashed", "waterlooCompany_values"], param2 : "com.linkedin.quasar.encoding.SomeEncodingClass” } feathr will be responsible of parsing the parameters map into a CustomParameters class defined by application: public class CustomParameters { List param1; String param2; } CustomParameters will be used in the constructor of the UserDefinedFunction. + */ + parameters: map[string, string] = {} +} \ No newline at end of file diff --git a/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/ValueType.pdl b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/ValueType.pdl new file mode 100644 index 000000000..598f6ccad --- /dev/null +++ b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/ValueType.pdl @@ -0,0 +1,23 @@ +// LINT_SUPPRESS: namespace.three.parts + +namespace com.linkedin.feathr.compute + +/** + * Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. In Quince Tensor specifically, the last column is designated as the value, and the rest of the columns are keys (or dimensions); Each row defines a single key/value pair. This enum defines supported value types for tensors in Quince and feathr. + */ +enum ValueType { + /** Integer. */ + INT + /** Long. */ + LONG + /** Float. */ + FLOAT + /** Double. */ + DOUBLE + /** String. */ + STRING + /** Boolean. */ + BOOLEAN + /** Byte array. */ + BYTES +} diff --git a/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/Window.pdl b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/Window.pdl new file mode 100644 index 000000000..6176ebc62 --- /dev/null +++ b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/compute/Window.pdl @@ -0,0 +1,25 @@ +namespace com.linkedin.feathr.compute + +/** + * Represents a time window used in sliding window algorithms. + */ +record Window { + /** + * Represents the duration of the window. + */ + size: int + + /** + * Represents a unit of time. + */ + unit: enum Unit { + /** A day. */ + DAY + /** An hour. */ + HOUR + /** A minute. */ + MINUTE + /** A second. */ + SECOND + } +} diff --git a/feathr-data-models/src/main/pegasus/com/linkedin/feathr/config/join/AbsoluteDateRange.pdl b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/config/join/AbsoluteDateRange.pdl new file mode 100644 index 000000000..6c2de6188 --- /dev/null +++ b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/config/join/AbsoluteDateRange.pdl @@ -0,0 +1,24 @@ +// LINT_SUPPRESS: namespace.three.parts + +namespace com.linkedin.feathr.config.join + +/** + * The absolute date range with start and end date being required fields. + * It accepts a start date and an end date which should be specifiied using the [[Date.pdl]] class. + * absoluteDateRange: { + * startDate: Date(day=1, month=1, year=2020) + * endDate: Date(day=3, month=1, year=2020) + * } + * In this case, the endDate > startDate. + */ +record AbsoluteDateRange { + /** + * start date of the date range, with the start date included in the range. + */ + startDate: Date + + /** + * end date of the date range, with the end date included in the range. + */ + endDate: Date +} diff --git a/feathr-data-models/src/main/pegasus/com/linkedin/feathr/config/join/AbsoluteTimeRange.pdl b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/config/join/AbsoluteTimeRange.pdl new file mode 100644 index 000000000..2a9787fd3 --- /dev/null +++ b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/config/join/AbsoluteTimeRange.pdl @@ -0,0 +1,31 @@ +// LINT_SUPPRESS: namespace.three.parts + +namespace com.linkedin.feathr.config.join + +/** + * The absolute time range with start and end time being required fields. + * It accepts a start time and an end time which should be specifiied using the [[Date.pdl]] or the [[HourTime.pdl]] class. + * This model can be used to represent time range in daily or hourly interval. + * absoluteTimeRange: { + * startTime: TimeHour(day=1, month=1, year=2020, hour=13) + * endTime: TimeHour(day=3, month=1, year=2020, hour=2) + * } + * (or) + * absoluteTimeRange: { + * startTime: Date(day=1, month=1, year=2020) + * endTime: Date(day=3, month=1, year=2020) + * } + * endTime and startTime should always have the same granularity, ie - Daily or Hourly. + * endTme > startTime + */ +record AbsoluteTimeRange { + /** + * start time of the date range, in daily or hourly format with the start date included in the range. + */ + startTime: union[date: Date, hourTime: HourTime] + + /** + * end date of the date range, in daily or hourly format with the end date included in the range. + */ + endTime: union[date: Date, hourTime: HourTime] +} diff --git a/feathr-data-models/src/main/pegasus/com/linkedin/feathr/config/join/Date.pdl b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/config/join/Date.pdl new file mode 100644 index 000000000..de094f88a --- /dev/null +++ b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/config/join/Date.pdl @@ -0,0 +1,29 @@ +// LINT_SUPPRESS: namespace.three.parts + +namespace com.linkedin.feathr.config.join + +/** + * Represents a date in a calendar year including day, year and month + */ +record Date { + /** + * day + */ + @validate.integerRange.min = 1 + @validate.integerRange.max = 31 + day: int + + /** + * month + */ + @validate.integerRange.min = 1 + @validate.integerRange.max = 12 + month: int + + /** + * year + */ + @validate.integerRange.min = 1970 + @validate.integerRange.max = 2099 + year: int +} diff --git a/feathr-data-models/src/main/pegasus/com/linkedin/feathr/config/join/FrameFeatureJoinConfig.pdl b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/config/join/FrameFeatureJoinConfig.pdl new file mode 100644 index 000000000..09fdc5e32 --- /dev/null +++ b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/config/join/FrameFeatureJoinConfig.pdl @@ -0,0 +1,72 @@ +// LINT_SUPPRESS: namespace.three.parts + +namespace com.linkedin.feathr.config.join + +/** + * The join config consists of 2 parts, settings and features section. + * Settings is related to the general settings corresponding to joining the input data set with the + * features, currently there are time related settings, but this can be extended to other settings as well. + * Features to be joined are described by list of Keys and featureName and featureAlias. + * Features in the feature list should be joined to the user's input data. + * matching the key in the input data. + * For example, + * key is ["key1"] and join feature1 and feature2 with input data + * settings: { // optional field + * inputDataTimeSettings: { + * absoluteTimeRange: { + * startTime: Date(year=2020, month=4, day=28) + * endTime: Date(year=2020, month=5, day=5) + * } + * } + * joinTimeSettings: { + * timestampColumn: { + * def: timestamp + * format: yyyy-MM-dd + * } + * simulateTimeDelay: 5d + * } + * } + * features=[ + * JoiningFeature{ + * keys: ["key1"] + * frameFeatureName: "feature1" + * AbsoluteDateRange(startDate: Date(year=2020, month=5, day=1), + * endTime: Date(year=2020, month=5, day=5)) + * }, JoiningFeature{ + * keys: ["key1"] + * frameFeatureName: "feature2" + * overrideTimeDelay: 5d + * }, JoiningFeature{ + * keys: ["key1"] + * frameFeatureName: "feature3" + * RelativeDateRange(numDays: 5, + * offset: 3) + * }, JoiningFeature{ + * keys: ["key1"] + * frameFeatureName: "feature4" + * } + * ] + * + * Here, the keys are corresponding to column names in the input FeaturizedDataset, which will be used + * to join the feature source. Feature name is canonical feathr feature names. + * Each feature can also have a set of optional time-related parameters. These parameter override the ones provided in + * the settings section and are applicable only to the particular feature. + * Feature join config operation. + * + * All these PDLs are moved to feathr MP:- https://rb.corp.linkedin.com/r/2356512/ + */ +record FrameFeatureJoinConfig { + /** + * settings required for joining input featurized dataset with the feature data. + */ + settings: optional Settings + + /** + * Array of joining features. + * + * Validation rules: + * - The array must be non-empty. + */ + features: array[JoiningFeature] + +} \ No newline at end of file diff --git a/feathr-data-models/src/main/pegasus/com/linkedin/feathr/config/join/HourTime.pdl b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/config/join/HourTime.pdl new file mode 100644 index 000000000..5729f5fea --- /dev/null +++ b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/config/join/HourTime.pdl @@ -0,0 +1,36 @@ +// LINT_SUPPRESS: namespace.three.parts + +namespace com.linkedin.feathr.config.join + +/** + * Time with hourly granularity + */ +record HourTime { + /** + * day + */ + @validate.integerRange.min = 1 + @validate.integerRange.max = 31 + day: int + + /** + * month + */ + @validate.integerRange.min = 1 + @validate.integerRange.max = 12 + month: int + + /** + * year + */ + @validate.integerRange.min = 1970 + @validate.integerRange.max = 2099 + year: int + + /** + * hour + */ + @validate.integerRange.min = 0 + @validate.integerRange.max = 23 + hour: int +} diff --git a/feathr-data-models/src/main/pegasus/com/linkedin/feathr/config/join/InputDataTimeSettings.pdl b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/config/join/InputDataTimeSettings.pdl new file mode 100644 index 000000000..718ff6feb --- /dev/null +++ b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/config/join/InputDataTimeSettings.pdl @@ -0,0 +1,37 @@ +// LINT_SUPPRESS: namespace.three.parts + +namespace com.linkedin.feathr.config.join + +/** + * The data time settings pertaining to how much of the input dataset is to be loaded from the timestamp column. This is a way in which + * the input data can be restricted to allow only a fixed interval of dates to be joined with the feature data. This restriction + * will apply on the timestamp column of the input data. + * inputDataTimeSettings: { + * absoluteTimeRange: { + * startTime: Date(year=2020, month=8, day=8) + * endTime: Date(year=2020, month=8, day=10) + * } + * (or) + * relativeTimeRange: { + * offset: TimeOffset(length=1, unit="DAY") + * window: TimeWindow(length=1, unit="DAY") + * } + * } + */ +record InputDataTimeSettings { + /** + * Union of [[AbsoluteTimeRange]] and [[RelativeTimeRange]]. + * It indicates the range of input data which is to be loaded. This field generally refers to how much of the input + * data should be restricted using the time in the timestamp column. + * + * For example, + * a. startDate: "20200522", endDate: "20200525" implies this feature should be joined with the input data starting from + * 22nd May 2020 to 25th May, 2020 with both dates included. + * We only support yyyyMMdd format for this. In future, if there is a request, we can + * add support for other date time formats as well. + * + * b. numDays - 5d implies, offset - 1d, if today's date is 11/09/2020, then the input data ranging from 11/08/2020 + * till 11/04/2020 willl be joined. + */ + timeRange: union[absoluteTimeRange: AbsoluteTimeRange, relativeTimeRange: RelativeTimeRange] +} diff --git a/feathr-data-models/src/main/pegasus/com/linkedin/feathr/config/join/JoinTimeSettings.pdl b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/config/join/JoinTimeSettings.pdl new file mode 100644 index 000000000..4570316ce --- /dev/null +++ b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/config/join/JoinTimeSettings.pdl @@ -0,0 +1,22 @@ +// LINT_SUPPRESS: namespace.three.parts + +namespace com.linkedin.feathr.config.join + +/** + * JoinTimeSettings contains all the parameters required to join the time sensitive input data with the feature data. + * The input data can be time sensitive in two ways:- + * a. Have a timestamp column + * b. Always join with the latest available feature data. In this case, we do not require a timestamp column. + * c. The file path is time-partition and the path time is used for the join + * (Todo - Add useTimePartitionPattern field in this section) + * In this section, the user needs to let feathr know which of the above properties is to be used for the join. + */ + +typeref JoinTimeSettings = union[ + + // Settings to join with the latest available feature data. In this case, we do not require a timestamp column. + useLatestJoinTimeSettings: UseLatestJoinTimeSettings, + + // Settiings to use the timestamp column to join with feature data. + timestampColJoinTimeSettings: TimestampColJoinTimeSettings +] diff --git a/feathr-data-models/src/main/pegasus/com/linkedin/feathr/config/join/JoiningFeature.pdl b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/config/join/JoiningFeature.pdl new file mode 100644 index 000000000..7b477eb29 --- /dev/null +++ b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/config/join/JoiningFeature.pdl @@ -0,0 +1,107 @@ +// LINT_SUPPRESS: namespace.three.parts + +namespace com.linkedin.feathr.config.join + +/** + * JoiningFeature is the feature section of the join config. This section consists of information pertaining to a feature + * which is to be joined:- + * a. The join keys of the input data, with which this feature is to be joined. + * b. name of the feature + * c. optional timeRange of the input data which is to be joined with this feature. + * d. optional overrideTimeDelay if this feature needs a different simulate time delay other than the one mentioned. + * + * This is a required section of the the join config. + * Example, + * a. JoiningFeature{ + * keys: ["key1"] + * frameFeatureName: "feature1" + * AbsoluteDateRange(startDate: Date(year=2020, month=5, day=5), + * endDate: Date(year=2020, month=5, day=7)) + * } + * b. JoiningFeature{ + * keys: ["key1"] + * frameFeatureName: "feature2" + * overrideTimeDelay: TimeDelay(length=1, unit="DAY") + * } + * c. JoiningFeature{ + * keys: ["key1"] + * frameFeatureName: "feature3" + * RelativeDateRange(numDays: 5, + * offset: 3) + * } + */ + +record JoiningFeature { + + /** + * Keys to join input with feature source, the field name of the key in the input featuized dataset. + */ + keys: array[string] + + /** + * Feature name as defined in feathr's feature definition configuration. + * + * Currently the column in the output FDS that holds this feature will have the same name as feature name. + * If multiple joined features have the same name and no alias is defined for them, feathr will prepend the keys to the feature name. + * + * In the future, if "featureAlias" is not set, the column in the output FDS that holds this feature will have the same name as feature name. + * If multiple joined features have the same name and no alias is defined for them, the join operation will fail + * (to avoid produciing two columns in the output FDS with the same name). + */ + frameFeatureName: string + + /** + * The development of this is in progress. This is not in use for now. + * + * The name to be used for the column in the output FDS that contains the values from this joined feature. + * If not set, the name of the feature (frameFeatureName) will be used for the output column. + * For example, if the user request joining a feature named "careers_job_listTime" and provides no alias, + * the output FDS will contain a column called "careers_job_listTime". However, if the user sets "featureAlias" to "list_time", + * the column will be named "list_time". + * + * feature alias can be useful for in a few cases: + * - If the user prefers to use a name different than the feathr name in their model, + * they can use an alias to control the name of the column in the output FDS. + * - Sometimes, the training datas needs to have two features that are from the same feathr feature. + * For example, if we are modeing the problem of the probability of a member A (viewer) seeing the profile of member B + * (viewee) and we want to use the skills of both viewer and viewee as features, we need to join feathr feature + * "member_skills" of member A with feathr feature "member_skills" of member B. That is, the two features are the same + * feature but for different entiity ids). The default behavior of join is to name the output column name using the feathr + * feature name, but in a case like the above case, that would result in two columns with the same name, + * which is not valid for FDS. In these cases, the user has to provide an alias for at least one of these joined features. + * For example, the user can use featureAliases such as "viewer_skills" and "viewee_skills". + * In these cases, featureAliases becomes mandatory. + */ + featureAlias: optional string + + /** + * dateRange is used in Time-based joins, which refers to the situation when one or multiple days of input data needs + * to be used for training. + * One of the common use cases where this is used, is in training with some time-insensitive features, or + * training pipeline that always use the full day data, one day before running (since there is only partial data for today). + * The time for the input featurized dataset can be set using this field. + * Hourly data is not allowed in this case. + * + * For example, + * a. startDate: "20200522", endDate: "20200525" implies this feature should be joined with the input data starting from + * 22nd May 2020 to 25th May, 2020 with both dates included. + * We only support yyyyMMdd format for this. In future, if there is a request, we can + * add support for other date time formats as well. + * + * b. numDays - 5d implies, offset - 1d, if today's date is 11/09/2020, then the input data ranging from 11/08/2020 + * till 11/04/2020 willl be joined. + * + * P.S - This is different from the timeRange used in settings as the settings startTime is applicable for the entire input data, + * while this a feature level setting. Also, we do not support hourly time here. + */ + dateRange: optional union[absoluteDateRange: AbsoluteDateRange, relativeDateRange: RelativeDateRange] + + /** + * The override time delay parameter which will override the global simulate time delay specified in the settings section for + * the particular feature. + * This parameter is only applicable when the simulate time delay is set in the settings section + * For example, let us say the global simulate delay was 5d, and the overrideTimeDelay is set to 3d. + * Then, for this specificc feature, a simulate delay of 3d will be applied. + */ + overrideTimeDelay: optional TimeOffset +} diff --git a/feathr-data-models/src/main/pegasus/com/linkedin/feathr/config/join/RelativeDateRange.pdl b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/config/join/RelativeDateRange.pdl new file mode 100644 index 000000000..427b6713e --- /dev/null +++ b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/config/join/RelativeDateRange.pdl @@ -0,0 +1,31 @@ +// LINT_SUPPRESS: namespace.three.parts + +namespace com.linkedin.feathr.config.join + +/** + * The date range represented relative to the current date. It uses the current system date as the reference and can be used to + * express a range of dates with respect to the current date. + * Example, - If current date is 01/01/2020, window is 3, and offset 1 (unit is number of days) + * then this corresponds to the following 3 days, ie- starting from (current date - offset), ie - 12/31/2019, 12/30/2019 and 12/29/2019. + * + * If dateOffset is not specified, it defaults to 0. + * relativeDateRange: RelativeDateRange(numDays=2, dateOffset=1) + * relativeDateRange: RelativeDateRange(numDays=5) + */ +record RelativeDateRange { + + /** + * Represents a length of time. + * numDays is the window from the reference date to look back to obtain a dateRange. + * For example, numDays - 5 implies, if reference date is 11/09/2020, then numDays will range from 11/09/2020 + * till 11/05/2020. + */ + @validate.positive = { } + numDays: long + + /** + * Number of days to backdate from current date, to obtain the reference date. For example, if dateOffset is 4, then reference date + * will be 4 days ago from today. + */ + dateOffset: long = 0 +} diff --git a/feathr-data-models/src/main/pegasus/com/linkedin/feathr/config/join/RelativeTimeRange.pdl b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/config/join/RelativeTimeRange.pdl new file mode 100644 index 000000000..4752bedd0 --- /dev/null +++ b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/config/join/RelativeTimeRange.pdl @@ -0,0 +1,32 @@ +// LINT_SUPPRESS: namespace.three.parts + +namespace com.linkedin.feathr.config.join + +/** + * The time range represented relative to the current timestamp. It uses the current system time as the reference and can be used to + * express a range of times with respect to the current time. + * Example, - If current time is 01/01/2020, window is 3 days, and offset is 1 day (unit can be day or hour). + * then this corresponds to the following 3 days, ie- starting from (current date - offset), ie - 12/31/2019, 12/30/2019 and 12/29/2019. + * + * relativeTimeRange: RelativeTimeRange(window=TimeWindow(length=2, unit="DAY"), offset=TimeOffset(length=1, unit="Day")) + * relativeTimeRange: RelativeTimeRange(window=TimeWindow(length=2, unit="HOUR")) + */ +record RelativeTimeRange { + /** + * Window is the number of time units from the reference time units to look back to obtain the timeRange. + * For example, window - 5days implies, if reference date is 11/09/2020, then range will be from 11/09/2020 + * till 11/05/2020 (both days included). + * window >= 1 TimeUnit + */ + window: TimeWindow + + /** + * Number of time units (corresponding to window's timeUnits) to backdate from current time, to obtain the reference time. + * For example, if dateOffset is 4, and window is 2 days, then reference time + * will be 4 days ago from today. + * Example - if today's date is 11th Dec, 2020 and offset is 4 days - Reference time will be 7th Dec, 2020. + * This will always take the window's timeUnits. + */ + @validate.integerRange.min = 0 + offset: long = 0 +} diff --git a/feathr-data-models/src/main/pegasus/com/linkedin/feathr/config/join/Settings.pdl b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/config/join/Settings.pdl new file mode 100644 index 000000000..9a4eccdc3 --- /dev/null +++ b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/config/join/Settings.pdl @@ -0,0 +1,37 @@ +// LINT_SUPPRESS: namespace.three.parts + +namespace com.linkedin.feathr.config.join + +/** + * The settings section contains all the config parameters required for the joining of input dataset with the + * feature data. As of now, we have only time related parameters, but in future this can be expanded. + * This section has configs related to:- + * a. How do I load the input dataset if it is time sensitive? + * b. How do I specify the join parameters for input dataset? + * For more details - https://docs.google.com/document/d/1C6u2CKWSmOmHDQEL8Ovm5V5ZZFKhC_HdxVxU9D1F9lg/edit# + * settings: { + * inputDataTimeSettings: { + * absoluteTimeRange: { + * startTime: 20200809 + * endTime: 20200810 + * timeFormat: yyyyMMdd + * } + * } + * joinTimeSettings: { + * useLatestFeatureData: true + * } + * } + */ +record Settings { + + /** + * Config parameters related to loading of the time sensitive input data. Contains parameters related to restricting the + * size of the input data with respect to the timestamp column. + */ + inputDataTimeSettings: optional InputDataTimeSettings + + /** + * This contains all the parameters required to join the time sensitive input data with the feature data. + */ + joinTimeSettings: optional JoinTimeSettings +} diff --git a/feathr-data-models/src/main/pegasus/com/linkedin/feathr/config/join/SparkSqlExpression.pdl b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/config/join/SparkSqlExpression.pdl new file mode 100644 index 000000000..f75bd1b42 --- /dev/null +++ b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/config/join/SparkSqlExpression.pdl @@ -0,0 +1,13 @@ +// LINT_SUPPRESS: namespace.three.parts + +namespace com.linkedin.feathr.config.join + +/** + * An expression in Spark SQL. + */ +record SparkSqlExpression { + /** + * The Spark SQL expression. + */ + expression: string +} diff --git a/feathr-data-models/src/main/pegasus/com/linkedin/feathr/config/join/TimeFormat.pdl b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/config/join/TimeFormat.pdl new file mode 100644 index 000000000..0e48109e9 --- /dev/null +++ b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/config/join/TimeFormat.pdl @@ -0,0 +1,9 @@ +// LINT_SUPPRESS: namespace.three.parts + +namespace com.linkedin.feathr.config.join + +/** + * The timeformat, which accepts the formats parsed by the DateTimeFormatter java class or epoch or epoch_millis. However in future, we can have + * the option of a stronger type. Example, dd/MM/yyyy, yyyy-MM-dd, epoch, epoch_millis, etc. + */ +typeref TimeFormat = string \ No newline at end of file diff --git a/feathr-data-models/src/main/pegasus/com/linkedin/feathr/config/join/TimeOffset.pdl b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/config/join/TimeOffset.pdl new file mode 100644 index 000000000..9f1be2657 --- /dev/null +++ b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/config/join/TimeOffset.pdl @@ -0,0 +1,20 @@ +// LINT_SUPPRESS: namespace.three.parts + +namespace com.linkedin.feathr.config.join + +/** + * TimeOffset is the amount of time we need to push back the current time wrt a reference time. Since, reference time can + * be any time in the past also, we do allow a positive or negative offset length. + * offset - 1 day implies the previous from the reference day. + */ +record TimeOffset { + /** + * Amount of the duration in TimeUnits. Can be positive or negative. + */ + length: long + + /** + * Time unit for "length". For example, TimeUnit.DAY or TimeUnit.HOUR. + */ + unit: TimeUnit +} diff --git a/feathr-data-models/src/main/pegasus/com/linkedin/feathr/config/join/TimeUnit.pdl b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/config/join/TimeUnit.pdl new file mode 100644 index 000000000..914cb23cd --- /dev/null +++ b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/config/join/TimeUnit.pdl @@ -0,0 +1,25 @@ +// LINT_SUPPRESS: namespace.three.parts + +namespace com.linkedin.feathr.config.join + +/** + * Unit of time used for defining a time range. + */ +enum TimeUnit { + /** + * Daily format + */ + DAY, + /** + * Hourly format + */ + HOUR, + /** + * minute format, this can be used in simulate time delay + */ + MINUTE, + /** + * second format, this can be used in simulate time delay + */ + SECOND +} diff --git a/feathr-data-models/src/main/pegasus/com/linkedin/feathr/config/join/TimeWindow.pdl b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/config/join/TimeWindow.pdl new file mode 100644 index 000000000..35f88a5ad --- /dev/null +++ b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/config/join/TimeWindow.pdl @@ -0,0 +1,19 @@ +// LINT_SUPPRESS: namespace.three.parts + +namespace com.linkedin.feathr.config.join + +/** + * Represents a length of time along with the corresponding time unit (DAY, HOUR). + */ +record TimeWindow { + /** + * Amount of the duration in TimeUnits. Can be greater or equal to 1. + */ + @validate.positive + length: long + + /** + * Time unit for "length". For example, TimeUnit.DAY or TimeUnit.HOUR. + */ + unit: TimeUnit +} diff --git a/feathr-data-models/src/main/pegasus/com/linkedin/feathr/config/join/TimestampColJoinTimeSettings.pdl b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/config/join/TimestampColJoinTimeSettings.pdl new file mode 100644 index 000000000..8b71e6cda --- /dev/null +++ b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/config/join/TimestampColJoinTimeSettings.pdl @@ -0,0 +1,33 @@ +// LINT_SUPPRESS: namespace.three.parts + +namespace com.linkedin.feathr.config.join + +/** + * Settings needed when the input data has a timestamp which should be used for the join. + * joinTimeSettings: { + * timestampColumn: { + * def: timestamp + * format: yyyy/MM/dd + * } + * simulateTimeDelay: 1d + * } + */ +record TimestampColJoinTimeSettings { + /** + * The timestamp column name and timeformat which should be used for joining with the feature data. + * Refer to [[TimestampColumn]]. + * Example, TimestampColumn: { + * def: timestamp + * format: yyyy/MM/dd + * } + */ + timestampColumn: TimestampColumn + + /** + * An optional simulate time delay parameter which can be set by the user. Indicates the amount of time that is to subtracted + * from the input data timestamp while joining with the feature data. + * We do support negative time delays. + */ + simulateTimeDelay: optional TimeOffset +} + diff --git a/feathr-data-models/src/main/pegasus/com/linkedin/feathr/config/join/TimestampColumn.pdl b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/config/join/TimestampColumn.pdl new file mode 100644 index 000000000..6e588363e --- /dev/null +++ b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/config/join/TimestampColumn.pdl @@ -0,0 +1,26 @@ +// LINT_SUPPRESS: namespace.three.parts + +namespace com.linkedin.feathr.config.join + +/** + * Timestamp column of the input featureiized dataset, which is to be used for the join. + * timestampColumn: { + * def: timestamp + * format: yyyyMMdd + * } + */ +record TimestampColumn { + /** + * The definiton of the timestamp column, which can be a sql expression involving the timestamp column + * or just the column name + * Example:- definition: timestamp, timestamp + 10000000. + */ + definition: union[columnName: string, sparkSqlExpression: SparkSqlExpression] + + /** + * Format of the timestamp column. Must confer to java's timestampFormatter or can be + * epoch or epoch_millis. + * Example:- epoch, epoch_millis, yyyy/MM/dd + */ + format: TimeFormat +} diff --git a/feathr-data-models/src/main/pegasus/com/linkedin/feathr/config/join/UseLatestJoinTimeSettings.pdl b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/config/join/UseLatestJoinTimeSettings.pdl new file mode 100644 index 000000000..9004cd339 --- /dev/null +++ b/feathr-data-models/src/main/pegasus/com/linkedin/feathr/config/join/UseLatestJoinTimeSettings.pdl @@ -0,0 +1,17 @@ +// LINT_SUPPRESS: namespace.three.parts + +namespace com.linkedin.feathr.config.join + +/** + * Settings needed when the input data is to be joined with the latest available feature data. + * joinTimeSettings: { + * useLatestFeatureData: true + * } + */ +record UseLatestJoinTimeSettings { + /** + * Boolean value, if set to true, indicates that the latest available feature data is to be used for joining. + * When useLatestFeatureData is set, there should be no other time-based parameters. + */ + useLatestFeatureData: boolean = true +} diff --git a/feathr-impl/build.gradle b/feathr-impl/build.gradle new file mode 100644 index 000000000..b15e0c5fa --- /dev/null +++ b/feathr-impl/build.gradle @@ -0,0 +1,140 @@ +plugins { + id 'scala' + id 'maven-publish' + id 'signing' + id "com.vanniktech.maven.publish.base" +} + +repositories { + mavenCentral() + mavenLocal() + maven { + url "https://repository.mulesoft.org/nexus/content/repositories/public/" + } + maven { + url "https://linkedin.jfrog.io/artifactory/open-source/" // GMA, pegasus + } + +} + +configurations { + // configuration that holds jars to include in the jar + extraLibs + + // Dependencies that will be provided at runtime in the cloud execution + provided + + compileOnly.extendsFrom(provided) + testImplementation.extendsFrom provided +} + +configurations.all { + resolutionStrategy.force "org.antlr:antlr4-runtime:4.8" + resolutionStrategy.force "org.antlr:antlr4-tool:4.8" +} + +dependencies { + implementation project(":feathr-compute") + implementation project(":feathr-config") + implementation project(":feathr-data-models") + implementation project(path: ':feathr-data-models', configuration: 'dataTemplate') + // needed to include data models in jar + extraLibs project(path: ':feathr-data-models', configuration: 'dataTemplate') + implementation spec.product.scala.scala_library + + implementation spec.product.jackson.dataformat_csv + implementation spec.product.jackson.dataformat_yaml + implementation spec.product.jackson.module_scala + implementation spec.product.jackson.dataformat_hocon + implementation spec.product.jackson.jackson_core + implementation spec.product.spark_redis + implementation spec.product.fastutil + implementation spec.product.hadoop.mapreduce_client_core + implementation spec.product.mvel + implementation spec.product.jackson.jackson_module_caseclass + implementation spec.product.protobuf + implementation spec.product.guava + implementation spec.product.xbean + implementation spec.product.json + implementation spec.product.avroUtil + implementation spec.product.antlr + implementation spec.product.antlrRuntime + + implementation spec.product.jackson.jackson_databind + provided spec.product.typesafe_config + provided spec.product.log4j + provided spec.product.hadoop.common + provided(spec.product.spark.spark_core) { + exclude group: 'org.apache.xbean', module: 'xbean-asm6-shaded' + } + provided(spec.product.spark.spark_avro) { + exclude group: 'org.apache.xbean', module: 'xbean-asm6-shaded' + } + provided(spec.product.spark.spark_hive) { + exclude group: 'com.tdunning', module: 'json' + } + provided spec.product.spark.spark_sql + + testImplementation spec.product.equalsverifier + testImplementation spec.product.spark.spark_catalyst + testImplementation spec.product.mockito + testImplementation spec.product.scala.scalatest + testImplementation spec.product.testing + testImplementation spec.product.jdiagnostics +} + +// Since there are cross-calls from Scala to Java, we use joint compiler +// to compile them at the same time with Scala compiler. +// See https://docs.gradle.org/current/userguide/scala_plugin.html +sourceSets { + main { + scala { + srcDirs = ['src/main/scala', 'src/main/java'] + } + java { + srcDirs = [] + } + } + test { + scala { + srcDirs = ['src/test/scala', 'src/test/java'] + } + java { + srcDirs = [] + } + } +} + +test { + useTestNG() +} + + +java { + withSourcesJar() + withJavadocJar() +} + +tasks.withType(Javadoc) { + options.addStringOption('Xdoclint:none', '-quiet') + options.addStringOption('encoding', 'UTF-8') + options.addStringOption('charSet', 'UTF-8') +} + +// Required for publishing to local maven +publishing { + publications { + mavenJava(MavenPublication) { + artifactId = 'feathr-impl' + from components.java + versionMapping { + usage('java-api') { + fromResolutionOf('runtimeClasspath') + } + usage('java-runtime') { + fromResolutionResult() + } + } + } + } +} diff --git a/src/main/java/com/linkedin/feathr/cli/FeatureExperimentEntryPoint.java b/feathr-impl/src/main/java/com/linkedin/feathr/cli/FeatureExperimentEntryPoint.java similarity index 80% rename from src/main/java/com/linkedin/feathr/cli/FeatureExperimentEntryPoint.java rename to feathr-impl/src/main/java/com/linkedin/feathr/cli/FeatureExperimentEntryPoint.java index c7a4c0279..bae2627fa 100644 --- a/src/main/java/com/linkedin/feathr/cli/FeatureExperimentEntryPoint.java +++ b/feathr-impl/src/main/java/com/linkedin/feathr/cli/FeatureExperimentEntryPoint.java @@ -3,14 +3,15 @@ import com.linkedin.feathr.offline.testfwk.generation.FeatureGenExperimentComponent; import py4j.GatewayServer; +import java.io.File; /** * The entry point for Py4j to access the feature experiment component in Java world. */ public class FeatureExperimentEntryPoint { public String getResult(String userWorkspaceDir, String featureNames) { - String mockDataDir = userWorkspaceDir + "/mockdata/"; - String featureDefFile = userWorkspaceDir + "/feature_conf/"; + String mockDataDir = new File(userWorkspaceDir, "mockdata").getAbsolutePath(); + String featureDefFile = new File(userWorkspaceDir, "feature_conf").getAbsolutePath(); FeatureGenExperimentComponent featureGenExperimentComponent = new FeatureGenExperimentComponent(); return featureGenExperimentComponent.prettyPrintFeatureGenResult(mockDataDir, featureNames, featureDefFile); } diff --git a/src/main/java/com/linkedin/feathr/common/AutoTensorizableTypes.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/AutoTensorizableTypes.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/AutoTensorizableTypes.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/AutoTensorizableTypes.java diff --git a/src/main/java/com/linkedin/feathr/common/CoercingTensorData.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/CoercingTensorData.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/CoercingTensorData.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/CoercingTensorData.java diff --git a/src/main/java/com/linkedin/feathr/common/CompatibilityUtils.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/CompatibilityUtils.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/CompatibilityUtils.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/CompatibilityUtils.java diff --git a/src/main/java/com/linkedin/feathr/common/Equal.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/Equal.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/Equal.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/Equal.java diff --git a/src/main/java/com/linkedin/feathr/common/ErasedEntityTaggedFeature.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/ErasedEntityTaggedFeature.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/ErasedEntityTaggedFeature.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/ErasedEntityTaggedFeature.java diff --git a/src/main/java/com/linkedin/feathr/common/Experimental.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/Experimental.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/Experimental.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/Experimental.java diff --git a/src/main/java/com/linkedin/feathr/common/FeatureAggregationType.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/FeatureAggregationType.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/FeatureAggregationType.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/FeatureAggregationType.java diff --git a/src/main/java/com/linkedin/feathr/common/FeatureDependencyGraph.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/FeatureDependencyGraph.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/FeatureDependencyGraph.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/FeatureDependencyGraph.java diff --git a/src/main/java/com/linkedin/feathr/common/FeatureError.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/FeatureError.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/FeatureError.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/FeatureError.java diff --git a/src/main/java/com/linkedin/feathr/common/FeatureErrorCode.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/FeatureErrorCode.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/FeatureErrorCode.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/FeatureErrorCode.java diff --git a/src/main/java/com/linkedin/feathr/common/FeatureExtractor.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/FeatureExtractor.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/FeatureExtractor.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/FeatureExtractor.java diff --git a/src/main/java/com/linkedin/feathr/common/FeatureTypeConfig.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/FeatureTypeConfig.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/FeatureTypeConfig.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/FeatureTypeConfig.java diff --git a/src/main/java/com/linkedin/feathr/common/FeatureTypeConfigDeserializer.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/FeatureTypeConfigDeserializer.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/FeatureTypeConfigDeserializer.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/FeatureTypeConfigDeserializer.java diff --git a/src/main/java/com/linkedin/feathr/common/FeatureTypes.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/FeatureTypes.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/FeatureTypes.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/FeatureTypes.java diff --git a/src/main/java/com/linkedin/feathr/common/FeatureValue.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/FeatureValue.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/FeatureValue.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/FeatureValue.java diff --git a/src/main/java/com/linkedin/feathr/common/FeatureVariableResolver.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/FeatureVariableResolver.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/FeatureVariableResolver.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/FeatureVariableResolver.java diff --git a/src/main/java/com/linkedin/feathr/common/GenericTypedTensor.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/GenericTypedTensor.java similarity index 96% rename from src/main/java/com/linkedin/feathr/common/GenericTypedTensor.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/GenericTypedTensor.java index 3b2240cd2..804d2fcca 100644 --- a/src/main/java/com/linkedin/feathr/common/GenericTypedTensor.java +++ b/feathr-impl/src/main/java/com/linkedin/feathr/common/GenericTypedTensor.java @@ -56,6 +56,9 @@ public TypedTensor slice(final Object val) { throw UNSUPPORTED_OPERATION_EXCEPTION; } + @Override + public TypedTensor subSlice(Object val) { throw UNSUPPORTED_OPERATION_EXCEPTION; } + /** * Returns human-readable summary suitable for debugging. */ diff --git a/src/main/java/com/linkedin/feathr/common/Hasher.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/Hasher.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/Hasher.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/Hasher.java diff --git a/src/main/java/com/linkedin/feathr/common/InternalApi.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/InternalApi.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/InternalApi.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/InternalApi.java diff --git a/src/main/java/com/linkedin/feathr/common/ParameterizedFeatureExtractor.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/ParameterizedFeatureExtractor.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/ParameterizedFeatureExtractor.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/ParameterizedFeatureExtractor.java diff --git a/feathr-impl/src/main/java/com/linkedin/feathr/common/PegasusDefaultFeatureValueResolver.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/PegasusDefaultFeatureValueResolver.java new file mode 100644 index 000000000..7c94ea5d8 --- /dev/null +++ b/feathr-impl/src/main/java/com/linkedin/feathr/common/PegasusDefaultFeatureValueResolver.java @@ -0,0 +1,206 @@ +package com.linkedin.feathr.common; + +import com.google.common.annotations.VisibleForTesting; +import com.linkedin.feathr.common.exception.ErrorLabel; +import com.linkedin.feathr.common.exception.FeathrException; +import com.linkedin.feathr.common.tensor.TensorType; +import com.linkedin.feathr.common.types.PrimitiveType; +import com.linkedin.feathr.compute.FeatureVersion; +import com.typesafe.config.Config; +import com.typesafe.config.ConfigFactory; +import com.typesafe.config.ConfigValue; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This class extracts default {@link FeatureValue} from pegasus models + */ +public class PegasusDefaultFeatureValueResolver { + private static final String DEFAULT_VALUE_PATH = "MOCK_DEFAULT_VALUE_PATH"; + private static final String HOCON_PREFIX = "{ "; + private static final String HOCON_SUFFIX = " }"; + private static final String HOCON_DELIM = " : "; + + private static final PegasusDefaultFeatureValueResolver INSTANCE = + new PegasusDefaultFeatureValueResolver(PegasusFeatureTypeResolver.getInstance()); + + private final PegasusFeatureTypeResolver _pegasusFeatureTypeResolver; + + private static final Logger LOG = LoggerFactory.getLogger(PegasusDefaultFeatureValueResolver.class.getSimpleName()); + + public static PegasusDefaultFeatureValueResolver getInstance() { + return INSTANCE; + } + + /** + * Package private constructor for testing with mock + */ + PegasusDefaultFeatureValueResolver(PegasusFeatureTypeResolver pegasusFeatureTypeResolver) { + _pegasusFeatureTypeResolver = pegasusFeatureTypeResolver; + } + + /** + * Resolve default value in the format of {@link FeatureValue} from {@link FeatureVersion}. + * The resolver does not cache the intermediate and final result. + * + * @param featureName the feature name + * @param featureVersion the Pegasus {@link FeatureVersion} record + * @return Optional of {@link FeatureValue}, empty if there is resolving exceptions, or if the input does not contain default value information + */ + public Optional resolveDefaultValue(String featureName, FeatureVersion featureVersion) { + if (!featureVersion.hasDefaultValue()) { + return Optional.empty(); + } + + if (!Objects.requireNonNull(featureVersion.getDefaultValue()).isString()) { + throw new RuntimeException("The default value type for " + featureName + + " is not supported, currently only support HOCON string"); + } + + String rawExpr = featureVersion.getDefaultValue().getString(); + + /* + * The default value stored in FeatureVersion is always a HOCON expression. + * The HOCON expression can not be directly parsed. + * Here we construct a valid HOCON string from the expression, and load the HOCON string with ConfigFactory. + * + * For instance, suppose the default value HOCON expression is "true", it can not be directly converted to a valid + * HOCON object. To correctly parse it, we build a valid HOCON string as follows + * "{ MOCK_DEFAULT_VALUE_PATH: true }". + */ + StringBuilder hoconStringBuilder = new StringBuilder(); + hoconStringBuilder.append(HOCON_PREFIX).append(DEFAULT_VALUE_PATH).append(HOCON_DELIM).append(rawExpr).append(HOCON_SUFFIX); + String hoconFullString = hoconStringBuilder.toString(); + Config config = ConfigFactory.parseString(hoconFullString); + + FeatureTypeConfig featureTypeConfig = _pegasusFeatureTypeResolver.resolveFeatureType(featureVersion); + Optional featureValue = resolveDefaultValue(featureTypeConfig, config); + + if (!featureValue.isPresent()) { + String errMessage = String.join("", "Fail to extract default FeatureValue for ", featureName, + " from raw expression:\n", rawExpr); + throw new RuntimeException(errMessage); + } + + LOG.info("The default value for feature {} is resolved as {}", featureName, featureValue.get()); + + return featureValue; + } + + private Optional resolveDefaultValue(FeatureTypeConfig featureTypeConfig, Config config) { + + ConfigValue defaultConfigValue = config.getValue(DEFAULT_VALUE_PATH); + // taking advantage of HOCON lib to extract default value Java object + // TODO - 14639) + // The behaviour here between JACKSON parser and TypeSafe config is slightly different. + // JACKSON parser allows us to specify the type via syntax like: 1.2f, 1.2d, 1.2L to respectively show they are + // float, double and Long. However, there is no way to do this in TypeSafe config. In TypeSafe config, + // 1.2f, 1.2d and 1.2L will all be considered as String. + Object defaultValueObj = defaultConfigValue.unwrapped(); + Optional normalizedDefaultValue = normalize(defaultValueObj); + + if (!normalizedDefaultValue.isPresent()) { + return Optional.empty(); + } + + Object defaultData = normalizedDefaultValue.get(); + FeatureTypes featureType = featureTypeConfig.getFeatureType(); + if (featureType != FeatureTypes.TENSOR) { + FeatureValue featureValue = new FeatureValue(defaultData, featureType); + return Optional.of(featureValue); + } else if (featureTypeConfig.getTensorType() != null) { + TensorType tensorType = featureTypeConfig.getTensorType(); + Object coercedDefault = defaultData; + // For float and double, we need to coerce it to make it more flexible. + // Otherwise it's quite common to see the two being incompatible. + // We are doing it here instead of inside FeatureValue.createTensor, since FeatureValue.createTensor is called + // more frequent and expensive and here it's usually called once during initialization. + if (tensorType.getDimensionTypes().size() == 0 && defaultData instanceof Number) { + Number num = (Number) defaultData; + // for scalar, defaultData is either double, string, or boolean so we need to coerce into corresponding types here. + if (tensorType.getValueType() == PrimitiveType.FLOAT) { + coercedDefault = num.floatValue(); + } else if (tensorType.getValueType() == PrimitiveType.DOUBLE) { + coercedDefault = num.doubleValue(); + } else if (tensorType.getValueType() == PrimitiveType.INT) { + coercedDefault = num.intValue(); + } else if (tensorType.getValueType() == PrimitiveType.LONG) { + coercedDefault = num.longValue(); + } + } + + FeatureValue featureValue = FeatureValue.createTensor(coercedDefault, featureTypeConfig.getTensorType()); + return Optional.of(featureValue); + } else { + throw new FeathrException(ErrorLabel.FEATHR_USER_ERROR, "Unknown default value "); + } + } + + @VisibleForTesting + Optional normalize(Object defaultValue) { + if (defaultValue instanceof Number) { + return Optional.of(normalizeNumber(defaultValue)); + } else if (defaultValue instanceof List) { + return normalizeList(defaultValue); + } else if (defaultValue instanceof Map) { + return normalizeMap(defaultValue); + } else { + // the rest type (String and Boolean) are directly supported + return Optional.of(defaultValue); + } + } + + private Optional normalizeList(Object defaultValue) { + ArrayList defaultList = new ArrayList<>(); + + List list = (List) defaultValue; + + for (Object elem : list) { + if (elem instanceof String) { + defaultList.add(elem); + } else if (elem instanceof Number) { + defaultList.add(normalizeNumber(elem)); + } else if (elem instanceof Boolean) { + defaultList.add(Boolean.valueOf(elem.toString())); + } else { + // value type can only be String or numeric + LOG.error("List element type not supported when resolving default value: {} .\n" + + "Only List and List are supported when defining List type default value.", elem); + return Optional.empty(); + } + } + return Optional.of(defaultList); + } + + private Optional normalizeMap(Object defaultValue) { + Map defaultMap = new HashMap<>(); + HashMap map = (HashMap) defaultValue; + for (String key : map.keySet()) { + Object valueObj = map.get(key); + if (valueObj instanceof Number) { + Number num = (Number) valueObj; + defaultMap.put(key, num.floatValue()); + } else if (valueObj instanceof Boolean) { + defaultMap.put(key, Boolean.valueOf(valueObj.toString())); + } else { + // The value type can only be numeric + LOG.error( + "Only Map type is supported when defining Map typed default value. The value type is not supported: " + + valueObj); + return Optional.empty(); + } + } + return Optional.of(defaultMap); + } + + private Double normalizeNumber(Object defaultValue) { + Number num = (Number) defaultValue; + return num.doubleValue(); + } +} \ No newline at end of file diff --git a/feathr-impl/src/main/java/com/linkedin/feathr/common/PegasusFeatureTypeResolver.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/PegasusFeatureTypeResolver.java new file mode 100644 index 000000000..76753bd53 --- /dev/null +++ b/feathr-impl/src/main/java/com/linkedin/feathr/common/PegasusFeatureTypeResolver.java @@ -0,0 +1,157 @@ +package com.linkedin.feathr.common; + +import com.google.common.annotations.VisibleForTesting; +import com.linkedin.feathr.compute.Dimension; +import com.linkedin.feathr.compute.FeatureVersion; +import com.linkedin.feathr.compute.TensorFeatureFormat; +import com.linkedin.feathr.common.tensor.DimensionType; +import com.linkedin.feathr.common.tensor.Primitive; +import com.linkedin.feathr.common.tensor.PrimitiveDimensionType; +import com.linkedin.feathr.common.types.PrimitiveType; +import com.linkedin.feathr.common.tensor.TensorCategory; +import com.linkedin.feathr.common.tensor.TensorType; +import com.linkedin.feathr.common.types.ValueType; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + + +/** + * This class maps from the pegasus models for feature types to Frame's common domain models for feature types and vice + * versa. + * + * This creates a layer of indirection from the feature definition models expressed in Pegasus to the domain models used + * by the frame's runtime engine (e.g. frame-online and frame-offline) + * + * @author bowu + */ +public class PegasusFeatureTypeResolver { + + private static final PegasusFeatureTypeResolver INSTANCE = new PegasusFeatureTypeResolver(); + + public static PegasusFeatureTypeResolver getInstance() { + return INSTANCE; + } + + private PegasusFeatureTypeResolver() { } + + /** + * Resolves the {@link FeatureTypeConfig} from the the pegasus {@link FeatureVersion} model. + * + * It's based on the following mapping rules: + * - if `type` is TENSOR without `format` field, it is a FML tensor type + * - if `type` is TENSOR with `format`, it is a Tensor feature type with FeatureTypeConfig in the feature definition + * - if `type` is non-TENSOR without `format`, it is a legacy type + * - if `type` is non-TENSOR with `format`, it is a legacy type with the format storing other info like embedding size + * that can be resolved using resolveEmbeddingSize(FeatureVersion) + */ + public FeatureTypeConfig resolveFeatureType(FeatureVersion featureVersion) { + FeatureTypes featureType = FeatureTypes.valueOf(featureVersion.getType().name()); + TensorType tensorType = null; + + // Even when featureType is not TENSOR, FeatureVersion still have format built + if (featureType == FeatureTypes.TENSOR && featureVersion.hasFormat()) { + tensorType = fromFeatureFormat(featureVersion.getFormat()); + // When the tensor format is present, then the frame feature type has to be TENSOR in case it is passed in + // as the default value of UNSPECIFIED + featureType = FeatureTypes.TENSOR; + } + + // NOTE: it is possible to resolve the TensorType for FML tensor based features (FeatureTypes == TENSOR) here it is + // purposely left out here to honor how {@link FeatureTypeConfig} should be handling FML tensor based features where + // tensorType = null + return tensorType != null ? new FeatureTypeConfig(featureType, tensorType, "No documentation") : new FeatureTypeConfig(featureType); + } + + /** + * Resolves the possible SWA embedding size from the pegasus {@link FeatureVersion} model. + * The embedding size is valid only when the feature is a possible embedding feature (1-d vector), which means + * the feature type can only be DENSE_VECTOR, or TENSOR, or UNSPECIFIED. Meanwhile, the input FeatureVersion + * should have valid format information: 1) the format filed exists and is not null, 2) the shape size is 1. + * + * The API is scheduled to be deprecated after dropping legacy feature type support in Frame, after which the + * embedding size information will always be inside the {@link FeatureTypeConfig} built from {@link #resolveFeatureType}. + * + * Warning: this should be only used when you know the feature is an embedding feature. + */ + @Deprecated + public Optional resolveEmbeddingSize(FeatureVersion featureVersion) { + FeatureTypes featureType = FeatureTypes.valueOf(featureVersion.getType().name()); + // embedding size is meaningful only when the feature is embedding feature + // embedding feature can only have type DENSE_VECTOR, or TENSOR, or UNSPECIFIED + if (featureType != FeatureTypes.UNSPECIFIED && featureType != FeatureTypes.DENSE_VECTOR && featureType != FeatureTypes.TENSOR) { + return Optional.empty(); + } + // if FeatureVersion does not have format field, then there is no valid embedding size information + if (!featureVersion.hasFormat()) { + return Optional.empty(); + } + + TensorType tensorType = fromFeatureFormat(featureVersion.getFormat()); + int[] shape = tensorType.getShape(); + // if the shape length is not 1, the tensor type is not an equivalence of embedding (1-d vector) + if (shape.length != 1) { + return Optional.empty(); + } + + return Optional.of(shape[0]); + } + + /** + * Maps the {@link TensorFeatureFormat} pegasus model to the {@link TensorType} in quince. + */ + private TensorType fromFeatureFormat(TensorFeatureFormat featureFormat) { + ValueType valType = fromValueTypeEnum(featureFormat.getValueType()); + TensorCategory tensorCategory = TensorCategory.valueOf(featureFormat.getTensorCategory().name()); + List dimensionTypes = + featureFormat.getDimensions().stream().map(this::fromDimension).collect(Collectors.toList()); + // NOTE: TensorFeatureFormat does not model the dimensionNames so using null to trigger the default handling which + // is to default to names taken from the dimensionTypes + return new TensorType(tensorCategory, valType, dimensionTypes, null); + } + + /** + * Maps the {@link Dimension} in the pegasus model to the {@link DimensionType} from quince + */ + @VisibleForTesting + DimensionType fromDimension(Dimension pegasusDimension) { + Integer shape = pegasusDimension.getShape(); + switch (pegasusDimension.getType()) { + case LONG: + return shape != null ? new PrimitiveDimensionType(Primitive.LONG, shape) : PrimitiveDimensionType.LONG; + case INT: + return shape != null ? new PrimitiveDimensionType(Primitive.INT, shape) : PrimitiveDimensionType.INT; + case STRING: + return shape != null ? new PrimitiveDimensionType(Primitive.STRING, shape) : PrimitiveDimensionType.STRING; + // TODO: seems that Boolean primitive dimension types are not modeled in FR + default: + throw new IllegalArgumentException( + "Unsupported dimension types from pegasus model: " + pegasusDimension.getType()); + } + } + + /** + * Maps the {@link com.linkedin.feathr.compute.ValueType} enum to the {@link ValueType} from quince + * + * Note: only primitives are supported at the moment + */ + @VisibleForTesting + ValueType fromValueTypeEnum(com.linkedin.feathr.compute.ValueType pegasusValType) { + switch (pegasusValType) { + case INT: + return PrimitiveType.INT; + case LONG: + return PrimitiveType.LONG; + case FLOAT: + return PrimitiveType.FLOAT; + case DOUBLE: + return PrimitiveType.DOUBLE; + case STRING: + return PrimitiveType.STRING; + case BOOLEAN: + return PrimitiveType.BOOLEAN; + default: + throw new IllegalArgumentException("Unsupported value type from the pegasus model: " + pegasusValType); + } + } +} \ No newline at end of file diff --git a/src/main/java/com/linkedin/feathr/common/TaggedFeatureName.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/TaggedFeatureName.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/TaggedFeatureName.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/TaggedFeatureName.java diff --git a/src/main/java/com/linkedin/feathr/common/TaggedFeatureUtils.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/TaggedFeatureUtils.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/TaggedFeatureUtils.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/TaggedFeatureUtils.java diff --git a/src/main/java/com/linkedin/feathr/common/TensorUtils.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/TensorUtils.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/TensorUtils.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/TensorUtils.java diff --git a/src/main/java/com/linkedin/feathr/common/TypedTensor.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/TypedTensor.java similarity index 92% rename from src/main/java/com/linkedin/feathr/common/TypedTensor.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/TypedTensor.java index fc4614397..3d498716a 100644 --- a/src/main/java/com/linkedin/feathr/common/TypedTensor.java +++ b/feathr-impl/src/main/java/com/linkedin/feathr/common/TypedTensor.java @@ -14,6 +14,8 @@ public interface TypedTensor { TypedTensor slice(Object val); + TypedTensor subSlice(Object val); + String toDebugString(); String toDebugString(int maxStringLenLimit); diff --git a/src/main/java/com/linkedin/feathr/common/configObj/ConfigObj.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/configObj/ConfigObj.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/configObj/ConfigObj.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/configObj/ConfigObj.java diff --git a/src/main/java/com/linkedin/feathr/common/configObj/DateTimeConfig.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/configObj/DateTimeConfig.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/configObj/DateTimeConfig.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/configObj/DateTimeConfig.java diff --git a/src/main/java/com/linkedin/feathr/common/configObj/configbuilder/ConfigBuilderException.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/configObj/configbuilder/ConfigBuilderException.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/configObj/configbuilder/ConfigBuilderException.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/configObj/configbuilder/ConfigBuilderException.java diff --git a/src/main/java/com/linkedin/feathr/common/configObj/configbuilder/ConfigUtils.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/configObj/configbuilder/ConfigUtils.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/configObj/configbuilder/ConfigUtils.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/configObj/configbuilder/ConfigUtils.java diff --git a/src/main/java/com/linkedin/feathr/common/configObj/configbuilder/DateTimeConfigBuilder.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/configObj/configbuilder/DateTimeConfigBuilder.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/configObj/configbuilder/DateTimeConfigBuilder.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/configObj/configbuilder/DateTimeConfigBuilder.java diff --git a/src/main/java/com/linkedin/feathr/common/configObj/configbuilder/FeatureGenConfigBuilder.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/configObj/configbuilder/FeatureGenConfigBuilder.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/configObj/configbuilder/FeatureGenConfigBuilder.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/configObj/configbuilder/FeatureGenConfigBuilder.java diff --git a/src/main/java/com/linkedin/feathr/common/configObj/configbuilder/OperationalConfigBuilder.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/configObj/configbuilder/OperationalConfigBuilder.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/configObj/configbuilder/OperationalConfigBuilder.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/configObj/configbuilder/OperationalConfigBuilder.java diff --git a/src/main/java/com/linkedin/feathr/common/configObj/configbuilder/OutputProcessorBuilder.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/configObj/configbuilder/OutputProcessorBuilder.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/configObj/configbuilder/OutputProcessorBuilder.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/configObj/configbuilder/OutputProcessorBuilder.java diff --git a/src/main/java/com/linkedin/feathr/common/configObj/generation/FeatureGenConfig.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/configObj/generation/FeatureGenConfig.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/configObj/generation/FeatureGenConfig.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/configObj/generation/FeatureGenConfig.java diff --git a/src/main/java/com/linkedin/feathr/common/configObj/generation/OfflineOperationalConfig.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/configObj/generation/OfflineOperationalConfig.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/configObj/generation/OfflineOperationalConfig.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/configObj/generation/OfflineOperationalConfig.java diff --git a/src/main/java/com/linkedin/feathr/common/configObj/generation/OperationalConfig.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/configObj/generation/OperationalConfig.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/configObj/generation/OperationalConfig.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/configObj/generation/OperationalConfig.java diff --git a/src/main/java/com/linkedin/feathr/common/configObj/generation/OutputProcessorConfig.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/configObj/generation/OutputProcessorConfig.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/configObj/generation/OutputProcessorConfig.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/configObj/generation/OutputProcessorConfig.java diff --git a/src/main/java/com/linkedin/feathr/common/exception/ErrorLabel.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/exception/ErrorLabel.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/exception/ErrorLabel.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/exception/ErrorLabel.java diff --git a/src/main/java/com/linkedin/feathr/common/exception/FeathrConfigException.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/exception/FeathrConfigException.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/exception/FeathrConfigException.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/exception/FeathrConfigException.java diff --git a/src/main/java/com/linkedin/feathr/common/exception/FeathrDataOutputException.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/exception/FeathrDataOutputException.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/exception/FeathrDataOutputException.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/exception/FeathrDataOutputException.java diff --git a/src/main/java/com/linkedin/feathr/common/exception/FeathrException.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/exception/FeathrException.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/exception/FeathrException.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/exception/FeathrException.java diff --git a/src/main/java/com/linkedin/feathr/common/exception/FeathrFeatureJoinException.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/exception/FeathrFeatureJoinException.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/exception/FeathrFeatureJoinException.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/exception/FeathrFeatureJoinException.java diff --git a/src/main/java/com/linkedin/feathr/common/exception/FeathrFeatureTransformationException.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/exception/FeathrFeatureTransformationException.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/exception/FeathrFeatureTransformationException.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/exception/FeathrFeatureTransformationException.java diff --git a/src/main/java/com/linkedin/feathr/common/exception/FeathrInputDataException.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/exception/FeathrInputDataException.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/exception/FeathrInputDataException.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/exception/FeathrInputDataException.java diff --git a/src/main/java/com/linkedin/feathr/common/featurizeddataset/BaseDenseTensorIterator.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/featurizeddataset/BaseDenseTensorIterator.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/featurizeddataset/BaseDenseTensorIterator.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/featurizeddataset/BaseDenseTensorIterator.java diff --git a/src/main/java/com/linkedin/feathr/common/featurizeddataset/DenseTensorList.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/featurizeddataset/DenseTensorList.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/featurizeddataset/DenseTensorList.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/featurizeddataset/DenseTensorList.java diff --git a/src/main/java/com/linkedin/feathr/common/featurizeddataset/FDSDenseTensorWrapper.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/featurizeddataset/FDSDenseTensorWrapper.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/featurizeddataset/FDSDenseTensorWrapper.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/featurizeddataset/FDSDenseTensorWrapper.java diff --git a/src/main/java/com/linkedin/feathr/common/featurizeddataset/FDSSparseTensorWrapper.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/featurizeddataset/FDSSparseTensorWrapper.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/featurizeddataset/FDSSparseTensorWrapper.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/featurizeddataset/FDSSparseTensorWrapper.java diff --git a/src/main/java/com/linkedin/feathr/common/featurizeddataset/FeatureDeserializer.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/featurizeddataset/FeatureDeserializer.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/featurizeddataset/FeatureDeserializer.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/featurizeddataset/FeatureDeserializer.java diff --git a/src/main/java/com/linkedin/feathr/common/featurizeddataset/InternalFeaturizedDatasetMetadataUtils.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/featurizeddataset/InternalFeaturizedDatasetMetadataUtils.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/featurizeddataset/InternalFeaturizedDatasetMetadataUtils.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/featurizeddataset/InternalFeaturizedDatasetMetadataUtils.java diff --git a/src/main/java/com/linkedin/feathr/common/featurizeddataset/SchemaMetadataUtils.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/featurizeddataset/SchemaMetadataUtils.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/featurizeddataset/SchemaMetadataUtils.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/featurizeddataset/SchemaMetadataUtils.java diff --git a/src/main/java/com/linkedin/feathr/common/featurizeddataset/SparkDeserializerFactory.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/featurizeddataset/SparkDeserializerFactory.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/featurizeddataset/SparkDeserializerFactory.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/featurizeddataset/SparkDeserializerFactory.java diff --git a/src/main/java/com/linkedin/feathr/common/time/TimeUnit.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/time/TimeUnit.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/time/TimeUnit.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/time/TimeUnit.java diff --git a/src/main/java/com/linkedin/feathr/common/types/BooleanFeatureType.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/types/BooleanFeatureType.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/types/BooleanFeatureType.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/types/BooleanFeatureType.java diff --git a/src/main/java/com/linkedin/feathr/common/types/CategoricalFeatureType.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/types/CategoricalFeatureType.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/types/CategoricalFeatureType.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/types/CategoricalFeatureType.java diff --git a/src/main/java/com/linkedin/feathr/common/types/CategoricalSetFeatureType.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/types/CategoricalSetFeatureType.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/types/CategoricalSetFeatureType.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/types/CategoricalSetFeatureType.java diff --git a/src/main/java/com/linkedin/feathr/common/types/DenseVectorFeatureType.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/types/DenseVectorFeatureType.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/types/DenseVectorFeatureType.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/types/DenseVectorFeatureType.java diff --git a/src/main/java/com/linkedin/feathr/common/types/FeatureType.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/types/FeatureType.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/types/FeatureType.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/types/FeatureType.java diff --git a/src/main/java/com/linkedin/feathr/common/types/NumericFeatureType.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/types/NumericFeatureType.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/types/NumericFeatureType.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/types/NumericFeatureType.java diff --git a/src/main/java/com/linkedin/feathr/common/types/PrimitiveType.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/types/PrimitiveType.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/types/PrimitiveType.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/types/PrimitiveType.java diff --git a/src/main/java/com/linkedin/feathr/common/types/TensorFeatureType.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/types/TensorFeatureType.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/types/TensorFeatureType.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/types/TensorFeatureType.java diff --git a/src/main/java/com/linkedin/feathr/common/types/TermVectorFeatureType.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/types/TermVectorFeatureType.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/types/TermVectorFeatureType.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/types/TermVectorFeatureType.java diff --git a/src/main/java/com/linkedin/feathr/common/types/ValueType.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/types/ValueType.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/types/ValueType.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/types/ValueType.java diff --git a/src/main/java/com/linkedin/feathr/common/types/protobuf/FeatureValueOuterClass.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/types/protobuf/FeatureValueOuterClass.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/types/protobuf/FeatureValueOuterClass.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/types/protobuf/FeatureValueOuterClass.java diff --git a/src/main/java/com/linkedin/feathr/common/util/CoercionUtils.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/util/CoercionUtils.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/util/CoercionUtils.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/util/CoercionUtils.java diff --git a/src/main/java/com/linkedin/feathr/common/util/MvelContextUDFs.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/util/MvelContextUDFs.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/util/MvelContextUDFs.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/util/MvelContextUDFs.java diff --git a/src/main/java/com/linkedin/feathr/common/value/AbstractFeatureFormatMapper.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/value/AbstractFeatureFormatMapper.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/value/AbstractFeatureFormatMapper.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/value/AbstractFeatureFormatMapper.java diff --git a/src/main/java/com/linkedin/feathr/common/value/BooleanFeatureValue.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/value/BooleanFeatureValue.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/value/BooleanFeatureValue.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/value/BooleanFeatureValue.java diff --git a/src/main/java/com/linkedin/feathr/common/value/CategoricalFeatureValue.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/value/CategoricalFeatureValue.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/value/CategoricalFeatureValue.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/value/CategoricalFeatureValue.java diff --git a/src/main/java/com/linkedin/feathr/common/value/CategoricalSetFeatureValue.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/value/CategoricalSetFeatureValue.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/value/CategoricalSetFeatureValue.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/value/CategoricalSetFeatureValue.java diff --git a/src/main/java/com/linkedin/feathr/common/value/DenseVectorFeatureValue.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/value/DenseVectorFeatureValue.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/value/DenseVectorFeatureValue.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/value/DenseVectorFeatureValue.java diff --git a/src/main/java/com/linkedin/feathr/common/value/FeatureFormatMapper.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/value/FeatureFormatMapper.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/value/FeatureFormatMapper.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/value/FeatureFormatMapper.java diff --git a/src/main/java/com/linkedin/feathr/common/value/FeatureValue.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/value/FeatureValue.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/value/FeatureValue.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/value/FeatureValue.java diff --git a/src/main/java/com/linkedin/feathr/common/value/FeatureValues.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/value/FeatureValues.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/value/FeatureValues.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/value/FeatureValues.java diff --git a/src/main/java/com/linkedin/feathr/common/value/NTVFeatureFormatMapper.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/value/NTVFeatureFormatMapper.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/value/NTVFeatureFormatMapper.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/value/NTVFeatureFormatMapper.java diff --git a/src/main/java/com/linkedin/feathr/common/value/NumericFeatureValue.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/value/NumericFeatureValue.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/value/NumericFeatureValue.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/value/NumericFeatureValue.java diff --git a/src/main/java/com/linkedin/feathr/common/value/QuinceFeatureFormatMapper.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/value/QuinceFeatureFormatMapper.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/value/QuinceFeatureFormatMapper.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/value/QuinceFeatureFormatMapper.java diff --git a/src/main/java/com/linkedin/feathr/common/value/QuinceFeatureTypeMapper.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/value/QuinceFeatureTypeMapper.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/value/QuinceFeatureTypeMapper.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/value/QuinceFeatureTypeMapper.java diff --git a/src/main/java/com/linkedin/feathr/common/value/TensorFeatureValue.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/value/TensorFeatureValue.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/value/TensorFeatureValue.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/value/TensorFeatureValue.java diff --git a/src/main/java/com/linkedin/feathr/common/value/TermVectorFeatureValue.java b/feathr-impl/src/main/java/com/linkedin/feathr/common/value/TermVectorFeatureValue.java similarity index 100% rename from src/main/java/com/linkedin/feathr/common/value/TermVectorFeatureValue.java rename to feathr-impl/src/main/java/com/linkedin/feathr/common/value/TermVectorFeatureValue.java diff --git a/src/main/protobuf/featureValue.proto b/feathr-impl/src/main/protobuf/featureValue.proto similarity index 100% rename from src/main/protobuf/featureValue.proto rename to feathr-impl/src/main/protobuf/featureValue.proto diff --git a/src/main/scala/com/databricks/spark/avro/SchemaConverterUtils.scala b/feathr-impl/src/main/scala/com/databricks/spark/avro/SchemaConverterUtils.scala similarity index 100% rename from src/main/scala/com/databricks/spark/avro/SchemaConverterUtils.scala rename to feathr-impl/src/main/scala/com/databricks/spark/avro/SchemaConverterUtils.scala diff --git a/src/main/scala/com/databricks/spark/avro/SchemaConverters.scala b/feathr-impl/src/main/scala/com/databricks/spark/avro/SchemaConverters.scala similarity index 100% rename from src/main/scala/com/databricks/spark/avro/SchemaConverters.scala rename to feathr-impl/src/main/scala/com/databricks/spark/avro/SchemaConverters.scala diff --git a/src/main/scala/com/linkedin/feathr/common/AnchorExtractor.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/common/AnchorExtractor.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/common/AnchorExtractor.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/common/AnchorExtractor.scala diff --git a/src/main/scala/com/linkedin/feathr/common/AnchorExtractorBase.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/common/AnchorExtractorBase.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/common/AnchorExtractorBase.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/common/AnchorExtractorBase.scala diff --git a/src/main/scala/com/linkedin/feathr/common/CanConvertToAvroRDD.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/common/CanConvertToAvroRDD.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/common/CanConvertToAvroRDD.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/common/CanConvertToAvroRDD.scala diff --git a/src/main/scala/com/linkedin/feathr/common/ColumnUtils.java b/feathr-impl/src/main/scala/com/linkedin/feathr/common/ColumnUtils.java similarity index 100% rename from src/main/scala/com/linkedin/feathr/common/ColumnUtils.java rename to feathr-impl/src/main/scala/com/linkedin/feathr/common/ColumnUtils.java diff --git a/src/main/scala/com/linkedin/feathr/common/DateTimeUtils.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/common/DateTimeUtils.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/common/DateTimeUtils.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/common/DateTimeUtils.scala diff --git a/src/main/scala/com/linkedin/feathr/common/FeatureDerivationFunction.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/common/FeatureDerivationFunction.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/common/FeatureDerivationFunction.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/common/FeatureDerivationFunction.scala diff --git a/src/main/scala/com/linkedin/feathr/common/FeatureDerivationFunctionBase.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/common/FeatureDerivationFunctionBase.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/common/FeatureDerivationFunctionBase.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/common/FeatureDerivationFunctionBase.scala diff --git a/src/main/scala/com/linkedin/feathr/common/FeatureRef.java b/feathr-impl/src/main/scala/com/linkedin/feathr/common/FeatureRef.java similarity index 100% rename from src/main/scala/com/linkedin/feathr/common/FeatureRef.java rename to feathr-impl/src/main/scala/com/linkedin/feathr/common/FeatureRef.java diff --git a/src/main/scala/com/linkedin/feathr/common/FrameJacksonScalaModule.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/common/FrameJacksonScalaModule.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/common/FrameJacksonScalaModule.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/common/FrameJacksonScalaModule.scala diff --git a/src/main/scala/com/linkedin/feathr/common/Params.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/common/Params.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/common/Params.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/common/Params.scala diff --git a/src/main/scala/com/linkedin/feathr/common/SparkRowExtractor.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/common/SparkRowExtractor.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/common/SparkRowExtractor.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/common/SparkRowExtractor.scala diff --git a/src/main/scala/com/linkedin/feathr/common/Types.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/common/Types.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/common/Types.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/common/Types.scala diff --git a/feathr-impl/src/main/scala/com/linkedin/feathr/common/common.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/common/common.scala new file mode 100644 index 000000000..8fcd5c232 --- /dev/null +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/common/common.scala @@ -0,0 +1,89 @@ +package com.linkedin.feathr + +import com.typesafe.config.Config +import scala.collection.JavaConverters._ + +/** + * parameter map(config) utility class, help user to get parameter value with a default value, + * example usage: + * + * import com.linkedin.feathr.common.RichConfig._ + * val batchValue = _params.map(_.getBooleanWithDefault(batchPath, true)).get + * + */ +package object common { + + val SELECTED_FEATURES = "selectedFeatures" + implicit class RichConfig(val config: Config) { + /* + get a parameter at 'path' with default value + */ + def getStringWithDefault(path: String, default: String): String = if (config.hasPath(path)) { + config.getString(path) + } else { + default + } + + /* + get a parameter at 'path' with default value + */ + def getBooleanWithDefault(path: String, default: Boolean): Boolean = if (config.hasPath(path)) { + config.getBoolean(path) + } else { + default + } + + /* + get a parameter at 'path' with default value + */ + def getIntWithDefault(path: String, default: Int): Int = if (config.hasPath(path)) { + config.getInt(path) + } else { + default + } + + /* + get a parameter at 'path' with default value + */ + def getDoubleWithDefault(path: String, default: Double): Double = if (config.hasPath(path)) { + config.getDouble(path) + } else { + default + } + /* + get a parameter at 'path' with default value + */ + def getMapWithDefault(path: String, default: Map[String, Object]): Map[String, Object] = if (config.hasPath(path)) { + config.getObject(path).unwrapped().asScala.toMap + } else { + default + } + + /* + get a parameter with optional string list + */ + def getStringListOpt(path: String): Option[Seq[String]] = if (config.hasPath(path)) { + Some(config.getStringList(path).asScala.toSeq) + } else { + None + } + + /* + get a parameter with optional string + */ + def getStringOpt(path: String): Option[String] = if (config.hasPath(path)) { + Some(config.getString(path)) + } else { + None + } + + /* + get a parameter with optional number + */ + def getNumberOpt(path: String): Option[Number] = if (config.hasPath(path)) { + Some(config.getNumber(path)) + } else { + None + } + } +} diff --git a/src/main/scala/com/linkedin/feathr/common/tensor/DenseTensor.java b/feathr-impl/src/main/scala/com/linkedin/feathr/common/tensor/DenseTensor.java similarity index 100% rename from src/main/scala/com/linkedin/feathr/common/tensor/DenseTensor.java rename to feathr-impl/src/main/scala/com/linkedin/feathr/common/tensor/DenseTensor.java diff --git a/src/main/scala/com/linkedin/feathr/common/tensor/DimensionType.java b/feathr-impl/src/main/scala/com/linkedin/feathr/common/tensor/DimensionType.java similarity index 70% rename from src/main/scala/com/linkedin/feathr/common/tensor/DimensionType.java rename to feathr-impl/src/main/scala/com/linkedin/feathr/common/tensor/DimensionType.java index 1af41f9f1..19f1eda1d 100644 --- a/src/main/scala/com/linkedin/feathr/common/tensor/DimensionType.java +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/common/tensor/DimensionType.java @@ -63,4 +63,34 @@ public int getShape() { public String getName() { return DUMMY_NAME; } + + + /** + * Convert a numeric index to a string representation. + * @param index the numeric index. 0 is reserved for out-of-vocab. + * @return the string representation + * @deprecated Use {@link #getDimensionValue(ReadableTuple, int)} instead + */ + @Deprecated + // LONG_TERM_TECH_DEBT_ALERT + public String indexToString(long index) { + // Default implementation, to be overridden by subclasses. + return Long.toString(index); + } + + /** + * Convert a string representation to a numeric index. + * @param string the string representation + * @return the numeric index. Categoricals return 0 if out-of-vocab, others will throw unchecked exceptions. + * @deprecated Use {@link #setDimensionValue(WriteableTuple, int, Object)} instead + */ + @Deprecated + // LONG_TERM_TECH_DEBT_ALERT + public long stringToIndex(String string) { + long index = Long.parseLong(string); + if (index < 0) { + throw new IllegalArgumentException(string + " must be >= 0."); + } + return index; + } } diff --git a/src/main/scala/com/linkedin/feathr/common/tensor/LOLTensorData.java b/feathr-impl/src/main/scala/com/linkedin/feathr/common/tensor/LOLTensorData.java similarity index 100% rename from src/main/scala/com/linkedin/feathr/common/tensor/LOLTensorData.java rename to feathr-impl/src/main/scala/com/linkedin/feathr/common/tensor/LOLTensorData.java diff --git a/src/main/scala/com/linkedin/feathr/common/tensor/Primitive.java b/feathr-impl/src/main/scala/com/linkedin/feathr/common/tensor/Primitive.java similarity index 100% rename from src/main/scala/com/linkedin/feathr/common/tensor/Primitive.java rename to feathr-impl/src/main/scala/com/linkedin/feathr/common/tensor/Primitive.java diff --git a/src/main/scala/com/linkedin/feathr/common/tensor/PrimitiveDimensionType.java b/feathr-impl/src/main/scala/com/linkedin/feathr/common/tensor/PrimitiveDimensionType.java similarity index 100% rename from src/main/scala/com/linkedin/feathr/common/tensor/PrimitiveDimensionType.java rename to feathr-impl/src/main/scala/com/linkedin/feathr/common/tensor/PrimitiveDimensionType.java diff --git a/src/main/scala/com/linkedin/feathr/common/tensor/ReadableTuple.java b/feathr-impl/src/main/scala/com/linkedin/feathr/common/tensor/ReadableTuple.java similarity index 100% rename from src/main/scala/com/linkedin/feathr/common/tensor/ReadableTuple.java rename to feathr-impl/src/main/scala/com/linkedin/feathr/common/tensor/ReadableTuple.java diff --git a/src/main/scala/com/linkedin/feathr/common/tensor/Representable.java b/feathr-impl/src/main/scala/com/linkedin/feathr/common/tensor/Representable.java similarity index 100% rename from src/main/scala/com/linkedin/feathr/common/tensor/Representable.java rename to feathr-impl/src/main/scala/com/linkedin/feathr/common/tensor/Representable.java diff --git a/src/main/scala/com/linkedin/feathr/common/tensor/SimpleWriteableTuple.java b/feathr-impl/src/main/scala/com/linkedin/feathr/common/tensor/SimpleWriteableTuple.java similarity index 100% rename from src/main/scala/com/linkedin/feathr/common/tensor/SimpleWriteableTuple.java rename to feathr-impl/src/main/scala/com/linkedin/feathr/common/tensor/SimpleWriteableTuple.java diff --git a/src/main/scala/com/linkedin/feathr/common/tensor/StandaloneReadableTuple.java b/feathr-impl/src/main/scala/com/linkedin/feathr/common/tensor/StandaloneReadableTuple.java similarity index 100% rename from src/main/scala/com/linkedin/feathr/common/tensor/StandaloneReadableTuple.java rename to feathr-impl/src/main/scala/com/linkedin/feathr/common/tensor/StandaloneReadableTuple.java diff --git a/src/main/scala/com/linkedin/feathr/common/tensor/TensorCategory.java b/feathr-impl/src/main/scala/com/linkedin/feathr/common/tensor/TensorCategory.java similarity index 100% rename from src/main/scala/com/linkedin/feathr/common/tensor/TensorCategory.java rename to feathr-impl/src/main/scala/com/linkedin/feathr/common/tensor/TensorCategory.java diff --git a/src/main/scala/com/linkedin/feathr/common/tensor/TensorData.java b/feathr-impl/src/main/scala/com/linkedin/feathr/common/tensor/TensorData.java similarity index 100% rename from src/main/scala/com/linkedin/feathr/common/tensor/TensorData.java rename to feathr-impl/src/main/scala/com/linkedin/feathr/common/tensor/TensorData.java diff --git a/src/main/scala/com/linkedin/feathr/common/tensor/TensorIterator.java b/feathr-impl/src/main/scala/com/linkedin/feathr/common/tensor/TensorIterator.java similarity index 100% rename from src/main/scala/com/linkedin/feathr/common/tensor/TensorIterator.java rename to feathr-impl/src/main/scala/com/linkedin/feathr/common/tensor/TensorIterator.java diff --git a/src/main/scala/com/linkedin/feathr/common/tensor/TensorType.java b/feathr-impl/src/main/scala/com/linkedin/feathr/common/tensor/TensorType.java similarity index 100% rename from src/main/scala/com/linkedin/feathr/common/tensor/TensorType.java rename to feathr-impl/src/main/scala/com/linkedin/feathr/common/tensor/TensorType.java diff --git a/src/main/scala/com/linkedin/feathr/common/tensor/TensorTypes.java b/feathr-impl/src/main/scala/com/linkedin/feathr/common/tensor/TensorTypes.java similarity index 100% rename from src/main/scala/com/linkedin/feathr/common/tensor/TensorTypes.java rename to feathr-impl/src/main/scala/com/linkedin/feathr/common/tensor/TensorTypes.java diff --git a/src/main/scala/com/linkedin/feathr/common/tensor/Tensors.java b/feathr-impl/src/main/scala/com/linkedin/feathr/common/tensor/Tensors.java similarity index 100% rename from src/main/scala/com/linkedin/feathr/common/tensor/Tensors.java rename to feathr-impl/src/main/scala/com/linkedin/feathr/common/tensor/Tensors.java diff --git a/src/main/scala/com/linkedin/feathr/common/tensor/WriteableTuple.java b/feathr-impl/src/main/scala/com/linkedin/feathr/common/tensor/WriteableTuple.java similarity index 100% rename from src/main/scala/com/linkedin/feathr/common/tensor/WriteableTuple.java rename to feathr-impl/src/main/scala/com/linkedin/feathr/common/tensor/WriteableTuple.java diff --git a/src/main/scala/com/linkedin/feathr/common/tensor/dense/ByteBufferDenseTensor.java b/feathr-impl/src/main/scala/com/linkedin/feathr/common/tensor/dense/ByteBufferDenseTensor.java similarity index 100% rename from src/main/scala/com/linkedin/feathr/common/tensor/dense/ByteBufferDenseTensor.java rename to feathr-impl/src/main/scala/com/linkedin/feathr/common/tensor/dense/ByteBufferDenseTensor.java diff --git a/src/main/scala/com/linkedin/feathr/common/tensor/dense/DenseBooleanTensor.java b/feathr-impl/src/main/scala/com/linkedin/feathr/common/tensor/dense/DenseBooleanTensor.java similarity index 100% rename from src/main/scala/com/linkedin/feathr/common/tensor/dense/DenseBooleanTensor.java rename to feathr-impl/src/main/scala/com/linkedin/feathr/common/tensor/dense/DenseBooleanTensor.java diff --git a/src/main/scala/com/linkedin/feathr/common/tensor/dense/DenseBytesTensor.java b/feathr-impl/src/main/scala/com/linkedin/feathr/common/tensor/dense/DenseBytesTensor.java similarity index 100% rename from src/main/scala/com/linkedin/feathr/common/tensor/dense/DenseBytesTensor.java rename to feathr-impl/src/main/scala/com/linkedin/feathr/common/tensor/dense/DenseBytesTensor.java diff --git a/src/main/scala/com/linkedin/feathr/common/tensor/dense/DenseDoubleTensor.java b/feathr-impl/src/main/scala/com/linkedin/feathr/common/tensor/dense/DenseDoubleTensor.java similarity index 100% rename from src/main/scala/com/linkedin/feathr/common/tensor/dense/DenseDoubleTensor.java rename to feathr-impl/src/main/scala/com/linkedin/feathr/common/tensor/dense/DenseDoubleTensor.java diff --git a/src/main/scala/com/linkedin/feathr/common/tensor/dense/DenseFloatTensor.java b/feathr-impl/src/main/scala/com/linkedin/feathr/common/tensor/dense/DenseFloatTensor.java similarity index 100% rename from src/main/scala/com/linkedin/feathr/common/tensor/dense/DenseFloatTensor.java rename to feathr-impl/src/main/scala/com/linkedin/feathr/common/tensor/dense/DenseFloatTensor.java diff --git a/src/main/scala/com/linkedin/feathr/common/tensor/dense/DenseIntTensor.java b/feathr-impl/src/main/scala/com/linkedin/feathr/common/tensor/dense/DenseIntTensor.java similarity index 100% rename from src/main/scala/com/linkedin/feathr/common/tensor/dense/DenseIntTensor.java rename to feathr-impl/src/main/scala/com/linkedin/feathr/common/tensor/dense/DenseIntTensor.java diff --git a/src/main/scala/com/linkedin/feathr/common/tensor/dense/DenseLongTensor.java b/feathr-impl/src/main/scala/com/linkedin/feathr/common/tensor/dense/DenseLongTensor.java similarity index 100% rename from src/main/scala/com/linkedin/feathr/common/tensor/dense/DenseLongTensor.java rename to feathr-impl/src/main/scala/com/linkedin/feathr/common/tensor/dense/DenseLongTensor.java diff --git a/src/main/scala/com/linkedin/feathr/common/tensor/dense/DenseStringTensor.java b/feathr-impl/src/main/scala/com/linkedin/feathr/common/tensor/dense/DenseStringTensor.java similarity index 100% rename from src/main/scala/com/linkedin/feathr/common/tensor/dense/DenseStringTensor.java rename to feathr-impl/src/main/scala/com/linkedin/feathr/common/tensor/dense/DenseStringTensor.java diff --git a/src/main/scala/com/linkedin/feathr/common/tensor/scalar/ScalarBooleanTensor.java b/feathr-impl/src/main/scala/com/linkedin/feathr/common/tensor/scalar/ScalarBooleanTensor.java similarity index 100% rename from src/main/scala/com/linkedin/feathr/common/tensor/scalar/ScalarBooleanTensor.java rename to feathr-impl/src/main/scala/com/linkedin/feathr/common/tensor/scalar/ScalarBooleanTensor.java diff --git a/src/main/scala/com/linkedin/feathr/common/tensor/scalar/ScalarBytesTensor.java b/feathr-impl/src/main/scala/com/linkedin/feathr/common/tensor/scalar/ScalarBytesTensor.java similarity index 100% rename from src/main/scala/com/linkedin/feathr/common/tensor/scalar/ScalarBytesTensor.java rename to feathr-impl/src/main/scala/com/linkedin/feathr/common/tensor/scalar/ScalarBytesTensor.java diff --git a/src/main/scala/com/linkedin/feathr/common/tensor/scalar/ScalarDoubleTensor.java b/feathr-impl/src/main/scala/com/linkedin/feathr/common/tensor/scalar/ScalarDoubleTensor.java similarity index 100% rename from src/main/scala/com/linkedin/feathr/common/tensor/scalar/ScalarDoubleTensor.java rename to feathr-impl/src/main/scala/com/linkedin/feathr/common/tensor/scalar/ScalarDoubleTensor.java diff --git a/src/main/scala/com/linkedin/feathr/common/tensor/scalar/ScalarFloatTensor.java b/feathr-impl/src/main/scala/com/linkedin/feathr/common/tensor/scalar/ScalarFloatTensor.java similarity index 100% rename from src/main/scala/com/linkedin/feathr/common/tensor/scalar/ScalarFloatTensor.java rename to feathr-impl/src/main/scala/com/linkedin/feathr/common/tensor/scalar/ScalarFloatTensor.java diff --git a/src/main/scala/com/linkedin/feathr/common/tensor/scalar/ScalarIntTensor.java b/feathr-impl/src/main/scala/com/linkedin/feathr/common/tensor/scalar/ScalarIntTensor.java similarity index 100% rename from src/main/scala/com/linkedin/feathr/common/tensor/scalar/ScalarIntTensor.java rename to feathr-impl/src/main/scala/com/linkedin/feathr/common/tensor/scalar/ScalarIntTensor.java diff --git a/src/main/scala/com/linkedin/feathr/common/tensor/scalar/ScalarLongTensor.java b/feathr-impl/src/main/scala/com/linkedin/feathr/common/tensor/scalar/ScalarLongTensor.java similarity index 100% rename from src/main/scala/com/linkedin/feathr/common/tensor/scalar/ScalarLongTensor.java rename to feathr-impl/src/main/scala/com/linkedin/feathr/common/tensor/scalar/ScalarLongTensor.java diff --git a/src/main/scala/com/linkedin/feathr/common/tensor/scalar/ScalarStringTensor.java b/feathr-impl/src/main/scala/com/linkedin/feathr/common/tensor/scalar/ScalarStringTensor.java similarity index 100% rename from src/main/scala/com/linkedin/feathr/common/tensor/scalar/ScalarStringTensor.java rename to feathr-impl/src/main/scala/com/linkedin/feathr/common/tensor/scalar/ScalarStringTensor.java diff --git a/src/main/scala/com/linkedin/feathr/common/tensor/scalar/ScalarTensor.java b/feathr-impl/src/main/scala/com/linkedin/feathr/common/tensor/scalar/ScalarTensor.java similarity index 100% rename from src/main/scala/com/linkedin/feathr/common/tensor/scalar/ScalarTensor.java rename to feathr-impl/src/main/scala/com/linkedin/feathr/common/tensor/scalar/ScalarTensor.java diff --git a/src/main/scala/com/linkedin/feathr/common/tensorbuilder/BufferUtils.java b/feathr-impl/src/main/scala/com/linkedin/feathr/common/tensorbuilder/BufferUtils.java similarity index 100% rename from src/main/scala/com/linkedin/feathr/common/tensorbuilder/BufferUtils.java rename to feathr-impl/src/main/scala/com/linkedin/feathr/common/tensorbuilder/BufferUtils.java diff --git a/src/main/scala/com/linkedin/feathr/common/tensorbuilder/BulkTensorBuilder.java b/feathr-impl/src/main/scala/com/linkedin/feathr/common/tensorbuilder/BulkTensorBuilder.java similarity index 100% rename from src/main/scala/com/linkedin/feathr/common/tensorbuilder/BulkTensorBuilder.java rename to feathr-impl/src/main/scala/com/linkedin/feathr/common/tensorbuilder/BulkTensorBuilder.java diff --git a/src/main/scala/com/linkedin/feathr/common/tensorbuilder/DenseTensorBuilder.java b/feathr-impl/src/main/scala/com/linkedin/feathr/common/tensorbuilder/DenseTensorBuilder.java similarity index 100% rename from src/main/scala/com/linkedin/feathr/common/tensorbuilder/DenseTensorBuilder.java rename to feathr-impl/src/main/scala/com/linkedin/feathr/common/tensorbuilder/DenseTensorBuilder.java diff --git a/src/main/scala/com/linkedin/feathr/common/tensorbuilder/DenseTensorBuilderFactory.java b/feathr-impl/src/main/scala/com/linkedin/feathr/common/tensorbuilder/DenseTensorBuilderFactory.java similarity index 100% rename from src/main/scala/com/linkedin/feathr/common/tensorbuilder/DenseTensorBuilderFactory.java rename to feathr-impl/src/main/scala/com/linkedin/feathr/common/tensorbuilder/DenseTensorBuilderFactory.java diff --git a/src/main/scala/com/linkedin/feathr/common/tensorbuilder/SortUtils.java b/feathr-impl/src/main/scala/com/linkedin/feathr/common/tensorbuilder/SortUtils.java similarity index 100% rename from src/main/scala/com/linkedin/feathr/common/tensorbuilder/SortUtils.java rename to feathr-impl/src/main/scala/com/linkedin/feathr/common/tensorbuilder/SortUtils.java diff --git a/src/main/scala/com/linkedin/feathr/common/tensorbuilder/TensorBuilder.java b/feathr-impl/src/main/scala/com/linkedin/feathr/common/tensorbuilder/TensorBuilder.java similarity index 100% rename from src/main/scala/com/linkedin/feathr/common/tensorbuilder/TensorBuilder.java rename to feathr-impl/src/main/scala/com/linkedin/feathr/common/tensorbuilder/TensorBuilder.java diff --git a/src/main/scala/com/linkedin/feathr/common/tensorbuilder/TensorBuilderFactory.java b/feathr-impl/src/main/scala/com/linkedin/feathr/common/tensorbuilder/TensorBuilderFactory.java similarity index 100% rename from src/main/scala/com/linkedin/feathr/common/tensorbuilder/TensorBuilderFactory.java rename to feathr-impl/src/main/scala/com/linkedin/feathr/common/tensorbuilder/TensorBuilderFactory.java diff --git a/src/main/scala/com/linkedin/feathr/common/tensorbuilder/TypedOperator.java b/feathr-impl/src/main/scala/com/linkedin/feathr/common/tensorbuilder/TypedOperator.java similarity index 100% rename from src/main/scala/com/linkedin/feathr/common/tensorbuilder/TypedOperator.java rename to feathr-impl/src/main/scala/com/linkedin/feathr/common/tensorbuilder/TypedOperator.java diff --git a/src/main/scala/com/linkedin/feathr/common/tensorbuilder/UniversalTensor.java b/feathr-impl/src/main/scala/com/linkedin/feathr/common/tensorbuilder/UniversalTensor.java similarity index 100% rename from src/main/scala/com/linkedin/feathr/common/tensorbuilder/UniversalTensor.java rename to feathr-impl/src/main/scala/com/linkedin/feathr/common/tensorbuilder/UniversalTensor.java diff --git a/src/main/scala/com/linkedin/feathr/common/tensorbuilder/UniversalTensorBuilder.java b/feathr-impl/src/main/scala/com/linkedin/feathr/common/tensorbuilder/UniversalTensorBuilder.java similarity index 100% rename from src/main/scala/com/linkedin/feathr/common/tensorbuilder/UniversalTensorBuilder.java rename to feathr-impl/src/main/scala/com/linkedin/feathr/common/tensorbuilder/UniversalTensorBuilder.java diff --git a/src/main/scala/com/linkedin/feathr/common/tensorbuilder/UniversalTensorBuilderFactory.java b/feathr-impl/src/main/scala/com/linkedin/feathr/common/tensorbuilder/UniversalTensorBuilderFactory.java similarity index 100% rename from src/main/scala/com/linkedin/feathr/common/tensorbuilder/UniversalTensorBuilderFactory.java rename to feathr-impl/src/main/scala/com/linkedin/feathr/common/tensorbuilder/UniversalTensorBuilderFactory.java diff --git a/src/main/scala/com/linkedin/feathr/offline/ErasedEntityTaggedFeature.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/ErasedEntityTaggedFeature.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/ErasedEntityTaggedFeature.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/ErasedEntityTaggedFeature.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/FeatureDataFrame.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/FeatureDataFrame.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/FeatureDataFrame.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/FeatureDataFrame.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/FeatureValue.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/FeatureValue.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/FeatureValue.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/FeatureValue.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/PostTransformationUtil.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/PostTransformationUtil.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/PostTransformationUtil.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/PostTransformationUtil.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/anchored/WindowTimeUnit.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/anchored/WindowTimeUnit.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/anchored/WindowTimeUnit.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/anchored/WindowTimeUnit.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/anchored/anchorExtractor/DebugMvelAnchorExtractor.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/anchored/anchorExtractor/DebugMvelAnchorExtractor.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/anchored/anchorExtractor/DebugMvelAnchorExtractor.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/anchored/anchorExtractor/DebugMvelAnchorExtractor.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/anchored/anchorExtractor/SQLConfigurableAnchorExtractor.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/anchored/anchorExtractor/SQLConfigurableAnchorExtractor.scala similarity index 98% rename from src/main/scala/com/linkedin/feathr/offline/anchored/anchorExtractor/SQLConfigurableAnchorExtractor.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/anchored/anchorExtractor/SQLConfigurableAnchorExtractor.scala index f80593116..e17319f76 100644 --- a/src/main/scala/com/linkedin/feathr/offline/anchored/anchorExtractor/SQLConfigurableAnchorExtractor.scala +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/anchored/anchorExtractor/SQLConfigurableAnchorExtractor.scala @@ -6,7 +6,7 @@ import com.linkedin.feathr.offline.config.SQLFeatureDefinition import com.linkedin.feathr.offline.transformation.FeatureColumnFormat.{FeatureColumnFormat, RAW} import com.linkedin.feathr.sparkcommon.SimpleAnchorExtractorSpark import org.apache.log4j.Logger -import org.apache.spark.sql.functions._ +import org.apache.spark.sql.functions.expr import org.apache.spark.sql.types._ import org.apache.spark.sql.{Column, DataFrame} diff --git a/src/main/scala/com/linkedin/feathr/offline/anchored/anchorExtractor/SimpleConfigurableAnchorExtractor.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/anchored/anchorExtractor/SimpleConfigurableAnchorExtractor.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/anchored/anchorExtractor/SimpleConfigurableAnchorExtractor.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/anchored/anchorExtractor/SimpleConfigurableAnchorExtractor.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/anchored/anchorExtractor/TimeWindowConfigurableAnchorExtractor.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/anchored/anchorExtractor/TimeWindowConfigurableAnchorExtractor.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/anchored/anchorExtractor/TimeWindowConfigurableAnchorExtractor.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/anchored/anchorExtractor/TimeWindowConfigurableAnchorExtractor.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/anchored/feature/FeatureAnchor.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/anchored/feature/FeatureAnchor.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/anchored/feature/FeatureAnchor.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/anchored/feature/FeatureAnchor.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/anchored/feature/FeatureAnchorWithSource.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/anchored/feature/FeatureAnchorWithSource.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/anchored/feature/FeatureAnchorWithSource.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/anchored/feature/FeatureAnchorWithSource.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/anchored/keyExtractor/MVELSourceKeyExtractor.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/anchored/keyExtractor/MVELSourceKeyExtractor.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/anchored/keyExtractor/MVELSourceKeyExtractor.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/anchored/keyExtractor/MVELSourceKeyExtractor.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/anchored/keyExtractor/SQLSourceKeyExtractor.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/anchored/keyExtractor/SQLSourceKeyExtractor.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/anchored/keyExtractor/SQLSourceKeyExtractor.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/anchored/keyExtractor/SQLSourceKeyExtractor.scala diff --git a/feathr-impl/src/main/scala/com/linkedin/feathr/offline/anchored/keyExtractor/SpecificRecordSourceKeyExtractor.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/anchored/keyExtractor/SpecificRecordSourceKeyExtractor.scala new file mode 100644 index 000000000..c89a5236a --- /dev/null +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/anchored/keyExtractor/SpecificRecordSourceKeyExtractor.scala @@ -0,0 +1,54 @@ +package com.linkedin.feathr.offline.anchored.keyExtractor + +import com.linkedin.feathr.common.AnchorExtractor +import com.linkedin.feathr.exception.{ErrorLabel, FeathrException} +import com.linkedin.feathr.sparkcommon.SourceKeyExtractor +import com.typesafe.config.ConfigRenderOptions +import org.apache.spark.sql._ + +/** + * This is the source key extractor class for user defined AnchorExtractor class + * @param anchorExtractorV1 + */ +private[feathr] class SpecificRecordSourceKeyExtractor( + anchorExtractorV1: AnchorExtractor[Any], + private val keyExprs: Seq[String] = Seq(), + private val keyAlias: Option[Seq[String]] = None) + extends SourceKeyExtractor { + val JOIN_KEY_PREFIX = anchorExtractorV1.toString.replaceAll("[^\\w]", "") + "_" + val MAX_KEY_FIELD_NUM = 5 + + override def appendKeyColumns(dataFrame: DataFrame): DataFrame = { + throw new FeathrException(ErrorLabel.FEATHR_ERROR, "appendKeyColumns function is not supported SpecificRecordSourceKeyExtractor") + } + + def getKey(datum: Any): Seq[String] = { + anchorExtractorV1.getKey(datum) + } + + /** + * Return the key column name of the current source, since appendKeyColumns is not supported by this source key + * extractor (will special handle it), we just return place holders. + * when the rdd is empty, pass None as datum, then this function + * will return empty Seq to signal empty dataframe + * + * @param datum + * @return + */ + override def getKeyColumnNames(datum: Option[Any]): Seq[String] = { + if (datum.isDefined) { + val size = anchorExtractorV1.getKey(datum.get).size + (1 to size).map(JOIN_KEY_PREFIX + _) + } else { + Seq() + } + } + + override def getKeyColumnAlias(datum: Option[Any]): Seq[String] = { + keyAlias.getOrElse(keyExprs) + } + + override def toString(): String = + super.toString() + anchorExtractorV1.getClass.getCanonicalName + + " withParams:" + params.map(_.root().render(ConfigRenderOptions.concise()).mkString(",")) +} diff --git a/src/main/scala/com/linkedin/feathr/offline/client/DataFrameColName.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/client/DataFrameColName.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/client/DataFrameColName.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/client/DataFrameColName.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/client/FeathrClient.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/client/FeathrClient.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/client/FeathrClient.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/client/FeathrClient.scala diff --git a/feathr-impl/src/main/scala/com/linkedin/feathr/offline/client/FeathrClient2.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/client/FeathrClient2.scala new file mode 100644 index 000000000..33c59228c --- /dev/null +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/client/FeathrClient2.scala @@ -0,0 +1,262 @@ +package com.linkedin.feathr.offline.client + +import com.linkedin.feathr.common.{FeatureTypeConfig, JoiningFeatureParams, TaggedFeatureName} +import com.linkedin.feathr.compute._ +import com.linkedin.feathr.compute.converter.FeatureDefinitionsConverter +import com.linkedin.feathr.config.FeatureDefinitionLoaderFactory +import com.linkedin.feathr.config.join.FrameFeatureJoinConfig +import com.linkedin.feathr.core.configdataprovider.{ResourceConfigDataProvider, StringConfigDataProvider} +import com.linkedin.feathr.exception.{ErrorLabel, FeathrConfigException} +import com.linkedin.feathr.offline.FeatureDataFrame +import com.linkedin.feathr.offline.config.join.converters.PegasusRecordFrameFeatureJoinConfigConverter +import com.linkedin.feathr.offline.config.{FeathrConfig, FeatureJoinConfig} +import com.linkedin.feathr.offline.exception.DataFrameApiUnsupportedOperationException +import com.linkedin.feathr.offline.graph.NodeUtils.getFeatureTypeConfigsMap +import com.linkedin.feathr.offline.graph.{FCMGraphTraverser, NodeUtils} +import com.linkedin.feathr.offline.job.{FeatureGenSpec, JoinJobContext} +import com.linkedin.feathr.offline.mvel.plugins.FeathrExpressionExecutionContext +import com.linkedin.feathr.offline.source.accessor.DataPathHandler +import com.linkedin.feathr.offline.util.FCMUtils.makeFeatureNameForDuplicates +import com.linkedin.feathr.offline.util.{AnchorUtils, FeaturizedDatasetUtils, SparkFeaturizedDataset} +import org.apache.log4j.Logger +import org.apache.spark.sql.SparkSession + +import scala.collection.JavaConverters._ +import scala.collection.mutable + +sealed trait VisitedState +case object NOT_VISITED extends VisitedState +case object IN_PROGRESS extends VisitedState +case object VISITED extends VisitedState + +/** + * FrameClient2 is the new entry point into Feathr for joining observation data with features. To achieve this, instantiate this class + * via the FrameClient2 builder which will take your feature config files and prepare a FrameClient2 instance which can join observation + * data with a join config via the joinFeatures API. + * + * The FrameClient takes in a [[ComputeGraph]] object, which can be created from the featureDefConf files using the [[FeatureDefinitionsConverter]] + * class. + */ +class FeathrClient2(ss: SparkSession, computeGraph: ComputeGraph, dataPathHandlers: List[DataPathHandler], mvelContext: Option[FeathrExpressionExecutionContext]) { + private val log = Logger.getLogger(getClass.getName) + + def joinFeatures(frameJoinConfig: FrameFeatureJoinConfig, obsData: SparkFeaturizedDataset, jobContext: JoinJobContext): + (FeatureDataFrame, Map[String, FeatureTypeConfig], Seq[String]) = { + val joinConfig = PegasusRecordFrameFeatureJoinConfigConverter.convert(frameJoinConfig) + joinFeatures(joinConfig, obsData, jobContext) + } + + private def findInvalidFeatureRefs(features: Seq[String]): List[String] = { + features.foldLeft(List.empty[String]) { (acc, f) => + // featureRefStr could have '-' now. + // TODO - 8037) unify featureRef/featureName and check for '-' + val featureRefStrInDF = DataFrameColName.getEncodedFeatureRefStrForColName(f) + val isValidSyntax = AnchorUtils.featureNamePattern.matcher(featureRefStrInDF).matches() + if (isValidSyntax) acc + else f :: acc + } + } + + /** + * Validate feature names in compute graph. Two things are checked here: + * 1. Feature names conform to regular expression as defined in feathr specs + * 2. Feature names don't conflict with any field names in the observation data + * TODO: Add ACL validation for all data sources + * TODO: Move validation to core library as this is shared among all environments. + * @param obsFieldNames Field names in observation data feathr + */ + private def validateFeatureNames(obsFieldNames: Array[String])= { + val allFeaturesInGraph = computeGraph.getFeatureNames.asScala.keys.toSeq + val invalidFeatureNames = findInvalidFeatureRefs(allFeaturesInGraph) + if (invalidFeatureNames.nonEmpty) { + throw new DataFrameApiUnsupportedOperationException( + "Feature names must conform to " + + s"regular expression: ${AnchorUtils.featureNamePattern}, but found feature names: $invalidFeatureNames") + } + val conflictFeatureNames: Seq[String] = allFeaturesInGraph.intersect(obsFieldNames) + if (conflictFeatureNames.nonEmpty) { + throw new FeathrConfigException( + ErrorLabel.FEATHR_USER_ERROR, + "Feature names must be different from field names in the observation data. " + + s"Please rename feature ${conflictFeatureNames} or rename the same field names in the observation data.") + } + } + + /** + * Joins observation data on the feature data. Observation data is loaded as SparkFeaturizedDataset, and the + * joined data is returned as a SparkFeaturizedDataset. + * @param joinConfig HOCON based join config + * @param obsData Observation data in the form of SparkFeaturizedDataset + * @param jobContext [[JoinJobContext]] + * @return Feature data join with observation data in the form of SparkFeaturizedDataset + */ + def joinFeatures(joinConfig: FeatureJoinConfig, obsData: SparkFeaturizedDataset, jobContext: JoinJobContext = JoinJobContext()): + (FeatureDataFrame, Map[String, FeatureTypeConfig], Seq[String]) = { + // Set up spark conf parameters needed. This call is crucial otherwise scala UDFs will cause errors when running in spark. + prepareExecuteEnv() + + val featureNames = joinConfig.joinFeatures.map(_.featureName) + val duplicateFeatureNames = featureNames.diff(featureNames.distinct).distinct + val joinFeatures = NodeUtils.getFeatureRequestsFromJoinConfig(joinConfig).asJava + + // Check for invalid feature names + validateFeatureNames(obsData.data.schema.fieldNames) + + // Create resolved graph using the joinFeatures + val resolvedGraph = new Resolver(computeGraph).resolveForRequest(joinFeatures) + + // Execute the resolved graph + val graphTraverser = new FCMGraphTraverser(ss, joinConfig, resolvedGraph, obsData.data, dataPathHandlers, mvelContext) + val newDf = graphTraverser.traverseGraph() + + val passthroughFeaturesList = resolvedGraph.getNodes.asScala.filter(node => node.getTransformation != null + && node.getTransformation.getFunction.getOperator().contains("passthrough")).map(node => node.getTransformation.getFeatureName) + + val userProvidedFeatureTypeConfigs = getFeatureTypeConfigsMap(resolvedGraph.getNodes.asScala) + (newDf, userProvidedFeatureTypeConfigs, passthroughFeaturesList) + } + + private def prepareExecuteEnv() = { + ss.conf.set("spark.sql.legacy.allowUntypedScalaUDF", "true") + ss.conf.set("spark.sql.unionToStructConversion.avro.useNativeSchema", "true") + } + + def generateFeatures(featureGenSpec: FeatureGenSpec): Map[TaggedFeatureName, SparkFeaturizedDataset] = { + throw new UnsupportedOperationException() + } +} + +object FeathrClient2 { + + /** + * Create an instance of a builder for constructing a FrameClient2 + * @param sparkSession the SparkSession required for the FrameClient2 to perform its operations + * @return Builder class + */ + def builder(sparkSession: SparkSession): Builder = { + new Builder(sparkSession) + } + + class Builder(ss: SparkSession) { + private val featureDefinitionLoader = FeatureDefinitionLoaderFactory.getInstance() + + private var featureDef: List[String] = List() + private var localOverrideDef: List[String] = List() + private var featureDefPath: List[String] = List() + private var localOverrideDefPath: List[String] = List() + private var dataPathHandlers: List[DataPathHandler] = List() + private var mvelContext: Option[FeathrExpressionExecutionContext] = None; + + def addFeatureDef(featureDef: String): Builder = { + this.featureDef = featureDef :: this.featureDef + this + } + + def addFeatureDef(featureDef: Option[String]): Builder = { + if (featureDef.isDefined) addFeatureDef(featureDef.get) else this + } + + def addLocalOverrideDef(localOverrideDef: String): Builder = { + this.localOverrideDef = localOverrideDef :: this.localOverrideDef + this + } + + def addLocalOverrideDef(localOverrideDef: Option[String]): Builder = { + if (localOverrideDef.isDefined) addFeatureDef(localOverrideDef.get) else this + } + + def addFeatureDefPath(featureDefPath: String): Builder = { + this.featureDefPath = featureDefPath :: this.featureDefPath + this + } + + def addFeatureDefPath(featureDefPath: Option[String]): Builder = { + if (featureDefPath.isDefined) addFeatureDefPath(featureDefPath.get) else this + } + + def addLocalOverrideDefPath(localOverrideDefPath: String): Builder = { + this.localOverrideDefPath = localOverrideDefPath :: this.localOverrideDefPath + this + } + + def addLocalOverrideDefPath(localOverrideDefPath: Option[String]): Builder = { + if (localOverrideDefPath.isDefined) addLocalOverrideDefPath(localOverrideDefPath.get) else this + } + + private[offline] def addFeatureDefConfs(featureDefConfs: Option[List[FeathrConfig]]): Builder = { + // Unlike FrameClient, we can't support this right now, since we only can convert to ComputeGraph from FR definitions + // and NOT from "FrameConfig" (at least for now – but this seems rarely used so probably not worth it.) + throw new UnsupportedOperationException() + } + + private[offline] def addFeatureDefConfs(featureDefConfs: List[FeathrConfig]): Builder = { + // Unlike FrameClient, we can't support this right now, since we only can convert to ComputeGraph from FR definitions + // and NOT from "FrameConfig" (at least for now – but this seems rarely used so probably not worth it.) + throw new UnsupportedOperationException() + } + + /** + * Add a list of data path handlers to the builder. Used to handle accessing and loading paths caught by user's udf, validatePath + * + * @param dataPathHandlers custom data path handlers + * @return FeathrClient.Builder + */ + def addDataPathHandlers(dataPathHandlers: List[DataPathHandler]): Builder = { + this.dataPathHandlers = dataPathHandlers ++ this.dataPathHandlers + this + } + + /** + * Add a data path handler to the builder. Used to handle accessing and loading paths caught by user's udf, validatePath + * + * @param dataPathHandler custom data path handler + * @return FeathrClient.Builder + */ + def addDataPathHandler(dataPathHandler: DataPathHandler): Builder = { + this.dataPathHandlers = dataPathHandler :: this.dataPathHandlers + this + } + def addFeathrExpressionContext(_mvelContext: Option[FeathrExpressionExecutionContext]): Builder = { + this.mvelContext = _mvelContext + this + } + + /** + * Same as {@code addDataPathHandler(DataPathHandler)} but the input dataPathHandlers is optional and when it is missing, + * this method performs an no-op. + * + * @param dataPathHandler custom data path handler + * @return FeathrClient.Builder + */ + def addDataPathHandler(dataPathHandler: Option[DataPathHandler]): Builder = { + if (dataPathHandler.isDefined) addDataPathHandler(dataPathHandler.get) else this + } + + /** + * Build a new instance of the FrameClient2 from the added feathr definition configs and any local overrides. + * + * @throws [[IllegalArgumentException]] an error when no feature definitions nor local overrides are configured. + */ + def build(): FeathrClient2 = { + import scala.collection.JavaConverters._ + + require( + localOverrideDefPath.nonEmpty || localOverrideDef.nonEmpty || featureDefPath.nonEmpty || featureDef.nonEmpty, + "Cannot build frameClient without a feature def conf file/string or local override def conf file/string") + + // Append all the configs to this empty list, with the local override def config going last + val configDocsInOrder = featureDef ::: featureDefPath.flatMap(x => readHdfsFile(Some(x))) ::: + localOverrideDef ::: localOverrideDefPath.flatMap(x => readHdfsFile(Some(x))) + + val partialComputeGraphs = configDocsInOrder.map(new StringConfigDataProvider(_)).map (config => + new FeatureDefinitionsConverter().convert(FeatureDefinitionLoaderFactory.getInstance.loadAllFeatureDefinitions(config))) + val graph = ComputeGraphs.removeRedundancies(ComputeGraphs.merge(partialComputeGraphs.asJava)) + + new FeathrClient2(ss, graph, dataPathHandlers, mvelContext) + } + + private def readHdfsFile(path: Option[String]): Option[String] = + path.map(p => ss.sparkContext.textFile(p).collect.mkString("\n")) + } +} +// scalastyle:on \ No newline at end of file diff --git a/src/main/scala/com/linkedin/feathr/offline/client/InputData.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/client/InputData.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/client/InputData.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/client/InputData.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/client/TypedRef.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/client/TypedRef.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/client/TypedRef.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/client/TypedRef.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/client/plugins/FeathrUdfPluginContext.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/client/plugins/FeathrUdfPluginContext.scala similarity index 99% rename from src/main/scala/com/linkedin/feathr/offline/client/plugins/FeathrUdfPluginContext.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/client/plugins/FeathrUdfPluginContext.scala index d67e5b6d5..cd0b0705f 100644 --- a/src/main/scala/com/linkedin/feathr/offline/client/plugins/FeathrUdfPluginContext.scala +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/client/plugins/FeathrUdfPluginContext.scala @@ -1,4 +1,5 @@ package com.linkedin.feathr.offline.client.plugins + import org.apache.spark.SparkContext import org.apache.spark.broadcast.Broadcast diff --git a/src/main/scala/com/linkedin/feathr/offline/client/plugins/UdfAdaptor.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/client/plugins/UdfAdaptor.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/client/plugins/UdfAdaptor.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/client/plugins/UdfAdaptor.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/config/ConfigLoaderUtils.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/ConfigLoaderUtils.scala similarity index 96% rename from src/main/scala/com/linkedin/feathr/offline/config/ConfigLoaderUtils.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/ConfigLoaderUtils.scala index ae2ff83b0..4dad3a5c1 100644 --- a/src/main/scala/com/linkedin/feathr/offline/config/ConfigLoaderUtils.scala +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/ConfigLoaderUtils.scala @@ -46,7 +46,7 @@ private[offline] object ConfigLoaderUtils { /** * Convert Java List[String] to Scala Seq[String], and make a deep copy to avoid any not-serializable exception */ - private[config] def javaListToSeqWithDeepCopy(inputList: JavaList[String]): Seq[String] = { + private[feathr] def javaListToSeqWithDeepCopy(inputList: JavaList[String]): Seq[String] = { Seq(inputList.asScala: _*) } } diff --git a/src/main/scala/com/linkedin/feathr/offline/config/DerivedFeatureConfig.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/DerivedFeatureConfig.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/config/DerivedFeatureConfig.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/DerivedFeatureConfig.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/config/FeathrConfigLoader.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/FeathrConfigLoader.scala similarity index 98% rename from src/main/scala/com/linkedin/feathr/offline/config/FeathrConfigLoader.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/FeathrConfigLoader.scala index e2ec6e588..1e18d5e4a 100644 --- a/src/main/scala/com/linkedin/feathr/offline/config/FeathrConfigLoader.scala +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/FeathrConfigLoader.scala @@ -14,7 +14,7 @@ import com.linkedin.feathr.offline.anchored.anchorExtractor.{SQLConfigurableAnch import com.linkedin.feathr.offline.anchored.feature.{FeatureAnchor, FeatureAnchorWithSource} import com.linkedin.feathr.offline.anchored.keyExtractor.{MVELSourceKeyExtractor, SQLSourceKeyExtractor} import com.linkedin.feathr.offline.client.plugins.{AnchorExtractorAdaptor, FeathrUdfPluginContext, FeatureDerivationFunctionAdaptor, SimpleAnchorExtractorSparkAdaptor, SourceKeyExtractorAdaptor} -import com.linkedin.feathr.offline.config.location.{DataLocation, KafkaEndpoint, LocationUtils, SimplePath} +import com.linkedin.feathr.offline.config.location.{DataLocation, KafkaEndpoint, LocationUtils, SimplePath, Snowflake} import com.linkedin.feathr.offline.derived._ import com.linkedin.feathr.offline.derived.functions.{MvelFeatureDerivationFunction, SQLFeatureDerivationFunction, SeqJoinDerivationFunction, SimpleMvelDerivationFunction} import com.linkedin.feathr.offline.source.{DataSource, SourceFormatType, TimeWindowParams} @@ -711,7 +711,6 @@ private[offline] class DataSourceLoader extends JsonDeserializer[DataSource] { override def deserialize(p: JsonParser, ctxt: DeserializationContext): DataSource = { val codec = p.getCodec val node = codec.readTree[TreeNode](p).asInstanceOf[ObjectNode] - // for now only HDFS can be set, in the future, here may allow more options // also to form a unified interface with online val dataSourceType = Option(node.get("type")) match { @@ -719,7 +718,7 @@ private[offline] class DataSourceLoader extends JsonDeserializer[DataSource] { case _ => "HDFS" } - if (dataSourceType != "HDFS" && dataSourceType != "PASSTHROUGH" && dataSourceType != "KAFKA") { + if (dataSourceType != "HDFS" && dataSourceType != "PASSTHROUGH" && dataSourceType != "KAFKA" && dataSourceType != "SNOWFLAKE") { throw new FeathrConfigException(ErrorLabel.FEATHR_USER_ERROR, s"Unknown source type parameter $dataSourceType is used") } @@ -733,7 +732,6 @@ private[offline] class DataSourceLoader extends JsonDeserializer[DataSource] { } else { SourceFormatType.FIXED_PATH } - /* * path here can be: * @@ -752,6 +750,15 @@ private[offline] class DataSourceLoader extends JsonDeserializer[DataSource] { s"Illegal setting for Kafka source ${node.toPrettyString()}, expected map") } case "PASSTHROUGH" => SimplePath("PASSTHROUGH") + case "SNOWFLAKE" => + Option(node.get("location")) match { + case Some(field: ObjectNode) => + LocationUtils.getMapper().treeToValue(field, classOf[Snowflake]) + case None => throw new FeathrConfigException(ErrorLabel.FEATHR_USER_ERROR, + s"Snowflake config is not defined for Snowflake source ${node.toPrettyString()}") + case _ => throw new FeathrConfigException(ErrorLabel.FEATHR_USER_ERROR, + s"Illegal setting for Snowflake source ${node.toPrettyString()}, expected map") + } case _ => Option(node.get("location")) match { case Some(field: ObjectNode) => LocationUtils.getMapper().treeToValue(field, classOf[DataLocation]) @@ -792,7 +799,6 @@ private[offline] class DataSourceLoader extends JsonDeserializer[DataSource] { } case None => null } - if (path.isInstanceOf[KafkaEndpoint]) { DataSource(path, sourceFormatType) } else { diff --git a/src/main/scala/com/linkedin/feathr/offline/config/FeatureDefinition.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/FeatureDefinition.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/config/FeatureDefinition.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/FeatureDefinition.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/config/FeatureGroupsGenerator.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/FeatureGroupsGenerator.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/config/FeatureGroupsGenerator.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/FeatureGroupsGenerator.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/config/FeatureJoinConfig.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/FeatureJoinConfig.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/config/FeatureJoinConfig.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/FeatureJoinConfig.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/config/FeatureJoinConfigDeserializer.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/FeatureJoinConfigDeserializer.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/config/FeatureJoinConfigDeserializer.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/FeatureJoinConfigDeserializer.scala diff --git a/feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/PegasusRecordDefaultValueConverter.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/PegasusRecordDefaultValueConverter.scala new file mode 100644 index 000000000..2733ac4f2 --- /dev/null +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/PegasusRecordDefaultValueConverter.scala @@ -0,0 +1,29 @@ +package com.linkedin.feathr.offline.config + +import com.linkedin.feathr.common.{FeatureValue, PegasusDefaultFeatureValueResolver} +import com.linkedin.feathr.compute.FeatureVersion + +private[offline] class PegasusRecordDefaultValueConverter private ( + pegasusDefaultFeatureValueResolver: PegasusDefaultFeatureValueResolver) { + + private val _pegasusDefaultFeatureValueResolver = pegasusDefaultFeatureValueResolver + + /** + * Convert feathr-Core FeatureTypeConfig to Offline [[FeatureTypeConfig]] + */ + def convert(features: Map[String, FeatureVersion]): Map[String, FeatureValue] = { + features + .transform((k, v) => _pegasusDefaultFeatureValueResolver.resolveDefaultValue(k, v)) + .filter(_._2.isPresent) + .mapValues(_.get) + // get rid of not serializable exception: + // https://stackoverflow.com/questions/32900862/map-can-not-be-serializable-in-scala/32945184 + .map(identity) + } +} + +private[offline] object PegasusRecordDefaultValueConverter { + def apply(): PegasusRecordDefaultValueConverter = { + new PegasusRecordDefaultValueConverter(PegasusDefaultFeatureValueResolver.getInstance) + } +} diff --git a/feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/PegasusRecordFeatureTypeConverter.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/PegasusRecordFeatureTypeConverter.scala new file mode 100644 index 000000000..53784400c --- /dev/null +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/PegasusRecordFeatureTypeConverter.scala @@ -0,0 +1,51 @@ +package com.linkedin.feathr.offline.config + +import com.linkedin.feathr.common.{FeatureTypeConfig, PegasusFeatureTypeResolver} +import com.linkedin.feathr.compute.FeatureVersion + +/** + * Class to convert [[FeatureTypeConfig]] from [[FeatureVersion]] + */ +private[offline] class PegasusRecordFeatureTypeConverter private (pegasusFeatureTypeResolver: PegasusFeatureTypeResolver) { + + private val _pegasusFeatureTypeResolver = pegasusFeatureTypeResolver + + /** + * Convert feathr-Core FeatureTypeConfig to Offline [[FeatureTypeConfig]] + */ + def convert(featureVersion: FeatureVersion): Option[FeatureTypeConfig] = { + // for now, convert CommonFeatureTypeConfig to CoreFeatureTypeConfig + // TODO after integ, remove CoreFeatureTypeConfig, and use CommonFeautreTypeConfig everywhere + if (featureVersion.hasType) { + val commonFeatureTypeConfig = _pegasusFeatureTypeResolver.resolveFeatureType(featureVersion) + val featureTypeConfig = new FeatureTypeConfig(commonFeatureTypeConfig.getFeatureType, commonFeatureTypeConfig.getTensorType, "No documentation") + Some(featureTypeConfig) + } else None + } + + /** + * Convert [[Option[FeatureTypeConfig]]] to a Map: + * 1. if [[FeatureTypeConfig]] exist, then create a singleton map from feature name to the [[FeatureTypeConfig]] object + * 2. otherwise return an empty Map + * @param featureNameRef feature name + * @param typeConfig Option of [[FeatureTypeConfig]] + * @return mapping from feature name to the [[FeatureTypeConfig]] object + */ + def parseFeatureTypeAsMap(featureNameRef: String, typeConfig: Option[FeatureTypeConfig]): Map[String, FeatureTypeConfig] = { + typeConfig match { + case Some(typeInfo) => Map(featureNameRef -> typeInfo) + case None => Map.empty + } + } +} + +private[offline] object PegasusRecordFeatureTypeConverter { + def apply(): PegasusRecordFeatureTypeConverter = { + new PegasusRecordFeatureTypeConverter(PegasusFeatureTypeResolver.getInstance) + } + + def apply(pegasusFeatureTypeResolver: PegasusFeatureTypeResolver): PegasusRecordFeatureTypeConverter = { + new PegasusRecordFeatureTypeConverter(pegasusFeatureTypeResolver) + } +} + diff --git a/src/main/scala/com/linkedin/feathr/offline/config/TimeWindowFeatureDefinition.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/TimeWindowFeatureDefinition.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/config/TimeWindowFeatureDefinition.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/TimeWindowFeatureDefinition.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/config/datasource/ADLSResourceInfoSetter.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/datasource/ADLSResourceInfoSetter.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/config/datasource/ADLSResourceInfoSetter.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/datasource/ADLSResourceInfoSetter.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/config/datasource/BlobResourceInfoSetter.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/datasource/BlobResourceInfoSetter.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/config/datasource/BlobResourceInfoSetter.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/datasource/BlobResourceInfoSetter.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/config/datasource/DataSourceConfig.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/datasource/DataSourceConfig.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/config/datasource/DataSourceConfig.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/datasource/DataSourceConfig.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/config/datasource/DataSourceConfigUtils.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/datasource/DataSourceConfigUtils.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/config/datasource/DataSourceConfigUtils.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/datasource/DataSourceConfigUtils.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/config/datasource/DataSourceConfigs.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/datasource/DataSourceConfigs.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/config/datasource/DataSourceConfigs.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/datasource/DataSourceConfigs.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/config/datasource/KafkaResourceInfoSetter.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/datasource/KafkaResourceInfoSetter.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/config/datasource/KafkaResourceInfoSetter.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/datasource/KafkaResourceInfoSetter.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/config/datasource/MonitoringResourceInfoSetter.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/datasource/MonitoringResourceInfoSetter.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/config/datasource/MonitoringResourceInfoSetter.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/datasource/MonitoringResourceInfoSetter.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/config/datasource/RedisResourceInfoSetter.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/datasource/RedisResourceInfoSetter.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/config/datasource/RedisResourceInfoSetter.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/datasource/RedisResourceInfoSetter.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/config/datasource/Resource.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/datasource/Resource.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/config/datasource/Resource.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/datasource/Resource.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/config/datasource/ResourceInfoSetter.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/datasource/ResourceInfoSetter.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/config/datasource/ResourceInfoSetter.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/datasource/ResourceInfoSetter.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/config/datasource/S3ResourceInfoSetter.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/datasource/S3ResourceInfoSetter.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/config/datasource/S3ResourceInfoSetter.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/datasource/S3ResourceInfoSetter.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/config/datasource/SQLResourceInfoSetter.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/datasource/SQLResourceInfoSetter.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/config/datasource/SQLResourceInfoSetter.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/datasource/SQLResourceInfoSetter.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/config/datasource/SnowflakeResourceInfoSetter.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/datasource/SnowflakeResourceInfoSetter.scala similarity index 92% rename from src/main/scala/com/linkedin/feathr/offline/config/datasource/SnowflakeResourceInfoSetter.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/datasource/SnowflakeResourceInfoSetter.scala index 004470b43..3e02f3ed2 100644 --- a/src/main/scala/com/linkedin/feathr/offline/config/datasource/SnowflakeResourceInfoSetter.scala +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/datasource/SnowflakeResourceInfoSetter.scala @@ -10,6 +10,7 @@ private[feathr] class SnowflakeResourceInfoSetter extends ResourceInfoSetter() { ss.conf.set("sfURL", getAuthFromContext("JDBC_SF_URL", dataSourceConfig)) ss.conf.set("sfUser", getAuthFromContext("JDBC_SF_USER", dataSourceConfig)) ss.conf.set("sfRole", getAuthFromContext("JDBC_SF_ROLE", dataSourceConfig)) + ss.conf.set("sfWarehouse", getAuthFromContext("JDBC_SF_WAREHOUSE", dataSourceConfig)) ss.conf.set("sfPassword", getAuthFromContext("JDBC_SF_PASSWORD", dataSourceConfig)) }) } diff --git a/feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/join/converters/PegasusRecordDateTimeConverter.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/join/converters/PegasusRecordDateTimeConverter.scala new file mode 100644 index 000000000..9bbbcfee8 --- /dev/null +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/join/converters/PegasusRecordDateTimeConverter.scala @@ -0,0 +1,43 @@ +package com.linkedin.feathr.offline.config.join.converters + +import java.time.{LocalDate, LocalDateTime} +import java.time.format.DateTimeFormatter +import com.linkedin.feathr.config.join.{Date, HourTime, TimeUnit} +import com.linkedin.feathr.exception.{ErrorLabel, FeathrConfigException} + +private[converters] object PegasusRecordDateTimeConverter { + + /** + * convert PDL duration with a length and time unit to DateParam's string representation, e.g., 1d or 2h + */ + def convertDuration(length: Long, unit: TimeUnit): String = { + unit match { + case TimeUnit.DAY => s"${length}d" + case TimeUnit.HOUR => s"${length}h" + case TimeUnit.MINUTE => s"${length}m" + case TimeUnit.SECOND => s"${length}s" + case _ => + throw new FeathrConfigException(ErrorLabel.FEATHR_USER_ERROR, s"Invalid TimeUnit $unit. It should be DAY, HOUR, MINUTE or SECOND.") + } + } + + /** + * convert PDL [[Date]] object to string with the given format + * @param date the PDL date object + * @param format the date pattern described in [[DateTimeFormatter]], e.g., yyyyMMdd + * @return the date string, e,g. "20201113" + */ + def convertDate(date: Date, format: String): String = { + LocalDate.of(date.getYear, date.getMonth, date.getDay).format(DateTimeFormatter.ofPattern(format)) + } + + /** + * convert PDL [[HourTime]] object to string with the given format + * @param hourTime the PDL hourly time object + * @param format the date pattern described in [[DateTimeFormatter]], e.g, yyyyMMddHH + * @return the time string, e.g, 2020111310 + */ + def convertHourTime(hourTime: HourTime, format: String): String = { + LocalDateTime.of(hourTime.getYear, hourTime.getMonth, hourTime.getDay, hourTime.getHour, 0).format(DateTimeFormatter.ofPattern(format)) + } +} diff --git a/feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/join/converters/PegasusRecordFrameFeatureJoinConfigConverter.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/join/converters/PegasusRecordFrameFeatureJoinConfigConverter.scala new file mode 100644 index 000000000..bb7fa7955 --- /dev/null +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/join/converters/PegasusRecordFrameFeatureJoinConfigConverter.scala @@ -0,0 +1,68 @@ +package com.linkedin.feathr.offline.config.join.converters + +import com.linkedin.data.template.GetMode +import com.linkedin.feathr.config.join.{FrameFeatureJoinConfig, JoiningFeature, TimeUnit} +import com.linkedin.feathr.exception.{ErrorLabel, FeathrConfigException} +import com.linkedin.feathr.offline.config.{FeatureJoinConfig, KeyedFeatureList} +import com.linkedin.feathr.offline.util.datetime.OfflineDateTimeUtils + +import scala.collection.JavaConverters._ + +/** + * Convert PDL [[FrameFeatureJoinConfig]] to offline's [[FeatureJoinConfig]] + * @param pegasusRecordSettingsConverter the convert for the settings section of the join config + */ +private[offline] class PegasusRecordFrameFeatureJoinConfigConverter(private val pegasusRecordSettingsConverter: PegasusRecordSettingsConverter) { + val FEATURE_GROUP_NAME = "FeatureJoinConfigConverterGeneratedGroupName" + + /** + * Convert PDL [[FrameFeatureJoinConfig]] to offline's [[FeatureJoinConfig]] + */ + def convert(frameFeatureJoinConfig: FrameFeatureJoinConfig): FeatureJoinConfig = { + // convert the features + val joiningFeatures = frameFeatureJoinConfig.getFeatures.asScala + val features = joiningFeatures.map(convertFeature) + val groups = Map(FEATURE_GROUP_NAME -> features) + val settings = Option(frameFeatureJoinConfig.getSettings(GetMode.DEFAULT)).map(pegasusRecordSettingsConverter.convert) + FeatureJoinConfig(groups, settings) + } + + /** + * convert PDL [[JoiningFeature]] to offline's [[KeyedFeatureList]] + */ + private def convertFeature(feature: JoiningFeature): KeyedFeatureList = { + val keys = feature.getKeys.asScala + + var startDate: Option[String] = None + var endDate: Option[String] = None + var numDays: Option[String] = None + var dateOffset: Option[String] = None + if (feature.hasDateRange) { + val dateRange = feature.getDateRange + if (dateRange.isAbsoluteDateRange) { + val absoluteRange = dateRange.getAbsoluteDateRange + startDate = Some(PegasusRecordDateTimeConverter.convertDate(absoluteRange.getStartDate, OfflineDateTimeUtils.DEFAULT_TIME_FORMAT)) + endDate = Some(PegasusRecordDateTimeConverter.convertDate(absoluteRange.getEndDate, OfflineDateTimeUtils.DEFAULT_TIME_FORMAT)) + } else if (dateRange.isRelativeDateRange) { + val relativeRange = dateRange.getRelativeDateRange + numDays = Some(PegasusRecordDateTimeConverter.convertDuration(relativeRange.getNumDays, TimeUnit.DAY)) + dateOffset = Some(PegasusRecordDateTimeConverter.convertDuration(relativeRange.getDateOffset, TimeUnit.DAY)) + } else { + throw new FeathrConfigException( + ErrorLabel.FEATHR_USER_ERROR, + s"RelativeTimeRange and AbsoluteTimeRange are not set in DateRange $dateRange of feature $feature.") + } + } + + val featureAliasName = Option(feature.getFeatureAlias()) + + val overrideTimeDelay = + Option(feature.getOverrideTimeDelay(GetMode.DEFAULT)).map(delay => PegasusRecordDateTimeConverter.convertDuration(delay.getLength, delay.getUnit)) + KeyedFeatureList(keys, Seq(feature.getFrameFeatureName), startDate, endDate, dateOffset, numDays, overrideTimeDelay, featureAliasName) + } +} + +/** + * Default FrameFeatureJoinConfig converter with default settings converter. + */ +object PegasusRecordFrameFeatureJoinConfigConverter extends PegasusRecordFrameFeatureJoinConfigConverter(PegasusRecordSettingsConverter) diff --git a/feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/join/converters/PegasusRecordSettingsConverter.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/join/converters/PegasusRecordSettingsConverter.scala new file mode 100644 index 000000000..a31e18de1 --- /dev/null +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/join/converters/PegasusRecordSettingsConverter.scala @@ -0,0 +1,103 @@ +package com.linkedin.feathr.offline.config.join.converters + +import com.linkedin.data.template.GetMode +import com.linkedin.feathr.common.DateParam +import com.linkedin.feathr.config.join.{InputDataTimeSettings, JoinTimeSettings, Settings} +import com.linkedin.feathr.exception.{ErrorLabel, FeathrConfigException} +import com.linkedin.feathr.offline.anchored.WindowTimeUnit +import com.linkedin.feathr.offline.config.{JoinConfigSettings, JoinTimeSetting, ObservationDataTimeSetting, TimestampColumn} +import com.linkedin.feathr.offline.util.datetime.OfflineDateTimeUtils + +/** + * trait for converting PDL [[Settings]] of the [[FrameJoinConfig]] to offline's [[JoinConfigSettings]] + */ +private[converters] trait PegasusRecordSettingsConverter { + + /** + * Convert PDL [[Settings]] of the [[FrameJoinConfig]] to offline's [[JoinConfigSettings]] + */ + def convert(settings: Settings): JoinConfigSettings +} + +/** + * default implementation of PegasusRecordSettingsConverter. + */ +private[converters] object PegasusRecordSettingsConverter extends PegasusRecordSettingsConverter { + + /** + * Convert PDL [[Settings]] of the [[FrameJoinConfig]] to offline's [[JoinConfigSettings]] + */ + override def convert(settings: Settings): JoinConfigSettings = { + val inputDataTimeSettings = Option(settings.getInputDataTimeSettings(GetMode.DEFAULT)).map(convertInputDataTimeSettings) + val joinTimeSetting = Option(settings.getJoinTimeSettings(GetMode.DEFAULT)).map(convertJoinTimeSettings) + JoinConfigSettings(inputDataTimeSettings, joinTimeSetting) + } + + /** + * Convert PDL[[JoinTimeSettings]] to offline's [[JoinTimeSetting]] + */ + private def convertJoinTimeSettings(joinTimeSettings: JoinTimeSettings): JoinTimeSetting = { + if (joinTimeSettings.isTimestampColJoinTimeSettings) { + val settings = joinTimeSettings.getTimestampColJoinTimeSettings + val pdlTimestampColumn = settings.getTimestampColumn + val timestampColumnDefinition = if (pdlTimestampColumn.getDefinition.isColumnName) { + pdlTimestampColumn.getDefinition.getColumnName + } else { + pdlTimestampColumn.getDefinition.getSparkSqlExpression.getExpression + } + val timeStampColumn = TimestampColumn(timestampColumnDefinition, pdlTimestampColumn.getFormat) + val simulateTimeDelay = + Option(settings.getSimulateTimeDelay(GetMode.DEFAULT)).map(delay => + WindowTimeUnit.parseWindowTime(PegasusRecordDateTimeConverter.convertDuration(delay.getLength, delay.getUnit))) + JoinTimeSetting(timeStampColumn, simulateTimeDelay, useLatestFeatureData = false) + } else if (joinTimeSettings.isUseLatestJoinTimeSettings) { + val useLatestFeatureData = joinTimeSettings.getUseLatestJoinTimeSettings.isUseLatestFeatureData + JoinTimeSetting(TimestampColumn("", ""), None, useLatestFeatureData) + } else { + throw new FeathrConfigException( + ErrorLabel.FEATHR_USER_ERROR, + s"joinTimeSettings $joinTimeSettings should have either SettingsWithTimestampCol or SettingsWithUseLatestFeatureData.") + } + } + + /** + * Convert PDL[[ObservationDataTimeSettings]] to offline's [[ObservationDataTimeSetting]] + */ + private def convertInputDataTimeSettings(inputDataTimeSettings: InputDataTimeSettings): ObservationDataTimeSetting = { + val timeRange = inputDataTimeSettings.getTimeRange + if (timeRange.isAbsoluteTimeRange) { + val absoluteTimeRange = timeRange.getAbsoluteTimeRange + val startTime = absoluteTimeRange.getStartTime + val endTime = absoluteTimeRange.getEndTime + if (!((startTime.isDate && endTime.isDate) || (startTime.isHourTime && endTime.isHourTime))) { + throw new FeathrConfigException( + ErrorLabel.FEATHR_USER_ERROR, + s"AbsoluteTimeRange $absoluteTimeRange has different granularity for startTime and endTime. One is daily and the other is hourly.") + } + val formatString = if (startTime.isDate) OfflineDateTimeUtils.DEFAULT_TIME_FORMAT else OfflineDateTimeUtils.DEFAULT_HOURLY_TIME_FORMAT + val startTimeString = if (startTime.isDate) { + PegasusRecordDateTimeConverter.convertDate(startTime.getDate, formatString) + } else { + PegasusRecordDateTimeConverter.convertHourTime(startTime.getHourTime, formatString) + } + val endTimeString = if (endTime.isDate) { + PegasusRecordDateTimeConverter.convertDate(endTime.getDate, formatString) + } else { + PegasusRecordDateTimeConverter.convertHourTime(endTime.getHourTime, formatString) + } + val dateParam = DateParam(Some(startTimeString), Some(endTimeString)) + ObservationDataTimeSetting(dateParam, Some(formatString)) + } else if (timeRange.isRelativeTimeRange) { + val relativeTimeRange = timeRange.getRelativeTimeRange + val offset = PegasusRecordDateTimeConverter.convertDuration(relativeTimeRange.getOffset, relativeTimeRange.getWindow.getUnit) + val window = PegasusRecordDateTimeConverter.convertDuration(relativeTimeRange.getWindow.getLength, relativeTimeRange.getWindow.getUnit) + val dateParam = DateParam(None, None, Some(offset), Some(window)) + ObservationDataTimeSetting(dateParam, None) + } else { + throw new FeathrConfigException( + ErrorLabel.FEATHR_USER_ERROR, + s"RelativeTimeRange and AbsoluteTimeRange are not set in InputDataTimeSettings $inputDataTimeSettings. " + + "If intention is to not restrict the size of the input data, please remove the inputDataTimeSettings section completely.") + } + } +} diff --git a/src/main/scala/com/linkedin/feathr/offline/config/location/DataLocation.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/location/DataLocation.scala similarity index 98% rename from src/main/scala/com/linkedin/feathr/offline/config/location/DataLocation.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/location/DataLocation.scala index 37bece6b8..83bb093a3 100644 --- a/src/main/scala/com/linkedin/feathr/offline/config/location/DataLocation.scala +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/location/DataLocation.scala @@ -25,6 +25,7 @@ import scala.collection.JavaConverters._ new JsonSubTypes.Type(value = classOf[PathList], name = "pathlist"), new JsonSubTypes.Type(value = classOf[Jdbc], name = "jdbc"), new JsonSubTypes.Type(value = classOf[GenericLocation], name = "generic"), + new JsonSubTypes.Type(value = classOf[Snowflake], name = "snowflake"), )) trait DataLocation { /** diff --git a/src/main/scala/com/linkedin/feathr/offline/config/location/GenericLocation.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/location/GenericLocation.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/config/location/GenericLocation.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/location/GenericLocation.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/config/location/Jdbc.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/location/Jdbc.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/config/location/Jdbc.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/location/Jdbc.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/config/location/KafkaEndpoint.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/location/KafkaEndpoint.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/config/location/KafkaEndpoint.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/location/KafkaEndpoint.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/config/location/PathList.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/location/PathList.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/config/location/PathList.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/location/PathList.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/config/location/SimplePath.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/location/SimplePath.scala similarity index 88% rename from src/main/scala/com/linkedin/feathr/offline/config/location/SimplePath.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/location/SimplePath.scala index d2d1e2db6..9bb110bf3 100644 --- a/src/main/scala/com/linkedin/feathr/offline/config/location/SimplePath.scala +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/location/SimplePath.scala @@ -19,7 +19,13 @@ case class SimplePath(@JsonProperty("path") path: String) extends DataLocation { override def getPathList: List[String] = List(path) - override def isFileBasedLocation(): Boolean = true + override def isFileBasedLocation(): Boolean = { + if (path.startsWith("jdbc:")) { + false + } else { + true + } + } override def toString: String = s"SimplePath(path=${path})" } diff --git a/feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/location/Snowflake.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/location/Snowflake.scala new file mode 100644 index 000000000..8421ca280 --- /dev/null +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/location/Snowflake.scala @@ -0,0 +1,64 @@ +package com.linkedin.feathr.offline.config.location + +import com.fasterxml.jackson.annotation.{JsonAlias, JsonIgnoreProperties} +import com.fasterxml.jackson.module.caseclass.annotation.CaseClassDeserialize +import com.linkedin.feathr.common.Header +import org.apache.spark.sql.{DataFrame, SparkSession} +import org.codehaus.jackson.annotate.JsonProperty +import com.linkedin.feathr.offline.generation.SparkIOUtils +import org.apache.hadoop.mapred.JobConf + +/** + * Snowflake source config. + * Example: + * snowflakeBatchSource: { + type: SNOWFLAKE + config: { + dbtable: "SNOWFLAKE_TABLE" + database: "SNOWFLAKE_DB" + schema: "SNOWFLAKE_SCHEMA" + } + } + * + * + */ +@CaseClassDeserialize() +@JsonIgnoreProperties(ignoreUnknown = true) +case class Snowflake(@JsonProperty("database") database: String, + @JsonProperty("schema") schema: String, + @JsonProperty("dbtable") dbtable: String = "", + @JsonProperty("query") query: String = "") extends DataLocation { + + override def loadDf(ss: SparkSession, dataIOParameters: Map[String, String] = Map()): DataFrame = { + SparkIOUtils.createUnionDataFrame(getPathList, dataIOParameters, new JobConf(), List()) + } + + override def writeDf(ss: SparkSession, df: DataFrame, header: Option[Header]): Unit = ??? + + override def getPath: String = { + val baseUrl = s"snowflake://snowflake_account/?sfDatabase=${database}&sfSchema=${schema}" + if (dbtable.isEmpty) { + baseUrl + s"&query=${query}" + } else { + baseUrl + s"&dbtable=${dbtable}" + } + } + + override def getPathList: List[String] = List(getPath) + + override def isFileBasedLocation(): Boolean = false +} + +object Snowflake { + /** + * Create Snowflake InputLocation with required info + * + * @param database + * @param schema + * @param dbtable + * @param query + * @return Newly created InputLocation instance + */ + def apply(database: String, schema: String, dbtable: String, query: String): Snowflake = Snowflake(database, schema, dbtable=dbtable, query=query) + +} \ No newline at end of file diff --git a/src/main/scala/com/linkedin/feathr/offline/config/sources/FeatureGroupsUpdater.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/sources/FeatureGroupsUpdater.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/config/sources/FeatureGroupsUpdater.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/config/sources/FeatureGroupsUpdater.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/derived/DerivedFeature.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/derived/DerivedFeature.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/derived/DerivedFeature.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/derived/DerivedFeature.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/derived/DerivedFeatureEvaluator.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/derived/DerivedFeatureEvaluator.scala similarity index 95% rename from src/main/scala/com/linkedin/feathr/offline/derived/DerivedFeatureEvaluator.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/derived/DerivedFeatureEvaluator.scala index ff16ebe18..59dd8ea8e 100644 --- a/src/main/scala/com/linkedin/feathr/offline/derived/DerivedFeatureEvaluator.scala +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/derived/DerivedFeatureEvaluator.scala @@ -1,19 +1,19 @@ package com.linkedin.feathr.offline.derived -import com.linkedin.feathr.{common, offline} -import com.linkedin.feathr.common.{FeatureDerivationFunction, FeatureTypeConfig} import com.linkedin.feathr.common.exception.{ErrorLabel, FeathrException} -import com.linkedin.feathr.offline.{ErasedEntityTaggedFeature, FeatureDataFrame} +import com.linkedin.feathr.common.{FeatureDerivationFunction, FeatureTypeConfig} import com.linkedin.feathr.offline.client.DataFrameColName import com.linkedin.feathr.offline.client.plugins.{FeathrUdfPluginContext, FeatureDerivationFunctionAdaptor} -import com.linkedin.feathr.offline.derived.functions.{MvelFeatureDerivationFunction, SeqJoinDerivationFunction} -import com.linkedin.feathr.offline.derived.strategies.{DerivationStrategies, RowBasedDerivation, SequentialJoinAsDerivation, SparkUdfDerivation} +import com.linkedin.feathr.offline.derived.functions.{MvelFeatureDerivationFunction, SQLFeatureDerivationFunction, SeqJoinDerivationFunction} +import com.linkedin.feathr.offline.derived.strategies._ import com.linkedin.feathr.offline.join.algorithms.{SequentialJoinConditionBuilder, SparkJoinWithJoinCondition} import com.linkedin.feathr.offline.logical.FeatureGroups import com.linkedin.feathr.offline.mvel.plugins.FeathrExpressionExecutionContext -import com.linkedin.feathr.offline.util.FeaturizedDatasetUtils import com.linkedin.feathr.offline.source.accessor.DataPathHandler +import com.linkedin.feathr.offline.util.FeaturizedDatasetUtils +import com.linkedin.feathr.offline.{ErasedEntityTaggedFeature, FeatureDataFrame} import com.linkedin.feathr.sparkcommon.FeatureDerivationFunctionSpark +import com.linkedin.feathr.{common, offline} import org.apache.log4j.Logger import org.apache.spark.sql.{DataFrame, SparkSession} @@ -45,6 +45,9 @@ private[offline] class DerivedFeatureEvaluator(derivationStrategies: DerivationS case h: FeatureDerivationFunctionSpark => val resultDF = derivationStrategies.customDerivationSparkStrategy(keyTag, keyTagList, contextDF, derivedFeature, h, mvelContext) convertFeatureColumnToQuinceFds(producedFeatureColName, derivedFeature, resultDF) + case s: SQLFeatureDerivationFunction => + val resultDF = derivationStrategies.sqlDerivationSparkStrategy(keyTag, keyTagList, contextDF, derivedFeature, s, mvelContext) + convertFeatureColumnToQuinceFds(producedFeatureColName, derivedFeature, resultDF) case x: FeatureDerivationFunction => // We should do the FDS conversion inside the rowBasedDerivationStrategy here. The result of rowBasedDerivationStrategy // can be NTV FeatureValue or TensorData-based Feature. NTV FeatureValue has fixed FDS schema. However, TensorData @@ -118,8 +121,8 @@ private[offline] object DerivedFeatureEvaluator { val defaultStrategies = strategies.DerivationStrategies( new SparkUdfDerivation(), new RowBasedDerivation(featureGroups.allTypeConfigs, mvelContext), - new SequentialJoinAsDerivation(ss, featureGroups, SparkJoinWithJoinCondition(SequentialJoinConditionBuilder), dataPathHandlers) - ) + new SequentialJoinAsDerivation(ss, featureGroups, SparkJoinWithJoinCondition(SequentialJoinConditionBuilder), dataPathHandlers), + new SqlDerivationSpark()) new DerivedFeatureEvaluator(defaultStrategies, mvelContext) } diff --git a/src/main/scala/com/linkedin/feathr/offline/derived/functions/MvelFeatureDerivationFunction.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/derived/functions/MvelFeatureDerivationFunction.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/derived/functions/MvelFeatureDerivationFunction.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/derived/functions/MvelFeatureDerivationFunction.scala diff --git a/feathr-impl/src/main/scala/com/linkedin/feathr/offline/derived/functions/MvelFeatureDerivationFunction1.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/derived/functions/MvelFeatureDerivationFunction1.scala new file mode 100644 index 000000000..2d5e30fb8 --- /dev/null +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/derived/functions/MvelFeatureDerivationFunction1.scala @@ -0,0 +1,59 @@ +package com.linkedin.feathr.offline.derived.functions + +import com.linkedin.feathr.common +import com.linkedin.feathr.common.{FeatureDerivationFunction, FeatureTypeConfig} +import com.linkedin.feathr.offline.FeatureValue +import com.linkedin.feathr.offline.mvel.plugins.FeathrExpressionExecutionContext +import com.linkedin.feathr.offline.mvel.{FeatureVariableResolverFactory, MvelContext, MvelUtils} +import org.mvel2.MVEL + +/** + * A derivation function defined via an MVEL expression. + * Unlike SimpleMvelDerivationFunction, this class is not for one-liners, and is useful for situations where + * the feature names aren't (or can't be) given directly in a single expression. For example, see the example + * config below: + * + * example_derived_feature: { + * key: [viewerId, vieweeId] + * input: { + * x: { keyTag: viewerId, feature: member_connectionCount } + * y: { keyTag: vieweeId, feature: member_connectionCount } + * } + * definition: "x - y" + * } + */ +private[offline] class MvelFeatureDerivationFunction1( + inputFeatures: Seq[String], + expression: String, + featureName: String, + featureTypeConfigOpt: Option[FeatureTypeConfig] = None) + extends FeatureDerivationFunction { + var mvelContext: Option[FeathrExpressionExecutionContext] = None + + val parameterNames: Seq[String] = inputFeatures + + private val compiledExpression = { + val parserContext = MvelContext.newParserContext() + MVEL.compileExpression(expression, parserContext) + } + + override def getFeatures(inputs: Seq[Option[common.FeatureValue]]): Seq[Option[common.FeatureValue]] = { + val argMap = (parameterNames zip inputs).toMap + val variableResolverFactory = new FeatureVariableResolverFactory(argMap) + + MvelUtils.executeExpression(compiledExpression, null, variableResolverFactory, featureName, mvelContext) match { + case Some(value) => + val featureTypeConfig = featureTypeConfigOpt.getOrElse(FeatureTypeConfig.UNDEFINED_TYPE_CONFIG) + if (value.isInstanceOf[common.FeatureValue]) { + // The dependent feature values could have been converted to FeatureValue already, e.g. using MVEL + // to rename an anchored feature where MVEL is just returning the original feature value + Seq(Some(value.asInstanceOf[common.FeatureValue])) + } else { + // If mvel returns some 'raw' value, use feature value to build FeatureValue object + Seq(Some(FeatureValue.fromTypeConfig(value, featureTypeConfig))) + } + case None => Seq(None) // undefined + } + } +} + diff --git a/src/main/scala/com/linkedin/feathr/offline/derived/functions/SQLFeatureDerivationFunction.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/derived/functions/SQLFeatureDerivationFunction.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/derived/functions/SQLFeatureDerivationFunction.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/derived/functions/SQLFeatureDerivationFunction.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/derived/functions/SeqJoinDerivationFunction.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/derived/functions/SeqJoinDerivationFunction.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/derived/functions/SeqJoinDerivationFunction.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/derived/functions/SeqJoinDerivationFunction.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/derived/functions/SimpleMvelDerivationFunction.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/derived/functions/SimpleMvelDerivationFunction.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/derived/functions/SimpleMvelDerivationFunction.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/derived/functions/SimpleMvelDerivationFunction.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/derived/strategies/DerivationStrategies.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/derived/strategies/DerivationStrategies.scala similarity index 87% rename from src/main/scala/com/linkedin/feathr/offline/derived/strategies/DerivationStrategies.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/derived/strategies/DerivationStrategies.scala index e54d68f59..13fbec9c7 100644 --- a/src/main/scala/com/linkedin/feathr/offline/derived/strategies/DerivationStrategies.scala +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/derived/strategies/DerivationStrategies.scala @@ -1,8 +1,8 @@ package com.linkedin.feathr.offline.derived.strategies import com.linkedin.feathr.common.{FeatureDerivationFunction, FeatureDerivationFunctionBase} -import com.linkedin.feathr.offline.derived.functions.SeqJoinDerivationFunction import com.linkedin.feathr.offline.derived.DerivedFeature +import com.linkedin.feathr.offline.derived.functions.{SQLFeatureDerivationFunction, SeqJoinDerivationFunction} import com.linkedin.feathr.offline.mvel.plugins.FeathrExpressionExecutionContext import com.linkedin.feathr.sparkcommon.FeatureDerivationFunctionSpark import org.apache.spark.sql.DataFrame @@ -41,10 +41,17 @@ private[offline] trait RowBasedDerivationStrategy extends DerivationStrategy[Fea */ private[offline] trait SequentialJoinDerivationStrategy extends DerivationStrategy[SeqJoinDerivationFunction] +/** + * Implementation should define how a SQL-expression based derivation is evaluated. + */ +private[offline] trait SqlDerivationSparkStrategy extends DerivationStrategy[SQLFeatureDerivationFunction] + /** * This case class holds the implementations of supported strategies. */ private[offline] case class DerivationStrategies( customDerivationSparkStrategy: SparkUdfDerivationStrategy, rowBasedDerivationStrategy: RowBasedDerivationStrategy, - sequentialJoinDerivationStrategy: SequentialJoinDerivationStrategy) + sequentialJoinDerivationStrategy: SequentialJoinDerivationStrategy, + sqlDerivationSparkStrategy: SqlDerivationSparkStrategy) { +} diff --git a/src/main/scala/com/linkedin/feathr/offline/derived/strategies/RowBasedDerivation.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/derived/strategies/RowBasedDerivation.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/derived/strategies/RowBasedDerivation.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/derived/strategies/RowBasedDerivation.scala diff --git a/feathr-impl/src/main/scala/com/linkedin/feathr/offline/derived/strategies/SeqJoinAggregator.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/derived/strategies/SeqJoinAggregator.scala new file mode 100644 index 000000000..66c5963da --- /dev/null +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/derived/strategies/SeqJoinAggregator.scala @@ -0,0 +1,435 @@ +package com.linkedin.feathr.offline.derived.strategies + +import com.linkedin.feathr.common +import com.linkedin.feathr.common.{FeatureAggregationType, FeatureValue} +import com.linkedin.feathr.common.FeatureAggregationType.{AVG, ELEMENTWISE_AVG, ELEMENTWISE_MAX, ELEMENTWISE_MIN, ELEMENTWISE_SUM, MAX, MIN, SUM, UNION} +import com.linkedin.feathr.exception.ErrorLabel.FEATHR_USER_ERROR +import com.linkedin.feathr.exception.FeathrConfigException +import com.linkedin.feathr.offline.join.algorithms.SeqJoinExplodedJoinKeyColumnAppender +import com.linkedin.feathr.offline.transformation.DataFrameDefaultValueSubstituter.substituteDefaults +import com.linkedin.feathr.offline.util.{CoercionUtilsScala, FeaturizedDatasetUtils, FeathrUtils} +import com.linkedin.feathr.sparkcommon.SeqJoinCustomAggregation +import org.apache.spark.sql.functions.{avg, collect_list, expr, first, max, min, sum, udf} +import org.apache.spark.sql.{Column, DataFrame, Row, SparkSession} +import org.apache.spark.sql.types.{ArrayType, DataType, DoubleType, FloatType, IntegerType, LongType, MapType, NumericType, StringType, StructType} + +import scala.collection.JavaConverters._ +import scala.collection.mutable + +/** + * This class contains the various functions needed to perform sequential join. These functions include substituting default + * values, performing the aggregation, etc. Most functions were copied from [[SequentialJoinAsDerivation]] and slightly + * rewritten to work with the compute model inputs. + */ +private[offline] object SeqJoinAggregator { + def substituteDefaultValuesForSeqJoinFeature( + inputDF: DataFrame, + seqJoinFeatureColumnName: String, + expansionDefaultValue: Option[FeatureValue], + ss: SparkSession): DataFrame = { + val defaultValue = expansionDefaultValue match { + case Some(x) => Map(seqJoinFeatureColumnName -> x) + case None => Map.empty[String, FeatureValue] + } + // derived feature does not have feature type + substituteDefaults(inputDF, Seq(seqJoinFeatureColumnName), defaultValue, Map(), ss) + } + + def coerceLeftDfForSeqJoin( + featureColumnNames: Seq[String], + contextDF: DataFrame + ): DataFrame = { + + // Transform the features with the provided transformations + val featureValueColumn = featureColumnNames.map { + case columnName => + val fieldIndex = contextDF.schema.fieldIndex(columnName.split("\\.").head) + val fieldType = contextDF.schema.toList(fieldIndex) + getDefaultTransformation(fieldType.dataType, columnName) + } + + val featureValueToJoinKeyColumnName = featureValueColumn zip featureColumnNames + featureValueToJoinKeyColumnName.foldLeft(contextDF)((s, x) => s.withColumn(x._2, x._1)) + } + /** + * Utility method to coerce left join key columns for seq join. + * @param dataType + * @param columnName + * @return + */ + def getDefaultTransformation(dataType: DataType, columnName: String): Column = { + // Convert 1d tensor FDS row to seq[string] for sequential join + def oneDTensorFDSStructToString(row: Row): Seq[String] = { + if (row != null) { + val dimensions = row.getAs[Seq[_]](FeaturizedDatasetUtils.FDS_1D_TENSOR_DIM) + if (dimensions.nonEmpty) { + dimensions.map(_.toString) + } else null + } else null + } + + def fvArrayToString(inputArray: Seq[Any]): Seq[String] = { + if (inputArray == null) { + Seq() + } else { + CoercionUtilsScala.coerceFeatureValueToStringKey(new common.FeatureValue(inputArray.asJava)) + } + } + + def fvMapToString(inputMap: Map[String, Float]): Seq[String] = { + if (inputMap == null) { + Seq() + } else { + CoercionUtilsScala.coerceFeatureValueToStringKey(new common.FeatureValue(inputMap.asJava)) + } + } + val coerceMapToStringKey = udf(fvMapToString(_: Map[String, Float])) + val coerceArrayToStringKey = udf(fvArrayToString(_: Seq[Any])) + val coerce1dTensorFDSStructToStringKey = udf(oneDTensorFDSStructToString(_: Row)) + dataType match { + case _: StringType => expr(columnName) + case _: NumericType => expr(columnName) + case _: MapType => coerceMapToStringKey(expr(columnName)) + case _: ArrayType => coerceArrayToStringKey(expr(columnName)) + case _: StructType => coerce1dTensorFDSStructToStringKey(expr(columnName)) + case fType => throw new FeathrConfigException(FEATHR_USER_ERROR, s"Cannot coerce feature with type ${fType} to join key in SequentialJoin") + } + } + + /** + * Apply aggregation for SeqJoin. We always groupBy the entire left dataframe to keep the original number of rows intact. + * @param derivedFeature Name of the derived feature + * @param seqJoinProducedFeatureName name of the column which will have the seqJoin feature + * @param joined Dataframe produced after the SeqJoin and before aggregation + * @param aggregationFunction Name of the aggregation function, could be a class extending [[ComplexAggregation]] or + * one of the functions mentioned in [[FeatureAggregationType]] + * @return dataframe with only the groupBy columns and the aggregated feature value result + */ + def applyAggregationFunction( + producedFeatureName: String, + seqJoinProducedFeatureName: String, + joined: DataFrame, + aggregationFunction: String, + groupByCol: String): DataFrame = { + if (aggregationFunction.isEmpty) { + // Sequential Join does not support empty aggregation function. + // This is checked when loading config but also here to cover all cases. + throw new FeathrConfigException( + FEATHR_USER_ERROR, + s"Empty aggregation is not supported for feature ${producedFeatureName}, in sequential join.") + } else if (aggregationFunction == UNION.toString) { + applyUnionAggregation(seqJoinProducedFeatureName, joined, groupByCol) + } else if (Seq(SUM, MAX, MIN, AVG).map(_.toString).contains(aggregationFunction)) { + applyNumericAggregation(FeatureAggregationType.valueOf(aggregationFunction), seqJoinProducedFeatureName, joined, groupByCol) + } else if (Seq(ELEMENTWISE_MIN, ELEMENTWISE_MAX, ELEMENTWISE_SUM, ELEMENTWISE_AVG).map(_.toString).contains(aggregationFunction)) { + applyElementWiseAggregation(FeatureAggregationType.valueOf(aggregationFunction), seqJoinProducedFeatureName, joined, groupByCol) + } else { + val aggTypeClass = Class.forName(aggregationFunction).newInstance() + aggTypeClass match { + case derivationFunction: SeqJoinCustomAggregation => // Custom aggregation class + val featureNameToJoinedColMap = Map(producedFeatureName -> seqJoinProducedFeatureName) + val (groupedDF, preservedColumns) = getGroupedDF(joined, groupByCol, seqJoinProducedFeatureName) + groupedDF.agg( + derivationFunction + .applyAggregation(featureNameToJoinedColMap)(producedFeatureName) + .alias(seqJoinProducedFeatureName), + preservedColumns: _*) + case _ => // Unsupported Aggregation type + throw new FeathrConfigException( + FEATHR_USER_ERROR, + s"Unsupported aggregation type ${aggregationFunction} for the seqJoin feature ${producedFeatureName}") + } + } + } + + /** + * Explode left join key column if necessary. The spark join condition for sequential join is capable of handling an array + * type as the left join key (it will join if element from right is in the array in the left). However, in some cases, + * we have seen performance improvements when instead the left join key array is exploded into individual rows. Thus this + * function will perform the explode as necessary. The following conditions should be satisfied - + * 1. The optimization should be enabled. + * 2. The join key column should contain an array type column. + * @param ss spark session + * @param inputDF Input Datafeathr. + * @param joinKeys Join key columns for the Datafeathr. + * @param seqJoinFeatureName Sequential Join feature name (used for providing more context in case of errors). + * @return adjusted join key column names and DataFrame with exploded column appended. + */ + private[feathr] def explodeLeftJoinKey(ss: SparkSession, inputDF: DataFrame, joinKeys: Seq[String], seqJoinFeatureName: String): (Seq[String], DataFrame) = { + // isSeqJoinArrayExplodeEnabled flag is controlled "spark.feathr.seq.join.array.explode.enabled" config. + // When enabled, array columns are exploded to avoid BroadcastNestedLoopJoin + val isSeqJoinArrayExplodeEnabled = FeathrUtils.getFeathrJobParam(ss, FeathrUtils.SEQ_JOIN_ARRAY_EXPLODE_ENABLED).toBoolean + if (isSeqJoinArrayExplodeEnabled) { + val joinKeyColumnAppender = new SeqJoinExplodedJoinKeyColumnAppender(seqJoinFeatureName) + joinKeyColumnAppender.appendJoinKeyColunmns(joinKeys, inputDF) + } else { + (joinKeys, inputDF) + } + } + + /** + * Apply Union aggregation for SeqJoin. + * @param groupByCol groupby column + * @param seqJoinProducedFeatureName name of the column which will have the seqJoin feature + * @param joinedDF Dataframe produced after the SeqJoin and before aggregation + * @return dataframe with only the groupBy columns and the aggregated feature value result + */ + private[feathr] def applyUnionAggregation(seqJoinProducedFeatureName: String, joinedDF: DataFrame, groupByCol: String): DataFrame = { + def union1DFDSTensor(row: Row, otherRow: Row): Row = { + val indices = row.getAs[mutable.WrappedArray[_]](0).union(otherRow.getAs[mutable.WrappedArray[_]](0)) + val values = row.getAs[mutable.WrappedArray[_]](1) ++ otherRow.getAs[mutable.WrappedArray[_]](1) + Row.apply(indices, values) + } + val flatten_map = udf((featureValues: Seq[Map[String, Float]]) => featureValues.flatten.toMap) + val fieldIndex = joinedDF.schema.fieldIndex(seqJoinProducedFeatureName) + val fieldType = joinedDF.schema.toList(fieldIndex) + val (groupedDF, preservedColumns) = getGroupedDF(joinedDF, groupByCol, seqJoinProducedFeatureName) + val aggDF: DataFrame = { + fieldType.dataType match { + case _: StringType => groupedDF.agg(collect_list(seqJoinProducedFeatureName).alias(seqJoinProducedFeatureName), preservedColumns: _*) + case _: NumericType => groupedDF.agg(collect_list(seqJoinProducedFeatureName).alias(seqJoinProducedFeatureName), preservedColumns: _*) + case _: MapType => groupedDF.agg(flatten_map(collect_list(seqJoinProducedFeatureName)).alias(seqJoinProducedFeatureName), preservedColumns: _*) + // FDS 1d Tensor + case structType: StructType if structType.fields.length == 2 => + val flatten_FDSStruct = udf((featureValues: Seq[Row]) => { + val mergedRow = + // If the feature values are null then return empty indices and values for 1d FDS tensor + if (featureValues.isEmpty) Row.apply(mutable.WrappedArray.empty, mutable.WrappedArray.empty) + else featureValues.reduce((row, otherRow) => union1DFDSTensor(row, otherRow)) + mergedRow + }, structType) + groupedDF.agg(flatten_FDSStruct(collect_list(seqJoinProducedFeatureName)).alias(seqJoinProducedFeatureName), preservedColumns: _*) + case fType => throw new FeathrConfigException(FEATHR_USER_ERROR, s"Union aggregation of type {$fType} for SeqJoin is not supported.") + } + } + aggDF + } + + /** + * utility function for sequential join wit aggregation + * @param joinedDF dataframe after sequential expansion feature joined + * @param groupByCol groupby column for the sequential join aggregation + * @param excludeColumn column that should not be included in the output column + * @return (grouped input dataframe, column to preserved in the output dataframe) + */ + private def getGroupedDF(joinedDF: DataFrame, groupByCol: String, excludeColumn: String) = { + val groupedDF = joinedDF.groupBy(expr(groupByCol)) + val presevedColumns = joinedDF.columns.collect { + case colName if (!colName.equals(groupByCol) && !colName.equals(excludeColumn)) => + first(expr(colName)).as(colName) + } + (groupedDF, presevedColumns) + } + + /* Given input parameters of the indices and values arrays of 2 FDS 1d sparse tensors, this function will apply + * the appropriate elementwise aggregation (max, min, or sum). Note that we apply sum in the case of ELEMENTWISE_AVG + * and ELEMENTWISE_SUM because we will be dividing by the number of rows at the end for ELEMENTWISE_AVG. The elementwise + * component is accomplished by converting the tensor into a map where indices are the keys and values are the values. + * The map is then converted to a list which we can then apply elementwise aggregation functions via groupBy. + */ + private def applyElementwiseOnRow[T: Numeric]( + indices1: mutable.WrappedArray[_], + indices2: mutable.WrappedArray[_], + values1: mutable.WrappedArray[T], + values2: mutable.WrappedArray[T], + aggType: FeatureAggregationType) = { + val map1 = (indices1 zip values1).toMap + val map2 = (indices2 zip values2).toMap + val union_list = map1.toList ++ map2.toList + aggType match { + case ELEMENTWISE_AVG | ELEMENTWISE_SUM => union_list.groupBy(_._1).mapValues(_.map(_._2).sum) + case ELEMENTWISE_MIN => union_list.groupBy(_._1).mapValues(_.map(_._2).min) + case ELEMENTWISE_MAX => union_list.groupBy(_._1).mapValues(_.map(_._2).max) + } + } + + /* Element wise aggregation UDF that takes 2 rows that are of the format of 1d FDS tensor and performs the appropriate + * elementwise aggregation between the two rows. The DataType of the values in the FDS tensor is also passed in as + * the last parameter so we can extract the values. + */ + private def tensorElementWiseAggregate(row: Row, otherRow: Row, valueType: DataType, aggType: FeatureAggregationType): Row = { + // Grab the indicies and values of the tensor + val indices1 = row.getAs[mutable.WrappedArray[_]](0) + val indices2 = otherRow.getAs[mutable.WrappedArray[_]](0) + val union_map = valueType match { + case _: FloatType => + val values1 = row.getAs[mutable.WrappedArray[Float]](1) + val values2 = otherRow.getAs[mutable.WrappedArray[Float]](1) + applyElementwiseOnRow(indices1, indices2, values1, values2, aggType) + case _: IntegerType => + val values1 = row.getAs[mutable.WrappedArray[Int]](1) + val values2 = otherRow.getAs[mutable.WrappedArray[Int]](1) + applyElementwiseOnRow(indices1, indices2, values1, values2, aggType) + case _: DoubleType => + val values1 = row.getAs[mutable.WrappedArray[Double]](1) + val values2 = otherRow.getAs[mutable.WrappedArray[Double]](1) + applyElementwiseOnRow(indices1, indices2, values1, values2, aggType) + case _: LongType => + val values1 = row.getAs[mutable.WrappedArray[Long]](1) + val values2 = otherRow.getAs[mutable.WrappedArray[Long]](1) + applyElementwiseOnRow(indices1, indices2, values1, values2, aggType) + case badType => throw new UnsupportedOperationException( + s"${badType} is not supported as a value type for 1d sparse tensors in elementwise aggregation. The only types" + + s"supported are Floats, Integers, Doubles, and Longs.") + } + Row.apply(union_map.keySet.toList, union_map.values.toList) + } + + /** + * Apply element wise aggregation for SeqJoin + * @param groupByCol groupby column + * @param aggType Name of the aggregation function as mentioned in [[FeatureAggregationType]] + * @param seqJoinProducedFeatureName name of the column which will have the seqJoin feature + * @param joinedDF Dataframe produced after thee SeqJoin and before aggregation + * @return dataframe with only the groupBy columns and the aggregated feature value result + */ + private[offline] def applyElementWiseAggregation( + aggType: FeatureAggregationType, + seqJoinProducedFeatureName: String, + joinedDF: DataFrame, + groupByCol: String): DataFrame = { + val fieldIndex = joinedDF.schema.fieldIndex(seqJoinProducedFeatureName) + val fieldType = joinedDF.schema.toList(fieldIndex) + def sumArr = + udf((a: Seq[Seq[Float]]) => { + if (a.isEmpty) { + Seq() + } else { + val zeroSeq = Seq.fill[Float](a.head.size)(0.0f) + a.foldLeft(zeroSeq)((a, x) => (a zip x).map { case (u, v) => u + v }) + } + }) + def avgArr = + udf((a: Seq[Seq[Float]]) => { + if (a.isEmpty) { + Seq() + } else { + val zeroSeq = Seq.fill[Float](a.head.size)(0.0f) + val sum = a.foldLeft(zeroSeq)((a, x) => (a zip x).map { case (u, v) => u + v }) + sum map (value => value / a.size) + } + }) + def minArr = + udf((a: Seq[Seq[Float]]) => { + val newList = a.transpose + newList map (list => list.min) + }) + def maxArr = + udf((a: Seq[Seq[Float]]) => { + val newList = a.transpose + newList map (list => list.max) + }) + // Explicitly cast Array(Double) to Float before applying aggregate + def transformToFloat(elementType: DataType, column: Column): Column = { + elementType match { + case _: NumericType if elementType != FloatType => column.cast("array") + case _: FloatType => column + case _ => + throw new UnsupportedOperationException( + s"${aggType} aggregation type not supported for feature '${seqJoinProducedFeatureName}', " + + s"${aggType} only supports array of numeric type but found array of ${elementType}") + + } + } + + // Return element-wise aggregate UDF based on the element type of the array. + def aggregate(elementType: DataType, column: Column): Column = { + val columnAsList = collect_list(transformToFloat(elementType, column)) + aggType match { + case ELEMENTWISE_SUM => sumArr(columnAsList) + case ELEMENTWISE_AVG => avgArr(columnAsList) + case ELEMENTWISE_MIN => minArr(columnAsList) + case ELEMENTWISE_MAX => maxArr(columnAsList) + } + } + + val (groupedDF, preservedColumns) = getGroupedDF(joinedDF, groupByCol, seqJoinProducedFeatureName) + fieldType.dataType match { + case ftype: ArrayType => + groupedDF.agg( + aggregate(ftype.elementType, expr(seqJoinProducedFeatureName)) + .alias(seqJoinProducedFeatureName), + preservedColumns: _*) + // 1D Sparse tensor case + case structType: StructType if structType.fields.length == 2 => + val valueType = structType.apply("values").dataType.asInstanceOf[ArrayType].elementType + val flatten_FDSStruct = udf((featureValues: Seq[Row]) => { + val mergedRow = + // If the feature values are null then return empty indices and values for 1d FDS tensor + if (featureValues.isEmpty) Row.apply(List.empty, List.empty) + else featureValues.reduce((row, nextRow) => tensorElementWiseAggregate(row, nextRow, valueType, aggType)) + // Note the elementWiseSum1DFDSTensor function returns the row where the values are Lists and not WrappedArray + // Note that here we have to duplicate the code to divide by the length to get the average because we can't + // easily extract out the division operation into a method that takes numerics. + val indices = mergedRow.getAs[List[_]](0) + val values = valueType match { + case _: FloatType => + val rawValues = mergedRow.getAs[List[Float]](1) + if (aggType == ELEMENTWISE_AVG) { + rawValues.map(_ / featureValues.length) + } else { + rawValues + } + case _: IntegerType => + val rawValues = mergedRow.getAs[List[Int]](1) + if (aggType == ELEMENTWISE_AVG) { + rawValues.map(_ / featureValues.length) + } else { + rawValues + } + case _: DoubleType => + val rawValues = mergedRow.getAs[List[Double]](1) + if (aggType == ELEMENTWISE_AVG) { + rawValues.map(_ / featureValues.length) + } else { + rawValues + } + case _: LongType => + val rawValues = mergedRow.getAs[List[Long]](1) + if (aggType == ELEMENTWISE_AVG) { + rawValues.map(_ / featureValues.length) + } else { + rawValues + } + case badType => throw new UnsupportedOperationException( + s"${badType} is not supported as a value type for 1d sparse tensors in elementwise aggregation.") + } + Row.apply(indices, values) + }, structType) + groupedDF.agg(flatten_FDSStruct(collect_list(seqJoinProducedFeatureName)).alias(seqJoinProducedFeatureName), preservedColumns: _*) + case _ => + throw new UnsupportedOperationException( + s"${aggType} aggregation type not supported for feature ${seqJoinProducedFeatureName}, " + + s"${aggType} only supports array and 1d sparse tensor type features") + } + } + + /** + * Apply arithmetic aggregation for SeqJoin + * @param groupByCol groupby column + * @param aggType Name of the aggregation function as mentioned in [[FeatureAggregationType]] + * @param seqJoinproducedFeatureName name of the column which will have the seqJoin feature + * @param joinedDF Dataframe produced after thee SeqJoin and before aggregation + * @return dataframe with only the groupBy columns and the aggregated feature value result + */ + private def applyNumericAggregation( + aggType: FeatureAggregationType, + seqJoinproducedFeatureName: String, + joinedDF: DataFrame, + groupByCol: String): DataFrame = { + val fieldIndex = joinedDF.schema.fieldIndex(seqJoinproducedFeatureName) + val fieldType = joinedDF.schema.toList(fieldIndex) + val (groupedDF, presevedColumns) = getGroupedDF(joinedDF, groupByCol, seqJoinproducedFeatureName) + fieldType.dataType match { + case ftype: NumericType => + val aggDF: DataFrame = aggType match { + case SUM => groupedDF.agg(sum(seqJoinproducedFeatureName).alias(seqJoinproducedFeatureName), presevedColumns: _*) + case MAX => groupedDF.agg(max(seqJoinproducedFeatureName).alias(seqJoinproducedFeatureName), presevedColumns: _*) + case MIN => groupedDF.agg(min(seqJoinproducedFeatureName).alias(seqJoinproducedFeatureName), presevedColumns: _*) + case AVG => groupedDF.agg(avg(seqJoinproducedFeatureName).alias(seqJoinproducedFeatureName), presevedColumns: _*) + } + aggDF + case _ => throw new FeathrConfigException(FEATHR_USER_ERROR, s"${aggType} aggregation type is not supported for type ${fieldType}") + } + } +} \ No newline at end of file diff --git a/src/main/scala/com/linkedin/feathr/offline/derived/strategies/SequentialJoinAsDerivation.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/derived/strategies/SequentialJoinAsDerivation.scala similarity index 99% rename from src/main/scala/com/linkedin/feathr/offline/derived/strategies/SequentialJoinAsDerivation.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/derived/strategies/SequentialJoinAsDerivation.scala index 2cee39d95..d9874d522 100644 --- a/src/main/scala/com/linkedin/feathr/offline/derived/strategies/SequentialJoinAsDerivation.scala +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/derived/strategies/SequentialJoinAsDerivation.scala @@ -185,7 +185,7 @@ private[offline] class SequentialJoinAsDerivation(ss: SparkSession, * @param seqJoinFeatureName Sequential Join feature name (used for providing more context in case of errors). * @return adjusted join key column names and DataFrame with exploded column appended. */ - private def explodeLeftJoinKey(inputDF: DataFrame, joinKeys: Seq[String], seqJoinFeatureName: String): (Seq[String], DataFrame) = { + def explodeLeftJoinKey(inputDF: DataFrame, joinKeys: Seq[String], seqJoinFeatureName: String): (Seq[String], DataFrame) = { // isSeqJoinArrayExplodeEnabled flag is controlled "spark.feathr.seq.join.array.explode.enabled" config. // This is a hidden config used by FEATHR DEV ONLY. This knob is required for performance tuning. // When enabled, array columns are exploded to avoid BroadcastNestedLoopJoin diff --git a/src/main/scala/com/linkedin/feathr/offline/derived/strategies/SparkUdfDerivation.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/derived/strategies/SparkUdfDerivation.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/derived/strategies/SparkUdfDerivation.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/derived/strategies/SparkUdfDerivation.scala diff --git a/feathr-impl/src/main/scala/com/linkedin/feathr/offline/derived/strategies/SqlDerivationSpark.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/derived/strategies/SqlDerivationSpark.scala new file mode 100644 index 000000000..c7b44c1cf --- /dev/null +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/derived/strategies/SqlDerivationSpark.scala @@ -0,0 +1,118 @@ +package com.linkedin.feathr.offline.derived.strategies + +import com.linkedin.feathr.common.exception.{ErrorLabel, FeathrFeatureTransformationException} +import com.linkedin.feathr.offline.client.DataFrameColName +import com.linkedin.feathr.offline.derived.DerivedFeature +import com.linkedin.feathr.offline.derived.functions.SQLFeatureDerivationFunction +import com.linkedin.feathr.offline.job.FeatureTransformation.FEATURE_NAME_PREFIX +import com.linkedin.feathr.offline.mvel.plugins.FeathrExpressionExecutionContext +import org.apache.spark.sql.functions.expr +import org.apache.spark.sql.{DataFrame, SparkSession} + +import scala.collection.JavaConverters._ + +/** + * This class executes SQL-expression based derived feature. + */ +class SqlDerivationSpark extends SqlDerivationSparkStrategy { + + + /** + * Rewrite sqlExpression for a derived feature, e.g, replace the feature name/argument name with Frame internal dataframe column name + * @param deriveFeature derived feature definition + * @param keyTag list of tags represented by integer + * @param keyTagId2StringMap Map from the tag integer id to the string tag + * @param asIsFeatureNames features names that does not to be rewritten, i.e. passthrough features, as they do not have key tags + * @return Rewritten SQL expression + */ + private[offline] def rewriteDerivedFeatureExpression( + deriveFeature: DerivedFeature, + keyTag: Seq[Int], + keyTagId2StringMap: Seq[String], + asIsFeatureNames: Set[String]): String = { + if (!deriveFeature.derivation.isInstanceOf[SQLFeatureDerivationFunction]) { + throw new FeathrFeatureTransformationException(ErrorLabel.FEATHR_ERROR, "Should not rewrite derived feature expression for non-SQLDerivedFeatures") + } + val sqlDerivation = deriveFeature.derivation.asInstanceOf[SQLFeatureDerivationFunction] + val deriveExpr = sqlDerivation.getExpression() + val parameterNames: Seq[String] = sqlDerivation.getParameterNames().getOrElse(Seq[String]()) + val consumedFeatureNames = deriveFeature.consumedFeatureNames.zipWithIndex.map { + case (consumeFeatureName, index) => + // begin of string, or other char except number and alphabet + // val featureStartPattern = """(^|[^a-zA-Z0-9])""" + // end of string, or other char except number and alphabet + // val featureEndPattern = """($|[^a-zA-Z0-9])""" + val namePattern = if (parameterNames.isEmpty) consumeFeatureName.getFeatureName else parameterNames(index) + // getBinding.map(keyTag.get) resolves the call tags + val newName = + if (!asIsFeatureNames.contains(FEATURE_NAME_PREFIX + consumeFeatureName.getFeatureName) + // Feature generation code path does not create columns with tags. + // The check ensures we do not run into IndexOutOfBoundsException when keyTag & keyTagId2StringMap are empty. + && keyTag.nonEmpty + && keyTagId2StringMap.nonEmpty) { + DataFrameColName.genFeatureColumnName( + consumeFeatureName.getFeatureName, + Some(consumeFeatureName.getBinding.asScala.map(keyTag(_)).map(keyTagId2StringMap))) + } else { + DataFrameColName.genFeatureColumnName(consumeFeatureName.getFeatureName) + } + (namePattern, newName) + }.toMap + + // replace all feature name to column names + // featureName is consist of numAlphabetic + val ss: SparkSession = SparkSession.builder().getOrCreate() + val dependencyFeatures = ss.sessionState.sqlParser.parseExpression(deriveExpr).references.map(_.name).toSeq + // \w is [a-zA-Z0-9_], not inclusion of _ and exclusion of -, as - is ambiguous, e.g, a-b could be a feature name or feature a minus feature b + val rewrittenExpr = dependencyFeatures.foldLeft(deriveExpr)((acc, ca) => { + // in scala \W does not work as ^\w + // "a+B+1".replaceAll("([^\w])B([^\w])", "$1abc$2" = A+abc+1 + // "a+B".replaceAll("([^\w])B$", "$1abc" = a+abc + // "B+1".replaceAll("^B([^\w])", "abc$1" = abc+1 + // "B".replaceAll("^B$", "abc" = abc + val newVal = consumedFeatureNames.getOrElse(ca, ca) + val patterns = Seq("([^\\w])" + ca + "([^\\w])", "([^\\w])" + ca + "$", "^" + ca + "([^\\w])", "^" + ca + "$") + val replacements = Seq("$1" + newVal + "$2", "$1" + newVal, newVal + "$1", newVal) + val replacedExpr = patterns + .zip(replacements) + .toMap + .foldLeft(acc)((orig, pairs) => { + orig.replaceAll(pairs._1, pairs._2) + }) + replacedExpr + }) + rewrittenExpr + } + + /** + * Apply the derivation strategy. + * + * @param keyTags keyTags for the derived feature. + * @param keyTagList integer keyTag to string keyTag map. + * @param df input DataFrame. + * @param derivedFeature Derived feature metadata. + * @param derivationFunction Derivation function to evaluate the derived feature + * @return output DataFrame with derived feature. + */ + override def apply(keyTags: Seq[Int], + keyTagList: Seq[String], + df: DataFrame, + derivedFeature: DerivedFeature, + derivationFunction: SQLFeatureDerivationFunction, + mvelContext: Option[FeathrExpressionExecutionContext]): DataFrame = { + // sql expression based derived feature needs rewrite, e.g, replace the feature names with feature column names in the dataframe + // Passthrough fields do not need rewrite as they do not have tags. + val passthroughFieldNames = df.schema.fields.map(f => + if (f.name.startsWith(FEATURE_NAME_PREFIX)) { + f.name + } else { + FEATURE_NAME_PREFIX + f.name + } + ).toSet + val rewrittenExpr = rewriteDerivedFeatureExpression(derivedFeature, keyTags, keyTagList, passthroughFieldNames) + val tags = Some(keyTags.map(keyTagList).toList) + val featureColumnName = DataFrameColName.genFeatureColumnName(derivedFeature.producedFeatureNames.head, tags) + df.withColumn(featureColumnName, expr(rewrittenExpr)) + } + +} diff --git a/src/main/scala/com/linkedin/feathr/offline/evaluator/DerivedFeatureGenStage.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/evaluator/DerivedFeatureGenStage.scala similarity index 88% rename from src/main/scala/com/linkedin/feathr/offline/evaluator/DerivedFeatureGenStage.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/evaluator/DerivedFeatureGenStage.scala index a270450e4..ebb6b2809 100644 --- a/src/main/scala/com/linkedin/feathr/offline/evaluator/DerivedFeatureGenStage.scala +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/evaluator/DerivedFeatureGenStage.scala @@ -1,20 +1,19 @@ package com.linkedin.feathr.offline.evaluator -import com.linkedin.feathr.common.exception.{ErrorLabel, FeathrException} -import com.linkedin.feathr.offline -import com.linkedin.feathr.offline.client.DataFrameColName +import com.linkedin.feathr.exception.{ErrorLabel, FeathrException} +import com.linkedin.feathr.offline.{FeatureDataFrame, FeatureDataWithJoinKeys, client} +import com.linkedin.feathr.offline.client.{DataFrameColName} import com.linkedin.feathr.offline.derived.{DerivedFeature, DerivedFeatureEvaluator} import com.linkedin.feathr.offline.job.FeatureTransformation.FEATURE_TAGS_PREFIX import com.linkedin.feathr.offline.logical.{FeatureGroups, MultiStageJoinPlan} -import com.linkedin.feathr.offline.{FeatureDataFrame, FeatureDataWithJoinKeys} import org.apache.spark.sql.DataFrame /** * The case class represents DataFrame and associated metadata required to compute a derived feature. - * @param featureDataFrame base DataFrame. + * @param featureDataFrame base Datafeathr. * @param joinKeys columns of DataFrame used for joins. - * @param featureNames evaluated features on the DataFrame. + * @param featureNames evaluated features on the Datafeathr. */ private[offline] case class BaseDataFrameMetadata(featureDataFrame: FeatureDataFrame, joinKeys: Seq[String], featureNames: Seq[String]) @@ -26,11 +25,11 @@ private[offline] case class BaseDataFrameMetadata(featureDataFrame: FeatureDataF * @param derivedFeatureUtils reference to derivations executor. */ private[offline] class DerivedFeatureGenStage(featureGroups: FeatureGroups, logicalPlan: MultiStageJoinPlan, derivedFeatureUtils: DerivedFeatureEvaluator) - extends StageEvaluator[FeatureDataWithJoinKeys, FeatureDataWithJoinKeys] { + extends StageEvaluator[FeatureDataWithJoinKeys, FeatureDataWithJoinKeys] { /** * Computes derivations for the input features. Before applying the derivations, it ensures that - * the dependent features required for computation are available on a single DataFrame. + * the dependent features required for computation are available on a single Datafeathr. * @param features derived features to evaluate in this stage. * @param keyTags key tags for the stage. * @param context features evaluated thus far. @@ -50,29 +49,29 @@ private[offline] class DerivedFeatureGenStage(featureGroups: FeatureGroups, logi } else { derivedFeatureUtils.evaluate(keyTags, logicalPlan.keyTagIntsToStrings, baseFeatureDataFrame.df, derivation) } - val columnRenamedDf = dropFeathrTagsAndRenameColumn(derivedFeatureDataFrame.df, featureColumnName) + val columnRenamedDf = dropFrameTagsAndRenameColumn(derivedFeatureDataFrame.df, featureColumnName) // Update featureTypeMap and features on DataFrame metadata val updatedFeatureTypeMap = baseFeatureDataFrame.inferredFeatureType ++ derivedFeatureDataFrame.inferredFeatureType val updatedFeaturesOnDf = featuresOnBaseDf :+ derivedFeatureName - accumulator ++ updatedFeaturesOnDf.map(f => f -> (offline.FeatureDataFrame(columnRenamedDf, updatedFeatureTypeMap), joinKeys)).toMap + accumulator ++ updatedFeaturesOnDf.map(f => f -> (FeatureDataFrame(columnRenamedDf, updatedFeatureTypeMap), joinKeys)).toMap }) } /** * Prepares a Base DataFrame that can be used to compute the derived features. * The dependent features of the derived feature may be present on different DataFrames. - * In such cases, the DataFrames are joined so that the dependent features are available on a single DataFrame. + * In such cases, the DataFrames are joined so that the dependent features are available on a single Datafeathr. * @param derivedFeatureName derived feature name. * @param derivedFeatureRef derived feature representation. * @param evaluatedFeatures features evaluated thus far. * @return BaseDataFrameMetadata that contains all required features to compute a derived feature. */ def evaluateBaseDataFrameForDerivation( - derivedFeatureName: String, - derivedFeatureRef: DerivedFeature, - evaluatedFeatures: FeatureDataWithJoinKeys): BaseDataFrameMetadata = { + derivedFeatureName: String, + derivedFeatureRef: DerivedFeature, + evaluatedFeatures: FeatureDataWithJoinKeys): BaseDataFrameMetadata = { val featuresGroupedByDf = evaluatedFeatures.groupBy(_._2._1.df).mapValues(_.keySet) // features grouped by DataFrames - val consumedFeatures = derivedFeatureRef.consumedFeatureNames.map(_.getFeatureName) + val consumedFeatures = derivedFeatureRef.consumedFeatureNames.map(_.getFeatureName.toString) if (!consumedFeatures.forall(evaluatedFeatures.contains)) { throw new FeathrException( ErrorLabel.FEATHR_ERROR, @@ -108,7 +107,7 @@ private[offline] class DerivedFeatureGenStage(featureGroups: FeatureGroups, logi .reduce(_ and _) val joinedDataFrame = leftDf.join(rightDataFrame, joinConditions, "full_outer") // "full" is same as full_outer BaseDataFrameMetadata( - // merge feature type mapping for features joined to the DataFrame. + // merge feature type mapping for features joined to the Datafeathr. FeatureDataFrame(joinedDataFrame.drop(rightJoinKey: _*), leftFeatureType ++ currFeatureType), joinKeys, (accumulator.featureNames ++ featuresOnCurrentDf).distinct) @@ -122,7 +121,7 @@ private[offline] class DerivedFeatureGenStage(featureGroups: FeatureGroups, logi * However, derived feature columns are created with tags. This helper method bridges the gap. * This helper method */ - private def dropFeathrTagsAndRenameColumn(df: DataFrame, featureName: String): DataFrame = { + private def dropFrameTagsAndRenameColumn(df: DataFrame, featureName: String): DataFrame = { val columnsInDf = df.columns columnsInDf.find(c => c.contains(featureName)) match { case Some(x) => diff --git a/feathr-impl/src/main/scala/com/linkedin/feathr/offline/evaluator/NodeEvaluator.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/evaluator/NodeEvaluator.scala new file mode 100644 index 000000000..2bc682712 --- /dev/null +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/evaluator/NodeEvaluator.scala @@ -0,0 +1,52 @@ +package com.linkedin.feathr.offline.evaluator + +import com.linkedin.feathr.compute.AnyNode +import com.linkedin.feathr.offline.graph.FCMGraphTraverser +import com.linkedin.feathr.offline.source.accessor.DataPathHandler +import org.apache.spark.sql.DataFrame + +/** + * Base trait class for all node evaluators. For each node type, the evaluate API should take a single node along with + * the necessary inputs, perform the necessary dataloading or transformations specific to the node type, and return the + * the context df. The batchEvaluate API is the batch version of the evaluate API. Node evaluators must ONLY evaluate the node + * in the inputs and not evaluate any other nodes within the graph out of order. + * + * Note that the graphTraverser is a class object which contains metadata regarding the graph and graph traversal state + * which are needed for node evaluation which is why FCMGraphTraverser is needed in the evaluation functions. + * Graph metadata available in graphTraverser: + * 1. nodeIdToDataframeAndColumnMetadataMap: Map of node id to node feature df and node metadata. + * See scaladocs of DataframeAndColumnMetadata for more info. + * 2. featureColumnFormatsMap: Map of output format of feature column (RAW vs FDS) + * 3. nodes: all nodes in resolved graph + * 4. nodeIdToFeatureName: node id to feature name + * 5. joinSettings: settings from join config + observation data time range for EVENT and AGGREGATION node processing + * 6. ss: spark session for spark calls + * + * GRAPHTRAVERSER UPDATE REQUIREMENTS: + * 1. nodeIdToDataframeAndColumnMetadataMap needs to be updated for datasource nodes and look up expansion nodes. + * 2. all node evaluators which produce a feature column in the context df must mark the format in featureColumnFormatsMap + * if the feature column is already in FDS format. + */ +trait NodeEvaluator { + /** + * Evaluate a single node according to the node type and return the context df. ContextDf should contain the output + * of the node evaluation in all cases except for Datasource nodes and seq join expansion feature nodes. Output of + * node evaluation is feature column and feature column is joined to context df based on feature join key. + * @param node Node to evaluate + * @param graphTraverser FCMGraphTraverser + * @param contextDf Context df + * @return DataFrame + */ + def evaluate(node: AnyNode, graphTraverser: FCMGraphTraverser, contextDf: DataFrame, dataPathHandlers: List[DataPathHandler]): DataFrame + + /** + * Evaluate a group of nodes and return the context df. ContextDf should contain the output + * of all the node evaluation in all cases except for Datasource nodes and seq join expansion feature nodes. Output of + * node evaluation is feature column and feature column is joined to context df based on feature join key. + * @param nodes Nodes to evaluate + * @param graphTraverser FCMGraphTraverser + * @param contextDf Context df + * @return DataFrame + */ + def batchEvaluate(nodes: Seq[AnyNode], graphTraverser: FCMGraphTraverser, contextDf: DataFrame, dataPathHandlers: List[DataPathHandler]): DataFrame +} \ No newline at end of file diff --git a/src/main/scala/com/linkedin/feathr/offline/evaluator/StageEvaluator.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/evaluator/StageEvaluator.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/evaluator/StageEvaluator.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/evaluator/StageEvaluator.scala diff --git a/feathr-impl/src/main/scala/com/linkedin/feathr/offline/evaluator/aggregation/AggregationNodeEvaluator.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/evaluator/aggregation/AggregationNodeEvaluator.scala new file mode 100644 index 000000000..d0f8a2c78 --- /dev/null +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/evaluator/aggregation/AggregationNodeEvaluator.scala @@ -0,0 +1,244 @@ +package com.linkedin.feathr.offline.evaluator.aggregation + +import com.linkedin.feathr.compute.{Aggregation, AnyNode} +import com.linkedin.feathr.exception.{ErrorLabel, FeathrConfigException} +import com.linkedin.feathr.offline.anchored.WindowTimeUnit +import com.linkedin.feathr.offline.client.{NOT_VISITED, VISITED, VisitedState} +import com.linkedin.feathr.offline.config.JoinConfigSettings +import com.linkedin.feathr.offline.evaluator.NodeEvaluator +import com.linkedin.feathr.offline.graph.NodeUtils.{getDefaultConverter, getFeatureTypeConfigsMap} +import com.linkedin.feathr.offline.graph.NodeGrouper.groupSWANodes +import com.linkedin.feathr.offline.graph.{DataframeAndColumnMetadata, FCMGraphTraverser} +import com.linkedin.feathr.offline.job.FeatureTransformation +import com.linkedin.feathr.offline.source.accessor.DataPathHandler +import com.linkedin.feathr.offline.swa.SlidingWindowFeatureUtils +import com.linkedin.feathr.offline.transformation.DataFrameDefaultValueSubstituter.substituteDefaults +import com.linkedin.feathr.offline.transformation.FeatureColumnFormat +import com.linkedin.feathr.offline.transformation.FeatureColumnFormat.{FDS_TENSOR, FeatureColumnFormat, RAW} +import com.linkedin.feathr.swj.{FactData, GroupBySpec, LabelData, LateralViewParams, SlidingWindowFeature, SlidingWindowJoin, WindowSpec} +import com.linkedin.feathr.swj.aggregate.{AggregationSpec, AggregationType, AvgAggregate, AvgPoolingAggregate, CountAggregate, LatestAggregate, MaxAggregate, MaxPoolingAggregate, MinAggregate, MinPoolingAggregate, SumAggregate} +import org.apache.spark.sql.functions.col +import org.apache.spark.sql.DataFrame + +import scala.collection.JavaConverters._ +import java.time.Duration +import scala.collection.mutable + +/** + * This aggregation node evaluator class executes sliding window aggregation as defined by the Aggregation node. The inputs + * to Aggregation nodes will always be Event Nodes which represent time aware feature data. The main function here is + * processAggregationNode which will be called by the FCMGraphTraverser to evaluate aggregation nodes. + */ +object AggregationNodeEvaluator extends NodeEvaluator { + + /** + * Construct the label data required for SWA join. + * @param aggregation + * @param featureJoinConfig + * @param df + * @param nodeIdToDataframeAndColumnMetadataMap + * @return + */ + private def getLabelData(aggregation: Aggregation, joinConfigSettings: Option[JoinConfigSettings], df: DataFrame, + nodeIdToDataframeAndColumnMetadataMap: mutable.Map[Int, DataframeAndColumnMetadata]): LabelData = { + val concreteKeys = aggregation.getConcreteKey.getKey.asScala.flatMap(x => nodeIdToDataframeAndColumnMetadataMap(x).keyExpression) + val obsKeys = concreteKeys.map(k => s"CAST (${k} AS string)") + val timestampCol = SlidingWindowFeatureUtils.constructTimeStampExpr(joinConfigSettings.get.joinTimeSetting.get.timestampColumn.name, + joinConfigSettings.get.joinTimeSetting.get.timestampColumn.format) + val updatedTimestampExpr = if (joinConfigSettings.isDefined && joinConfigSettings.get.joinTimeSetting.isDefined && + joinConfigSettings.get.joinTimeSetting.get.useLatestFeatureData) { + "unix_timestamp()" + } else timestampCol + LabelData(df, obsKeys, updatedTimestampExpr) + } + + private def getLateralViewParams(aggregation: Aggregation): Option[LateralViewParams] = { + val lateralViewDef = aggregation.getFunction.getParameters.get("lateral_view_expression_0") match { + case x: String => Some(x) + case null => None + } + + val lateralViewAlias = aggregation.getFunction.getParameters.get("lateral_view_table_alias_0") match { + case x: String => Some(x) + case null => None + } + + val lateralViewParams = if (lateralViewDef.isDefined && lateralViewAlias.isDefined) { + Some(LateralViewParams(lateralViewDef.get, lateralViewAlias.get, None)) + } else None + lateralViewParams + } + + private def getAggSpec(aggType: AggregationType.Value, featureDef: String): AggregationSpec = { + aggType match { + case AggregationType.SUM => new SumAggregate(featureDef) + case AggregationType.COUNT => + // The count aggregation in spark-algorithms MP is implemented as Sum over partial counts. + // In feathr's use case, we want to treat the count aggregation as simple count of non-null items. + val rewrittenDef = s"CASE WHEN ${featureDef} IS NOT NULL THEN 1 ELSE 0 END" + new CountAggregate(rewrittenDef) + case AggregationType.AVG => new AvgAggregate(featureDef) // TODO: deal with avg. of pre-aggregated data + case AggregationType.MAX => new MaxAggregate(featureDef) + case AggregationType.MIN => new MinAggregate(featureDef) + case AggregationType.LATEST => new LatestAggregate(featureDef) + case AggregationType.MAX_POOLING => new MaxPoolingAggregate(featureDef) + case AggregationType.MIN_POOLING => new MinPoolingAggregate(featureDef) + case AggregationType.AVG_POOLING => new AvgPoolingAggregate(featureDef) + } + } + + private def getSimTimeDelay(featureName: String, joinConfigSettings: Option[JoinConfigSettings], + featuresToTimeDelayMap: Map[String, String]): Duration = { + if (featuresToTimeDelayMap.contains(featureName)) { + if (joinConfigSettings.isEmpty || joinConfigSettings.get.joinTimeSetting.isEmpty || + joinConfigSettings.get.joinTimeSetting.get.simulateTimeDelay.isEmpty) { + throw new FeathrConfigException( + ErrorLabel.FEATHR_USER_ERROR, + "overrideTimeDelay cannot be defined without setting a simulateTimeDelay in the " + + "joinTimeSettings") + } + WindowTimeUnit.parseWindowTime(featuresToTimeDelayMap(featureName)) + } else { + if (joinConfigSettings.isDefined && joinConfigSettings.get.joinTimeSetting.isDefined && + joinConfigSettings.get.joinTimeSetting.get.simulateTimeDelay.isDefined) { + joinConfigSettings.get.joinTimeSetting.get.simulateTimeDelay.get + } else { + Duration.ZERO + } + } + } + + // Get a set of [[FactData]] grouped by feature data source, keys and lateral view params. + private def getFactDataSet(swaNodeIdToNode: Map[Integer, AnyNode], swaMegaNodeMap: Map[Integer, Seq[Integer]], + aggregation: Aggregation, nodeIdToDataframeAndColumnMetadataMap: mutable.Map[Int, DataframeAndColumnMetadata], + featureColumnFormatsMap: mutable.HashMap[String, FeatureColumnFormat], + joinConfigSettings: Option[JoinConfigSettings], + featuresToTimeDelayMap: Map[String, String], + nodeIdToFeatureName: Map[Integer, String]): List[FactData] = { + val allSwaFeatures = swaMegaNodeMap(aggregation.getId) + val nodes = allSwaFeatures.map(swaNodeIdToNode(_)) + + // We will group the nodes by the feature datasource, key expression and the lateral view params as prescribed by the SWA library + val groupedNodes = nodes.groupBy(x => { + val lateralViewParams = getLateralViewParams(x.getAggregation) + (nodeIdToDataframeAndColumnMetadataMap(x.getAggregation.getInput.getId()).dataSource, + nodeIdToDataframeAndColumnMetadataMap(x.getAggregation.getInput.getId()).keyExpression, + lateralViewParams) + }) + + // Again sort the acc to size of the groupings to reduce shuffle size. + groupedNodes.values.toList.sortBy(p => p.size).reverse.map(nodesAtSameLevel => { + val exampleNode = nodesAtSameLevel.filter(x => nodeIdToDataframeAndColumnMetadataMap.contains(x.getAggregation.getInput.getId())).head.getAggregation + val featureDf = nodeIdToDataframeAndColumnMetadataMap(exampleNode.getInput.getId()).df + val featureKeys = nodeIdToDataframeAndColumnMetadataMap(exampleNode.getInput.getId()).keyExpression + val timestampExpr = nodeIdToDataframeAndColumnMetadataMap(exampleNode.getInput.getId()).timestampColumn.get + val featureKeysAsString = featureKeys.map(k => s"CAST (${k} AS string)") + + val lateralViewParams = getLateralViewParams(exampleNode) + val slidingWindowFeatureList = nodesAtSameLevel.map(node => { + val aggNode = node.getAggregation + val featureName = nodeIdToFeatureName(aggNode.getId()) + + val aggType = AggregationType.withName(aggNode.getFunction.getParameters.get("aggregation_type")) + val featureDef = aggNode.getFunction.getParameters.get("target_column") + val rewrittenFeatureDef = if (featureDef.contains(FeatureTransformation.USER_FACING_MULTI_DIM_FDS_TENSOR_UDF_NAME)) { + // If the feature definition contains USER_FACING_MULTI_DIM_FDS_TENSOR_UDF_NAME then the feature column is already in FDS format. + // So we strip the udf name and return only the feature name. + (FeatureTransformation.parseMultiDimTensorExpr(featureDef), FDS_TENSOR) + } else (featureDef, RAW) + val aggregationSpec = getAggSpec(aggType, rewrittenFeatureDef._1) + + val window = Duration.parse(aggNode.getFunction.getParameters.get("window_size")) + val simTimeDelay = getSimTimeDelay(featureName, joinConfigSettings, featuresToTimeDelayMap) + + val filterCondition = aggNode.getFunction.getParameters.get("filter_expression") match { + case x: String => Some(x) + case null => None + } + + val groupBy = aggNode.getFunction.getParameters.get("group_by_expression") match { + case x: String => Some(x) + case null => None + } + + val limit = aggNode.getFunction.getParameters.get("max_number_groups") match { + case x: String => Some(x.toInt) + case null => Some(0) + } + + val groupbySpec = if (groupBy.isDefined) { + Some(GroupBySpec(groupBy.get, limit.get)) + } else None + + featureColumnFormatsMap(featureName) = rewrittenFeatureDef._2 + SlidingWindowFeature(featureName, aggregationSpec, WindowSpec(window, simTimeDelay), filterCondition, groupbySpec, lateralViewParams) + }) + FactData(featureDf, featureKeysAsString, timestampExpr, slidingWindowFeatureList.toList) + } + ) + } + + /** + * The nodes are first grouped by the label data, and then further grouped by the feature datasource, + * feature keys and lateral view params. We invoke the SWA library achieve the SWA join. + * + * @param nodes Seq[AnyNode] + * @param graphTraverser FCMGraphTraverser + * @param contextDf Context df + * @return DataFrame + */ + override def batchEvaluate(nodes: Seq[AnyNode], graphTraverser: FCMGraphTraverser, contextDf: DataFrame, dataPathHandlers: List[DataPathHandler]): DataFrame = { + val groupedAggregationNodeMap = groupSWANodes(nodes) + val swaNodeIdToNode = graphTraverser.nodes.filter(node => node.isAggregation).map(node => node.getAggregation.getId() -> node).toMap + val featureColumnFormatsMap = graphTraverser.featureColumnFormatsMap + val defaultConverter = getDefaultConverter(nodes) + val featureTypeConfigs = getFeatureTypeConfigsMap(nodes) + + var df: DataFrame = contextDf + + // We sort the group of nodes in ascending order. This is because we want to join the + // smallest group of features first to reduce shuffle partitions. + val processedState = Array.fill[VisitedState](graphTraverser.nodes.length)(NOT_VISITED) + groupedAggregationNodeMap.values.toList.sortBy(p => p.size).reverse.map(listOfnodeIds => { + // We can take any node from this group as they have been grouped by the same label data, keys, and timestamp column + val node = swaNodeIdToNode(listOfnodeIds.head) + if (processedState(node.getAggregation.getId()) != VISITED) { + val labelData = getLabelData(node.getAggregation, graphTraverser.timeConfigSettings.timeConfigSettings, df, + graphTraverser.nodeIdToDataframeAndColumnMetadataMap) + val featureDataSet = getFactDataSet(swaNodeIdToNode, groupedAggregationNodeMap.toMap, + node.getAggregation, graphTraverser.nodeIdToDataframeAndColumnMetadataMap, + featureColumnFormatsMap, + graphTraverser.timeConfigSettings.timeConfigSettings, + graphTraverser.timeConfigSettings.featuresToTimeDelayMap, + graphTraverser.nodeIdToFeatureName) + df = SlidingWindowJoin.join(labelData, featureDataSet) + val allSwaFeatures = groupedAggregationNodeMap(node.getAggregation.getId) + // Mark all the nodes evaluated at this stage as visited. + allSwaFeatures.map(nId => { + val featureName = graphTraverser.nodeIdToFeatureName(nId) + // Convert to FDS before applying default values + df = SlidingWindowFeatureUtils.convertSWADFToFDS(df, Set(featureName), featureColumnFormatsMap.toMap, featureTypeConfigs).df + // Mark feature as converted to FDS + featureColumnFormatsMap(featureName) = FeatureColumnFormat.FDS_TENSOR + df = substituteDefaults(df, Seq(featureName), defaultConverter, featureTypeConfigs, graphTraverser.ss) + // NOTE: This appending of a dummy column is CRUCIAL to forcing the RDD of the df to have the appropriate schema. + // Same behavior is present in feathr but feathr unintentionally resolves it by using internal naming for features + // and only converting to use the real feature name at the end. This step in theory does nothing at all to the data + // but somehow it affects the schema of the RDD. + df = df.withColumnRenamed(featureName, featureName + "__dummy__") + df = df.withColumn(featureName, col(featureName + "__dummy__")) + df = df.drop(featureName + "__dummy__") + graphTraverser.nodeIdToDataframeAndColumnMetadataMap(nId) = + DataframeAndColumnMetadata(df, Seq.empty, Some(featureName)) // Key column for SWA feature is not needed in node context. + processedState(nId) = VISITED + }) + } + }) + df + } + + override def evaluate(node: AnyNode, graphTraverser: FCMGraphTraverser, contextDf: DataFrame, dataPathHandlers: List[DataPathHandler]): DataFrame = { + batchEvaluate(Seq(node), graphTraverser, contextDf, dataPathHandlers: List[DataPathHandler]) + } +} + diff --git a/feathr-impl/src/main/scala/com/linkedin/feathr/offline/evaluator/datasource/DataSourceNodeEvaluator.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/evaluator/datasource/DataSourceNodeEvaluator.scala new file mode 100644 index 000000000..9b11444ed --- /dev/null +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/evaluator/datasource/DataSourceNodeEvaluator.scala @@ -0,0 +1,219 @@ +package com.linkedin.feathr.offline.evaluator.datasource + +import com.linkedin.feathr.common.exception.{ErrorLabel, FeathrConfigException} +import com.linkedin.feathr.common.{AnchorExtractor, DateTimeResolution} +import com.linkedin.feathr.compute.{AnyNode, DataSourceType, KeyExpressionType} +import com.linkedin.feathr.core.config.producer.common.KeyListExtractor +import com.linkedin.feathr.offline.client.plugins.{AnchorExtractorAdaptor, FeathrUdfPluginContext, SourceKeyExtractorAdaptor} +import com.linkedin.feathr.offline.config.ConfigLoaderUtils +import com.linkedin.feathr.offline.evaluator.NodeEvaluator +import com.linkedin.feathr.offline.graph.{DataframeAndColumnMetadata, FCMGraphTraverser} +import com.linkedin.feathr.offline.source.{DataSource, SourceFormatType, TimeWindowParams} +import com.linkedin.feathr.offline.source.accessor.{DataPathHandler, DataSourceAccessor} +import com.linkedin.feathr.offline.source.dataloader.DataLoaderHandler +import com.linkedin.feathr.offline.source.pathutil.{PathChecker, TimeBasedHdfsPathAnalyzer} +import com.linkedin.feathr.offline.swa.SlidingWindowFeatureUtils.{TIMESTAMP_PARTITION_COLUMN, constructTimeStampExpr} +import com.linkedin.feathr.offline.util.datetime.{DateTimeInterval, OfflineDateTimeUtils} +import com.linkedin.feathr.sparkcommon.SourceKeyExtractor +import org.apache.log4j.Logger +import org.apache.spark.sql.{DataFrame, SparkSession} + +import java.time.Duration +import scala.collection.JavaConverters.asScalaBufferConverter +import scala.collection.mutable + +/** + * Node evaluator class for data source nodes. We have one private function per data source node type which are responsible + * for handling the 3 different data source types we support: CONTEXT, EVENT, and TABLE. + */ +object DataSourceNodeEvaluator extends NodeEvaluator{ + val log = Logger.getLogger(getClass) + /** + * Process datasource node of type CONTEXT but with no concrete key (non-passthrough feature context nodes). + * @param contextDataFrame + * @param dataSource + * @return + */ + private def processContextNode(contextDataFrame: DataFrame, dataSource: com.linkedin.feathr.compute.DataSource): DataframeAndColumnMetadata = { + // This is the feature column being extracted + val colName = dataSource.getExternalSourceRef + DataframeAndColumnMetadata(contextDataFrame, Seq(colName)) + } + + /** + * Process an event node. Event nodes represent SWA data sources. Here we load in the appropriate time range for the datasource + * given the time parameters. + * @param ss Spark session + * @param dataSourceNode Event node + * @param timeRange Optional time range to load in for data source. + * @return DataframeAndColumnMetadata with df loaded + */ + private def processEventNode(ss: SparkSession, dataSourceNode: com.linkedin.feathr.compute.DataSource, + timeRange: Option[DateTimeInterval], dataPathHandlers: List[DataPathHandler]): DataframeAndColumnMetadata = { + assert(dataSourceNode.hasConcreteKey) + assert(dataSourceNode.getConcreteKey.getKey.asScala.nonEmpty) + val path = dataSourceNode.getExternalSourceRef // We are using ExternalSourceRef for way too many things at this point. + + // Augment time information also here. Table node should not have time info? + val source = com.linkedin.feathr.offline.source.DataSource(path, SourceFormatType.TIME_SERIES_PATH, if (dataSourceNode.hasTimestampColumnInfo) { + Some(TimeWindowParams(dataSourceNode.getTimestampColumnInfo().getExpression(), + dataSourceNode.getTimestampColumnInfo().getFormat)) + } else None, if (dataSourceNode.hasFilePartitionFormat) { + Some(dataSourceNode.getFilePartitionFormat) + } else None) + + val timeWindowParam = if (dataSourceNode.hasTimestampColumnInfo) { + TimeWindowParams(dataSourceNode.getTimestampColumnInfo().getExpression, dataSourceNode.getTimestampColumnInfo().getFormat) + } else { + TimeWindowParams(TIMESTAMP_PARTITION_COLUMN, "epoch") + } + val timeStampExpr = constructTimeStampExpr(timeWindowParam.timestampColumn, timeWindowParam.timestampColumnFormat) + val needTimestampColumn = if (dataSourceNode.hasTimestampColumnInfo) false else true + val dataSourceAccessor = DataSourceAccessor(ss, source, timeRange, None, failOnMissingPartition = false, needTimestampColumn, dataPathHandlers = dataPathHandlers) + val sourceDF = dataSourceAccessor.get() + val (df, keyExtractor, timestampExpr) = if (dataSourceNode.getKeyExpressionType == KeyExpressionType.UDF) { + val className = Class.forName(dataSourceNode.getKeyExpression()) + val keyExtractorClass = className.newInstance match { + case keyExtractorClass: SourceKeyExtractor => + keyExtractorClass + case _ => + FeathrUdfPluginContext.getRegisteredUdfAdaptor(className) match { + case Some(adaptor: SourceKeyExtractorAdaptor) => + adaptor.adaptUdf(className.getDeclaredConstructor().newInstance().asInstanceOf[AnyRef]) + case _ => + throw new UnsupportedOperationException("Unknown extractor type: " + className) + } + } + (keyExtractorClass.appendKeyColumns(sourceDF), keyExtractorClass.getKeyColumnNames(), timeStampExpr) + } else { + val featureKeys = ConfigLoaderUtils.javaListToSeqWithDeepCopy(KeyListExtractor.getInstance(). + extractFromHocon(dataSourceNode.getKeyExpression)).map(k => s"CAST (${k} AS string)") + (sourceDF, featureKeys, timeStampExpr) + } + + // Only for datasource node, we will append the timestampExpr with the key field. TODO - find a better way of doing this. + DataframeAndColumnMetadata(df, keyExtractor, None, Some(source), Some(timestampExpr)) + } + + /** + * Process table nodes. Table nodes represent HDFS sources with a fixed path and no time partition data. Here we load + * in the data specified in the data source node and apply key extractor logic here if there is one. + * @param ss Spark session + * @param dataSourceNode Table node + * @return DataframeAndColumnMetadata with source loaded into df + */ + private def processTableNode(ss: SparkSession, dataSourceNode: com.linkedin.feathr.compute.DataSource, dataPathHandlers: List[DataPathHandler]): DataframeAndColumnMetadata = { + assert(dataSourceNode.hasConcreteKey) + assert(dataSourceNode.getConcreteKey.getKey.asScala.nonEmpty) + val path = dataSourceNode.getExternalSourceRef // We are using ExternalSourceRef for way too many things at this point. + + // Augment time information also here. Table node should not have time info? + val dataSource = com.linkedin.feathr.offline.source.DataSource(path, SourceFormatType.FIXED_PATH) + val dataSourceAccessor = DataSourceAccessor(ss, dataSource, None, None, failOnMissingPartition = false, dataPathHandlers = dataPathHandlers) + val sourceDF = dataSourceAccessor.get() + val (df, keyExtractor) = if (dataSourceNode.getKeyExpressionType == KeyExpressionType.UDF) { + val className = Class.forName(dataSourceNode.getKeyExpression()) + className.newInstance match { + case keyExtractorClass: SourceKeyExtractor => + val updatedDf = keyExtractorClass.appendKeyColumns(sourceDF) + (updatedDf, keyExtractorClass.getKeyColumnNames()) + case _: AnchorExtractor[_] => + // key will be evaluated at the time of anchor evaluation. + (sourceDF, Seq()) + case _ => + val x = FeathrUdfPluginContext.getRegisteredUdfAdaptor(className) + log.info("x is " + x + " and x type is " + x.getClass) + FeathrUdfPluginContext.getRegisteredUdfAdaptor(className) match { + case Some(adaptor: SourceKeyExtractorAdaptor) => + val keyExtractor = adaptor.adaptUdf(className.getDeclaredConstructor().newInstance().asInstanceOf[AnyRef]) + val updatedDf = keyExtractor.appendKeyColumns(sourceDF) + (updatedDf, keyExtractor.getKeyColumnNames()) + case Some(adaptor: AnchorExtractorAdaptor) => + (sourceDF, Seq()) + case _ => + throw new UnsupportedOperationException("Unknown extractor type: " + className + " FeathrUdfPluginContext" + + ".getRegisteredUdfAdaptor(className) is " + FeathrUdfPluginContext.getRegisteredUdfAdaptor(className) + "and type is " + x.get.isInstanceOf[AnchorExtractorAdaptor]) + } + } + } else { + val featureKeys = ConfigLoaderUtils.javaListToSeqWithDeepCopy(KeyListExtractor.getInstance().extractFromHocon(dataSourceNode.getKeyExpression())) + (sourceDF, featureKeys) + } + + DataframeAndColumnMetadata(df, keyExtractor, dataSource = Some(dataSource)) + } + + private def getOptimizedDurationMap(nodes: Seq[AnyNode]): Map[String, Duration] = { + val allSWANodes = nodes.filter(node => node.getAggregation != null) + // Create a map from SWA's event node to window duration in order to compute event node. + val swaDurationMap = allSWANodes.map(node => node.getAggregation.getInput.getId() -> Duration.parse(node.getAggregation.getFunction.getParameters + .get("window_size"))).toMap + val allEventSourceNodes = nodes.filter(node => node.isDataSource && node.getDataSource.getSourceType() == DataSourceType.EVENT) + val pathToDurationMap = mutable.HashMap.empty[String, Duration] + allEventSourceNodes.map(node => { + val sourcePath = node.getDataSource.getExternalSourceRef + if (!pathToDurationMap.contains(sourcePath)) { + pathToDurationMap.put(sourcePath, swaDurationMap(node.getDataSource.getId)) + } else { + val duration = pathToDurationMap(sourcePath) + if (duration.toHours < swaDurationMap(node.getDataSource.getId()).toHours) pathToDurationMap.put(sourcePath, swaDurationMap(node.getDataSource.getId)) + } + }) + pathToDurationMap.toMap + } + + /** + * Evaluate a single data source node according to the datasource type and return the context df. + * In this case only the graphTraverser's nodeIdToDataframeAndColumnMetadataMap is updated for the datasource node evaluation and the context df + * is not modified. Note that we don't process passthrough features at this point. + * + * @param graphTraverser FCMGraphTraverser + * @param contextDf Context df + * @return DataFrame + */ + override def evaluate(node: AnyNode, graphTraverser: FCMGraphTraverser, contextDf: DataFrame, dataPathHandlers: List[DataPathHandler]): DataFrame = { + val dataSource = node.getDataSource + val nodeId = node.getDataSource.getId + dataSource.getSourceType match { + case DataSourceType.CONTEXT => + if (dataSource.hasConcreteKey) { + val key = dataSource.getKeyExpression + val df = contextDf + graphTraverser.nodeIdToDataframeAndColumnMetadataMap(nodeId) = DataframeAndColumnMetadata(df, Seq(key)) + } else { + graphTraverser.nodeIdToDataframeAndColumnMetadataMap(nodeId) = processContextNode(contextDf, dataSource) + } + case DataSourceType.UPDATE => + graphTraverser.nodeIdToDataframeAndColumnMetadataMap(nodeId) = processTableNode(graphTraverser.ss, dataSource, dataPathHandlers: List[DataPathHandler]) + case DataSourceType.EVENT => + val dataLoaderHandlers: List[DataLoaderHandler] = dataPathHandlers.map(_.dataLoaderHandler) + val pathChecker = PathChecker(graphTraverser.ss, dataLoaderHandlers = dataLoaderHandlers) + val pathAnalyzer = new TimeBasedHdfsPathAnalyzer(pathChecker, dataLoaderHandlers = dataLoaderHandlers) + val pathInfo = pathAnalyzer.analyze(node.getDataSource.getExternalSourceRef) + val adjustedObsTimeRange = if (pathInfo.dateTimeResolution == DateTimeResolution.DAILY) + { + graphTraverser.timeConfigSettings.obsTimeRange.adjustWithDateTimeResolution(DateTimeResolution.DAILY) + } else graphTraverser.timeConfigSettings.obsTimeRange + + val eventPathToDurationMap = getOptimizedDurationMap(graphTraverser.nodes) + val duration = eventPathToDurationMap(node.getDataSource.getExternalSourceRef()) + if (graphTraverser.timeConfigSettings.timeConfigSettings.isEmpty || graphTraverser.timeConfigSettings.timeConfigSettings.get.joinTimeSetting.isEmpty) { + throw new FeathrConfigException( + ErrorLabel.FEATHR_USER_ERROR, + "joinTimeSettings section is not defined in join config," + + " cannot perform window aggregation operation") + } + + val adjustedTimeRange = OfflineDateTimeUtils.getFactDataTimeRange(adjustedObsTimeRange, duration, + Array(graphTraverser.timeConfigSettings.timeConfigSettings.get.joinTimeSetting.get.simulateTimeDelay.getOrElse(Duration.ZERO))) + graphTraverser.nodeIdToDataframeAndColumnMetadataMap(node.getDataSource.getId) = + processEventNode(graphTraverser.ss, node.getDataSource, Some(adjustedTimeRange), dataPathHandlers: List[DataPathHandler]) + } + contextDf + } + + override def batchEvaluate(nodes: Seq[AnyNode], graphTraverser: FCMGraphTraverser, contextDf: DataFrame, dataPathHandlers: List[DataPathHandler]): DataFrame = { + nodes.foreach(evaluate(_, graphTraverser, contextDf, dataPathHandlers)) + contextDf + } +} diff --git a/feathr-impl/src/main/scala/com/linkedin/feathr/offline/evaluator/lookup/LookupNodeEvaluator.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/evaluator/lookup/LookupNodeEvaluator.scala new file mode 100644 index 000000000..b595ba5ab --- /dev/null +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/evaluator/lookup/LookupNodeEvaluator.scala @@ -0,0 +1,171 @@ +package com.linkedin.feathr.offline.evaluator.lookup + +import com.linkedin.feathr.common.FeatureValue +import com.linkedin.feathr.compute.{AnyNode, Lookup} +import com.linkedin.feathr.offline.PostTransformationUtil +import com.linkedin.feathr.offline.graph.{DataframeAndColumnMetadata, FCMGraphTraverser} +import com.linkedin.feathr.offline.derived.strategies.SeqJoinAggregator +import com.linkedin.feathr.offline.derived.strategies.SequentialJoinAsDerivation.getDefaultTransformation +import com.linkedin.feathr.offline.evaluator.NodeEvaluator +import com.linkedin.feathr.offline.graph.NodeUtils.getDefaultConverter +import com.linkedin.feathr.offline.join.algorithms.{JoinType, SequentialJoinConditionBuilder, SparkJoinWithJoinCondition} +import com.linkedin.feathr.offline.source.accessor.DataPathHandler +import com.linkedin.feathr.offline.transformation.MvelDefinition +import com.linkedin.feathr.offline.util.DataFrameSplitterMerger +import org.apache.spark.sql.{DataFrame, SparkSession} +import org.apache.spark.sql.functions.{col, lit, monotonically_increasing_id} + +import scala.collection.JavaConverters.asScalaBufferConverter + +/** + * LookupNodeEvaluator contains processLookupNode function needed to evaluate Lookup Nodes which represent seq join where we have an + * expansion feature which will be keyed on a base feature. + */ +object LookupNodeEvaluator extends NodeEvaluator { + /** + * Process look up node which represents seq join. The graph traverser is responsible for gathering the necessary info + * to complete the look up node processing and call processLookupNode. This function will perform the seq join where + * the expansion feature will be joined to the context df based on the base feature. + * @param lookupNode Lookup Node + * @param baseNode DataframeAndColumnMetadata of base feature node. + * @param baseKeyColumns Column name of base feature. + * @param expansionNode DataframeAndColumnMetadata of expansion feature node. + * @param contextDf Context df + * @param seqJoinFeatureName Seq join feature name + * @param seqJoinJoiner Seq join joiner with seq join spark join condition + * @param defaultValueMap Default values map to be used for default value substitution + * @param ss Spark session + * @return DataframeAndColumnMetadata + */ + def processLookupNode(lookupNode: Lookup, + baseNode: DataframeAndColumnMetadata, + baseKeyColumns: Seq[String], + expansionNode: DataframeAndColumnMetadata, + contextDf: DataFrame, + seqJoinFeatureName: String, + seqJoinJoiner: SparkJoinWithJoinCondition, + defaultValueMap: Map[String, FeatureValue], + ss: SparkSession): DataframeAndColumnMetadata = { + // Get only required expansion features + val expansionFeatureName = expansionNode.featureColumn.get + val expansionNodeCols = expansionNode.keyExpression ++ Seq(expansionNode.featureColumn.get) + val expansionNodeDF = expansionNode.df.select(expansionNodeCols.map(col): _*) + // rename columns to know which columns are to be dropped + val expansionNodeRenamedCols = expansionNodeDF.columns.map(c => "__expansion__" + c).toSeq + val expansionNodeDfWithRenamedCols = expansionNodeDF.toDF(expansionNodeRenamedCols: _*) + + // coerce left join keys before joining base and expansion features + val left: DataFrame = PostTransformationUtil.transformFeatures(Seq((baseNode.featureColumn.get, baseNode.featureColumn.get)), contextDf, + Map.empty[String, MvelDefinition], getDefaultTransformation, None) + + // Partition base feature (left) side of the join based on null values. This is an optimization so we don't waste + // time joining nulls from the left df. + val (coercedBaseDfWithNoNull, coercedBaseDfWithNull) = DataFrameSplitterMerger.splitOnNull(left, baseNode.featureColumn.get) + + val groupByColumn = "__frame_seq_join_group_by_id" + /* We group by the monotonically_increasing_id to ensure we do not lose any of the observation data. + * This is essentially grouping by all the columns in the left table + * Note: we cannot add the monotonically_increasing_id before DataFrameSplitterMerger.splitOnNull. + * the implementation of monotonically_increasing_id is non-deterministic because its result depends on partition IDs. + * and it can generate duplicate ids between the withNoNull and WithNull part. + * see: https://godatadriven.com/blog/spark-surprises-for-the-uninitiated + */ + val leftWithUidDF = coercedBaseDfWithNoNull.withColumn(groupByColumn, monotonically_increasing_id) + val (adjustedLeftJoinKey, explodedLeft) = SeqJoinAggregator.explodeLeftJoinKey(ss, leftWithUidDF, baseKeyColumns, seqJoinFeatureName) + + // join base feature's results with expansion feature's results + val intermediateResult = seqJoinJoiner.join(adjustedLeftJoinKey, explodedLeft, + expansionNode.keyExpression.map(c => "__expansion__" + c), expansionNodeDfWithRenamedCols, JoinType.left_outer) + val producedFeatureName = "__expansion__" + expansionFeatureName + + /* + * Substitute defaults. The Sequential Join inherits the default values from the expansion feature definition. + * This step is done before applying aggregations becaUSE the default values should be factored in. + */ + val expansionFeatureDefaultValue = defaultValueMap.get(expansionFeatureName) + val intermediateResultWithDefault = + SeqJoinAggregator.substituteDefaultValuesForSeqJoinFeature(intermediateResult, producedFeatureName, expansionFeatureDefaultValue, ss) + + // apply aggregation to non-null part + val aggregationType = lookupNode.getAggregation + val aggDf = SeqJoinAggregator.applyAggregationFunction( + seqJoinFeatureName, producedFeatureName, intermediateResultWithDefault, aggregationType, groupByColumn) + + // Similarly, substitute the default values and apply aggregation function to the null part. + val coercedBaseDfWithNullWithDefault = SeqJoinAggregator.substituteDefaultValuesForSeqJoinFeature( + coercedBaseDfWithNull.withColumn(producedFeatureName, lit(null).cast(intermediateResult.schema(producedFeatureName).dataType)), + producedFeatureName, + expansionFeatureDefaultValue, + ss) + val coercedBaseDfWithNullWithAgg = SeqJoinAggregator.applyAggregationFunction( + seqJoinFeatureName, + producedFeatureName, + coercedBaseDfWithNullWithDefault.withColumn(groupByColumn, monotonically_increasing_id), + aggregationType, + groupByColumn) + + // Union the rows that participated in the join and the rows with nulls + val finalRes = DataFrameSplitterMerger.merge(aggDf, coercedBaseDfWithNullWithAgg) + + val resWithDroppedCols = finalRes.drop(expansionNode.keyExpression.map(c => "__expansion__" + c): _*) + .drop("__base__" + baseNode.featureColumn.get) + val finalResAfterDroppingCols = resWithDroppedCols.withColumnRenamed(producedFeatureName, seqJoinFeatureName) + + DataframeAndColumnMetadata(finalResAfterDroppingCols, baseNode.keyExpression.map(x => x.split("__").last), Some(seqJoinFeatureName)) + } + + /** + * Given a node, return its concrete keys as a Seq[Integer] + * @param node + * @return + */ + private def getLookupNodeKeys(node: AnyNode): Seq[Integer] = { + node match { + case n if n.isLookup => n.getLookup.getConcreteKey.getKey.asScala + case n if n.isDataSource => if (n.getDataSource.hasConcreteKey) n.getDataSource.getConcreteKey.getKey().asScala else null + case n if n.isTransformation => n.getTransformation.getConcreteKey.getKey.asScala + } + } + + /** + * Evaluate a lookup node and set the node's DataframeAndColumnMetadata in the graph traverser to be the output of the node evaluation. Returns + * the output of lookup joined to the context df. + * + * @param node Lookup Node to evaluate + * @param graphTraverser FCMGraphTraverser + * @param contextDf Context df + * @return DataFrame + */ + override def evaluate(node: AnyNode, graphTraverser: FCMGraphTraverser, contextDf: DataFrame, dataPathHandlers: List[DataPathHandler]): DataFrame = { + val lookUpNode = node.getLookup + // Assume there is only one lookup key that is a node reference. In the future this may not be true and will have to be changed. + // NOTE: We currently assume there is only 1 base node because that is what is supported currently in the feathr HOCON config + // there is no such constraint on the graph model. TODO: Modify the implementation of lookup such that multiple base nodes + // are supported. + val baseNodeRef = lookUpNode.getLookupKey.asScala.find(x => x.isNodeReference).get.getNodeReference + val baseNode = graphTraverser.nodeIdToDataframeAndColumnMetadataMap(baseNodeRef.getId) + val baseKeyColumns = getLookupNodeKeys(graphTraverser.nodes(lookUpNode.getLookupNode)) + .flatMap(x => if (graphTraverser.nodeIdToDataframeAndColumnMetadataMap(x).featureColumn.isDefined) { + Seq(graphTraverser.nodeIdToDataframeAndColumnMetadataMap(x).featureColumn.get) + } else { + graphTraverser.nodeIdToDataframeAndColumnMetadataMap(x).keyExpression + }) + val expansionNodeId = lookUpNode.getLookupNode() + val expansionNode = graphTraverser.nodeIdToDataframeAndColumnMetadataMap(expansionNodeId) + val seqJoinFeatureName = graphTraverser.nodeIdToFeatureName(lookUpNode.getId) + + val expansionNodeDefaultConverter = getDefaultConverter(Seq(graphTraverser.nodes(expansionNodeId))) + val lookupNodeContext = LookupNodeEvaluator.processLookupNode(lookUpNode, baseNode, + baseKeyColumns, expansionNode, contextDf, seqJoinFeatureName, SparkJoinWithJoinCondition(SequentialJoinConditionBuilder), + expansionNodeDefaultConverter, graphTraverser.ss) + + // Update nodeIdToDataframeAndColumnMetadataMap and return new contextDf + graphTraverser.nodeIdToDataframeAndColumnMetadataMap(lookUpNode.getId) = lookupNodeContext + lookupNodeContext.df + } + + // Batch evaluate just calls single evaluate sequentially + override def batchEvaluate(nodes: Seq[AnyNode], graphTraverser: FCMGraphTraverser, contextDf: DataFrame, dataPathHandlers: List[DataPathHandler]): DataFrame = { + nodes.foldLeft(contextDf)((updatedContextDf, node) => evaluate(node, graphTraverser, updatedContextDf, dataPathHandlers)) + } +} diff --git a/feathr-impl/src/main/scala/com/linkedin/feathr/offline/evaluator/transformation/AnchorMvelOperator.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/evaluator/transformation/AnchorMvelOperator.scala new file mode 100644 index 000000000..0552cc829 --- /dev/null +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/evaluator/transformation/AnchorMvelOperator.scala @@ -0,0 +1,64 @@ +package com.linkedin.feathr.offline.evaluator.transformation + +import com.linkedin.feathr.compute.Transformation +import com.linkedin.feathr.offline.anchored.anchorExtractor.SimpleConfigurableAnchorExtractor +import com.linkedin.feathr.offline.anchored.keyExtractor.MVELSourceKeyExtractor +import com.linkedin.feathr.offline.config.MVELFeatureDefinition +import com.linkedin.feathr.offline.evaluator.transformation.TransformationOperatorUtils.{dropAndRenameCols, joinResultToContextDfAndApplyDefaults} +import com.linkedin.feathr.offline.graph.FCMGraphTraverser +import com.linkedin.feathr.offline.graph.NodeUtils.getFeatureTypeConfigsMapForTransformationNodes +import com.linkedin.feathr.offline.job.FeatureTransformation.{getFeatureKeyColumnNames} +import com.linkedin.feathr.offline.source.accessor.DataPathHandler +import com.linkedin.feathr.offline.transformation.DataFrameBasedRowEvaluator +import org.apache.spark.sql.DataFrame + +object AnchorMVELOperator extends TransformationOperator { + + /** + * Compute the anchor MVEL transformation and return the result df and output key columns. + * @param nodes + * @param graphTraverser + * @return (DataFrame, Seq[String]) + */ + def computeMVELResult(nodes: Seq[Transformation], graphTraverser: FCMGraphTraverser, contextDf: DataFrame, + appendKeyColumns: Boolean): (DataFrame, Seq[String]) = { + // All nodes in MVEL anchor group will have the same key expression and input node so we can just use the head. + val inputNodeId = nodes.head.getInputs.get(0).getId // Anchor operators should only have a single input + val keySeq = graphTraverser.nodeIdToDataframeAndColumnMetadataMap(inputNodeId).keyExpression + val inputDf = if (appendKeyColumns) graphTraverser.nodeIdToDataframeAndColumnMetadataMap(inputNodeId).df else contextDf + + val featureTypeConfigs = getFeatureTypeConfigsMapForTransformationNodes(nodes) + val featureNameToMvelExpr = nodes.map(node => graphTraverser.nodeIdToFeatureName(node.getId) -> MVELFeatureDefinition( + node.getFunction.getParameters.get("expression"), featureTypeConfigs.get(node.getFeatureName))).toMap + val featureNamesInBatch = featureNameToMvelExpr.keys.toSeq + val mvelExtractor = new SimpleConfigurableAnchorExtractor(keySeq, featureNameToMvelExpr) + + // Here we make the assumption that the key expression is of the same type of operator as the feature definition and + // evaluate and append the key columns. Same logic is repeated for SQL expressions too + val mvelKeyExtractor = new MVELSourceKeyExtractor(mvelExtractor) + val withKeyColumnDF = if (appendKeyColumns) mvelKeyExtractor.appendKeyColumns(inputDf) else inputDf + val outputJoinKeyColumnNames = getFeatureKeyColumnNames(mvelKeyExtractor, withKeyColumnDF) + val transformationResult = DataFrameBasedRowEvaluator.transform(mvelExtractor, withKeyColumnDF, + featureNamesInBatch.map((_, " ")), featureTypeConfigs, graphTraverser.mvelExpressionContext).df + (transformationResult, outputJoinKeyColumnNames) + } + + /** + * Operator for batch anchor MVEL transformations. Given context df and a grouped set of MVEL transformation nodes, + * perform the MVEL transformations and return the context df with all the MVEL features joined. + * @param nodes Seq of nodes with MVEL anchor as operator + * @param graphTraverser FCMGraphTraverser + * @param contextDf Context df + * @return Dataframe + */ + override def batchCompute(nodes: Seq[Transformation], graphTraverser: FCMGraphTraverser, contextDf: DataFrame, dataPathHandlers: List[DataPathHandler]): DataFrame = { + val (transformationResult, outputJoinKeyColumnNames) = computeMVELResult(nodes, graphTraverser, contextDf, appendKeyColumns = true) + val featureNamesInBatch = nodes.map(node => graphTraverser.nodeIdToFeatureName(node.getId)) + val (prunedResult, keyColumns) = dropAndRenameCols(transformationResult, outputJoinKeyColumnNames, featureNamesInBatch) + joinResultToContextDfAndApplyDefaults(nodes, graphTraverser, prunedResult, keyColumns, contextDf) + } + + override def compute(node: Transformation, graphTraverser: FCMGraphTraverser, contextDf: DataFrame, dataPathHandlers: List[DataPathHandler]): DataFrame = { + batchCompute(Seq(node), graphTraverser, contextDf, dataPathHandlers) + } +} diff --git a/feathr-impl/src/main/scala/com/linkedin/feathr/offline/evaluator/transformation/AnchorSQLOperator.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/evaluator/transformation/AnchorSQLOperator.scala new file mode 100644 index 000000000..1827369e0 --- /dev/null +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/evaluator/transformation/AnchorSQLOperator.scala @@ -0,0 +1,80 @@ +package com.linkedin.feathr.offline.evaluator.transformation + +import com.linkedin.feathr.common.FeatureTypeConfig +import com.linkedin.feathr.compute.Transformation +import com.linkedin.feathr.offline.anchored.anchorExtractor.{SQLConfigurableAnchorExtractor, SQLKeys} +import com.linkedin.feathr.offline.anchored.keyExtractor.SQLSourceKeyExtractor +import com.linkedin.feathr.offline.config.SQLFeatureDefinition +import com.linkedin.feathr.offline.evaluator.transformation.TransformationOperatorUtils.{createFeatureDF, dropAndRenameCols, joinResultToContextDfAndApplyDefaults} +import com.linkedin.feathr.offline.graph.FCMGraphTraverser +import com.linkedin.feathr.offline.graph.NodeUtils.getFeatureTypeConfigsMapForTransformationNodes +import com.linkedin.feathr.offline.job.FeatureTransformation.getFeatureKeyColumnNames +import com.linkedin.feathr.offline.source.accessor.DataPathHandler +import com.linkedin.feathr.offline.transformation.FeatureColumnFormat +import com.linkedin.feathr.offline.util.FeaturizedDatasetUtils +import org.apache.spark.sql.DataFrame + +object AnchorSQLOperator extends TransformationOperator { + private val USER_FACING_MULTI_DIM_FDS_TENSOR_UDF_NAME = "FDSExtract" + + /** + * Compute the SQL transformation and return the result dataframe and key columns. + * @param nodes + * @param graphTraverser + * @return + */ + def computeSQLResult(nodes: Seq[Transformation], graphTraverser: FCMGraphTraverser, contextDf: DataFrame, + appendKeyColumns: Boolean): (DataFrame, Seq[String]) = { + // All nodes in SQL anchor group will have the same key expression and input node so we can just use the head. + val inputNodeId = nodes.head.getInputs.get(0).getId // Anchor operators should only have a single input + val keySeq = graphTraverser.nodeIdToDataframeAndColumnMetadataMap(inputNodeId).keyExpression + val inputDf = if (appendKeyColumns) graphTraverser.nodeIdToDataframeAndColumnMetadataMap(inputNodeId).df else contextDf + + val featureTypeConfigs = getFeatureTypeConfigsMapForTransformationNodes(nodes) + val featureNameToSqlExpr = nodes.map(node => graphTraverser.nodeIdToFeatureName(node.getId) -> SQLFeatureDefinition( + node.getFunction.getParameters.get("expression"))).toMap + val featureNamesInBatch = featureNameToSqlExpr.keys.toSeq + val featureSchemas = featureNamesInBatch + .map(featureName => { + // Currently assumes that tensor type is undefined + val tensorType = FeaturizedDatasetUtils.lookupTensorTypeForFeatureRef(featureName, None, + featureTypeConfigs.getOrElse(featureName, FeatureTypeConfig.UNDEFINED_TYPE_CONFIG)) + val schema = FeaturizedDatasetUtils.tensorTypeToDataFrameSchema(tensorType) + featureName -> schema + }) + .toMap + val sqlExtractor = new SQLConfigurableAnchorExtractor(SQLKeys(keySeq), featureNameToSqlExpr) + + // Apply SQL transformation and append key columns to inputDf. + val transformedCols = sqlExtractor.getTensorFeatures(inputDf, featureSchemas) + val sqlKeyExtractor = new SQLSourceKeyExtractor(keySeq) + val withKeyColumnDF = if (appendKeyColumns) sqlKeyExtractor.appendKeyColumns(inputDf) else inputDf + val withFeaturesDf = createFeatureDF(withKeyColumnDF, transformedCols.keys.toSeq) + val outputJoinKeyColumnNames = getFeatureKeyColumnNames(sqlKeyExtractor, withFeaturesDf) + + // Mark as FDS format if it is the FDSExtract SQL function + featureNameToSqlExpr.filter(ele => ele._2.featureExpr.contains(USER_FACING_MULTI_DIM_FDS_TENSOR_UDF_NAME)) + .foreach(nameToSql => graphTraverser.featureColumnFormatsMap(nameToSql._1) = FeatureColumnFormat.FDS_TENSOR) + + (withFeaturesDf, outputJoinKeyColumnNames) + } + /** + * Operator for batch anchor SQL transformations. Given context df and a grouped set of SQL transformation nodes, + * perform the SQL transformations and return the context df with all the SQL features joined. + * @param nodes Seq of nodes with SQL anchor as operator + * @param graphTraverser FCMGraphTraverser + * @param contextDf Context df + * @return Dataframe + */ + override def batchCompute(nodes: Seq[Transformation], graphTraverser: FCMGraphTraverser, contextDf: DataFrame, + dataPathHandlers: List[DataPathHandler]): DataFrame = { + val (transformationResult, outputJoinKeyColumnNames) = computeSQLResult(nodes, graphTraverser, contextDf, appendKeyColumns = true) + val featureNamesInBatch = nodes.map(node => graphTraverser.nodeIdToFeatureName(node.getId)) + val (prunedResult, keyColumns) = dropAndRenameCols(transformationResult, outputJoinKeyColumnNames, featureNamesInBatch) + joinResultToContextDfAndApplyDefaults(nodes, graphTraverser, prunedResult, keyColumns, contextDf) + } + + override def compute(node: Transformation, graphTraverser: FCMGraphTraverser, contextDf: DataFrame, dataPathHandlers: List[DataPathHandler]): DataFrame = { + batchCompute(Seq(node), graphTraverser, contextDf, dataPathHandlers) + } +} diff --git a/feathr-impl/src/main/scala/com/linkedin/feathr/offline/evaluator/transformation/AnchorUDFOperator.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/evaluator/transformation/AnchorUDFOperator.scala new file mode 100644 index 000000000..f2921aac2 --- /dev/null +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/evaluator/transformation/AnchorUDFOperator.scala @@ -0,0 +1,165 @@ +package com.linkedin.feathr.offline.evaluator.transformation + +import com.linkedin.feathr.common.{AnchorExtractor, AnchorExtractorBase, CanConvertToAvroRDD, FeatureTypeConfig} +import com.linkedin.feathr.compute.Transformation +import com.linkedin.feathr.offline.anchored.anchorExtractor.SQLConfigurableAnchorExtractor +import com.linkedin.feathr.offline.anchored.keyExtractor.{SQLSourceKeyExtractor, SpecificRecordSourceKeyExtractor} +import com.linkedin.feathr.offline.client.plugins.{AnchorExtractorAdaptor, FeathrUdfPluginContext, SimpleAnchorExtractorSparkAdaptor} +import com.linkedin.feathr.offline.evaluator.transformation.TransformationOperatorUtils.{createFeatureDF, dropAndRenameCols, joinResultToContextDfAndApplyDefaults} +import com.linkedin.feathr.offline.graph.FCMGraphTraverser +import com.linkedin.feathr.offline.graph.NodeUtils.getFeatureTypeConfigsMapForTransformationNodes +import com.linkedin.feathr.offline.job.FeatureTransformation.{applyRowBasedTransformOnRdd, getFeatureKeyColumnNames} +import com.linkedin.feathr.offline.source.accessor.{DataPathHandler, DataSourceAccessor, NonTimeBasedDataSourceAccessor} +import com.linkedin.feathr.offline.transformation.FeatureColumnFormat +import com.linkedin.feathr.offline.transformation.FeatureColumnFormat.FeatureColumnFormat +import com.linkedin.feathr.offline.util.{FeaturizedDatasetUtils, SourceUtils} +import com.linkedin.feathr.sparkcommon.{FDSExtractor, GenericAnchorExtractorSpark, SimpleAnchorExtractorSpark} +import org.apache.spark.rdd.RDD +import org.apache.spark.sql.{Column, DataFrame} + +object AnchorUDFOperator extends TransformationOperator { + private val FDSExtractorUserFacingName = "com.linkedin.feathr.sparkcommon.FDSExtractor" + /** + * Compute the anchor UDF transformation and return the result df and output key columns. + * @param nodes + * @param graphTraverser + * @return (DataFrame, Seq[String]) + */ + def computeUDFResult(nodes: Seq[Transformation], graphTraverser: FCMGraphTraverser, contextDf: DataFrame, + appendKeyColumns: Boolean, dataPathHandlers: List[DataPathHandler]): (DataFrame, Seq[String]) = { + // All nodes in UDF anchor group will have the same key expression and input node so we can just use the head. + val inputNodeId = nodes.head.getInputs.get(0).getId // Anchor operators should only have a single input + val keySeq = graphTraverser.nodeIdToDataframeAndColumnMetadataMap(inputNodeId).keyExpression + val inputDf = if (appendKeyColumns) graphTraverser.nodeIdToDataframeAndColumnMetadataMap(inputNodeId).df else contextDf + val featureTypeConfigs = getFeatureTypeConfigsMapForTransformationNodes(nodes) + + // Grab extractor class and create appropriate extractor. All extractors in batch should have the same class. + val className = nodes.head.getFunction.getParameters.get("class") + val featureNamesInBatch = nodes.map(node => graphTraverser.nodeIdToFeatureName(node.getId)) + val extractor = if (className.equals(FDSExtractorUserFacingName)) { // Support for FDSExtractor, which is a canned extractor. + new FDSExtractor(featureNamesInBatch.toSet) + } else { + Class.forName(className).newInstance + } + + val newExtractor = FeathrUdfPluginContext.getRegisteredUdfAdaptor(Class.forName(className)) match { + case Some(adaptor: SimpleAnchorExtractorSparkAdaptor) => + adaptor.adaptUdf(extractor.asInstanceOf[AnyRef]) + case Some(adaptor: AnchorExtractorAdaptor) => + adaptor.adaptUdf(extractor.asInstanceOf[AnyRef]) + case None => extractor + } + + val (withFeaturesDf, outputJoinKeyColumnNames) = newExtractor match { + case sparkExtractor: SimpleAnchorExtractorSpark => + // Note that for Spark UDFs we only support SQL keys. + print("in simpleanchorextractorspark = " + newExtractor) + val sqlKeyExtractor = new SQLSourceKeyExtractor(keySeq) + val withKeyColumnDF = if (appendKeyColumns) sqlKeyExtractor.appendKeyColumns(inputDf) else inputDf + val outputJoinKeyColumnNames = getFeatureKeyColumnNames(sqlKeyExtractor, withKeyColumnDF) + + val tensorizedFeatureColumns = sparkExtractor.getFeatures(inputDf, Map()) + val transformedColsAndFormats: Map[(String, Column), FeatureColumnFormat] = extractor match { + case extractor2: SQLConfigurableAnchorExtractor => + print("in SQLConfigurableAnchorExtractor = " + newExtractor) + // If instance of SQLConfigurableAnchorExtractor, get Tensor features + // Get DataFrame schema for tensor based on FML or inferred tensor type. + val featureSchemas = featureNamesInBatch.map(featureName => { + // Currently assumes that tensor type is undefined + val featureTypeConfig = featureTypeConfigs.getOrElse(featureName, FeatureTypeConfig.UNDEFINED_TYPE_CONFIG) + val tensorType = FeaturizedDatasetUtils.lookupTensorTypeForFeatureRef(featureName, None, featureTypeConfig) + val schema = FeaturizedDatasetUtils.tensorTypeToDataFrameSchema(tensorType) + featureName -> schema + }) + .toMap + extractor2.getTensorFeatures(inputDf, featureSchemas) + case _ => newExtractor match { + case extractor1: FDSExtractor => + // While using the FDS extractor, the feature columns are already in FDS format. + featureNamesInBatch.foreach(featureName => graphTraverser.featureColumnFormatsMap(featureName) = FeatureColumnFormat.FDS_TENSOR) + extractor1.transformAsColumns(inputDf).map(c => (c, FeatureColumnFormat.FDS_TENSOR)).toMap + case _ => if (tensorizedFeatureColumns.isEmpty) { + // If transform.getFeatures() returns empty Seq, then transform using transformAsColumns + sparkExtractor.transformAsColumns(inputDf).map(c => (c, FeatureColumnFormat.RAW)).toMap + } else { + // transform.getFeature() expects user to return FDS tensor + featureNamesInBatch.foreach(featureName => graphTraverser.featureColumnFormatsMap(featureName) = FeatureColumnFormat.FDS_TENSOR) + tensorizedFeatureColumns.map(c => (c, FeatureColumnFormat.FDS_TENSOR)).toMap + } + } + } + val transformedDF = createFeatureDF(withKeyColumnDF, transformedColsAndFormats.keys.toSeq) + (transformedDF, outputJoinKeyColumnNames) + case sparkExtractor: GenericAnchorExtractorSpark => + // Note that for Spark UDFs we only support SQL keys. + val sqlKeyExtractor = new SQLSourceKeyExtractor(keySeq) + val withKeyColumnDF = if (appendKeyColumns) sqlKeyExtractor.appendKeyColumns(inputDf) else inputDf + val outputJoinKeyColumnNames = getFeatureKeyColumnNames(sqlKeyExtractor, withKeyColumnDF) + + val transformedDF = sparkExtractor.transform(inputDf) + (transformedDF, outputJoinKeyColumnNames) + case _ => newExtractor match { + case rowBasedExtractor: AnchorExtractorBase[Any] => + // Note that for row based extractors we will be using MVEL source key extractor and row based extractor requires us + // to create a rdd so we can't just use the input df. + val userProvidedFeatureTypes = featureTypeConfigs map { case (key, value) => (key, value.getFeatureType) } + val dataSource = graphTraverser.nodeIdToDataframeAndColumnMetadataMap(nodes.head.getInputs.get(0).getId).dataSource.get + val expectDatumType = SourceUtils.getExpectDatumType(Seq(rowBasedExtractor)) + val dataSourceAccessor = DataSourceAccessor(graphTraverser.ss, dataSource, None, Some(expectDatumType), failOnMissingPartition = false, dataPathHandlers = dataPathHandlers) + val rdd = newExtractor.asInstanceOf[CanConvertToAvroRDD].convertToAvroRdd(dataSourceAccessor.asInstanceOf[NonTimeBasedDataSourceAccessor].get()) + val sourceKeyExtractors = nodes.map(node => { + val className = node.getFunction.getParameters.get("class") + val createdExtractor = FeathrUdfPluginContext.getRegisteredUdfAdaptor(Class.forName(className)) match { + case Some(adaptor: SimpleAnchorExtractorSparkAdaptor) => + adaptor.adaptUdf(extractor.asInstanceOf[AnyRef]) + case Some(adaptor: AnchorExtractorAdaptor) => + adaptor.adaptUdf(extractor.asInstanceOf[AnyRef]) + case None => extractor + } + new SpecificRecordSourceKeyExtractor(createdExtractor.asInstanceOf[AnchorExtractor[Any]], Seq.empty[String]) + }) + + val anchorExtractors = nodes.map(node => { + val className = node.getFunction.getParameters.get("class") + val createdExtractor = FeathrUdfPluginContext.getRegisteredUdfAdaptor(Class.forName(className)) match { + case Some(adaptor: SimpleAnchorExtractorSparkAdaptor) => + adaptor.adaptUdf(extractor.asInstanceOf[AnyRef]) + case Some(adaptor: AnchorExtractorAdaptor) => + adaptor.adaptUdf(extractor.asInstanceOf[AnyRef]) + case None => extractor + } + createdExtractor.asInstanceOf[AnchorExtractorBase[Any]] + }) + + val (transformedDf, keyNames) = applyRowBasedTransformOnRdd(userProvidedFeatureTypes, featureNamesInBatch, + rdd, + sourceKeyExtractors, + anchorExtractors, featureTypeConfigs) + (transformedDf, keyNames) + case _ => + throw new UnsupportedOperationException("Unknow extractor type : " + extractor + " and it's class is " + extractor.getClass) + } + } + (withFeaturesDf, outputJoinKeyColumnNames) + } + + /** + * Operator for batch anchor UDF transformations. Given context df and a grouped set of UDF transformation nodes, + * perform the UDF transformations and return the context df with all the UDF features joined. + * @param nodes Seq of nodes with UDF anchor as operator + * @param graphTraverser FCMGraphTraverser + * @param contextDf Context df + * @return Dataframe + */ + override def batchCompute(nodes: Seq[Transformation], graphTraverser: FCMGraphTraverser, contextDf: DataFrame, dataPathHandlers: List[DataPathHandler]): DataFrame = { + val (transformationResult, outputJoinKeyColumnNames) = computeUDFResult(nodes, graphTraverser, contextDf, appendKeyColumns = true, dataPathHandlers) + val featureNamesInBatch = nodes.map(node => graphTraverser.nodeIdToFeatureName(node.getId)) + val (prunedResult, keyColumns) = dropAndRenameCols(transformationResult, outputJoinKeyColumnNames, featureNamesInBatch) + joinResultToContextDfAndApplyDefaults(nodes, graphTraverser, prunedResult, keyColumns, contextDf) + } + + override def compute(node: Transformation, graphTraverser: FCMGraphTraverser, contextDf: DataFrame, dataPathHandlers: List[DataPathHandler]): DataFrame = { + batchCompute(Seq(node), graphTraverser, contextDf, dataPathHandlers) + } + +} diff --git a/feathr-impl/src/main/scala/com/linkedin/feathr/offline/evaluator/transformation/BaseDerivedFeatureOperator.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/evaluator/transformation/BaseDerivedFeatureOperator.scala new file mode 100644 index 000000000..dee6dfc92 --- /dev/null +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/evaluator/transformation/BaseDerivedFeatureOperator.scala @@ -0,0 +1,118 @@ +package com.linkedin.feathr.offline.evaluator.transformation + +import com.linkedin.feathr.common +import com.linkedin.feathr.common.{FeatureDerivationFunction, FeatureTypeConfig, FeatureTypes} +import com.linkedin.feathr.compute.{NodeReference, Transformation} +import com.linkedin.feathr.exception.{ErrorLabel, FrameFeatureTransformationException} +import com.linkedin.feathr.offline.derived.functions.{MvelFeatureDerivationFunction, SimpleMvelDerivationFunction} +import com.linkedin.feathr.offline.graph.FCMGraphTraverser +import com.linkedin.feathr.offline.graph.NodeUtils.{getFeatureTypeConfigsMap, getFeatureTypeConfigsMapForTransformationNodes} +import com.linkedin.feathr.offline.mvel.plugins.FeathrExpressionExecutionContext +import com.linkedin.feathr.offline.transformation.{FDSConversionUtils, FeatureColumnFormat} +import com.linkedin.feathr.offline.util.{CoercionUtilsScala, FeaturizedDatasetUtils} +import com.linkedin.feathr.offline.util.FeaturizedDatasetUtils.tensorTypeToDataFrameSchema +import org.apache.spark.sql.catalyst.encoders.RowEncoder +import org.apache.spark.sql.types.{StructField, StructType} +import org.apache.spark.sql.{DataFrame, Row} + +import scala.collection.JavaConverters.mapAsScalaMapConverter +import scala.collection.mutable + +/** + * BaseDerivedFeatureOperator contains the function applyDerivationFunction is used by the 4 different derived operators we support + * (OPERATOR_ID_DERIVED_MVEL, OPERATOR_ID_DERIVED_JAVA_UDF_FEATURE_EXTRACTOR, OPERATOR_ID_DERIVED_SPARK_SQL_FEATURE_EXTRACTOR, + * and OPERATOR_ID_EXTRACT_FROM_TUPLE) + * to apply their respective derivation functions to the context dataframe. Note that this function expects the columns which + * the derivation function requires as inputs to be joined to the contextDf + */ +object BaseDerivedFeatureOperator { + def applyDerivationFunction(node: Transformation, + derivationFunction: FeatureDerivationFunction, + graphTraverser: FCMGraphTraverser, + contextDf: DataFrame): DataFrame = { + val featureName = if (node.getFeatureName == null) graphTraverser.nodeIdToFeatureName(node.getId) else node.getFeatureName + // If the feature name is already in the contextDf, drop that column + val inputDf = if (contextDf.columns.contains(featureName)) { + contextDf.drop(featureName) + } else { + contextDf + } + + // Gather inputs from node + val inputs = node.getInputs + val inputFeatureNames = inputs.toArray.map(input => { + val inp = input.asInstanceOf[NodeReference] + graphTraverser.nodeIdToFeatureName(inp.getId) + }).sorted + val inputNodes = inputs.toArray.map(input => { + val inp = input.asInstanceOf[NodeReference] + graphTraverser.nodes(inp.getId) + }).toSeq + val inputFeatureTypeConfigs = getFeatureTypeConfigsMap(inputNodes) + + // Prepare schema values needed for computation. + val featureTypeConfigs = getFeatureTypeConfigsMapForTransformationNodes(Seq(node)) + val featureTypeConfig = featureTypeConfigs.getOrElse(featureName, new FeatureTypeConfig(FeatureTypes.UNSPECIFIED)) + val tensorType = FeaturizedDatasetUtils.lookupTensorTypeForNonFMLFeatureRef(featureName, FeatureTypes.UNSPECIFIED, featureTypeConfig) + val newSchema = tensorTypeToDataFrameSchema(tensorType) + val inputSchema = inputDf.schema + val mvelContext: Option[FeathrExpressionExecutionContext] = graphTraverser.mvelExpressionContext + val outputSchema = StructType(inputSchema.union(StructType(Seq(StructField(featureName, newSchema, nullable = true))))) + val encoder = RowEncoder(outputSchema) + val outputDf = inputDf.map(row => { + try { + val contextFeatureValues = mutable.Map.empty[String, common.FeatureValue] + inputFeatureNames.map(inputFeatureName => { + val featureTypeConfig = inputFeatureTypeConfigs.getOrElse(inputFeatureName, FeatureTypeConfig.UNDEFINED_TYPE_CONFIG) + val featureValue = CoercionUtilsScala.coerceFieldToFeatureValue(row, inputSchema, inputFeatureName, featureTypeConfig) + contextFeatureValues.put(inputFeatureName, featureValue) + } + ) + // Sort by input feature name to be consistent with how the derivation function is created. + val featureValues = contextFeatureValues.toSeq.sortBy(_._1).map(fv => Option(fv._2)) + val derivedFunc = derivationFunction match { + case derivedFunc: MvelFeatureDerivationFunction => + derivedFunc.mvelContext = mvelContext + derivedFunc + case func => func + } + val unlinkedOutput = derivedFunc.getFeatures(featureValues) + val featureType = featureTypeConfigs + .getOrElse(featureName, FeatureTypeConfig.UNDEFINED_TYPE_CONFIG).getFeatureType + val fdFeatureValue = unlinkedOutput.map(fv => { + if (fv.isDefined) { + if (featureType == FeatureTypes.TENSOR && !derivationFunction.isInstanceOf[SimpleMvelDerivationFunction]) { + // Convert to FDS directly when tensor type is specified + FDSConversionUtils.rawToFDSRow(fv.get.getAsTensorData, newSchema) + } else { + FDSConversionUtils.rawToFDSRow(fv.get.getAsTermVector.asScala, newSchema) + } + } else { + null + } + }) + Row.fromSeq(outputSchema.indices.map { i => { + if (i >= inputSchema.size) { + fdFeatureValue(i - inputSchema.size) + } else { + row.get(i) + } + } + }) + } catch { + case e: Exception => + throw new FrameFeatureTransformationException( + ErrorLabel.FEATHR_USER_ERROR, + s"Fail to calculate derived feature " + featureName, + e) + } + })(encoder) + + // Apply feature alias if there is one defined. + if (graphTraverser.nodeIdToFeatureName(node.getId) != node.getFeatureName) { + val featureAlias = graphTraverser.nodeIdToFeatureName(node.getId) + graphTraverser.featureColumnFormatsMap(featureAlias) = FeatureColumnFormat.RAW + outputDf.withColumnRenamed(featureName, featureAlias) + } else outputDf + } +} \ No newline at end of file diff --git a/feathr-impl/src/main/scala/com/linkedin/feathr/offline/evaluator/transformation/DeriveSimpleMVELOperator.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/evaluator/transformation/DeriveSimpleMVELOperator.scala new file mode 100644 index 000000000..cd3ace728 --- /dev/null +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/evaluator/transformation/DeriveSimpleMVELOperator.scala @@ -0,0 +1,32 @@ +package com.linkedin.feathr.offline.evaluator.transformation + +import com.linkedin.feathr.common.FeatureDerivationFunction +import com.linkedin.feathr.compute.Transformation +import com.linkedin.feathr.offline.config.PegasusRecordFeatureTypeConverter +import com.linkedin.feathr.offline.derived.functions.SimpleMvelDerivationFunction +import com.linkedin.feathr.offline.evaluator.transformation.BaseDerivedFeatureOperator.applyDerivationFunction +import com.linkedin.feathr.offline.evaluator.transformation.TransformationOperatorUtils.updateDataframeMapAndApplyDefaults +import com.linkedin.feathr.offline.graph.FCMGraphTraverser +import com.linkedin.feathr.offline.source.accessor.DataPathHandler +import org.apache.spark.sql.DataFrame + +/** + * Transformation operator for simple MVEL operator. + */ +object DerivedSimpleMVELOperator extends TransformationOperator { + + override def compute(node: Transformation, graphTraverser: FCMGraphTraverser, contextDf: DataFrame, dataPathHandlers: List[DataPathHandler]): DataFrame = { + val transformationFunction = node.getFunction + val featureName = if (node.getFeatureName == null) graphTraverser.nodeIdToFeatureName(node.getId) else node.getFeatureName + val featureTypeConfig = PegasusRecordFeatureTypeConverter().convert(node.getFeatureVersion) + val derivationFunction = new SimpleMvelDerivationFunction(transformationFunction.getParameters.get("expression"), + featureName, featureTypeConfig) + .asInstanceOf[FeatureDerivationFunction] + val newContextDf = applyDerivationFunction(node, derivationFunction, graphTraverser, contextDf) + updateDataframeMapAndApplyDefaults(Seq(node), graphTraverser, newContextDf, Seq.empty) // Note here derived features don't have output key columns + } + + override def batchCompute(nodes: Seq[Transformation], graphTraverser: FCMGraphTraverser, contextDf: DataFrame, dataPathHandlers: List[DataPathHandler]): DataFrame = { + nodes.foldLeft(contextDf)((newContextDf, node) => compute(node, graphTraverser, newContextDf, dataPathHandlers)) + } +} diff --git a/feathr-impl/src/main/scala/com/linkedin/feathr/offline/evaluator/transformation/DerivedComplexMVELOperator.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/evaluator/transformation/DerivedComplexMVELOperator.scala new file mode 100644 index 000000000..1413a802c --- /dev/null +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/evaluator/transformation/DerivedComplexMVELOperator.scala @@ -0,0 +1,35 @@ +package com.linkedin.feathr.offline.evaluator.transformation + +import com.linkedin.feathr.compute.{NodeReference, Transformation} +import com.linkedin.feathr.offline.config.{PegasusRecordFeatureTypeConverter, TaggedDependency} +import com.linkedin.feathr.offline.derived.functions.MvelFeatureDerivationFunction +import com.linkedin.feathr.offline.evaluator.transformation.BaseDerivedFeatureOperator.applyDerivationFunction +import com.linkedin.feathr.offline.evaluator.transformation.TransformationOperatorUtils.updateDataframeMapAndApplyDefaults +import com.linkedin.feathr.offline.graph.FCMGraphTraverser +import com.linkedin.feathr.offline.source.accessor.DataPathHandler +import org.apache.spark.sql.DataFrame + +/** + * Transformation operator for complex MVEL operator. + */ +object DerivedComplexMVELOperator extends TransformationOperator { + override def compute(node: Transformation, graphTraverser: FCMGraphTraverser, contextDf: DataFrame, dataPathHandlers: List[DataPathHandler]): DataFrame = { + val featureName = if (node.getFeatureName == null) graphTraverser.nodeIdToFeatureName(node.getId) else node.getFeatureName + val inputFeatureNames = node.getInputs.toArray.map(input => { + val inp = input.asInstanceOf[NodeReference] + graphTraverser.nodeIdToFeatureName(inp.getId) + }).sorted // Sort by input feature name to create the derivation function. Sort is crucial here to properly link input features. + + // We convert from array to map with dummy values in order to reuse MvelFeatureDerivationFunction from feathr. + val featureTypeConfig = PegasusRecordFeatureTypeConverter().convert(node.getFeatureVersion) + val featuresMap = inputFeatureNames.map(name => (name, TaggedDependency(Seq(""), ""))).toMap + val derivationFunction = new MvelFeatureDerivationFunction(featuresMap, node.getFunction.getParameters.get("expression"), featureName, + featureTypeConfig) + val newContextDf = applyDerivationFunction(node, derivationFunction, graphTraverser, contextDf) + updateDataframeMapAndApplyDefaults(Seq(node), graphTraverser, newContextDf, Seq.empty) // Note here derived features don't have output key columns + } + + override def batchCompute(nodes: Seq[Transformation], graphTraverser: FCMGraphTraverser, contextDf: DataFrame, dataPathHandlers: List[DataPathHandler]): DataFrame = { + nodes.foldLeft(contextDf)((newContextDf, node) => compute(node, graphTraverser, newContextDf, dataPathHandlers)) + } +} diff --git a/feathr-impl/src/main/scala/com/linkedin/feathr/offline/evaluator/transformation/DerivedUDFOperator.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/evaluator/transformation/DerivedUDFOperator.scala new file mode 100644 index 000000000..888d7f349 --- /dev/null +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/evaluator/transformation/DerivedUDFOperator.scala @@ -0,0 +1,35 @@ +package com.linkedin.feathr.offline.evaluator.transformation + +import com.linkedin.feathr.common.FeatureDerivationFunction +import com.linkedin.feathr.compute.Transformation +import com.linkedin.feathr.offline.client.plugins.{FeathrUdfPluginContext, FeatureDerivationFunctionAdaptor} +import com.linkedin.feathr.offline.evaluator.transformation.BaseDerivedFeatureOperator.applyDerivationFunction +import com.linkedin.feathr.offline.evaluator.transformation.TransformationOperatorUtils.updateDataframeMapAndApplyDefaults +import com.linkedin.feathr.offline.graph.FCMGraphTraverser +import com.linkedin.feathr.offline.source.accessor.DataPathHandler +import org.apache.spark.sql.DataFrame + +/** + * Transformation operator for derived UDF operator. + */ +object DerivedUDFOperator extends TransformationOperator { + override def compute(node: Transformation, graphTraverser: FCMGraphTraverser, contextDf: DataFrame, dataPathHandlers: List[DataPathHandler]): DataFrame = { + val udfClass = Class.forName(node.getFunction.getParameters.get("class")) + print(udfClass) + val derivationFunction = udfClass.getDeclaredConstructor().newInstance().asInstanceOf[AnyRef] + // possibly "adapt" the derivation function, in case it doesn't implement Feathr's FeatureDerivationFunction, + // using FeathrUdfPluginContext + val maybeAdaptedDerivationFunction = FeathrUdfPluginContext.getRegisteredUdfAdaptor(udfClass) match { + case Some(adaptor: FeatureDerivationFunctionAdaptor) => adaptor.adaptUdf(derivationFunction) + case _ => derivationFunction + } + + val derivedFunction = maybeAdaptedDerivationFunction.asInstanceOf[FeatureDerivationFunction] + val newContextDf = applyDerivationFunction(node, derivedFunction, graphTraverser, contextDf) + updateDataframeMapAndApplyDefaults(Seq(node), graphTraverser, newContextDf, Seq.empty) // Note here derived features don't have output key columns + } + + override def batchCompute(nodes: Seq[Transformation], graphTraverser: FCMGraphTraverser, contextDf: DataFrame, dataPathHandlers: List[DataPathHandler]): DataFrame = { + nodes.foldLeft(contextDf)((newContextDf, node) => compute(node, graphTraverser, newContextDf, dataPathHandlers)) + } +} diff --git a/feathr-impl/src/main/scala/com/linkedin/feathr/offline/evaluator/transformation/FeatureAliasOperator.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/evaluator/transformation/FeatureAliasOperator.scala new file mode 100644 index 000000000..d91c0fbf2 --- /dev/null +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/evaluator/transformation/FeatureAliasOperator.scala @@ -0,0 +1,30 @@ +package com.linkedin.feathr.offline.evaluator.transformation + +import com.linkedin.feathr.compute.Transformation +import com.linkedin.feathr.offline.graph.{DataframeAndColumnMetadata, FCMGraphTraverser} +import com.linkedin.feathr.offline.source.accessor.DataPathHandler +import org.apache.spark.sql.DataFrame +import org.apache.spark.sql.functions.col + +object FeatureAliasOperator extends TransformationOperator { + /** + * Compute feature alias via a withColumn call on the context df. + * @param node + * @param graphTraverser + * @param contextDf + * @return + */ + override def compute(node: Transformation, graphTraverser: FCMGraphTraverser, contextDf: DataFrame, dataPathHandlers: List[DataPathHandler]): DataFrame = { + // In the case of a feature alias operator we can optimize this by just doing a withColumn call on the contextDf instead of doing a join. + val inputNodeId = node.getInputs.get(0).getId + val featureName = if (node.getFeatureName == null) graphTraverser.nodeIdToFeatureName(node.getId) else node.getFeatureName + val modifiedContextDf = contextDf.withColumn(featureName, col(graphTraverser.nodeIdToFeatureName(inputNodeId))) + graphTraverser.nodeIdToDataframeAndColumnMetadataMap(node.getId) = DataframeAndColumnMetadata(modifiedContextDf, + graphTraverser.nodeIdToDataframeAndColumnMetadataMap(inputNodeId).keyExpression) + modifiedContextDf + } + + override def batchCompute(nodes: Seq[Transformation], graphTraverser: FCMGraphTraverser, contextDf: DataFrame, dataPathHandlers: List[DataPathHandler]): DataFrame = { + nodes.foldLeft(contextDf)((newContextDf, node) => compute(node, graphTraverser, newContextDf, dataPathHandlers)) + } +} diff --git a/feathr-impl/src/main/scala/com/linkedin/feathr/offline/evaluator/transformation/LookupMVELOperator.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/evaluator/transformation/LookupMVELOperator.scala new file mode 100644 index 000000000..0a45f3c5d --- /dev/null +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/evaluator/transformation/LookupMVELOperator.scala @@ -0,0 +1,43 @@ +package com.linkedin.feathr.offline.evaluator.transformation + +import com.linkedin.feathr.compute.Transformation +import com.linkedin.feathr.offline.anchored.anchorExtractor.SimpleConfigurableAnchorExtractor +import com.linkedin.feathr.offline.config.{MVELFeatureDefinition, PegasusRecordFeatureTypeConverter} +import com.linkedin.feathr.offline.evaluator.transformation.TransformationOperatorUtils.updateDataframeMapAndApplyDefaults +import com.linkedin.feathr.offline.graph.FCMGraphTraverser +import com.linkedin.feathr.offline.source.accessor.DataPathHandler +import com.linkedin.feathr.offline.transformation.{DataFrameBasedRowEvaluator, FeatureColumnFormat} +import org.apache.spark.sql.DataFrame +import org.apache.spark.sql.functions.col + +/** + * Operator for specifically the transformation applied for look up base nodes. Note that we have to treat this + * differently than a derived MVEL feature for parity sakes with feathr v16. + */ +object LookupMVELOperator extends TransformationOperator { + + override def compute(node: Transformation, graphTraverser: FCMGraphTraverser, contextDf: DataFrame, dataPathHandlers: List[DataPathHandler]): DataFrame = { + val featureName = if (node.getFeatureName == null) graphTraverser.nodeIdToFeatureName(node.getId) else node.getFeatureName + val featureTypeConfig = PegasusRecordFeatureTypeConverter().convert(node.getFeatureVersion) + val mvelExpr = node.getFunction.getParameters.get("expression") + val mvelExtractor = new SimpleConfigurableAnchorExtractor(Seq.empty, + Map(featureName -> MVELFeatureDefinition(mvelExpr, featureTypeConfig))) + + + val transformedDf = DataFrameBasedRowEvaluator.transform(mvelExtractor, contextDf, Seq((featureName, "")), + Map(featureName -> featureTypeConfig.get), graphTraverser.mvelExpressionContext).df + + // Apply feature alias here if needed. + val result = if (graphTraverser.nodeIdToFeatureName(node.getId) != node.getFeatureName) { + val featureAlias = graphTraverser.nodeIdToFeatureName(node.getId) + graphTraverser.featureColumnFormatsMap(featureAlias) = FeatureColumnFormat.RAW + transformedDf.withColumn(featureAlias, col(featureName)) + } else transformedDf + updateDataframeMapAndApplyDefaults(Seq(node), graphTraverser, result, Seq.empty) // Note here lookup MVEL features don't have output key columns + } + + override def batchCompute(nodes: Seq[Transformation], graphTraverser: FCMGraphTraverser, contextDf: DataFrame, + dataPathHandlers: List[DataPathHandler]): DataFrame = { + nodes.foldLeft(contextDf)((newContextDf, node) => compute(node, graphTraverser, newContextDf, dataPathHandlers)) + } +} diff --git a/feathr-impl/src/main/scala/com/linkedin/feathr/offline/evaluator/transformation/PassthroughMVELOperator.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/evaluator/transformation/PassthroughMVELOperator.scala new file mode 100644 index 000000000..15be2bef3 --- /dev/null +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/evaluator/transformation/PassthroughMVELOperator.scala @@ -0,0 +1,27 @@ +package com.linkedin.feathr.offline.evaluator.transformation + +import com.linkedin.feathr.compute.Transformation +import com.linkedin.feathr.offline.evaluator.transformation.AnchorMVELOperator.computeMVELResult +import com.linkedin.feathr.offline.evaluator.transformation.TransformationOperatorUtils.updateDataframeMapAndApplyDefaults +import com.linkedin.feathr.offline.graph.FCMGraphTraverser +import com.linkedin.feathr.offline.source.accessor.DataPathHandler +import org.apache.spark.sql.DataFrame + +object PassthroughMVELOperator extends TransformationOperator { + /** + * Operator for batch passthrough MVEL transformations. Given context df and a grouped set of MVEL transformation nodes, + * perform the MVEL transformations. Since this is a passthrough operator, we don't append key columns or join to context. + * @param nodes Seq of nodes with MVEL anchor as operator + * @param graphTraverser FCMGraphTraverser + * @param contextDf Context df + * @return Dataframe + */ + override def batchCompute(nodes: Seq[Transformation], graphTraverser: FCMGraphTraverser, contextDf: DataFrame, dataPathHandlers: List[DataPathHandler]): DataFrame = { + val (result, keyColumns) = computeMVELResult(nodes, graphTraverser, contextDf, appendKeyColumns = false) + updateDataframeMapAndApplyDefaults(nodes, graphTraverser, result, keyColumns) + } + + override def compute(node: Transformation, graphTraverser: FCMGraphTraverser, contextDf: DataFrame, dataPathHandlers: List[DataPathHandler]): DataFrame = { + batchCompute(Seq(node), graphTraverser, contextDf, dataPathHandlers) + } +} diff --git a/feathr-impl/src/main/scala/com/linkedin/feathr/offline/evaluator/transformation/PassthroughSQLOperator.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/evaluator/transformation/PassthroughSQLOperator.scala new file mode 100644 index 000000000..f10104e55 --- /dev/null +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/evaluator/transformation/PassthroughSQLOperator.scala @@ -0,0 +1,27 @@ +package com.linkedin.feathr.offline.evaluator.transformation + +import com.linkedin.feathr.compute.Transformation +import com.linkedin.feathr.offline.evaluator.transformation.AnchorSQLOperator.computeSQLResult +import com.linkedin.feathr.offline.evaluator.transformation.TransformationOperatorUtils.updateDataframeMapAndApplyDefaults +import com.linkedin.feathr.offline.graph.FCMGraphTraverser +import com.linkedin.feathr.offline.source.accessor.DataPathHandler +import org.apache.spark.sql.DataFrame + +object PassthroughSQLOperator extends TransformationOperator { + /** + * Operator for batch passthrough SQL transformations. Given context df and a grouped set of SQL transformation nodes, + * perform the SQL transformations. Since this is a passthrough operator, we don't append key columns or join to context. + * @param nodes Seq of nodes with UDF anchor as operator + * @param graphTraverser FCMGraphTraverser + * @param contextDf Context df + * @return Dataframe + */ + override def batchCompute(nodes: Seq[Transformation], graphTraverser: FCMGraphTraverser, contextDf: DataFrame, dataPathHandlers: List[DataPathHandler]): DataFrame = { + val (result, keyColumns) = computeSQLResult(nodes, graphTraverser, contextDf, appendKeyColumns = false) + updateDataframeMapAndApplyDefaults(nodes, graphTraverser, result, keyColumns) + } + + override def compute(node: Transformation, graphTraverser: FCMGraphTraverser, contextDf: DataFrame, dataPathHandlers: List[DataPathHandler]): DataFrame = { + batchCompute(Seq(node), graphTraverser, contextDf, dataPathHandlers) + } +} diff --git a/feathr-impl/src/main/scala/com/linkedin/feathr/offline/evaluator/transformation/PassthroughUDFOperator.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/evaluator/transformation/PassthroughUDFOperator.scala new file mode 100644 index 000000000..06ae58922 --- /dev/null +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/evaluator/transformation/PassthroughUDFOperator.scala @@ -0,0 +1,27 @@ +package com.linkedin.feathr.offline.evaluator.transformation + +import com.linkedin.feathr.compute.Transformation +import com.linkedin.feathr.offline.evaluator.transformation.AnchorUDFOperator.computeUDFResult +import com.linkedin.feathr.offline.evaluator.transformation.TransformationOperatorUtils.updateDataframeMapAndApplyDefaults +import com.linkedin.feathr.offline.graph.FCMGraphTraverser +import com.linkedin.feathr.offline.source.accessor.DataPathHandler +import org.apache.spark.sql.DataFrame + +object PassthroughUDFOperator extends TransformationOperator { + /** + * Operator for batch passthrough UDF transformations. Given context df and a grouped set of UDF transformation nodes, + * perform the UDF transformations. Since this is a passthrough operator, we don't append key columns or join to context. + * @param nodes Seq of nodes with UDF anchor as operator + * @param graphTraverser FCMGraphTraverser + * @param contextDf Context df + * @return Dataframe + */ + override def batchCompute(nodes: Seq[Transformation], graphTraverser: FCMGraphTraverser, contextDf: DataFrame, dataPathHandlers: List[DataPathHandler]): DataFrame = { + val (result, keyColumns) = computeUDFResult(nodes, graphTraverser, contextDf, appendKeyColumns = false, dataPathHandlers) + updateDataframeMapAndApplyDefaults(nodes, graphTraverser, result, keyColumns) + } + + override def compute(node: Transformation, graphTraverser: FCMGraphTraverser, contextDf: DataFrame, dataPathHandlers: List[DataPathHandler]): DataFrame = { + batchCompute(Seq(node), graphTraverser, contextDf, dataPathHandlers) + } +} diff --git a/feathr-impl/src/main/scala/com/linkedin/feathr/offline/evaluator/transformation/TransformationNodeEvaluator.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/evaluator/transformation/TransformationNodeEvaluator.scala new file mode 100644 index 000000000..1a3b5176d --- /dev/null +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/evaluator/transformation/TransformationNodeEvaluator.scala @@ -0,0 +1,42 @@ +package com.linkedin.feathr.offline.evaluator.transformation + +import com.linkedin.feathr.compute.{AnyNode, Operators} +import com.linkedin.feathr.offline.evaluator.NodeEvaluator +import com.linkedin.feathr.offline.graph.FCMGraphTraverser +import com.linkedin.feathr.offline.source.accessor.DataPathHandler +import org.apache.spark.sql.DataFrame + +object TransformationNodeEvaluator extends NodeEvaluator { + /** + * Evaluate all the transformation nodes in the batch. Note that with the current grouping criteria, we expect all nodes + * in a batch to have the same operator. + * @param nodes Nodes to evaluate + * @param graphTraverser FCMGraphTraverser + * @param contextDf Context df + * @return DataFrame + */ + override def batchEvaluate(nodes: Seq[AnyNode], graphTraverser: FCMGraphTraverser, contextDf: DataFrame, dataPathHandlers: List[DataPathHandler]): DataFrame = { + // We require that all batch transformation nodes have the same operator so we can pattern match on the head of the + // node seq to decide on the appropriate TransformationOperator to call. + val transformationNodes = nodes.map(_.getTransformation) + val transformationOperator = transformationNodes.head.getFunction.getOperator + transformationOperator match { + case Operators.OPERATOR_ID_ANCHOR_MVEL => AnchorMVELOperator.batchCompute(transformationNodes, graphTraverser, contextDf, dataPathHandlers) + case Operators.OPERATOR_ID_ANCHOR_SPARK_SQL_FEATURE_EXTRACTOR => AnchorSQLOperator.batchCompute(transformationNodes, graphTraverser, contextDf, dataPathHandlers) + case Operators.OPERATOR_ID_ANCHOR_JAVA_UDF_FEATURE_EXTRACTOR => AnchorUDFOperator.batchCompute(transformationNodes, graphTraverser, contextDf, dataPathHandlers) + case Operators.OPERATOR_ID_PASSTHROUGH_MVEL => PassthroughMVELOperator.batchCompute(transformationNodes, graphTraverser, contextDf, dataPathHandlers) + case Operators.OPERATOR_ID_PASSTHROUGH_SPARK_SQL_FEATURE_EXTRACTOR => PassthroughSQLOperator.batchCompute(transformationNodes, graphTraverser, contextDf, dataPathHandlers) + case Operators.OPERATOR_ID_PASSTHROUGH_JAVA_UDF_FEATURE_EXTRACTOR => PassthroughUDFOperator.batchCompute(transformationNodes, graphTraverser, contextDf, dataPathHandlers) + case Operators.OPERATOR_ID_DERIVED_MVEL => DerivedSimpleMVELOperator.batchCompute(transformationNodes, graphTraverser, contextDf, dataPathHandlers) + case Operators.OPERATOR_ID_EXTRACT_FROM_TUPLE => DerivedComplexMVELOperator.batchCompute(transformationNodes, graphTraverser, contextDf, dataPathHandlers) + case Operators.OPERATOR_ID_DERIVED_JAVA_UDF_FEATURE_EXTRACTOR => DerivedUDFOperator.batchCompute(transformationNodes, graphTraverser, contextDf, dataPathHandlers) + case Operators.OPERATOR_ID_LOOKUP_MVEL => LookupMVELOperator.batchCompute(transformationNodes, graphTraverser, contextDf, dataPathHandlers) + case Operators.OPERATOR_FEATURE_ALIAS => FeatureAliasOperator.batchCompute(transformationNodes, graphTraverser, contextDf, dataPathHandlers) + case _ => throw new UnsupportedOperationException("Unsupported operator found in Transformation node.") + } + } + + override def evaluate(node: AnyNode, graphTraverser: FCMGraphTraverser, contextDf: DataFrame, dataPathHandlers: List[DataPathHandler]): DataFrame = { + batchEvaluate(Seq(node), graphTraverser, contextDf, dataPathHandlers) + } +} diff --git a/feathr-impl/src/main/scala/com/linkedin/feathr/offline/evaluator/transformation/TransformationOperator.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/evaluator/transformation/TransformationOperator.scala new file mode 100644 index 000000000..1e0ba181e --- /dev/null +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/evaluator/transformation/TransformationOperator.scala @@ -0,0 +1,31 @@ +package com.linkedin.feathr.offline.evaluator.transformation + +import com.linkedin.feathr.compute.Transformation +import com.linkedin.feathr.offline.graph.FCMGraphTraverser +import com.linkedin.feathr.offline.source.accessor.DataPathHandler +import org.apache.spark.sql.DataFrame + +/** + * Trait class for transformation operators. The task of operators is to compute their operation (i.e. MVEL, SQL, etc) + * and ensure that the result is available in the graphTraverser nodeIdToDataframeAndColumnMetadataMap map, + * the result is present in the context dataframe, and return the context df. + */ +trait TransformationOperator { + /** + * Perform operation on seq of transformation nodes and return context df. + * + * @param nodes + * @param graphTraverser + * @param contextDf + */ + def batchCompute(nodes: Seq[Transformation], graphTraverser: FCMGraphTraverser, contextDf: DataFrame, dataPathHandlers: List[DataPathHandler]): DataFrame + + /** + * Perform operation on a single transformation node and return context df. + * + * @param node + * @param graphTraverser + * @param contextDf + */ + def compute(node: Transformation, graphTraverser: FCMGraphTraverser, contextDf: DataFrame, dataPathHandlers: List[DataPathHandler]): DataFrame +} diff --git a/feathr-impl/src/main/scala/com/linkedin/feathr/offline/evaluator/transformation/TransformationOperatorUtils.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/evaluator/transformation/TransformationOperatorUtils.scala new file mode 100644 index 000000000..631e399f3 --- /dev/null +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/evaluator/transformation/TransformationOperatorUtils.scala @@ -0,0 +1,141 @@ +package com.linkedin.feathr.offline.evaluator.transformation + +import com.linkedin.feathr.compute.Transformation +import com.linkedin.feathr.offline.graph.NodeUtils.{getDefaultConverterForTransformationNodes, getFeatureTypeConfigsMapForTransformationNodes} +import com.linkedin.feathr.offline.graph.{DataframeAndColumnMetadata, FCMGraphTraverser} +import com.linkedin.feathr.offline.join.algorithms.{EqualityJoinConditionBuilder, JoinType, SparkJoinWithJoinCondition} +import com.linkedin.feathr.offline.transformation.DataFrameDefaultValueSubstituter.substituteDefaults +import org.apache.spark.sql.{Column, DataFrame} +import org.apache.spark.sql.functions._ + +import scala.collection.JavaConverters.asScalaBufferConverter + +/** + * Util functions which are shared among different operators. + */ +object TransformationOperatorUtils { + /** + * Keeps only feature column + key columns and drops all other columns. Key columns are renamed with __frame__key__column__ prefix. + * @param df + * @param keyCols + * @param featureName + * @return + */ + def dropAndRenameCols(df: DataFrame, keyCols: Seq[String], featureName: Seq[String]): (DataFrame, Seq[String]) = { + val toDropCols = df.columns diff (keyCols ++ featureName) + val modifiedDf = df.drop(toDropCols: _*) + val renamedKeyColumns = keyCols.map(c => "__frame__key__column__" + c) + val oldKeyColToNewKeyCOl = (keyCols zip renamedKeyColumns).toMap + val withRenamedColsDF = modifiedDf.select( + modifiedDf.columns.map(c => modifiedDf(c).alias(oldKeyColToNewKeyCOl.getOrElse(c, c))): _* + ) + (withRenamedColsDF, renamedKeyColumns) + } + + /** + * Create data frame by combining inputDf and Seq of feature name -> spark Column. Some extractors in Frame outputs the result + * in the form of Seq[(String, Column)] so we need this utility to append the result to the input df. + * @param inputDf + * @param featureColumnDefs + * @return + */ + def createFeatureDF(inputDf: DataFrame, featureColumnDefs: Seq[(String, Column)]): DataFrame = { + // first add a prefix to the feature column name in the schema + val featureColumnNamePrefix = "_frame_sql_feature_prefix_" + print(inputDf.columns.mkString("Array(", ", ", ")")) + val transformedDF = featureColumnDefs.foldLeft(inputDf)((baseDF, columnWithName) => { + print("COLUMN NAME = " + columnWithName) + val columnName = featureColumnNamePrefix + columnWithName._1 + baseDF.withColumn(columnName, expr(columnWithName._2.toString())) + }) + val featureNames = featureColumnDefs.map(_._1) + // drop the context column that have the same name as feature names + val withoutDupContextFieldDF = transformedDF.drop(featureNames: _*) + // remove the prefix we just added, so that we have a dataframe with feature names as their column names + featureNames + .zip(featureNames) + .foldLeft(withoutDupContextFieldDF)((baseDF, namePair) => { + baseDF.withColumnRenamed(featureColumnNamePrefix + namePair._1, namePair._2) + }) + } + + /** + * Joins result df to context df using concrete keys and applies default values. Returns new context df. + * @param nodes + * @param graphTraverser + * @param resultDf + * @param resultKeyColumns + * @param contextDf + * @return + */ + def joinResultToContextDfAndApplyDefaults(nodes: Seq[Transformation], + graphTraverser: FCMGraphTraverser, + resultDf: DataFrame, + resultKeyColumns: Seq[String], + contextDf: DataFrame): DataFrame = { + val featureNamesInBatch = nodes.map(node => graphTraverser.nodeIdToFeatureName(node.getId)) + // Update node context map for all nodes in this batch + nodes.foreach(node => { + graphTraverser.nodeIdToDataframeAndColumnMetadataMap(node.getId) = + DataframeAndColumnMetadata(resultDf, resultKeyColumns, Some(graphTraverser.nodeIdToFeatureName(node.getId))) + }) + + // Get concrete keys from nodeIdToDataframeAndColumnMetadataMap to join transformation result to contextDf + val concreteKeys = nodes.head.getConcreteKey.getKey.asScala.flatMap(x => { + if (graphTraverser.nodeIdToDataframeAndColumnMetadataMap(x).featureColumn.isDefined) { + Seq(graphTraverser.nodeIdToDataframeAndColumnMetadataMap(x).featureColumn.get) + } else { + graphTraverser.nodeIdToDataframeAndColumnMetadataMap(x).keyExpression + } + }) + + // Join result to context df and drop transformation node key columns. + // NOTE: If the batch of nodes only contains look up expansion features, we can not join to the context df at this point. + val featureTypeConfigs = getFeatureTypeConfigsMapForTransformationNodes(nodes) + val defaultConverter = getDefaultConverterForTransformationNodes(nodes) + val allLookupExpansionNodes = graphTraverser.nodes.filter(node => node.getLookup != null).map(node => node.getLookup.getLookupNode) + val isLookupExpansionGroup = nodes.forall(node => allLookupExpansionNodes.contains(node.getId)) + if (isLookupExpansionGroup) { + val withDefaultsDf = substituteDefaults(resultDf, featureNamesInBatch, + defaultConverter, featureTypeConfigs, graphTraverser.ss) + nodes.foreach(node => { + graphTraverser.nodeIdToDataframeAndColumnMetadataMap(node.getId) = + DataframeAndColumnMetadata(withDefaultsDf, resultKeyColumns, Some(graphTraverser.nodeIdToFeatureName(node.getId))) + }) + contextDf + } else { + // If the feature name is already present in the contextDf, it must have been needed for a derived feature. Drop the + // column and join the new one. + val newContextDf = featureNamesInBatch.foldLeft(contextDf)((currContextDf, featureName) => { + if (currContextDf.columns.contains(featureName)) currContextDf.drop(featureName) else currContextDf + }) + val result = SparkJoinWithJoinCondition(EqualityJoinConditionBuilder).join(concreteKeys, newContextDf, resultKeyColumns, resultDf, JoinType.left_outer) + .drop(resultKeyColumns: _*) + substituteDefaults(result, featureNamesInBatch, defaultConverter, featureTypeConfigs, graphTraverser.ss) + } + } + + /** + * Given a seq of transformation nodes, updates graphTraverser's nodeIdToDataframeAndColumnMetadataMap with the result + * and returns the new context df. This function is used by passthrough and derived operators as they don't perform any joins. + * @param nodes + * @param graphTraverser + * @param resultDf + * @param resultKeyColumns + * @return + */ + def updateDataframeMapAndApplyDefaults(nodes: Seq[Transformation], + graphTraverser: FCMGraphTraverser, + resultDf: DataFrame, + resultKeyColumns: Seq[String]): DataFrame = { + // Update node context map for all processed nodes this stage. + nodes.foreach(node => { + graphTraverser.nodeIdToDataframeAndColumnMetadataMap(node.getId) = + DataframeAndColumnMetadata(resultDf, resultKeyColumns, Some(graphTraverser.nodeIdToFeatureName(node.getId))) + }) + val featureNamesInBatch = nodes.map(node => graphTraverser.nodeIdToFeatureName(node.getId)) + val featureTypeConfigs = getFeatureTypeConfigsMapForTransformationNodes(nodes) + val defaultConverter = getDefaultConverterForTransformationNodes(nodes) + substituteDefaults(resultDf, featureNamesInBatch, defaultConverter, featureTypeConfigs, graphTraverser.ss) + } +} diff --git a/feathr-impl/src/main/scala/com/linkedin/feathr/offline/exception/DataFrameApiUnsupportedOperationException.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/exception/DataFrameApiUnsupportedOperationException.scala new file mode 100644 index 000000000..2a83c33d5 --- /dev/null +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/exception/DataFrameApiUnsupportedOperationException.scala @@ -0,0 +1,13 @@ +package com.linkedin.feathr.offline.exception + +/** + * This exception is thrown when operation is not supported in DataFrame API (vs RDD api) + * It will be caught in local running mode, and just logging warning message. + */ +private[offline] class DataFrameApiUnsupportedOperationException(message: String) extends Exception(message) { + + def this(message: String, cause: Throwable) { + this(message) + initCause(cause) + } +} diff --git a/src/main/scala/com/linkedin/feathr/offline/exception/FeathrIllegalStateException.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/exception/FeathrIllegalStateException.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/exception/FeathrIllegalStateException.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/exception/FeathrIllegalStateException.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/exception/FeatureTransformationException.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/exception/FeatureTransformationException.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/exception/FeatureTransformationException.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/exception/FeatureTransformationException.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/generation/DataFrameFeatureGenerator.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/generation/DataFrameFeatureGenerator.scala similarity index 98% rename from src/main/scala/com/linkedin/feathr/offline/generation/DataFrameFeatureGenerator.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/generation/DataFrameFeatureGenerator.scala index 310c3931e..57f4def55 100644 --- a/src/main/scala/com/linkedin/feathr/offline/generation/DataFrameFeatureGenerator.scala +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/generation/DataFrameFeatureGenerator.scala @@ -5,7 +5,7 @@ import com.linkedin.feathr.common.{Header, JoiningFeatureParams, TaggedFeatureNa import com.linkedin.feathr.offline import com.linkedin.feathr.offline.anchored.feature.FeatureAnchorWithSource.{getDefaultValues, getFeatureTypes} import com.linkedin.feathr.offline.derived.functions.SeqJoinDerivationFunction -import com.linkedin.feathr.offline.derived.strategies.{DerivationStrategies, RowBasedDerivation, SequentialJoinDerivationStrategy, SparkUdfDerivation} +import com.linkedin.feathr.offline.derived.strategies.{DerivationStrategies, RowBasedDerivation, SequentialJoinDerivationStrategy, SparkUdfDerivation, SqlDerivationSpark} import com.linkedin.feathr.offline.derived.{DerivedFeature, DerivedFeatureEvaluator} import com.linkedin.feathr.offline.evaluator.DerivedFeatureGenStage import com.linkedin.feathr.offline.job.{FeatureGenSpec, FeatureTransformation} @@ -133,5 +133,7 @@ private[offline] class DataFrameFeatureGenerator(logicalPlan: MultiStageJoinPlan ErrorLabel.FEATHR_ERROR, s"Feature Generation does not support Sequential Join features : ${derivedFeature.producedFeatureNames.head}") } - }), mvelContext) + }, + new SqlDerivationSpark() + ), mvelContext) } diff --git a/src/main/scala/com/linkedin/feathr/offline/generation/FeatureDataHDFSProcessUtils.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/generation/FeatureDataHDFSProcessUtils.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/generation/FeatureDataHDFSProcessUtils.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/generation/FeatureDataHDFSProcessUtils.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/generation/FeatureGenDefaultsSubstituter.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/generation/FeatureGenDefaultsSubstituter.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/generation/FeatureGenDefaultsSubstituter.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/generation/FeatureGenDefaultsSubstituter.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/generation/FeatureGenFeatureGrouper.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/generation/FeatureGenFeatureGrouper.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/generation/FeatureGenFeatureGrouper.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/generation/FeatureGenFeatureGrouper.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/generation/FeatureGenKeyTagAnalyzer.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/generation/FeatureGenKeyTagAnalyzer.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/generation/FeatureGenKeyTagAnalyzer.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/generation/FeatureGenKeyTagAnalyzer.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/generation/FeatureGenUtils.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/generation/FeatureGenUtils.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/generation/FeatureGenUtils.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/generation/FeatureGenUtils.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/generation/FeatureGenerationPathName.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/generation/FeatureGenerationPathName.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/generation/FeatureGenerationPathName.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/generation/FeatureGenerationPathName.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/generation/IncrementalAggSnapshotLoader.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/generation/IncrementalAggSnapshotLoader.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/generation/IncrementalAggSnapshotLoader.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/generation/IncrementalAggSnapshotLoader.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/generation/PostGenPruner.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/generation/PostGenPruner.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/generation/PostGenPruner.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/generation/PostGenPruner.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/generation/RawDataWriterUtils.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/generation/RawDataWriterUtils.scala similarity index 94% rename from src/main/scala/com/linkedin/feathr/offline/generation/RawDataWriterUtils.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/generation/RawDataWriterUtils.scala index 6351c701b..7b1e0c254 100644 --- a/src/main/scala/com/linkedin/feathr/offline/generation/RawDataWriterUtils.scala +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/generation/RawDataWriterUtils.scala @@ -1,5 +1,6 @@ package com.linkedin.feathr.offline.generation +import com.linkedin.avroutil1.compatibility.AvroCompatibilityHelper import com.linkedin.feathr.common.exception.{ErrorLabel, FeathrDataOutputException} import com.linkedin.feathr.common.{Header, TaggedFeatureName} import com.linkedin.feathr.offline.generation.FeatureDataHDFSProcessUtils._ @@ -102,10 +103,10 @@ private[offline] object RawDataWriterUtils { // single key does not have to be record? private def makeSingleWrappedSchema(schema: Schema, recordName: String, wrapperName: String): Schema.Field = { val outputKeySchemaFields = schema.getFields.map(f => { - new Schema.Field(f.name(), f.schema(), f.doc(), SourceUtils.getDefaultValueFromAvroRecord(f), f.order()) + AvroCompatibilityHelper.createSchemaField(f.name(), f.schema(), f.doc(), SourceUtils.getDefaultValueFromAvroRecord(f), f.order()) }) val outputKeySchema = Schema.createRecord(recordName, null, null, false) outputKeySchema.setFields(outputKeySchemaFields) - new Schema.Field(wrapperName, outputKeySchema, null, null) + AvroCompatibilityHelper.createSchemaField(wrapperName, outputKeySchema, null, null) } } diff --git a/src/main/scala/com/linkedin/feathr/offline/generation/SparkIOUtils.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/generation/SparkIOUtils.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/generation/SparkIOUtils.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/generation/SparkIOUtils.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/generation/StreamingFeatureGenerator.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/generation/StreamingFeatureGenerator.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/generation/StreamingFeatureGenerator.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/generation/StreamingFeatureGenerator.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/generation/aggregations/AvgPooling.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/generation/aggregations/AvgPooling.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/generation/aggregations/AvgPooling.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/generation/aggregations/AvgPooling.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/generation/aggregations/CollectTermValueMap.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/generation/aggregations/CollectTermValueMap.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/generation/aggregations/CollectTermValueMap.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/generation/aggregations/CollectTermValueMap.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/generation/aggregations/MaxPooling.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/generation/aggregations/MaxPooling.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/generation/aggregations/MaxPooling.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/generation/aggregations/MaxPooling.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/generation/aggregations/MinPooling.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/generation/aggregations/MinPooling.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/generation/aggregations/MinPooling.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/generation/aggregations/MinPooling.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/generation/outputProcessor/FeatureMonitoringProcessor.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/generation/outputProcessor/FeatureMonitoringProcessor.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/generation/outputProcessor/FeatureMonitoringProcessor.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/generation/outputProcessor/FeatureMonitoringProcessor.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/generation/outputProcessor/FeatureMonitoringUtils.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/generation/outputProcessor/FeatureMonitoringUtils.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/generation/outputProcessor/FeatureMonitoringUtils.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/generation/outputProcessor/FeatureMonitoringUtils.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/generation/outputProcessor/PushToRedisOutputProcessor.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/generation/outputProcessor/PushToRedisOutputProcessor.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/generation/outputProcessor/PushToRedisOutputProcessor.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/generation/outputProcessor/PushToRedisOutputProcessor.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/generation/outputProcessor/RedisOutputUtils.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/generation/outputProcessor/RedisOutputUtils.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/generation/outputProcessor/RedisOutputUtils.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/generation/outputProcessor/RedisOutputUtils.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/generation/outputProcessor/WriteToHDFSOutputProcessor.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/generation/outputProcessor/WriteToHDFSOutputProcessor.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/generation/outputProcessor/WriteToHDFSOutputProcessor.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/generation/outputProcessor/WriteToHDFSOutputProcessor.scala diff --git a/feathr-impl/src/main/scala/com/linkedin/feathr/offline/graph/FCMGraphTraverser.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/graph/FCMGraphTraverser.scala new file mode 100644 index 000000000..b35133b59 --- /dev/null +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/graph/FCMGraphTraverser.scala @@ -0,0 +1,218 @@ +package com.linkedin.feathr.offline.graph + +import com.linkedin.feathr.compute.{AnyNode, ComputeGraph, Dependencies} +import com.linkedin.feathr.offline.FeatureDataFrame +import com.linkedin.feathr.offline.client.{IN_PROGRESS, NOT_VISITED, VISITED, VisitedState} +import com.linkedin.feathr.offline.config.{FeatureJoinConfig, JoinConfigSettings} +import com.linkedin.feathr.offline.evaluator.aggregation.AggregationNodeEvaluator +import com.linkedin.feathr.offline.evaluator.datasource.DataSourceNodeEvaluator +import com.linkedin.feathr.offline.evaluator.lookup.LookupNodeEvaluator +import com.linkedin.feathr.offline.evaluator.transformation.TransformationNodeEvaluator +import com.linkedin.feathr.offline.graph.NodeGrouper.{groupAllSWANodes, groupTransformationNodes} +import com.linkedin.feathr.offline.graph.NodeUtils.getFeatureTypeConfigsMap +import com.linkedin.feathr.offline.job.FeatureTransformation.convertFCMResultDFToFDS +import com.linkedin.feathr.offline.mvel.plugins.FeathrExpressionExecutionContext +import com.linkedin.feathr.offline.source.DataSource +import com.linkedin.feathr.offline.source.accessor.DataPathHandler +import com.linkedin.feathr.offline.swa.SlidingWindowFeatureUtils +import com.linkedin.feathr.offline.transformation.FeatureColumnFormat +import com.linkedin.feathr.offline.transformation.FeatureColumnFormat.FeatureColumnFormat +import com.linkedin.feathr.offline.util.datetime.DateTimeInterval +import org.apache.log4j.Logger +import org.apache.spark.sql.{DataFrame, SparkSession} + +import scala.collection.JavaConverters._ +import scala.collection.mutable + +/** + * Case class to hold DataFrame and column metadata. + * @param df + * @param keyExpression + * @param featureColumn + * @param dataSource + * @param timestampColumn + */ +case class DataframeAndColumnMetadata(df: DataFrame, keyExpression: Seq[String], featureColumn: Option[String] = None, + dataSource: Option[DataSource] = None, timestampColumn: Option[String] = None) + +/** + * Case class to hold config settings extracted from the join config + observation data which is needed for evaluation + * of EVENT and AGGREGATION nodes. + * @param timeConfigSettings + * @param featuresToTimeDelayMap + * @param obsTimeRange + */ +case class TimeConfigSettings(timeConfigSettings: Option[JoinConfigSettings], featuresToTimeDelayMap: Map[String, String], obsTimeRange: DateTimeInterval) + +/** + * The main purpose of the FCMGraphTraverser is to walk a resolved compute graph and perform the feature join specified by the graph + join config. + * The main API is traverseGraph() which will actually execute the resolve graph. In the initialization of the class, the necessary information + * like nodes, join config settings, spark session etc. will be extracted from the inputs and the public member variables needed for graph + * traversal will be created. See the scaladocs of traverseGraph for more info on graph traversal algo. + * @param inputSparkSession + * @param featureJoinConfig + * @param resolvedGraph + * @param observationDf + */ +class FCMGraphTraverser(inputSparkSession: SparkSession, + featureJoinConfig: FeatureJoinConfig, + resolvedGraph: ComputeGraph, + observationDf: DataFrame, + dataPathHandlers: List[DataPathHandler], + mvelContext: Option[FeathrExpressionExecutionContext]) { + private val log = Logger.getLogger(getClass.getName) + // nodeIdToDataframeAndColumnMetadataMap will be a map of node id -> DataframeAndColumnMetadata which will be updated as each node is processed. + val nodeIdToDataframeAndColumnMetadataMap: mutable.HashMap[Int, DataframeAndColumnMetadata] = mutable.HashMap[Int, DataframeAndColumnMetadata]() + + // Create a map of requested feature names to FeatureColumnFormat (Raw or FDS) for FDS conversion sake at the end of + // execution. All features will default to Raw unless specified otherwise. Purpose is that some operators will do + // FDS conversion while others will not. + val featureColumnFormatsMap: mutable.HashMap[String, FeatureColumnFormat] = + mutable.HashMap[String, FeatureColumnFormat](featureJoinConfig.joinFeatures.map(joinFeature => (joinFeature.featureName, FeatureColumnFormat.RAW)): _*) + + val nodes: mutable.Buffer[AnyNode] = resolvedGraph.getNodes().asScala + val nodeIdToFeatureName: Map[Integer, String] = getNodeIdToFeatureNameMap(nodes) + val mvelExpressionContext: Option[FeathrExpressionExecutionContext] = mvelContext + + // Join info needed from join config + obs data for EVENT and AGGREGATION nodes + val timeConfigSettings: TimeConfigSettings = getJoinSettings + val ss: SparkSession = inputSparkSession + + /** + * Create join settings case object from join config + observation data time range. + * @return + */ + private def getJoinSettings: TimeConfigSettings = { + val obsTimeRange: DateTimeInterval = if (featureJoinConfig.settings.isDefined) { + SlidingWindowFeatureUtils.getObsSwaDataTimeRange(observationDf, featureJoinConfig.settings)._1.get + } else null + TimeConfigSettings(timeConfigSettings = featureJoinConfig.settings, + featuresToTimeDelayMap = featureJoinConfig.featuresToTimeDelayMap, obsTimeRange = obsTimeRange) + } + + /** + * Create map of node ID to feature name + * @param nodes Buffer of all nodes in compute graph + * @return Map of node id to feature name + */ + private def getNodeIdToFeatureNameMap(nodes: mutable.Buffer[AnyNode]): Map[Integer, String] = { + val derivedFeatureAliasMap: Map[Integer, String] = resolvedGraph.getFeatureNames.asScala.map(x => x._2 -> x._1).toMap + nodes.filter(node => node.isLookup || node.isAggregation || node.isTransformation).map(node => + if (node.isLookup) { + if (derivedFeatureAliasMap.contains(node.getLookup.getId)) { + (node.getLookup.getId, derivedFeatureAliasMap(node.getLookup.getId)) + } else { + (node.getLookup.getId, node.getLookup.getFeatureName) + } + } else if (node.isAggregation) { + if (derivedFeatureAliasMap.contains(node.getAggregation.getId)) { + (node.getAggregation.getId, derivedFeatureAliasMap(node.getAggregation.getId)) + } else { + (node.getAggregation.getId, node.getAggregation.getFeatureName) + } + } else { + if (derivedFeatureAliasMap.contains(node.getTransformation.getId)) { + (node.getTransformation.getId, derivedFeatureAliasMap(node.getTransformation.getId)) + } else if (node.getTransformation.hasFeatureName) { + (node.getTransformation.getId, node.getTransformation.getFeatureName) + } else { + (node.getTransformation.getId, "__seq__join__feature") // TODO: Currently have hacky hard coded names, should add logic for generating names. + } + } + ).toMap + } + + /** + * Given a node, return the unfinished dependencies as a set of node ids. + * @param node + * @return + */ + private def getUnfinishedDependencies(node: AnyNode, visitedState: Array[VisitedState]): Set[Integer] = { + val dependencies = new Dependencies().getDependencies(node).asScala + dependencies.filter(visitedState(_) != VISITED).toSet + } + + /** + * The main graph traversal function for FCMGraphTraverser. Graph traversal algo: + * 1. Create optimizedGrouping map which specifies if nodes should be executed in the same group. + * 2. Push all requested nodes onto a stack. + * 3. Pop a node and evaluate it. + * a. For each node evaluation, first check if all the node's dependecies have been visited. If they have not, + * push all dependency nodes onto the stack and push the node back onto the stack after marking it as IN_PROGRESS. + * b. If all node's dependecies have been visited, pass the node to the appropriate node evaluator. + * c. Update the contextDf with the output of the node evaluation. + * d. Mark node as VISITED + * 4. Convert contextDf to FDS and return as FeatureDataFrame + * @return FeatureDataFrame + */ + def traverseGraph(): FeatureDataFrame = { + // Set up stack for graph traversal + val stack = mutable.Stack[Int]() + var contextDf: DataFrame = observationDf + + // Optimization: Group all transformation nodes with the same input nodes, keys and transformation function operators. + val optimizedGroupingMap = groupTransformationNodes(nodes) ++ groupAllSWANodes(nodes) + val nodeRankingMap = resolvedGraph.getFeatureNames.asScala.values.map(x => if (nodes(x).isAggregation) x -> 1 else x -> 2).toMap + // Push all requested nodes onto stack processing. + val visitedState: Array[VisitedState] = Array.fill[VisitedState](nodes.length)(NOT_VISITED) + resolvedGraph.getFeatureNames.asScala.values.foreach(x => stack.push(x)) + while (stack.nonEmpty) { + stack.sortBy {case(i) => nodeRankingMap.get(i) } + val nodeId = stack.pop + if (visitedState(nodeId) != VISITED) { + val node = nodes(nodeId) + // If node is part of an optimized grouping, we have to consider the dependencies of the other nodes in the group also + val unfinishedDependencies = optimizedGroupingMap.getOrElse(nodeId, Seq(new Integer(nodeId))) + .foldLeft(Set.empty[Integer])((unfinishedSet, currNodeId) => { + unfinishedSet ++ getUnfinishedDependencies(nodes(currNodeId), visitedState) + }) + if (unfinishedDependencies.nonEmpty) { + if (visitedState(nodeId) == IN_PROGRESS) { + throw new RuntimeException("Encountered dependency cycle involving node " + nodeId) + } + stack.push(nodeId) // revisit this node after its dependencies + unfinishedDependencies.foreach(stack.push(_)) // visit dependencies + visitedState(nodeId) = IN_PROGRESS + } else { + // actually handle this node, since all its dependencies (if any) are ready + assert(!nodeIdToDataframeAndColumnMetadataMap.contains(nodeId)) + // If the optimized grouping map contains this nodeId and all the dependencies are finished, we know we can batch evaluate these nodes now. + // We assume all nodes in a group are the same type, if the grouping fails this criteria then we will throw an error within the evaluator. + contextDf = if (optimizedGroupingMap.contains(nodeId)) { + node match { + // Currently the batch datasource and batch lookup case will not be used as we do not have an optimization for those node types. + case node if node.isDataSource => DataSourceNodeEvaluator.batchEvaluate(optimizedGroupingMap(nodeId).map(nodes(_)), this, contextDf, + dataPathHandlers) + case node if node.isLookup => LookupNodeEvaluator.batchEvaluate(optimizedGroupingMap(nodeId).map(nodes(_)), this, contextDf, dataPathHandlers) + case node if node.isTransformation => TransformationNodeEvaluator.batchEvaluate(optimizedGroupingMap(nodeId).map(nodes(_)), this, contextDf, dataPathHandlers) + case node if node.isAggregation => AggregationNodeEvaluator.batchEvaluate(optimizedGroupingMap(nodeId).map(nodes(_)), this, contextDf, dataPathHandlers) + case node if node.isExternal => throw new RuntimeException(s"External node found in resolved graph traversal. Node information: $node") + } + } else { + node match { + case node if node.isDataSource => DataSourceNodeEvaluator.evaluate(node, this, contextDf, dataPathHandlers) + case node if node.isLookup => LookupNodeEvaluator.evaluate(node, this, contextDf, dataPathHandlers) + case node if node.isTransformation => TransformationNodeEvaluator.evaluate(node, this, contextDf, dataPathHandlers) + case node if node.isAggregation => AggregationNodeEvaluator.evaluate(node, this, contextDf, dataPathHandlers) // No processing needed for SWA nodes at this stage. + case node if node.isExternal => throw new RuntimeException(s"External node found in resolved graph traversal. Node information: $node") + } + } + // Mark batch or single node as visited. + if (optimizedGroupingMap.contains(nodeId)) { + optimizedGroupingMap(nodeId).foreach(visitedState(_) = VISITED) + } else { + visitedState(nodeId) = VISITED + } + } + } + } + + // Drop all unneeded columns and return the result after FDS conversion + val featureTypeConfigs = getFeatureTypeConfigsMap(nodes) + val necessaryColumns = resolvedGraph.getFeatureNames.asScala.keys ++ observationDf.columns + val toDropCols = contextDf.columns diff necessaryColumns.toSeq + contextDf = contextDf.drop(toDropCols: _*) + convertFCMResultDFToFDS(resolvedGraph.getFeatureNames.asScala.keys.toSeq, + featureColumnFormatsMap.toMap, contextDf, featureTypeConfigs) + } +} diff --git a/feathr-impl/src/main/scala/com/linkedin/feathr/offline/graph/NodeGrouper.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/graph/NodeGrouper.scala new file mode 100644 index 000000000..9c4a7c247 --- /dev/null +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/graph/NodeGrouper.scala @@ -0,0 +1,97 @@ +package com.linkedin.feathr.offline.graph + +import com.linkedin.feathr.compute.{AnyNode, ConcreteKey, NodeReference, Operators} +import com.linkedin.feathr.offline.client.plugins.{AnchorExtractorAdaptor, FeathrUdfPluginContext, SimpleAnchorExtractorSparkAdaptor} + +import scala.collection.mutable + +/** + * This NodeGrouper class contains utility functions which group nodes into batches. This exists because we have optimizations + * where SWA and anchor features are best transformed together in a group so we need to signal to the node evaluators via + * these groupings that certain nodes (like all SWA, all transformation nodes with same extractor, etc.) can be executed + * together as a group. + */ +object NodeGrouper { + /** + * Given a set of nodes, group the Aggregation nodes and return a map of node id to seq of nodes in the same group. + * By grouping the nodes we can minimize the number of calls to the SWJ library and minimize the number of spark operations needed. + * Grouping criteria: is we group all aggregation nodes which have the same concrete key. + * @param nodes Buffer of nodes + * @return Map of node id to seq of node id's in the same group + */ + def groupSWANodes(nodes: Seq[AnyNode]): mutable.HashMap[Integer, Seq[Integer]] = { + val allSWANodes = nodes.filter(node => node.getAggregation != null) + val swaMap = mutable.Map[ConcreteKey, Seq[Integer]]() + allSWANodes.map (node => { + val concreteKey = node.getAggregation.getConcreteKey + if (!swaMap.contains(concreteKey)) swaMap.put(concreteKey, Seq(node.getAggregation.getId())) + else { + val existingGroup = swaMap(concreteKey) + val updatedGroup = existingGroup :+ node.getAggregation.getId() + swaMap.put(concreteKey, updatedGroup) + } + }) + val groupedAggregationNodeMap = mutable.HashMap.empty[Integer, Seq[Integer]] + swaMap.values.map(nodeArray => { + nodeArray.map(node => groupedAggregationNodeMap.put(node, nodeArray)) + }) + groupedAggregationNodeMap + } + + /** + * Given a buffer of nodes, return a map of all SWA nodes. Map keys are node id of swa nodes and value will be + * a seq of all swa node ids. Purpose of this grouping is that all SWA nodes should be evaluated together as a + * group to optimize performance of SWJ library. + * @param nodes + * @return + */ + def groupAllSWANodes(nodes: mutable.Buffer[AnyNode]): Map[Integer, Seq[Integer]] = { + val allSWANodes = nodes.filter(node => node.getAggregation != null).map(node => node.getAggregation.getId) + allSWANodes.map(node => (node, allSWANodes)).toMap + } + + /** + * Given a set of nodes, group specifically the anchor feature nodes and return a map of node id to seq of node id's in the same + * group. Note here that the definition of an anchor feature node is a transformation node which has a data source node as input. + * The purpose of grouping here is to minimize the number of calls to the different operators such that nodes that can be + * computed in the same step will be computed in the same step. For example, we want to group all MVEL operations so that we apply + * the MVEL transformations on each row only one time and not one time per node. + * Grouping criteria: nodes with the same concrete key and same transformation operator will be grouped together. + * @param nodes Buffer of nodes + * @return Map of node id to seq of node id's in the same group + */ + def groupTransformationNodes(nodes: mutable.Buffer[AnyNode]): Map[Integer, Seq[Integer]] = { + val allAnchorTransformationNodes = nodes.filter(node => node.getTransformation != null && node.getTransformation.getInputs.size() == 1 && + nodes(node.getTransformation.getInputs.get(0).getId()).isDataSource) + val transformationNodesMap = mutable.Map[(NodeReference, ConcreteKey, String, String), Seq[Integer]]() + allAnchorTransformationNodes.map(node => { + val inputNode = node.getTransformation.getInputs().get(0) // Already assumed that it is an anchored transformation node + val concreteKey = node.getTransformation.getConcreteKey + val transformationOperator = node.getTransformation.getFunction().getOperator() + val extractorClass = if (transformationOperator == Operators.OPERATOR_ID_ANCHOR_JAVA_UDF_FEATURE_EXTRACTOR) { + val className = node.getTransformation.getFunction().getParameters.get("class") + FeathrUdfPluginContext.getRegisteredUdfAdaptor(Class.forName(className)) match { + case Some(adaptor: AnchorExtractorAdaptor) => + "rowExtractor" + case _ => className + case None => className + } + } else { + "non_java_udf" + } + + if (!transformationNodesMap.contains((inputNode, concreteKey, transformationOperator, extractorClass))) { + transformationNodesMap.put((inputNode, concreteKey, transformationOperator, extractorClass), Seq(node.getTransformation.getId())) + } else { + val existingGroup = transformationNodesMap(inputNode, concreteKey, transformationOperator, extractorClass) + val updatedGroup = existingGroup :+ node.getTransformation.getId() + transformationNodesMap.put((inputNode, concreteKey, transformationOperator, extractorClass), updatedGroup) + } + }) + + transformationNodesMap.values.foldLeft(Map.empty[Integer, Seq[Integer]])((groupMap, nodes) => { + groupMap ++ nodes.map(node => (node, nodes)).toMap + }) + } + +} diff --git a/feathr-impl/src/main/scala/com/linkedin/feathr/offline/graph/NodeUtils.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/graph/NodeUtils.scala new file mode 100644 index 000000000..bd22f2ad5 --- /dev/null +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/graph/NodeUtils.scala @@ -0,0 +1,95 @@ +package com.linkedin.feathr.offline.graph + +import com.linkedin.feathr.common.{FeatureTypeConfig, FeatureValue, JoiningFeatureParams} +import com.linkedin.feathr.compute.{AnyNode, Transformation} +import com.linkedin.feathr.compute.Resolver.FeatureRequest +import com.linkedin.feathr.offline.anchored.WindowTimeUnit +import com.linkedin.feathr.offline.config.{FeatureJoinConfig, PegasusRecordDefaultValueConverter, PegasusRecordFeatureTypeConverter} +import com.linkedin.feathr.offline.util.FCMUtils.makeFeatureNameForDuplicates + +import java.time.Duration +import scala.collection.JavaConverters.seqAsJavaListConverter + +/** + * This object class contains helper functions which extract information (like feature type and default values) from nodes + * and returns them in data formats which our API's can work with. + */ +object NodeUtils { + /** + * Given the feathr join config, create the list of FeatureRequest to be consumed by the FCM graph resolver. + * @param joinConfig feathr join config + * @return List of FeatureRequest to be consumed by FCM graph resolver + */ + def getFeatureRequestsFromJoinConfig(joinConfig: FeatureJoinConfig): List[FeatureRequest] = { + val featureNames = joinConfig.joinFeatures.map(_.featureName) + val duplicateFeatureNames = featureNames.diff(featureNames.distinct).distinct + joinConfig.joinFeatures.map { + case JoiningFeatureParams(keyTags, featureName, dateParam, timeDelay, featureAlias) => + val delay = if (timeDelay.isDefined) { + WindowTimeUnit.parseWindowTime(timeDelay.get) + } else { + if (joinConfig.settings.isDefined && joinConfig.settings.get.joinTimeSetting.isDefined && + joinConfig.settings.get.joinTimeSetting.get.simulateTimeDelay.isDefined) { + joinConfig.settings.get.joinTimeSetting.get.simulateTimeDelay.get + } else { + Duration.ZERO + } + } + // In the case of duplicate feature names in the join config, according to feathr offline specs the feature name will be created as + // keys + __ + name. For example a feature "foo" with keys key0 and key1 will be named key0_key1__foo. + if (duplicateFeatureNames.contains(featureName)) { + new FeatureRequest(featureName, keyTags.toList.asJava, delay, makeFeatureNameForDuplicates(keyTags, featureName)) + } else { + new FeatureRequest(featureName, keyTags.toList.asJava, delay, featureAlias.orNull) + } + }.toList + } + + /** + * Create map of feature name to feature type config + * @param nodes Seq of any nodes. + * @return Map of node id to feature type config + */ + def getFeatureTypeConfigsMap(nodes: Seq[AnyNode]): Map[String, FeatureTypeConfig] = { + nodes.filter(node => node.isLookup || node.isAggregation || node.isTransformation).map { + case n if n.isTransformation => n.getTransformation.getFeatureName -> PegasusRecordFeatureTypeConverter().convert(n.getTransformation.getFeatureVersion) + case n if n.isLookup => n.getLookup.getFeatureName -> PegasusRecordFeatureTypeConverter().convert(n.getLookup.getFeatureVersion) + case n if n.isAggregation => n.getAggregation.getFeatureName -> PegasusRecordFeatureTypeConverter().convert(n.getAggregation.getFeatureVersion) + }.collect { case (key, Some(value)) => (key, value) }.toMap // filter out Nones and get rid of Option + } + + /** + * Create map of feature name to feature type config + * @param nodes Seq of Transformation nodes + * @return Map of node id to feature type config + */ + def getFeatureTypeConfigsMapForTransformationNodes(nodes: Seq[Transformation]): Map[String, FeatureTypeConfig] = { + nodes.map { n => n.getFeatureName -> PegasusRecordFeatureTypeConverter().convert(n.getFeatureVersion) + }.collect { case (key, Some(value)) => (key, value) }.toMap // filter out Nones and get rid of Option + } + + /** + * Create default value converter for nodes + * @param nodes Seq of any nodes + * @return Map[String, FeatureValue] where key is feature name. + */ + def getDefaultConverter(nodes: Seq[AnyNode]): Map[String, FeatureValue] = { + val featureVersionMap = nodes.filter(node => node.isLookup || node.isAggregation || node.isTransformation).map { + case n if n.isTransformation => n.getTransformation.getFeatureName -> n.getTransformation.getFeatureVersion + case n if n.isLookup => n.getLookup.getFeatureName -> n.getLookup.getFeatureVersion + case n if n.isAggregation => n.getAggregation.getFeatureName -> n.getAggregation.getFeatureVersion + }.toMap + PegasusRecordDefaultValueConverter().convert(featureVersionMap) + } + + /** + * Create default value converter for Transformation nodes + * @param nodes Seq of Transformation + * @return Map[String, FeatureValue] where key is feature name. + */ + def getDefaultConverterForTransformationNodes(nodes: Seq[Transformation]): Map[String, FeatureValue] = { + val featureVersionMap = nodes.map { n => n.getFeatureName -> n.getFeatureVersion }.toMap + PegasusRecordDefaultValueConverter().convert(featureVersionMap) + } +} + diff --git a/src/main/scala/com/linkedin/feathr/offline/job/DataFrameStatFunctions.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/job/DataFrameStatFunctions.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/job/DataFrameStatFunctions.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/job/DataFrameStatFunctions.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/job/DataSourceUtils.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/job/DataSourceUtils.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/job/DataSourceUtils.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/job/DataSourceUtils.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/job/FeathrUdfRegistry.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/job/FeathrUdfRegistry.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/job/FeathrUdfRegistry.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/job/FeathrUdfRegistry.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/job/FeatureGenConfigOverrider.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/job/FeatureGenConfigOverrider.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/job/FeatureGenConfigOverrider.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/job/FeatureGenConfigOverrider.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/job/FeatureGenContext.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/job/FeatureGenContext.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/job/FeatureGenContext.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/job/FeatureGenContext.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/job/FeatureGenJob.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/job/FeatureGenJob.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/job/FeatureGenJob.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/job/FeatureGenJob.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/job/FeatureGenSpec.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/job/FeatureGenSpec.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/job/FeatureGenSpec.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/job/FeatureGenSpec.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/job/FeatureJoinJob.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/job/FeatureJoinJob.scala similarity index 86% rename from src/main/scala/com/linkedin/feathr/offline/job/FeatureJoinJob.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/job/FeatureJoinJob.scala index ef01044d1..3f3f7be05 100644 --- a/src/main/scala/com/linkedin/feathr/offline/job/FeatureJoinJob.scala +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/job/FeatureJoinJob.scala @@ -73,11 +73,12 @@ object FeatureJoinJob { checkAuthorization(ss, hadoopConf, jobContext, dataLoaderHandlers) feathrJoinRun(ss=ss, - hadoopConf=hadoopConf, - joinConfig=joinConfig, - jobContext=jobContext.jobJoinContext, - localTestConfig=None, - dataPathHandlers=dataPathHandlers) + hadoopConf=hadoopConf, + joinConfig=joinConfig, + jobContext=jobContext.jobJoinContext, + localTestConfig=None, + dataPathHandlers=dataPathHandlers, + useFCM = jobContext.useFCM) } // Log the feature names for bookkeeping. Global config may be merged with local config(s). @@ -163,6 +164,52 @@ object FeatureJoinJob { } } + /** + * This function will get the FCM client using the spark session and jobContext, and call FCM client (FeathrClient2)#joinObsAndFeatures + * method. + * @param ss spark session + * @param observations observations DF + * @param featureGroupings feature groups to join + * @param joinConfig join config + * @param jobContext job context + * @param localTestConfigOpt Local test config + * @return Dataframe and header associated with it. + */ + private[offline] def getFCMClientAndJoinFeatures( + ss: SparkSession, + observations: DataFrame, + featureGroupings: Map[String, Seq[JoiningFeatureParams]], + joinConfig: FeatureJoinConfig, + jobContext: JoinJobContext, + dataPathHandlers: List[DataPathHandler], + localTestConfigOpt: Option[LocalTestConfig] = None): DataFrame = { + + val feathrClient2 = getFCMClient(ss, jobContext, dataPathHandlers, localTestConfigOpt) + feathrClient2.joinFeatures(joinConfig, SparkFeaturizedDataset(observations, FeaturizedDatasetMetadata()), jobContext)._1.df + } + + private[offline] def getFCMClient( + ss: SparkSession, + jobContext: JoinJobContext, + dataPathHandlers: List[DataPathHandler], + localTestConfigOpt: Option[LocalTestConfig] = None): FeathrClient2 = { + + localTestConfigOpt match { + case None => + FeathrClient2.builder(ss) + .addFeatureDefPath(jobContext.feathrFeatureConfig) + .addLocalOverrideDefPath(jobContext.feathrLocalConfig) + .addDataPathHandlers(dataPathHandlers) + .build() + case Some(localTestConfig) => + FeathrClient2.builder(ss) + .addFeatureDef(localTestConfig.featureConfig) + .addLocalOverrideDef(localTestConfig.localConfig) + .addDataPathHandlers(dataPathHandlers) + .build() + } + } + /** * This function will collect the data, build the schema and do the join work for hdfs records. * @@ -179,7 +226,8 @@ object FeatureJoinJob { joinConfig: FeatureJoinConfig, jobContext: JoinJobContext, dataPathHandlers: List[DataPathHandler], - localTestConfig: Option[LocalTestConfig] = None): (Option[RDD[GenericRecord]], Option[DataFrame]) = { + localTestConfig: Option[LocalTestConfig] = None, + useFCM: Boolean = false): (Option[RDD[GenericRecord]], Option[DataFrame]) = { val sparkConf = ss.sparkContext.getConf val dataLoaderHandlers: List[DataLoaderHandler] = dataPathHandlers.map(_.dataLoaderHandler) val featureGroupings = joinConfig.featureGroupings @@ -190,7 +238,11 @@ object FeatureJoinJob { val failOnMissing = FeathrUtils.getFeathrJobParam(ss, FeathrUtils.FAIL_ON_MISSING_PARTITION).toBoolean val observationsDF = SourceUtils.loadObservationAsDF(ss, hadoopConf, jobContext.inputData.get, dataLoaderHandlers, failOnMissing) - val (joinedDF, _) = getFeathrClientAndJoinFeatures(ss, observationsDF, featureGroupings, joinConfig, jobContext, dataPathHandlers, localTestConfig) + val joinedDF = if (useFCM) { + getFCMClientAndJoinFeatures(ss, observationsDF, featureGroupings, joinConfig, jobContext, dataPathHandlers, localTestConfig) + } else { + getFeathrClientAndJoinFeatures(ss, observationsDF, featureGroupings, joinConfig, jobContext, dataPathHandlers, localTestConfig)._1 + } val parameters = Map(SparkIOUtils.OUTPUT_PARALLELISM -> jobContext.numParts.toString, SparkIOUtils.OVERWRITE_MODE -> "ALL") @@ -231,6 +283,8 @@ object FeatureJoinJob { "blob-config" -> OptionParam("bc", "Authentication config for Azure Blob Storage (wasb)", "BLOB_CONFIG", ""), "sql-config" -> OptionParam("sqlc", "Authentication config for Azure SQL Database (jdbc)", "SQL_CONFIG", ""), "snowflake-config" -> OptionParam("sfc", "Authentication config for Snowflake Database (jdbc)", "SNOWFLAKE_CONFIG", ""), + "use-fcm" -> OptionParam("ufcm", "If set to true, use FCM client, else use Feathr Client", "USE_FCM", "false"), + "snowflake-config" -> OptionParam("sfc", "Authentication config for Snowflake Database (jdbc)", "SNOWFLAKE_CONFIG", ""), "system-properties" -> OptionParam("sps", "Additional System Properties", "SYSTEM_PROPERTIES_CONFIG", "") ) @@ -280,7 +334,8 @@ object FeatureJoinJob { } val dataSourceConfigs = DataSourceConfigUtils.getConfigs(cmdParser) - FeathrJoinJobContext(joinConfig, joinJobContext, dataSourceConfigs) + val useFCM = cmdParser.extractRequiredValue("use-fcm").toBoolean + FeathrJoinJobContext(joinConfig, joinJobContext, dataSourceConfigs, useFCM) } type KeyTag = Seq[String] @@ -383,7 +438,7 @@ object FeatureJoinJob { case class FeathrJoinPreparationInfo(sparkSession: SparkSession, hadoopConf: Configuration, jobContext: FeathrJoinJobContext) -case class FeathrJoinJobContext(joinConfig: String, jobJoinContext: JoinJobContext, dataSourceConfigs: DataSourceConfigs) {} +case class FeathrJoinJobContext(joinConfig: String, jobJoinContext: JoinJobContext, dataSourceConfigs: DataSourceConfigs, useFCM: Boolean) {} /** * This case class describes feature record after join process diff --git a/src/main/scala/com/linkedin/feathr/offline/job/FeatureTransformation.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/job/FeatureTransformation.scala similarity index 90% rename from src/main/scala/com/linkedin/feathr/offline/job/FeatureTransformation.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/job/FeatureTransformation.scala index 7b106572b..aa0d7c038 100644 --- a/src/main/scala/com/linkedin/feathr/offline/job/FeatureTransformation.scala +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/job/FeatureTransformation.scala @@ -6,7 +6,7 @@ import com.linkedin.feathr.common.types.FeatureType import com.linkedin.feathr.common.{AnchorExtractorBase, _} import com.linkedin.feathr.offline.anchored.anchorExtractor.{SQLConfigurableAnchorExtractor, SimpleConfigurableAnchorExtractor, TimeWindowConfigurableAnchorExtractor} import com.linkedin.feathr.offline.anchored.feature.{FeatureAnchor, FeatureAnchorWithSource} -import com.linkedin.feathr.offline.anchored.keyExtractor.MVELSourceKeyExtractor +import com.linkedin.feathr.offline.anchored.keyExtractor.{MVELSourceKeyExtractor, SpecificRecordSourceKeyExtractor} import com.linkedin.feathr.offline.client.DataFrameColName import com.linkedin.feathr.offline.config.{MVELFeatureDefinition, TimeWindowFeatureDefinition} import com.linkedin.feathr.offline.generation.IncrementalAggContext @@ -888,6 +888,7 @@ private[offline] object FeatureTransformation { val features = transformers map { case extractor: AnchorExtractor[IndexedRecord] => val features = extractor.getFeatures(record) + print(features) FeatureValueTypeValidator.validate(features, featureTypeConfigs) features case extractor => @@ -1298,6 +1299,159 @@ private[offline] object FeatureTransformation { } } + + /** + * Convert the dataframe that results are the end of all node execution to QUINCE_FDS tensors. Note that we expect some + * columns to already be in FDS format and FeatureColumnFormats map will tell us that. Some transformation operators + * and nodes will return the column in FDS format so we do not need to do conversion in that instance. + * @param allFeaturesToConvert all features to convert + * @param featureColumnFormatsMap transformer returned result + * @param withFeatureDF input dataframe with all requested features + * @param userProvidedFeatureTypeConfigs user provided feature types + * @return dataframe in FDS format + */ + def convertFCMResultDFToFDS( + allFeaturesToConvert: Seq[String], + featureColumnFormatsMap: Map[String, FeatureColumnFormat], + withFeatureDF: DataFrame, + userProvidedFeatureTypeConfigs: Map[String, FeatureTypeConfig] = Map()): FeatureDataFrame = { + // 1. infer the feature types if they are not done by the transformers above + val defaultInferredFeatureTypes = inferFeatureTypesFromRawDF(withFeatureDF, allFeaturesToConvert) + val transformedInferredFeatureTypes = defaultInferredFeatureTypes + val featureColNameToFeatureNameAndType = + allFeaturesToConvert.map { featureName => + val userProvidedConfig = userProvidedFeatureTypeConfigs.getOrElse(featureName, FeatureTypeConfig.UNDEFINED_TYPE_CONFIG) + val userProvidedFeatureType = userProvidedConfig.getFeatureType + val processedFeatureTypeConfig = if (userProvidedFeatureType == FeatureTypes.UNSPECIFIED) { + transformedInferredFeatureTypes.getOrElse(featureName, FeatureTypeConfig.UNDEFINED_TYPE_CONFIG) + } else userProvidedConfig + val colName = featureName + (colName, (featureName, processedFeatureTypeConfig)) + }.toMap + val inferredFeatureTypes = featureColNameToFeatureNameAndType.map { + case (_, (featureName, featureType)) => + featureName -> featureType + } + + // 2. convert to QUINCE_FDS + val convertedDF = featureColNameToFeatureNameAndType + .groupBy(pair => featureColumnFormatsMap(pair._1)) + .foldLeft(withFeatureDF)((inputDF, featureColNameToFeatureNameAndTypeWithFormat) => { + val fdsDF = featureColNameToFeatureNameAndTypeWithFormat._1 match { + case FeatureColumnFormat.FDS_TENSOR => + inputDF + case FeatureColumnFormat.RAW => + // sql extractor return rawDerivedFeatureEvaluator.scala (Diff rev + val convertedDF = FeaturizedDatasetUtils.convertRawDFtoQuinceFDS(inputDF, featureColNameToFeatureNameAndType) + convertedDF + } + fdsDF + }) + FeatureDataFrame(convertedDF, inferredFeatureTypes) + } + + /** + * This method is used to strip off the function name, ie - USER_FACING_MULTI_DIM_FDS_TENSOR_UDF_NAME. + * For example, if the featureDef: FDSExtract(f1), then only f1 will be returned. + * @param featureDef feature definition expression with the keyword (USER_FACING_MULTI_DIM_FDS_TENSOR_UDF_NAME) + * @return feature def expression after stripping off the keyword (USER_FACING_MULTI_DIM_FDS_TENSOR_UDF_NAME) + */ + def parseMultiDimTensorExpr(featureDef: String): String = { + // String char should be one more than the len of the keyword to account for '('. The end should be 1 less than length of feature string + // to account for ')'. + featureDef.substring(featureDef.indexOf("(") + 1, featureDef.indexOf(")")) + } + + + def applyRowBasedTransformOnRdd(userProvidedFeatureTypes: Map[String, FeatureTypes], requestedFeatureNames: Seq[String], + inputRdd: RDD[_], sourceKeyExtractors: Seq[SourceKeyExtractor], transformers: Seq[AnchorExtractorBase[Any]], + featureTypeConfigs: Map[String, FeatureTypeConfig]): (DataFrame, Seq[String]) = { + /* + * Transform the given RDD by applying extractors to each row to create an RDD[Row] where each Row + * represents keys and feature values + */ + val spark = SparkSession.builder().getOrCreate() + val FeatureTypeInferenceContext(featureTypeAccumulators) = + FeatureTransformation.getTypeInferenceContext(spark, userProvidedFeatureTypes, requestedFeatureNames) + val transformedRdd = inputRdd map { row => + val (keys, featureValuesWithType) = transformRow(requestedFeatureNames, sourceKeyExtractors, transformers, row, featureTypeConfigs) + requestedFeatureNames.zip(featureValuesWithType).foreach { + case (featureRef, (_, featureType)) => + if (featureTypeAccumulators(featureRef).isZero && featureType != null) { + // This is lazy evaluated + featureTypeAccumulators(featureRef).add(FeatureTypes.valueOf(featureType.getBasicType.toString)) + } + } + // Create a row by merging a row created from keys and a row created from term-vectors/tensors + Row.merge(Row.fromSeq(keys), Row.fromSeq(featureValuesWithType.map(_._1))) + } + + // Create a DataFrame from the above obtained RDD + val keyNames = getFeatureKeyColumnNamesRdd(sourceKeyExtractors.head, inputRdd) + val (outputSchema, inferredFeatureTypeConfigs) = { + val inferredFeatureTypes = inferFeatureTypes(featureTypeAccumulators, transformedRdd, requestedFeatureNames) + val inferredFeatureTypeConfigs = inferredFeatureTypes.map(x => x._1 -> new FeatureTypeConfig(x._2)) + val mergedFeatureTypeConfig = inferredFeatureTypeConfigs ++ featureTypeConfigs + val colPrefix = "" + val featureTensorTypeInfo = getFDSSchemaFields(requestedFeatureNames, mergedFeatureTypeConfig, colPrefix) + val structFields = keyNames.foldRight(List.empty[StructField]) { + case (colName, acc) => + StructField(colName, StringType) :: acc + } + val outputSchema = StructType(StructType(structFields ++ featureTensorTypeInfo)) + (outputSchema, mergedFeatureTypeConfig) + } + (spark.createDataFrame(transformedRdd, outputSchema), keyNames) + } + + private def transformRow( + requestedFeatureNames: Seq[FeatureName], + sourceKeyExtractors: Seq[SourceKeyExtractor], + transformers: Seq[AnchorExtractorBase[Any]], + row: Any, + featureTypeConfigs: Map[String, FeatureTypeConfig] = Map()): (Seq[String], Seq[(Any, FeatureType)]) = { + val keys = sourceKeyExtractors.head match { + case mvelSourceKeyExtractor: MVELSourceKeyExtractor => mvelSourceKeyExtractor.getKey(row) + case specificSourceKeyExtractor: SpecificRecordSourceKeyExtractor => specificSourceKeyExtractor.getKey(row) + case _ => throw new FeathrFeatureTransformationException(ErrorLabel.FEATHR_USER_ERROR, s"${sourceKeyExtractors.head} is not a valid extractor on RDD") + } + + /* + * For the given row, apply all extractors to extract feature values. If requested as tensors, each feature value + * contains a tensor else a term-vector. + */ + val features = transformers map { + case extractor: AnchorExtractor[Any] => + val features = extractor.getFeatures(row) + print(features) + FeatureValueTypeValidator.validate(features, featureTypeConfigs) + features + case extractor => + throw new FeathrFeatureTransformationException( + ErrorLabel.FEATHR_USER_ERROR, + s"Invalid extractor $extractor for features:" + + s"$requestedFeatureNames requested as tensors") + } reduce (_ ++ _) + if (logger.isTraceEnabled) { + logger.trace(s"Extracted features: $features") + } + + /* + * Retain feature values for only the requested features, and represent each feature value as a term-vector or as + * a tensor, as specified. If tensors are required, create a row for each feature value (that is, the tensor). + */ + val featureValuesWithType = requestedFeatureNames map { name => + features.get(name) map { + case featureValue => + val tensorData: TensorData = featureValue.getAsTensorData() + val featureType: FeatureType = featureValue.getFeatureType() + val row = FeaturizedDatasetUtils.tensorToFDSDataFrameRow(tensorData) + (row, featureType) + } getOrElse ((null, null)) // return null if no feature value present + } + (keys, featureValuesWithType) + } + /** * Get standardized key names for feature generation, e.g. key0, key1, key2, etc. * @param joinKeySize number of join keys diff --git a/src/main/scala/com/linkedin/feathr/offline/job/JoinJobContext.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/job/JoinJobContext.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/job/JoinJobContext.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/job/JoinJobContext.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/job/LocalFeatureGenJob.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/job/LocalFeatureGenJob.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/job/LocalFeatureGenJob.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/job/LocalFeatureGenJob.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/job/LocalFeatureJoinJob.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/job/LocalFeatureJoinJob.scala similarity index 90% rename from src/main/scala/com/linkedin/feathr/offline/job/LocalFeatureJoinJob.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/job/LocalFeatureJoinJob.scala index 4a38d2304..aa92cd546 100644 --- a/src/main/scala/com/linkedin/feathr/offline/job/LocalFeatureJoinJob.scala +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/job/LocalFeatureJoinJob.scala @@ -1,6 +1,6 @@ package com.linkedin.feathr.offline.job -import com.linkedin.feathr.offline.client.FeathrClient +import com.linkedin.feathr.offline.client.{FeathrClient, FeathrClient2} import com.linkedin.feathr.offline.config.FeatureJoinConfig import com.linkedin.feathr.offline.mvel.plugins.FeathrExpressionExecutionContext import com.linkedin.feathr.offline.source.dataloader.DataLoaderHandler @@ -23,6 +23,7 @@ object LocalFeatureJoinJob { /** * local debug API, used in unit test and local debug + * * @param joinConfigAsHoconString feature join config as HOCON config string * @param featureDefAsString feature def config * @param observationData observation data @@ -38,11 +39,8 @@ object LocalFeatureJoinJob { dataPathHandlers: List[DataPathHandler], mvelContext: Option[FeathrExpressionExecutionContext]): SparkFeaturizedDataset = { val joinConfig = FeatureJoinConfig.parseJoinConfig(joinConfigAsHoconString) - val feathrClient = FeathrClient.builder(ss) - .addFeatureDef(featureDefAsString) - .addDataPathHandlers(dataPathHandlers) - .addFeathrExpressionContext(mvelContext) - .build() + val feathrClient = FeathrClient.builder(ss).addFeatureDef(featureDefAsString).addDataPathHandlers(dataPathHandlers) + .addFeathrExpressionContext(mvelContext).build() val outputPath: String = FeatureJoinJob.SKIP_OUTPUT val defaultParams = Array( @@ -53,7 +51,7 @@ object LocalFeatureJoinJob { outputPath) val jobContext = FeatureJoinJob.parseInputArgument(defaultParams ++ extraParams).jobJoinContext - feathrClient.joinFeatures(joinConfig, observationData, jobContext) + SparkFeaturizedDataset(feathrClient.joinFeatures(joinConfig, observationData, jobContext).data, FeaturizedDatasetMetadata()) } /** @@ -87,7 +85,7 @@ object LocalFeatureJoinJob { val dataLoaderFactory = DataLoaderFactory(ss, dataLoaderHandlers=dataLoaderHandlers) val data = source.pathList.map(dataLoaderFactory.create(_).loadDataFrame()).reduce(_ union _) - SparkFeaturizedDataset(data,FeaturizedDatasetMetadata()) + SparkFeaturizedDataset(data, FeaturizedDatasetMetadata()) } } diff --git a/src/main/scala/com/linkedin/feathr/offline/job/OutputUtils.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/job/OutputUtils.scala similarity index 73% rename from src/main/scala/com/linkedin/feathr/offline/job/OutputUtils.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/job/OutputUtils.scala index 54d75eccb..c271a1b3b 100644 --- a/src/main/scala/com/linkedin/feathr/offline/job/OutputUtils.scala +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/job/OutputUtils.scala @@ -1,5 +1,6 @@ package com.linkedin.feathr.offline.job +import com.linkedin.avroutil1.compatibility.AvroCompatibilityHelper import com.linkedin.feathr.common.{Header, JoiningFeatureParams} import org.apache.avro.Schema @@ -48,7 +49,7 @@ private[offline] object OutputUtils { val compactFeatureSchemaFloat = { val schema = Schema.createRecord("Feature", null, null, false) schema.setFields(util.Arrays - .asList(new Schema.Field("term", Schema.create(Schema.Type.STRING), null, null), new Schema.Field("value", Schema.create(Schema.Type.FLOAT), null, null))) + .asList(AvroCompatibilityHelper.createSchemaField("term", Schema.create(Schema.Type.STRING), null, null), AvroCompatibilityHelper.createSchemaField("value", Schema.create(Schema.Type.FLOAT), null, null))) schema } @@ -57,8 +58,8 @@ private[offline] object OutputUtils { val schema = Schema.createRecord("Feature", null, null, false) schema.setFields( util.Arrays.asList( - new Schema.Field("term", Schema.create(Schema.Type.STRING), null, null), - new Schema.Field("value", Schema.create(Schema.Type.DOUBLE), null, null))) + AvroCompatibilityHelper.createSchemaField("term", Schema.create(Schema.Type.STRING), null, null), + AvroCompatibilityHelper.createSchemaField("value", Schema.create(Schema.Type.DOUBLE), null, null))) schema } @@ -67,9 +68,9 @@ private[offline] object OutputUtils { val schema = Schema.createRecord("Feature", null, null, false) schema.setFields( util.Arrays.asList( - new Schema.Field("name", Schema.create(Schema.Type.STRING), null, null), - new Schema.Field("term", Schema.create(Schema.Type.STRING), null, null), - new Schema.Field("value", Schema.create(Schema.Type.FLOAT), null, null))) + AvroCompatibilityHelper.createSchemaField("name", Schema.create(Schema.Type.STRING), null, null), + AvroCompatibilityHelper.createSchemaField("term", Schema.create(Schema.Type.STRING), null, null), + AvroCompatibilityHelper.createSchemaField("value", Schema.create(Schema.Type.FLOAT), null, null))) schema } @@ -78,9 +79,9 @@ private[offline] object OutputUtils { val schema = Schema.createRecord("Feature", null, null, false) schema.setFields( util.Arrays.asList( - new Schema.Field("name", Schema.create(Schema.Type.STRING), null, null), - new Schema.Field("term", Schema.create(Schema.Type.STRING), null, null), - new Schema.Field("value", Schema.create(Schema.Type.DOUBLE), null, null))) + AvroCompatibilityHelper.createSchemaField("name", Schema.create(Schema.Type.STRING), null, null), + AvroCompatibilityHelper.createSchemaField("term", Schema.create(Schema.Type.STRING), null, null), + AvroCompatibilityHelper.createSchemaField("value", Schema.create(Schema.Type.DOUBLE), null, null))) schema } diff --git a/src/main/scala/com/linkedin/feathr/offline/job/PreprocessedDataFrameManager.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/job/PreprocessedDataFrameManager.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/job/PreprocessedDataFrameManager.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/job/PreprocessedDataFrameManager.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/join/DataFrameFeatureJoiner.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/join/DataFrameFeatureJoiner.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/join/DataFrameFeatureJoiner.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/join/DataFrameFeatureJoiner.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/join/DataFrameKeyCombiner.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/join/DataFrameKeyCombiner.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/join/DataFrameKeyCombiner.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/join/DataFrameKeyCombiner.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/join/ExecutionContext.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/join/ExecutionContext.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/join/ExecutionContext.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/join/ExecutionContext.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/join/OptimizerUtils.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/join/OptimizerUtils.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/join/OptimizerUtils.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/join/OptimizerUtils.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/join/algorithms/Join.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/join/algorithms/Join.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/join/algorithms/Join.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/join/algorithms/Join.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/join/algorithms/JoinConditionBuilder.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/join/algorithms/JoinConditionBuilder.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/join/algorithms/JoinConditionBuilder.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/join/algorithms/JoinConditionBuilder.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/join/algorithms/JoinKeyColumnsAppender.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/join/algorithms/JoinKeyColumnsAppender.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/join/algorithms/JoinKeyColumnsAppender.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/join/algorithms/JoinKeyColumnsAppender.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/join/algorithms/JoinType.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/join/algorithms/JoinType.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/join/algorithms/JoinType.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/join/algorithms/JoinType.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/join/algorithms/SaltedSparkJoin.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/join/algorithms/SaltedSparkJoin.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/join/algorithms/SaltedSparkJoin.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/join/algorithms/SaltedSparkJoin.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/join/algorithms/SparkJoinWithJoinCondition.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/join/algorithms/SparkJoinWithJoinCondition.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/join/algorithms/SparkJoinWithJoinCondition.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/join/algorithms/SparkJoinWithJoinCondition.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/join/algorithms/SparkJoinWithNoJoinCondition.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/join/algorithms/SparkJoinWithNoJoinCondition.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/join/algorithms/SparkJoinWithNoJoinCondition.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/join/algorithms/SparkJoinWithNoJoinCondition.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/join/util/CountMinSketchFrequentItemEstimator.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/join/util/CountMinSketchFrequentItemEstimator.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/join/util/CountMinSketchFrequentItemEstimator.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/join/util/CountMinSketchFrequentItemEstimator.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/join/util/FrequentItemEstimator.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/join/util/FrequentItemEstimator.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/join/util/FrequentItemEstimator.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/join/util/FrequentItemEstimator.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/join/util/FrequentItemEstimatorType.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/join/util/FrequentItemEstimatorType.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/join/util/FrequentItemEstimatorType.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/join/util/FrequentItemEstimatorType.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/join/util/FrequetItemEstimatorFactory.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/join/util/FrequetItemEstimatorFactory.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/join/util/FrequetItemEstimatorFactory.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/join/util/FrequetItemEstimatorFactory.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/join/util/GroupAndCountFrequentItemEstimator.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/join/util/GroupAndCountFrequentItemEstimator.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/join/util/GroupAndCountFrequentItemEstimator.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/join/util/GroupAndCountFrequentItemEstimator.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/join/util/PreComputedFrequentItemEstimator.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/join/util/PreComputedFrequentItemEstimator.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/join/util/PreComputedFrequentItemEstimator.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/join/util/PreComputedFrequentItemEstimator.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/join/util/SparkFrequentItemEstimator.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/join/util/SparkFrequentItemEstimator.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/join/util/SparkFrequentItemEstimator.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/join/util/SparkFrequentItemEstimator.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/join/workflow/AnchoredFeatureJoinStep.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/join/workflow/AnchoredFeatureJoinStep.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/join/workflow/AnchoredFeatureJoinStep.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/join/workflow/AnchoredFeatureJoinStep.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/join/workflow/DerivedFeatureJoinStep.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/join/workflow/DerivedFeatureJoinStep.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/join/workflow/DerivedFeatureJoinStep.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/join/workflow/DerivedFeatureJoinStep.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/join/workflow/FeatureJoinStep.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/join/workflow/FeatureJoinStep.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/join/workflow/FeatureJoinStep.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/join/workflow/FeatureJoinStep.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/join/workflow/JoinStepInput.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/join/workflow/JoinStepInput.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/join/workflow/JoinStepInput.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/join/workflow/JoinStepInput.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/join/workflow/JoinStepOutput.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/join/workflow/JoinStepOutput.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/join/workflow/JoinStepOutput.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/join/workflow/JoinStepOutput.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/logical/FeatureGroups.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/logical/FeatureGroups.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/logical/FeatureGroups.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/logical/FeatureGroups.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/logical/LogicalPlanner.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/logical/LogicalPlanner.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/logical/LogicalPlanner.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/logical/LogicalPlanner.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/logical/MultiStageJoinPlan.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/logical/MultiStageJoinPlan.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/logical/MultiStageJoinPlan.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/logical/MultiStageJoinPlan.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/logical/MultiStageJoinPlanner.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/logical/MultiStageJoinPlanner.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/logical/MultiStageJoinPlanner.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/logical/MultiStageJoinPlanner.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/mvel/FeatureVariableResolverFactory.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/mvel/FeatureVariableResolverFactory.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/mvel/FeatureVariableResolverFactory.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/mvel/FeatureVariableResolverFactory.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/mvel/MvelContext.java b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/mvel/MvelContext.java similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/mvel/MvelContext.java rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/mvel/MvelContext.java diff --git a/src/main/scala/com/linkedin/feathr/offline/mvel/MvelUtils.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/mvel/MvelUtils.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/mvel/MvelUtils.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/mvel/MvelUtils.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/mvel/plugins/FeathrExpressionExecutionContext.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/mvel/plugins/FeathrExpressionExecutionContext.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/mvel/plugins/FeathrExpressionExecutionContext.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/mvel/plugins/FeathrExpressionExecutionContext.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/mvel/plugins/FeatureValueTypeAdaptor.java b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/mvel/plugins/FeatureValueTypeAdaptor.java similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/mvel/plugins/FeatureValueTypeAdaptor.java rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/mvel/plugins/FeatureValueTypeAdaptor.java diff --git a/src/main/scala/com/linkedin/feathr/offline/package.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/package.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/package.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/package.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/source/DataSource.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/DataSource.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/source/DataSource.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/DataSource.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/source/SourceFormatType.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/SourceFormatType.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/source/SourceFormatType.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/SourceFormatType.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/source/accessor/DataSourceAccessor.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/accessor/DataSourceAccessor.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/source/accessor/DataSourceAccessor.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/accessor/DataSourceAccessor.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/source/accessor/NonTimeBasedDataSourceAccessor.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/accessor/NonTimeBasedDataSourceAccessor.scala similarity index 84% rename from src/main/scala/com/linkedin/feathr/offline/source/accessor/NonTimeBasedDataSourceAccessor.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/accessor/NonTimeBasedDataSourceAccessor.scala index 2eaca9db0..181feefff 100644 --- a/src/main/scala/com/linkedin/feathr/offline/source/accessor/NonTimeBasedDataSourceAccessor.scala +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/accessor/NonTimeBasedDataSourceAccessor.scala @@ -1,10 +1,13 @@ package com.linkedin.feathr.offline.source.accessor -import com.linkedin.feathr.offline.config.location.{GenericLocation, Jdbc, PathList, SimplePath} +import com.linkedin.feathr.offline.config.location.{GenericLocation, Jdbc, PathList, SimplePath, Snowflake} import com.linkedin.feathr.offline.source.DataSource -import com.linkedin.feathr.offline.source.dataloader.DataLoaderFactory +import com.linkedin.feathr.offline.source.dataloader.{CaseInsensitiveGenericRecordWrapper, DataLoaderFactory} import com.linkedin.feathr.offline.testfwk.TestFwkUtils import com.linkedin.feathr.offline.transformation.DataFrameExt._ +import org.apache.avro.generic.{GenericRecord, IndexedRecord} +import org.apache.avro.specific.SpecificRecordBase +import org.apache.spark.rdd.RDD import org.apache.spark.sql.{DataFrame, SparkSession} /** * load a dataset from a non-partitioned source. @@ -32,6 +35,7 @@ private[offline] class NonTimeBasedDataSourceAccessor( case PathList(paths) => paths.map(fileLoaderFactory.create(_).loadDataFrame()).reduce((x, y) => x.fuzzyUnion(y)) case Jdbc(_, _, _, _, _) => source.location.loadDf(SparkSession.builder().getOrCreate()) case GenericLocation(_, _) => source.location.loadDf(SparkSession.builder().getOrCreate()) + case Snowflake(_, _, _, _) => source.location.loadDf(SparkSession.builder().getOrCreate()) case _ => fileLoaderFactory.createFromLocation(source.location).loadDataFrame() } diff --git a/src/main/scala/com/linkedin/feathr/offline/source/accessor/PathPartitionedTimeSeriesSourceAccessor.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/accessor/PathPartitionedTimeSeriesSourceAccessor.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/source/accessor/PathPartitionedTimeSeriesSourceAccessor.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/accessor/PathPartitionedTimeSeriesSourceAccessor.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/source/accessor/StreamDataSourceAccessor.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/accessor/StreamDataSourceAccessor.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/source/accessor/StreamDataSourceAccessor.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/accessor/StreamDataSourceAccessor.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/source/accessor/TimeBasedDataSourceAccessor.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/accessor/TimeBasedDataSourceAccessor.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/source/accessor/TimeBasedDataSourceAccessor.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/accessor/TimeBasedDataSourceAccessor.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/source/dataloader/AvroJsonDataLoader.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/dataloader/AvroJsonDataLoader.scala similarity index 99% rename from src/main/scala/com/linkedin/feathr/offline/source/dataloader/AvroJsonDataLoader.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/dataloader/AvroJsonDataLoader.scala index 2f00cb9d0..06dd5c45a 100644 --- a/src/main/scala/com/linkedin/feathr/offline/source/dataloader/AvroJsonDataLoader.scala +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/dataloader/AvroJsonDataLoader.scala @@ -42,7 +42,6 @@ private[offline] class AvroJsonDataLoader(ss: SparkSession, path: String) extend val res = AvroJsonDataLoader.loadJsonFileAsAvroToRDD(ss, path) AvroJsonDataLoader.convertRDD2DF(ss, res) } - } private[offline] object AvroJsonDataLoader { diff --git a/src/main/scala/com/linkedin/feathr/offline/source/dataloader/BatchDataLoader.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/dataloader/BatchDataLoader.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/source/dataloader/BatchDataLoader.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/dataloader/BatchDataLoader.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/source/dataloader/BatchDataLoaderFactory.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/dataloader/BatchDataLoaderFactory.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/source/dataloader/BatchDataLoaderFactory.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/dataloader/BatchDataLoaderFactory.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/source/dataloader/CaseInsensitiveGenericRecordWrapper.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/dataloader/CaseInsensitiveGenericRecordWrapper.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/source/dataloader/CaseInsensitiveGenericRecordWrapper.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/dataloader/CaseInsensitiveGenericRecordWrapper.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/source/dataloader/CsvDataLoader.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/dataloader/CsvDataLoader.scala similarity index 94% rename from src/main/scala/com/linkedin/feathr/offline/source/dataloader/CsvDataLoader.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/dataloader/CsvDataLoader.scala index c726113a7..6efdf2444 100644 --- a/src/main/scala/com/linkedin/feathr/offline/source/dataloader/CsvDataLoader.scala +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/dataloader/CsvDataLoader.scala @@ -1,6 +1,7 @@ package com.linkedin.feathr.offline.source.dataloader import com.fasterxml.jackson.dataformat.csv.{CsvMapper, CsvSchema} +import com.linkedin.avroutil1.compatibility.AvroCompatibilityHelper import org.apache.avro.Schema import org.apache.avro.generic.GenericData.{Array, Record} import org.apache.avro.generic.GenericRecord @@ -71,7 +72,7 @@ private[offline] class CsvDataLoader(ss: SparkSession, path: String) extends Dat // hackishly convert to Avro GenericRecord format val avroSchema = Schema.createRecord(getArbitraryRecordName(fields), null, null, false) avroSchema.setFields( - fields.map(new Schema.Field(_, Schema.createUnion(List(Schema.create(Schema.Type.STRING), Schema.create(Schema.Type.NULL))), null, null))) + fields.map(AvroCompatibilityHelper.createSchemaField(_, Schema.createUnion(List(Schema.create(Schema.Type.STRING), Schema.create(Schema.Type.NULL))), null, null))) val genericRecords = rowsCleaned.map(coerceToAvro(avroSchema, _).asInstanceOf[GenericRecord]) (genericRecords, avroSchema) diff --git a/src/main/scala/com/linkedin/feathr/offline/source/dataloader/DataLoader.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/dataloader/DataLoader.scala similarity index 95% rename from src/main/scala/com/linkedin/feathr/offline/source/dataloader/DataLoader.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/dataloader/DataLoader.scala index 3976802d1..de8c1865e 100644 --- a/src/main/scala/com/linkedin/feathr/offline/source/dataloader/DataLoader.scala +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/dataloader/DataLoader.scala @@ -2,6 +2,7 @@ package com.linkedin.feathr.offline.source.dataloader import org.apache.avro.Schema import org.apache.log4j.Logger +import org.apache.spark.rdd.RDD import org.apache.spark.sql.DataFrame /** diff --git a/src/main/scala/com/linkedin/feathr/offline/source/dataloader/DataLoaderFactory.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/dataloader/DataLoaderFactory.scala similarity index 96% rename from src/main/scala/com/linkedin/feathr/offline/source/dataloader/DataLoaderFactory.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/dataloader/DataLoaderFactory.scala index 057be7e9b..29459174c 100644 --- a/src/main/scala/com/linkedin/feathr/offline/source/dataloader/DataLoaderFactory.scala +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/dataloader/DataLoaderFactory.scala @@ -45,7 +45,7 @@ private[offline] object DataLoaderFactory { /** * Class that encloses hooks for creating/writing data frames depends on the data/path type. * @param validatePath used to validate if path should be routed to data handler - * @param createDataFrame used to create a data frame given a path. + * @param createDataFrame used to create a data feathr given a path. * @param createUnionDataFrame used to create a data frame given multiple paths * @param writeDataFrame used to write a data frame to a path */ diff --git a/src/main/scala/com/linkedin/feathr/offline/source/dataloader/JDBCDataLoader.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/dataloader/JDBCDataLoader.scala similarity index 84% rename from src/main/scala/com/linkedin/feathr/offline/source/dataloader/JDBCDataLoader.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/dataloader/JDBCDataLoader.scala index 590f83152..d7bb74feb 100644 --- a/src/main/scala/com/linkedin/feathr/offline/source/dataloader/JDBCDataLoader.scala +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/dataloader/JDBCDataLoader.scala @@ -1,7 +1,9 @@ package com.linkedin.feathr.offline.source.dataloader +import com.linkedin.feathr.common.exception.{ErrorLabel, FeathrException} import com.linkedin.feathr.offline.source.dataloader.jdbc.JdbcUtils import org.apache.avro.Schema +import org.apache.spark.rdd.RDD import org.apache.spark.sql.{DataFrame, SparkSession} /** diff --git a/src/main/scala/com/linkedin/feathr/offline/source/dataloader/JDBCDataLoaderFactory.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/dataloader/JDBCDataLoaderFactory.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/source/dataloader/JDBCDataLoaderFactory.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/dataloader/JDBCDataLoaderFactory.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/source/dataloader/JsonWithSchemaDataLoader.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/dataloader/JsonWithSchemaDataLoader.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/source/dataloader/JsonWithSchemaDataLoader.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/dataloader/JsonWithSchemaDataLoader.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/source/dataloader/LocalDataLoaderFactory.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/dataloader/LocalDataLoaderFactory.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/source/dataloader/LocalDataLoaderFactory.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/dataloader/LocalDataLoaderFactory.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/source/dataloader/ParquetDataLoader.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/dataloader/ParquetDataLoader.scala similarity index 84% rename from src/main/scala/com/linkedin/feathr/offline/source/dataloader/ParquetDataLoader.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/dataloader/ParquetDataLoader.scala index 33718d961..914f3b8d7 100644 --- a/src/main/scala/com/linkedin/feathr/offline/source/dataloader/ParquetDataLoader.scala +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/dataloader/ParquetDataLoader.scala @@ -1,6 +1,8 @@ package com.linkedin.feathr.offline.source.dataloader +import com.linkedin.feathr.common.exception.{ErrorLabel, FeathrException} import org.apache.avro.Schema +import org.apache.spark.rdd.RDD import org.apache.spark.sql.{DataFrame, SparkSession} /** diff --git a/src/main/scala/com/linkedin/feathr/offline/source/dataloader/StreamingDataLoaderFactory.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/dataloader/StreamingDataLoaderFactory.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/source/dataloader/StreamingDataLoaderFactory.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/dataloader/StreamingDataLoaderFactory.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/source/dataloader/hdfs/FileFormat.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/dataloader/hdfs/FileFormat.scala similarity index 94% rename from src/main/scala/com/linkedin/feathr/offline/source/dataloader/hdfs/FileFormat.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/dataloader/hdfs/FileFormat.scala index 8061e2618..b405b0728 100644 --- a/src/main/scala/com/linkedin/feathr/offline/source/dataloader/hdfs/FileFormat.scala +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/dataloader/hdfs/FileFormat.scala @@ -2,7 +2,7 @@ package com.linkedin.feathr.offline.source.dataloader.hdfs import com.linkedin.feathr.common.exception.FeathrException import com.linkedin.feathr.offline.source.dataloader._ -import com.linkedin.feathr.offline.source.dataloader.jdbc.JdbcUtils +import com.linkedin.feathr.offline.source.dataloader.jdbc.{JdbcUtils, SnowflakeUtils} import org.apache.spark.sql.SparkSession import org.apache.spark.sql.DataFrame import com.linkedin.feathr.offline.util.DelimiterUtils.checkDelimiterOption @@ -23,6 +23,8 @@ object FileFormat { val PATHLIST = "PATHLIST" // Detail JDBC Sql Type, please refer to dataloader.jdbc.SqlDbType val JDBC = "JDBC" + // Snowflake type + val SNOWFLAKE = "SNOWFLAKE" private val AVRO_DATASOURCE = "avro" // Use Spark native orc reader instead of hive-orc since Spark 2.3 @@ -44,6 +46,7 @@ object FileFormat { case p if p.endsWith(".avro.json") => AVRO_JSON case p if p.endsWith(".avro") => AVRO case p if p.startsWith("jdbc:") => JDBC + case p if p.startsWith("snowflake:") => SNOWFLAKE case _ => // if we cannot tell the file format from the file extensions, we should read from `spark.feathr.inputFormat` to get the format that's sepcified by user. if (ss.conf.get("spark.feathr.inputFormat","").nonEmpty) ss.conf.get("spark.feathr.inputFormat") else PATHLIST @@ -81,6 +84,7 @@ object FileFormat { case p if p.endsWith(".avro.json") => AVRO_JSON case p if p.endsWith(".avro") => AVRO case p if p.startsWith("jdbc:") => JDBC + case p if p.startsWith("snowflake:") => SNOWFLAKE case _ => // if we cannot tell the file format from the file extensions, we should read from `spark.feathr.inputFormat` to get the format that's sepcified by user. dataIOParameters.getOrElse(DATA_FORMAT, ss.conf.get("spark.feathr.inputFormat", AVRO)).toUpperCase @@ -106,6 +110,8 @@ object FileFormat { case JDBC => // TODO: We should stop using JDBC URL as simple path, otherwise the code will be full of such hack JdbcUtils.loadDataFrame(ss, existingHdfsPaths.head) + case SNOWFLAKE => + SnowflakeUtils.loadDataFrame(ss, existingHdfsPaths.head) case _ => // Allow dynamic config of the file format if users want to use one if (ss.conf.getOption("spark.feathr.inputFormat").nonEmpty) ss.read.format(ss.conf.get("spark.feathr.inputFormat")).load(existingHdfsPaths: _*) diff --git a/src/main/scala/com/linkedin/feathr/offline/source/dataloader/jdbc/JDBCConnector.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/dataloader/jdbc/JDBCConnector.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/source/dataloader/jdbc/JDBCConnector.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/dataloader/jdbc/JDBCConnector.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/source/dataloader/jdbc/JDBCUtils.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/dataloader/jdbc/JDBCUtils.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/source/dataloader/jdbc/JDBCUtils.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/dataloader/jdbc/JDBCUtils.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/source/dataloader/jdbc/JdbcConnectorChooser.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/dataloader/jdbc/JdbcConnectorChooser.scala similarity index 86% rename from src/main/scala/com/linkedin/feathr/offline/source/dataloader/jdbc/JdbcConnectorChooser.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/dataloader/jdbc/JdbcConnectorChooser.scala index d96648122..f35117844 100644 --- a/src/main/scala/com/linkedin/feathr/offline/source/dataloader/jdbc/JdbcConnectorChooser.scala +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/dataloader/jdbc/JdbcConnectorChooser.scala @@ -15,13 +15,11 @@ sealed trait JdbcConnectorChooser object JdbcConnectorChooser { case object SqlServer extends JdbcConnectorChooser case object Postgres extends JdbcConnectorChooser - case object SnowflakeSql extends JdbcConnectorChooser case object DefaultJDBC extends JdbcConnectorChooser def getType (url: String): JdbcConnectorChooser = url match { case url if url.startsWith("jdbc:sqlserver") => SqlServer case url if url.startsWith("jdbc:postgresql:") => Postgres - case url if url.startsWith("jdbc:snowflake:") => SnowflakeSql case _ => DefaultJDBC } @@ -29,7 +27,6 @@ object JdbcConnectorChooser { val sqlDbType = getType(url) val dataLoader = sqlDbType match { case SqlServer => new SqlServerDataLoader(ss) - case SnowflakeSql => new SnowflakeSqlDataLoader(ss) case _ => new SqlServerDataLoader(ss) //default jdbc data loader place holder } dataLoader diff --git a/feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/dataloader/jdbc/SnowflakeDataLoader.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/dataloader/jdbc/SnowflakeDataLoader.scala new file mode 100644 index 000000000..9f27db0a0 --- /dev/null +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/dataloader/jdbc/SnowflakeDataLoader.scala @@ -0,0 +1,51 @@ +package com.linkedin.feathr.offline.source.dataloader.jdbc + +import org.apache.commons.httpclient.URI +import org.apache.http.client.utils.URLEncodedUtils +import org.apache.spark.sql.{DataFrame, DataFrameReader, SparkSession} + +import scala.collection.JavaConverters.asScalaBufferConverter +import java.nio.charset.Charset + +/** + * This is used for Snowflake data source JDBC connector + * + */ +class SnowflakeDataLoader(ss: SparkSession) { + val SNOWFLAKE_SOURCE_NAME = "net.snowflake.spark.snowflake" + + def getDFReader(jdbcOptions: Map[String, String]): DataFrameReader = { + val dfReader = ss.read + .format(SNOWFLAKE_SOURCE_NAME) + .options(jdbcOptions) + dfReader + } + + def extractSFOptions(ss: SparkSession, url: String): Map[String, String] = { + var authParams = getSfParams(ss) + + val uri = new URI(url) + val charset = Charset.forName("UTF-8") + val params = URLEncodedUtils.parse(uri.getQuery, charset).asScala + params.foreach(x => { + authParams = authParams.updated(x.getName, x.getValue) + }) + authParams + } + + def getSfParams(ss: SparkSession): Map[String, String] = { + Map[String, String]( + "sfURL" -> ss.conf.get("sfURL"), + "sfUser" -> ss.conf.get("sfUser"), + "sfRole" -> ss.conf.get("sfRole"), + "sfWarehouse" -> ss.conf.get("sfWarehouse"), + "sfPassword" -> ss.conf.get("sfPassword"), + ) + } + + def loadDataFrame(url: String, sfOptions: Map[String, String] = Map[String, String]()): DataFrame = { + val sparkReader = getDFReader(sfOptions) + sparkReader + .load() + } +} diff --git a/feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/dataloader/jdbc/SnowflakeUtils.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/dataloader/jdbc/SnowflakeUtils.scala new file mode 100644 index 000000000..3336487c5 --- /dev/null +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/dataloader/jdbc/SnowflakeUtils.scala @@ -0,0 +1,19 @@ +package com.linkedin.feathr.offline.source.dataloader.jdbc + +import org.apache.spark.SparkConf +import org.apache.spark.sql.{DataFrame, SparkSession} + +/** + * This Utils contains all + * Custom Spark Config Keys For Snowflake Options + * Functions to parse Snowflake Configs + * Basic Function to load dataframe from Snowflake data source + */ +object SnowflakeUtils { + + def loadDataFrame(ss: SparkSession, url: String): DataFrame = { + val snowflakeLoader = new SnowflakeDataLoader(ss) + val snowflakeOptions = snowflakeLoader.extractSFOptions(ss, url) + snowflakeLoader.loadDataFrame(url, snowflakeOptions) + } +} diff --git a/src/main/scala/com/linkedin/feathr/offline/source/dataloader/jdbc/SqlServerDataLoader.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/dataloader/jdbc/SqlServerDataLoader.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/source/dataloader/jdbc/SqlServerDataLoader.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/dataloader/jdbc/SqlServerDataLoader.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/source/dataloader/stream/KafkaDataLoader.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/dataloader/stream/KafkaDataLoader.scala similarity index 95% rename from src/main/scala/com/linkedin/feathr/offline/source/dataloader/stream/KafkaDataLoader.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/dataloader/stream/KafkaDataLoader.scala index d152b26c5..98baa86e5 100644 --- a/src/main/scala/com/linkedin/feathr/offline/source/dataloader/stream/KafkaDataLoader.scala +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/dataloader/stream/KafkaDataLoader.scala @@ -1,8 +1,10 @@ package com.linkedin.feathr.offline.source.dataloader.stream +import com.linkedin.feathr.common.exception.{ErrorLabel, FeathrException} import com.linkedin.feathr.offline.config.datasource.KafkaResourceInfoSetter import com.linkedin.feathr.offline.config.location.KafkaEndpoint import org.apache.avro.Schema +import org.apache.spark.rdd.RDD import org.apache.spark.sql.streaming.DataStreamReader import org.apache.spark.sql.{DataFrame, SparkSession} diff --git a/src/main/scala/com/linkedin/feathr/offline/source/dataloader/stream/StreamDataLoader.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/dataloader/stream/StreamDataLoader.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/source/dataloader/stream/StreamDataLoader.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/dataloader/stream/StreamDataLoader.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/source/pathutil/HdfsPathChecker.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/pathutil/HdfsPathChecker.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/source/pathutil/HdfsPathChecker.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/pathutil/HdfsPathChecker.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/source/pathutil/LocalPathChecker.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/pathutil/LocalPathChecker.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/source/pathutil/LocalPathChecker.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/pathutil/LocalPathChecker.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/source/pathutil/PathChecker.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/pathutil/PathChecker.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/source/pathutil/PathChecker.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/pathutil/PathChecker.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/source/pathutil/TimeBasedHdfsPathAnalyzer.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/pathutil/TimeBasedHdfsPathAnalyzer.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/source/pathutil/TimeBasedHdfsPathAnalyzer.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/pathutil/TimeBasedHdfsPathAnalyzer.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/source/pathutil/TimeBasedHdfsPathGenerator.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/pathutil/TimeBasedHdfsPathGenerator.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/source/pathutil/TimeBasedHdfsPathGenerator.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/pathutil/TimeBasedHdfsPathGenerator.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/swa/SlidingWindowAggregationJoiner.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/swa/SlidingWindowAggregationJoiner.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/swa/SlidingWindowAggregationJoiner.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/swa/SlidingWindowAggregationJoiner.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/swa/SlidingWindowFeatureUtils.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/swa/SlidingWindowFeatureUtils.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/swa/SlidingWindowFeatureUtils.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/swa/SlidingWindowFeatureUtils.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/testfwk/DataConfiguration.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/testfwk/DataConfiguration.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/testfwk/DataConfiguration.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/testfwk/DataConfiguration.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/testfwk/DataConfigurationMockContext.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/testfwk/DataConfigurationMockContext.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/testfwk/DataConfigurationMockContext.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/testfwk/DataConfigurationMockContext.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/testfwk/FeatureDefContext.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/testfwk/FeatureDefContext.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/testfwk/FeatureDefContext.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/testfwk/FeatureDefContext.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/testfwk/FeatureDefMockContext.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/testfwk/FeatureDefMockContext.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/testfwk/FeatureDefMockContext.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/testfwk/FeatureDefMockContext.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/testfwk/SourceMockParam.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/testfwk/SourceMockParam.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/testfwk/SourceMockParam.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/testfwk/SourceMockParam.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/testfwk/TestFwkUtils.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/testfwk/TestFwkUtils.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/testfwk/TestFwkUtils.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/testfwk/TestFwkUtils.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/testfwk/generation/FeathrGenTestComponent.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/testfwk/generation/FeathrGenTestComponent.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/testfwk/generation/FeathrGenTestComponent.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/testfwk/generation/FeathrGenTestComponent.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/testfwk/generation/FeatureGenDataConfiguration.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/testfwk/generation/FeatureGenDataConfiguration.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/testfwk/generation/FeatureGenDataConfiguration.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/testfwk/generation/FeatureGenDataConfiguration.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/testfwk/generation/FeatureGenDataConfigurationMockContext.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/testfwk/generation/FeatureGenDataConfigurationMockContext.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/testfwk/generation/FeatureGenDataConfigurationMockContext.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/testfwk/generation/FeatureGenDataConfigurationMockContext.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/testfwk/generation/FeatureGenDataConfigurationWithMockContext.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/testfwk/generation/FeatureGenDataConfigurationWithMockContext.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/testfwk/generation/FeatureGenDataConfigurationWithMockContext.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/testfwk/generation/FeatureGenDataConfigurationWithMockContext.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/testfwk/generation/FeatureGenExperimentComponent.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/testfwk/generation/FeatureGenExperimentComponent.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/testfwk/generation/FeatureGenExperimentComponent.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/testfwk/generation/FeatureGenExperimentComponent.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/transformation/AnchorToDataSourceMapper.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/transformation/AnchorToDataSourceMapper.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/transformation/AnchorToDataSourceMapper.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/transformation/AnchorToDataSourceMapper.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/transformation/DataFrameBasedRowEvaluator.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/transformation/DataFrameBasedRowEvaluator.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/transformation/DataFrameBasedRowEvaluator.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/transformation/DataFrameBasedRowEvaluator.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/transformation/DataFrameBasedSqlEvaluator.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/transformation/DataFrameBasedSqlEvaluator.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/transformation/DataFrameBasedSqlEvaluator.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/transformation/DataFrameBasedSqlEvaluator.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/transformation/DataFrameExt.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/transformation/DataFrameExt.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/transformation/DataFrameExt.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/transformation/DataFrameExt.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/transformation/DefaultValueSubstituter.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/transformation/DefaultValueSubstituter.scala similarity index 99% rename from src/main/scala/com/linkedin/feathr/offline/transformation/DefaultValueSubstituter.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/transformation/DefaultValueSubstituter.scala index 1b67d9558..bf5d70c75 100644 --- a/src/main/scala/com/linkedin/feathr/offline/transformation/DefaultValueSubstituter.scala +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/transformation/DefaultValueSubstituter.scala @@ -112,7 +112,7 @@ private[offline] object DataFrameDefaultValueSubstituter extends DataFrameDefaul // For tensor default, since we don't have type, so we need to use expr to construct the default column val schema = field.dataType val tensorData = defaultFeatureValue.getAsTensorData - val ts = FeaturizedDatasetUtils.tensorToFDSDataFrameRow(tensorData) + val ts = FeaturizedDatasetUtils.tensorToFDSDataFrameRow(tensorData, Some(schema)) val fdsTensorDefaultUDF = getFDSTensorDefaultUDF(schema, ts) ss.udf.register("tz_udf", fdsTensorDefaultUDF) expr(s"tz_udf($featureColumnName)") diff --git a/src/main/scala/com/linkedin/feathr/offline/transformation/FDS1dTensor.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/transformation/FDS1dTensor.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/transformation/FDS1dTensor.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/transformation/FDS1dTensor.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/transformation/FDSConversionUtils.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/transformation/FDSConversionUtils.scala similarity index 98% rename from src/main/scala/com/linkedin/feathr/offline/transformation/FDSConversionUtils.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/transformation/FDSConversionUtils.scala index 25d96af11..e2196fe2f 100644 --- a/src/main/scala/com/linkedin/feathr/offline/transformation/FDSConversionUtils.scala +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/transformation/FDSConversionUtils.scala @@ -380,7 +380,13 @@ private[offline] object FDSConversionUtils { // Auto tz case. If user does not explicitly give a valType and the the values are Numbers, auto tz logic sets // valType to Float and we will coerce the output to Float. if (valType == FloatType) { - arrays(0).zip(arrays(1).map(_.toString.toFloat)).sortBy(p => p._1.toString).unzip + val dimToValArray = arrays(0).zip(arrays(1).map(_.toString.toFloat)) + val sortedArray = try { + dimToValArray.sortBy(p => java.lang.Float.valueOf(p._1.toString)) + } catch { + case e: Exception => dimToValArray.sortBy(p => p._1.toString) + } + sortedArray.unzip } else { // Explicit tz case arrays(0).zip(arrays(1)).sortBy(p => p._1.toString).unzip } diff --git a/src/main/scala/com/linkedin/feathr/offline/transformation/FeatureColumnFormat.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/transformation/FeatureColumnFormat.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/transformation/FeatureColumnFormat.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/transformation/FeatureColumnFormat.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/transformation/FeatureValueToColumnConverter.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/transformation/FeatureValueToColumnConverter.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/transformation/FeatureValueToColumnConverter.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/transformation/FeatureValueToColumnConverter.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/transformation/MvelDefinition.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/transformation/MvelDefinition.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/transformation/MvelDefinition.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/transformation/MvelDefinition.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/transformation/WindowAggregationEvaluator.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/transformation/WindowAggregationEvaluator.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/transformation/WindowAggregationEvaluator.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/transformation/WindowAggregationEvaluator.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/util/AclCheckUtils.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/util/AclCheckUtils.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/util/AclCheckUtils.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/util/AclCheckUtils.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/util/AnchorUtils.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/util/AnchorUtils.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/util/AnchorUtils.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/util/AnchorUtils.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/util/CmdLineParser.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/util/CmdLineParser.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/util/CmdLineParser.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/util/CmdLineParser.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/util/CoercionUtilsScala.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/util/CoercionUtilsScala.scala similarity index 98% rename from src/main/scala/com/linkedin/feathr/offline/util/CoercionUtilsScala.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/util/CoercionUtilsScala.scala index 8c7cc1ed2..69dce9b57 100644 --- a/src/main/scala/com/linkedin/feathr/offline/util/CoercionUtilsScala.scala +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/util/CoercionUtilsScala.scala @@ -76,6 +76,7 @@ private[offline] object CoercionUtilsScala { } def coerceFieldToFeatureValue(row: Row, schema: StructType, fieldName: String, featureTypeConfig: FeatureTypeConfig): FeatureValue = { + print("ROW IS " + row + " and featureTypeConfig is " + featureTypeConfig + " and feature name is " + fieldName) val fieldIndex = schema.fieldIndex(fieldName) val fieldType = schema.toList(fieldIndex) val valueMap = if (row.get(fieldIndex) == null) { diff --git a/src/main/scala/com/linkedin/feathr/offline/util/ColumnMetadataMap.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/util/ColumnMetadataMap.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/util/ColumnMetadataMap.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/util/ColumnMetadataMap.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/util/DataFrameSplitterMerger.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/util/DataFrameSplitterMerger.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/util/DataFrameSplitterMerger.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/util/DataFrameSplitterMerger.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/util/DelimiterUtils.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/util/DelimiterUtils.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/util/DelimiterUtils.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/util/DelimiterUtils.scala diff --git a/feathr-impl/src/main/scala/com/linkedin/feathr/offline/util/FCMUtils.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/util/FCMUtils.scala new file mode 100644 index 000000000..b9bbad007 --- /dev/null +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/util/FCMUtils.scala @@ -0,0 +1,7 @@ +package com.linkedin.feathr.offline.util + +object FCMUtils { + def makeFeatureNameForDuplicates(keyTags: Seq[String], featureName: String): String = { + keyTags.mkString("_") + "__" + featureName + } +} diff --git a/src/main/scala/com/linkedin/feathr/offline/util/FeathrTestUtils.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/util/FeathrTestUtils.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/util/FeathrTestUtils.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/util/FeathrTestUtils.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/util/FeathrUtils.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/util/FeathrUtils.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/util/FeathrUtils.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/util/FeathrUtils.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/util/FeatureGenUtils.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/util/FeatureGenUtils.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/util/FeatureGenUtils.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/util/FeatureGenUtils.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/util/FeatureValueTypeValidator.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/util/FeatureValueTypeValidator.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/util/FeatureValueTypeValidator.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/util/FeatureValueTypeValidator.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/util/FeaturizedDatasetMetadata.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/util/FeaturizedDatasetMetadata.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/util/FeaturizedDatasetMetadata.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/util/FeaturizedDatasetMetadata.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/util/FeaturizedDatasetUtils.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/util/FeaturizedDatasetUtils.scala similarity index 93% rename from src/main/scala/com/linkedin/feathr/offline/util/FeaturizedDatasetUtils.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/util/FeaturizedDatasetUtils.scala index 534881f7a..8b52ce72e 100644 --- a/src/main/scala/com/linkedin/feathr/offline/util/FeaturizedDatasetUtils.scala +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/util/FeaturizedDatasetUtils.scala @@ -149,6 +149,23 @@ private[offline] object FeaturizedDatasetUtils { tensorType } + def lookupTensorTypeForNonFMLFeatureRef(featureRefStr: String, featureType: FeatureTypes, featureTypeConfig: FeatureTypeConfig): TensorType = { + // For backward-compatibility, we are using following order to dertermin the tensor type: + // 1. always use FML metadata for tensor type, + // 2. then use tensor type specified in the config, + // 3. then use get auto-tensorized tensor type. + val autoTzTensorTypeOpt = AutoTensorizableTypes.getDefaultTensorType(featureType) + + val tensorType = if (featureType == FeatureTypes.DENSE_VECTOR) { + DENSE_VECTOR_FDS_TENSOR_TYPE + } else if (featureTypeConfig.hasTensorType) { + featureTypeConfig.getTensorType + } else if (autoTzTensorTypeOpt.isPresent) { + autoTzTensorTypeOpt.get() + } else throw new FeathrException(ErrorLabel.FEATHR_ERROR, s"Cannot get tensor type for ${featureRefStr} with type ${featureType}") + tensorType + } + /** * For a given Quince TensorData, converts the tensor into its Quince-FDS representation, which will be either a diff --git a/src/main/scala/com/linkedin/feathr/offline/util/HdfsUtils.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/util/HdfsUtils.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/util/HdfsUtils.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/util/HdfsUtils.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/util/LocalFeatureJoinUtils.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/util/LocalFeatureJoinUtils.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/util/LocalFeatureJoinUtils.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/util/LocalFeatureJoinUtils.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/util/PartitionLimiter.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/util/PartitionLimiter.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/util/PartitionLimiter.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/util/PartitionLimiter.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/util/SourceUtils.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/util/SourceUtils.scala similarity index 99% rename from src/main/scala/com/linkedin/feathr/offline/util/SourceUtils.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/util/SourceUtils.scala index a70c11fd0..1673786f5 100644 --- a/src/main/scala/com/linkedin/feathr/offline/util/SourceUtils.scala +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/util/SourceUtils.scala @@ -15,7 +15,7 @@ import com.linkedin.feathr.offline.source.SourceFormatType import com.linkedin.feathr.offline.source.SourceFormatType.SourceFormatType import com.linkedin.feathr.offline.source.dataloader.DataLoaderHandler import com.linkedin.feathr.offline.source.dataloader.hdfs.FileFormat -import com.linkedin.feathr.offline.source.dataloader.jdbc.JdbcUtils +import com.linkedin.feathr.offline.source.dataloader.jdbc.{JdbcUtils, SnowflakeUtils} import com.linkedin.feathr.offline.source.pathutil.{PathChecker, TimeBasedHdfsPathAnalyzer, TimeBasedHdfsPathGenerator} import com.linkedin.feathr.offline.util.AclCheckUtils.getLatestPath import com.linkedin.feathr.offline.util.datetime.OfflineDateTimeUtils @@ -648,6 +648,9 @@ private[offline] object SourceUtils { case FileFormat.JDBC => { JdbcUtils.loadDataFrame(ss, inputData.inputPath) } + case FileFormat.SNOWFLAKE => { + SnowflakeUtils.loadDataFrame(ss, inputData.inputPath) + } case FileFormat.CSV => { ss.read.format("csv").option("header", "true").option("delimiter", csvDelimiterOption).load(inputData.inputPath) } diff --git a/src/main/scala/com/linkedin/feathr/offline/util/SparkFeaturizedDataset.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/util/SparkFeaturizedDataset.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/util/SparkFeaturizedDataset.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/util/SparkFeaturizedDataset.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/util/datetime/DateTimeInterval.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/util/datetime/DateTimeInterval.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/util/datetime/DateTimeInterval.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/util/datetime/DateTimeInterval.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/util/datetime/DateTimePeriod.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/util/datetime/DateTimePeriod.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/util/datetime/DateTimePeriod.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/util/datetime/DateTimePeriod.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/util/datetime/OfflineDateTimeUtils.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/util/datetime/OfflineDateTimeUtils.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/util/datetime/OfflineDateTimeUtils.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/util/datetime/OfflineDateTimeUtils.scala diff --git a/src/main/scala/com/linkedin/feathr/offline/util/transformations.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/util/transformations.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/offline/util/transformations.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/offline/util/transformations.scala diff --git a/src/main/scala/com/linkedin/feathr/sparkcommon/ComplexAggregation.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/sparkcommon/ComplexAggregation.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/sparkcommon/ComplexAggregation.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/sparkcommon/ComplexAggregation.scala diff --git a/feathr-impl/src/main/scala/com/linkedin/feathr/sparkcommon/FDSExtractor.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/sparkcommon/FDSExtractor.scala new file mode 100644 index 000000000..72284e4e6 --- /dev/null +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/sparkcommon/FDSExtractor.scala @@ -0,0 +1,39 @@ +package com.linkedin.feathr.sparkcommon + +import com.linkedin.feathr.exception.{ErrorLabel, FrameFeatureJoinException} +import org.apache.spark.sql.{Column, DataFrame} + +/** + * A canned extractor class to extract features which are already present in FDS format. We do not support any type of + * SQL or MVEL expressions to extract the features. These features will be joined to the observation data as is. Also, it is + * a pre-requisite for these columns to already be in the FDS format. + * Usage - Please specify the class name "com.linkedin.frame.sparkcommon.FDSExtractor" in the extractor field of the anchor. + * All the features contained within that anchor will be extracted using this class. + * This class is final and cannot be further inherited. + * @param features List of features to be extracted. + */ +final class FDSExtractor(val features: Set[String]) extends SimpleAnchorExtractorSpark { + + override def getProvidedFeatureNames: Seq[String] = features.toSeq + + /** + * Return the sequence of feature names to the respective column using the input ddataframe. + * In this case, as the features are already in the FDS format, the columns will be return as is, without any processing. + * + * @param inputDF input dataframe + * @return Seq of extracted feature names with the columns. + */ + override def transformAsColumns(inputDF: DataFrame): Seq[(String, Column)] = { + val schema = inputDF.schema + features + .map(featureName => { + try { + (featureName, inputDF.col(featureName)) + } catch { + case e: Exception => throw new FrameFeatureJoinException(ErrorLabel.FEATHR_ERROR, s"Unable to extract column" + + s" $featureName from the input dataframe with schema $schema.") + } + }) + }.toSeq +} + diff --git a/src/main/scala/com/linkedin/feathr/sparkcommon/FeatureDerivationFunctionSpark.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/sparkcommon/FeatureDerivationFunctionSpark.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/sparkcommon/FeatureDerivationFunctionSpark.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/sparkcommon/FeatureDerivationFunctionSpark.scala diff --git a/feathr-impl/src/main/scala/com/linkedin/feathr/sparkcommon/GenericAnchorExtractorSpark.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/sparkcommon/GenericAnchorExtractorSpark.scala new file mode 100644 index 000000000..ad50c07e7 --- /dev/null +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/sparkcommon/GenericAnchorExtractorSpark.scala @@ -0,0 +1,46 @@ +package com.linkedin.feathr.sparkcommon + +import com.linkedin.feathr.common.AnchorExtractorBase +import org.apache.spark.sql.types.StructType +import org.apache.spark.sql.{Column, DataFrame, Dataset} + +/** + * Spark DataFrame-based generic anchor extractor (Warning: performance impact). + * + * We strongly recommend developer extends the other trait, [[SimpleAnchorExtractorSpark]] + * (when SQL based syntax is not able to express the transformation logic) to implement customized transformation logic, + * instead of extending this [[GenericAnchorExtractorSpark]]. As this trait is LESS efficient than SQL syntax based or the + * [[SimpleAnchorExtractorSpark]] in feathr. + * + * Each use of this GenericAnchorExtractorSpark will trigger an expensive join between the observation and + * transformed feature data (i.e, the output dataframe of the transform() method). + * + * Only extends this trait when if is NOT possible to use [[SimpleAnchorExtractorSpark]] + [[SourceKeyExtractor]], + * such case should be rare, e.g, even when you need to filter input rows/columns, explode rows, you could apply some + * of the transformations in the SourceKeyExtractor's appendKeyColumns, and use [[SimpleAnchorExtractorSpark]] + * to apply the rest of your transformations. + */ + +abstract class GenericAnchorExtractorSpark extends AnchorExtractorBase[Any] { + /** + * + * Transform input dataframe to generate feature columns + * The column names for the features should be the same as the declared feature names, + * which are the feature names returned by getProvidedFeatureNames(). + * + * + * @param dataFrameWithKeyColumns input dataframe with join key columns appended + * @return input dataframe with feature columns appended. + */ + def transform(dataFrameWithKeyColumns: DataFrame): DataFrame + + /** + * Check the validity of the input DataFrame, raise an exception if the schema is invalid, + * e.g, does not contain required input columns or has incorrect column types + * It is the developer's responsibility to validate the input schema's validity + * @param schema the schema of input dataframe (i.e dataFrameWithKeyColumns in transform) + */ + def validateInputSchema(schema: StructType): Unit = {} + + +} diff --git a/src/main/scala/com/linkedin/feathr/sparkcommon/OutputProcessor.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/sparkcommon/OutputProcessor.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/sparkcommon/OutputProcessor.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/sparkcommon/OutputProcessor.scala diff --git a/src/main/scala/com/linkedin/feathr/sparkcommon/SeqJoinCustomAggregation.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/sparkcommon/SeqJoinCustomAggregation.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/sparkcommon/SeqJoinCustomAggregation.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/sparkcommon/SeqJoinCustomAggregation.scala diff --git a/src/main/scala/com/linkedin/feathr/sparkcommon/SimpleAnchorExtractorSpark.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/sparkcommon/SimpleAnchorExtractorSpark.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/sparkcommon/SimpleAnchorExtractorSpark.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/sparkcommon/SimpleAnchorExtractorSpark.scala diff --git a/src/main/scala/com/linkedin/feathr/sparkcommon/SourceKeyExtractor.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/sparkcommon/SourceKeyExtractor.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/sparkcommon/SourceKeyExtractor.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/sparkcommon/SourceKeyExtractor.scala diff --git a/src/main/scala/com/linkedin/feathr/swj/SlidingWindowDataDef.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/swj/SlidingWindowDataDef.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/swj/SlidingWindowDataDef.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/swj/SlidingWindowDataDef.scala diff --git a/src/main/scala/com/linkedin/feathr/swj/SlidingWindowJoin.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/swj/SlidingWindowJoin.scala similarity index 93% rename from src/main/scala/com/linkedin/feathr/swj/SlidingWindowJoin.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/swj/SlidingWindowJoin.scala index f810fc2e5..966f234fe 100644 --- a/src/main/scala/com/linkedin/feathr/swj/SlidingWindowJoin.scala +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/swj/SlidingWindowJoin.scala @@ -1,8 +1,10 @@ package com.linkedin.feathr.swj +import com.linkedin.feathr.offline.evaluator.datasource.DataSourceNodeEvaluator.getClass import com.linkedin.feathr.swj.join.{FeatureColumnMetaData, SlidingWindowJoinIterator} import com.linkedin.feathr.swj.transformer.FeatureTransformer import com.linkedin.feathr.swj.transformer.FeatureTransformer._ +import org.apache.log4j.Logger import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{ArrayType, StructField, StructType} import org.apache.spark.sql.{DataFrame, Row, SparkSession} @@ -10,6 +12,7 @@ import org.apache.spark.sql.{DataFrame, Row, SparkSession} object SlidingWindowJoin { + val log = Logger.getLogger(getClass) lazy val spark: SparkSession = SparkSession.builder().getOrCreate() private val LABEL_VIEW_NAME = "label_data" @@ -28,6 +31,13 @@ object SlidingWindowJoin { labelDataset: LabelData, factDatasets: List[FactData], numPartitions: Int = spark.sparkContext.getConf.getInt(SQLConf.SHUFFLE_PARTITIONS.key, 200)): DataFrame = { + factDatasets.foreach(factDataset => { + factDataset.aggFeatures.foreach(swaFeature => { + log.info("Evaluating feature " + swaFeature.name + "\n") + }) + log.info("Feature's keys are " + factDataset.joinKey + "\n") + }) + val labelDF = addLabelDataCols(labelDataset.dataSource, labelDataset) // Partition the label DataFrame by join_key and sort each partition with (join_key, timestamp) var result = labelDF.repartition(numPartitions, labelDF.col(JOIN_KEY_COL_NAME)) diff --git a/src/main/scala/com/linkedin/feathr/swj/aggregate/AggregationSpec.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/swj/aggregate/AggregationSpec.scala similarity index 97% rename from src/main/scala/com/linkedin/feathr/swj/aggregate/AggregationSpec.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/swj/aggregate/AggregationSpec.scala index 6ad57e35b..a69453fbf 100644 --- a/src/main/scala/com/linkedin/feathr/swj/aggregate/AggregationSpec.scala +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/swj/aggregate/AggregationSpec.scala @@ -12,7 +12,7 @@ import org.apache.spark.sql.types.DataType * fields except metricCol. The field metricCol is supposed to be passed in via * the constructor of the concrete AggregationSpec class. */ -private[swj] trait AggregationSpec extends Serializable { +private[feathr] trait AggregationSpec extends Serializable { // Type of the aggregation as an AggregationType def aggregation: AggregationType // It can be either the name of the metric column or a Spark SQL column expression diff --git a/src/main/scala/com/linkedin/feathr/swj/aggregate/AggregationType.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/swj/aggregate/AggregationType.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/swj/aggregate/AggregationType.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/swj/aggregate/AggregationType.scala diff --git a/src/main/scala/com/linkedin/feathr/swj/aggregate/AggregationWithDeaggBase.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/swj/aggregate/AggregationWithDeaggBase.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/swj/aggregate/AggregationWithDeaggBase.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/swj/aggregate/AggregationWithDeaggBase.scala diff --git a/src/main/scala/com/linkedin/feathr/swj/aggregate/AvgAggregate.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/swj/aggregate/AvgAggregate.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/swj/aggregate/AvgAggregate.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/swj/aggregate/AvgAggregate.scala diff --git a/src/main/scala/com/linkedin/feathr/swj/aggregate/AvgPoolingAggregate.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/swj/aggregate/AvgPoolingAggregate.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/swj/aggregate/AvgPoolingAggregate.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/swj/aggregate/AvgPoolingAggregate.scala diff --git a/src/main/scala/com/linkedin/feathr/swj/aggregate/CountAggregate.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/swj/aggregate/CountAggregate.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/swj/aggregate/CountAggregate.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/swj/aggregate/CountAggregate.scala diff --git a/src/main/scala/com/linkedin/feathr/swj/aggregate/CountDistinctAggregate.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/swj/aggregate/CountDistinctAggregate.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/swj/aggregate/CountDistinctAggregate.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/swj/aggregate/CountDistinctAggregate.scala diff --git a/src/main/scala/com/linkedin/feathr/swj/aggregate/DummyAggregate.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/swj/aggregate/DummyAggregate.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/swj/aggregate/DummyAggregate.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/swj/aggregate/DummyAggregate.scala diff --git a/src/main/scala/com/linkedin/feathr/swj/aggregate/LatestAggregate.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/swj/aggregate/LatestAggregate.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/swj/aggregate/LatestAggregate.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/swj/aggregate/LatestAggregate.scala diff --git a/src/main/scala/com/linkedin/feathr/swj/aggregate/MaxAggregate.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/swj/aggregate/MaxAggregate.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/swj/aggregate/MaxAggregate.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/swj/aggregate/MaxAggregate.scala diff --git a/src/main/scala/com/linkedin/feathr/swj/aggregate/MaxPoolingAggregate.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/swj/aggregate/MaxPoolingAggregate.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/swj/aggregate/MaxPoolingAggregate.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/swj/aggregate/MaxPoolingAggregate.scala diff --git a/src/main/scala/com/linkedin/feathr/swj/aggregate/MinAggregate.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/swj/aggregate/MinAggregate.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/swj/aggregate/MinAggregate.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/swj/aggregate/MinAggregate.scala diff --git a/src/main/scala/com/linkedin/feathr/swj/aggregate/MinPoolingAggregate.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/swj/aggregate/MinPoolingAggregate.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/swj/aggregate/MinPoolingAggregate.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/swj/aggregate/MinPoolingAggregate.scala diff --git a/src/main/scala/com/linkedin/feathr/swj/aggregate/SumAggregate.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/swj/aggregate/SumAggregate.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/swj/aggregate/SumAggregate.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/swj/aggregate/SumAggregate.scala diff --git a/src/main/scala/com/linkedin/feathr/swj/aggregate/SumPoolingAggregate.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/swj/aggregate/SumPoolingAggregate.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/swj/aggregate/SumPoolingAggregate.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/swj/aggregate/SumPoolingAggregate.scala diff --git a/src/main/scala/com/linkedin/feathr/swj/aggregate/TimesinceAggregate.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/swj/aggregate/TimesinceAggregate.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/swj/aggregate/TimesinceAggregate.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/swj/aggregate/TimesinceAggregate.scala diff --git a/src/main/scala/com/linkedin/feathr/swj/join/FeatureColumnMetaData.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/swj/join/FeatureColumnMetaData.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/swj/join/FeatureColumnMetaData.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/swj/join/FeatureColumnMetaData.scala diff --git a/src/main/scala/com/linkedin/feathr/swj/join/SlidingWindowJoinIterator.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/swj/join/SlidingWindowJoinIterator.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/swj/join/SlidingWindowJoinIterator.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/swj/join/SlidingWindowJoinIterator.scala diff --git a/src/main/scala/com/linkedin/feathr/swj/transformer/FeatureTransformer.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/swj/transformer/FeatureTransformer.scala similarity index 100% rename from src/main/scala/com/linkedin/feathr/swj/transformer/FeatureTransformer.scala rename to feathr-impl/src/main/scala/com/linkedin/feathr/swj/transformer/FeatureTransformer.scala diff --git a/src/main/scala/org/apache/spark/customized/CustomGenericRowWithSchema.scala b/feathr-impl/src/main/scala/org/apache/spark/customized/CustomGenericRowWithSchema.scala similarity index 100% rename from src/main/scala/org/apache/spark/customized/CustomGenericRowWithSchema.scala rename to feathr-impl/src/main/scala/org/apache/spark/customized/CustomGenericRowWithSchema.scala diff --git a/src/test/avro/AggregationActorFact.avsc b/feathr-impl/src/test/avro/AggregationActorFact.avsc similarity index 100% rename from src/test/avro/AggregationActorFact.avsc rename to feathr-impl/src/test/avro/AggregationActorFact.avsc diff --git a/src/test/avro/AggregationFact.avsc b/feathr-impl/src/test/avro/AggregationFact.avsc similarity index 100% rename from src/test/avro/AggregationFact.avsc rename to feathr-impl/src/test/avro/AggregationFact.avsc diff --git a/src/test/avro/AggregationLabel.avsc b/feathr-impl/src/test/avro/AggregationLabel.avsc similarity index 100% rename from src/test/avro/AggregationLabel.avsc rename to feathr-impl/src/test/avro/AggregationLabel.avsc diff --git a/src/test/avro/MultiKeyTrainingData.avsc b/feathr-impl/src/test/avro/MultiKeyTrainingData.avsc similarity index 100% rename from src/test/avro/MultiKeyTrainingData.avsc rename to feathr-impl/src/test/avro/MultiKeyTrainingData.avsc diff --git a/src/test/avro/SWARegularData.avsc b/feathr-impl/src/test/avro/SWARegularData.avsc similarity index 100% rename from src/test/avro/SWARegularData.avsc rename to feathr-impl/src/test/avro/SWARegularData.avsc diff --git a/src/test/avro/SimpleSpecificRecord.avsc b/feathr-impl/src/test/avro/SimpleSpecificRecord.avsc similarity index 100% rename from src/test/avro/SimpleSpecificRecord.avsc rename to feathr-impl/src/test/avro/SimpleSpecificRecord.avsc diff --git a/src/test/avro/TrainingData.avsc b/feathr-impl/src/test/avro/TrainingData.avsc similarity index 100% rename from src/test/avro/TrainingData.avsc rename to feathr-impl/src/test/avro/TrainingData.avsc diff --git a/src/test/generated/config/feathr.conf b/feathr-impl/src/test/generated/config/feathr.conf similarity index 100% rename from src/test/generated/config/feathr.conf rename to feathr-impl/src/test/generated/config/feathr.conf diff --git a/src/test/generated/config/featureJoin_singleKey.conf b/feathr-impl/src/test/generated/config/featureJoin_singleKey.conf similarity index 100% rename from src/test/generated/config/featureJoin_singleKey.conf rename to feathr-impl/src/test/generated/config/featureJoin_singleKey.conf diff --git a/src/test/generated/mockData/acl_user_no_read/.acl_user_no_read.txt.crc b/feathr-impl/src/test/generated/mockData/acl_user_no_read/.acl_user_no_read.txt.crc similarity index 100% rename from src/test/generated/mockData/acl_user_no_read/.acl_user_no_read.txt.crc rename to feathr-impl/src/test/generated/mockData/acl_user_no_read/.acl_user_no_read.txt.crc diff --git a/src/test/generated/mockData/acl_user_no_read/acl_user_no_read.txt b/feathr-impl/src/test/generated/mockData/acl_user_no_read/acl_user_no_read.txt similarity index 100% rename from src/test/generated/mockData/acl_user_no_read/acl_user_no_read.txt rename to feathr-impl/src/test/generated/mockData/acl_user_no_read/acl_user_no_read.txt diff --git a/src/test/generated/mockData/acl_user_no_read_2/.acl_user_no_read.txt.crc b/feathr-impl/src/test/generated/mockData/acl_user_no_read_2/.acl_user_no_read.txt.crc similarity index 100% rename from src/test/generated/mockData/acl_user_no_read_2/.acl_user_no_read.txt.crc rename to feathr-impl/src/test/generated/mockData/acl_user_no_read_2/.acl_user_no_read.txt.crc diff --git a/src/test/generated/mockData/acl_user_no_read_2/acl_user_no_read.txt b/feathr-impl/src/test/generated/mockData/acl_user_no_read_2/acl_user_no_read.txt similarity index 100% rename from src/test/generated/mockData/acl_user_no_read_2/acl_user_no_read.txt rename to feathr-impl/src/test/generated/mockData/acl_user_no_read_2/acl_user_no_read.txt diff --git a/src/test/generated/mockData/acl_user_no_write_execute/.acl_user_no_write_execute.txt.crc b/feathr-impl/src/test/generated/mockData/acl_user_no_write_execute/.acl_user_no_write_execute.txt.crc similarity index 100% rename from src/test/generated/mockData/acl_user_no_write_execute/.acl_user_no_write_execute.txt.crc rename to feathr-impl/src/test/generated/mockData/acl_user_no_write_execute/.acl_user_no_write_execute.txt.crc diff --git a/src/test/generated/mockData/acl_user_no_write_execute/acl_user_no_write_execute.txt b/feathr-impl/src/test/generated/mockData/acl_user_no_write_execute/acl_user_no_write_execute.txt similarity index 100% rename from src/test/generated/mockData/acl_user_no_write_execute/acl_user_no_write_execute.txt rename to feathr-impl/src/test/generated/mockData/acl_user_no_write_execute/acl_user_no_write_execute.txt diff --git a/src/test/generated/mockData/acl_user_no_write_execute_2/.acl_user_no_write_execute.txt.crc b/feathr-impl/src/test/generated/mockData/acl_user_no_write_execute_2/.acl_user_no_write_execute.txt.crc similarity index 100% rename from src/test/generated/mockData/acl_user_no_write_execute_2/.acl_user_no_write_execute.txt.crc rename to feathr-impl/src/test/generated/mockData/acl_user_no_write_execute_2/.acl_user_no_write_execute.txt.crc diff --git a/src/test/generated/mockData/acl_user_no_write_execute_2/acl_user_no_write_execute.txt b/feathr-impl/src/test/generated/mockData/acl_user_no_write_execute_2/acl_user_no_write_execute.txt similarity index 100% rename from src/test/generated/mockData/acl_user_no_write_execute_2/acl_user_no_write_execute.txt rename to feathr-impl/src/test/generated/mockData/acl_user_no_write_execute_2/acl_user_no_write_execute.txt diff --git a/src/test/generated/mockData/acl_user_read/.acl_user_read.txt.crc b/feathr-impl/src/test/generated/mockData/acl_user_read/.acl_user_read.txt.crc similarity index 100% rename from src/test/generated/mockData/acl_user_read/.acl_user_read.txt.crc rename to feathr-impl/src/test/generated/mockData/acl_user_read/.acl_user_read.txt.crc diff --git a/src/test/generated/mockData/acl_user_read/acl_user_read.txt b/feathr-impl/src/test/generated/mockData/acl_user_read/acl_user_read.txt similarity index 100% rename from src/test/generated/mockData/acl_user_read/acl_user_read.txt rename to feathr-impl/src/test/generated/mockData/acl_user_read/acl_user_read.txt diff --git a/src/test/generated/mockData/test_daysgap/2019/09/29/.test.avro.crc b/feathr-impl/src/test/generated/mockData/test_daysgap/2019/09/29/.test.avro.crc similarity index 100% rename from src/test/generated/mockData/test_daysgap/2019/09/29/.test.avro.crc rename to feathr-impl/src/test/generated/mockData/test_daysgap/2019/09/29/.test.avro.crc diff --git a/src/test/generated/mockData/test_daysgap/2019/09/29/test.avro b/feathr-impl/src/test/generated/mockData/test_daysgap/2019/09/29/test.avro similarity index 100% rename from src/test/generated/mockData/test_daysgap/2019/09/29/test.avro rename to feathr-impl/src/test/generated/mockData/test_daysgap/2019/09/29/test.avro diff --git a/src/test/generated/mockData/test_latest_path/2018_10_17/.test.avro.crc b/feathr-impl/src/test/generated/mockData/test_latest_path/2018_10_17/.test.avro.crc similarity index 100% rename from src/test/generated/mockData/test_latest_path/2018_10_17/.test.avro.crc rename to feathr-impl/src/test/generated/mockData/test_latest_path/2018_10_17/.test.avro.crc diff --git a/src/test/generated/mockData/test_latest_path/2018_10_17/test.avro b/feathr-impl/src/test/generated/mockData/test_latest_path/2018_10_17/test.avro similarity index 100% rename from src/test/generated/mockData/test_latest_path/2018_10_17/test.avro rename to feathr-impl/src/test/generated/mockData/test_latest_path/2018_10_17/test.avro diff --git a/src/test/generated/mockData/test_latest_path/2018_11_15/.test.avro.crc b/feathr-impl/src/test/generated/mockData/test_latest_path/2018_11_15/.test.avro.crc similarity index 100% rename from src/test/generated/mockData/test_latest_path/2018_11_15/.test.avro.crc rename to feathr-impl/src/test/generated/mockData/test_latest_path/2018_11_15/.test.avro.crc diff --git a/src/test/generated/mockData/test_latest_path/2018_11_15/test.avro b/feathr-impl/src/test/generated/mockData/test_latest_path/2018_11_15/test.avro similarity index 100% rename from src/test/generated/mockData/test_latest_path/2018_11_15/test.avro rename to feathr-impl/src/test/generated/mockData/test_latest_path/2018_11_15/test.avro diff --git a/src/test/generated/mockData/test_latest_path/2018_11_16/.test.avro.crc b/feathr-impl/src/test/generated/mockData/test_latest_path/2018_11_16/.test.avro.crc similarity index 100% rename from src/test/generated/mockData/test_latest_path/2018_11_16/.test.avro.crc rename to feathr-impl/src/test/generated/mockData/test_latest_path/2018_11_16/.test.avro.crc diff --git a/src/test/generated/mockData/test_latest_path/2018_11_16/test.avro b/feathr-impl/src/test/generated/mockData/test_latest_path/2018_11_16/test.avro similarity index 100% rename from src/test/generated/mockData/test_latest_path/2018_11_16/test.avro rename to feathr-impl/src/test/generated/mockData/test_latest_path/2018_11_16/test.avro diff --git a/src/test/generated/mockData/test_multi_latest_path/2018/.08.crc b/feathr-impl/src/test/generated/mockData/test_multi_latest_path/2018/.08.crc similarity index 100% rename from src/test/generated/mockData/test_multi_latest_path/2018/.08.crc rename to feathr-impl/src/test/generated/mockData/test_multi_latest_path/2018/.08.crc diff --git a/src/test/generated/mockData/test_multi_latest_path/2018/01/17/.test.avro.crc b/feathr-impl/src/test/generated/mockData/test_multi_latest_path/2018/01/17/.test.avro.crc similarity index 100% rename from src/test/generated/mockData/test_multi_latest_path/2018/01/17/.test.avro.crc rename to feathr-impl/src/test/generated/mockData/test_multi_latest_path/2018/01/17/.test.avro.crc diff --git a/src/test/generated/mockData/test_multi_latest_path/2018/01/17/.test1.avro.crc b/feathr-impl/src/test/generated/mockData/test_multi_latest_path/2018/01/17/.test1.avro.crc similarity index 100% rename from src/test/generated/mockData/test_multi_latest_path/2018/01/17/.test1.avro.crc rename to feathr-impl/src/test/generated/mockData/test_multi_latest_path/2018/01/17/.test1.avro.crc diff --git a/src/test/generated/mockData/test_multi_latest_path/2018/01/17/.test2.avro.crc b/feathr-impl/src/test/generated/mockData/test_multi_latest_path/2018/01/17/.test2.avro.crc similarity index 100% rename from src/test/generated/mockData/test_multi_latest_path/2018/01/17/.test2.avro.crc rename to feathr-impl/src/test/generated/mockData/test_multi_latest_path/2018/01/17/.test2.avro.crc diff --git a/src/test/generated/mockData/test_multi_latest_path/2018/01/17/test.avro b/feathr-impl/src/test/generated/mockData/test_multi_latest_path/2018/01/17/test.avro similarity index 100% rename from src/test/generated/mockData/test_multi_latest_path/2018/01/17/test.avro rename to feathr-impl/src/test/generated/mockData/test_multi_latest_path/2018/01/17/test.avro diff --git a/src/test/generated/mockData/test_multi_latest_path/2018/01/17/test1.avro b/feathr-impl/src/test/generated/mockData/test_multi_latest_path/2018/01/17/test1.avro similarity index 100% rename from src/test/generated/mockData/test_multi_latest_path/2018/01/17/test1.avro rename to feathr-impl/src/test/generated/mockData/test_multi_latest_path/2018/01/17/test1.avro diff --git a/src/test/generated/mockData/test_multi_latest_path/2018/01/17/test2.avro b/feathr-impl/src/test/generated/mockData/test_multi_latest_path/2018/01/17/test2.avro similarity index 100% rename from src/test/generated/mockData/test_multi_latest_path/2018/01/17/test2.avro rename to feathr-impl/src/test/generated/mockData/test_multi_latest_path/2018/01/17/test2.avro diff --git a/src/test/generated/mockData/test_multi_latest_path/2018/08 b/feathr-impl/src/test/generated/mockData/test_multi_latest_path/2018/08 similarity index 100% rename from src/test/generated/mockData/test_multi_latest_path/2018/08 rename to feathr-impl/src/test/generated/mockData/test_multi_latest_path/2018/08 diff --git a/src/test/generated/mockData/test_multi_latest_path/2018/11/15/.test.avro.crc b/feathr-impl/src/test/generated/mockData/test_multi_latest_path/2018/11/15/.test.avro.crc similarity index 100% rename from src/test/generated/mockData/test_multi_latest_path/2018/11/15/.test.avro.crc rename to feathr-impl/src/test/generated/mockData/test_multi_latest_path/2018/11/15/.test.avro.crc diff --git a/src/test/generated/mockData/test_multi_latest_path/2018/11/15/test.avro b/feathr-impl/src/test/generated/mockData/test_multi_latest_path/2018/11/15/test.avro similarity index 100% rename from src/test/generated/mockData/test_multi_latest_path/2018/11/15/test.avro rename to feathr-impl/src/test/generated/mockData/test_multi_latest_path/2018/11/15/test.avro diff --git a/src/test/generated/mockData/test_multi_latest_path/2018/11/16/.test.avro.crc b/feathr-impl/src/test/generated/mockData/test_multi_latest_path/2018/11/16/.test.avro.crc similarity index 100% rename from src/test/generated/mockData/test_multi_latest_path/2018/11/16/.test.avro.crc rename to feathr-impl/src/test/generated/mockData/test_multi_latest_path/2018/11/16/.test.avro.crc diff --git a/src/test/generated/mockData/test_multi_latest_path/2018/11/16/.test1.avro.crc b/feathr-impl/src/test/generated/mockData/test_multi_latest_path/2018/11/16/.test1.avro.crc similarity index 100% rename from src/test/generated/mockData/test_multi_latest_path/2018/11/16/.test1.avro.crc rename to feathr-impl/src/test/generated/mockData/test_multi_latest_path/2018/11/16/.test1.avro.crc diff --git a/src/test/generated/mockData/test_multi_latest_path/2018/11/16/test.avro b/feathr-impl/src/test/generated/mockData/test_multi_latest_path/2018/11/16/test.avro similarity index 100% rename from src/test/generated/mockData/test_multi_latest_path/2018/11/16/test.avro rename to feathr-impl/src/test/generated/mockData/test_multi_latest_path/2018/11/16/test.avro diff --git a/src/test/generated/mockData/test_multi_latest_path/2018/11/16/test1.avro b/feathr-impl/src/test/generated/mockData/test_multi_latest_path/2018/11/16/test1.avro similarity index 100% rename from src/test/generated/mockData/test_multi_latest_path/2018/11/16/test1.avro rename to feathr-impl/src/test/generated/mockData/test_multi_latest_path/2018/11/16/test1.avro diff --git a/src/test/java/com/linkedin/feathr/common/AutoTensorizableTypesTest.java b/feathr-impl/src/test/java/com/linkedin/feathr/common/AutoTensorizableTypesTest.java similarity index 100% rename from src/test/java/com/linkedin/feathr/common/AutoTensorizableTypesTest.java rename to feathr-impl/src/test/java/com/linkedin/feathr/common/AutoTensorizableTypesTest.java diff --git a/src/test/java/com/linkedin/feathr/common/FeatureTypeConfigTest.java b/feathr-impl/src/test/java/com/linkedin/feathr/common/FeatureTypeConfigTest.java similarity index 100% rename from src/test/java/com/linkedin/feathr/common/FeatureTypeConfigTest.java rename to feathr-impl/src/test/java/com/linkedin/feathr/common/FeatureTypeConfigTest.java diff --git a/src/test/java/com/linkedin/feathr/common/TestFeatureDependencyGraph.java b/feathr-impl/src/test/java/com/linkedin/feathr/common/TestFeatureDependencyGraph.java similarity index 100% rename from src/test/java/com/linkedin/feathr/common/TestFeatureDependencyGraph.java rename to feathr-impl/src/test/java/com/linkedin/feathr/common/TestFeatureDependencyGraph.java diff --git a/src/test/java/com/linkedin/feathr/common/TestFeatureValue.java b/feathr-impl/src/test/java/com/linkedin/feathr/common/TestFeatureValue.java similarity index 100% rename from src/test/java/com/linkedin/feathr/common/TestFeatureValue.java rename to feathr-impl/src/test/java/com/linkedin/feathr/common/TestFeatureValue.java diff --git a/src/test/java/com/linkedin/feathr/common/types/TestFeatureTypes.java b/feathr-impl/src/test/java/com/linkedin/feathr/common/types/TestFeatureTypes.java similarity index 100% rename from src/test/java/com/linkedin/feathr/common/types/TestFeatureTypes.java rename to feathr-impl/src/test/java/com/linkedin/feathr/common/types/TestFeatureTypes.java diff --git a/src/test/java/com/linkedin/feathr/common/types/TestQuinceFeatureTypeMapper.java b/feathr-impl/src/test/java/com/linkedin/feathr/common/types/TestQuinceFeatureTypeMapper.java similarity index 100% rename from src/test/java/com/linkedin/feathr/common/types/TestQuinceFeatureTypeMapper.java rename to feathr-impl/src/test/java/com/linkedin/feathr/common/types/TestQuinceFeatureTypeMapper.java diff --git a/src/test/java/com/linkedin/feathr/common/util/MvelUDFExpressionTests.java b/feathr-impl/src/test/java/com/linkedin/feathr/common/util/MvelUDFExpressionTests.java similarity index 100% rename from src/test/java/com/linkedin/feathr/common/util/MvelUDFExpressionTests.java rename to feathr-impl/src/test/java/com/linkedin/feathr/common/util/MvelUDFExpressionTests.java diff --git a/src/test/java/com/linkedin/feathr/common/util/TestMvelContextUDFs.java b/feathr-impl/src/test/java/com/linkedin/feathr/common/util/TestMvelContextUDFs.java similarity index 100% rename from src/test/java/com/linkedin/feathr/common/util/TestMvelContextUDFs.java rename to feathr-impl/src/test/java/com/linkedin/feathr/common/util/TestMvelContextUDFs.java diff --git a/src/test/java/com/linkedin/feathr/common/value/TestFeatureValueOldAPICompatibility.java b/feathr-impl/src/test/java/com/linkedin/feathr/common/value/TestFeatureValueOldAPICompatibility.java similarity index 100% rename from src/test/java/com/linkedin/feathr/common/value/TestFeatureValueOldAPICompatibility.java rename to feathr-impl/src/test/java/com/linkedin/feathr/common/value/TestFeatureValueOldAPICompatibility.java diff --git a/src/test/java/com/linkedin/feathr/common/value/TestFeatureValues.java b/feathr-impl/src/test/java/com/linkedin/feathr/common/value/TestFeatureValues.java similarity index 100% rename from src/test/java/com/linkedin/feathr/common/value/TestFeatureValues.java rename to feathr-impl/src/test/java/com/linkedin/feathr/common/value/TestFeatureValues.java diff --git a/src/test/java/com/linkedin/feathr/offline/MockAvroData.java b/feathr-impl/src/test/java/com/linkedin/feathr/offline/MockAvroData.java similarity index 100% rename from src/test/java/com/linkedin/feathr/offline/MockAvroData.java rename to feathr-impl/src/test/java/com/linkedin/feathr/offline/MockAvroData.java diff --git a/src/test/java/com/linkedin/feathr/offline/TestMvelContext.java b/feathr-impl/src/test/java/com/linkedin/feathr/offline/TestMvelContext.java similarity index 100% rename from src/test/java/com/linkedin/feathr/offline/TestMvelContext.java rename to feathr-impl/src/test/java/com/linkedin/feathr/offline/TestMvelContext.java diff --git a/src/test/java/com/linkedin/feathr/offline/TestMvelExpression.java b/feathr-impl/src/test/java/com/linkedin/feathr/offline/TestMvelExpression.java similarity index 100% rename from src/test/java/com/linkedin/feathr/offline/TestMvelExpression.java rename to feathr-impl/src/test/java/com/linkedin/feathr/offline/TestMvelExpression.java diff --git a/src/test/java/com/linkedin/feathr/offline/data/TrainingData.java b/feathr-impl/src/test/java/com/linkedin/feathr/offline/data/TrainingData.java similarity index 100% rename from src/test/java/com/linkedin/feathr/offline/data/TrainingData.java rename to feathr-impl/src/test/java/com/linkedin/feathr/offline/data/TrainingData.java diff --git a/src/test/java/com/linkedin/feathr/offline/plugins/AlienFeatureValue.java b/feathr-impl/src/test/java/com/linkedin/feathr/offline/plugins/AlienFeatureValue.java similarity index 100% rename from src/test/java/com/linkedin/feathr/offline/plugins/AlienFeatureValue.java rename to feathr-impl/src/test/java/com/linkedin/feathr/offline/plugins/AlienFeatureValue.java diff --git a/src/test/java/com/linkedin/feathr/offline/plugins/AlienFeatureValueMvelUDFs.java b/feathr-impl/src/test/java/com/linkedin/feathr/offline/plugins/AlienFeatureValueMvelUDFs.java similarity index 100% rename from src/test/java/com/linkedin/feathr/offline/plugins/AlienFeatureValueMvelUDFs.java rename to feathr-impl/src/test/java/com/linkedin/feathr/offline/plugins/AlienFeatureValueMvelUDFs.java diff --git a/src/test/java/com/linkedin/feathr/offline/plugins/AlienFeatureValueTypeAdaptor.java b/feathr-impl/src/test/java/com/linkedin/feathr/offline/plugins/AlienFeatureValueTypeAdaptor.java similarity index 100% rename from src/test/java/com/linkedin/feathr/offline/plugins/AlienFeatureValueTypeAdaptor.java rename to feathr-impl/src/test/java/com/linkedin/feathr/offline/plugins/AlienFeatureValueTypeAdaptor.java diff --git a/src/test/java/com/linkedin/feathr/offline/plugins/FeathrFeatureValueMvelUDFs.java b/feathr-impl/src/test/java/com/linkedin/feathr/offline/plugins/FeathrFeatureValueMvelUDFs.java similarity index 100% rename from src/test/java/com/linkedin/feathr/offline/plugins/FeathrFeatureValueMvelUDFs.java rename to feathr-impl/src/test/java/com/linkedin/feathr/offline/plugins/FeathrFeatureValueMvelUDFs.java diff --git a/src/test/resources/LocalSQLAnchorTest/feature.avro.json b/feathr-impl/src/test/resources/LocalSQLAnchorTest/feature.avro.json similarity index 100% rename from src/test/resources/LocalSQLAnchorTest/feature.avro.json rename to feathr-impl/src/test/resources/LocalSQLAnchorTest/feature.avro.json diff --git a/src/test/resources/LocalSQLAnchorTest/obs.avro.json b/feathr-impl/src/test/resources/LocalSQLAnchorTest/obs.avro.json similarity index 100% rename from src/test/resources/LocalSQLAnchorTest/obs.avro.json rename to feathr-impl/src/test/resources/LocalSQLAnchorTest/obs.avro.json diff --git a/src/test/resources/anchor1-source.csv b/feathr-impl/src/test/resources/anchor1-source.csv similarity index 100% rename from src/test/resources/anchor1-source.csv rename to feathr-impl/src/test/resources/anchor1-source.csv diff --git a/src/test/resources/anchor1-source.tsv b/feathr-impl/src/test/resources/anchor1-source.tsv similarity index 100% rename from src/test/resources/anchor1-source.tsv rename to feathr-impl/src/test/resources/anchor1-source.tsv diff --git a/src/test/resources/anchor2-source.csv b/feathr-impl/src/test/resources/anchor2-source.csv similarity index 100% rename from src/test/resources/anchor2-source.csv rename to feathr-impl/src/test/resources/anchor2-source.csv diff --git a/src/test/resources/anchor3-source.csv b/feathr-impl/src/test/resources/anchor3-source.csv similarity index 100% rename from src/test/resources/anchor3-source.csv rename to feathr-impl/src/test/resources/anchor3-source.csv diff --git a/src/test/resources/anchor4-source.csv b/feathr-impl/src/test/resources/anchor4-source.csv similarity index 100% rename from src/test/resources/anchor4-source.csv rename to feathr-impl/src/test/resources/anchor4-source.csv diff --git a/src/test/resources/anchor5-source.avro.json b/feathr-impl/src/test/resources/anchor5-source.avro.json similarity index 100% rename from src/test/resources/anchor5-source.avro.json rename to feathr-impl/src/test/resources/anchor5-source.avro.json diff --git a/src/test/resources/anchor6-source.csv b/feathr-impl/src/test/resources/anchor6-source.csv similarity index 100% rename from src/test/resources/anchor6-source.csv rename to feathr-impl/src/test/resources/anchor6-source.csv diff --git a/src/test/resources/anchorAndDerivations/derivations/anchor6-source.csv b/feathr-impl/src/test/resources/anchorAndDerivations/derivations/anchor6-source.csv similarity index 100% rename from src/test/resources/anchorAndDerivations/derivations/anchor6-source.csv rename to feathr-impl/src/test/resources/anchorAndDerivations/derivations/anchor6-source.csv diff --git a/src/test/resources/anchorAndDerivations/derivations/featureGeneration/Data.avro.json b/feathr-impl/src/test/resources/anchorAndDerivations/derivations/featureGeneration/Data.avro.json similarity index 100% rename from src/test/resources/anchorAndDerivations/derivations/featureGeneration/Data.avro.json rename to feathr-impl/src/test/resources/anchorAndDerivations/derivations/featureGeneration/Data.avro.json diff --git a/src/test/resources/anchorAndDerivations/derivations/featureGeneration/Names.avro.json b/feathr-impl/src/test/resources/anchorAndDerivations/derivations/featureGeneration/Names.avro.json similarity index 100% rename from src/test/resources/anchorAndDerivations/derivations/featureGeneration/Names.avro.json rename to feathr-impl/src/test/resources/anchorAndDerivations/derivations/featureGeneration/Names.avro.json diff --git a/src/test/resources/anchorAndDerivations/derivations/test2-observations.csv b/feathr-impl/src/test/resources/anchorAndDerivations/derivations/test2-observations.csv similarity index 100% rename from src/test/resources/anchorAndDerivations/derivations/test2-observations.csv rename to feathr-impl/src/test/resources/anchorAndDerivations/derivations/test2-observations.csv diff --git a/src/test/resources/anchorAndDerivations/nullValue-source4.avro.json b/feathr-impl/src/test/resources/anchorAndDerivations/nullValue-source4.avro.json similarity index 100% rename from src/test/resources/anchorAndDerivations/nullValue-source4.avro.json rename to feathr-impl/src/test/resources/anchorAndDerivations/nullValue-source4.avro.json diff --git a/src/test/resources/anchorAndDerivations/nullValue-source5.avro.json b/feathr-impl/src/test/resources/anchorAndDerivations/nullValue-source5.avro.json similarity index 100% rename from src/test/resources/anchorAndDerivations/nullValue-source5.avro.json rename to feathr-impl/src/test/resources/anchorAndDerivations/nullValue-source5.avro.json diff --git a/src/test/resources/anchorAndDerivations/nullValueSource.avro.json b/feathr-impl/src/test/resources/anchorAndDerivations/nullValueSource.avro.json similarity index 100% rename from src/test/resources/anchorAndDerivations/nullValueSource.avro.json rename to feathr-impl/src/test/resources/anchorAndDerivations/nullValueSource.avro.json diff --git a/src/test/resources/anchorAndDerivations/passThrough/passthrough.avro.json b/feathr-impl/src/test/resources/anchorAndDerivations/passThrough/passthrough.avro.json similarity index 100% rename from src/test/resources/anchorAndDerivations/passThrough/passthrough.avro.json rename to feathr-impl/src/test/resources/anchorAndDerivations/passThrough/passthrough.avro.json diff --git a/src/test/resources/anchorAndDerivations/simple-obs2.avro.json b/feathr-impl/src/test/resources/anchorAndDerivations/simple-obs2.avro.json similarity index 100% rename from src/test/resources/anchorAndDerivations/simple-obs2.avro.json rename to feathr-impl/src/test/resources/anchorAndDerivations/simple-obs2.avro.json diff --git a/src/test/resources/anchorAndDerivations/test5-observations.csv b/feathr-impl/src/test/resources/anchorAndDerivations/test5-observations.csv similarity index 100% rename from src/test/resources/anchorAndDerivations/test5-observations.csv rename to feathr-impl/src/test/resources/anchorAndDerivations/test5-observations.csv diff --git a/src/test/resources/anchorAndDerivations/testMVELLoopExpFeature-observations.csv b/feathr-impl/src/test/resources/anchorAndDerivations/testMVELLoopExpFeature-observations.csv similarity index 100% rename from src/test/resources/anchorAndDerivations/testMVELLoopExpFeature-observations.csv rename to feathr-impl/src/test/resources/anchorAndDerivations/testMVELLoopExpFeature-observations.csv diff --git a/src/test/resources/avro/2022/09/15/part-00000-a5fbb15b-11b1-4a96-9fb0-28f7b77de928-c000.avro b/feathr-impl/src/test/resources/avro/2022/09/15/part-00000-a5fbb15b-11b1-4a96-9fb0-28f7b77de928-c000.avro similarity index 100% rename from src/test/resources/avro/2022/09/15/part-00000-a5fbb15b-11b1-4a96-9fb0-28f7b77de928-c000.avro rename to feathr-impl/src/test/resources/avro/2022/09/15/part-00000-a5fbb15b-11b1-4a96-9fb0-28f7b77de928-c000.avro diff --git a/src/test/resources/avro/2022/09/15/part-00001-a5fbb15b-11b1-4a96-9fb0-28f7b77de928-c000.avro b/feathr-impl/src/test/resources/avro/2022/09/15/part-00001-a5fbb15b-11b1-4a96-9fb0-28f7b77de928-c000.avro similarity index 100% rename from src/test/resources/avro/2022/09/15/part-00001-a5fbb15b-11b1-4a96-9fb0-28f7b77de928-c000.avro rename to feathr-impl/src/test/resources/avro/2022/09/15/part-00001-a5fbb15b-11b1-4a96-9fb0-28f7b77de928-c000.avro diff --git a/src/test/resources/bloomfilter-s1.avro.json b/feathr-impl/src/test/resources/bloomfilter-s1.avro.json similarity index 100% rename from src/test/resources/bloomfilter-s1.avro.json rename to feathr-impl/src/test/resources/bloomfilter-s1.avro.json diff --git a/src/test/resources/bloomfilter-s2.avro.json b/feathr-impl/src/test/resources/bloomfilter-s2.avro.json similarity index 100% rename from src/test/resources/bloomfilter-s2.avro.json rename to feathr-impl/src/test/resources/bloomfilter-s2.avro.json diff --git a/src/test/resources/bloomfilter-s3.avro.json b/feathr-impl/src/test/resources/bloomfilter-s3.avro.json similarity index 100% rename from src/test/resources/bloomfilter-s3.avro.json rename to feathr-impl/src/test/resources/bloomfilter-s3.avro.json diff --git a/src/test/resources/decayTest/daily/2019/05/20/data.avro.json b/feathr-impl/src/test/resources/decayTest/daily/2019/05/20/data.avro.json similarity index 100% rename from src/test/resources/decayTest/daily/2019/05/20/data.avro.json rename to feathr-impl/src/test/resources/decayTest/daily/2019/05/20/data.avro.json diff --git a/src/test/resources/feathrConf-default.conf b/feathr-impl/src/test/resources/feathrConf-default.conf similarity index 100% rename from src/test/resources/feathrConf-default.conf rename to feathr-impl/src/test/resources/feathrConf-default.conf diff --git a/src/test/resources/featureAliasing/viewerFeatureData.avro.json b/feathr-impl/src/test/resources/featureAliasing/viewerFeatureData.avro.json similarity index 100% rename from src/test/resources/featureAliasing/viewerFeatureData.avro.json rename to feathr-impl/src/test/resources/featureAliasing/viewerFeatureData.avro.json diff --git a/src/test/resources/featureAliasing/viewerObsData.avro.json b/feathr-impl/src/test/resources/featureAliasing/viewerObsData.avro.json similarity index 100% rename from src/test/resources/featureAliasing/viewerObsData.avro.json rename to feathr-impl/src/test/resources/featureAliasing/viewerObsData.avro.json diff --git a/src/test/resources/featuresWithFilterObs.avro.json b/feathr-impl/src/test/resources/featuresWithFilterObs.avro.json similarity index 100% rename from src/test/resources/featuresWithFilterObs.avro.json rename to feathr-impl/src/test/resources/featuresWithFilterObs.avro.json diff --git a/src/test/resources/frameConf-default.conf b/feathr-impl/src/test/resources/frameConf-default.conf similarity index 100% rename from src/test/resources/frameConf-default.conf rename to feathr-impl/src/test/resources/frameConf-default.conf diff --git a/src/test/resources/generation/daily/2019/05/19/data.avro.json b/feathr-impl/src/test/resources/generation/daily/2019/05/19/data.avro.json similarity index 100% rename from src/test/resources/generation/daily/2019/05/19/data.avro.json rename to feathr-impl/src/test/resources/generation/daily/2019/05/19/data.avro.json diff --git a/src/test/resources/generation/daily/2019/05/20/data.avro.json b/feathr-impl/src/test/resources/generation/daily/2019/05/20/data.avro.json similarity index 100% rename from src/test/resources/generation/daily/2019/05/20/data.avro.json rename to feathr-impl/src/test/resources/generation/daily/2019/05/20/data.avro.json diff --git a/src/test/resources/generation/daily/2019/05/21/data.avro.json b/feathr-impl/src/test/resources/generation/daily/2019/05/21/data.avro.json similarity index 100% rename from src/test/resources/generation/daily/2019/05/21/data.avro.json rename to feathr-impl/src/test/resources/generation/daily/2019/05/21/data.avro.json diff --git a/src/test/resources/generation/daily/2019/05/22/data.avro.json b/feathr-impl/src/test/resources/generation/daily/2019/05/22/data.avro.json similarity index 100% rename from src/test/resources/generation/daily/2019/05/22/data.avro.json rename to feathr-impl/src/test/resources/generation/daily/2019/05/22/data.avro.json diff --git a/src/test/resources/generation/hourly/2019/05/19/01/data.avro.json b/feathr-impl/src/test/resources/generation/hourly/2019/05/19/01/data.avro.json similarity index 100% rename from src/test/resources/generation/hourly/2019/05/19/01/data.avro.json rename to feathr-impl/src/test/resources/generation/hourly/2019/05/19/01/data.avro.json diff --git a/src/test/resources/generation/hourly/2019/05/19/02/data.avro.json b/feathr-impl/src/test/resources/generation/hourly/2019/05/19/02/data.avro.json similarity index 100% rename from src/test/resources/generation/hourly/2019/05/19/02/data.avro.json rename to feathr-impl/src/test/resources/generation/hourly/2019/05/19/02/data.avro.json diff --git a/src/test/resources/generation/hourly/2019/05/19/03/data.avro.json b/feathr-impl/src/test/resources/generation/hourly/2019/05/19/03/data.avro.json similarity index 100% rename from src/test/resources/generation/hourly/2019/05/19/03/data.avro.json rename to feathr-impl/src/test/resources/generation/hourly/2019/05/19/03/data.avro.json diff --git a/src/test/resources/generation/hourly/2019/05/19/04/data.avro.json b/feathr-impl/src/test/resources/generation/hourly/2019/05/19/04/data.avro.json similarity index 100% rename from src/test/resources/generation/hourly/2019/05/19/04/data.avro.json rename to feathr-impl/src/test/resources/generation/hourly/2019/05/19/04/data.avro.json diff --git a/src/test/resources/generation/hourly/2019/05/19/05/data.avro.json b/feathr-impl/src/test/resources/generation/hourly/2019/05/19/05/data.avro.json similarity index 100% rename from src/test/resources/generation/hourly/2019/05/19/05/data.avro.json rename to feathr-impl/src/test/resources/generation/hourly/2019/05/19/05/data.avro.json diff --git a/src/test/resources/generation/hourly/2019/05/20/01/data.avro.json b/feathr-impl/src/test/resources/generation/hourly/2019/05/20/01/data.avro.json similarity index 100% rename from src/test/resources/generation/hourly/2019/05/20/01/data.avro.json rename to feathr-impl/src/test/resources/generation/hourly/2019/05/20/01/data.avro.json diff --git a/src/test/resources/generation/hourly/2019/05/21/01/data.avro.json b/feathr-impl/src/test/resources/generation/hourly/2019/05/21/01/data.avro.json similarity index 100% rename from src/test/resources/generation/hourly/2019/05/21/01/data.avro.json rename to feathr-impl/src/test/resources/generation/hourly/2019/05/21/01/data.avro.json diff --git a/src/test/resources/generation/hourly/2019/05/22/01/data.avro.json b/feathr-impl/src/test/resources/generation/hourly/2019/05/22/01/data.avro.json similarity index 100% rename from src/test/resources/generation/hourly/2019/05/22/01/data.avro.json rename to feathr-impl/src/test/resources/generation/hourly/2019/05/22/01/data.avro.json diff --git a/src/test/resources/generationHourly/hourly/2019/05/19/00/data.avro.json b/feathr-impl/src/test/resources/generationHourly/hourly/2019/05/19/00/data.avro.json similarity index 100% rename from src/test/resources/generationHourly/hourly/2019/05/19/00/data.avro.json rename to feathr-impl/src/test/resources/generationHourly/hourly/2019/05/19/00/data.avro.json diff --git a/src/test/resources/generationHourly/hourly/2019/05/19/01/data.avro.json b/feathr-impl/src/test/resources/generationHourly/hourly/2019/05/19/01/data.avro.json similarity index 100% rename from src/test/resources/generationHourly/hourly/2019/05/19/01/data.avro.json rename to feathr-impl/src/test/resources/generationHourly/hourly/2019/05/19/01/data.avro.json diff --git a/src/test/resources/generationHourly/hourly/2019/05/19/02/data.avro.json b/feathr-impl/src/test/resources/generationHourly/hourly/2019/05/19/02/data.avro.json similarity index 100% rename from src/test/resources/generationHourly/hourly/2019/05/19/02/data.avro.json rename to feathr-impl/src/test/resources/generationHourly/hourly/2019/05/19/02/data.avro.json diff --git a/src/test/resources/incrementalTestSource1/daily/2019/05/17/data.avro.json b/feathr-impl/src/test/resources/incrementalTestSource1/daily/2019/05/17/data.avro.json similarity index 100% rename from src/test/resources/incrementalTestSource1/daily/2019/05/17/data.avro.json rename to feathr-impl/src/test/resources/incrementalTestSource1/daily/2019/05/17/data.avro.json diff --git a/src/test/resources/incrementalTestSource1/daily/2019/05/18/data.avro.json b/feathr-impl/src/test/resources/incrementalTestSource1/daily/2019/05/18/data.avro.json similarity index 100% rename from src/test/resources/incrementalTestSource1/daily/2019/05/18/data.avro.json rename to feathr-impl/src/test/resources/incrementalTestSource1/daily/2019/05/18/data.avro.json diff --git a/src/test/resources/incrementalTestSource1/daily/2019/05/19/data.avro.json b/feathr-impl/src/test/resources/incrementalTestSource1/daily/2019/05/19/data.avro.json similarity index 100% rename from src/test/resources/incrementalTestSource1/daily/2019/05/19/data.avro.json rename to feathr-impl/src/test/resources/incrementalTestSource1/daily/2019/05/19/data.avro.json diff --git a/src/test/resources/incrementalTestSource1/daily/2019/05/20/data.avro.json b/feathr-impl/src/test/resources/incrementalTestSource1/daily/2019/05/20/data.avro.json similarity index 100% rename from src/test/resources/incrementalTestSource1/daily/2019/05/20/data.avro.json rename to feathr-impl/src/test/resources/incrementalTestSource1/daily/2019/05/20/data.avro.json diff --git a/src/test/resources/incrementalTestSource1/daily/2019/05/21/data.avro.json b/feathr-impl/src/test/resources/incrementalTestSource1/daily/2019/05/21/data.avro.json similarity index 100% rename from src/test/resources/incrementalTestSource1/daily/2019/05/21/data.avro.json rename to feathr-impl/src/test/resources/incrementalTestSource1/daily/2019/05/21/data.avro.json diff --git a/src/test/resources/incrementalTestSource2/daily/2019/05/17/data.avro.json b/feathr-impl/src/test/resources/incrementalTestSource2/daily/2019/05/17/data.avro.json similarity index 100% rename from src/test/resources/incrementalTestSource2/daily/2019/05/17/data.avro.json rename to feathr-impl/src/test/resources/incrementalTestSource2/daily/2019/05/17/data.avro.json diff --git a/src/test/resources/incrementalTestSource2/daily/2019/05/18/data.avro.json b/feathr-impl/src/test/resources/incrementalTestSource2/daily/2019/05/18/data.avro.json similarity index 100% rename from src/test/resources/incrementalTestSource2/daily/2019/05/18/data.avro.json rename to feathr-impl/src/test/resources/incrementalTestSource2/daily/2019/05/18/data.avro.json diff --git a/src/test/resources/incrementalTestSource2/daily/2019/05/19/data.avro.json b/feathr-impl/src/test/resources/incrementalTestSource2/daily/2019/05/19/data.avro.json similarity index 100% rename from src/test/resources/incrementalTestSource2/daily/2019/05/19/data.avro.json rename to feathr-impl/src/test/resources/incrementalTestSource2/daily/2019/05/19/data.avro.json diff --git a/src/test/resources/incrementalTestSource2/daily/2019/05/20/data.avro.json b/feathr-impl/src/test/resources/incrementalTestSource2/daily/2019/05/20/data.avro.json similarity index 100% rename from src/test/resources/incrementalTestSource2/daily/2019/05/20/data.avro.json rename to feathr-impl/src/test/resources/incrementalTestSource2/daily/2019/05/20/data.avro.json diff --git a/src/test/resources/incrementalTestSource2/daily/2019/05/21/data.avro.json b/feathr-impl/src/test/resources/incrementalTestSource2/daily/2019/05/21/data.avro.json similarity index 100% rename from src/test/resources/incrementalTestSource2/daily/2019/05/21/data.avro.json rename to feathr-impl/src/test/resources/incrementalTestSource2/daily/2019/05/21/data.avro.json diff --git a/src/test/resources/localAnchorTestObsData.avro.json b/feathr-impl/src/test/resources/localAnchorTestObsData.avro.json similarity index 100% rename from src/test/resources/localAnchorTestObsData.avro.json rename to feathr-impl/src/test/resources/localAnchorTestObsData.avro.json diff --git a/src/test/resources/localSWAAnchorTestFeatureData/daily/2018/05/01/data.avro.json b/feathr-impl/src/test/resources/localSWAAnchorTestFeatureData/daily/2018/05/01/data.avro.json similarity index 100% rename from src/test/resources/localSWAAnchorTestFeatureData/daily/2018/05/01/data.avro.json rename to feathr-impl/src/test/resources/localSWAAnchorTestFeatureData/daily/2018/05/01/data.avro.json diff --git a/src/test/resources/localTimeAwareTestFeatureData/daily/2018/04/30/data.avro.json b/feathr-impl/src/test/resources/localTimeAwareTestFeatureData/daily/2018/04/30/data.avro.json similarity index 100% rename from src/test/resources/localTimeAwareTestFeatureData/daily/2018/04/30/data.avro.json rename to feathr-impl/src/test/resources/localTimeAwareTestFeatureData/daily/2018/04/30/data.avro.json diff --git a/src/test/resources/localTimeAwareTestFeatureData/daily/2018/05/01/data.avro.json b/feathr-impl/src/test/resources/localTimeAwareTestFeatureData/daily/2018/05/01/data.avro.json similarity index 100% rename from src/test/resources/localTimeAwareTestFeatureData/daily/2018/05/01/data.avro.json rename to feathr-impl/src/test/resources/localTimeAwareTestFeatureData/daily/2018/05/01/data.avro.json diff --git a/src/test/resources/localTimeAwareTestFeatureData/daily/2018/05/02/data.avro.json b/feathr-impl/src/test/resources/localTimeAwareTestFeatureData/daily/2018/05/02/data.avro.json similarity index 100% rename from src/test/resources/localTimeAwareTestFeatureData/daily/2018/05/02/data.avro.json rename to feathr-impl/src/test/resources/localTimeAwareTestFeatureData/daily/2018/05/02/data.avro.json diff --git a/src/test/resources/metric.properties b/feathr-impl/src/test/resources/metric.properties similarity index 100% rename from src/test/resources/metric.properties rename to feathr-impl/src/test/resources/metric.properties diff --git a/src/test/resources/mockdata/driver_data/copy_green_tripdata_2021-01.csv b/feathr-impl/src/test/resources/mockdata/driver_data/copy_green_tripdata_2021-01.csv similarity index 100% rename from src/test/resources/mockdata/driver_data/copy_green_tripdata_2021-01.csv rename to feathr-impl/src/test/resources/mockdata/driver_data/copy_green_tripdata_2021-01.csv diff --git a/src/test/resources/mockdata/driver_data/green_tripdata_2021-01.csv b/feathr-impl/src/test/resources/mockdata/driver_data/green_tripdata_2021-01.csv similarity index 100% rename from src/test/resources/mockdata/driver_data/green_tripdata_2021-01.csv rename to feathr-impl/src/test/resources/mockdata/driver_data/green_tripdata_2021-01.csv diff --git a/src/test/resources/mockdata/feature_monitoring_mock_data/feature_monitoring_data.csv b/feathr-impl/src/test/resources/mockdata/feature_monitoring_mock_data/feature_monitoring_data.csv similarity index 100% rename from src/test/resources/mockdata/feature_monitoring_mock_data/feature_monitoring_data.csv rename to feathr-impl/src/test/resources/mockdata/feature_monitoring_mock_data/feature_monitoring_data.csv diff --git a/src/test/resources/mockdata/simple-obs2/mockData.json b/feathr-impl/src/test/resources/mockdata/simple-obs2/mockData.json similarity index 100% rename from src/test/resources/mockdata/simple-obs2/mockData.json rename to feathr-impl/src/test/resources/mockdata/simple-obs2/mockData.json diff --git a/src/test/resources/mockdata/simple-obs2/schema.avsc b/feathr-impl/src/test/resources/mockdata/simple-obs2/schema.avsc similarity index 100% rename from src/test/resources/mockdata/simple-obs2/schema.avsc rename to feathr-impl/src/test/resources/mockdata/simple-obs2/schema.avsc diff --git a/src/test/resources/mockdata/sqlite/test.db b/feathr-impl/src/test/resources/mockdata/sqlite/test.db similarity index 100% rename from src/test/resources/mockdata/sqlite/test.db rename to feathr-impl/src/test/resources/mockdata/sqlite/test.db diff --git a/src/test/resources/nullValue-source.avro.json b/feathr-impl/src/test/resources/nullValue-source.avro.json similarity index 100% rename from src/test/resources/nullValue-source.avro.json rename to feathr-impl/src/test/resources/nullValue-source.avro.json diff --git a/src/test/resources/nullValue-source1.avro.json b/feathr-impl/src/test/resources/nullValue-source1.avro.json similarity index 100% rename from src/test/resources/nullValue-source1.avro.json rename to feathr-impl/src/test/resources/nullValue-source1.avro.json diff --git a/src/test/resources/nullValue-source2.avro.json b/feathr-impl/src/test/resources/nullValue-source2.avro.json similarity index 100% rename from src/test/resources/nullValue-source2.avro.json rename to feathr-impl/src/test/resources/nullValue-source2.avro.json diff --git a/src/test/resources/nullValue-source3.avro.json b/feathr-impl/src/test/resources/nullValue-source3.avro.json similarity index 100% rename from src/test/resources/nullValue-source3.avro.json rename to feathr-impl/src/test/resources/nullValue-source3.avro.json diff --git a/src/test/resources/nullValueSource.avro.json b/feathr-impl/src/test/resources/nullValueSource.avro.json similarity index 100% rename from src/test/resources/nullValueSource.avro.json rename to feathr-impl/src/test/resources/nullValueSource.avro.json diff --git a/src/test/resources/obs/obs.csv b/feathr-impl/src/test/resources/obs/obs.csv similarity index 100% rename from src/test/resources/obs/obs.csv rename to feathr-impl/src/test/resources/obs/obs.csv diff --git a/src/test/resources/sampleFeatureDef.conf b/feathr-impl/src/test/resources/sampleFeatureDef.conf similarity index 100% rename from src/test/resources/sampleFeatureDef.conf rename to feathr-impl/src/test/resources/sampleFeatureDef.conf diff --git a/src/test/resources/simple-obs.csv b/feathr-impl/src/test/resources/simple-obs.csv similarity index 100% rename from src/test/resources/simple-obs.csv rename to feathr-impl/src/test/resources/simple-obs.csv diff --git a/src/test/resources/simple-obs2.avro.json b/feathr-impl/src/test/resources/simple-obs2.avro.json similarity index 100% rename from src/test/resources/simple-obs2.avro.json rename to feathr-impl/src/test/resources/simple-obs2.avro.json diff --git a/src/test/resources/slidingWindowAgg/csvTypeTimeFile1.csv b/feathr-impl/src/test/resources/slidingWindowAgg/csvTypeTimeFile1.csv similarity index 100% rename from src/test/resources/slidingWindowAgg/csvTypeTimeFile1.csv rename to feathr-impl/src/test/resources/slidingWindowAgg/csvTypeTimeFile1.csv diff --git a/src/test/resources/slidingWindowAgg/daily/2018/04/25/data.avro.json b/feathr-impl/src/test/resources/slidingWindowAgg/daily/2018/04/25/data.avro.json similarity index 100% rename from src/test/resources/slidingWindowAgg/daily/2018/04/25/data.avro.json rename to feathr-impl/src/test/resources/slidingWindowAgg/daily/2018/04/25/data.avro.json diff --git a/src/test/resources/slidingWindowAgg/featureDataWithUnionNull.avro.json b/feathr-impl/src/test/resources/slidingWindowAgg/featureDataWithUnionNull.avro.json similarity index 100% rename from src/test/resources/slidingWindowAgg/featureDataWithUnionNull.avro.json rename to feathr-impl/src/test/resources/slidingWindowAgg/featureDataWithUnionNull.avro.json diff --git a/src/test/resources/slidingWindowAgg/foo/daily/2019/01/05/data.avro.json b/feathr-impl/src/test/resources/slidingWindowAgg/foo/daily/2019/01/05/data.avro.json similarity index 100% rename from src/test/resources/slidingWindowAgg/foo/daily/2019/01/05/data.avro.json rename to feathr-impl/src/test/resources/slidingWindowAgg/foo/daily/2019/01/05/data.avro.json diff --git a/src/test/resources/slidingWindowAgg/hourlyObsData.avro.json b/feathr-impl/src/test/resources/slidingWindowAgg/hourlyObsData.avro.json similarity index 100% rename from src/test/resources/slidingWindowAgg/hourlyObsData.avro.json rename to feathr-impl/src/test/resources/slidingWindowAgg/hourlyObsData.avro.json diff --git a/src/test/resources/slidingWindowAgg/localAnchorTestObsData.avro.json b/feathr-impl/src/test/resources/slidingWindowAgg/localAnchorTestObsData.avro.json similarity index 100% rename from src/test/resources/slidingWindowAgg/localAnchorTestObsData.avro.json rename to feathr-impl/src/test/resources/slidingWindowAgg/localAnchorTestObsData.avro.json diff --git a/src/test/resources/slidingWindowAgg/localSWAAnchorTestFeatureData/daily/2018/05/01/data.avro.json b/feathr-impl/src/test/resources/slidingWindowAgg/localSWAAnchorTestFeatureData/daily/2018/05/01/data.avro.json similarity index 100% rename from src/test/resources/slidingWindowAgg/localSWAAnchorTestFeatureData/daily/2018/05/01/data.avro.json rename to feathr-impl/src/test/resources/slidingWindowAgg/localSWAAnchorTestFeatureData/daily/2018/05/01/data.avro.json diff --git a/src/test/resources/slidingWindowAgg/localSWADefaultTest/daily/2018/05/01/data.avro.json b/feathr-impl/src/test/resources/slidingWindowAgg/localSWADefaultTest/daily/2018/05/01/data.avro.json similarity index 100% rename from src/test/resources/slidingWindowAgg/localSWADefaultTest/daily/2018/05/01/data.avro.json rename to feathr-impl/src/test/resources/slidingWindowAgg/localSWADefaultTest/daily/2018/05/01/data.avro.json diff --git a/src/test/resources/slidingWindowAgg/localSWASimulateTimeDelay/daily/2018/04/25/data.avro.json b/feathr-impl/src/test/resources/slidingWindowAgg/localSWASimulateTimeDelay/daily/2018/04/25/data.avro.json similarity index 100% rename from src/test/resources/slidingWindowAgg/localSWASimulateTimeDelay/daily/2018/04/25/data.avro.json rename to feathr-impl/src/test/resources/slidingWindowAgg/localSWASimulateTimeDelay/daily/2018/04/25/data.avro.json diff --git a/src/test/resources/slidingWindowAgg/localSWASimulateTimeDelay/daily/2018/04/28/data.avro.json b/feathr-impl/src/test/resources/slidingWindowAgg/localSWASimulateTimeDelay/daily/2018/04/28/data.avro.json similarity index 100% rename from src/test/resources/slidingWindowAgg/localSWASimulateTimeDelay/daily/2018/04/28/data.avro.json rename to feathr-impl/src/test/resources/slidingWindowAgg/localSWASimulateTimeDelay/daily/2018/04/28/data.avro.json diff --git a/src/test/resources/slidingWindowAgg/localSWASimulateTimeDelay/daily/2018/05/01/data.avro.json b/feathr-impl/src/test/resources/slidingWindowAgg/localSWASimulateTimeDelay/daily/2018/05/01/data.avro.json similarity index 100% rename from src/test/resources/slidingWindowAgg/localSWASimulateTimeDelay/daily/2018/05/01/data.avro.json rename to feathr-impl/src/test/resources/slidingWindowAgg/localSWASimulateTimeDelay/daily/2018/05/01/data.avro.json diff --git a/src/test/resources/slidingWindowAgg/obsWithPassthrough.avro.json b/feathr-impl/src/test/resources/slidingWindowAgg/obsWithPassthrough.avro.json similarity index 100% rename from src/test/resources/slidingWindowAgg/obsWithPassthrough.avro.json rename to feathr-impl/src/test/resources/slidingWindowAgg/obsWithPassthrough.avro.json diff --git a/src/test/resources/tensors/allTensorsFeatureData.avro.json b/feathr-impl/src/test/resources/tensors/allTensorsFeatureData.avro.json similarity index 100% rename from src/test/resources/tensors/allTensorsFeatureData.avro.json rename to feathr-impl/src/test/resources/tensors/allTensorsFeatureData.avro.json diff --git a/src/test/resources/tensors/featureData.avro.json b/feathr-impl/src/test/resources/tensors/featureData.avro.json similarity index 100% rename from src/test/resources/tensors/featureData.avro.json rename to feathr-impl/src/test/resources/tensors/featureData.avro.json diff --git a/src/test/resources/tensors/obsData.avro.json b/feathr-impl/src/test/resources/tensors/obsData.avro.json similarity index 100% rename from src/test/resources/tensors/obsData.avro.json rename to feathr-impl/src/test/resources/tensors/obsData.avro.json diff --git a/src/test/resources/test1-observations.csv b/feathr-impl/src/test/resources/test1-observations.csv similarity index 100% rename from src/test/resources/test1-observations.csv rename to feathr-impl/src/test/resources/test1-observations.csv diff --git a/src/test/resources/test2-observations.csv b/feathr-impl/src/test/resources/test2-observations.csv similarity index 100% rename from src/test/resources/test2-observations.csv rename to feathr-impl/src/test/resources/test2-observations.csv diff --git a/src/test/resources/test3-observations.csv b/feathr-impl/src/test/resources/test3-observations.csv similarity index 100% rename from src/test/resources/test3-observations.csv rename to feathr-impl/src/test/resources/test3-observations.csv diff --git a/src/test/resources/test4-observations.csv b/feathr-impl/src/test/resources/test4-observations.csv similarity index 100% rename from src/test/resources/test4-observations.csv rename to feathr-impl/src/test/resources/test4-observations.csv diff --git a/src/test/resources/testAnchorsAsIs/featureGenConfig.conf b/feathr-impl/src/test/resources/testAnchorsAsIs/featureGenConfig.conf similarity index 100% rename from src/test/resources/testAnchorsAsIs/featureGenConfig.conf rename to feathr-impl/src/test/resources/testAnchorsAsIs/featureGenConfig.conf diff --git a/src/test/resources/testAnchorsAsIs/featureGenConfig_need_override.conf b/feathr-impl/src/test/resources/testAnchorsAsIs/featureGenConfig_need_override.conf similarity index 100% rename from src/test/resources/testAnchorsAsIs/featureGenConfig_need_override.conf rename to feathr-impl/src/test/resources/testAnchorsAsIs/featureGenConfig_need_override.conf diff --git a/src/test/resources/testAnchorsAsIs/joinconfig.conf b/feathr-impl/src/test/resources/testAnchorsAsIs/joinconfig.conf similarity index 100% rename from src/test/resources/testAnchorsAsIs/joinconfig.conf rename to feathr-impl/src/test/resources/testAnchorsAsIs/joinconfig.conf diff --git a/src/test/resources/testAnchorsAsIs/joinconfig_with_passthrough.conf b/feathr-impl/src/test/resources/testAnchorsAsIs/joinconfig_with_passthrough.conf similarity index 100% rename from src/test/resources/testAnchorsAsIs/joinconfig_with_passthrough.conf rename to feathr-impl/src/test/resources/testAnchorsAsIs/joinconfig_with_passthrough.conf diff --git a/src/test/resources/testAnchorsAsIs/localframe.conf b/feathr-impl/src/test/resources/testAnchorsAsIs/localframe.conf similarity index 100% rename from src/test/resources/testAnchorsAsIs/localframe.conf rename to feathr-impl/src/test/resources/testAnchorsAsIs/localframe.conf diff --git a/src/test/resources/testAnchorsAsIs/localframe_need_override.conf b/feathr-impl/src/test/resources/testAnchorsAsIs/localframe_need_override.conf similarity index 100% rename from src/test/resources/testAnchorsAsIs/localframe_need_override.conf rename to feathr-impl/src/test/resources/testAnchorsAsIs/localframe_need_override.conf diff --git a/src/test/resources/testAvroUnionType.avro.json b/feathr-impl/src/test/resources/testAvroUnionType.avro.json similarity index 100% rename from src/test/resources/testAvroUnionType.avro.json rename to feathr-impl/src/test/resources/testAvroUnionType.avro.json diff --git a/src/test/resources/testBloomfilter-observations.csv b/feathr-impl/src/test/resources/testBloomfilter-observations.csv similarity index 100% rename from src/test/resources/testBloomfilter-observations.csv rename to feathr-impl/src/test/resources/testBloomfilter-observations.csv diff --git a/src/test/resources/testBloomfilter.conf b/feathr-impl/src/test/resources/testBloomfilter.conf similarity index 100% rename from src/test/resources/testBloomfilter.conf rename to feathr-impl/src/test/resources/testBloomfilter.conf diff --git a/src/test/resources/testFlatten.avro.json b/feathr-impl/src/test/resources/testFlatten.avro.json similarity index 100% rename from src/test/resources/testFlatten.avro.json rename to feathr-impl/src/test/resources/testFlatten.avro.json diff --git a/src/test/resources/testFlatten_obs.csv b/feathr-impl/src/test/resources/testFlatten_obs.csv similarity index 100% rename from src/test/resources/testFlatten_obs.csv rename to feathr-impl/src/test/resources/testFlatten_obs.csv diff --git a/src/test/resources/testInferenceTakeout-observations.csv b/feathr-impl/src/test/resources/testInferenceTakeout-observations.csv similarity index 100% rename from src/test/resources/testInferenceTakeout-observations.csv rename to feathr-impl/src/test/resources/testInferenceTakeout-observations.csv diff --git a/src/test/resources/testMVELDerivedFeatureCheckingNull-observations.csv b/feathr-impl/src/test/resources/testMVELDerivedFeatureCheckingNull-observations.csv similarity index 100% rename from src/test/resources/testMVELDerivedFeatureCheckingNull-observations.csv rename to feathr-impl/src/test/resources/testMVELDerivedFeatureCheckingNull-observations.csv diff --git a/src/test/resources/testMVELDerivedFeatureCheckingNull.conf b/feathr-impl/src/test/resources/testMVELDerivedFeatureCheckingNull.conf similarity index 100% rename from src/test/resources/testMVELDerivedFeatureCheckingNull.conf rename to feathr-impl/src/test/resources/testMVELDerivedFeatureCheckingNull.conf diff --git a/src/test/resources/testMVELFeatureWithNullValue-observations.csv b/feathr-impl/src/test/resources/testMVELFeatureWithNullValue-observations.csv similarity index 100% rename from src/test/resources/testMVELFeatureWithNullValue-observations.csv rename to feathr-impl/src/test/resources/testMVELFeatureWithNullValue-observations.csv diff --git a/src/test/resources/testMVELFeatureWithNullValue.conf b/feathr-impl/src/test/resources/testMVELFeatureWithNullValue.conf similarity index 100% rename from src/test/resources/testMVELFeatureWithNullValue.conf rename to feathr-impl/src/test/resources/testMVELFeatureWithNullValue.conf diff --git a/src/test/resources/testMVELLoopExpFeature-observations.csv b/feathr-impl/src/test/resources/testMVELLoopExpFeature-observations.csv similarity index 100% rename from src/test/resources/testMVELLoopExpFeature-observations.csv rename to feathr-impl/src/test/resources/testMVELLoopExpFeature-observations.csv diff --git a/src/test/resources/testMVELLoopExpFeature.conf b/feathr-impl/src/test/resources/testMVELLoopExpFeature.conf similarity index 100% rename from src/test/resources/testMVELLoopExpFeature.conf rename to feathr-impl/src/test/resources/testMVELLoopExpFeature.conf diff --git a/src/test/resources/testMultiKeyDerived-observations.csv b/feathr-impl/src/test/resources/testMultiKeyDerived-observations.csv similarity index 100% rename from src/test/resources/testMultiKeyDerived-observations.csv rename to feathr-impl/src/test/resources/testMultiKeyDerived-observations.csv diff --git a/src/test/resources/testWrongMVELExpressionFeature.conf b/feathr-impl/src/test/resources/testWrongMVELExpressionFeature.conf similarity index 100% rename from src/test/resources/testWrongMVELExpressionFeature.conf rename to feathr-impl/src/test/resources/testWrongMVELExpressionFeature.conf diff --git a/src/test/resources/timeAwareJoin/creatorPopularityFeatureData/daily/2020/11/15/data.avro.json b/feathr-impl/src/test/resources/timeAwareJoin/creatorPopularityFeatureData/daily/2020/11/15/data.avro.json similarity index 100% rename from src/test/resources/timeAwareJoin/creatorPopularityFeatureData/daily/2020/11/15/data.avro.json rename to feathr-impl/src/test/resources/timeAwareJoin/creatorPopularityFeatureData/daily/2020/11/15/data.avro.json diff --git a/src/test/resources/timeAwareJoin/creatorPopularityFeatureData/daily/2020/11/16/data.avro.json b/feathr-impl/src/test/resources/timeAwareJoin/creatorPopularityFeatureData/daily/2020/11/16/data.avro.json similarity index 100% rename from src/test/resources/timeAwareJoin/creatorPopularityFeatureData/daily/2020/11/16/data.avro.json rename to feathr-impl/src/test/resources/timeAwareJoin/creatorPopularityFeatureData/daily/2020/11/16/data.avro.json diff --git a/src/test/resources/timeAwareJoin/localTimeAwareTestFeatureData/daily/2018/04/30/data.avro.json b/feathr-impl/src/test/resources/timeAwareJoin/localTimeAwareTestFeatureData/daily/2018/04/30/data.avro.json similarity index 100% rename from src/test/resources/timeAwareJoin/localTimeAwareTestFeatureData/daily/2018/04/30/data.avro.json rename to feathr-impl/src/test/resources/timeAwareJoin/localTimeAwareTestFeatureData/daily/2018/04/30/data.avro.json diff --git a/src/test/resources/timeAwareJoin/localTimeAwareTestFeatureData/daily/2018/05/01/data.avro.json b/feathr-impl/src/test/resources/timeAwareJoin/localTimeAwareTestFeatureData/daily/2018/05/01/data.avro.json similarity index 100% rename from src/test/resources/timeAwareJoin/localTimeAwareTestFeatureData/daily/2018/05/01/data.avro.json rename to feathr-impl/src/test/resources/timeAwareJoin/localTimeAwareTestFeatureData/daily/2018/05/01/data.avro.json diff --git a/src/test/resources/timeAwareJoin/localTimeAwareTestFeatureData/daily/2018/05/02/data.avro.json b/feathr-impl/src/test/resources/timeAwareJoin/localTimeAwareTestFeatureData/daily/2018/05/02/data.avro.json similarity index 100% rename from src/test/resources/timeAwareJoin/localTimeAwareTestFeatureData/daily/2018/05/02/data.avro.json rename to feathr-impl/src/test/resources/timeAwareJoin/localTimeAwareTestFeatureData/daily/2018/05/02/data.avro.json diff --git a/src/test/resources/timeAwareJoin/timeAwareFeedObservationData.avro.json b/feathr-impl/src/test/resources/timeAwareJoin/timeAwareFeedObservationData.avro.json similarity index 100% rename from src/test/resources/timeAwareJoin/timeAwareFeedObservationData.avro.json rename to feathr-impl/src/test/resources/timeAwareJoin/timeAwareFeedObservationData.avro.json diff --git a/src/test/resources/timeAwareJoin/timeAwareObsData.avro.json b/feathr-impl/src/test/resources/timeAwareJoin/timeAwareObsData.avro.json similarity index 100% rename from src/test/resources/timeAwareJoin/timeAwareObsData.avro.json rename to feathr-impl/src/test/resources/timeAwareJoin/timeAwareObsData.avro.json diff --git a/src/test/resources/xFeatureData_NewSchema.avsc b/feathr-impl/src/test/resources/xFeatureData_NewSchema.avsc similarity index 100% rename from src/test/resources/xFeatureData_NewSchema.avsc rename to feathr-impl/src/test/resources/xFeatureData_NewSchema.avsc diff --git a/src/test/scala/com/linkedin/feathr/offline/AnchoredFeaturesIntegTest.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/AnchoredFeaturesIntegTest.scala similarity index 93% rename from src/test/scala/com/linkedin/feathr/offline/AnchoredFeaturesIntegTest.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/AnchoredFeaturesIntegTest.scala index 061b42598..02964dab2 100644 --- a/src/test/scala/com/linkedin/feathr/offline/AnchoredFeaturesIntegTest.scala +++ b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/AnchoredFeaturesIntegTest.scala @@ -58,6 +58,16 @@ class AnchoredFeaturesIntegTest extends FeathrIntegTest { | type: "DENSE_VECTOR" | default: [7,8,9] | } + | ee2: { + | def: "c" + | type: { + | type: TENSOR + | tensorCategory: DENSE + | dimensionType: [INT] + | valType: FLOAT + | } + | default: [] + | } | ff: { | def: "c" | default: [6,7] @@ -155,7 +165,7 @@ class AnchoredFeaturesIntegTest extends FeathrIntegTest { */ @Test def testSingleKeyJoinWithDifferentFeatureTypes(): Unit = { - val selectedColumns = Seq("x", "aa", "bb", "cc", "dd", "ee", "ff", "multiply_a_b", "categorical_b") // , "z") + val selectedColumns = Seq("x", "aa", "bb", "cc", "dd", "ee", "ee2", "ff", "multiply_a_b", "categorical_b") // , "z") val featureJoinConf = s""" | @@ -186,6 +196,8 @@ class AnchoredFeaturesIntegTest extends FeathrIntegTest { null, // ee mutable.WrappedArray.make(Array(7.0f, 8.0f, 9.0f)), + // ee2 + mutable.WrappedArray.empty, // ff mutable.WrappedArray.make(Array(6.0f, 7.0f)), // multiply_a_b @@ -207,6 +219,8 @@ class AnchoredFeaturesIntegTest extends FeathrIntegTest { mutable.WrappedArray.make(Array(1.0f, 2.0f, 3.0f)), // ee mutable.WrappedArray.make(Array(1.0f, 2.0f, 3.0f)), + // ee2 + mutable.WrappedArray.make(Array(1.0f, 2.0f, 3.0f)), // ff mutable.WrappedArray.make(Array(1.0f, 2.0f, 3.0f)), // multiply_a_b @@ -228,6 +242,8 @@ class AnchoredFeaturesIntegTest extends FeathrIntegTest { mutable.WrappedArray.make(Array(4.0f, 5.0f, 6.0f)), // ee mutable.WrappedArray.make(Array(4.0f, 5.0f, 6.0f)), + // ee2 + mutable.WrappedArray.make(Array(4.0f, 5.0f, 6.0f)), // ff mutable.WrappedArray.make(Array(4.0f, 5.0f, 6.0f)), // multiply_a_b @@ -246,6 +262,7 @@ class AnchoredFeaturesIntegTest extends FeathrIntegTest { StructField("cc", FloatType, true), StructField("dd", ArrayType(FloatType, true), true), StructField("ee", ArrayType(FloatType, false), true), + StructField("ee2", ArrayType(FloatType, false), true), StructField("ff", ArrayType(FloatType, false), true), StructField( "multiply_a_b", @@ -317,9 +334,9 @@ class AnchoredFeaturesIntegTest extends FeathrIntegTest { /** * This test validates that Passthrough features specified over multiple anchors - * do not get dropped silently in the output. + * do not get dropped silently in the output. TODO: Enable test after FCM can handle new config syntax */ - @Test + @Test(enabled = false) def testPassthroughFeaturesNotDroppedWithMultipleAnchors(): Unit = { val featureDefAsString = """ @@ -423,7 +440,8 @@ class AnchoredFeaturesIntegTest extends FeathrIntegTest { ds.data.show() } - @Test + // TODO: Enable after FCM can handle new syntax + @Test(enabled = false) def testPassthroughFeaturesWithSWA(): Unit = { val featureDefAsString = """ @@ -467,7 +485,16 @@ class AnchoredFeaturesIntegTest extends FeathrIntegTest { | |derivations: { | f_trip_time_distance: { - | definition: "f_trip_distance * f_trip_time_duration" + | definition: "f_trip_distance * f_trip_time_duration" + | type: NUMERIC + | } + | f_trip_time_distance_sql: { + | key: [trip] + | inputs: { + | trip_distance: { key: [trip], feature: f_trip_distance } + | trip_time_duration: { key: [trip], feature: f_trip_time_duration } + | } + | definition.sqlExpr: "trip_distance * trip_time_duration" | type: NUMERIC | } |} @@ -497,7 +524,8 @@ class AnchoredFeaturesIntegTest extends FeathrIntegTest { |featureList: [ | { | key: DOLocationID - | featureList: [f_location_avg_fare, f_trip_time_distance, f_trip_distance, f_trip_time_duration, f_is_long_trip_distance, f_day_of_week] + | featureList: [f_location_avg_fare, f_trip_time_distance, f_trip_distance, + | f_trip_time_duration, f_is_long_trip_distance, f_day_of_week, f_trip_time_distance_sql] | } |] """.stripMargin @@ -506,7 +534,8 @@ class AnchoredFeaturesIntegTest extends FeathrIntegTest { df.data.show() } - @Test + // TODO: Enable after FCM can handle new syntax + @Test(enabled = false) def tesSWAWithPreprocessing(): Unit = { val featureDefAsString = """ diff --git a/src/test/scala/com/linkedin/feathr/offline/AssertFeatureUtils.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/AssertFeatureUtils.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/AssertFeatureUtils.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/AssertFeatureUtils.scala diff --git a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/DerivationsIntegTest.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/DerivationsIntegTest.scala new file mode 100644 index 000000000..94e92e06d --- /dev/null +++ b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/DerivationsIntegTest.scala @@ -0,0 +1,146 @@ +package com.linkedin.feathr.offline + +import com.linkedin.feathr.offline.util.FeathrTestUtils.assertDataFrameApproximatelyEquals +import org.apache.spark.sql.Row +import org.apache.spark.sql.types._ +import org.testng.annotations.Test + +class DerivationsIntegTest extends FeathrIntegTest { + + /** + * Test multi-key derived feature and multi-tagged feature. + * This test covers the following:- + * -> sql based custom extractor + */ + @Test + def testMultiKeyDerivedFeatureDFWithSQL: Unit = { + val df = runLocalFeatureJoinForTest( + joinConfigAsString = """ + | features: [ { + | key: ["concat('',viewer)", viewee] + | featureList: [ "foo_square_distance_sql"] + | } , + | { + | key: [viewee, viewer] + | featureList: [ "foo_square_distance_sql"] + | }, + | { + | key: [viewee, viewer] + | featureList: [ "square_fooFeature_sql"] + | } + | ] + """.stripMargin, + featureDefAsString = """ + | anchors: { + | anchor1: { + | source: anchorAndDerivations/derivations/anchor6-source.csv + | key.sqlExpr: [sourceId, destId] + | features: { + | fooFeature: { + | def.sqlExpr: cast(source as int) + | type: NUMERIC + | } + | } + | } + | } + | derivations: { + | + | square_fooFeature_sql: { + | key: [m1, m2] + | inputs: { + | a: { key: [m1, m2], feature: fooFeature } + | } + | definition.sqlExpr: "a * a" + | } + | foo_square_distance_sql: { + | key: [m1, m2] + | inputs: { + | a1: { key: [m1, m2], feature: square_fooFeature_sql } + | a2: { key: [m2, m1], feature: square_fooFeature_sql } + | } + | definition.sqlExpr: "a1 - a2" + | } + | } + """.stripMargin, + observationDataPath = "anchorAndDerivations/derivations/test2-observations.csv") + + val expectedDf = ss.createDataFrame( + ss.sparkContext.parallelize( + Seq( + Row( + // viewer + "1", + // viewee + "3", + // label + "1.0", + // square_fooFeature_sql + 4.0f, + // viewee_viewer__foo_square_distance_sql + -21.0f, + // concat____viewer__viewee__foo_square_distance_sql + 21.0f), + Row( + // viewer + "2", + // viewee + "1", + // label + "-1.0", + // square_fooFeature_sql + 9.0f, + // viewee_viewer__foo_square_distance_sql + -27.0f, + // concat____viewer__viewee__foo_square_distance_sql + 27.0f), + Row( + // viewer + "3", + // viewee + "6", + // label + "1.0", + // square_fooFeature_sql + null, + // viewee_viewer__foo_square_distance_sql + null, + // concat____viewer__viewee__foo_square_distance_sql + null), + Row( + // viewer + "3", + // viewee + "5", + // label + "-1.0", + // square_fooFeature_sql + null, + // viewee_viewer__foo_square_distance_sql + null, + // concat____viewer__viewee__foo_square_distance_sql + null), + Row( + // viewer + "5", + // viewee + "10", + // label + "1.0", + // square_fooFeature_sql + null, + // viewee_viewer__foo_square_distance_sql + null, + // concat____viewer__viewee__foo_square_distance_sql + null))), + StructType( + List( + StructField("viewer", StringType, true), + StructField("viewee", StringType, true), + StructField("label", StringType, true), + StructField("square_fooFeature_sql", FloatType, true), + StructField("viewee_viewer__foo_square_distance_sql", FloatType, true), + StructField("concat____viewer__viewee__foo_square_distance_sql", FloatType, true)))) + def cmpFunc(row: Row): String = if (row.get(0) != null) row.get(0).toString else "null" + assertDataFrameApproximatelyEquals(df.data, expectedDf, cmpFunc) + } +} diff --git a/src/test/scala/com/linkedin/feathr/offline/FeathrIntegTest.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/FeathrIntegTest.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/FeathrIntegTest.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/FeathrIntegTest.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/FeatureGenIntegTest.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/FeatureGenIntegTest.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/FeatureGenIntegTest.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/FeatureGenIntegTest.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/FeatureMonitoringIntegTest.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/FeatureMonitoringIntegTest.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/FeatureMonitoringIntegTest.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/FeatureMonitoringIntegTest.scala diff --git a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/GatewayTest.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/GatewayTest.scala new file mode 100644 index 000000000..359b1c85b --- /dev/null +++ b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/GatewayTest.scala @@ -0,0 +1,15 @@ +package com.linkedin.feathr.offline + +import com.linkedin.feathr.cli.FeatureExperimentEntryPoint +import org.testng.annotations.{Ignore, Test} + +/** + * Execute FeatureExperimentEntryPoint.main in the context of test environment + * that has all the `provided` jars, and can be run from the IDE + */ +object GatewayTest { + def main(args: Array[String]): Unit = { + FeatureExperimentEntryPoint.main(Array()) + Thread.sleep(Long.MaxValue) + } +} \ No newline at end of file diff --git a/src/test/scala/com/linkedin/feathr/offline/SlidingWindowAggIntegTest.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/SlidingWindowAggIntegTest.scala similarity index 99% rename from src/test/scala/com/linkedin/feathr/offline/SlidingWindowAggIntegTest.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/SlidingWindowAggIntegTest.scala index 4ef4c8c5e..dd7fd7f27 100644 --- a/src/test/scala/com/linkedin/feathr/offline/SlidingWindowAggIntegTest.scala +++ b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/SlidingWindowAggIntegTest.scala @@ -130,7 +130,7 @@ class SlidingWindowAggIntegTest extends FeathrIntegTest { | } | } | } - | swaAnchorWithKeyExtractor: { + | swaAnchorWithKeyExtractor3: { | source: "swaSource" | keyExtractor: "com.linkedin.feathr.offline.anchored.keyExtractor.SimpleSampleKeyExtractor2" | lateralViewParameters: { @@ -680,8 +680,10 @@ class SlidingWindowAggIntegTest extends FeathrIntegTest { /** * test invalid case when there is an overrideTimeDelay with no simulateTimeDelay set. + * TODO: Enable after adding validation code in FCM. */ @Test( + enabled = false, expectedExceptions = Array(classOf[RuntimeException]), expectedExceptionsMessageRegExp = "\\[FEATHR_USER_ERROR\\] overrideTimeDelay cannot be defined without setting a simulateTimeDelay(.*)") def testInvalidCaseWithOverrideTimeDelay: Unit = { @@ -985,6 +987,7 @@ class SlidingWindowAggIntegTest extends FeathrIntegTest { } + /** @Test def testSWACountDistinct(): Unit = { val featureDefAsString = @@ -1064,5 +1067,5 @@ class SlidingWindowAggIntegTest extends FeathrIntegTest { val dfs = runLocalFeatureJoinForTest(featureJoinAsString, featureDefAsString, "featuresWithFilterObs.avro.json").data validateRows(dfs.select(keyField, features: _*).collect().sortBy(row => row.getAs[Int](keyField)), expectedRows) - } + }*/ } diff --git a/src/test/scala/com/linkedin/feathr/offline/TestFeathr.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/TestFeathr.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/TestFeathr.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/TestFeathr.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/TestFeathrDefaultValue.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/TestFeathrDefaultValue.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/TestFeathrDefaultValue.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/TestFeathrDefaultValue.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/TestFeathrKeyTag.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/TestFeathrKeyTag.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/TestFeathrKeyTag.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/TestFeathrKeyTag.scala diff --git a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/TestFeathrUdfPlugins.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/TestFeathrUdfPlugins.scala new file mode 100644 index 000000000..64d2cee62 --- /dev/null +++ b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/TestFeathrUdfPlugins.scala @@ -0,0 +1,141 @@ +package com.linkedin.feathr.offline + +import com.linkedin.feathr.common.FeatureTypes +import com.linkedin.feathr.offline.anchored.keyExtractor.AlienSourceKeyExtractorAdaptor +import com.linkedin.feathr.offline.client.plugins.FeathrUdfPluginContext +import com.linkedin.feathr.offline.derived.AlienDerivationFunctionAdaptor +import com.linkedin.feathr.offline.mvel.plugins.FeathrExpressionExecutionContext +import com.linkedin.feathr.offline.plugins.{AlienFeatureValue, AlienFeatureValueTypeAdaptor} +import com.linkedin.feathr.offline.util.FeathrTestUtils +import org.apache.spark.sql.Row +import org.apache.spark.sql.types.{FloatType, StringType, StructField, StructType} +import org.testng.Assert.assertEquals +import org.testng.annotations.Test + +class TestFeathrUdfPlugins extends FeathrIntegTest { + + val MULTILINE_QUOTE = "\"\"\"" + + private val mvelContext = new FeathrExpressionExecutionContext() + + // todo - support udf plugins through FCM + @Test (enabled = false) + def testMvelUdfPluginSupport: Unit = { + mvelContext.setupExecutorMvelContext(classOf[AlienFeatureValue], new AlienFeatureValueTypeAdaptor(), ss.sparkContext) + FeathrUdfPluginContext.registerUdfAdaptor(new AlienDerivationFunctionAdaptor(), ss.sparkContext) + FeathrUdfPluginContext.registerUdfAdaptor(new AlienSourceKeyExtractorAdaptor(), ss.sparkContext) + val df = runLocalFeatureJoinForTest( + joinConfigAsString = """ + | features: { + | key: a_id + | featureList: ["f1", "f2", "f3", "f4", "f5", "f6", "f7", "f8", "fA"] + | } + """.stripMargin, + featureDefAsString = s""" + |anchors: { + | anchor1: { + | source: "anchor1-source.csv" + | key: "mId" + | features: { + | // create an alien-type feature value, and expect Feathr to consume it via plugin + | f1: $MULTILINE_QUOTE + | import com.linkedin.feathr.offline.plugins.AlienFeatureValueMvelUDFs; + | AlienFeatureValueMvelUDFs.sqrt_float(gamma) + | $MULTILINE_QUOTE + | + | // create an alien-type feature value, and pass it to a UDF that expects Feathr feature value + | f2: $MULTILINE_QUOTE + | import com.linkedin.feathr.offline.plugins.AlienFeatureValueMvelUDFs; + | import com.linkedin.feathr.offline.plugins.FeathrFeatureValueMvelUDFs; + | FeathrFeatureValueMvelUDFs.inverse_ffv(AlienFeatureValueMvelUDFs.sqrt_float(gamma)) + | $MULTILINE_QUOTE + | + | // create a Feathr feature value, and pass it to a UDF that expects the alien feature value + | f3: $MULTILINE_QUOTE + | import com.linkedin.feathr.offline.plugins.AlienFeatureValueMvelUDFs; + | import com.linkedin.feathr.offline.plugins.FeathrFeatureValueMvelUDFs; + | AlienFeatureValueMvelUDFs.sqrt_afv(FeathrFeatureValueMvelUDFs.inverse_float(gamma)) + | $MULTILINE_QUOTE + | + | f4: { + | type: CATEGORICAL + | def: $MULTILINE_QUOTE + | import com.linkedin.feathr.offline.plugins.AlienFeatureValueMvelUDFs; + | AlienFeatureValueMvelUDFs.uppercase_string(alpha); + | $MULTILINE_QUOTE + | } + | } + | } + | anchor2: { + | source: "anchor1-source.csv" + | keyExtractor: "com.linkedin.feathr.offline.anchored.keyExtractor.AlienSampleKeyExtractor" + | features: { + | fA: { + | def: cast_float(beta) + | type: NUMERIC + | default: 0 + | } + | } + | } + |} + | + |derivations: { + | // use an UDF that expects/returns alien-valued feature value + | f5: { + | type: NUMERIC + | definition: $MULTILINE_QUOTE + | import com.linkedin.feathr.offline.plugins.AlienFeatureValueMvelUDFs; + | AlienFeatureValueMvelUDFs.sqrt_float(f3) + | $MULTILINE_QUOTE + | } + | f6: { + | type: NUMERIC + | definition: $MULTILINE_QUOTE + | import com.linkedin.feathr.offline.plugins.AlienFeatureValueMvelUDFs; + | AlienFeatureValueMvelUDFs.sqrt_float(f2) + | $MULTILINE_QUOTE + | } + | f7: { + | type: CATEGORICAL + | definition: $MULTILINE_QUOTE + | import com.linkedin.feathr.offline.plugins.AlienFeatureValueMvelUDFs; + | AlienFeatureValueMvelUDFs.lowercase_string_afv(f4); + | $MULTILINE_QUOTE + | } + | f8: { + | key: ["mId"] + | inputs: [{ key: "mId", feature: "f6" }] + | class: "com.linkedin.feathr.offline.derived.SampleAlienFeatureDerivationFunction" + | type: NUMERIC + | } + |} + """.stripMargin, + observationDataPath = "anchorAndDerivations/testMVELLoopExpFeature-observations.csv", + mvelContext = Some(mvelContext)) + + val f8Type = df.fdsMetadata.header.get.featureInfoMap.filter(_._1.getFeatureName == "f8").head._2.featureType.getFeatureType + assertEquals(f8Type, FeatureTypes.NUMERIC) + + val selectedColumns = Seq("a_id", "fA") + val filteredDf = df.data.select(selectedColumns.head, selectedColumns.tail: _*) + + val expectedDf = ss.createDataFrame( + ss.sparkContext.parallelize( + Seq( + Row( + "1", + 10.0f), + Row( + "2", + 10.0f), + Row( + "3", + 10.0f))), + StructType( + List( + StructField("a_id", StringType, true), + StructField("fA", FloatType, true)))) + def cmpFunc(row: Row): String = row.get(0).toString + FeathrTestUtils.assertDataFrameApproximatelyEquals(filteredDf, expectedDf, cmpFunc) + } +} diff --git a/src/test/scala/com/linkedin/feathr/offline/TestFeathrUtils.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/TestFeathrUtils.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/TestFeathrUtils.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/TestFeathrUtils.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/TestIOUtils.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/TestIOUtils.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/TestIOUtils.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/TestIOUtils.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/TestUtils.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/TestUtils.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/TestUtils.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/TestUtils.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/ValidationCodeGenerator.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/ValidationCodeGenerator.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/ValidationCodeGenerator.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/ValidationCodeGenerator.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/anchored/TestWindowTimeUnit.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/anchored/TestWindowTimeUnit.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/anchored/TestWindowTimeUnit.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/anchored/TestWindowTimeUnit.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/anchored/keyExtractor/AlienSampleKeyExtractor.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/anchored/keyExtractor/AlienSampleKeyExtractor.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/anchored/keyExtractor/AlienSampleKeyExtractor.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/anchored/keyExtractor/AlienSampleKeyExtractor.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/anchored/keyExtractor/AlienSourceKeyExtractor.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/anchored/keyExtractor/AlienSourceKeyExtractor.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/anchored/keyExtractor/AlienSourceKeyExtractor.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/anchored/keyExtractor/AlienSourceKeyExtractor.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/anchored/keyExtractor/AlienSourceKeyExtractorAdaptor.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/anchored/keyExtractor/AlienSourceKeyExtractorAdaptor.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/anchored/keyExtractor/AlienSourceKeyExtractorAdaptor.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/anchored/keyExtractor/AlienSourceKeyExtractorAdaptor.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/anchored/keyExtractor/SimpleSampleKeyExtractor.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/anchored/keyExtractor/SimpleSampleKeyExtractor.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/anchored/keyExtractor/SimpleSampleKeyExtractor.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/anchored/keyExtractor/SimpleSampleKeyExtractor.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/anchored/keyExtractor/SimpleSampleKeyExtractor2.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/anchored/keyExtractor/SimpleSampleKeyExtractor2.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/anchored/keyExtractor/SimpleSampleKeyExtractor2.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/anchored/keyExtractor/SimpleSampleKeyExtractor2.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/anchored/keyExtractor/SimpleSampleKeyExtractorWithOtherKey.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/anchored/keyExtractor/SimpleSampleKeyExtractorWithOtherKey.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/anchored/keyExtractor/SimpleSampleKeyExtractorWithOtherKey.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/anchored/keyExtractor/SimpleSampleKeyExtractorWithOtherKey.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/client/TestDataFrameColName.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/client/TestDataFrameColName.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/client/TestDataFrameColName.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/client/TestDataFrameColName.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/client/TestFeathrClientBuilder.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/client/TestFeathrClientBuilder.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/client/TestFeathrClientBuilder.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/client/TestFeathrClientBuilder.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/config/TestDataSourceLoader.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/config/TestDataSourceLoader.scala similarity index 75% rename from src/test/scala/com/linkedin/feathr/offline/config/TestDataSourceLoader.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/config/TestDataSourceLoader.scala index 585e2eab6..627f2af73 100644 --- a/src/test/scala/com/linkedin/feathr/offline/config/TestDataSourceLoader.scala +++ b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/config/TestDataSourceLoader.scala @@ -5,7 +5,7 @@ import com.fasterxml.jackson.databind.{DeserializationFeature, ObjectMapper} import com.fasterxml.jackson.module.scala.DefaultScalaModule import com.jasonclawson.jackson.dataformat.hocon.HoconFactory import com.linkedin.feathr.common.FeathrJacksonScalaModule -import com.linkedin.feathr.offline.config.location.{Jdbc, LocationUtils} +import com.linkedin.feathr.offline.config.location.{Jdbc, LocationUtils, Snowflake} import com.linkedin.feathr.offline.source.{DataSource, SourceFormatType} import org.scalatest.FunSuite @@ -35,6 +35,38 @@ class TestDataSourceLoader extends FunSuite { assert(ds.sourceType == SourceFormatType.FIXED_PATH) } + test("Test Deserialize Snowflake DataSource") { + val jackson = LocationUtils.getMapper() + val configDoc = + """ + |{ + | location: { + | type: "snowflake" + | database: "DATABASE" + | schema: "SCHEMA" + | dbtable: "TABLE" + | } + | timeWindowParameters: { + | timestampColumn: "lpep_dropoff_datetime" + | timestampColumnFormat: "yyyy-MM-dd HH:mm:ss" + | } + |} + |""".stripMargin + val ds = jackson.readValue(configDoc, classOf[DataSource]) + ds.location match { + case Snowflake(database, schema, dbtable, query) => { + assert(database == "DATABASE") + assert(schema == "SCHEMA") + assert(dbtable == "TABLE") + } + case _ => assert(false) + } + assert(ds.timeWindowParams.nonEmpty) + assert(ds.timePartitionPattern.isEmpty) + assert(ds.timeWindowParams.get.timestampColumn == "lpep_dropoff_datetime") + assert(ds.timeWindowParams.get.timestampColumnFormat == "yyyy-MM-dd HH:mm:ss") + } + test("Test Deserialize DataSource") { val jackson = LocationUtils.getMapper() val configDoc = diff --git a/src/test/scala/com/linkedin/feathr/offline/config/TestFeatureGroupsGenerator.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/config/TestFeatureGroupsGenerator.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/config/TestFeatureGroupsGenerator.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/config/TestFeatureGroupsGenerator.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/config/TestFeatureJoinConfig.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/config/TestFeatureJoinConfig.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/config/TestFeatureJoinConfig.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/config/TestFeatureJoinConfig.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/config/location/TestDesLocation.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/config/location/TestDesLocation.scala similarity index 91% rename from src/test/scala/com/linkedin/feathr/offline/config/location/TestDesLocation.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/config/location/TestDesLocation.scala index 1be2adf77..7b1f6fed0 100644 --- a/src/test/scala/com/linkedin/feathr/offline/config/location/TestDesLocation.scala +++ b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/config/location/TestDesLocation.scala @@ -78,6 +78,26 @@ class TestDesLocation extends FunSuite { } } + test("Deserialize Snowflake") { + val configDoc = + """ + |{ + | type: "snowflake" + | dbtable: "TABLE" + | database: "DATABASE" + | schema: "SCHEMA" + |}""".stripMargin + val ds = jackson.readValue(configDoc, classOf[Snowflake]) + ds match { + case Snowflake(database, schema, dbtable, query) => { + assert(database == "DATABASE") + assert(schema == "SCHEMA") + assert(dbtable == "TABLE") + } + case _ => assert(false) + } + } + test("Test load Sqlite") { val path = s"${System.getProperty("user.dir")}/src/test/resources/mockdata/sqlite/test.db" val configDoc = diff --git a/src/test/scala/com/linkedin/feathr/offline/config/sources/TestFeatureGroupsUpdater.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/config/sources/TestFeatureGroupsUpdater.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/config/sources/TestFeatureGroupsUpdater.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/config/sources/TestFeatureGroupsUpdater.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/derived/AlienDerivationFunctionAdaptor.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/derived/AlienDerivationFunctionAdaptor.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/derived/AlienDerivationFunctionAdaptor.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/derived/AlienDerivationFunctionAdaptor.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/derived/AlienFeatureDerivationFunction.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/derived/AlienFeatureDerivationFunction.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/derived/AlienFeatureDerivationFunction.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/derived/AlienFeatureDerivationFunction.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/derived/SampleAdvancedDerivationFunctionExtractor.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/derived/SampleAdvancedDerivationFunctionExtractor.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/derived/SampleAdvancedDerivationFunctionExtractor.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/derived/SampleAdvancedDerivationFunctionExtractor.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/derived/SampleAlienFeatureDerivationFunction.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/derived/SampleAlienFeatureDerivationFunction.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/derived/SampleAlienFeatureDerivationFunction.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/derived/SampleAlienFeatureDerivationFunction.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/derived/TestDataFrameDerivationFunctionExtractor.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/derived/TestDataFrameDerivationFunctionExtractor.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/derived/TestDataFrameDerivationFunctionExtractor.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/derived/TestDataFrameDerivationFunctionExtractor.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/derived/TestDerivationFunctionExtractor.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/derived/TestDerivationFunctionExtractor.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/derived/TestDerivationFunctionExtractor.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/derived/TestDerivationFunctionExtractor.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/derived/TestSequentialJoinAsDerivation.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/derived/TestSequentialJoinAsDerivation.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/derived/TestSequentialJoinAsDerivation.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/derived/TestSequentialJoinAsDerivation.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/generation/TestFeatureGenFeatureGrouper.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/generation/TestFeatureGenFeatureGrouper.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/generation/TestFeatureGenFeatureGrouper.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/generation/TestFeatureGenFeatureGrouper.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/generation/TestFeatureGenKeyTagAnalyzer.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/generation/TestFeatureGenKeyTagAnalyzer.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/generation/TestFeatureGenKeyTagAnalyzer.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/generation/TestFeatureGenKeyTagAnalyzer.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/generation/TestIncrementalAggSnapshotLoader.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/generation/TestIncrementalAggSnapshotLoader.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/generation/TestIncrementalAggSnapshotLoader.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/generation/TestIncrementalAggSnapshotLoader.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/generation/TestPostGenPruner.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/generation/TestPostGenPruner.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/generation/TestPostGenPruner.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/generation/TestPostGenPruner.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/generation/TestPushToRedisOutputProcessor.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/generation/TestPushToRedisOutputProcessor.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/generation/TestPushToRedisOutputProcessor.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/generation/TestPushToRedisOutputProcessor.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/generation/TestStageEvaluator.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/generation/TestStageEvaluator.scala similarity index 99% rename from src/test/scala/com/linkedin/feathr/offline/generation/TestStageEvaluator.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/generation/TestStageEvaluator.scala index c115d4e8b..65e80bb14 100644 --- a/src/test/scala/com/linkedin/feathr/offline/generation/TestStageEvaluator.scala +++ b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/generation/TestStageEvaluator.scala @@ -1,6 +1,6 @@ package com.linkedin.feathr.offline.generation -import com.linkedin.feathr.common.exception.FeathrException +import com.linkedin.feathr.exception.FeathrException import com.linkedin.feathr.common.{ErasedEntityTaggedFeature, FeatureTypeConfig} import com.linkedin.feathr.offline.derived.{DerivedFeature, DerivedFeatureEvaluator} import com.linkedin.feathr.offline.evaluator.{BaseDataFrameMetadata, DerivedFeatureGenStage} diff --git a/src/test/scala/com/linkedin/feathr/offline/job/SeqJoinAggregationClass.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/job/SeqJoinAggregationClass.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/job/SeqJoinAggregationClass.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/job/SeqJoinAggregationClass.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/job/TestFeatureGenJob.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/job/TestFeatureGenJob.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/job/TestFeatureGenJob.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/job/TestFeatureGenJob.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/job/TestFeatureJoinJob.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/job/TestFeatureJoinJob.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/job/TestFeatureJoinJob.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/job/TestFeatureJoinJob.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/job/TestFeatureJoinJobUtils.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/job/TestFeatureJoinJobUtils.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/job/TestFeatureJoinJobUtils.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/job/TestFeatureJoinJobUtils.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/job/TestFeatureTransformation.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/job/TestFeatureTransformation.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/job/TestFeatureTransformation.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/job/TestFeatureTransformation.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/job/TestTimeBasedJoin.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/job/TestTimeBasedJoin.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/job/TestTimeBasedJoin.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/job/TestTimeBasedJoin.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/job/featureGen/TestFeatureGenConfigOverrider.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/job/featureGen/TestFeatureGenConfigOverrider.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/job/featureGen/TestFeatureGenConfigOverrider.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/job/featureGen/TestFeatureGenConfigOverrider.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/job/featureGen/TestFeatureGenJobParser.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/job/featureGen/TestFeatureGenJobParser.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/job/featureGen/TestFeatureGenJobParser.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/job/featureGen/TestFeatureGenJobParser.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/job/featureGen/TestFeatureGenSpecParser.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/job/featureGen/TestFeatureGenSpecParser.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/job/featureGen/TestFeatureGenSpecParser.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/job/featureGen/TestFeatureGenSpecParser.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/join/TestDataFrameKeyCombiner.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/join/TestDataFrameKeyCombiner.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/join/TestDataFrameKeyCombiner.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/join/TestDataFrameKeyCombiner.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/join/algorithms/TestJoinConditionBuilder.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/join/algorithms/TestJoinConditionBuilder.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/join/algorithms/TestJoinConditionBuilder.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/join/algorithms/TestJoinConditionBuilder.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/join/algorithms/TestJoinKeyColumnsAppender.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/join/algorithms/TestJoinKeyColumnsAppender.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/join/algorithms/TestJoinKeyColumnsAppender.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/join/algorithms/TestJoinKeyColumnsAppender.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/join/algorithms/TestSparkJoin.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/join/algorithms/TestSparkJoin.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/join/algorithms/TestSparkJoin.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/join/algorithms/TestSparkJoin.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/join/algorithms/TestSparkSaltedJoin.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/join/algorithms/TestSparkSaltedJoin.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/join/algorithms/TestSparkSaltedJoin.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/join/algorithms/TestSparkSaltedJoin.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/join/workflow/TestAnchoredFeatureJoinStep.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/join/workflow/TestAnchoredFeatureJoinStep.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/join/workflow/TestAnchoredFeatureJoinStep.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/join/workflow/TestAnchoredFeatureJoinStep.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/join/workflow/TestDerivedFeatureJoinStep.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/join/workflow/TestDerivedFeatureJoinStep.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/join/workflow/TestDerivedFeatureJoinStep.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/join/workflow/TestDerivedFeatureJoinStep.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/logical/TestMultiStageJoinPlan.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/logical/TestMultiStageJoinPlan.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/logical/TestMultiStageJoinPlan.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/logical/TestMultiStageJoinPlan.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/mvel/FeathrMvelFixture.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/mvel/FeathrMvelFixture.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/mvel/FeathrMvelFixture.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/mvel/FeathrMvelFixture.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/mvel/TestFrameMVEL.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/mvel/TestFrameMVEL.scala similarity index 97% rename from src/test/scala/com/linkedin/feathr/offline/mvel/TestFrameMVEL.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/mvel/TestFrameMVEL.scala index d22db66de..6237a284c 100644 --- a/src/test/scala/com/linkedin/feathr/offline/mvel/TestFrameMVEL.scala +++ b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/mvel/TestFrameMVEL.scala @@ -18,8 +18,10 @@ class TestFeathrMVEL extends TestFeathr { * When test runs successfully, an MVEL PropertyAccessException containing an NPE * should be caught from applying SimpleConfigurableAnchorExtractor, because we deliberately * used in the feature definition a method that doesn't exist. + * TODO: org.apache.avro.AvroRuntimeException: Not a valid schema field: foo is thrown and this is not + * gracefully handled. Modify test to reflect this behavior. */ - @Test + @Test(enabled = false) def testWrongMVELExpressionFeature(): Unit = { val feathrClient = FeathrClient.builder(ss).addFeatureDef(Some(FeathrMvelFixture.wrongMVELExpressionFeatureConf)).build() diff --git a/src/test/scala/com/linkedin/feathr/offline/source/accessor/TestDataSourceAccessor.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/source/accessor/TestDataSourceAccessor.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/source/accessor/TestDataSourceAccessor.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/source/accessor/TestDataSourceAccessor.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/source/accessor/TestPathPartitionedTimeSeriesSourceAccessor.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/source/accessor/TestPathPartitionedTimeSeriesSourceAccessor.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/source/accessor/TestPathPartitionedTimeSeriesSourceAccessor.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/source/accessor/TestPathPartitionedTimeSeriesSourceAccessor.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/source/dataloader/TestAvroJsonDataLoader.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/source/dataloader/TestAvroJsonDataLoader.scala similarity index 89% rename from src/test/scala/com/linkedin/feathr/offline/source/dataloader/TestAvroJsonDataLoader.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/source/dataloader/TestAvroJsonDataLoader.scala index 2bdd35756..1f65b5a1e 100644 --- a/src/test/scala/com/linkedin/feathr/offline/source/dataloader/TestAvroJsonDataLoader.scala +++ b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/source/dataloader/TestAvroJsonDataLoader.scala @@ -1,5 +1,6 @@ package com.linkedin.feathr.offline.source.dataloader +import com.linkedin.avroutil1.compatibility.AvroCompatibilityHelper import com.linkedin.feathr.offline.TestFeathr import org.apache.avro.Schema import org.apache.spark.sql.Row @@ -28,7 +29,7 @@ class TestAvroJsonDataLoader extends TestFeathr { val schema = dataLoader.loadSchema() val expectedFields = List( - new Schema.Field("mId", Schema.create(Schema.Type.LONG), null, null) + AvroCompatibilityHelper.createSchemaField("mId", Schema.create(Schema.Type.LONG), null, null) ).asJava val expectedSchema = Schema.createRecord("FeathrTest", null, null, false) expectedSchema.setFields(expectedFields) diff --git a/src/test/scala/com/linkedin/feathr/offline/source/dataloader/TestBatchDataLoader.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/source/dataloader/TestBatchDataLoader.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/source/dataloader/TestBatchDataLoader.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/source/dataloader/TestBatchDataLoader.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/source/dataloader/TestCaseInsensitiveGenericRecordWrapper.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/source/dataloader/TestCaseInsensitiveGenericRecordWrapper.scala similarity index 87% rename from src/test/scala/com/linkedin/feathr/offline/source/dataloader/TestCaseInsensitiveGenericRecordWrapper.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/source/dataloader/TestCaseInsensitiveGenericRecordWrapper.scala index 47e7f65aa..7234869cc 100644 --- a/src/test/scala/com/linkedin/feathr/offline/source/dataloader/TestCaseInsensitiveGenericRecordWrapper.scala +++ b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/source/dataloader/TestCaseInsensitiveGenericRecordWrapper.scala @@ -1,5 +1,6 @@ package com.linkedin.feathr.offline.source.dataloader +import com.linkedin.avroutil1.compatibility.AvroCompatibilityHelper import org.apache.avro.generic.{GenericData, GenericRecord} import org.apache.avro.{AvroRuntimeException, Schema} import org.scalatest.testng.TestNGSuite @@ -73,11 +74,11 @@ class TestCaseInsensitiveGenericRecordWrapper extends TestNGSuite{ * @return */ def createRecord(): GenericData.Record = { - val childSchema = Schema.createRecord(List(new Schema.Field("f", Schema.create(Schema.Type.INT), null, null)).asJava) + val childSchema = Schema.createRecord(List(AvroCompatibilityHelper.createSchemaField("f", Schema.create(Schema.Type.INT), null, null)).asJava) val childRecord = new GenericData.Record(childSchema) childRecord.put("f", 2) val schema = - Schema.createRecord(List(new Schema.Field("a", Schema.create(Schema.Type.INT), null, null), new Schema.Field("child", childSchema, null, null)).asJava) + Schema.createRecord(List(AvroCompatibilityHelper.createSchemaField("a", Schema.create(Schema.Type.INT), null, null), AvroCompatibilityHelper.createSchemaField("child", childSchema, null, null)).asJava) val record = new GenericData.Record(schema) record.put("a", 1) record.put("child", childRecord) diff --git a/src/test/scala/com/linkedin/feathr/offline/source/dataloader/TestCsvDataLoader.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/source/dataloader/TestCsvDataLoader.scala similarity index 82% rename from src/test/scala/com/linkedin/feathr/offline/source/dataloader/TestCsvDataLoader.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/source/dataloader/TestCsvDataLoader.scala index ef838f0cb..caf334d4e 100644 --- a/src/test/scala/com/linkedin/feathr/offline/source/dataloader/TestCsvDataLoader.scala +++ b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/source/dataloader/TestCsvDataLoader.scala @@ -1,5 +1,6 @@ package com.linkedin.feathr.offline.source.dataloader +import com.linkedin.avroutil1.compatibility.AvroCompatibilityHelper import com.linkedin.feathr.offline.TestFeathr import org.apache.avro.Schema import org.apache.spark.sql.Row @@ -36,11 +37,11 @@ class TestCsvDataLoader extends TestFeathr { val fieldSchema = Schema.createUnion(List(Schema.create(Schema.Type.STRING), Schema.create(Schema.Type.NULL)).asJava) val expectedFields = List( - new Schema.Field("alpha", fieldSchema, null, null), - new Schema.Field("beta", fieldSchema, null, null), - new Schema.Field("gamma", fieldSchema, null, null), - new Schema.Field("mId", fieldSchema, null, null), - new Schema.Field("omega", fieldSchema, null, null) + AvroCompatibilityHelper.createSchemaField("alpha", fieldSchema, null, null), + AvroCompatibilityHelper.createSchemaField("beta", fieldSchema, null, null), + AvroCompatibilityHelper.createSchemaField("gamma", fieldSchema, null, null), + AvroCompatibilityHelper.createSchemaField("mId", fieldSchema, null, null), + AvroCompatibilityHelper.createSchemaField("omega", fieldSchema, null, null) ).asJava val expectedSchema = Schema.createRecord(expectedFields) assertEquals(schema.getFields, expectedSchema.getFields) diff --git a/src/test/scala/com/linkedin/feathr/offline/source/dataloader/TestDataLoaderFactory.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/source/dataloader/TestDataLoaderFactory.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/source/dataloader/TestDataLoaderFactory.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/source/dataloader/TestDataLoaderFactory.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/source/dataloader/TestJsonWithSchemaDataLoader.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/source/dataloader/TestJsonWithSchemaDataLoader.scala similarity index 88% rename from src/test/scala/com/linkedin/feathr/offline/source/dataloader/TestJsonWithSchemaDataLoader.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/source/dataloader/TestJsonWithSchemaDataLoader.scala index df0ee2525..312b13994 100644 --- a/src/test/scala/com/linkedin/feathr/offline/source/dataloader/TestJsonWithSchemaDataLoader.scala +++ b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/source/dataloader/TestJsonWithSchemaDataLoader.scala @@ -1,5 +1,6 @@ package com.linkedin.feathr.offline.source.dataloader +import com.linkedin.avroutil1.compatibility.AvroCompatibilityHelper import com.linkedin.feathr.offline.TestFeathr import com.linkedin.feathr.offline.util.LocalFeatureJoinUtils import org.apache.avro.Schema @@ -29,7 +30,7 @@ class TestJsonWithSchemaDataLoader extends TestFeathr { val schema = dataLoader.loadSchema() val expectedFields = List( - new Schema.Field("mId", Schema.create(Schema.Type.LONG), null, null) + AvroCompatibilityHelper.createSchemaField("mId", Schema.create(Schema.Type.LONG), null, null) ).asJava val expectedSchema = Schema.createRecord("FeathrTest", null, null, false) expectedSchema.setFields(expectedFields) diff --git a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/source/dataloader/TestSnowflakeDataLoader.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/source/dataloader/TestSnowflakeDataLoader.scala new file mode 100644 index 000000000..8e09cb86a --- /dev/null +++ b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/source/dataloader/TestSnowflakeDataLoader.scala @@ -0,0 +1,39 @@ +package com.linkedin.feathr.offline.source.dataloader + +import com.linkedin.feathr.offline.TestFeathr +import org.testng.annotations.{BeforeClass, Test} +import com.linkedin.feathr.offline.source.dataloader.jdbc.SnowflakeDataLoader +import org.testng.Assert.assertEquals + +/** + * unit tests for [[SnowflakeDataLoader]] + */ +class TestSnowflakeDataLoader extends TestFeathr { + + @BeforeClass + def ssVarSetUp(): Unit = { + ss.conf.set("sfURL", "snowflake_account") + ss.conf.set("sfUser", "snowflake_usr") + ss.conf.set("sfRole", "snowflake_role") + ss.conf.set("sfWarehouse", "snowflake_warehouse") + ss.conf.set("sfPassword", "snowflake_password") + } + + @Test(description = "Test Extract SF Options") + def testExtractSfOptions() : Unit = { + val snowflakeUrl = "snowflake://snowflake_account/?sfDatabase=DATABASE&sfSchema=SCHEMA&dbtable=TABLE" + val dataloader = new SnowflakeDataLoader(ss) + val actualOptions = dataloader.extractSFOptions(ss, snowflakeUrl) + val expectedOptions = Map[String, String]( + "sfURL" -> "snowflake_account", + "sfUser" -> "snowflake_usr", + "sfRole" -> "snowflake_role", + "sfWarehouse" -> "snowflake_warehouse", + "sfPassword" -> "snowflake_password", + "sfSchema" -> "SCHEMA", + "sfDatabase" -> "DATABASE", + "dbtable" -> "TABLE" + ) + assertEquals(actualOptions, expectedOptions) + } +} \ No newline at end of file diff --git a/src/test/scala/com/linkedin/feathr/offline/source/dataloader/hdfs/TestFileFormat.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/source/dataloader/hdfs/TestFileFormat.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/source/dataloader/hdfs/TestFileFormat.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/source/dataloader/hdfs/TestFileFormat.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/source/pathutil/TestPathChecker.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/source/pathutil/TestPathChecker.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/source/pathutil/TestPathChecker.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/source/pathutil/TestPathChecker.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/source/pathutil/TestTimeBasedHdfsPathAnalyzer.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/source/pathutil/TestTimeBasedHdfsPathAnalyzer.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/source/pathutil/TestTimeBasedHdfsPathAnalyzer.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/source/pathutil/TestTimeBasedHdfsPathAnalyzer.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/source/pathutil/TestTimeBasedHdfsPathGenerator.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/source/pathutil/TestTimeBasedHdfsPathGenerator.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/source/pathutil/TestTimeBasedHdfsPathGenerator.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/source/pathutil/TestTimeBasedHdfsPathGenerator.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/swa/TestSlidingWindowFeatureUtils.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/swa/TestSlidingWindowFeatureUtils.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/swa/TestSlidingWindowFeatureUtils.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/swa/TestSlidingWindowFeatureUtils.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/transformation/TestAnchorToDataSourceMapper.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/transformation/TestAnchorToDataSourceMapper.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/transformation/TestAnchorToDataSourceMapper.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/transformation/TestAnchorToDataSourceMapper.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/transformation/TestDataFrameExt.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/transformation/TestDataFrameExt.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/transformation/TestDataFrameExt.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/transformation/TestDataFrameExt.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/transformation/TestDefaultValueToColumnConverter.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/transformation/TestDefaultValueToColumnConverter.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/transformation/TestDefaultValueToColumnConverter.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/transformation/TestDefaultValueToColumnConverter.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/transformation/TestFDSConversionUtils.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/transformation/TestFDSConversionUtils.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/transformation/TestFDSConversionUtils.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/transformation/TestFDSConversionUtils.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/util/TestCoercionUtilsScala.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/util/TestCoercionUtilsScala.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/util/TestCoercionUtilsScala.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/util/TestCoercionUtilsScala.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/util/TestDataFrameSplitterMerger.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/util/TestDataFrameSplitterMerger.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/util/TestDataFrameSplitterMerger.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/util/TestDataFrameSplitterMerger.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/util/TestDataSource.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/util/TestDataSource.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/util/TestDataSource.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/util/TestDataSource.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/util/TestFDSConversionUtil.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/util/TestFDSConversionUtil.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/util/TestFDSConversionUtil.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/util/TestFDSConversionUtil.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/util/TestFeatureGenUtils.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/util/TestFeatureGenUtils.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/util/TestFeatureGenUtils.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/util/TestFeatureGenUtils.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/util/TestFeatureValueTypeValidator.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/util/TestFeatureValueTypeValidator.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/util/TestFeatureValueTypeValidator.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/util/TestFeatureValueTypeValidator.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/util/TestPartitionLimiter.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/util/TestPartitionLimiter.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/util/TestPartitionLimiter.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/util/TestPartitionLimiter.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/util/TestSourceUtils.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/util/TestSourceUtils.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/util/TestSourceUtils.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/util/TestSourceUtils.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/util/datetime/TestDateTimeInterval.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/util/datetime/TestDateTimeInterval.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/util/datetime/TestDateTimeInterval.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/util/datetime/TestDateTimeInterval.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/util/datetime/TestDateTimePeriod.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/util/datetime/TestDateTimePeriod.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/util/datetime/TestDateTimePeriod.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/util/datetime/TestDateTimePeriod.scala diff --git a/src/test/scala/com/linkedin/feathr/offline/util/datetime/TestOfflineDateTimeUtils.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/util/datetime/TestOfflineDateTimeUtils.scala similarity index 100% rename from src/test/scala/com/linkedin/feathr/offline/util/datetime/TestOfflineDateTimeUtils.scala rename to feathr-impl/src/test/scala/com/linkedin/feathr/offline/util/datetime/TestOfflineDateTimeUtils.scala diff --git a/feathr_project/MANIFEST.in b/feathr_project/MANIFEST.in index 2a296fe4f..18e8f9020 100644 --- a/feathr_project/MANIFEST.in +++ b/feathr_project/MANIFEST.in @@ -1 +1,2 @@ -recursive-include feathrcli/data * \ No newline at end of file +recursive-include feathrcli/data * +include feathr/spark_provider/noop-1.0.jar \ No newline at end of file diff --git a/feathr_project/docs/index.rst b/feathr_project/docs/index.rst index 672217f79..49f6fbff8 100644 --- a/feathr_project/docs/index.rst +++ b/feathr_project/docs/index.rst @@ -10,7 +10,7 @@ If you are an end user, read `Feathr User APIs`. If you have any suggestions for our API documentation, please help us improve it by creating_ a Github issue for us. -.. _creating: https://github.com/linkedin/feathr/issues/new +.. _creating: https://github.com/feathr-ai/feathr/issues/new Feathr APIs for End Users ================================== diff --git a/feathr_project/docs/make.bat b/feathr_project/docs/make.bat index 27f573b87..7893348a1 100644 --- a/feathr_project/docs/make.bat +++ b/feathr_project/docs/make.bat @@ -1,35 +1,35 @@ -@ECHO OFF - -pushd %~dp0 - -REM Command file for Sphinx documentation - -if "%SPHINXBUILD%" == "" ( - set SPHINXBUILD=sphinx-build -) -set SOURCEDIR=. -set BUILDDIR=_build - -if "%1" == "" goto help - -%SPHINXBUILD% >NUL 2>NUL -if errorlevel 9009 ( - echo. - echo.The 'sphinx-build' command was not found. Make sure you have Sphinx - echo.installed, then set the SPHINXBUILD environment variable to point - echo.to the full path of the 'sphinx-build' executable. Alternatively you - echo.may add the Sphinx directory to PATH. - echo. - echo.If you don't have Sphinx installed, grab it from - echo.http://sphinx-doc.org/ - exit /b 1 -) - -%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% -goto end - -:help -%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% - -:end -popd +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% + +:end +popd diff --git a/feathr_project/feathr/__init__.py b/feathr_project/feathr/__init__.py index 74809fd81..5c279b7d5 100644 --- a/feathr_project/feathr/__init__.py +++ b/feathr_project/feathr/__init__.py @@ -56,6 +56,7 @@ 'Source', 'InputContext', 'HdfsSource', + 'SnowflakeSource', 'KafkaConfig', 'KafKaSource', 'ValueType', diff --git a/feathr_project/feathr/client.py b/feathr_project/feathr/client.py index 216b5f97c..80119563e 100644 --- a/feathr_project/feathr/client.py +++ b/feathr_project/feathr/client.py @@ -10,6 +10,7 @@ from jinja2 import Template from pyhocon import ConfigFactory import redis +from loguru import logger from feathr.constants import * from feathr.definition._materialization_utils import _to_materialization_config @@ -20,9 +21,8 @@ from feathr.definition.monitoring_settings import MonitoringSettings from feathr.definition.query_feature_list import FeatureQuery from feathr.definition.settings import ObservationSettings -from feathr.definition.sink import Sink +from feathr.definition.sink import Sink, HdfsSink from feathr.protobuf.featureValue_pb2 import FeatureValue -from feathr.registry.feature_registry import default_registry_client from feathr.spark_provider._databricks_submission import _FeathrDatabricksJobLauncher from feathr.spark_provider._localspark_submission import _FeathrLocalSparkJobLauncher from feathr.spark_provider._synapse_submission import _FeathrSynapseJobLauncher @@ -32,8 +32,15 @@ from feathr.utils._file_utils import write_to_file from feathr.utils.feature_printer import FeaturePrinter from feathr.utils.spark_job_params import FeatureGenerationJobParams, FeatureJoinJobParams - - +from feathr.definition.source import InputContext +from azure.identity import DefaultAzureCredential +from jinja2 import Template +from loguru import logger +from feathr.definition.config_helper import FeathrConfigHelper +from pyhocon import ConfigFactory +from feathr.registry._feathr_registry_client import _FeatureRegistry +from feathr.registry._feature_registry_purview import _PurviewRegistry +from feathr.version import get_version class FeathrClient(object): """Feathr client. @@ -165,8 +172,25 @@ def __init__(self, config_path:str = "./feathr_config.yaml", local_workspace_dir self.secret_names = [] + # initialize config helper + self.config_helper = FeathrConfigHelper() + # initialize registry - self.registry = default_registry_client(self.project_name, config_path=config_path, credential=self.credential) + self.registry = None + registry_endpoint = self.envutils.get_environment_variable_with_default("feature_registry", "api_endpoint") + azure_purview_name = self.envutils.get_environment_variable_with_default('feature_registry', 'purview', 'purview_name') + if registry_endpoint: + self.registry = _FeatureRegistry(self.project_name, endpoint=registry_endpoint, project_tags=project_registry_tag, credential=credential) + elif azure_purview_name: + registry_delimiter = self.envutils.get_environment_variable_with_default('feature_registry', 'purview', 'delimiter') + # initialize the registry no matter whether we set purview name or not, given some of the methods are used there. + self.registry = _PurviewRegistry(self.project_name, azure_purview_name, registry_delimiter, project_registry_tag, config_path = config_path, credential=credential) + logger.warning("FEATURE_REGISTRY__PURVIEW__PURVIEW_NAME will be deprecated soon. Please use FEATURE_REGISTRY__API_ENDPOINT instead.") + else: + # no registry configured + logger.info("Feathr registry is not configured. Consider setting the Feathr registry component for richer feature store experience.") + + logger.info(f"Feathr client {get_version()} initialized successfully.") def register_features(self, from_context: bool = True): """Registers features based on the current workspace @@ -179,7 +203,7 @@ def register_features(self, from_context: bool = True): if from_context: # make sure those items are in `self` if 'anchor_list' in dir(self) and 'derived_feature_list' in dir(self): - self.registry.save_to_feature_config_from_context(self.anchor_list, self.derived_feature_list, self.local_workspace_dir) + self.config_helper.save_to_feature_config_from_context(self.anchor_list, self.derived_feature_list, self.local_workspace_dir) self.registry.register_features(self.local_workspace_dir, from_context=from_context, anchor_list=self.anchor_list, derived_feature_list=self.derived_feature_list) else: raise RuntimeError("Please call FeathrClient.build_features() first in order to register features") @@ -206,9 +230,8 @@ def build_features(self, anchor_list: List[FeatureAnchor] = [], derived_feature_ else: source_names[anchor.source.name] = anchor.source - preprocessingPyudfManager = _PreprocessingPyudfManager() _PreprocessingPyudfManager.build_anchor_preprocessing_metadata(anchor_list, self.local_workspace_dir) - self.registry.save_to_feature_config_from_context(anchor_list, derived_feature_list, self.local_workspace_dir) + self.config_helper.save_to_feature_config_from_context(anchor_list, derived_feature_list, self.local_workspace_dir) self.anchor_list = anchor_list self.derived_feature_list = derived_feature_list @@ -224,11 +247,37 @@ def build_features(self, anchor_list: List[FeatureAnchor] = [], derived_feature_ if verbose and self.anchor_list: FeaturePrinter.pretty_print_anchors(self.anchor_list) + def get_snowflake_path(self, database: str, schema: str, dbtable: str = None, query: str = None) -> str: + """ + Returns snowflake path given dataset location information. + Either dbtable or query must be specified but not both. + """ + if dbtable is not None and query is not None: + raise RuntimeError("Both dbtable and query are specified. Can only specify one..") + if dbtable is None and query is None: + raise RuntimeError("One of dbtable or query must be specified..") + if dbtable: + return f"snowflake://snowflake_account/?sfDatabase={database}&sfSchema={schema}&dbtable={dbtable}" + else: + return f"snowflake://snowflake_account/?sfDatabase={database}&sfSchema={schema}&query={query}" + def list_registered_features(self, project_name: str = None) -> List[str]: """List all the already registered features under the given project. `project_name` must not be None or empty string because it violates the RBAC policy """ return self.registry.list_registered_features(project_name) + + def list_dependent_entities(self, qualified_name: str): + """ + Lists all dependent/downstream entities for a given entity + """ + return self.registry.list_dependent_entities(qualified_name) + + def delete_entity(self, qualified_name: str): + """ + Deletes a single entity if it has no downstream/dependent entities + """ + return self.registry.delete_entity(qualified_name) def _get_registry_client(self): """ @@ -400,7 +449,6 @@ def get_offline_features(self, output_path: Union[str, Sink], execution_configurations: Union[SparkExecutionConfiguration ,Dict[str,str]] = {}, config_file_name:str = "feature_join_conf/feature_join.conf", - udf_files = None, verbose: bool = False ): """ @@ -437,7 +485,7 @@ def get_offline_features(self, # otherwise users will be confused on what are the available features # in build_features it will assign anchor_list and derived_feature_list variable, hence we are checking if those two variables exist to make sure the above condition is met if 'anchor_list' in dir(self) and 'derived_feature_list' in dir(self): - self.registry.save_to_feature_config_from_context(self.anchor_list, self.derived_feature_list, self.local_workspace_dir) + self.config_helper.save_to_feature_config_from_context(self.anchor_list, self.derived_feature_list, self.local_workspace_dir) else: raise RuntimeError("Please call FeathrClient.build_features() first in order to get offline features") @@ -468,10 +516,12 @@ def _get_offline_features_with_config(self, observation_path=feathr_feature['observationPath'], feature_config=os.path.join(self.local_workspace_dir, 'feature_conf/'), job_output_path=output_path) - job_tags = {OUTPUT_PATH_TAG:feature_join_job_params.job_output_path} + job_tags = { OUTPUT_PATH_TAG: feature_join_job_params.job_output_path } # set output format in job tags if it's set by user, so that it can be used to parse the job result in the helper function if execution_configurations is not None and OUTPUT_FORMAT in execution_configurations: - job_tags[OUTPUT_FORMAT]= execution_configurations[OUTPUT_FORMAT] + job_tags[OUTPUT_FORMAT] = execution_configurations[OUTPUT_FORMAT] + else: + job_tags[OUTPUT_FORMAT] = "avro" ''' - Job tags are for job metadata and it's not passed to the actual spark job (i.e. not visible to spark job), more like a platform related thing that Feathr want to add (currently job tags only have job output URL and job output format, ). They are carried over with the job and is visible to every Feathr client. Think this more like some customized metadata for the job which would be weird to be put in the spark job itself. - Job arguments (or sometimes called job parameters)are the arguments which are command line arguments passed into the actual spark job. This is usually highly related with the spark job. In Feathr it's like the input to the scala spark CLI. They are usually not spark specific (for example if we want to specify the location of the feature files, or want to @@ -594,7 +644,7 @@ def _valid_materialize_keys(self, features: List[str], allow_empty_key=False): self.logger.error(f"Inconsistent feature keys. Current keys are {str(keys)}") return False return True - + def materialize_features(self, settings: MaterializationSettings, execution_configurations: Union[SparkExecutionConfiguration ,Dict[str,str]] = {}, verbose: bool = False, allow_materialize_non_agg_feature: bool = False): """Materialize feature data @@ -604,9 +654,16 @@ def materialize_features(self, settings: MaterializationSettings, execution_conf allow_materialize_non_agg_feature: Materializing non-aggregated features (the features without WindowAggTransformation) doesn't output meaningful results so it's by default set to False, but if you really want to materialize non-aggregated features, set this to True. """ feature_list = settings.feature_names - if len(feature_list) > 0 and not self._valid_materialize_keys(feature_list): - raise RuntimeError(f"Invalid materialization features: {feature_list}, since they have different keys. Currently Feathr only supports materializing features of the same keys.") - + if len(feature_list) > 0: + if 'anchor_list' in dir(self): + anchors = [anchor for anchor in self.anchor_list if isinstance(anchor.source, InputContext)] + anchor_feature_names = set(feature.name for anchor in anchors for feature in anchor.features) + for feature in feature_list: + if feature in anchor_feature_names: + raise RuntimeError(f"Materializing features that are defined on INPUT_CONTEXT is not supported. {feature} is defined on INPUT_CONTEXT so you should remove it from the feature list in MaterializationSettings.") + if not self._valid_materialize_keys(feature_list): + raise RuntimeError(f"Invalid materialization features: {feature_list}, since they have different keys. Currently Feathr only supports materializing features of the same keys.") + if not allow_materialize_non_agg_feature: # Check if there are non-aggregation features in the list for fn in feature_list: @@ -620,11 +677,16 @@ def materialize_features(self, settings: MaterializationSettings, execution_conf if feature.name == fn and not isinstance(feature.transform, WindowAggTransformation): raise RuntimeError(f"Feature {fn} is not an aggregation feature. Currently Feathr only supports materializing aggregation features. If you want to materialize {fn}, please set allow_materialize_non_agg_feature to True.") - # Collect secrets from sinks + # Collect secrets from sinks. Get output_path as well if the sink is offline sink (HdfsSink) for later use. secrets = [] + output_path = None for sink in settings.sinks: if hasattr(sink, "get_required_properties"): secrets.extend(sink.get_required_properties()) + if isinstance(sink, HdfsSink): + # Note, for now we only cache one output path from one of HdfsSinks (if one passed multiple sinks). + output_path = sink.output_path + results = [] # produce materialization config for end in settings.get_backfill_cutoff_time(): @@ -638,13 +700,19 @@ def materialize_features(self, settings: MaterializationSettings, execution_conf # otherwise users will be confused on what are the available features # in build_features it will assign anchor_list and derived_feature_list variable, hence we are checking if those two variables exist to make sure the above condition is met if 'anchor_list' in dir(self) and 'derived_feature_list' in dir(self): - self.registry.save_to_feature_config_from_context(self.anchor_list, self.derived_feature_list, self.local_workspace_dir) + self.config_helper.save_to_feature_config_from_context(self.anchor_list, self.derived_feature_list, self.local_workspace_dir) else: raise RuntimeError("Please call FeathrClient.build_features() first in order to materialize the features") udf_files = _PreprocessingPyudfManager.prepare_pyspark_udf_files(settings.feature_names, self.local_workspace_dir) # CLI will directly call this so the experience won't be broken - result = self._materialize_features_with_config(config_file_path, execution_configurations, udf_files, secrets) + result = self._materialize_features_with_config( + feature_gen_conf_path=config_file_path, + execution_configurations=execution_configurations, + udf_files=udf_files, + secrets=secrets, + output_path=output_path, + ) if os.path.exists(config_file_path) and self.spark_runtime != 'local': os.remove(config_file_path) results.append(result) @@ -655,12 +723,23 @@ def materialize_features(self, settings: MaterializationSettings, execution_conf return results - def _materialize_features_with_config(self, feature_gen_conf_path: str = 'feature_gen_conf/feature_gen.conf',execution_configurations: Dict[str,str] = {}, udf_files=[], secrets=[]): + def _materialize_features_with_config( + self, + feature_gen_conf_path: str = 'feature_gen_conf/feature_gen.conf', + execution_configurations: Dict[str,str] = {}, + udf_files: List = [], + secrets: List = [], + output_path: str = None, + ): """Materializes feature data based on the feature generation config. The feature data will be materialized to the destination specified in the feature generation config. Args - feature_gen_conf_path: Relative path to the feature generation config you want to materialize. + feature_gen_conf_path: Relative path to the feature generation config you want to materialize. + execution_configurations: Spark job execution configurations. + udf_files: UDF files. + secrets: Secrets to access sinks. + output_path: The output path of the materialized features when using an offline sink. """ cloud_udf_paths = [self.feathr_spark_launcher.upload_or_get_cloud_path(udf_local_path) for udf_local_path in udf_files] @@ -668,6 +747,13 @@ def _materialize_features_with_config(self, feature_gen_conf_path: str = 'featur generation_config = FeatureGenerationJobParams( generation_config_path=os.path.abspath(feature_gen_conf_path), feature_config=os.path.join(self.local_workspace_dir, "feature_conf/")) + + job_tags = { OUTPUT_PATH_TAG: output_path } + # set output format in job tags if it's set by user, so that it can be used to parse the job result in the helper function + if execution_configurations is not None and OUTPUT_FORMAT in execution_configurations: + job_tags[OUTPUT_FORMAT] = execution_configurations[OUTPUT_FORMAT] + else: + job_tags[OUTPUT_FORMAT] = "avro" ''' - Job tags are for job metadata and it's not passed to the actual spark job (i.e. not visible to spark job), more like a platform related thing that Feathr want to add (currently job tags only have job output URL and job output format, ). They are carried over with the job and is visible to every Feathr client. Think this more like some customized metadata for the job which would be weird to be put in the spark job itself. - Job arguments (or sometimes called job parameters)are the arguments which are command line arguments passed into the actual spark job. This is usually highly related with the spark job. In Feathr it's like the input to the scala spark CLI. They are usually not spark specific (for example if we want to specify the location of the feature files, or want to @@ -693,6 +779,7 @@ def _materialize_features_with_config(self, feature_gen_conf_path: str = 'featur job_name=self.project_name + '_feathr_feature_materialization_job', main_jar_path=self._FEATHR_JOB_JAR_PATH, python_files=cloud_udf_paths, + job_tags=job_tags, main_class_name=GEN_CLASS_NAME, arguments=arguments, reference_files_path=[], @@ -700,7 +787,6 @@ def _materialize_features_with_config(self, feature_gen_conf_path: str = 'featur properties=self._collect_secrets(secrets) ) - def wait_job_to_finish(self, timeout_sec: int = 300): """Waits for the job to finish in a blocking way unless it times out """ @@ -810,14 +896,16 @@ def _get_snowflake_config_str(self): sf_url = self.envutils.get_environment_variable_with_default('offline_store', 'snowflake', 'url') sf_user = self.envutils.get_environment_variable_with_default('offline_store', 'snowflake', 'user') sf_role = self.envutils.get_environment_variable_with_default('offline_store', 'snowflake', 'role') + sf_warehouse = self.envutils.get_environment_variable_with_default('offline_store', 'snowflake', 'warehouse') sf_password = self.envutils.get_environment_variable('JDBC_SF_PASSWORD') # HOCON format will be parsed by the Feathr job config_str = """ JDBC_SF_URL: {JDBC_SF_URL} JDBC_SF_USER: {JDBC_SF_USER} JDBC_SF_ROLE: {JDBC_SF_ROLE} + JDBC_SF_WAREHOUSE: {JDBC_SF_WAREHOUSE} JDBC_SF_PASSWORD: {JDBC_SF_PASSWORD} - """.format(JDBC_SF_URL=sf_url, JDBC_SF_USER=sf_user, JDBC_SF_PASSWORD=sf_password, JDBC_SF_ROLE=sf_role) + """.format(JDBC_SF_URL=sf_url, JDBC_SF_USER=sf_user, JDBC_SF_PASSWORD=sf_password, JDBC_SF_ROLE=sf_role, JDBC_SF_WAREHOUSE=sf_warehouse) return self._reshape_config_str(config_str) def _get_kafka_config_str(self): diff --git a/feathr_project/feathr/constants.py b/feathr_project/feathr/constants.py index b2222e2b6..31e64ad25 100644 --- a/feathr_project/feathr/constants.py +++ b/feathr_project/feathr/constants.py @@ -28,11 +28,6 @@ TYPEDEF_ARRAY_DERIVED_FEATURE=f"array" TYPEDEF_ARRAY_ANCHOR_FEATURE=f"array" -# Decouple Feathr MAVEN Version from Feathr Python SDK Version -import os -from feathr.version import __version__ -FEATHR_MAVEN_VERSION = os.environ.get("FEATHR_MAVEN_VERSION", __version__) -FEATHR_MAVEN_ARTIFACT=f"com.linkedin.feathr:feathr_2.12:{FEATHR_MAVEN_VERSION}" JOIN_CLASS_NAME="com.linkedin.feathr.offline.job.FeatureJoinJob" GEN_CLASS_NAME="com.linkedin.feathr.offline.job.FeatureGenJob" \ No newline at end of file diff --git a/feathr_project/feathr/datasets/__init__.py b/feathr_project/feathr/datasets/__init__.py new file mode 100644 index 000000000..a1e2e5bf3 --- /dev/null +++ b/feathr_project/feathr/datasets/__init__.py @@ -0,0 +1,9 @@ +"""Utilities for downloading sample datasets""" + +from feathr.datasets.constants import ( + NYC_TAXI_SMALL_URL +) + +__all__ = [ + "NYC_TAXI_SMALL_URL", +] diff --git a/feathr_project/feathr/datasets/constants.py b/feathr_project/feathr/datasets/constants.py new file mode 100644 index 000000000..849865570 --- /dev/null +++ b/feathr_project/feathr/datasets/constants.py @@ -0,0 +1,3 @@ +NYC_TAXI_SMALL_URL = ( + "https://azurefeathrstorage.blob.core.windows.net/public/sample_data/green_tripdata_2020-04_with_index.csv" +) diff --git a/feathr_project/feathr/datasets/nyc_taxi.py b/feathr_project/feathr/datasets/nyc_taxi.py new file mode 100644 index 000000000..e00fa7150 --- /dev/null +++ b/feathr_project/feathr/datasets/nyc_taxi.py @@ -0,0 +1,87 @@ +from pathlib import Path +from tempfile import TemporaryDirectory +from threading import local +from urllib.parse import urlparse + +import pandas as pd +from pyspark.sql import DataFrame, SparkSession + +from feathr.datasets import NYC_TAXI_SMALL_URL +from feathr.datasets.utils import maybe_download +from feathr.utils.platform import is_databricks + + +def get_pandas_df( + local_cache_path: str = None, +) -> pd.DataFrame: + """Get NYC taxi fare prediction data samples as a pandas DataFrame. + + Refs: + https://www1.nyc.gov/site/tlc/about/tlc-trip-record-data.page + + Args: + local_cache_path (optional): Local cache file path to download the data set. + If local_cache_path is a directory, the source file name will be added. + + Returns: + pandas DataFrame + """ + # if local_cache_path params is not provided then create a temporary folder + if local_cache_path is None: + local_cache_path = TemporaryDirectory().name + + # If local_cache_path is a directory, add the source file name. + src_filepath = Path(urlparse(NYC_TAXI_SMALL_URL).path) + dst_path = Path(local_cache_path) + if dst_path.suffix != src_filepath.suffix: + local_cache_path = str(dst_path.joinpath(src_filepath.name)) + + maybe_download(src_url=NYC_TAXI_SMALL_URL, dst_filepath=local_cache_path) + + pdf = pd.read_csv(local_cache_path) + + return pdf + + +def get_spark_df( + spark: SparkSession, + local_cache_path: str, +) -> DataFrame: + """Get NYC taxi fare prediction data samples as a spark DataFrame. + + Refs: + https://www1.nyc.gov/site/tlc/about/tlc-trip-record-data.page + + Args: + spark: Spark session. + local_cache_path: Local cache file path to download the data set. + If local_cache_path is a directory, the source file name will be added. + + Returns: + Spark DataFrame + """ + # In spark, local_cache_path should be a persist directory or file path + if local_cache_path is None: + raise ValueError("In spark, `local_cache_path` should be a persist directory or file path.") + + # If local_cache_path is a directory, add the source file name. + src_filepath = Path(urlparse(NYC_TAXI_SMALL_URL).path) + dst_path = Path(local_cache_path) + if dst_path.suffix != src_filepath.suffix: + local_cache_path = str(dst_path.joinpath(src_filepath.name)) + + if is_databricks(): + # Databricks uses "dbfs:/" prefix for spark paths + if not local_cache_path.startswith("dbfs:"): + local_cache_path = f"dbfs:/{local_cache_path.lstrip('/')}" + # Databricks uses "/dbfs/" prefix for python paths + python_local_cache_path = local_cache_path.replace("dbfs:", "/dbfs") + # TODO add "if is_synapse()" + else: + python_local_cache_path = local_cache_path + + maybe_download(src_url=NYC_TAXI_SMALL_URL, dst_filepath=python_local_cache_path) + + df = spark.read.option("header", True).csv(local_cache_path) + + return df diff --git a/feathr_project/feathr/datasets/utils.py b/feathr_project/feathr/datasets/utils.py new file mode 100644 index 000000000..5dcfb6e87 --- /dev/null +++ b/feathr_project/feathr/datasets/utils.py @@ -0,0 +1,64 @@ +"""Dataset utilities +""" +import logging +import math +from pathlib import Path +import requests +from urllib.parse import urlparse + +from tqdm import tqdm + + +log = logging.getLogger(__name__) + + +def maybe_download(src_url: str, dst_filepath: str, expected_bytes=None) -> bool: + """Check if file exists. If not, download and return True. Else, return False. + + Refs: + https://github.com/microsoft/recommenders/blob/main/recommenders/datasets/download_utils.py + + Args: + src_url: Source file URL. + dst_filepath: Destination file path. + expected_bytes (optional): Expected bytes of the file to verify. + + Returns: + bool: Whether the file was downloaded or not + """ + dst_filepath = Path(dst_filepath) + + if dst_filepath.is_file(): + log.info(f"File {str(dst_filepath)} already exists") + return False + + # Check dir if exists. If not, create one + dst_filepath.parent.mkdir(parents=True, exist_ok=True) + + response = requests.get(src_url, stream=True) + if response.status_code == 200: + log.info(f"Downloading {src_url}") + total_size = int(response.headers.get("content-length", 0)) + block_size = 1024 + num_iterables = math.ceil(total_size / block_size) + with open(str(dst_filepath.resolve()), "wb") as file: + for data in tqdm( + response.iter_content(block_size), + total=num_iterables, + unit="KB", + unit_scale=True, + ): + file.write(data) + + # Verify the file size + if expected_bytes is not None and expected_bytes != dst_filepath.stat().st_size: + # Delete the file since the size is not the same as the expected one. + dst_filepath.unlink() + raise IOError(f"Failed to verify {str(dst_filepath)}. Maybe interrupted while downloading?") + else: + return True + + else: + response.raise_for_status() + # If not HTTPError yet still cannot download + raise Exception(f"Problem downloading {src_url}") diff --git a/feathr_project/feathr/definition/config_helper.py b/feathr_project/feathr/definition/config_helper.py new file mode 100644 index 000000000..a2e63e977 --- /dev/null +++ b/feathr_project/feathr/definition/config_helper.py @@ -0,0 +1,193 @@ +from feathr.definition.dtype import * +from feathr.registry.registry_utils import * +from feathr.utils._file_utils import write_to_file +from feathr.definition.anchor import FeatureAnchor +from feathr.constants import * +from feathr.definition.feature import Feature, FeatureType,FeatureBase +from feathr.definition.feature_derivations import DerivedFeature +from feathr.definition.repo_definitions import RepoDefinitions +from feathr.definition.source import HdfsSource, InputContext, JdbcSource, Source +from feathr.definition.transformation import (ExpressionTransformation, Transformation, + WindowAggTransformation) +from feathr.definition.typed_key import TypedKey +from feathr.registry.feature_registry import FeathrRegistry +from feathr.definition.repo_definitions import RepoDefinitions +from pathlib import Path +from jinja2 import Template +import sys +from feathr.utils._file_utils import write_to_file +import importlib +import os + +class FeathrConfigHelper(object): + def __init__(self) -> None: + pass + def _get_py_files(self, path: Path) -> List[Path]: + """Get all Python files under path recursively, excluding __init__.py""" + py_files = [] + for item in path.glob('**/*.py'): + if "__init__.py" != item.name: + py_files.append(item) + return py_files + + def _convert_to_module_path(self, path: Path, workspace_path: Path) -> str: + """Convert a Python file path to its module path so that we can import it later""" + prefix = os.path.commonprefix( + [path.resolve(), workspace_path.resolve()]) + resolved_path = str(path.resolve()) + module_path = resolved_path[len(prefix): -len(".py")] + # Convert features under nested folder to module name + # e.g. /path/to/pyfile will become path.to.pyfile + return ( + module_path + .lstrip('/') + .replace("/", ".") + ) + + def _extract_features_from_context(self, anchor_list, derived_feature_list, result_path: Path) -> RepoDefinitions: + """Collect feature definitions from the context instead of python files""" + definitions = RepoDefinitions( + sources=set(), + features=set(), + transformations=set(), + feature_anchors=set(), + derived_features=set() + ) + for derived_feature in derived_feature_list: + if isinstance(derived_feature, DerivedFeature): + definitions.derived_features.add(derived_feature) + definitions.transformations.add( + vars(derived_feature)["transform"]) + else: + raise RuntimeError(f"Please make sure you pass a list of `DerivedFeature` objects to the `derived_feature_list` argument. {str(type(derived_feature))} is detected.") + + for anchor in anchor_list: + # obj is `FeatureAnchor` + definitions.feature_anchors.add(anchor) + # add the source section of this `FeatureAnchor` object + definitions.sources.add(vars(anchor)['source']) + for feature in vars(anchor)['features']: + # get the transformation object from `Feature` or `DerivedFeature` + if isinstance(feature, Feature): + # feature is of type `Feature` + definitions.features.add(feature) + definitions.transformations.add(vars(feature)["transform"]) + else: + + raise RuntimeError(f"Please make sure you pass a list of `Feature` objects. {str(type(feature))} is detected.") + + return definitions + + def _extract_features(self, workspace_path: Path) -> RepoDefinitions: + """Collect feature definitions from the python file, convert them into feature config and save them locally""" + os.chdir(workspace_path) + # Add workspace path to system path so that we can load features defined in Python via import_module + sys.path.append(str(workspace_path)) + definitions = RepoDefinitions( + sources=set(), + features=set(), + transformations=set(), + feature_anchors=set(), + derived_features=set() + ) + for py_file in self._get_py_files(workspace_path): + module_path = self._convert_to_module_path(py_file, workspace_path) + module = importlib.import_module(module_path) + for attr_name in dir(module): + obj = getattr(module, attr_name) + if isinstance(obj, Source): + definitions.sources.add(obj) + elif isinstance(obj, Feature): + definitions.features.add(obj) + elif isinstance(obj, DerivedFeature): + definitions.derived_features.add(obj) + elif isinstance(obj, FeatureAnchor): + definitions.feature_anchors.add(obj) + elif isinstance(obj, Transformation): + definitions.transformations.add(obj) + return definitions + + def save_to_feature_config(self, workspace_path: Path, config_save_dir: Path): + """Save feature definition within the workspace into HOCON feature config files""" + repo_definitions = self._extract_features(workspace_path) + self._save_request_feature_config(repo_definitions, config_save_dir) + self._save_anchored_feature_config(repo_definitions, config_save_dir) + self._save_derived_feature_config(repo_definitions, config_save_dir) + + def save_to_feature_config_from_context(self, anchor_list, derived_feature_list, local_workspace_dir: Path): + """Save feature definition within the workspace into HOCON feature config files from current context, rather than reading from python files""" + repo_definitions = self._extract_features_from_context( + anchor_list, derived_feature_list, local_workspace_dir) + self._save_request_feature_config(repo_definitions, local_workspace_dir) + self._save_anchored_feature_config(repo_definitions, local_workspace_dir) + self._save_derived_feature_config(repo_definitions, local_workspace_dir) + + def _save_request_feature_config(self, repo_definitions: RepoDefinitions, local_workspace_dir="./"): + config_file_name = "feature_conf/auto_generated_request_features.conf" + tm = Template( + """ +// THIS FILE IS AUTO GENERATED. PLEASE DO NOT EDIT. +anchors: { + {% for anchor in feature_anchors %} + {% if anchor.source.name == "PASSTHROUGH" %} + {{anchor.to_feature_config()}} + {% endif %} + {% endfor %} +} +""" + ) + + request_feature_configs = tm.render( + feature_anchors=repo_definitions.feature_anchors) + config_file_path = os.path.join(local_workspace_dir, config_file_name) + write_to_file(content=request_feature_configs, + full_file_name=config_file_path) + + @classmethod + def _save_anchored_feature_config(self, repo_definitions: RepoDefinitions, local_workspace_dir="./"): + config_file_name = "feature_conf/auto_generated_anchored_features.conf" + tm = Template( + """ +// THIS FILE IS AUTO GENERATED. PLEASE DO NOT EDIT. +anchors: { + {% for anchor in feature_anchors %} + {% if not anchor.source.name == "PASSTHROUGH" %} + {{anchor.to_feature_config()}} + {% endif %} + {% endfor %} +} + +sources: { + {% for source in sources%} + {% if not source.name == "PASSTHROUGH" %} + {{source.to_feature_config()}} + {% endif %} + {% endfor %} +} +""" + ) + anchored_feature_configs = tm.render(feature_anchors=repo_definitions.feature_anchors, + sources=repo_definitions.sources) + config_file_path = os.path.join(local_workspace_dir, config_file_name) + write_to_file(content=anchored_feature_configs, + full_file_name=config_file_path) + + @classmethod + def _save_derived_feature_config(self, repo_definitions: RepoDefinitions, local_workspace_dir="./"): + config_file_name = "feature_conf/auto_generated_derived_features.conf" + tm = Template( + """ +anchors: {} +derivations: { + {% for derived_feature in derived_features %} + {{derived_feature.to_feature_config()}} + {% endfor %} +} +""" + ) + derived_feature_configs = tm.render( + derived_features=repo_definitions.derived_features) + config_file_path = os.path.join(local_workspace_dir, config_file_name) + write_to_file(content=derived_feature_configs, + full_file_name=config_file_path) + diff --git a/feathr_project/feathr/definition/feature.py b/feathr_project/feathr/definition/feature.py index 5ba577498..0720aced7 100644 --- a/feathr_project/feathr/definition/feature.py +++ b/feathr_project/feathr/definition/feature.py @@ -30,6 +30,11 @@ def __init__(self, registry_tags: Optional[Dict[str, str]] = None, ): FeatureBase.validate_feature_name(name) + + # Validate the feature type + if not isinstance(feature_type, FeatureType): + raise KeyError(f'Feature type must be a FeatureType class, like INT32, but got {feature_type}') + self.name = name self.feature_type = feature_type self.registry_tags=registry_tags diff --git a/feathr_project/feathr/definition/source.py b/feathr_project/feathr/definition/source.py index 3ca8925df..232dcc542 100644 --- a/feathr_project/feathr/definition/source.py +++ b/feathr_project/feathr/definition/source.py @@ -6,6 +6,7 @@ from jinja2 import Template from loguru import logger +from urllib.parse import urlparse, parse_qs import json @@ -100,7 +101,6 @@ class HdfsSource(Source): - `epoch` (seconds since epoch), for example `1647737463` - `epoch_millis` (milliseconds since epoch), for example `1647737517761` - Any date formats supported by [SimpleDateFormat](https://docs.oracle.com/javase/8/docs/api/java/text/SimpleDateFormat.html). - registry_tags: A dict of (str, str) that you can pass to feature registry for better organization. For example, you can use {"deprecated": "true"} to indicate this source is deprecated, etc. time_partition_pattern(Optional[str]): Format of the time partitioned feature data. e.g. yyyy/MM/DD. All formats supported in dateTimeFormatter. config: @@ -115,7 +115,6 @@ class HdfsSource(Source): Given the above HDFS path: /data/somePath/daily, then the expectation is that the following sub directorie(s) should exist: /data/somePath/daily/{yyyy}/{MM}/{dd} - """ def __init__(self, name: str, path: str, preprocessing: Optional[Callable] = None, event_timestamp_column: Optional[str] = None, timestamp_format: Optional[str] = "epoch", registry_tags: Optional[Dict[str, str]] = None, time_partition_pattern: Optional[str] = None) -> None: @@ -152,6 +151,91 @@ def __str__(self): def to_argument(self): return self.path +class SnowflakeSource(Source): + """ + A data source for Snowflake + + Attributes: + name (str): name of the source + database (str): Snowflake Database + schema (str): Snowflake Schema + dbtable (Optional[str]): Snowflake Table + query (Optional[str]): Query instead of snowflake table + Either one of query or dbtable must be specified but not both. + preprocessing (Optional[Callable]): A preprocessing python function that transforms the source data for further feature transformation. + event_timestamp_column (Optional[str]): The timestamp field of your record. As sliding window aggregation feature assume each record in the source data should have a timestamp column. + timestamp_format (Optional[str], optional): The format of the timestamp field. Defaults to "epoch". Possible values are: + - `epoch` (seconds since epoch), for example `1647737463` + - `epoch_millis` (milliseconds since epoch), for example `1647737517761` + - Any date formats supported by [SimpleDateFormat](https://docs.oracle.com/javase/8/docs/api/java/text/SimpleDateFormat.html). + registry_tags: A dict of (str, str) that you can pass to feature registry for better organization. For example, you can use {"deprecated": "true"} to indicate this source is deprecated, etc. + """ + def __init__(self, name: str, database: str, schema: str, dbtable: Optional[str] = None, query: Optional[str] = None, preprocessing: Optional[Callable] = None, event_timestamp_column: Optional[str] = None, timestamp_format: Optional[str] = "epoch", registry_tags: Optional[Dict[str, str]] = None) -> None: + super().__init__(name, event_timestamp_column, + timestamp_format, registry_tags=registry_tags) + self.preprocessing=preprocessing + if dbtable is not None and query is not None: + raise RuntimeError("Both dbtable and query are specified. Can only specify one..") + if dbtable is None and query is None: + raise RuntimeError("One of dbtable or query must be specified..") + if dbtable is not None: + self.dbtable = dbtable + if query is not None: + self.query = query + self.database = database + self.schema = schema + self.path = self._get_snowflake_path(dbtable, query) + + def _get_snowflake_path(self, dbtable: Optional[str] = None, query: Optional[str] = None) -> str: + """ + Returns snowflake path for registry. + """ + if dbtable: + return f"snowflake://snowflake_account/?sfDatabase={self.database}&sfSchema={self.schema}&dbtable={dbtable}" + else: + return f"snowflake://snowflake_account/?sfDatabase={self.database}&sfSchema={self.schema}&query={query}" + + def parse_snowflake_path(url: str) -> Dict[str, str]: + """ + Parses snowflake path into dictionary of components for registry. + """ + parse_result = urlparse(url) + parsed_queries = parse_qs(parse_result.query) + updated_dict = {key: parsed_queries[key][0] for key in parsed_queries} + return updated_dict + + def to_feature_config(self) -> str: + tm = Template(""" + {{source.name}}: { + type: SNOWFLAKE + location: { + type: "snowflake" + {% if source.dbtable is defined %} + dbtable: "{{source.dbtable}}" + {% endif %} + {% if source.query is defined %} + query: "{{source.query}}" + {% endif %} + database: "{{source.database}}" + schema: "{{source.schema}}" + } + {% if source.event_timestamp_column %} + timeWindowParameters: { + timestampColumn: "{{source.event_timestamp_column}}" + timestampColumnFormat: "{{source.timestamp_format}}" + } + {% endif %} + } + """) + msg = tm.render(source=self) + return msg + + def __str__(self): + return str(self.preprocessing) + '\n' + self.to_feature_config() + + def to_argument(self): + return self.path + class JdbcSource(Source): def __init__(self, name: str, url: str = "", dbtable: Optional[str] = None, query: Optional[str] = None, auth: Optional[str] = None, preprocessing: Optional[Callable] = None, event_timestamp_column: Optional[str] = None, timestamp_format: Optional[str] = "epoch", registry_tags: Optional[Dict[str, str]] = None) -> None: super().__init__(name, event_timestamp_column, timestamp_format, registry_tags) diff --git a/feathr_project/feathr/definition/typed_key.py b/feathr_project/feathr/definition/typed_key.py index 16274698d..c2732a476 100644 --- a/feathr_project/feathr/definition/typed_key.py +++ b/feathr_project/feathr/definition/typed_key.py @@ -20,6 +20,10 @@ def __init__(self, full_name: Optional[str] = None, description: Optional[str] = None, key_column_alias: Optional[str] = None) -> None: + # Validate the key_column type + if not isinstance(key_column_type, ValueType): + raise KeyError(f'key_column_type must be a ValueType, like Value.INT32, but got {key_column_type}') + self.key_column = key_column self.key_column_type = key_column_type self.full_name = full_name diff --git a/feathr_project/feathr/registry/_feathr_registry_client.py b/feathr_project/feathr/registry/_feathr_registry_client.py index 98397627a..0851d5aae 100644 --- a/feathr_project/feathr/registry/_feathr_registry_client.py +++ b/feathr_project/feathr/registry/_feathr_registry_client.py @@ -21,7 +21,7 @@ from feathr.definition.feature import Feature, FeatureBase from feathr.definition.feature_derivations import DerivedFeature from feathr.definition.repo_definitions import RepoDefinitions -from feathr.definition.source import GenericSource, HdfsSource, InputContext, JdbcSource, Source +from feathr.definition.source import GenericSource, HdfsSource, InputContext, JdbcSource, SnowflakeSource, Source from feathr.definition.transformation import ExpressionTransformation, Transformation, WindowAggTransformation from feathr.definition.typed_key import TypedKey from feathr.registry.feature_registry import FeathrRegistry @@ -89,7 +89,7 @@ def __init__(self, project_name: str, endpoint: str, project_tags: Dict[str, str exclude_interactive_browser_credential=False) if credential is None else credential self.project_id = None - def register_features(self, workspace_path: Optional[Path] = None, from_context: bool = True, anchor_list=[], derived_feature_list=[]): + def register_features(self, workspace_path: Optional[Path] = None, from_context: bool = True, anchor_list: List[FeatureAnchor]=[], derived_feature_list=[]): """Register Features for the specified workspace. Args: workspace_path (str, optional): path to a workspace. Defaults to None, not used in this implementation. @@ -136,6 +136,23 @@ def list_registered_features(self, project_name: str) -> List[str]: "id": r["guid"], "qualifiedName": r["attributes"]["qualifiedName"], } for r in resp] + + def list_dependent_entities(self, qualified_name: str): + """ + Returns list of dependent entities for provided entity + """ + resp = self._get(f"/dependent/{qualified_name}") + return [{ + "name": r["attributes"]["name"], + "id": r["guid"], + "qualifiedName": r["attributes"]["qualifiedName"], + } for r in resp] + + def delete_entity(self, qualified_name: str): + """ + Deletes entity if it has no dependent entities + """ + self._delete(f"/entity/{qualified_name}") def get_features_from_registry(self, project_name: str) -> Tuple[List[FeatureAnchor], List[DerivedFeature]]: """ @@ -187,6 +204,10 @@ def _create_derived_feature(self, s: DerivedFeature) -> UUID: def _get(self, path: str) -> dict: logging.debug("PATH: ", path) return check(requests.get(f"{self.endpoint}{path}", headers=self._get_auth_header())).json() + + def _delete(self, path: str) -> dict: + logging.debug("PATH: ", path) + return check(requests.delete(f"{self.endpoint}{path}", headers=self._get_auth_header())).json() def _post(self, path: str, body: dict) -> dict: logging.debug("PATH: ", path) @@ -196,185 +217,6 @@ def _post(self, path: str, body: dict) -> dict: def _get_auth_header(self) -> dict: return {"Authorization": f'Bearer {self.credential.get_token("https://management.azure.com/.default").token}'} - @classmethod - def _get_py_files(self, path: Path) -> List[Path]: - """Get all Python files under path recursively, excluding __init__.py""" - py_files = [] - for item in path.glob('**/*.py'): - if "__init__.py" != item.name: - py_files.append(item) - return py_files - - @classmethod - def _convert_to_module_path(self, path: Path, workspace_path: Path) -> str: - """Convert a Python file path to its module path so that we can import it later""" - prefix = os.path.commonprefix( - [path.resolve(), workspace_path.resolve()]) - resolved_path = str(path.resolve()) - module_path = resolved_path[len(prefix): -len(".py")] - # Convert features under nested folder to module name - # e.g. /path/to/pyfile will become path.to.pyfile - return ( - module_path - .lstrip('/') - .replace("/", ".") - ) - - @classmethod - def _extract_features_from_context(self, anchor_list, derived_feature_list, result_path: Path) -> RepoDefinitions: - """Collect feature definitions from the context instead of python files""" - definitions = RepoDefinitions( - sources=set(), - features=set(), - transformations=set(), - feature_anchors=set(), - derived_features=set() - ) - for derived_feature in derived_feature_list: - if isinstance(derived_feature, DerivedFeature): - definitions.derived_features.add(derived_feature) - definitions.transformations.add( - vars(derived_feature)["transform"]) - else: - raise RuntimeError( - "Object cannot be parsed. `derived_feature_list` should be a list of `DerivedFeature`.") - - for anchor in anchor_list: - # obj is `FeatureAnchor` - definitions.feature_anchors.add(anchor) - # add the source section of this `FeatureAnchor` object - definitions.sources.add(vars(anchor)['source']) - for feature in vars(anchor)['features']: - # get the transformation object from `Feature` or `DerivedFeature` - if isinstance(feature, Feature): - # feature is of type `Feature` - definitions.features.add(feature) - definitions.transformations.add(vars(feature)["transform"]) - else: - raise RuntimeError("Object cannot be parsed.") - - return definitions - - @classmethod - def _extract_features(self, workspace_path: Path) -> RepoDefinitions: - """Collect feature definitions from the python file, convert them into feature config and save them locally""" - os.chdir(workspace_path) - # Add workspace path to system path so that we can load features defined in Python via import_module - sys.path.append(str(workspace_path)) - definitions = RepoDefinitions( - sources=set(), - features=set(), - transformations=set(), - feature_anchors=set(), - derived_features=set() - ) - for py_file in self._get_py_files(workspace_path): - module_path = self._convert_to_module_path(py_file, workspace_path) - module = importlib.import_module(module_path) - for attr_name in dir(module): - obj = getattr(module, attr_name) - if isinstance(obj, Source): - definitions.sources.add(obj) - elif isinstance(obj, Feature): - definitions.features.add(obj) - elif isinstance(obj, DerivedFeature): - definitions.derived_features.add(obj) - elif isinstance(obj, FeatureAnchor): - definitions.feature_anchors.add(obj) - elif isinstance(obj, Transformation): - definitions.transformations.add(obj) - return definitions - - @classmethod - def save_to_feature_config(self, workspace_path: Path, config_save_dir: Path): - """Save feature definition within the workspace into HOCON feature config files""" - repo_definitions = self._extract_features(workspace_path) - self._save_request_feature_config(repo_definitions, config_save_dir) - self._save_anchored_feature_config(repo_definitions, config_save_dir) - self._save_derived_feature_config(repo_definitions, config_save_dir) - - @classmethod - def save_to_feature_config_from_context(self, anchor_list, derived_feature_list, local_workspace_dir: Path): - """Save feature definition within the workspace into HOCON feature config files from current context, rather than reading from python files""" - repo_definitions = self._extract_features_from_context( - anchor_list, derived_feature_list, local_workspace_dir) - self._save_request_feature_config( - repo_definitions, local_workspace_dir) - self._save_anchored_feature_config( - repo_definitions, local_workspace_dir) - self._save_derived_feature_config( - repo_definitions, local_workspace_dir) - - @classmethod - def _save_request_feature_config(self, repo_definitions: RepoDefinitions, local_workspace_dir="./"): - config_file_name = "feature_conf/auto_generated_request_features.conf" - tm = Template( - """ -// THIS FILE IS AUTO GENERATED. PLEASE DO NOT EDIT. -anchors: { - {% for anchor in feature_anchors %} - {% if anchor.source.name == "PASSTHROUGH" %} - {{anchor.to_feature_config()}} - {% endif %} - {% endfor %} -} -""" - ) - - request_feature_configs = tm.render( - feature_anchors=repo_definitions.feature_anchors) - config_file_path = os.path.join(local_workspace_dir, config_file_name) - write_to_file(content=request_feature_configs, - full_file_name=config_file_path) - - @classmethod - def _save_anchored_feature_config(self, repo_definitions: RepoDefinitions, local_workspace_dir="./"): - config_file_name = "feature_conf/auto_generated_anchored_features.conf" - tm = Template( - """ -// THIS FILE IS AUTO GENERATED. PLEASE DO NOT EDIT. -anchors: { - {% for anchor in feature_anchors %} - {% if not anchor.source.name == "PASSTHROUGH" %} - {{anchor.to_feature_config()}} - {% endif %} - {% endfor %} -} - -sources: { - {% for source in sources%} - {% if not source.name == "PASSTHROUGH" %} - {{source.to_feature_config()}} - {% endif %} - {% endfor %} -} -""" - ) - anchored_feature_configs = tm.render(feature_anchors=repo_definitions.feature_anchors, - sources=repo_definitions.sources) - config_file_path = os.path.join(local_workspace_dir, config_file_name) - write_to_file(content=anchored_feature_configs, - full_file_name=config_file_path) - - @classmethod - def _save_derived_feature_config(self, repo_definitions: RepoDefinitions, local_workspace_dir="./"): - config_file_name = "feature_conf/auto_generated_derived_features.conf" - tm = Template( - """ -anchors: {} -derivations: { - {% for derived_feature in derived_features %} - {{derived_feature.to_feature_config()}} - {% endfor %} -} -""" - ) - derived_feature_configs = tm.render( - derived_features=repo_definitions.derived_features) - config_file_path = os.path.join(local_workspace_dir, config_file_name) - write_to_file(content=derived_feature_configs, - full_file_name=config_file_path) - def check(r): if not r.ok: @@ -397,6 +239,12 @@ def source_to_def(v: Source) -> dict: "type": urlparse(v.path).scheme, "path": v.path, } + elif isinstance(v, SnowflakeSource): + ret = { + "name": v.name, + "type": "SNOWFLAKE", + "path": v.path, + } elif isinstance(v, JdbcSource): ret = { "name": v.name, @@ -446,6 +294,21 @@ def dict_to_source(v: dict) -> Source: timestamp_format=v["attributes"].get( "timestampFormat"), registry_tags=v["attributes"].get("tags", {})) + elif type == "SNOWFLAKE": + snowflake_path = v["attributes"]["path"] + snowflake_parameters = SnowflakeSource.parse_snowflake_path(snowflake_path) + source = SnowflakeSource(name=v["attributes"]["name"], + dbtable=snowflake_parameters.get("dbtable", None), + query=snowflake_parameters.get("query", None), + database=snowflake_parameters["sfDatabase"], + schema=snowflake_parameters["sfSchema"], + preprocessing=_correct_function_indentation( + v["attributes"].get("preprocessing")), + event_timestamp_column=v["attributes"].get( + "eventTimestampColumn"), + timestamp_format=v["attributes"].get( + "timestampFormat"), + registry_tags=v["attributes"].get("tags", {})) elif type == "generic": options = v["attributes"].copy() # These are not options diff --git a/feathr_project/feathr/registry/_feature_registry_purview.py b/feathr_project/feathr/registry/_feature_registry_purview.py index a6b2c17e4..d47105a37 100644 --- a/feathr_project/feathr/registry/_feature_registry_purview.py +++ b/feathr_project/feathr/registry/_feature_registry_purview.py @@ -1,12 +1,8 @@ import glob -import importlib import inspect import itertools import os import re -import sys -import ast -import types from graphlib import TopologicalSorter from pathlib import Path from tracemalloc import stop @@ -16,9 +12,7 @@ from uuid import UUID from azure.identity import DefaultAzureCredential -from jinja2 import Template from loguru import logger -from pyapacheatlas.auth import ServicePrincipalAuthentication from pyapacheatlas.auth.azcredential import AzCredentialWrapper from pyapacheatlas.core import (AtlasClassification, AtlasEntity, AtlasProcess, PurviewClient, TypeCategory) @@ -31,13 +25,12 @@ from feathr.definition.dtype import * from feathr.registry.registry_utils import * -from feathr.utils._file_utils import write_to_file from feathr.definition.anchor import FeatureAnchor from feathr.constants import * -from feathr.definition.feature import Feature, FeatureType,FeatureBase +from feathr.definition.feature import Feature, FeatureType from feathr.definition.feature_derivations import DerivedFeature from feathr.definition.repo_definitions import RepoDefinitions -from feathr.definition.source import HdfsSource, InputContext, JdbcSource, Source +from feathr.definition.source import HdfsSource, InputContext, JdbcSource, SnowflakeSource, Source from feathr.definition.transformation import (ExpressionTransformation, Transformation, WindowAggTransformation) from feathr.definition.typed_key import TypedKey @@ -320,7 +313,7 @@ def _merge_anchor(self,original_anchor:Dict, new_anchor:Dict)->List[Dict[str,any transformed_original_elements.setdefault(elem['qualifiedName'],elem) return list(transformed_original_elements.values()) - def _parse_source(self, source: Union[Source, HdfsSource, JdbcSource]) -> AtlasEntity: + def _parse_source(self, source: Union[Source, HdfsSource, JdbcSource, SnowflakeSource]) -> AtlasEntity: """ parse the input sources """ @@ -336,7 +329,7 @@ def _parse_source(self, source: Union[Source, HdfsSource, JdbcSource]) -> AtlasE attrs = {} if isinstance(source, JdbcSource): - { + attrs = { "type": INPUT_CONTEXT if input_context else urlparse(source.path).scheme, "url": INPUT_CONTEXT if input_context else source.url, "timestamp_format": source.timestamp_format, @@ -350,6 +343,20 @@ def _parse_source(self, source: Union[Source, HdfsSource, JdbcSource]) -> AtlasE attrs["dbtable"] = source.dbtable if source.query is not None: attrs["query"] = source.query + elif isinstance(source, SnowflakeSource): + attrs = { + "type": INPUT_CONTEXT if input_context else "SNOWFLAKE", + "database": source.database, + "schema": source.schema, + "timestamp_format": source.timestamp_format, + "event_timestamp_column": source.event_timestamp_column, + "tags": source.registry_tags, + "preprocessing": preprocessing_func # store the UDF as a string + } + if source.dbtable is not None: + attrs["dbtable"] = source.dbtable + if source.query is not None: + attrs["query"] = source.query else: attrs = { "type": INPUT_CONTEXT if input_context else urlparse(source.path).scheme, @@ -398,8 +405,6 @@ def _add_all_derived_features(self, derived_features: List[DerivedFeature], ts:T # if the amount of features is huge, consider only add the derived features into the function call self._add_all_derived_features(input_feature.input_features, ts) - - def _parse_derived_features(self, derived_features: List[DerivedFeature]) -> List[AtlasEntity]: """parse derived feature @@ -542,182 +547,6 @@ def _parse_features_from_context(self, workspace_path: str, anchor_list, derived self.entity_batch_queue.extend(anchor_entities) self.entity_batch_queue.extend(derived_feature_entities) - @classmethod - def _get_py_files(self, path: Path) -> List[Path]: - """Get all Python files under path recursively, excluding __init__.py""" - py_files = [] - for item in path.glob('**/*.py'): - if "__init__.py" != item.name: - py_files.append(item) - return py_files - - @classmethod - def _convert_to_module_path(self, path: Path, workspace_path: Path) -> str: - """Convert a Python file path to its module path so that we can import it later""" - prefix = os.path.commonprefix( - [path.resolve(), workspace_path.resolve()]) - resolved_path = str(path.resolve()) - module_path = resolved_path[len(prefix): -len(".py")] - # Convert features under nested folder to module name - # e.g. /path/to/pyfile will become path.to.pyfile - return ( - module_path - .lstrip('/') - .replace("/", ".") - ) - - @classmethod - def _extract_features_from_context(self, anchor_list, derived_feature_list, result_path: Path) -> RepoDefinitions: - """Collect feature definitions from the context instead of python files""" - definitions = RepoDefinitions( - sources=set(), - features=set(), - transformations=set(), - feature_anchors=set(), - derived_features=set() - ) - for derived_feature in derived_feature_list: - if isinstance(derived_feature, DerivedFeature): - definitions.derived_features.add(derived_feature) - definitions.transformations.add( - vars(derived_feature)["transform"]) - else: - raise RuntimeError( - "Object cannot be parsed. `derived_feature_list` should be a list of `DerivedFeature`.") - - for anchor in anchor_list: - # obj is `FeatureAnchor` - definitions.feature_anchors.add(anchor) - # add the source section of this `FeatureAnchor` object - definitions.sources.add(vars(anchor)['source']) - for feature in vars(anchor)['features']: - # get the transformation object from `Feature` or `DerivedFeature` - if isinstance(feature, Feature): - # feature is of type `Feature` - definitions.features.add(feature) - definitions.transformations.add(vars(feature)["transform"]) - else: - raise RuntimeError("Object cannot be parsed.") - - return definitions - - @classmethod - def _extract_features(self, workspace_path: Path) -> RepoDefinitions: - """Collect feature definitions from the python file, convert them into feature config and save them locally""" - os.chdir(workspace_path) - # Add workspace path to system path so that we can load features defined in Python via import_module - sys.path.append(str(workspace_path)) - definitions = RepoDefinitions( - sources=set(), - features=set(), - transformations=set(), - feature_anchors=set(), - derived_features=set() - ) - for py_file in self._get_py_files(workspace_path): - module_path = self._convert_to_module_path(py_file, workspace_path) - module = importlib.import_module(module_path) - for attr_name in dir(module): - obj = getattr(module, attr_name) - if isinstance(obj, Source): - definitions.sources.add(obj) - elif isinstance(obj, Feature): - definitions.features.add(obj) - elif isinstance(obj, DerivedFeature): - definitions.derived_features.add(obj) - elif isinstance(obj, FeatureAnchor): - definitions.feature_anchors.add(obj) - elif isinstance(obj, Transformation): - definitions.transformations.add(obj) - return definitions - - @classmethod - def save_to_feature_config(self, workspace_path: Path, config_save_dir: Path): - """Save feature definition within the workspace into HOCON feature config files""" - repo_definitions = self._extract_features(workspace_path) - self._save_request_feature_config(repo_definitions, config_save_dir) - self._save_anchored_feature_config(repo_definitions, config_save_dir) - self._save_derived_feature_config(repo_definitions, config_save_dir) - - @classmethod - def save_to_feature_config_from_context(self, anchor_list, derived_feature_list, local_workspace_dir: Path): - """Save feature definition within the workspace into HOCON feature config files from current context, rather than reading from python files""" - repo_definitions = self._extract_features_from_context( - anchor_list, derived_feature_list, local_workspace_dir) - self._save_request_feature_config(repo_definitions, local_workspace_dir) - self._save_anchored_feature_config(repo_definitions, local_workspace_dir) - self._save_derived_feature_config(repo_definitions, local_workspace_dir) - - @classmethod - def _save_request_feature_config(self, repo_definitions: RepoDefinitions, local_workspace_dir="./"): - config_file_name = "feature_conf/auto_generated_request_features.conf" - tm = Template( - """ -// THIS FILE IS AUTO GENERATED. PLEASE DO NOT EDIT. -anchors: { - {% for anchor in feature_anchors %} - {% if anchor.source.name == "PASSTHROUGH" %} - {{anchor.to_feature_config()}} - {% endif %} - {% endfor %} -} -""" - ) - - request_feature_configs = tm.render( - feature_anchors=repo_definitions.feature_anchors) - config_file_path = os.path.join(local_workspace_dir, config_file_name) - write_to_file(content=request_feature_configs, - full_file_name=config_file_path) - - @classmethod - def _save_anchored_feature_config(self, repo_definitions: RepoDefinitions, local_workspace_dir="./"): - config_file_name = "feature_conf/auto_generated_anchored_features.conf" - tm = Template( - """ -// THIS FILE IS AUTO GENERATED. PLEASE DO NOT EDIT. -anchors: { - {% for anchor in feature_anchors %} - {% if not anchor.source.name == "PASSTHROUGH" %} - {{anchor.to_feature_config()}} - {% endif %} - {% endfor %} -} - -sources: { - {% for source in sources%} - {% if not source.name == "PASSTHROUGH" %} - {{source.to_feature_config()}} - {% endif %} - {% endfor %} -} -""" - ) - anchored_feature_configs = tm.render(feature_anchors=repo_definitions.feature_anchors, - sources=repo_definitions.sources) - config_file_path = os.path.join(local_workspace_dir, config_file_name) - write_to_file(content=anchored_feature_configs, - full_file_name=config_file_path) - - @classmethod - def _save_derived_feature_config(self, repo_definitions: RepoDefinitions, local_workspace_dir="./"): - config_file_name = "feature_conf/auto_generated_derived_features.conf" - tm = Template( - """ -anchors: {} -derivations: { - {% for derived_feature in derived_features %} - {{derived_feature.to_feature_config()}} - {% endfor %} -} -""" - ) - derived_feature_configs = tm.render( - derived_features=repo_definitions.derived_features) - config_file_path = os.path.join(local_workspace_dir, config_file_name) - write_to_file(content=derived_feature_configs, - full_file_name=config_file_path) - def _create_project(self) -> UUID: ''' create a project entity @@ -744,8 +573,7 @@ def upload_single_entity_to_purview(self,entity:Union[AtlasEntity,AtlasProcess]) """ Try to find existing entity/process first, if found, return the existing entity's GUID """ - id = self.get_entity_id(entity.qualifiedName) - response = self.purview_client.get_entity(id)['entities'][0] + response = self.purview_client.get_entity(qualifiedName=entity.qualifiedName)['entities'][0] j = entity.to_json() if j["typeName"] == response["typeName"]: if j["typeName"] == "Process": @@ -1084,6 +912,18 @@ def list_registered_features(self, project_name: str, limit=1000, starting_offse feature_list.append({"name":entity["name"],'id':entity['id'],"qualifiedName":entity['qualifiedName']}) return feature_list + + def list_dependent_entities(self, qualified_name: str): + """ + Returns list of dependent entities for provided entity + """ + raise NotImplementedError("Delete functionality supported through API") + + def delete_entity(self, qualified_name: str): + """ + Deletes entity if it has no dependent entities + """ + raise NotImplementedError("Delete functionality supported through API") def get_feature_by_fqdn_type(self, qualifiedName, typeName): """ diff --git a/feathr_project/feathr/registry/feature_registry.py b/feathr_project/feathr/registry/feature_registry.py index 3f10fb3fb..b511b1ee3 100644 --- a/feathr_project/feathr/registry/feature_registry.py +++ b/feathr_project/feathr/registry/feature_registry.py @@ -28,6 +28,20 @@ def list_registered_features(self, project_name: str) -> List[str]: """ pass + @abstractmethod + def list_dependent_entities(self, qualified_name: str): + """ + Returns list of dependent entities for provided entity + """ + pass + + @abstractmethod + def delete_entity(self, qualified_name: str): + """ + Deletes entity if it has no dependent entities + """ + pass + @abstractmethod def get_features_from_registry(self, project_name: str) -> Tuple[List[FeatureAnchor], List[DerivedFeature]]: """[Sync Features from registry to local workspace, given a project_name, will write project's features from registry to to user's local workspace] @@ -39,28 +53,5 @@ def get_features_from_registry(self, project_name: str) -> Tuple[List[FeatureAnc bool: Returns true if the job completed successfully, otherwise False """ pass - - @classmethod - @abstractmethod - def save_to_feature_config(self, workspace_path: Path, config_save_dir: Path): - """Save feature definition within the workspace into HOCON feature config files""" - pass - @classmethod - @abstractmethod - def save_to_feature_config_from_context(self, anchor_list, derived_feature_list, local_workspace_dir: Path): - """Save feature definition within the workspace into HOCON feature config files from current context, rather than reading from python files""" - pass -def default_registry_client(project_name: str, config_path:str = "./feathr_config.yaml", project_registry_tag: Dict[str, str]=None, credential = None) -> FeathrRegistry: - from feathr.registry._feathr_registry_client import _FeatureRegistry - from feathr.registry._feature_registry_purview import _PurviewRegistry - envutils = _EnvVaraibleUtil(config_path) - registry_endpoint = envutils.get_environment_variable_with_default("feature_registry", "api_endpoint") - if registry_endpoint: - return _FeatureRegistry(project_name, endpoint=registry_endpoint, project_tags=project_registry_tag, credential=credential) - else: - registry_delimiter = envutils.get_environment_variable_with_default('feature_registry', 'purview', 'delimiter') - azure_purview_name = envutils.get_environment_variable_with_default('feature_registry', 'purview', 'purview_name') - # initialize the registry no matter whether we set purview name or not, given some of the methods are used there. - return _PurviewRegistry(project_name, azure_purview_name, registry_delimiter, project_registry_tag, config_path = config_path, credential=credential) diff --git a/feathr_project/feathr/registry/registry_utils.py b/feathr_project/feathr/registry/registry_utils.py index cc064259d..027b0305b 100644 --- a/feathr_project/feathr/registry/registry_utils.py +++ b/feathr_project/feathr/registry/registry_utils.py @@ -7,7 +7,7 @@ from feathr.definition.dtype import FeatureType, str_to_value_type, value_type_to_str from feathr.definition.feature import Feature from feathr.definition.feature_derivations import DerivedFeature -from feathr.definition.source import HdfsSource, JdbcSource, Source +from feathr.definition.source import HdfsSource, JdbcSource, Source, SnowflakeSource from pyapacheatlas.core import AtlasProcess,AtlasEntity from feathr.definition.transformation import ExpressionTransformation, Transformation, WindowAggTransformation @@ -41,6 +41,12 @@ def source_to_def(v: Source) -> dict: "type": "hdfs", "path": v.path, } + elif isinstance(v, SnowflakeSource): + ret = { + "name": v.name, + "type": "SNOWFLAKE", + "path": v.path, + } elif isinstance(v, JdbcSource): ret = { "name": v.name, diff --git a/feathr_project/feathr/spark_provider/_databricks_submission.py b/feathr_project/feathr/spark_provider/_databricks_submission.py index ac4d7f7fb..51303a922 100644 --- a/feathr_project/feathr/spark_provider/_databricks_submission.py +++ b/feathr_project/feathr/spark_provider/_databricks_submission.py @@ -1,67 +1,66 @@ -from ast import Raise +from collections import namedtuple import copy import json import os -import time -from collections import namedtuple from os.path import basename from pathlib import Path -from typing import Any, Dict, List, Optional, Union +import time +from typing import Dict, List, Optional, Union from urllib.parse import urlparse from urllib.request import urlopen -import requests from databricks_cli.dbfs.api import DbfsApi from databricks_cli.runs.api import RunsApi from databricks_cli.sdk.api_client import ApiClient -from feathr.constants import * -from feathr.spark_provider._abc import SparkJobLauncher from loguru import logger +import requests from requests.structures import CaseInsensitiveDict +from feathr.constants import * +from feathr.version import get_maven_artifact_fullname +from feathr.spark_provider._abc import SparkJobLauncher + class _FeathrDatabricksJobLauncher(SparkJobLauncher): """Class to interact with Databricks Spark cluster - This is a light-weight databricks job runner, users should use the provided template json string to get more fine controlled environment for databricks cluster. - For example, user can control whether to use a new cluster to run the job or not, specify the cluster ID, running frequency, node size, workder no., whether to send out failed notification email, etc. - This runner will only fill in necessary arguments in the JSON template. - - This class will read from the provided configs string, and do the following steps. - This default template can be overwritten by users, but users need to make sure the template is compatible with the default template. Specifically: - 1. it's a SparkJarTask (rather than other types of jobs, say NotebookTask or others). See https://docs.microsoft.com/en-us/azure/databricks/dev-tools/api/2.0/jobs#--runs-submit for more details - 2. Use the Feathr Jar to run the job (hence will add an entry in `libraries` section) - 3. Only supports `new_cluster` type for now - 4. Will override `main_class_name` and `parameters` field in the JSON template `spark_jar_task` field - 5. will override the name of this job + This is a light-weight databricks job runner, users should use the provided template json string to get more fine controlled environment for databricks cluster. + For example, user can control whether to use a new cluster to run the job or not, specify the cluster ID, running frequency, node size, workder no., whether to send out failed notification email, etc. + This runner will only fill in necessary arguments in the JSON template. + + This class will read from the provided configs string, and do the following steps. + This default template can be overwritten by users, but users need to make sure the template is compatible with the default template. Specifically: + 1. it's a SparkJarTask (rather than other types of jobs, say NotebookTask or others). See https://docs.microsoft.com/en-us/azure/databricks/dev-tools/api/2.0/jobs#--runs-submit for more details + 2. Use the Feathr Jar to run the job (hence will add an entry in `libraries` section) + 3. Will override `main_class_name` and `parameters` field in the JSON template `spark_jar_task` field + 4. will override the name of this job + + Args: + workspace_instance_url (str): the workinstance url. Document to get workspace_instance_url: https://docs.microsoft.com/en-us/azure/databricks/workspace/workspace-details#workspace-url + token_value (str): see here on how to get tokens: https://docs.microsoft.com/en-us/azure/databricks/dev-tools/api/latest/authentication + config_template (str): config template for databricks cluster. See https://docs.microsoft.com/en-us/azure/databricks/dev-tools/api/2.0/jobs#--runs-submit for more details. + databricks_work_dir (_type_, optional): databricks_work_dir must start with dbfs:/. Defaults to 'dbfs:/feathr_jobs'. + """ - Args: - workspace_instance_url (str): the workinstance url. Document to get workspace_instance_url: https://docs.microsoft.com/en-us/azure/databricks/workspace/workspace-details#workspace-url - token_value (str): see here on how to get tokens: https://docs.microsoft.com/en-us/azure/databricks/dev-tools/api/latest/authentication - config_template (str): config template for databricks cluster. See https://docs.microsoft.com/en-us/azure/databricks/dev-tools/api/2.0/jobs#--runs-submit for more details. - databricks_work_dir (_type_, optional): databricks_work_dir must start with dbfs:/. Defaults to 'dbfs:/feathr_jobs'. - """ def __init__( - self, - workspace_instance_url: str, - token_value: str, - config_template: Union[str,Dict], - databricks_work_dir: str = 'dbfs:/feathr_jobs', + self, + workspace_instance_url: str, + token_value: str, + config_template: Union[str, Dict], + databricks_work_dir: str = "dbfs:/feathr_jobs", ): - - # Below we will use Databricks job APIs (as well as many other APIs) to submit jobs or transfer files # For Job APIs, see https://docs.microsoft.com/en-us/azure/databricks/dev-tools/api/2.0/jobs # for DBFS APIs, see: https://docs.microsoft.com/en-us/azure/databricks/dev-tools/api/latest/dbfs self.config_template = config_template # remove possible trailing '/' due to wrong input format - self.workspace_instance_url = workspace_instance_url.rstrip('/') + self.workspace_instance_url = workspace_instance_url.rstrip("/") self.auth_headers = CaseInsensitiveDict() # Authenticate the REST APIs. Documentation: https://docs.microsoft.com/en-us/azure/databricks/dev-tools/api/latest/authentication - self.auth_headers['Accept'] = 'application/json' - self.auth_headers['Authorization'] = f'Bearer {token_value}' + self.auth_headers["Accept"] = "application/json" + self.auth_headers["Authorization"] = f"Bearer {token_value}" self.databricks_work_dir = databricks_work_dir - self.api_client = ApiClient(host=self.workspace_instance_url,token=token_value) + self.api_client = ApiClient(host=self.workspace_instance_url, token=token_value) def upload_or_get_cloud_path(self, local_path_or_http_path: str): """ @@ -69,56 +68,75 @@ def upload_or_get_cloud_path(self, local_path_or_http_path: str): """ src_parse_result = urlparse(local_path_or_http_path) file_name = os.path.basename(local_path_or_http_path) - # returned paths for the uploaded file - returned_path = os.path.join(self.databricks_work_dir, file_name) + # returned paths for the uploaded file. Note that we cannot use os.path.join here, since in Windows system it will yield paths like this: + # dbfs:/feathrazure_cijob_snowflake_9_30_157692\auto_generated_derived_features.conf, where the path sep is mixed, and won't be able to be parsed by databricks. + # so we force the path to be Linux style here. + cloud_dest_path = self.databricks_work_dir + "/" + file_name if src_parse_result.scheme.startswith('http'): with urlopen(local_path_or_http_path) as f: # use REST API to avoid local temp file data = f.read() - files = {'file': data} + files = {"file": data} # for DBFS APIs, see: https://docs.microsoft.com/en-us/azure/databricks/dev-tools/api/latest/dbfs r = requests.post(url=self.workspace_instance_url+'/api/2.0/dbfs/put', - headers=self.auth_headers, files=files, data={'overwrite': 'true', 'path': returned_path}) + headers=self.auth_headers, files=files, data={'overwrite': 'true', 'path': cloud_dest_path}) logger.info('{} is downloaded and then uploaded to location: {}', - local_path_or_http_path, returned_path) + local_path_or_http_path, cloud_dest_path) elif src_parse_result.scheme.startswith('dbfs'): # passed a cloud path logger.info( 'Skip uploading file {} as the file starts with dbfs:/', local_path_or_http_path) - returned_path = local_path_or_http_path + cloud_dest_path = local_path_or_http_path elif src_parse_result.scheme.startswith(('wasb','s3','gs')): # if the path starts with a location that's not a local path - logger.error("File {} cannot be downloaded. Please upload the file to dbfs manually.", local_path_or_http_path) - raise RuntimeError(f"File {local_path_or_http_path} cannot be downloaded. Please upload the file to dbfs manually.") + logger.error( + "File {} cannot be downloaded. Please upload the file to dbfs manually.", local_path_or_http_path + ) + raise RuntimeError( + f"File {local_path_or_http_path} cannot be downloaded. Please upload the file to dbfs manually." + ) else: # else it should be a local file path or dir if os.path.isdir(local_path_or_http_path): logger.info("Uploading folder {}", local_path_or_http_path) dest_paths = [] for item in Path(local_path_or_http_path).glob('**/*.conf'): - returned_path = self.upload_local_file(item.resolve()) - dest_paths.extend([returned_path]) - returned_path = ','.join(dest_paths) + cloud_dest_path = self._upload_local_file_to_workspace(item.resolve()) + dest_paths.extend([cloud_dest_path]) + cloud_dest_path = ','.join(dest_paths) else: - returned_path = self.upload_local_file(local_path_or_http_path) - return returned_path + cloud_dest_path = self._upload_local_file_to_workspace(local_path_or_http_path) + return cloud_dest_path - def upload_local_file(self, local_path: str) -> str: + def _upload_local_file_to_workspace(self, local_path: str) -> str: """ Supports transferring file from a local path to cloud working storage. """ file_name = os.path.basename(local_path) - # returned paths for the uploaded file - returned_path = os.path.join(self.databricks_work_dir, file_name) + # returned paths for the uploaded file. Note that we cannot use os.path.join here, since in Windows system it will yield paths like this: + # dbfs:/feathrazure_cijob_snowflake_9_30_157692\auto_generated_derived_features.conf, where the path sep is mixed, and won't be able to be parsed by databricks. + # so we force the path to be Linux style here. + cloud_dest_path = self.databricks_work_dir + "/" + file_name # `local_path_or_http_path` will be either string or PathLib object, so normalize it to string local_path = str(local_path) try: - DbfsApi(self.api_client).cp(recursive=True, overwrite=True, src=local_path, dst=returned_path) + DbfsApi(self.api_client).cp(recursive=True, overwrite=True, src=local_path, dst=cloud_dest_path) except RuntimeError as e: - raise RuntimeError(f"The source path: {local_path}, or the destination path: {returned_path}, is/are not valid.") from e - return returned_path - - def submit_feathr_job(self, job_name: str, main_jar_path: str, main_class_name: str, arguments: List[str], python_files: List[str], reference_files_path: List[str] = [], job_tags: Dict[str, str] = None, configuration: Dict[str, str] = {}, properties: Dict[str, str] = {}): + raise RuntimeError(f"The source path: {local_path}, or the destination path: {cloud_dest_path}, is/are not valid.") from e + return cloud_dest_path + + def submit_feathr_job( + self, + job_name: str, + main_jar_path: str, + main_class_name: str, + arguments: List[str], + python_files: List[str], + reference_files_path: List[str] = [], + job_tags: Dict[str, str] = None, + configuration: Dict[str, str] = {}, + properties: Dict[str, str] = {}, + ): """ submit the feathr job to databricks Refer to the databricks doc for more details on the meaning of the parameters: @@ -142,72 +160,93 @@ def submit_feathr_job(self, job_name: str, main_jar_path: str, main_class_name: # otherwise users might have missed the quotes in the config. Treat them as dict # Note that we need to use deep copy here, in order to make `self.config_template` immutable # Otherwise, since we need to change submission_params later, which will modify `self.config_template` and cause unexpected behaviors - submission_params = copy.deepcopy(self.config_template) - - submission_params['run_name'] = job_name - if 'existing_cluster_id' not in submission_params: + submission_params = copy.deepcopy(self.config_template) + + submission_params["run_name"] = job_name + cfg = configuration.copy() + if "existing_cluster_id" in submission_params: + logger.info("Using an existing general purpose cluster to run the feathr job...") + if cfg: + logger.warning( + "Spark execution configuration will be ignored. To use job-specific spark configs, please use a new job cluster or set the configs via Databricks UI." + ) + if job_tags: + logger.warning( + "Job tags will be ignored. To assign job tags to the cluster, please use a new job cluster." + ) + elif "new_cluster" in submission_params: + logger.info("Using a new job cluster to run the feathr job...") # if users don't specify existing_cluster_id # Solving this issue: Handshake fails trying to connect from Azure Databricks to Azure PostgreSQL with SSL # https://docs.microsoft.com/en-us/answers/questions/170730/handshake-fails-trying-to-connect-from-azure-datab.html - configuration['spark.executor.extraJavaOptions'] = '-Djava.security.properties=' - configuration['spark.driver.extraJavaOptions'] = '-Djava.security.properties=' - submission_params['new_cluster']['spark_conf'] = configuration + cfg["spark.executor.extraJavaOptions"] = "-Djava.security.properties=" + cfg["spark.driver.extraJavaOptions"] = "-Djava.security.properties=" + submission_params["new_cluster"]["spark_conf"] = cfg if job_tags: - custom_tags = submission_params['new_cluster'].get('custom_tags', {}) + custom_tags = submission_params["new_cluster"].get("custom_tags", {}) for tag, value in job_tags.items(): custom_tags[tag] = value - submission_params['new_cluster']['custom_tags'] = custom_tags + submission_params["new_cluster"]["custom_tags"] = custom_tags + else: + # TODO we should fail fast -- maybe check this in config verification while initializing the client. + raise ValueError( + "No cluster specifications are found. Either 'existing_cluster_id' or 'new_cluster' should be configured via feathr config." + ) # the feathr main jar file is anyway needed regardless it's pyspark or scala spark if not main_jar_path: - logger.info(f"Main JAR file is not set, using default package '{FEATHR_MAVEN_ARTIFACT}' from Maven") - submission_params['libraries'][0]['maven'] = { "coordinates": FEATHR_MAVEN_ARTIFACT } + logger.info(f"Main JAR file is not set, using default package '{get_maven_artifact_fullname()}' from Maven") + submission_params['libraries'][0]['maven'] = { "coordinates": get_maven_artifact_fullname() } else: - submission_params['libraries'][0]['jar'] = self.upload_or_get_cloud_path(main_jar_path) + submission_params["libraries"][0]["jar"] = self.upload_or_get_cloud_path(main_jar_path) # see here for the submission parameter definition https://docs.microsoft.com/en-us/azure/databricks/dev-tools/api/2.0/jobs#--request-structure-6 if python_files: # this is a pyspark job. definition here: https://docs.microsoft.com/en-us/azure/databricks/dev-tools/api/2.0/jobs#--sparkpythontask # the first file is the pyspark driver code. we only need the driver code to execute pyspark - param_and_file_dict = {"parameters": arguments, "python_file": self.upload_or_get_cloud_path(python_files[0])} + param_and_file_dict = { + "parameters": arguments, + "python_file": self.upload_or_get_cloud_path(python_files[0]), + } # indicates this is a pyspark job # `setdefault` method will get the value of the "spark_python_task" item, if the "spark_python_task" item does not exist, insert "spark_python_task" with the value "param_and_file_dict": - submission_params.setdefault('spark_python_task',param_and_file_dict) + submission_params.setdefault("spark_python_task", param_and_file_dict) else: # this is a scala spark job - submission_params['spark_jar_task']['parameters'] = arguments - submission_params['spark_jar_task']['main_class_name'] = main_class_name + submission_params["spark_jar_task"]["parameters"] = arguments + submission_params["spark_jar_task"]["main_class_name"] = main_class_name result = RunsApi(self.api_client).submit_run(submission_params) try: # see if we can parse the returned result - self.res_job_id = result['run_id'] + self.res_job_id = result["run_id"] except: - logger.error("Submitting Feathr job to Databricks cluster failed. Message returned from Databricks: {}", result) + logger.error( + "Submitting Feathr job to Databricks cluster failed. Message returned from Databricks: {}", result + ) exit(1) result = RunsApi(self.api_client).get_run(self.res_job_id) - self.job_url = result['run_page_url'] - logger.info('Feathr job Submitted Successfully. View more details here: {}', self.job_url) + self.job_url = result["run_page_url"] + logger.info("Feathr job Submitted Successfully. View more details here: {}", self.job_url) # return ID as the submission result return self.res_job_id def wait_for_completion(self, timeout_seconds: Optional[int] = 600) -> bool: - """ Returns true if the job completed successfully - """ + """Returns true if the job completed successfully""" start_time = time.time() while (timeout_seconds is None) or (time.time() - start_time < timeout_seconds): status = self.get_status() - logger.debug('Current Spark job status: {}', status) + logger.debug("Current Spark job status: {}", status) # see all the status here: # https://docs.microsoft.com/en-us/azure/databricks/dev-tools/api/2.0/jobs#--runlifecyclestate # https://docs.microsoft.com/en-us/azure/databricks/dev-tools/api/2.0/jobs#--runresultstate - if status in {'SUCCESS'}: + if status in {"SUCCESS"}: return True - elif status in {'INTERNAL_ERROR', 'FAILED', 'TIMEDOUT', 'CANCELED'}: + elif status in {"INTERNAL_ERROR", "FAILED", "TIMEDOUT", "CANCELED"}: result = RunsApi(self.api_client).get_run_output(self.res_job_id) # See here for the returned fields: https://docs.microsoft.com/en-us/azure/databricks/dev-tools/api/2.0/jobs#--response-structure-8 # print out logs and stack trace if the job has failed @@ -220,14 +259,14 @@ def wait_for_completion(self, timeout_seconds: Optional[int] = 600) -> bool: else: time.sleep(30) else: - raise TimeoutError('Timeout waiting for Feathr job to complete') + raise TimeoutError("Timeout waiting for Feathr job to complete") def get_status(self) -> str: assert self.res_job_id is not None result = RunsApi(self.api_client).get_run(self.res_job_id) # first try to get result state. it might not be available, and if that's the case, try to get life_cycle_state # see result structure: https://docs.microsoft.com/en-us/azure/databricks/dev-tools/api/2.0/jobs#--response-structure-6 - res_state = result['state'].get('result_state') or result['state']['life_cycle_state'] + res_state = result["state"].get("result_state") or result["state"]["life_cycle_state"] assert res_state is not None return res_state @@ -241,7 +280,6 @@ def get_job_result_uri(self) -> str: # in case users call this API even when there's no tags available return None if custom_tags is None else custom_tags[OUTPUT_PATH_TAG] - def get_job_tags(self) -> Dict[str, str]: """Get job tags @@ -252,21 +290,23 @@ def get_job_tags(self) -> Dict[str, str]: # For result structure, see https://docs.microsoft.com/en-us/azure/databricks/dev-tools/api/2.0/jobs#--response-structure-6 result = RunsApi(self.api_client).get_run(self.res_job_id) - if 'new_cluster' in result['cluster_spec']: - custom_tags = result['cluster_spec']['new_cluster']['custom_tags'] + if "new_cluster" in result["cluster_spec"]: + custom_tags = result["cluster_spec"]["new_cluster"].get("custom_tags") return custom_tags else: # this is not a new cluster; it's an existing cluster. - logger.warning("Job tags are not available since you are using an existing Databricks cluster. Consider using 'new_cluster' in databricks configuration.") + logger.warning( + "Job tags are not available since you are using an existing Databricks cluster. Consider using 'new_cluster' in databricks configuration." + ) return None - def download_result(self, result_path: str, local_folder: str): """ Supports downloading files from the result folder. Only support paths starts with `dbfs:/` and only support downloading files in one folder (per Spark's design, everything will be in the result folder in a flat manner) """ - if not result_path.startswith('dbfs'): - raise RuntimeError('Currently only paths starting with dbfs is supported for downloading results from a databricks cluster. The path should start with \"dbfs:\" .') + if not result_path.startswith("dbfs"): + raise RuntimeError( + 'Currently only paths starting with dbfs is supported for downloading results from a databricks cluster. The path should start with "dbfs:" .' + ) DbfsApi(self.api_client).cp(recursive=True, overwrite=True, src=result_path, dst=local_folder) - diff --git a/feathr_project/feathr/spark_provider/_localspark_submission.py b/feathr_project/feathr/spark_provider/_localspark_submission.py index afed9683d..a5ef0e53d 100644 --- a/feathr_project/feathr/spark_provider/_localspark_submission.py +++ b/feathr_project/feathr/spark_provider/_localspark_submission.py @@ -1,3 +1,4 @@ +from copy import deepcopy from datetime import datetime import json import os @@ -10,7 +11,8 @@ from loguru import logger from pyspark import * -from feathr.constants import FEATHR_MAVEN_ARTIFACT +from feathr.constants import OUTPUT_PATH_TAG +from feathr.version import get_maven_artifact_fullname from feathr.spark_provider._abc import SparkJobLauncher @@ -40,6 +42,7 @@ def __init__( self.retry_sec = retry_sec self.packages = self._get_default_package() self.master = master or "local[*]" + self.job_tags = None def upload_or_get_cloud_path(self, local_path_or_http_path: str): """For Local Spark Case, no need to upload to cloud workspace.""" @@ -52,6 +55,7 @@ def submit_feathr_job( main_class_name: str, arguments: List[str] = None, python_files: List[str] = None, + job_tags: Dict[str, str] = None, configuration: Dict[str, str] = {}, properties: Dict[str, str] = {}, **_, @@ -66,9 +70,10 @@ def submit_feathr_job( main_class_name: name of your main class arguments: all the arguments you want to pass into the spark job python_files: required .zip, .egg, or .py files of spark job + job_tags: tags of the job, for example you might want to put your user ID, or a tag with a certain information configuration: Additional configs for the spark job properties: System properties configuration - **_: Not used arguments in local spark mode, such as reference_files_path and job_tags + **_: Not used arguments in local spark mode, such as reference_files_path """ logger.warning( f"Local Spark Mode only support basic params right now and should be used only for testing purpose." @@ -77,7 +82,7 @@ def submit_feathr_job( # Get conf and package arguments cfg = configuration.copy() if configuration else {} - maven_dependency = f"{cfg.pop('spark.jars.packages', self.packages)},{FEATHR_MAVEN_ARTIFACT}" + maven_dependency = f"{cfg.pop('spark.jars.packages', self.packages)},{get_maven_artifact_fullname()}" spark_args = self._init_args(job_name=job_name, confs=cfg) if not main_jar_path: @@ -86,7 +91,7 @@ def submit_feathr_job( # This is a JAR job # Azure Synapse/Livy doesn't allow JAR job starts from Maven directly, we must have a jar file uploaded. # so we have to use a dummy jar as the main file. - logger.info(f"Main JAR file is not set, using default package '{FEATHR_MAVEN_ARTIFACT}' from Maven") + logger.info(f"Main JAR file is not set, using default package '{get_maven_artifact_fullname()}' from Maven") # Use the no-op jar as the main file # This is a dummy jar which contains only one `org.example.Noop` class with one empty `main` function # which does nothing @@ -125,6 +130,8 @@ def submit_feathr_job( logger.info(f"Local Spark job submit with pid: {proc.pid}.") + self.job_tags = deepcopy(job_tags) + return proc def wait_for_completion(self, timeout_seconds: Optional[float] = 500) -> bool: @@ -198,6 +205,22 @@ def get_status(self) -> str: """Get the status of the job, only a placeholder for local spark""" return self.latest_spark_proc.returncode + def get_job_result_uri(self) -> str: + """Get job output path + + Returns: + str: output_path + """ + return self.job_tags.get(OUTPUT_PATH_TAG, None) if self.job_tags else None + + def get_job_tags(self) -> Dict[str, str]: + """Get job tags + + Returns: + Dict[str, str]: a dict of job tags + """ + return self.job_tags + def _init_args(self, job_name: str, confs: Dict[str, str]) -> List[str]: logger.info(f"Spark job: {job_name} is running on local spark with master: {self.master}.") args = [ diff --git a/feathr_project/feathr/spark_provider/_synapse_submission.py b/feathr_project/feathr/spark_provider/_synapse_submission.py index 5b55a3b06..6b56f6a3b 100644 --- a/feathr_project/feathr/spark_provider/_synapse_submission.py +++ b/feathr_project/feathr/spark_provider/_synapse_submission.py @@ -22,6 +22,7 @@ from feathr.spark_provider._abc import SparkJobLauncher from feathr.constants import * +from feathr.version import get_maven_artifact_fullname class LivyStates(Enum): """ Adapt LivyStates over to relax the dependency for azure-synapse-spark pacakge. @@ -114,17 +115,17 @@ def submit_feathr_job(self, job_name: str, main_jar_path: str = None, main_clas if not main_jar_path: # We don't have the main jar, use Maven # Add Maven dependency to the job configuration + logger.info(f"Main JAR file is not set, using default package '{get_maven_artifact_fullname()}' from Maven") if "spark.jars.packages" in cfg: cfg["spark.jars.packages"] = ",".join( - [cfg["spark.jars.packages"], FEATHR_MAVEN_ARTIFACT]) + [cfg["spark.jars.packages"], get_maven_artifact_fullname()]) else: - cfg["spark.jars.packages"] = FEATHR_MAVEN_ARTIFACT + cfg["spark.jars.packages"] = get_maven_artifact_fullname() if not python_files: # This is a JAR job # Azure Synapse/Livy doesn't allow JAR job starts from Maven directly, we must have a jar file uploaded. # so we have to use a dummy jar as the main file. - logger.info(f"Main JAR file is not set, using default package '{FEATHR_MAVEN_ARTIFACT}' from Maven") # Use the no-op jar as the main file # This is a dummy jar which contains only one `org.example.Noop` class with one empty `main` function which does nothing current_dir = pathlib.Path(__file__).parent.resolve() @@ -169,7 +170,7 @@ def submit_feathr_job(self, job_name: str, main_jar_path: str = None, main_clas def wait_for_completion(self, timeout_seconds: Optional[float]) -> bool: """ Returns true if the job completed successfully - """ + """ start_time = time.time() while (timeout_seconds is None) or (time.time() - start_time < timeout_seconds): status = self.get_status() @@ -178,7 +179,9 @@ def wait_for_completion(self, timeout_seconds: Optional[float]) -> bool: return True elif status in {LivyStates.ERROR.value, LivyStates.DEAD.value, LivyStates.KILLED.value}: logger.error("Feathr job has failed.") - logger.error(self._api.get_driver_log(self.current_job_info.id).decode('utf-8')) + error_msg = self._api.get_driver_log(self.current_job_info.id).decode('utf-8') + logger.error(error_msg) + logger.error("The size of the whole error log is: {}. The logs might be truncated in some cases (such as in Visual Studio Code) so only the top a few lines of the error message is displayed. If you cannot see the whole log, you may want to extend the setting for output size limit.", len(error_msg)) return False else: time.sleep(30) @@ -432,8 +435,6 @@ def download_file(self, target_adls_directory: str, local_dir_cache: str): # returns the paths to all the files in the target director in ADLS # get all the paths that are not under a directory - test_paths = self.file_system_client.get_paths( - path=parse_result.path, recursive=False) result_paths = [basename(file_path.name) for file_path in self.file_system_client.get_paths( path=parse_result.path, recursive=False) if not file_path.is_directory] diff --git a/feathr_project/feathr/udf/_preprocessing_pyudf_manager.py b/feathr_project/feathr/udf/_preprocessing_pyudf_manager.py index 55756ba3d..c4f102566 100644 --- a/feathr_project/feathr/udf/_preprocessing_pyudf_manager.py +++ b/feathr_project/feathr/udf/_preprocessing_pyudf_manager.py @@ -176,6 +176,7 @@ def prepare_pyspark_udf_files(feature_names: List[str], local_workspace_dir): for feature_name in feature_names: if feature_name in features_with_preprocessing: has_py_udf_preprocessing = True + break if has_py_udf_preprocessing: pyspark_driver_path = os.path.join(local_workspace_dir, FEATHR_PYSPARK_DRIVER_FILE_NAME) diff --git a/feathr_project/feathr/utils/config.py b/feathr_project/feathr/utils/config.py new file mode 100644 index 000000000..9a5f5fd89 --- /dev/null +++ b/feathr_project/feathr/utils/config.py @@ -0,0 +1,278 @@ +import collections.abc +from copy import deepcopy +import os +import json +from tempfile import NamedTemporaryFile +from typing import Dict +import yaml + +from feathr.utils.platform import is_databricks + + +DEFAULT_FEATHR_CONFIG = { + "api_version": 1, + "project_config": {}, # "project_name" + "feature_registry": {}, # "api_endpoint" + "spark_config": { + "spark_cluster": "local", # Currently support 'azure_synapse', 'databricks', and 'local' + "spark_result_output_parts": "1", + }, + "offline_store": { + "adls": {"adls_enabled": "true"}, + "wasb": {"wasb_enabled": "true"}, + }, + "online_store": { + "redis": { + # "host" + "port": "6380", + "ssl_enabled": "true", + } + } +} + + +# New databricks job cluster config +DEFAULT_DATABRICKS_CLUSTER_CONFIG = { + "spark_version": "11.2.x-scala2.12", + "node_type_id": "Standard_D3_v2", + "num_workers": 2, + "spark_conf": { + "FEATHR_FILL_IN": "FEATHR_FILL_IN", + # Exclude conflicting packages if use feathr <= v0.8.0: + "spark.jars.excludes": "commons-logging:commons-logging,org.slf4j:slf4j-api,com.google.protobuf:protobuf-java,javax.xml.bind:jaxb-api", + }, +} + + +# New Azure Synapse spark pool config +DEFAULT_AZURE_SYNAPSE_SPARK_POOL_CONFIG = { + "executor_size": "Small", + "executor_num": 2, +} + + +def generate_config( + resource_prefix: str, + project_name: str, + output_filepath: str = None, + databricks_workspace_token_value: str = None, + databricks_cluster_id: str = None, + redis_password: str = None, + adls_key: str = None, + use_env_vars: bool = True, + **kwargs, +) -> str: + """Generate a feathr config yaml file. + Note, `use_env_vars` argument gives an option to either use environment variables for generating the config file + or not. Feathr client will use environment variables anyway if they are set. + + Keyword arguments follow the same naming convention as the feathr config. E.g. to set Databricks as the target + cluster, use `spark_config__spark_cluster="databricks"`. + See https://feathr-ai.github.io/feathr/quickstart_synapse.html#step-4-update-feathr-config for more details. + + Note: + This utility function assumes Azure resources are deployed using the Azure Resource Manager (ARM) template, + and infers resource names based on the given `resource_prefix`. If you deploy resources manually, you may need + to pass each resource url manually, e.g. `spark_config__azure_synapse__dev_url="your-resource-url"`. + + Args: + resource_prefix: Resource name prefix used when deploying Feathr resources by using ARM template. + project_name: Feathr project name. + cluster_name (optional): Databricks cluster or Azure Synapse spark pool name to use an existing one. + output_filepath (optional): Output filepath. + use_env_vars (optional): Whether to use environment variables if they are set. + databricks_workspace_token_value (optional): Databricks workspace token. If provided, the value will be stored + as the environment variable. + databricks_cluster_id (optional): Databricks cluster id to use an existing cluster. + redis_password (optional): Redis password. If provided, the value will be stored as the environment variable. + adls_key (optional): ADLS key. If provided, the value will be stored as the environment variable. + + Returns: + str: Generated config file path. This will be identical to `output_filepath` if provided. + """ + # Set keys + if databricks_workspace_token_value: + os.environ["DATABRICKS_WORKSPACE_TOKEN_VALUE"] = databricks_workspace_token_value + if redis_password: + os.environ["REDIS_PASSWORD"] = redis_password + if adls_key: + os.environ["ADLS_KEY"] = adls_key + + # Set configs + config = deepcopy(DEFAULT_FEATHR_CONFIG) + config["project_config"]["project_name"] = project_name + config["feature_registry"]["api_endpoint"] = f"https://{resource_prefix}webapp.azurewebsites.net/api/v1" + config["online_store"]["redis"]["host"] = f"{resource_prefix}redis.redis.cache.windows.net" + + # Update configs using kwargs + new_config = _config_kwargs_to_dict(**kwargs) + _update_config(config, new_config) + + # Set platform specific configurations + if config["spark_config"]["spark_cluster"] == "local": + _set_local_spark_config() + elif config["spark_config"]["spark_cluster"] == "azure_synapse": + _set_azure_synapse_config( + config=config, + resource_prefix=resource_prefix, + project_name=project_name, + ) + elif config["spark_config"]["spark_cluster"] == "databricks": + _set_databricks_config( + config=config, + project_name=project_name, + cluster_id=databricks_cluster_id, + ) + + # Maybe update configs with environment variables + if use_env_vars: + _maybe_update_config_with_env_var(config, "SPARK_CONFIG__SPARK_CLUSTER") + _maybe_update_config_with_env_var(config, "SPARK_CONFIG__AZURE_SYNAPSE__DEV_URL") + _maybe_update_config_with_env_var(config, "SPARK_CONFIG__AZURE_SYNAPSE__POOL_NAME") + _maybe_update_config_with_env_var(config, "SPARK_CONFIG__AZURE_SYNAPSE__WORKSPACE_DIR") + _maybe_update_config_with_env_var(config, "SPARK_CONFIG__DATABRICKS__WORK_DIR") + _maybe_update_config_with_env_var(config, "SPARK_CONFIG__DATABRICKS__WORKSPACE_INSTANCE_URL") + _maybe_update_config_with_env_var(config, "SPARK_CONFIG__DATABRICKS__CONFIG_TEMPLATE") + + # Verify config + _verify_config(config) + + # Write config to file + if not output_filepath: + output_filepath = NamedTemporaryFile(mode="w", delete=False).name + + with open(output_filepath, "w") as f: + yaml.dump(config, f, default_flow_style=False) + + return output_filepath + + +def _set_local_spark_config(): + """Set environment variables for local spark cluster.""" + os.environ["SPARK_LOCAL_IP"] = os.getenv( + "SPARK_LOCAL_IP", + "127.0.0.1", + ) + + +def _set_azure_synapse_config( + config: Dict, + resource_prefix: str, + project_name: str, +): + """Set configs for Azure Synapse spark cluster.""" + + config["spark_config"]["azure_synapse"] = config["spark_config"].get("azure_synapse", {}) + + if not config["spark_config"]["azure_synapse"].get("dev_url"): + config["spark_config"]["azure_synapse"]["dev_url"] = f"https://{resource_prefix}syws.dev.azuresynapse.net" + + if not config["spark_config"]["azure_synapse"].get("workspace_dir"): + config["spark_config"]["azure_synapse"]["workspace_dir"] =\ + f"abfss://{resource_prefix}fs@{resource_prefix}dls.dfs.core.windows.net/{project_name}" + + for k, v in DEFAULT_AZURE_SYNAPSE_SPARK_POOL_CONFIG.items(): + if not config["spark_config"]["azure_synapse"].get(k): + config["spark_config"]["azure_synapse"][k] = v + + +def _set_databricks_config( + config: Dict, + project_name: str, + cluster_id: str = None, +): + """Set configs for Databricks spark cluster.""" + + config["spark_config"]["databricks"] = config["spark_config"].get("databricks", {}) + + if not config["spark_config"]["databricks"].get("work_dir"): + config["spark_config"]["databricks"]["work_dir"] = f"dbfs:/{project_name}" + + if not config["spark_config"]["databricks"].get("config_template"): + databricks_config = { + "run_name": "FEATHR_FILL_IN", + "libraries": [{"jar": "FEATHR_FILL_IN"}], + "spark_jar_task": { + "main_class_name": "FEATHR_FILL_IN", + "parameters": ["FEATHR_FILL_IN"], + }, + } + if cluster_id is None: + databricks_config["new_cluster"] = DEFAULT_DATABRICKS_CLUSTER_CONFIG + else: + databricks_config["existing_cluster_id"] = cluster_id + + config["spark_config"]["databricks"]["config_template"] = json.dumps(databricks_config) + + +def _config_kwargs_to_dict(**kwargs) -> Dict: + """Parse config's keyword arguments to dictionary. + e.g. `spark_config__spark_cluster="local"` will be parsed to `{"spark_config": {"spark_cluster": "local"}}`. + """ + config = dict() + + for conf_key, conf_value in kwargs.items(): + if conf_value is None: + continue + + conf = config + keys = conf_key.split("__") + for k in keys[:-1]: + if k not in conf: + conf[k] = dict() + conf = conf[k] + conf[keys[-1]] = conf_value + + return config + + +def _update_config(config: Dict, new_config: Dict): + """Update config dictionary with the values in `new_config`.""" + for k, v in new_config.items(): + if k in config and isinstance(v, collections.abc.Mapping): + _update_config(config[k], v) + else: + config[k] = v + + +def _verify_config(config: Dict): + """Verify config.""" + if config["spark_config"]["spark_cluster"] == "azure_synapse": + if not os.environ.get("ADLS_KEY"): + raise ValueError("ADLS_KEY must be set in environment variables") + elif ( + not os.environ.get("SPARK_CONFIG__AZURE_SYNAPSE__DEV_URL") and + config["spark_config"]["azure_synapse"].get("dev_url") is None + ): + raise ValueError("Azure Synapse dev endpoint is not provided.") + elif ( + not os.environ.get("SPARK_CONFIG__AZURE_SYNAPSE__POOL_NAME") and + config["spark_config"]["azure_synapse"].get("pool_name") is None + ): + raise ValueError("Azure Synapse pool name is not provided.") + + elif config["spark_config"]["spark_cluster"] == "databricks": + if not os.environ.get("DATABRICKS_WORKSPACE_TOKEN_VALUE"): + raise ValueError("Databricks workspace token is not provided.") + elif ( + not os.environ.get("SPARK_CONFIG__DATABRICKS__WORKSPACE_INSTANCE_URL") and + config["spark_config"]["databricks"].get("workspace_instance_url") is None + ): + raise ValueError("Databricks workspace url is not provided.") + + +def _maybe_update_config_with_env_var(config: Dict, env_var_name: str): + """Update config dictionary with the values in environment variables. + e.g. `SPARK_CONFIG__SPARK_CLUSTER` will be parsed to `{"spark_config": {"spark_cluster": "local"}}`. + """ + if not os.environ.get(env_var_name): + return + + keys = env_var_name.lower().split("__") + conf = config + for k in keys[:-1]: + if k not in conf: + conf[k] = dict() + conf = conf[k] + + conf[keys[-1]] = os.environ[env_var_name] diff --git a/feathr_project/feathr/utils/job_utils.py b/feathr_project/feathr/utils/job_utils.py index 6a6bd63c0..329814f12 100644 --- a/feathr_project/feathr/utils/job_utils.py +++ b/feathr_project/feathr/utils/job_utils.py @@ -1,77 +1,198 @@ -from feathr.client import FeathrClient -import os -import glob -from feathr.constants import OUTPUT_FORMAT +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import Union + from loguru import logger import pandas as pd -import tempfile -from pandas.errors import EmptyDataError +from pyspark.sql import DataFrame, SparkSession + +from feathr.client import FeathrClient +from feathr.constants import OUTPUT_FORMAT +from feathr.utils.platform import is_databricks + + +def get_result_pandas_df( + client: FeathrClient, + data_format: str = None, + res_url: str = None, + local_cache_path: str = None, +) -> pd.DataFrame: + """Download the job result dataset from cloud as a Pandas DataFrame. + + Args: + client: Feathr client + data_format: Format to read the downloaded files. Currently support `parquet`, `delta`, `avro`, and `csv`. + Default to use client's job tags if exists. + res_url: Result URL to download files from. Note that this will not block the job so you need to make sure + the job is finished and the result URL contains actual data. Default to use client's job tags if exists. + local_cache_path (optional): Specify the absolute download path. if the user does not provide this, + the function will create a temporary directory. + + Returns: + pandas DataFrame + """ + return get_result_df(client=client, data_format=data_format, res_url=res_url, local_cache_path=local_cache_path) +def get_result_spark_df( + spark: SparkSession, + client: FeathrClient, + data_format: str = None, + res_url: str = None, + local_cache_path: str = None, +) -> DataFrame: + """Download the job result dataset from cloud as a Spark DataFrame. -def get_result_df(client: FeathrClient, format: str = None, res_url: str = None, local_folder: str = None) -> pd.DataFrame: - """Download the job result dataset from cloud as a Pandas dataframe to make it easier for the client to read. + Args: + spark: Spark session + client: Feathr client + data_format: Format to read the downloaded files. Currently support `parquet`, `delta`, `avro`, and `csv`. + Default to use client's job tags if exists. + res_url: Result URL to download files from. Note that this will not block the job so you need to make sure + the job is finished and the result URL contains actual data. Default to use client's job tags if exists. + local_cache_path (optional): Specify the absolute download path. if the user does not provide this, + the function will create a temporary directory. - format: format to read the downloaded files. Currently support `parquet`, `delta`, `avro`, and `csv`. Default to `avro` if not specified. - res_url: output URL to download files. Note that this will not block the job so you need to make sure the job is finished and result URL contains actual data. - local_folder: optional parameter to specify the absolute download path. if the user does not provide this, function will create a temporary directory and delete it after reading the dataframe. + Returns: + Spark DataFrame """ - # use a result url if it's provided by the user, otherwise use the one provided by the job + return get_result_df( + client=client, + data_format=data_format, + res_url=res_url, + local_cache_path=local_cache_path, + spark=spark, + ) + + +def get_result_df( + client: FeathrClient, + data_format: str = None, + res_url: str = None, + local_cache_path: str = None, + spark: SparkSession = None, + format: str = None, +) -> Union[DataFrame, pd.DataFrame]: + """Download the job result dataset from cloud as a Spark DataFrame or pandas DataFrame. + + Args: + client: Feathr client + data_format: Format to read the downloaded files. Currently support `parquet`, `delta`, `avro`, and `csv`. + Default to use client's job tags if exists. + res_url: Result URL to download files from. Note that this will not block the job so you need to make sure + the job is finished and the result URL contains actual data. Default to use client's job tags if exists. + local_cache_path (optional): Specify the absolute download directory. if the user does not provide this, + the function will create a temporary directory. + spark (optional): Spark session. If provided, the function returns spark Dataframe. + Otherwise, it returns pd.DataFrame. + format: An alias for `data_format` (for backward compatibility). + + Returns: + Either Spark or pandas DataFrame. + """ + if format is not None: + data_format = format + + if data_format is None: + # May use data format from the job tags + if client.get_job_tags() and client.get_job_tags().get(OUTPUT_FORMAT): + data_format = client.get_job_tags().get(OUTPUT_FORMAT) + else: + raise ValueError("Cannot determine the data format. Please provide the data_format argument.") + + data_format = data_format.lower() + + if is_databricks() and client.spark_runtime != "databricks": + raise RuntimeError(f"The function is called from Databricks but the client.spark_runtime is {client.spark_runtime}.") + + # TODO Loading Synapse Delta table result into pandas has a bug: https://github.com/delta-io/delta-rs/issues/582 + if not spark and client.spark_runtime == "azure_synapse" and data_format == "delta": + raise RuntimeError(f"Loading Delta table result from Azure Synapse into pandas DataFrame is not supported. You maybe able to use spark DataFrame to load the result instead.") + + # use a result url if it's provided by the user, otherwise use the one provided by the job res_url: str = res_url or client.get_job_result_uri(block=True, timeout_sec=1200) if res_url is None: - raise RuntimeError("res_url is None. Please make sure either you provide a res_url or make sure the job finished in FeathrClient has a valid result URI.") + raise ValueError( + "`res_url` is None. Please make sure either you provide a res_url or make sure the job finished in FeathrClient has a valid result URI." + ) - # use user provided format, if there isn't one, then otherwise use the one provided by the job; - # if none of them is available, "avro" is the default format. - format: str = format or client.get_job_tags().get(OUTPUT_FORMAT, "") - if format is None or format == "": - format = "avro" + if client.spark_runtime == "local": + if local_cache_path is not None: + logger.warning( + "In local spark mode, the result files are expected to be stored at a local storage and thus `local_cache_path` argument will be ignored." + ) + local_cache_path = res_url - # if local_folder params is not provided then create a temporary folder - if local_folder is not None: - local_dir_path = local_folder - else: - tmp_dir = tempfile.TemporaryDirectory() - local_dir_path = tmp_dir.name - - client.feathr_spark_launcher.download_result(result_path=res_url, local_folder=local_dir_path) - dataframe_list = [] - # by default the result are in avro format - if format.casefold()=="parquet": - files = glob.glob(os.path.join(local_dir_path, '*.parquet')) - from pyarrow.parquet import ParquetDataset - ds = ParquetDataset(files) - result_df = ds.read().to_pandas() - elif format.casefold()=="delta": - from deltalake import DeltaTable - delta = DeltaTable(local_dir_path) - if not client.spark_runtime == 'azure_synapse': - # don't detect for synapse result with Delta as there's a problem with underlying system - # Issues are tracked here: https://github.com/delta-io/delta-rs/issues/582 - result_df = delta.to_pyarrow_table().to_pandas() + elif client.spark_runtime == "databricks": + if not res_url.startswith("dbfs:"): + logger.warning( + f"In Databricks, the result files are expected to be stored in DBFS, but the res_url {res_url} is not a dbfs path. Prefixing it with 'dbfs:/'" + ) + res_url = f"dbfs:/{res_url.lstrip('/')}" + + if is_databricks(): # Check if the function is being called from Databricks + if local_cache_path is not None: + logger.warning( + "Result files are already in DBFS and thus `local_cache_path` will be ignored." + ) + local_cache_path = res_url + + if local_cache_path is None: + local_cache_path = TemporaryDirectory().name + + if local_cache_path != res_url: + logger.info(f"{res_url} files will be downloaded into {local_cache_path}") + client.feathr_spark_launcher.download_result(result_path=res_url, local_folder=local_cache_path) + + result_df = None + try: + if spark is not None: + if data_format == "csv": + result_df = spark.read.option("header", True).csv(local_cache_path) + else: + result_df = spark.read.format(data_format).load(local_cache_path) else: - logger.info("Please use Azure Synapse to read the result in the Azure Synapse cluster. Reading local results is not supported for Azure Synapse. Empty DataFrame is returned.") - result_df = pd.DataFrame() - elif format.casefold()=="avro": + result_df = _load_files_to_pandas_df( + dir_path=local_cache_path.replace("dbfs:", "/dbfs"), # replace to python path if spark path is provided. + data_format=data_format, + ) + except Exception as e: + logger.error(f"Failed to load result files from {local_cache_path} with format {data_format}.") + raise e + + return result_df + + +def _load_files_to_pandas_df(dir_path: str, data_format: str = "avro") -> pd.DataFrame: + + if data_format == "parquet": + return pd.read_parquet(dir_path) + + elif data_format == "delta": + from deltalake import DeltaTable + delta = DeltaTable(dir_path) + return delta.to_pyarrow_table().to_pandas() + + elif data_format == "avro": import pandavro as pdx - for file in glob.glob(os.path.join(local_dir_path, '*.avro')): - dataframe_list.append(pdx.read_avro(file)) - result_df = pd.concat(dataframe_list, axis=0) - elif format.casefold()=="csv": - for file in glob.glob(os.path.join(local_dir_path, '*.csv')): + if Path(dir_path).is_file(): + return pdx.read_avro(dir_path) + else: try: - df = pd.read_csv(file, index_col=None, header=None) - except EmptyDataError: - # in case there are empty files - df = pd.DataFrame() - dataframe_list.append(df) - result_df = pd.concat(dataframe_list, axis=0) - # Reset index to avoid duplicated indices - result_df.reset_index(drop=True) - else: - raise RuntimeError(f"{format} is currently not supported in get_result_df. Currently only parquet, delta, avro, and csv are supported, please consider writing a customized function to read the result.") + return pd.concat([pdx.read_avro(f) for f in Path(dir_path).glob("*.avro")]).reset_index(drop=True) + except ValueError: # No object to concat when the dir is empty + return pd.DataFrame() - - if local_folder is None: - tmp_dir.cleanup() - return result_df \ No newline at end of file + elif data_format == "csv": + if Path(dir_path).is_file(): + return pd.read_csv(dir_path) + else: + try: + return pd.concat([pd.read_csv(f) for f in Path(dir_path).glob("*.csv")]).reset_index(drop=True) + except ValueError: # No object to concat when the dir is empty + return pd.DataFrame() + + else: + raise ValueError( + f"{data_format} is currently not supported in get_result_df. Currently only parquet, delta, avro, and csv are supported, please consider writing a customized function to read the result." + ) diff --git a/feathr_project/feathr/utils/platform.py b/feathr_project/feathr/utils/platform.py new file mode 100644 index 000000000..8f832f22d --- /dev/null +++ b/feathr_project/feathr/utils/platform.py @@ -0,0 +1,45 @@ +"""Platform utilities. +Refs: https://github.com/microsoft/recommenders/blob/main/recommenders/utils/notebook_utils.py +""" +from pathlib import Path + + +def is_jupyter() -> bool: + """Check if the module is running on Jupyter notebook/console. + Note - there might be better way to check if the code is running on a jupyter notebook or not, + but this hacky way still works. + + Ref: + https://stackoverflow.com/questions/15411967/how-can-i-check-if-code-is-executed-in-the-ipython-notebook + + Returns: + bool: True if the module is running on Jupyter notebook or Jupyter console, False otherwise. + """ + try: + # Pre-loaded module `get_ipython()` tells you whether you are running inside IPython or not. + shell_name = get_ipython().__class__.__name__ + # `ZMQInteractiveShell` tells you if this is an interactive mode (notebook). + if shell_name == "ZMQInteractiveShell": + return True + else: + return False + except NameError: + return False + + +def is_databricks() -> bool: + """Check if the module is running on Databricks. + + Returns: + bool: True if the module is running on Databricks notebook, False otherwise. + """ + try: + if str(Path(".").resolve()) == "/databricks/driver": + return True + else: + return False + except NameError: + return False + + +# TODO maybe add is_synapse() diff --git a/feathr_project/feathr/version.py b/feathr_project/feathr/version.py index 807119de6..1b1d4559e 100644 --- a/feathr_project/feathr/version.py +++ b/feathr_project/feathr/version.py @@ -1 +1,10 @@ -__version__ = "0.8.0" \ No newline at end of file +__version__ = "0.9.0" + +def get_version(): + return __version__ + +# Decouple Feathr MAVEN Version from Feathr Python SDK Version +import os +def get_maven_artifact_fullname(): + maven_artifact_version = os.environ.get("MAVEN_ARTIFACT_VERSION", __version__) + return f"com.linkedin.feathr:feathr_2.12:{maven_artifact_version}" \ No newline at end of file diff --git a/feathr_project/feathrcli/data/feathr_user_workspace/feathr_config.yaml b/feathr_project/feathrcli/data/feathr_user_workspace/feathr_config.yaml index c40e7c45d..d76b63e3e 100644 --- a/feathr_project/feathrcli/data/feathr_user_workspace/feathr_config.yaml +++ b/feathr_project/feathrcli/data/feathr_user_workspace/feathr_config.yaml @@ -25,8 +25,8 @@ project_config: # the environemnt variables are optional, however you will need them if you want to use some of the services: - ADLS_ACCOUNT - ADLS_KEY - - WASB_ACCOUNT - - WASB_KEY + - BLOB_ACCOUNT + - BLOB_KEY - S3_ACCESS_KEY - S3_SECRET_KEY - JDBC_TABLE @@ -41,7 +41,7 @@ offline_store: adls_enabled: true # paths starts with wasb:// or wasbs:// - # WASB_ACCOUNT and WASB_KEY should be set in environment variable + # BLOB_ACCOUNT and BLOB_KEY should be set in environment variable wasb: wasb_enabled: true @@ -118,8 +118,8 @@ feature_registry: delimiter: "__" # controls whether the type system will be initialized or not. Usually this is only required to be executed once. type_system_initialization: false - - + + secrets: azure_key_vault: name: feathrazuretest3-kv \ No newline at end of file diff --git a/feathr_project/feathrcli/data/feathr_user_workspace/mockdata/feathrazuretest3fs@feathrazuretest3storage.dfs.core.windows.net/demo_data/green_tripdata_2020-04.csv b/feathr_project/feathrcli/data/feathr_user_workspace/mockdata/feathrazuretest3fs@feathrazuretest3storage.dfs.core.windows.net/demo_data/green_tripdata_2020-04.csv deleted file mode 100644 index ce34f255a..000000000 --- a/feathr_project/feathrcli/data/feathr_user_workspace/mockdata/feathrazuretest3fs@feathrazuretest3storage.dfs.core.windows.net/demo_data/green_tripdata_2020-04.csv +++ /dev/null @@ -1,14 +0,0 @@ -VendorID,lpep_pickup_datetime,lpep_dropoff_datetime,store_and_fwd_flag,RatecodeID,PULocationID,DOLocationID,passenger_count,trip_distance,fare_amount,extra,mta_tax,tip_amount,tolls_amount,ehail_fee,improvement_surcharge,total_amount,payment_type,trip_type,congestion_surcharge -2,2021-01-01 00:15:56,2021-01-01 00:19:52,N,1,43,151,1,1.01,5.5,0.5,0.5,0,0,,0.3,6.8,2,1,0 -22,2021-01-01 11:25:59,2021-01-01 11:34:44,N,1,166,239,1,2.53,10,0.5,0.5,2.81,0,,0.3,16.86,1,1,2.75 -23,2021-01-01 00:45:57,2021-01-01 00:51:55,N,1,41,42,1,1.12,6,0.5,0.5,1,0,,0.3,8.3,1,1,0 -24,2020-12-31 23:57:51,2021-01-01 23:04:56,N,1,168,75,1,1.99,8,0.5,0.5,0,0,,0.3,9.3,2,1,0 -25,2021-01-01 17:16:36,2021-01-01 17:16:40,N,2,265,265,3,.00,-52,0,-0.5,0,0,,-0.3,-52.8,3,1,0 -12,2021-01-01 00:16:36,2021-01-01 00:16:40,N,2,265,265,3,.00,52,0,0.5,0,0,,0.3,52.8,2,1,0 -42,2021-01-01 05:19:14,2021-01-01 00:19:21,N,5,265,265,1,.00,180,0,0,36.06,0,,0.3,216.36,1,2,0 -52,2021-01-01 00:26:31,2021-01-01 00:28:50,N,1,75,75,6,.45,3.5,0.5,0.5,0.96,0,,0.3,5.76,1,1,0 -2,2021-01-01 00:57:46,2021-01-01 00:57:57,N,1,225,225,1,.00,2.5,0.5,0.5,0,0,,0.3,3.8,2,1,0 -32,2021-01-01 00:58:32,2021-01-01 01:32:34,N,1,225,265,1,12.19,38,0.5,0.5,2.75,0,,0.3,42.05,1,1,0 -2,2021-01-01 18:39:57,2021-01-01 18:55:25,N,1,74,60,1,5.48,18,0.5,0.5,0,0,,0.3,19.3,2,1,0 -15,2021-01-01 00:51:27,2021-01-01 00:57:20,N,1,42,41,2,.90,6,0.5,0.5,0,0,,0.3,7.3,1,1,0 -15,2021-01-01 00:29:05,2021-01-01 00:29:07,N,5,42,264,1,9.00E-02,10,0,0,2.06,0,,0.3,12.36,1,2,0 \ No newline at end of file diff --git a/feathr_project/feathrcli/data/feathr_user_workspace/mockdata/feathrazuretest3fs@feathrazuretest3storage.dfs.core.windows.net/product_recommendation_sample/product_detail_mock_data.csv b/feathr_project/feathrcli/data/feathr_user_workspace/mockdata/feathrazuretest3fs@feathrazuretest3storage.dfs.core.windows.net/product_recommendation_sample/product_detail_mock_data.csv deleted file mode 100644 index 476ea06f3..000000000 --- a/feathr_project/feathrcli/data/feathr_user_workspace/mockdata/feathrazuretest3fs@feathrazuretest3storage.dfs.core.windows.net/product_recommendation_sample/product_detail_mock_data.csv +++ /dev/null @@ -1,11 +0,0 @@ -product_id,category,price,quantity,recent_sold,made_in_state,discount -1,1,22,100,0,CA,7.5 -2,2,17,300,1,CA,7.5 -3,1,40,0,2,WA,7.5 -4,1,25,100,3,WA,7.5 -5,1,33,0,2,PA,0 -6,2,19,0,2,CA,7.5 -7,2,22,200,1,WA,7.5 -8,2,59,300,0,PA,8.5 -9,0,80,100,1,WA,8.5 -10,0,39,100,0,WA,7.5 \ No newline at end of file diff --git a/feathr_project/feathrcli/data/feathr_user_workspace/mockdata/feathrazuretest3fs@feathrazuretest3storage.dfs.core.windows.net/product_recommendation_sample/user_observation_mock_data.csv b/feathr_project/feathrcli/data/feathr_user_workspace/mockdata/feathrazuretest3fs@feathrazuretest3storage.dfs.core.windows.net/product_recommendation_sample/user_observation_mock_data.csv deleted file mode 100644 index 38fe25ceb..000000000 --- a/feathr_project/feathrcli/data/feathr_user_workspace/mockdata/feathrazuretest3fs@feathrazuretest3storage.dfs.core.windows.net/product_recommendation_sample/user_observation_mock_data.csv +++ /dev/null @@ -1,35 +0,0 @@ -user_id,product_id,event_timestamp,product_rating -1,1,2021-04-01,4 -1,2,2021-04-01,4 -1,3,2021-04-01,4 -1,4,2021-04-01,4 -1,5,2021-04-01,4 -2,1,2021-04-01,5 -2,2,2021-04-01,5 -2,3,2021-04-01,5 -2,4,2021-04-01,5 -2,5,2021-04-01,5 -3,1,2021-04-01,5 -3,2,2021-04-01,5 -3,3,2021-04-01,5 -3,4,2021-04-01,5 -3,5,2021-04-01,5 -4,1,2021-04-01,1 -4,2,2021-04-01,1 -4,3,2021-04-01,1 -4,4,2021-04-01,1 -4,5,2021-04-01,1 -5,1,2021-04-01,5 -5,2,2021-04-01,5 -6,1,2021-04-01,2 -7,1,2021-04-01,5 -7,2,2021-04-01,5 -7,3,2021-04-01,5 -8,1,2021-04-01,2 -8,2,2021-04-01,2 -8,3,2021-04-01,2 -9,1,2021-04-01,5 -9,2,2021-04-01,5 -9,3,2021-04-01,5 -9,4,2021-04-01,5 -10,1,2021-04-01,3 \ No newline at end of file diff --git a/feathr_project/feathrcli/data/feathr_user_workspace/mockdata/feathrazuretest3fs@feathrazuretest3storage.dfs.core.windows.net/product_recommendation_sample/user_profile_mock_data.csv b/feathr_project/feathrcli/data/feathr_user_workspace/mockdata/feathrazuretest3fs@feathrazuretest3storage.dfs.core.windows.net/product_recommendation_sample/user_profile_mock_data.csv deleted file mode 100644 index 6c38f51d7..000000000 --- a/feathr_project/feathrcli/data/feathr_user_workspace/mockdata/feathrazuretest3fs@feathrazuretest3storage.dfs.core.windows.net/product_recommendation_sample/user_profile_mock_data.csv +++ /dev/null @@ -1,11 +0,0 @@ -user_id,gender,age,gift_card_balance,number_of_credit_cards,state,tax_rate -1,1,22,100,0,CA,7.5 -2,2,17,300,1,CA,7.5 -3,1,40,0,2,WA,7.5 -4,1,25,100,3,WA,7.5 -5,1,33,0,2,PA,0 -6,2,19,0,2,CA,7.5 -7,2,22,200,1,WA,7.5 -8,2,59,300,0,PA,8.5 -9,0,80,100,1,WA,8.5 -10,0,39,100,0,WA,7.5 \ No newline at end of file diff --git a/feathr_project/feathrcli/data/feathr_user_workspace/mockdata/feathrazuretest3fs@feathrazuretest3storage.dfs.core.windows.net/product_recommendation_sample/user_purchase_history_mock_data.csv b/feathr_project/feathrcli/data/feathr_user_workspace/mockdata/feathrazuretest3fs@feathrazuretest3storage.dfs.core.windows.net/product_recommendation_sample/user_purchase_history_mock_data.csv deleted file mode 100644 index 8c8481d1f..000000000 --- a/feathr_project/feathrcli/data/feathr_user_workspace/mockdata/feathrazuretest3fs@feathrazuretest3storage.dfs.core.windows.net/product_recommendation_sample/user_purchase_history_mock_data.csv +++ /dev/null @@ -1,31 +0,0 @@ -user_id,purchase_date,purchase_amount -1,2021-01-01,0.33 -1,2021-03-03,574.35 -1,2021-01-03,796.07 -2,2021-01-04,342.15 -2,2021-03-05,280.46 -2,2021-01-06,664.18 -3,2021-01-07,359.02 -3,2021-01-08,357.12 -3,2021-01-09,845.40 -4,2021-01-10,103.92 -4,2021-02-21,670.12 -4,2021-02-12,698.65 -5,2021-01-13,110.52 -5,2021-01-14,931.72 -5,2021-02-15,388.14 -6,2021-01-16,822.96 -6,2021-01-17,292.39 -6,2021-01-18,524.76 -7,2021-01-19,262.00 -7,2021-03-20,715.94 -7,2021-01-21,345.70 -8,2021-01-22,379.00 -8,2021-01-23,194.96 -8,2021-01-24,862.33 -9,2021-01-25,430.41 -9,2021-01-26,398.72 -9,2021-02-27,158.52 -10,2021-01-28,550.01 -10,2021-03-02,157.88 -10,2021-03-03,528.43 \ No newline at end of file diff --git a/feathr_project/feathrcli/data/feathr_user_workspace/nyc_driver_demo.ipynb b/feathr_project/feathrcli/data/feathr_user_workspace/nyc_driver_demo.ipynb deleted file mode 100644 index 38cec2ca9..000000000 --- a/feathr_project/feathrcli/data/feathr_user_workspace/nyc_driver_demo.ipynb +++ /dev/null @@ -1,720 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Feathr Feature Store on Azure Demo Notebook\n", - "\n", - "This notebook illustrates the use of Feature Store to create a model that predicts NYC Taxi fares. It includes these steps:\n", - "\n", - "\n", - "This tutorial demonstrates the key capabilities of Feathr, including:\n", - "\n", - "1. Install and set up Feathr with Azure\n", - "2. Create shareable features with Feathr feature definition configs.\n", - "3. Create a training dataset via point-in-time feature join.\n", - "4. Compute and write features.\n", - "5. Train a model using these features to predict fares.\n", - "6. Materialize feature value to online store.\n", - "7. Fetch feature value in real-time from online store for online scoring.\n", - "\n", - "In this tutorial, we use Feathr Feature Store to create a model that predicts NYC Taxi fares. The dataset comes from [here](https://www1.nyc.gov/site/tlc/about/tlc-trip-record-data.page). The feature flow is as below:\n", - "\n", - "![Feature Flow](https://github.com/linkedin/feathr/blob/main/docs/images/feature_flow.png?raw=true)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Prerequisite: Use Quick Start Template to Provision Azure Resources\n", - "First step is to provision required cloud resources if you want to use Feathr. Feathr provides a python based client to interact with cloud resources.\n", - "\n", - "Please follow the steps [here](https://feathr-ai.github.io/feathr/how-to-guides/azure-deployment-arm.html) to provision required cloud resources. Due to the complexity of the possible cloud environment, it is almost impossible to create a script that works for all the use cases. Because of this, [azure_resource_provision.sh](https://github.com/linkedin/feathr/blob/main/docs/how-to-guides/azure_resource_provision.sh) is a full end to end command line to create all the required resources, and you can tailor the script as needed, while [the companion documentation](https://feathr-ai.github.io/feathr/how-to-guides/azure-deployment-cli.html) can be used as a complete guide for using that shell script.\n", - "\n", - "\n", - "![Architecture](https://github.com/linkedin/feathr/blob/main/docs/images/architecture.png?raw=true)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Prerequisite: Install Feathr \n", - "\n", - "Install Feathr using pip:\n", - "\n", - "`pip install -U feathr pandavro scikit-learn`" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Prerequisite: Configure the required environment with Feathr Quick Start Template\n", - "\n", - "In the first step (Provision cloud resources), you should have provisioned all the required cloud resources. Run the code below to install Feathr, login to Azure to get the required credentials to access more cloud resources." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "**REQUIRED STEP: Fill in the resource prefix when provisioning the resources**" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "resource_prefix = \"feathr_resource_prefix\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "! pip install feathr azure-cli pandavro scikit-learn" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Login to Azure with a device code (You will see instructions in the output):" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "! az login --use-device-code" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import glob\n", - "import os\n", - "import tempfile\n", - "from datetime import datetime, timedelta\n", - "from math import sqrt\n", - "\n", - "import pandas as pd\n", - "import pandavro as pdx\n", - "from feathr import FeathrClient\n", - "from feathr import BOOLEAN, FLOAT, INT32, ValueType\n", - "from feathr import Feature, DerivedFeature, FeatureAnchor\n", - "from feathr import BackfillTime, MaterializationSettings\n", - "from feathr import FeatureQuery, ObservationSettings\n", - "from feathr import RedisSink\n", - "from feathr import INPUT_CONTEXT, HdfsSource\n", - "from feathr import WindowAggTransformation\n", - "from feathr import TypedKey\n", - "from sklearn.metrics import mean_squared_error\n", - "from sklearn.model_selection import train_test_split\n", - "from azure.identity import DefaultAzureCredential\n", - "from azure.keyvault.secrets import SecretClient\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Get all the required credentials from Azure KeyVault" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Get all the required credentials from Azure Key Vault\n", - "key_vault_name=resource_prefix+\"kv\"\n", - "synapse_workspace_url=resource_prefix+\"syws\"\n", - "adls_account=resource_prefix+\"dls\"\n", - "adls_fs_name=resource_prefix+\"fs\"\n", - "purview_name=resource_prefix+\"purview\"\n", - "key_vault_uri = f\"https://{key_vault_name}.vault.azure.net\"\n", - "credential = DefaultAzureCredential(exclude_interactive_browser_credential=False)\n", - "client = SecretClient(vault_url=key_vault_uri, credential=credential)\n", - "secretName = \"FEATHR-ONLINE-STORE-CONN\"\n", - "retrieved_secret = client.get_secret(secretName).value\n", - "\n", - "# Get redis credentials; This is to parse Redis connection string.\n", - "redis_port=retrieved_secret.split(',')[0].split(\":\")[1]\n", - "redis_host=retrieved_secret.split(',')[0].split(\":\")[0]\n", - "redis_password=retrieved_secret.split(',')[1].split(\"password=\",1)[1]\n", - "redis_ssl=retrieved_secret.split(',')[2].split(\"ssl=\",1)[1]\n", - "\n", - "# Set the resource link\n", - "os.environ['spark_config__azure_synapse__dev_url'] = f'https://{synapse_workspace_url}.dev.azuresynapse.net'\n", - "os.environ['spark_config__azure_synapse__pool_name'] = 'spark31'\n", - "os.environ['spark_config__azure_synapse__workspace_dir'] = f'abfss://{adls_fs_name}@{adls_account}.dfs.core.windows.net/feathr_project'\n", - "os.environ['online_store__redis__host'] = redis_host\n", - "os.environ['online_store__redis__port'] = redis_port\n", - "os.environ['online_store__redis__ssl_enabled'] = redis_ssl\n", - "os.environ['REDIS_PASSWORD']=redis_password\n", - "feathr_output_path = f'abfss://{adls_fs_name}@{adls_account}.dfs.core.windows.net/feathr_output'" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Prerequisite: Configure the required environment (Don't need to update if using the above Quick Start Template)\n", - "\n", - "In the first step (Provision cloud resources), you should have provisioned all the required cloud resources. If you use Feathr CLI to create a workspace, you should have a folder with a file called `feathr_config.yaml` in it with all the required configurations. Otherwise, update the configuration below.\n", - "\n", - "The code below will write this configuration string to a temporary location and load it to Feathr. Please still refer to [feathr_config.yaml](https://github.com/linkedin/feathr/blob/main/feathr_project/feathrcli/data/feathr_user_workspace/feathr_config.yaml) and use that as the source of truth. It should also have more explanations on the meaning of each variable." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import tempfile\n", - "yaml_config = \"\"\"\n", - "# Please refer to https://github.com/linkedin/feathr/blob/main/feathr_project/feathrcli/data/feathr_user_workspace/feathr_config.yaml for explanations on the meaning of each field.\n", - "api_version: 1\n", - "project_config:\n", - " project_name: 'feathr_getting_started'\n", - " required_environment_variables:\n", - " - 'REDIS_PASSWORD'\n", - " - 'AZURE_CLIENT_ID'\n", - " - 'AZURE_TENANT_ID'\n", - " - 'AZURE_CLIENT_SECRET'\n", - "offline_store:\n", - " adls:\n", - " adls_enabled: true\n", - " wasb:\n", - " wasb_enabled: true\n", - " s3:\n", - " s3_enabled: false\n", - " s3_endpoint: 's3.amazonaws.com'\n", - " jdbc:\n", - " jdbc_enabled: false\n", - " jdbc_database: 'feathrtestdb'\n", - " jdbc_table: 'feathrtesttable'\n", - " snowflake:\n", - " url: \"dqllago-ol19457.snowflakecomputing.com\"\n", - " user: \"feathrintegration\"\n", - " role: \"ACCOUNTADMIN\"\n", - "spark_config:\n", - " spark_cluster: 'azure_synapse'\n", - " spark_result_output_parts: '1'\n", - " azure_synapse:\n", - " dev_url: 'https://feathrazuretest3synapse.dev.azuresynapse.net'\n", - " pool_name: 'spark3'\n", - " workspace_dir: 'abfss://feathrazuretest3fs@feathrazuretest3storage.dfs.core.windows.net/feathr_getting_started'\n", - " executor_size: 'Small'\n", - " executor_num: 1\n", - " databricks:\n", - " workspace_instance_url: 'https://adb-2474129336842816.16.azuredatabricks.net'\n", - " config_template: {'run_name':'','new_cluster':{'spark_version':'9.1.x-scala2.12','node_type_id':'Standard_D3_v2','num_workers':2,'spark_conf':{}},'libraries':[{'jar':''}],'spark_jar_task':{'main_class_name':'','parameters':['']}}\n", - " work_dir: 'dbfs:/feathr_getting_started'\n", - "online_store:\n", - " redis:\n", - " host: 'feathrazuretest3redis.redis.cache.windows.net'\n", - " port: 6380\n", - " ssl_enabled: True\n", - "feature_registry:\n", - " api_endpoint: \"https://feathr-sql-registry.azurewebsites.net/api/v1\"\n", - "\"\"\"\n", - "tmp = tempfile.NamedTemporaryFile(mode='w', delete=False)\n", - "with open(tmp.name, \"w\") as text_file:\n", - " text_file.write(yaml_config)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Setup necessary environment variables (Skip if using the above Quick Start Template)\n", - "\n", - "You should setup the environment variables in order to run this sample. More environment variables can be set by referring to [feathr_config.yaml](https://github.com/linkedin/feathr/blob/main/feathr_project/feathrcli/data/feathr_user_workspace/feathr_config.yaml) and use that as the source of truth. It also has more explanations on the meaning of each variable.\n", - "\n", - "To run this notebook, for Azure users, you need AZURE_CLIENT_ID, AZURE_TENANT_ID, AZURE_CLIENT_SECRET and REDIS_PASSWORD.\n", - "To run this notebook, for Databricks useres, you need DATABRICKS_WORKSPACE_TOKEN_VALUE and REDIS_PASSWORD." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Initialize Feathr Client" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "client = FeathrClient(config_path=tmp.name)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## View the data\n", - "\n", - "In this tutorial, we use Feathr Feature Store to create a model that predicts NYC Taxi fares. The dataset comes from [here](https://www1.nyc.gov/site/tlc/about/tlc-trip-record-data.page). The data is as below" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import pandas as pd\n", - "pd.read_csv(\"https://azurefeathrstorage.blob.core.windows.net/public/sample_data/green_tripdata_2020-04_with_index.csv\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Defining Features with Feathr\n", - "\n", - "In Feathr, a feature is viewed as a function, mapping from entity id or key, and timestamp to a feature value. For more details on feature definition, please refer to the [Feathr Feature Definition Guide](https://github.com/linkedin/feathr/blob/main/docs/concepts/feature-definition.md)\n", - "\n", - "\n", - "1. The typed key (a.k.a. entity id) identifies the subject of feature, e.g. a user id, 123.\n", - "2. The feature name is the aspect of the entity that the feature is indicating, e.g. the age of the user.\n", - "3. The feature value is the actual value of that aspect at a particular time, e.g. the value is 30 at year 2022.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Note that, in some cases, such as features defined on top of request data, may have no entity key or timestamp.\n", - "It is merely a function/transformation executing against request data at runtime.\n", - "For example, the day of week of the request, which is calculated by converting the request UNIX timestamp.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Define Sources Section with UDFs\n", - "A feature source is needed for anchored features that describes the raw data in which the feature values are computed from. See the python documentation to get the details on each input column.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from pyspark.sql import SparkSession, DataFrame\n", - "def feathr_udf_day_calc(df: DataFrame) -> DataFrame:\n", - " from pyspark.sql.functions import dayofweek, dayofyear, col\n", - " df = df.withColumn(\"fare_amount_cents\", col(\"fare_amount\")*100)\n", - " return df\n", - "\n", - "batch_source = HdfsSource(name=\"nycTaxiBatchSource\",\n", - " path=\"wasbs://public@azurefeathrstorage.blob.core.windows.net/sample_data/green_tripdata_2020-04_with_index.csv\",\n", - " event_timestamp_column=\"lpep_dropoff_datetime\",\n", - " preprocessing=feathr_udf_day_calc,\n", - " timestamp_format=\"yyyy-MM-dd HH:mm:ss\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Define Anchors and Features\n", - "A feature is called an anchored feature when the feature is directly extracted from the source data, rather than computed on top of other features. The latter case is called derived feature." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "f_trip_distance = Feature(name=\"f_trip_distance\",\n", - " feature_type=FLOAT, transform=\"trip_distance\")\n", - "f_trip_time_duration = Feature(name=\"f_trip_time_duration\",\n", - " feature_type=INT32,\n", - " transform=\"(to_unix_timestamp(lpep_dropoff_datetime) - to_unix_timestamp(lpep_pickup_datetime))/60\")\n", - "\n", - "features = [\n", - " f_trip_distance,\n", - " f_trip_time_duration,\n", - " Feature(name=\"f_is_long_trip_distance\",\n", - " feature_type=BOOLEAN,\n", - " transform=\"cast_float(trip_distance)>30\"),\n", - " Feature(name=\"f_day_of_week\",\n", - " feature_type=INT32,\n", - " transform=\"dayofweek(lpep_dropoff_datetime)\"),\n", - "]\n", - "\n", - "request_anchor = FeatureAnchor(name=\"request_features\",\n", - " source=INPUT_CONTEXT,\n", - " features=features)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Window aggregation features\n", - "\n", - "For window aggregation features, see the supported fields below:\n", - "\n", - "Note that the `agg_func` should be any of these:\n", - "\n", - "| Aggregation Type | Input Type | Description |\n", - "| --- | --- | --- |\n", - "|SUM, COUNT, MAX, MIN, AVG\t|Numeric|Applies the the numerical operation on the numeric inputs. |\n", - "|MAX_POOLING, MIN_POOLING, AVG_POOLING\t| Numeric Vector | Applies the max/min/avg operation on a per entry bassis for a given a collection of numbers.|\n", - "|LATEST| Any |Returns the latest not-null values from within the defined time window |\n", - "\n", - "\n", - "After you have defined features and sources, bring them together to build an anchor:\n", - "\n", - "\n", - "Note that if the data source is from the observation data, the `source` section should be `INPUT_CONTEXT` to indicate the source of those defined anchors." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "location_id = TypedKey(key_column=\"DOLocationID\",\n", - " key_column_type=ValueType.INT32,\n", - " description=\"location id in NYC\",\n", - " full_name=\"nyc_taxi.location_id\")\n", - "agg_features = [Feature(name=\"f_location_avg_fare\",\n", - " key=location_id,\n", - " feature_type=FLOAT,\n", - " transform=WindowAggTransformation(agg_expr=\"cast_float(fare_amount)\",\n", - " agg_func=\"AVG\",\n", - " window=\"90d\")),\n", - " Feature(name=\"f_location_max_fare\",\n", - " key=location_id,\n", - " feature_type=FLOAT,\n", - " transform=WindowAggTransformation(agg_expr=\"cast_float(fare_amount)\",\n", - " agg_func=\"MAX\",\n", - " window=\"90d\")),\n", - " Feature(name=\"f_location_total_fare_cents\",\n", - " key=location_id,\n", - " feature_type=FLOAT,\n", - " transform=WindowAggTransformation(agg_expr=\"fare_amount_cents\",\n", - " agg_func=\"SUM\",\n", - " window=\"90d\")),\n", - " ]\n", - "\n", - "agg_anchor = FeatureAnchor(name=\"aggregationFeatures\",\n", - " source=batch_source,\n", - " features=agg_features)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Derived Features Section\n", - "Derived features are the features that are computed from other features. They could be computed from anchored features, or other derived features." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "f_trip_time_distance = DerivedFeature(name=\"f_trip_time_distance\",\n", - " feature_type=FLOAT,\n", - " input_features=[\n", - " f_trip_distance, f_trip_time_duration],\n", - " transform=\"f_trip_distance * f_trip_time_duration\")\n", - "\n", - "f_trip_time_rounded = DerivedFeature(name=\"f_trip_time_rounded\",\n", - " feature_type=INT32,\n", - " input_features=[f_trip_time_duration],\n", - " transform=\"f_trip_time_duration % 10\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "And then we need to build those features so that it can be consumed later. Note that we have to build both the \"anchor\" and the \"derived\" features (which is not anchored to a source)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "client.build_features(anchor_list=[agg_anchor, request_anchor], derived_feature_list=[\n", - " f_trip_time_distance, f_trip_time_rounded])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Create training data using point-in-time correct feature join\n", - "\n", - "A training dataset usually contains entity id columns, multiple feature columns, event timestamp column and label/target column. \n", - "\n", - "To create a training dataset using Feathr, one needs to provide a feature join configuration file to specify\n", - "what features and how these features should be joined to the observation data. \n", - "\n", - "To learn more on this topic, please refer to [Point-in-time Correctness](https://github.com/linkedin/feathr/blob/main/docs/concepts/point-in-time-join.md)\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "if client.spark_runtime == 'databricks':\n", - " output_path = 'dbfs:/feathrazure_test.avro'\n", - "else:\n", - " output_path = feathr_output_path\n", - "\n", - "\n", - "feature_query = FeatureQuery(\n", - " feature_list=[\"f_location_avg_fare\", \"f_trip_time_rounded\", \"f_is_long_trip_distance\", \"f_location_total_fare_cents\"], key=location_id)\n", - "settings = ObservationSettings(\n", - " observation_path=\"wasbs://public@azurefeathrstorage.blob.core.windows.net/sample_data/green_tripdata_2020-04_with_index.csv\",\n", - " event_timestamp_column=\"lpep_dropoff_datetime\",\n", - " timestamp_format=\"yyyy-MM-dd HH:mm:ss\")\n", - "client.get_offline_features(observation_settings=settings,\n", - " feature_query=feature_query,\n", - " output_path=output_path)\n", - "client.wait_job_to_finish(timeout_sec=500)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Download the result and show the result\n", - "\n", - "Let's use the helper function `get_result_df` to download the result and view it:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def get_result_df(client: FeathrClient) -> pd.DataFrame:\n", - " \"\"\"Download the job result dataset from cloud as a Pandas dataframe.\"\"\"\n", - " res_url = client.get_job_result_uri(block=True, timeout_sec=600)\n", - " tmp_dir = tempfile.TemporaryDirectory()\n", - " client.feathr_spark_launcher.download_result(result_path=res_url, local_folder=tmp_dir.name)\n", - " dataframe_list = []\n", - " # assuming the result are in avro format\n", - " for file in glob.glob(os.path.join(tmp_dir.name, '*.avro')):\n", - " dataframe_list.append(pdx.read_avro(file))\n", - " vertical_concat_df = pd.concat(dataframe_list, axis=0)\n", - " tmp_dir.cleanup()\n", - " return vertical_concat_df\n", - "\n", - "df_res = get_result_df(client)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "df_res" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Train a machine learning model\n", - "After getting all the features, let's train a machine learning model with the converted feature by Feathr:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# remove columns\n", - "from sklearn.ensemble import GradientBoostingRegressor\n", - "final_df = df_res\n", - "final_df.drop([\"lpep_pickup_datetime\", \"lpep_dropoff_datetime\",\n", - " \"store_and_fwd_flag\"], axis=1, inplace=True, errors='ignore')\n", - "final_df.fillna(0, inplace=True)\n", - "final_df['fare_amount'] = final_df['fare_amount'].astype(\"float64\")\n", - "\n", - "\n", - "train_x, test_x, train_y, test_y = train_test_split(final_df.drop([\"fare_amount\"], axis=1),\n", - " final_df[\"fare_amount\"],\n", - " test_size=0.2,\n", - " random_state=42)\n", - "model = GradientBoostingRegressor()\n", - "model.fit(train_x, train_y)\n", - "\n", - "y_predict = model.predict(test_x)\n", - "\n", - "y_actual = test_y.values.flatten().tolist()\n", - "rmse = sqrt(mean_squared_error(y_actual, y_predict))\n", - "\n", - "sum_actuals = sum_errors = 0\n", - "\n", - "for actual_val, predict_val in zip(y_actual, y_predict):\n", - " abs_error = actual_val - predict_val\n", - " if abs_error < 0:\n", - " abs_error = abs_error * -1\n", - "\n", - " sum_errors = sum_errors + abs_error\n", - " sum_actuals = sum_actuals + actual_val\n", - "\n", - "mean_abs_percent_error = sum_errors / sum_actuals\n", - "print(\"Model MAPE:\")\n", - "print(mean_abs_percent_error)\n", - "print()\n", - "print(\"Model Accuracy:\")\n", - "print(1 - mean_abs_percent_error)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Materialize feature value into offline/online storage\n", - "\n", - "While Feathr can compute the feature value from the feature definition on-the-fly at request time, it can also pre-compute\n", - "and materialize the feature value to offline and/or online storage. \n", - "\n", - "We can push the generated features to the online store like below:\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "backfill_time = BackfillTime(start=datetime(\n", - " 2020, 5, 20), end=datetime(2020, 5, 20), step=timedelta(days=1))\n", - "redisSink = RedisSink(table_name=\"nycTaxiDemoFeature\")\n", - "settings = MaterializationSettings(\"nycTaxiTable\",\n", - " backfill_time=backfill_time,\n", - " sinks=[redisSink],\n", - " feature_names=[\"f_location_avg_fare\", \"f_location_max_fare\"])\n", - "\n", - "client.materialize_features(settings)\n", - "client.wait_job_to_finish(timeout_sec=500)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can then get the features from the online store (Redis):\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Fetching feature value for online inference\n", - "\n", - "For features that are already materialized by the previous step, their latest value can be queried via the client's\n", - "`get_online_features` or `multi_get_online_features` API." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "res = client.get_online_features('nycTaxiDemoFeature', '265', [\n", - " 'f_location_avg_fare', 'f_location_max_fare'])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "client.multi_get_online_features(\"nycTaxiDemoFeature\", [\"239\", \"265\"], [\n", - " 'f_location_avg_fare', 'f_location_max_fare'])\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Registering and Fetching features\n", - "\n", - "We can also register the features with an Apache Atlas compatible service, such as Azure Purview, and share the registered features across teams:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "client.register_features()\n", - "client.list_registered_features(project_name=\"feathr_getting_started\")" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3.9.5 ('base')", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.5" - }, - "vscode": { - "interpreter": { - "hash": "3d597f4c481aa0f25dceb95d2a0067e73c0966dcbd003d741d821a7208527ecf" - } - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/feathr_project/project/build.properties b/feathr_project/project/build.properties deleted file mode 100644 index c8fcab543..000000000 --- a/feathr_project/project/build.properties +++ /dev/null @@ -1 +0,0 @@ -sbt.version=1.6.2 diff --git a/feathr_project/pyproject.toml b/feathr_project/pyproject.toml index 693233dc2..be0813090 100644 --- a/feathr_project/pyproject.toml +++ b/feathr_project/pyproject.toml @@ -9,6 +9,11 @@ known_first_party = ['feathr'] force_sort_within_sections = true multi_line_output = 3 +[tool.pytest.ini_options] +markers = [ + "notebooks: Jupyter notebook tests", +] + [build-system] requires = [ "setuptools", diff --git a/feathr_project/setup.py b/feathr_project/setup.py index a3cc4ee78..a2851b2c4 100644 --- a/feathr_project/setup.py +++ b/feathr_project/setup.py @@ -3,19 +3,45 @@ from setuptools import setup, find_packages from pathlib import Path + # Use the README.md from /docs root_path = Path(__file__).resolve().parent.parent -long_description = (root_path / "docs/README.md").read_text(encoding="utf8") +readme_path = root_path / "docs/README.md" +if readme_path.exists(): + long_description = readme_path.read_text(encoding="utf8") +else: + # In some build environments (specifically in conda), we may not have the README file + # readily available. In these cases, just set long_description to the URL of README.md. + long_description = "See https://github.com/feathr-ai/feathr/blob/main/docs/README.md" try: exec(open("feathr/version.py").read()) except IOError: print("Failed to load Feathr version file for packaging.", file=sys.stderr) - sys.exit(-1) + # Temp workaround for conda build. For long term fix, Jay will need to update manifest.in file. + VERSION = "0.9.0" VERSION = __version__ # noqa -os.environ["FEATHR_VERSION"] = VERSION +os.environ["FEATHR_VERSION"] = VERSION + +extras_require=dict( + dev=[ + "black>=22.1.0", # formatter + "isort", # sort import statements + "pytest>=7", + "pytest-cov", + "pytest-xdist", + "pytest-mock>=3.8.1", + ], + notebook=[ + "jupyter==1.0.0", + "matplotlib==3.6.1", + "papermill>=2.1.2,<3", # to test run notebooks + "scrapbook>=0.5.0,<1.0.0", # to scrap notebook outputs + ], +) +extras_require["all"] = list(set(sum([*extras_require.values()], []))) setup( name='feathr', @@ -73,14 +99,7 @@ tests_require=[ # TODO: This has been depricated "pytest", ], - extras_require=dict( - dev=[ - "black>=22.1.0", # formatter - "isort", # sort import statements - "pytest>=7", - "pytest-mock>=3.8.1", - ], - ), + extras_require=extras_require, entry_points={ 'console_scripts': ['feathr=feathrcli.cli:cli'] }, diff --git a/feathr_project/test/conftest.py b/feathr_project/test/conftest.py new file mode 100644 index 000000000..c2699e871 --- /dev/null +++ b/feathr_project/test/conftest.py @@ -0,0 +1,57 @@ +from pathlib import Path +from pyspark.sql import SparkSession +import pytest + +from feathr import FeathrClient + + +def pytest_addoption(parser): + """Pytest command line argument options. + E.g. + `python -m pytest feathr_project/test/ --resource-prefix your_feathr_resource_prefix` + """ + parser.addoption( + "--config-path", + action="store", + default=str(Path(__file__).parent.resolve().joinpath("test_user_workspace", "feathr_config.yaml")), + help="Test config path", + ) + + +@pytest.fixture +def config_path(request): + return request.config.getoption("--config-path") + + +@pytest.fixture(scope="session") +def workspace_dir() -> str: + """Workspace directory path containing data files and configs for testing.""" + return str(Path(__file__).parent.resolve().joinpath("test_user_workspace")) + + +@pytest.fixture(scope="function") +def feathr_client(workspace_dir) -> FeathrClient: + """Test function-scoped Feathr client. + Note, cluster target (local, databricks, synapse) maybe overriden by the environment variables set at test machine. + """ + return FeathrClient(config_path=str(Path(workspace_dir, "feathr_config.yaml"))) + + +@pytest.fixture(scope="module") +def spark() -> SparkSession: + """Generate a spark session for tests.""" + # Set ui port other than the default one (4040) so that feathr spark job may not fail. + spark_session = ( + SparkSession.builder + .appName("tests") + .config("spark.jars.packages", ",".join([ + "org.apache.spark:spark-avro_2.12:3.3.0", + "io.delta:delta-core_2.12:2.1.1", + ])) + .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") + .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") + .config("spark.ui.port", "8080") + .getOrCreate() + ) + yield spark_session + spark_session.stop() diff --git a/feathr_project/test/samples/test_notebooks.py b/feathr_project/test/samples/test_notebooks.py new file mode 100644 index 000000000..c8d1cbefc --- /dev/null +++ b/feathr_project/test/samples/test_notebooks.py @@ -0,0 +1,54 @@ +from pathlib import Path +from tempfile import TemporaryDirectory + +import pytest +try: + import papermill as pm + import scrapbook as sb +except ImportError: + pass # disable error while collecting tests for non-notebook environments + + +SAMPLES_DIR = ( + Path(__file__) + .parent # .../samples + .parent # .../test + .parent # .../feathr_project + .parent # .../feathr (root of the repo) + .joinpath("docs", "samples") +) +NOTEBOOK_PATHS = { + "nyc_taxi_demo": str(SAMPLES_DIR.joinpath("nyc_taxi_demo.ipynb")), +} + + +@pytest.mark.notebooks +def test__nyc_taxi_demo(config_path, tmp_path): + notebook_name = "nyc_taxi_demo" + + output_tmpdir = TemporaryDirectory() + output_notebook_path = str(tmp_path.joinpath(f"{notebook_name}.ipynb")) + + print(f"Running {notebook_name} notebook as {output_notebook_path}") + + pm.execute_notebook( + input_path=NOTEBOOK_PATHS[notebook_name], + output_path=output_notebook_path, + # kernel_name="python3", + parameters=dict( + FEATHR_CONFIG_PATH=config_path, + DATA_STORE_PATH=output_tmpdir.name, + USE_CLI_AUTH=False, + REGISTER_FEATURES=False, + SCRAP_RESULTS=True, + ), + ) + + # Read results from the Scrapbook and assert expected values + nb = sb.read_notebook(output_notebook_path) + outputs = nb.scraps + + assert outputs["materialized_feature_values"].data["239"] == pytest.approx([1480., 5707.], abs=1.) + assert outputs["materialized_feature_values"].data["265"] == pytest.approx([4160., 10000.], abs=1.) + assert outputs["rmse"].data == pytest.approx(5., abs=2.) + assert outputs["mae"].data == pytest.approx(2., abs=1.) diff --git a/feathr_project/test/test_azure_kafka_e2e.py b/feathr_project/test/test_azure_kafka_e2e.py index 6c1a9b7d9..f680f695a 100644 --- a/feathr_project/test/test_azure_kafka_e2e.py +++ b/feathr_project/test/test_azure_kafka_e2e.py @@ -19,5 +19,5 @@ def test_feathr_kafa_streaming_features(): sinks=[redisSink], feature_names=['f_modified_streaming_count'] ) - client.materialize_features(settings) + client.materialize_features(settings, allow_materialize_non_agg_feature=True) client.wait_job_to_finish(timeout_sec=Constants.SPARK_JOB_TIMEOUT_SECONDS) diff --git a/feathr_project/test/test_azure_snowflake_e2e.py b/feathr_project/test/test_azure_snowflake_e2e.py index 17474ab1b..d0aba78ae 100644 --- a/feathr_project/test/test_azure_snowflake_e2e.py +++ b/feathr_project/test/test_azure_snowflake_e2e.py @@ -66,9 +66,10 @@ def test_feathr_get_offline_features(): feature_query = FeatureQuery( feature_list=['f_snowflake_call_center_division_name', 'f_snowflake_call_center_zipcode'], key=call_sk_id) + + observation_path = client.get_snowflake_path(database="SNOWFLAKE_SAMPLE_DATA",schema="TPCDS_SF10TCL",dbtable="CALL_CENTER") settings = ObservationSettings( - observation_path='jdbc:snowflake://dqllago-ol19457.snowflakecomputing.com/?user=feathrintegration&sfWarehouse' - '=COMPUTE_WH&dbtable=CALL_CENTER&sfDatabase=SNOWFLAKE_SAMPLE_DATA&sfSchema=TPCDS_SF10TCL') + observation_path=observation_path) now = datetime.now() # set output folder based on different runtime @@ -87,3 +88,15 @@ def test_feathr_get_offline_features(): res = get_result_df(client) # just assume there are results. assert res.shape[0] > 1 + +def test_client_get_snowflake_observation_path(): + """ + Test get_snowflake_path() returns correct snowflake observation path + """ + test_workspace_dir = Path(__file__).parent.resolve() / "test_user_workspace" + + + client = snowflake_test_setup(os.path.join(test_workspace_dir, "feathr_config.yaml")) + snowflake_path_actual = client.get_snowflake_path(database="DATABASE", schema="SCHEMA", dbtable="TABLE") + snowflake_path_expected = "snowflake://snowflake_account/?sfDatabase=DATABASE&sfSchema=SCHEMA&dbtable=TABLE" + assert snowflake_path_actual == snowflake_path_expected diff --git a/feathr_project/test/test_azure_spark_e2e.py b/feathr_project/test/test_azure_spark_e2e.py index e82e0efe9..bbcf6b8c1 100644 --- a/feathr_project/test/test_azure_spark_e2e.py +++ b/feathr_project/test/test_azure_spark_e2e.py @@ -55,43 +55,9 @@ def test_feathr_materialize_to_offline(): # download result and just assert the returned result is not empty # by default, it will write to a folder appended with date - res_df = get_result_df(client, "avro", output_path + "/df0/daily/2020/05/20") + res_df = get_result_df(client, data_format="avro", res_url=output_path + "/df0/daily/2020/05/20") assert res_df.shape[0] > 0 -def test_feathr_materialize_with_time_partition_pattern(): - """ - Test FeathrClient() using HdfsSource with 'timePartitionPattern'. - """ - test_workspace_dir = Path( - __file__).parent.resolve() / "test_user_workspace" - # os.chdir(test_workspace_dir) - - client: FeathrClient = time_partition_pattern_test_setup(os.path.join(test_workspace_dir, "feathr_config.yaml")) - - backfill_time = BackfillTime(start=datetime( - 2020, 5, 20), end=datetime(2020, 5, 20), step=timedelta(days=1)) - - now = datetime.now() - if client.spark_runtime == 'databricks': - output_path = ''.join(['dbfs:/feathrazure_cijob_materialize_offline_','_', str(now.minute), '_', str(now.second), ""]) - else: - output_path = ''.join(['abfss://feathrazuretest3fs@feathrazuretest3storage.dfs.core.windows.net/demo_data/feathrazure_cijob_materialize_offline_','_', str(now.minute), '_', str(now.second), ""]) - offline_sink = HdfsSink(output_path=output_path) - settings = MaterializationSettings("nycTaxiTable", - sinks=[offline_sink], - feature_names=[ - "f_location_avg_fare", "f_location_max_fare"], - backfill_time=backfill_time) - client.materialize_features(settings) - # assuming the job can successfully run; otherwise it will throw exception - client.wait_job_to_finish(timeout_sec=Constants.SPARK_JOB_TIMEOUT_SECONDS) - - # download result and just assert the returned result is not empty - # by default, it will write to a folder appended with date - res_df = get_result_df(client, "avro", output_path + "/df0/daily/2020/05/20") - assert res_df.shape[0] > 0 - - def test_feathr_online_store_agg_features(): """ Test FeathrClient() get_online_features and batch_get can get data correctly. @@ -217,7 +183,7 @@ def test_feathr_get_offline_features(): full_name="nyc_taxi.location_id") feature_query = FeatureQuery( - feature_list=["f_location_avg_fare"], key=location_id) + feature_list=["f_location_avg_fare", "f_trip_time_rounded"], key=location_id) settings = ObservationSettings( observation_path="wasbs://public@azurefeathrstorage.blob.core.windows.net/sample_data/green_tripdata_2020-04.csv", event_timestamp_column="lpep_dropoff_datetime", @@ -279,6 +245,45 @@ def test_feathr_get_offline_features_to_sql(): # assuming the job can successfully run; otherwise it will throw exception client.wait_job_to_finish(timeout_sec=Constants.SPARK_JOB_TIMEOUT_SECONDS) +@pytest.mark.skip(reason="Marked as skipped as we need to setup token and enable SQL AAD login for this test") +def test_feathr_get_offline_features_to_sql_with_token(): + """ + Test get_offline_features() can save data to SQL. + """ + # runner.invoke(init, []) + test_workspace_dir = Path( + __file__).parent.resolve() / "test_user_workspace" + client: FeathrClient = basic_test_setup(os.path.join(test_workspace_dir, "feathr_config.yaml")) + + location_id = TypedKey(key_column="DOLocationID", + key_column_type=ValueType.INT32, + description="location id in NYC", + full_name="nyc_taxi.location_id") + + feature_query = FeatureQuery( + feature_list=["f_location_avg_fare"], key=location_id) + settings = ObservationSettings( + observation_path="wasbs://public@azurefeathrstorage.blob.core.windows.net/sample_data/green_tripdata_2020-04.csv", + event_timestamp_column="lpep_dropoff_datetime", + timestamp_format="yyyy-MM-dd HH:mm:ss") + + now = datetime.now() + + # Set DB token before submitting job + # os.environ[f"SQL1_TOKEN"] = "some_token" + os.environ["SQL1_TOKEN"] = client.credential.get_token("https://management.azure.com/.default").token + output_path = JdbcSink(name="sql1", + url="jdbc:sqlserver://feathrazureci.database.windows.net:1433;database=feathrci;encrypt=true;", + dbtable=f'feathr_ci_sql_token_{str(now)[:19].replace(" ", "_").replace(":", "_").replace("-", "_")}', + auth="TOKEN") + + client.get_offline_features(observation_settings=settings, + feature_query=feature_query, + output_path=output_path) + + # assuming the job can successfully run; otherwise it will throw exception + client.wait_job_to_finish(timeout_sec=Constants.SPARK_JOB_TIMEOUT_SECONDS) + def test_feathr_materialize_to_cosmosdb(): """ Test FeathrClient() CosmosDbSink. @@ -343,9 +348,9 @@ def test_feathr_materialize_to_aerospike(): # os.chdir(test_workspace_dir) now = datetime.now() # set workspace folder by time; make sure we don't have write conflict if there are many CI tests running - os.environ['SPARK_CONFIG__DATABRICKS__WORK_DIR'] = ''.join(['dbfs:/feathrazure_cijob','_', str(now.minute), '_', str(now.second), '_', str(now.microsecond)]) - os.environ['SPARK_CONFIG__AZURE_SYNAPSE__WORKSPACE_DIR'] = ''.join(['abfss://feathrazuretest3fs@feathrazuretest3storage.dfs.core.windows.net/feathr_github_ci','_', str(now.minute), '_', str(now.second) ,'_', str(now.microsecond)]) - + os.environ['SPARK_CONFIG__DATABRICKS__WORK_DIR'] = ''.join(['dbfs:/feathrazure_cijob','_', str(now.minute), '_', str(now.second), '_', str(now.microsecond)]) + os.environ['SPARK_CONFIG__AZURE_SYNAPSE__WORKSPACE_DIR'] = ''.join(['abfss://feathrazuretest3fs@feathrazuretest3storage.dfs.core.windows.net/feathr_github_ci','_', str(now.minute), '_', str(now.second) ,'_', str(now.microsecond)]) + client = FeathrClient(config_path="feathr_config.yaml") batch_source = HdfsSource(name="nycTaxiBatchSource", path="wasbs://public@azurefeathrstorage.blob.core.windows.net/sample_data/green_tripdata_2020-04.csv", @@ -427,7 +432,67 @@ def test_feathr_materialize_to_aerospike(): client.materialize_features(settings) # assuming the job can successfully run; otherwise it will throw exception client.wait_job_to_finish(timeout_sec=Constants.SPARK_JOB_TIMEOUT_SECONDS) + +def test_feathr_materialize_with_time_partition_pattern(): + """ + Test FeathrClient() using HdfsSource with 'timePartitionPattern'. + """ + test_workspace_dir = Path( + __file__).parent.resolve() / "test_user_workspace" + # os.chdir(test_workspace_dir) + # Create data source first + client_producer: FeathrClient = basic_test_setup(os.path.join(test_workspace_dir, "feathr_config.yaml")) + + backfill_time = BackfillTime(start=datetime( + 2020, 5, 20), end=datetime(2020, 5, 20), step=timedelta(days=1)) + + if client_producer.spark_runtime == 'databricks': + output_path = 'dbfs:/timePartitionPattern_test' + else: + output_path = 'abfss://feathrazuretest3fs@feathrazuretest3storage.dfs.core.windows.net/timePartitionPattern_test' + + offline_sink = HdfsSink(output_path=output_path) + settings = MaterializationSettings("nycTaxiTable", + sinks=[offline_sink], + feature_names=[ + "f_location_avg_fare", "f_location_max_fare"], + backfill_time=backfill_time) + client_producer.materialize_features(settings) + # assuming the job can successfully run; otherwise it will throw exception + client_producer.wait_job_to_finish(timeout_sec=Constants.SPARK_JOB_TIMEOUT_SECONDS) + + # download result and just assert the returned result is not empty + # by default, it will write to a folder appended with date + res_df = get_result_df(client_producer, "avro", output_path + "/df0/daily/2020/05/20") + assert res_df.shape[0] > 0 + + client_consumer: FeathrClient = time_partition_pattern_test_setup(os.path.join(test_workspace_dir, "feathr_config.yaml"), output_path+'/df0/daily') + + backfill_time_tpp = BackfillTime(start=datetime( + 2020, 5, 20), end=datetime(2020, 5, 20), step=timedelta(days=1)) + + now = datetime.now() + if client_consumer.spark_runtime == 'databricks': + output_path_tpp = ''.join(['dbfs:/feathrazure_cijob_materialize_offline_','_', str(now.minute), '_', str(now.second), ""]) + else: + output_path_tpp = ''.join(['abfss://feathrazuretest3fs@feathrazuretest3storage.dfs.core.windows.net/demo_data/feathrazure_cijob_materialize_offline_','_', str(now.minute), '_', str(now.second), ""]) + offline_sink_tpp = HdfsSink(output_path=output_path_tpp) + settings_tpp = MaterializationSettings("nycTaxiTable", + sinks=[offline_sink_tpp], + feature_names=[ + "f_loc_avg_output", "f_loc_max_output"], + backfill_time=backfill_time_tpp) + client_consumer.materialize_features(settings_tpp, allow_materialize_non_agg_feature=True) + # assuming the job can successfully run; otherwise it will throw exception + client_consumer.wait_job_to_finish(timeout_sec=Constants.SPARK_JOB_TIMEOUT_SECONDS) + + # download result and just assert the returned result is not empty + # by default, it will write to a folder appended with date + res_df = get_result_df(client_consumer, "avro", output_path_tpp + "/df0/daily/2020/05/20") + assert res_df.shape[0] > 0 + + if __name__ == "__main__": test_feathr_materialize_to_aerospike() test_feathr_get_offline_features_to_sql() - test_feathr_materialize_to_cosmosdb() \ No newline at end of file + test_feathr_materialize_to_cosmosdb() diff --git a/feathr_project/test/test_azure_spark_maven_e2e.py b/feathr_project/test/test_azure_spark_maven_e2e.py index 6b93bb7a8..a2f214020 100644 --- a/feathr_project/test/test_azure_spark_maven_e2e.py +++ b/feathr_project/test/test_azure_spark_maven_e2e.py @@ -45,7 +45,7 @@ def test_feathr_online_store_agg_features(): if client.spark_runtime == 'databricks': output_path = ''.join(['dbfs:/feathrazure_cijob','_', str(now.minute), '_', str(now.second), ".avro"]) else: - output_path = ''.join(['abfss://xchfeathrtest4fs@xchfeathrtest4sto.dfs.core.windows.net/demo_data/output','_', str(now.minute), '_', str(now.second), ".avro"]) + output_path = ''.join(['abfss://feathrazuretest3fs@feathrazuretest3storage.dfs.core.windows.net/demo_data/output','_', str(now.minute), '_', str(now.second), ".avro"]) client.get_offline_features(observation_settings=settings, diff --git a/feathr_project/test/test_feature_anchor.py b/feathr_project/test/test_feature_anchor.py index 1fabeec67..d5e6701b8 100644 --- a/feathr_project/test/test_feature_anchor.py +++ b/feathr_project/test/test_feature_anchor.py @@ -167,60 +167,4 @@ def test_agg_anchor_to_config(): } } """ - assert ''.join(agg_anchor.to_feature_config().split()) == ''.join(expected_agg_feature_config.split()) - -def test_time_partition_to_config(): - batch_source = HdfsSource(name="testTimePartitionSource", - path="abfss://public@azurefeathrstorage.blob.core.windows.net/sample_data/time_partition_pattern", - time_partition_pattern="yyyy/MM/dd" - ) - key = TypedKey(key_column="key0", - key_column_type=ValueType.INT32) - agg_features = [ - Feature(name="f_loc_avg", - key=[key], - feature_type=FLOAT, - transform="f_location_avg_fare"), - Feature(name="f_loc_max", - feature_type=FLOAT, - key=[key], - transform="f_location_max_fare"), - ] - agg_anchor = FeatureAnchor(name="testTimePartitionFeaturesSource", - source=batch_source, - features=agg_features) - expected_time_partition_config = """ - anchors: { - testTimePartitionFeatures: { - source: testTimePartitionSource - key.sqlExpr: [key0] - features: { - f_loc_avg: { - def.sqlExpr: "f_location_avg_fare" - type: { - type: TENSOR - tensorCategory: DENSE - dimensionType: [] - valType: FLOAT - } - } - f_loc_max: { - def.sqlExpr: "f_location_max_fare" - type: { - type: TENSOR - tensorCategory: DENSE - dimensionType: [] - valType: FLOAT - } - } - } - } - } - sources: { - testTimePartitionSource: { - location: {path: "abfss://public@azurefeathrstorage.blob.core.windows.net/sample_data/time_partition_pattern"} - timePartitionPattern: "yyyy/MM/dd" - } - } - """ - assert ''.join(agg_anchor.to_feature_config().split()) == ''.join(expected_time_partition_config.split()) \ No newline at end of file + assert ''.join(agg_anchor.to_feature_config().split()) == ''.join(expected_agg_feature_config.split()) \ No newline at end of file diff --git a/feathr_project/test/test_feature_materialization.py b/feathr_project/test/test_feature_materialization.py index e8100578c..754c12ebb 100644 --- a/feathr_project/test/test_feature_materialization.py +++ b/feathr_project/test/test_feature_materialization.py @@ -18,6 +18,8 @@ from test_fixture import basic_test_setup from test_fixture import get_online_test_table_name from test_utils.constants import Constants +from logging import raiseExceptions +import pytest def test_feature_materialization_config(): backfill_time = BackfillTime(start=datetime(2020, 5, 20), end=datetime(2020, 5,20), step=timedelta(days=1)) @@ -255,4 +257,21 @@ def test_delete_feature_from_redis(): res = client.get_online_features(online_test_table, '265', ['f_location_avg_fare']) assert len(res) == 1 - assert res[0] == None \ No newline at end of file + assert res[0] == None + +def test_feature_list_on_input_context(): + with pytest.raises(RuntimeError) as e_info: + test_workspace_dir = Path(__file__).parent.resolve() / "test_user_workspace" + + client: FeathrClient = basic_test_setup(os.path.join(test_workspace_dir, "feathr_config.yaml")) + online_test_table = get_online_test_table_name('nycTaxiCITableDeletion') + redisSink = RedisSink(table_name=online_test_table) + settings = MaterializationSettings(name="py_udf", + sinks=[redisSink], + feature_names=[ + "f_location_avg_fare", + "f_day_of_week" + ]) + client.materialize_features(settings, allow_materialize_non_agg_feature=True) + assert e_info is not None + assert e_info.value.args[0] == "Materializing features that are defined on INPUT_CONTEXT is not supported. f_day_of_week is defined on INPUT_CONTEXT so you should remove it from the feature list in MaterializationSettings." \ No newline at end of file diff --git a/feathr_project/test/test_feature_registry.py b/feathr_project/test/test_feature_registry.py index 5f2fea7d4..9fe66322a 100644 --- a/feathr_project/test/test_feature_registry.py +++ b/feathr_project/test/test_feature_registry.py @@ -14,7 +14,7 @@ from feathr.registry._feathr_registry_client import _FeatureRegistry from feathrcli.cli import init from test_fixture import registry_test_setup -from test_fixture import registry_test_setup_append, registry_test_setup_partially +from test_fixture import registry_test_setup_append, registry_test_setup_partially, registry_test_setup_for_409 from test_utils.constants import Constants class FeatureRegistryTests(unittest.TestCase): @@ -58,18 +58,15 @@ def test_feathr_register_features_e2e(self): # Sync workspace from registry, will get all conf files back client.get_features_from_registry(client.project_name) - - feature_query = FeatureQuery( - feature_list=["f_location_avg_fare", "f_trip_time_rounded", "f_is_long_trip_distance"], - key=TypedKey(key_column="DOLocationID",key_column_type=ValueType.INT32)) - settings = ObservationSettings( - observation_path="wasbs://public@azurefeathrstorage.blob.core.windows.net/sample_data/green_tripdata_2020-04_with_index.csv", - event_timestamp_column="lpep_dropoff_datetime", - timestamp_format="yyyy-MM-dd HH:mm:ss") - client.get_offline_features(observation_settings=settings, - feature_query=feature_query, - output_path=output_path) - client.wait_job_to_finish(timeout_sec=Constants.SPARK_JOB_TIMEOUT_SECONDS) + + # Register the same feature with different definition and expect an error. + client: FeathrClient = registry_test_setup_for_409(os.path.join(test_workspace_dir, config_path), client.project_name) + + with pytest.raises(RuntimeError) as exc_info: + client.register_features() + + # 30 + # update this to trigger 409 conflict with the existing one + features = [ + Feature(name="f_is_long_trip_distance", + feature_type=BOOLEAN, + transform="cast_float(trip_distance)>10"), + ] + + request_anchor = FeatureAnchor(name="request_features", + source=INPUT_CONTEXT, + features=features, + registry_tags={"for_test_purpose":"true"} + ) + + client.build_features(anchor_list=[request_anchor]) + return client + def get_online_test_table_name(table_name: str): # use different time for testing to avoid write conflicts now = datetime.now() res_table = '_'.join([table_name, str(now.minute), str(now.second)]) print("The online Redis table is", res_table) - return res_table \ No newline at end of file + return res_table + +def time_partition_pattern_test_setup(config_path: str, data_source_path: str): + now = datetime.now() + # set workspace folder by time; make sure we don't have write conflict if there are many CI tests running + os.environ['SPARK_CONFIG__DATABRICKS__WORK_DIR'] = ''.join(['dbfs:/feathrazure_cijob','_', str(now.minute), '_', str(now.second), '_', str(now.microsecond)]) + os.environ['SPARK_CONFIG__AZURE_SYNAPSE__WORKSPACE_DIR'] = ''.join(['abfss://feathrazuretest3fs@feathrazuretest3storage.dfs.core.windows.net/feathr_github_ci','_', str(now.minute), '_', str(now.second) ,'_', str(now.microsecond)]) + client = FeathrClient(config_path=config_path) + + batch_source = HdfsSource(name="testTimePartitionSource", + path=data_source_path, + time_partition_pattern="yyyy/MM/dd" + ) + key = TypedKey(key_column="key0", + key_column_type=ValueType.INT32) + agg_features = [ + Feature(name="f_loc_avg_output", + key=[key], + feature_type=FLOAT, + transform="f_location_avg_fare"), + Feature(name="f_loc_max_output", + feature_type=FLOAT, + key=[key], + transform="f_location_max_fare"), + ] + + agg_anchor = FeatureAnchor(name="testTimePartitionFeatures", + source=batch_source, + features=agg_features) + client.build_features(anchor_list=[agg_anchor]) + return client \ No newline at end of file diff --git a/feathr_project/test/test_input_output_sources.py b/feathr_project/test/test_input_output_sources.py index f4af85678..ba4b3921a 100644 --- a/feathr_project/test/test_input_output_sources.py +++ b/feathr_project/test/test_input_output_sources.py @@ -10,6 +10,7 @@ from test_fixture import basic_test_setup from test_utils.constants import Constants + # test parquet file read/write without an extension name def test_feathr_get_offline_features_with_parquet(): """ @@ -38,7 +39,7 @@ def test_feathr_get_offline_features_with_parquet(): else: output_path = ''.join(['abfss://feathrazuretest3fs@feathrazuretest3storage.dfs.core.windows.net/demo_data/output','_', str(now.minute), '_', str(now.second), ".parquet"]) - + client.get_offline_features(observation_settings=settings, feature_query=feature_query, output_path=output_path, @@ -47,14 +48,12 @@ def test_feathr_get_offline_features_with_parquet(): # assuming the job can successfully run; otherwise it will throw exception client.wait_job_to_finish(timeout_sec=Constants.SPARK_JOB_TIMEOUT_SECONDS) - + # download result and just assert the returned result is not empty res_df = get_result_df(client) assert res_df.shape[0] > 0 - - # test delta lake read/write without an extension name def test_feathr_get_offline_features_with_delta_lake(): """ @@ -83,7 +82,7 @@ def test_feathr_get_offline_features_with_delta_lake(): else: output_path = ''.join(['abfss://feathrazuretest3fs@feathrazuretest3storage.dfs.core.windows.net/demo_data/output','_', str(now.minute), '_', str(now.second), "_deltalake"]) - + client.get_offline_features(observation_settings=settings, feature_query=feature_query, output_path=output_path, @@ -92,15 +91,13 @@ def test_feathr_get_offline_features_with_delta_lake(): # assuming the job can successfully run; otherwise it will throw exception client.wait_job_to_finish(timeout_sec=Constants.SPARK_JOB_TIMEOUT_SECONDS) - + # wait for a few secs for the resource to come up in the databricks API time.sleep(5) - # download result and just assert the returned result is not empty - res_df = get_result_df(client) - + # download result and just assert the returned result is not empty + # if users are using delta format in synapse, skip this check, due to issue https://github.com/delta-io/delta-rs/issues/582 result_format: str = client.get_job_tags().get(OUTPUT_FORMAT, "") if not (client.spark_runtime == 'azure_synapse' and result_format == 'delta'): - # if users are using delta format in synapse, skip this check, due to issue https://github.com/delta-io/delta-rs/issues/582 + res_df = get_result_df(client) assert res_df.shape[0] > 0 - diff --git a/feathr_project/test/test_observation_setting.py b/feathr_project/test/test_observation_setting.py index f083a2eb0..aa9cd6f72 100644 --- a/feathr_project/test/test_observation_setting.py +++ b/feathr_project/test/test_observation_setting.py @@ -23,11 +23,11 @@ def test_observation_setting_with_timestamp(): def test_observation_setting_without_timestamp(): + observation_settings = ObservationSettings( - observation_path='jdbc:snowflake://dqllago-ol19457.snowflakecomputing.com/?user=feathrintegration&sfWarehouse' - '=COMPUTE_WH&dbtable=CALL_CENTER&sfDatabase=SNOWFLAKE_SAMPLE_DATA&sfSchema=TPCDS_SF10TCL') + observation_path='snowflake://snowflake_account/?dbtable=CALL_CENTER&sfDatabase=SNOWFLAKE_SAMPLE_DATA&sfSchema=TPCDS_SF10TCL') config = observation_settings.to_feature_config() expected_config = """ - observationPath:"jdbc:snowflake://dqllago-ol19457.snowflakecomputing.com/?user=feathrintegration&sfWarehouse=COMPUTE_WH&dbtable=CALL_CENTER&sfDatabase=SNOWFLAKE_SAMPLE_DATA&sfSchema=TPCDS_SF10TCL" + observationPath:"snowflake://snowflake_account/?dbtable=CALL_CENTER&sfDatabase=SNOWFLAKE_SAMPLE_DATA&sfSchema=TPCDS_SF10TCL" """ assert ''.join(config.split()) == ''.join(expected_config.split()) \ No newline at end of file diff --git a/feathr_project/test/test_pyduf_preprocessing_e2e.py b/feathr_project/test/test_pyduf_preprocessing_e2e.py index 83ace12ea..896eb3055 100644 --- a/feathr_project/test/test_pyduf_preprocessing_e2e.py +++ b/feathr_project/test/test_pyduf_preprocessing_e2e.py @@ -10,7 +10,7 @@ from feathr import Feature from feathr import FeatureAnchor from feathr import FeatureQuery -from feathr import HdfsSource +from feathr import HdfsSource, SnowflakeSource from feathr import ObservationSettings from feathr import RedisSink from feathr import STRING, FLOAT, INT32, ValueType @@ -402,14 +402,13 @@ def test_feathr_get_offline_features_from_snowflake(): """ test_workspace_dir = Path(__file__).parent.resolve() / "test_user_workspace" client = snowflake_test_setup(os.path.join(test_workspace_dir, "feathr_config.yaml")) - batch_source = HdfsSource(name="nycTaxiBatchSource", - path="jdbc:snowflake://dqllago-ol19457.snowflakecomputing.com/?user=feathrintegration" - "&sfWarehouse=COMPUTE_WH&dbtable=CALL_CENTER&sfDatabase=SNOWFLAKE_SAMPLE_DATA" - "&sfSchema=TPCDS_SF10TCL", - preprocessing=snowflake_preprocessing, - event_timestamp_column="lpep_dropoff_datetime", - timestamp_format="yyyy-MM-dd HH:mm:ss") - + batch_source = SnowflakeSource(name="nycTaxiBatchSource", + database="SNOWFLAKE_SAMPLE_DATA", + schema="TPCDS_SF10TCL", + dbtable="CALL_CENTER", + preprocessing=snowflake_preprocessing, + event_timestamp_column="lpep_dropoff_datetime", + timestamp_format="yyyy-MM-dd HH:mm:ss") call_sk_id = TypedKey(key_column="CC_CALL_CENTER_SK", key_column_type=ValueType.STRING, description="call center sk", @@ -435,9 +434,10 @@ def test_feathr_get_offline_features_from_snowflake(): feature_query = FeatureQuery( feature_list=['f_snowflake_call_center_division_name_with_preprocessing', 'f_snowflake_call_center_zipcode_with_preprocessing'], key=call_sk_id) + + observation_path = client.get_snowflake_path(database="SNOWFLAKE_SAMPLE_DATA", schema="TPCDS_SF10TCL", dbtable="CALL_CENTER") settings = ObservationSettings( - observation_path='jdbc:snowflake://dqllago-ol19457.snowflakecomputing.com/?user=feathrintegration&sfWarehouse' - '=COMPUTE_WH&dbtable=CALL_CENTER&sfDatabase=SNOWFLAKE_SAMPLE_DATA&sfSchema=TPCDS_SF10TCL') + observation_path=observation_path) now = datetime.now() # set output folder based on different runtime diff --git a/feathr_project/test/test_user_workspace/feathr_config.yaml b/feathr_project/test/test_user_workspace/feathr_config.yaml index e67c803ef..48fbf21f7 100644 --- a/feathr_project/test/test_user_workspace/feathr_config.yaml +++ b/feathr_project/test/test_user_workspace/feathr_config.yaml @@ -25,8 +25,8 @@ project_config: # the environemnt variables are optional, however you will need them if you want to use some of the services: - ADLS_ACCOUNT - ADLS_KEY - - WASB_ACCOUNT - - WASB_KEY + - BLOB_ACCOUNT + - BLOB_KEY - S3_ACCESS_KEY - S3_SECRET_KEY - JDBC_TABLE @@ -41,7 +41,7 @@ offline_store: adls_enabled: true # paths starts with wasb:// or wasbs:// - # WASB_ACCOUNT and WASB_KEY should be set in environment variable + # BLOB_ACCOUNT and BLOB_KEY should be set in environment variable wasb: wasb_enabled: true @@ -64,6 +64,7 @@ offline_store: url: "dqllago-ol19457.snowflakecomputing.com" user: "feathrintegration" role: "ACCOUNTADMIN" + warehouse: "COMPUTE_WH" spark_config: # choice for spark runtime. Currently support: azure_synapse, databricks @@ -82,10 +83,10 @@ spark_config: # Feathr Job configuration. Support local paths, path start with http(s)://, and paths start with abfs(s):// # this is the default location so end users don't have to compile the runtime again. # feathr_runtime_location: wasbs://public@azurefeathrstorage.blob.core.windows.net/feathr-assembly-LATEST.jar - feathr_runtime_location: "../../target/scala-2.12/feathr-assembly-0.8.0.jar" + feathr_runtime_location: "../../build/libs/feathr_2.12-0.11.1-rc1.jar" databricks: # workspace instance - workspace_instance_url: 'https://adb-2474129336842816.16.azuredatabricks.net/' + workspace_instance_url: 'https://adb-4121774437039026.6.azuredatabricks.net' workspace_token_value: '' # config string including run time information, spark version, machine size, etc. # the config follows the format in the databricks documentation: https://docs.microsoft.com/en-us/azure/databricks/dev-tools/api/2.0/jobs @@ -93,7 +94,7 @@ spark_config: # Feathr Job location. Support local paths, path start with http(s)://, and paths start with dbfs:/ work_dir: 'dbfs:/feathr_getting_started' # this is the default location so end users don't have to compile the runtime again. - feathr_runtime_location: "../../target/scala-2.12/feathr-assembly-0.8.0.jar" + feathr_runtime_location: "../../build/libs/feathr_2.12-0.11.1-rc1.jar" online_store: redis: diff --git a/feathr_project/test/test_user_workspace/feathr_config_local.yaml b/feathr_project/test/test_user_workspace/feathr_config_local.yaml index a30c972da..d34844208 100644 --- a/feathr_project/test/test_user_workspace/feathr_config_local.yaml +++ b/feathr_project/test/test_user_workspace/feathr_config_local.yaml @@ -17,8 +17,8 @@ project_config: # the environemnt variables are optional, however you will need them if you want to use some of the services: - ADLS_ACCOUNT - ADLS_KEY - - WASB_ACCOUNT - - WASB_KEY + - BLOB_ACCOUNT + - BLOB_KEY - S3_ACCESS_KEY - S3_SECRET_KEY - JDBC_TABLE diff --git a/feathr_project/test/test_user_workspace/feathr_config_maven.yaml b/feathr_project/test/test_user_workspace/feathr_config_maven.yaml index c86d5b00c..b319d0edc 100644 --- a/feathr_project/test/test_user_workspace/feathr_config_maven.yaml +++ b/feathr_project/test/test_user_workspace/feathr_config_maven.yaml @@ -25,8 +25,8 @@ project_config: # the environemnt variables are optional, however you will need them if you want to use some of the services: - ADLS_ACCOUNT - ADLS_KEY - - WASB_ACCOUNT - - WASB_KEY + - BLOB_ACCOUNT + - BLOB_KEY - S3_ACCESS_KEY - S3_SECRET_KEY - JDBC_TABLE @@ -41,7 +41,7 @@ offline_store: adls_enabled: true # paths starts with wasb:// or wasbs:// - # WASB_ACCOUNT and WASB_KEY should be set in environment variable + # BLOB_ACCOUNT and BLOB_KEY should be set in environment variable wasb: wasb_enabled: true @@ -64,6 +64,7 @@ offline_store: url: "dqllago-ol19457.snowflakecomputing.com" user: "feathrintegration" role: "ACCOUNTADMIN" + warehouse: "COMPUTE_WH" spark_config: # choice for spark runtime. Currently support: azure_synapse, databricks @@ -84,7 +85,7 @@ spark_config: # this is the default location so end users don't have to compile the runtime again. # feathr_runtime_location: wasbs://public@azurefeathrstorage.blob.core.windows.net/feathr-assembly-LATEST.jar # Unset this value will use default package on Maven - # feathr_runtime_location: "../../target/scala-2.12/feathr-assembly-0.x.x.jar" # Use latest version of the jar + # feathr_runtime_location: "../../build/libs/feathr-assembly-0.x.x.jar" # Use latest version of the jar databricks: # workspace instance workspace_instance_url: 'https://adb-5638037984879289.9.azuredatabricks.net/' @@ -97,7 +98,7 @@ spark_config: # this is the default location so end users don't have to compile the runtime again. # Unset this value will use default package on Maven - # feathr_runtime_location: "../../target/scala-2.12/feathr-assembly-0.x.x.jar" (Use latest jar) + # feathr_runtime_location: "../../build/libs/feathr-assembly-0.x.x.jar" (Use latest jar) online_store: redis: diff --git a/feathr_project/test/test_user_workspace/feathr_config_registry_purview.yaml b/feathr_project/test/test_user_workspace/feathr_config_registry_purview.yaml index f716da0b4..1b7b71f75 100644 --- a/feathr_project/test/test_user_workspace/feathr_config_registry_purview.yaml +++ b/feathr_project/test/test_user_workspace/feathr_config_registry_purview.yaml @@ -25,13 +25,13 @@ spark_config: workspace_dir: 'abfss://feathrazuretest3fs@feathrazuretest3storage.dfs.core.windows.net/feathr_test_workspace' executor_size: 'Small' executor_num: 1 - feathr_runtime_location: "../../target/scala-2.12/feathr-assembly-0.8.0.jar" + feathr_runtime_location: "../../build/libs/feathr_2.12-0.11.1-rc1.jar" databricks: workspace_instance_url: 'https://adb-2474129336842816.16.azuredatabricks.net/' workspace_token_value: '' config_template: {"run_name":"FEATHR_FILL_IN","new_cluster":{"spark_version":"9.1.x-scala2.12","num_workers":1,"spark_conf":{"FEATHR_FILL_IN":"FEATHR_FILL_IN"},"instance_pool_id":"0403-214809-inlet434-pool-l9dj3kwz"},"libraries":[{"jar":"FEATHR_FILL_IN"}],"spark_jar_task":{"main_class_name":"FEATHR_FILL_IN","parameters":["FEATHR_FILL_IN"]}} work_dir: 'dbfs:/feathr_getting_started' - feathr_runtime_location: "../../target/scala-2.12/feathr-assembly-0.8.0.jar" + feathr_runtime_location: "../../build/libs/feathr_2.12-0.11.1-rc1.jar" online_store: redis: diff --git a/feathr_project/test/test_user_workspace/feathr_config_registry_purview_rbac.yaml b/feathr_project/test/test_user_workspace/feathr_config_registry_purview_rbac.yaml index c842bc702..8b698f58a 100644 --- a/feathr_project/test/test_user_workspace/feathr_config_registry_purview_rbac.yaml +++ b/feathr_project/test/test_user_workspace/feathr_config_registry_purview_rbac.yaml @@ -25,13 +25,13 @@ spark_config: workspace_dir: 'abfss://feathrazuretest3fs@feathrazuretest3storage.dfs.core.windows.net/feathr_test_workspace' executor_size: 'Small' executor_num: 1 - feathr_runtime_location: "../../target/scala-2.12/feathr-assembly-0.8.0.jar" + feathr_runtime_location: "../../build/libs/feathr_2.12-0.11.1-rc1.jar" databricks: workspace_instance_url: 'https://adb-2474129336842816.16.azuredatabricks.net/' workspace_token_value: '' config_template: {"run_name":"FEATHR_FILL_IN","new_cluster":{"spark_version":"9.1.x-scala2.12","num_workers":1,"spark_conf":{"FEATHR_FILL_IN":"FEATHR_FILL_IN"},"instance_pool_id":"0403-214809-inlet434-pool-l9dj3kwz"},"libraries":[{"jar":"FEATHR_FILL_IN"}],"spark_jar_task":{"main_class_name":"FEATHR_FILL_IN","parameters":["FEATHR_FILL_IN"]}} work_dir: 'dbfs:/feathr_getting_started' - feathr_runtime_location: "../../target/scala-2.12/feathr-assembly-0.8.0.jar" + feathr_runtime_location: "../../build/libs/feathr_2.12-0.11.1-rc1.jar" online_store: redis: diff --git a/feathr_project/test/test_user_workspace/feathr_config_registry_sql.yaml b/feathr_project/test/test_user_workspace/feathr_config_registry_sql.yaml index dcb73d827..7743fa0e0 100644 --- a/feathr_project/test/test_user_workspace/feathr_config_registry_sql.yaml +++ b/feathr_project/test/test_user_workspace/feathr_config_registry_sql.yaml @@ -25,13 +25,13 @@ spark_config: workspace_dir: 'abfss://feathrazuretest3fs@feathrazuretest3storage.dfs.core.windows.net/feathr_test_workspace' executor_size: 'Small' executor_num: 1 - feathr_runtime_location: "../../target/scala-2.12/feathr-assembly-0.8.0.jar" + feathr_runtime_location: "../../build/libs/feathr_2.12-0.11.1-rc1.jar" databricks: workspace_instance_url: 'https://adb-2474129336842816.16.azuredatabricks.net/' workspace_token_value: '' config_template: {"run_name":"FEATHR_FILL_IN","new_cluster":{"spark_version":"9.1.x-scala2.12","num_workers":1,"spark_conf":{"FEATHR_FILL_IN":"FEATHR_FILL_IN"},"instance_pool_id":"0403-214809-inlet434-pool-l9dj3kwz"},"libraries":[{"jar":"FEATHR_FILL_IN"}],"spark_jar_task":{"main_class_name":"FEATHR_FILL_IN","parameters":["FEATHR_FILL_IN"]}} work_dir: 'dbfs:/feathr_getting_started' - feathr_runtime_location: "../../target/scala-2.12/feathr-assembly-0.8.0.jar" + feathr_runtime_location: "../../build/libs/feathr_2.12-0.11.1-rc1.jar" online_store: redis: diff --git a/feathr_project/test/test_user_workspace/feathr_config_registry_sql_rbac.yaml b/feathr_project/test/test_user_workspace/feathr_config_registry_sql_rbac.yaml index 29c6889e8..ed04932a6 100644 --- a/feathr_project/test/test_user_workspace/feathr_config_registry_sql_rbac.yaml +++ b/feathr_project/test/test_user_workspace/feathr_config_registry_sql_rbac.yaml @@ -25,13 +25,13 @@ spark_config: workspace_dir: 'abfss://feathrazuretest3fs@feathrazuretest3storage.dfs.core.windows.net/feathr_test_workspace' executor_size: 'Small' executor_num: 1 - feathr_runtime_location: "../../target/scala-2.12/feathr-assembly-0.8.0.jar" + feathr_runtime_location: "../../build/libs/feathr_2.12-0.11.1-rc1.jar" databricks: workspace_instance_url: 'https://adb-2474129336842816.16.azuredatabricks.net/' workspace_token_value: '' config_template: {"run_name":"FEATHR_FILL_IN","new_cluster":{"spark_version":"9.1.x-scala2.12","num_workers":1,"spark_conf":{"FEATHR_FILL_IN":"FEATHR_FILL_IN"},"instance_pool_id":"0403-214809-inlet434-pool-l9dj3kwz"},"libraries":[{"jar":"FEATHR_FILL_IN"}],"spark_jar_task":{"main_class_name":"FEATHR_FILL_IN","parameters":["FEATHR_FILL_IN"]}} work_dir: 'dbfs:/feathr_getting_started' - feathr_runtime_location: "../../target/scala-2.12/feathr-assembly-0.8.0.jar" + feathr_runtime_location: "../../build/libs/feathr_2.12-0.11.1-rc1.jar" online_store: redis: diff --git a/feathr_project/test/test_user_workspace/mock_results/output-delta/_delta_log/00000000000000000000.json b/feathr_project/test/test_user_workspace/mock_results/output-delta/_delta_log/00000000000000000000.json new file mode 100644 index 000000000..855c52b51 --- /dev/null +++ b/feathr_project/test/test_user_workspace/mock_results/output-delta/_delta_log/00000000000000000000.json @@ -0,0 +1,4 @@ +{"protocol":{"minReaderVersion":1,"minWriterVersion":2}} +{"metaData":{"id":"a3a34f62-adf4-428f-9595-dc1a0c1055e7","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"trip_id\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"VendorID\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"lpep_pickup_datetime\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"lpep_dropoff_datetime\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"store_and_fwd_flag\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"RatecodeID\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"PULocationID\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"DOLocationID\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"passenger_count\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"trip_distance\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"fare_amount\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"extra\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"mta_tax\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"tip_amount\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"tolls_amount\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"ehail_fee\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"improvement_surcharge\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"total_amount\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"payment_type\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"trip_type\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"congestion_surcharge\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1667325249843}} +{"add":{"path":"part-00000-5020f59b-ee83-45a6-a2cd-4b9a37427f86-c000.snappy.parquet","partitionValues":{},"size":6277,"modificationTime":1667325251596,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"trip_id\":\"0\",\"VendorID\":\"2.0\",\"lpep_pickup_datetime\":\"2020-04-01 00:00:23\",\"lpep_dropoff_datetime\":\"2020-04-01 00:16:13\",\"store_and_fwd_flag\":\"N\",\"RatecodeID\":\"1.0\",\"PULocationID\":\"244\",\"DOLocationID\":\"169\",\"passenger_count\":\"1.0\",\"trip_distance\":\"1.0\",\"fare_amount\":\"12.0\",\"extra\":\"0.5\",\"mta_tax\":\"0.5\",\"tip_amount\":\"0.0\",\"tolls_amount\":\"0.0\",\"improvement_surcharge\":\"0.3\",\"total_amount\":\"10.3\",\"payment_type\":\"1.0\",\"trip_type\":\"1.0\",\"congestion_surcharge\":\"0.0\"},\"maxValues\":{\"trip_id\":\"4\",\"VendorID\":\"2.0\",\"lpep_pickup_datetime\":\"2020-04-01 00:45:06\",\"lpep_dropoff_datetime\":\"2020-04-01 01:04:39\",\"store_and_fwd_flag\":\"N\",\"RatecodeID\":\"1.0\",\"PULocationID\":\"75\",\"DOLocationID\":\"41\",\"passenger_count\":\"3.0\",\"trip_distance\":\"6.79\",\"fare_amount\":\"9.0\",\"extra\":\"0.5\",\"mta_tax\":\"0.5\",\"tip_amount\":\"0.0\",\"tolls_amount\":\"0.0\",\"improvement_surcharge\":\"0.3\",\"total_amount\":\"9.3\",\"payment_type\":\"2.0\",\"trip_type\":\"1.0\",\"congestion_surcharge\":\"0.0\"},\"nullCount\":{\"trip_id\":0,\"VendorID\":0,\"lpep_pickup_datetime\":0,\"lpep_dropoff_datetime\":0,\"store_and_fwd_flag\":0,\"RatecodeID\":0,\"PULocationID\":0,\"DOLocationID\":0,\"passenger_count\":0,\"trip_distance\":0,\"fare_amount\":0,\"extra\":0,\"mta_tax\":0,\"tip_amount\":0,\"tolls_amount\":0,\"ehail_fee\":5,\"improvement_surcharge\":0,\"total_amount\":0,\"payment_type\":0,\"trip_type\":0,\"congestion_surcharge\":0}}"}} +{"commitInfo":{"timestamp":1667325251731,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"5","numOutputBytes":"6277"},"engineInfo":"Apache-Spark/3.2.2 Delta-Lake/2.1.1","txnId":"a5e436e6-dfb6-4956-9e0c-b31b883128a0"}} diff --git a/feathr_project/test/test_user_workspace/mock_results/output-delta/part-00000-5020f59b-ee83-45a6-a2cd-4b9a37427f86-c000.snappy.parquet b/feathr_project/test/test_user_workspace/mock_results/output-delta/part-00000-5020f59b-ee83-45a6-a2cd-4b9a37427f86-c000.snappy.parquet new file mode 100644 index 000000000..1d8214c42 Binary files /dev/null and b/feathr_project/test/test_user_workspace/mock_results/output-delta/part-00000-5020f59b-ee83-45a6-a2cd-4b9a37427f86-c000.snappy.parquet differ diff --git a/feathr_project/test/test_user_workspace/mock_results/output.avro/part-00000-979daf2d-d172-48cc-a65e-87a89526f97a-c000.avro b/feathr_project/test/test_user_workspace/mock_results/output.avro/part-00000-979daf2d-d172-48cc-a65e-87a89526f97a-c000.avro new file mode 100644 index 000000000..c97dec375 Binary files /dev/null and b/feathr_project/test/test_user_workspace/mock_results/output.avro/part-00000-979daf2d-d172-48cc-a65e-87a89526f97a-c000.avro differ diff --git a/feathr_project/test/test_user_workspace/mock_results/output.csv b/feathr_project/test/test_user_workspace/mock_results/output.csv new file mode 100644 index 000000000..0468eb1b6 --- /dev/null +++ b/feathr_project/test/test_user_workspace/mock_results/output.csv @@ -0,0 +1,6 @@ +trip_id,VendorID,lpep_pickup_datetime,lpep_dropoff_datetime,store_and_fwd_flag,RatecodeID,PULocationID,DOLocationID,passenger_count,trip_distance,fare_amount,extra,mta_tax,tip_amount,tolls_amount,ehail_fee,improvement_surcharge,total_amount,payment_type,trip_type,congestion_surcharge +0,2.0,2020-04-01 00:44:02,2020-04-01 00:52:23,N,1.0,42,41,1.0,1.68,8.0,0.5,0.5,0.0,0.0,,0.3,9.3,1.0,1.0,0.0 +1,2.0,2020-04-01 00:24:39,2020-04-01 00:33:06,N,1.0,244,247,2.0,1.94,9.0,0.5,0.5,0.0,0.0,,0.3,10.3,2.0,1.0,0.0 +2,2.0,2020-04-01 00:45:06,2020-04-01 00:51:13,N,1.0,244,243,3.0,1.0,6.5,0.5,0.5,0.0,0.0,,0.3,7.8,2.0,1.0,0.0 +3,2.0,2020-04-01 00:45:06,2020-04-01 01:04:39,N,1.0,244,243,2.0,2.81,12.0,0.5,0.5,0.0,0.0,,0.3,13.3,2.0,1.0,0.0 +4,2.0,2020-04-01 00:00:23,2020-04-01 00:16:13,N,1.0,75,169,1.0,6.79,21.0,0.5,0.5,0.0,0.0,,0.3,22.3,1.0,1.0,0.0 diff --git a/feathr_project/test/test_user_workspace/mock_results/output.parquet/part-00000-bfa76930-af3c-4d58-a6e6-c1050f57ab99-c000.snappy.parquet b/feathr_project/test/test_user_workspace/mock_results/output.parquet/part-00000-bfa76930-af3c-4d58-a6e6-c1050f57ab99-c000.snappy.parquet new file mode 100644 index 000000000..0e2f9d13f Binary files /dev/null and b/feathr_project/test/test_user_workspace/mock_results/output.parquet/part-00000-bfa76930-af3c-4d58-a6e6-c1050f57ab99-c000.snappy.parquet differ diff --git a/feathr_project/test/test_user_workspace/mock_results/output_dir.csv/part-00000-06dad06f-1275-434b-8d83-6b9ed6c73eab-c000.csv b/feathr_project/test/test_user_workspace/mock_results/output_dir.csv/part-00000-06dad06f-1275-434b-8d83-6b9ed6c73eab-c000.csv new file mode 100644 index 000000000..b5b08ca83 --- /dev/null +++ b/feathr_project/test/test_user_workspace/mock_results/output_dir.csv/part-00000-06dad06f-1275-434b-8d83-6b9ed6c73eab-c000.csv @@ -0,0 +1,5 @@ +0,2.0,2020-04-01 00:44:02,2020-04-01 00:52:23,N,1.0,42,41,1.0,1.68,8.0,0.5,0.5,0.0,0.0,"",0.3,9.3,1.0,1.0,0.0 +1,2.0,2020-04-01 00:24:39,2020-04-01 00:33:06,N,1.0,244,247,2.0,1.94,9.0,0.5,0.5,0.0,0.0,"",0.3,10.3,2.0,1.0,0.0 +2,2.0,2020-04-01 00:45:06,2020-04-01 00:51:13,N,1.0,244,243,3.0,1.0,6.5,0.5,0.5,0.0,0.0,"",0.3,7.8,2.0,1.0,0.0 +3,2.0,2020-04-01 00:45:06,2020-04-01 01:04:39,N,1.0,244,243,2.0,2.81,12.0,0.5,0.5,0.0,0.0,"",0.3,13.3,2.0,1.0,0.0 +4,2.0,2020-04-01 00:00:23,2020-04-01 00:16:13,N,1.0,75,169,1.0,6.79,21.0,0.5,0.5,0.0,0.0,"",0.3,22.3,1.0,1.0,0.0 diff --git a/feathr_project/test/unit/datasets/test_dataset_utils.py b/feathr_project/test/unit/datasets/test_dataset_utils.py new file mode 100644 index 000000000..2aabaa9a1 --- /dev/null +++ b/feathr_project/test/unit/datasets/test_dataset_utils.py @@ -0,0 +1,53 @@ +from pathlib import Path +from tempfile import TemporaryDirectory +from urllib.parse import urlparse + +import pytest + +from feathr.datasets.nyc_taxi import NYC_TAXI_SMALL_URL +from feathr.datasets.utils import maybe_download + + +@pytest.mark.parametrize( + # 3924447 is the nyc_taxi sample data's bytes + "expected_bytes", [3924447, None] +) +def test__maybe_download(expected_bytes: int): + """Test maybe_download utility function w/ nyc_taxi data cached at Azure blob.""" + + tmpdir = TemporaryDirectory() + dst_filepath = Path(tmpdir.name, "data.csv") + + # Assert the data is downloaded + assert maybe_download( + src_url=NYC_TAXI_SMALL_URL, + dst_filepath=str(dst_filepath), + expected_bytes=expected_bytes, + ) + + # Assert the downloaded file exists. + assert dst_filepath.is_file() + + # Assert the data is already exists and thus the function does not download + assert not maybe_download( + src_url=NYC_TAXI_SMALL_URL, + dst_filepath=str(dst_filepath), + expected_bytes=expected_bytes, + ) + + tmpdir.cleanup() + + +def test__maybe_download__raise_exception(): + """Test maby_download utility function to raise IOError when the expected bytes mismatches.""" + + tmpdir = TemporaryDirectory() + + with pytest.raises(IOError): + maybe_download( + src_url=NYC_TAXI_SMALL_URL, + dst_filepath=Path(tmpdir.name, "data.csv").resolve(), + expected_bytes=10, + ) + + tmpdir.cleanup() diff --git a/feathr_project/test/unit/datasets/test_datasets.py b/feathr_project/test/unit/datasets/test_datasets.py new file mode 100644 index 000000000..10d89c673 --- /dev/null +++ b/feathr_project/test/unit/datasets/test_datasets.py @@ -0,0 +1,97 @@ +from pathlib import Path +from unittest.mock import MagicMock + +from pyspark.sql import SparkSession +import pytest +from pytest_mock import MockerFixture + +from feathr.datasets import nyc_taxi + + +TEST_DATASET_DIR = Path(__file__).parent.parent.parent.joinpath("test_user_workspace") +NYC_TAXI_FILE_PATH = str(TEST_DATASET_DIR.joinpath("green_tripdata_2020-04_with_index.csv").resolve()) + + +@pytest.mark.parametrize( + "local_cache_path", + [ + None, # default temporary directory + NYC_TAXI_FILE_PATH, # full filepath + str(Path(NYC_TAXI_FILE_PATH).parent), # directory + ], +) +def test__nyc_taxi__get_pandas_df( + mocker: MockerFixture, + local_cache_path: str, +): + """Test if nyc_taxi.get_pandas_df returns pd.DataFrame. Also check if the proper modules are being called.""" + # Mock maybe_download and TempDirectory + mocked_maybe_download = mocker.patch("feathr.datasets.nyc_taxi.maybe_download") + mocked_tmpdir = MagicMock() + mocked_tmpdir.name = NYC_TAXI_FILE_PATH + mocked_TemporaryDirectory = mocker.patch("feathr.datasets.nyc_taxi.TemporaryDirectory", return_value=mocked_tmpdir) + + pdf = nyc_taxi.get_pandas_df(local_cache_path=local_cache_path) + assert len(pdf) == 35612 + + # Assert mock called + if local_cache_path: + mocked_TemporaryDirectory.assert_not_called() + else: + mocked_TemporaryDirectory.assert_called_once() + + # TODO check this is called w/ file extension added + mocked_maybe_download.assert_called_once_with(src_url=nyc_taxi.NYC_TAXI_SMALL_URL, dst_filepath=NYC_TAXI_FILE_PATH) + + +@pytest.mark.parametrize( + "local_cache_path", [ + NYC_TAXI_FILE_PATH, # full filepath + str(Path(NYC_TAXI_FILE_PATH).parent), # directory + ], +) +def test__nyc_taxi__get_spark_df( + spark, + mocker: MockerFixture, + local_cache_path: str, +): + """Test if nyc_taxi.get_spark_df returns spark.sql.DataFrame.""" + # Mock maybe_download + mocked_maybe_download = mocker.patch("feathr.datasets.nyc_taxi.maybe_download") + + df = nyc_taxi.get_spark_df(spark=spark, local_cache_path=local_cache_path) + assert df.count() == 35612 + + mocked_maybe_download.assert_called_once_with( + src_url=nyc_taxi.NYC_TAXI_SMALL_URL, dst_filepath=NYC_TAXI_FILE_PATH + ) + + +@pytest.mark.parametrize( + "local_cache_path", [ + NYC_TAXI_FILE_PATH, # full filepath + str(Path(NYC_TAXI_FILE_PATH).parent), # directory + ], +) +def test__nyc_taxi__get_spark_df__with_databricks( + mocker: MockerFixture, + local_cache_path: str, +): + # Mock maybe_download and spark session + mocked_maybe_download = mocker.patch("feathr.datasets.nyc_taxi.maybe_download") + mocked_is_databricks = mocker.patch("feathr.datasets.nyc_taxi.is_databricks", return_value=True) + mocked_spark = MagicMock(spec=SparkSession) + + nyc_taxi.get_spark_df(spark=mocked_spark, local_cache_path=local_cache_path) + + # Assert mock called with databricks paths + mocked_is_databricks.assert_called_once() + + expected_dst_filepath = str(Path("/dbfs", NYC_TAXI_FILE_PATH.lstrip("/"))) + mocked_maybe_download.assert_called_once_with( + src_url=nyc_taxi.NYC_TAXI_SMALL_URL, dst_filepath=expected_dst_filepath + ) + + mocked_spark.read.option.return_value.csv.assert_called_once_with( + str(Path("dbfs:", NYC_TAXI_FILE_PATH.lstrip("/"))) + ) diff --git a/feathr_project/test/unit/spark_provider/test_localspark_submission.py b/feathr_project/test/unit/spark_provider/test_localspark_submission.py index 9a9d7238b..992f2015e 100644 --- a/feathr_project/test/unit/spark_provider/test_localspark_submission.py +++ b/feathr_project/test/unit/spark_provider/test_localspark_submission.py @@ -4,6 +4,7 @@ import pytest from pytest_mock import MockerFixture +from feathr.constants import OUTPUT_PATH_TAG from feathr.spark_provider._localspark_submission import _FeathrLocalSparkJobLauncher @@ -15,9 +16,17 @@ def local_spark_job_launcher(tmp_path) -> _FeathrLocalSparkJobLauncher: ) +@pytest.mark.parametrize( + "job_tags,expected_result_uri", [ + (None, None), + ({OUTPUT_PATH_TAG: "output"}, "output"), + ] +) def test__local_spark_job_launcher__submit_feathr_job( mocker: MockerFixture, local_spark_job_launcher: _FeathrLocalSparkJobLauncher, + job_tags: Dict[str, str], + expected_result_uri: str, ): # Mock necessary components local_spark_job_launcher._init_args = MagicMock(return_value=[]) @@ -31,11 +40,16 @@ def test__local_spark_job_launcher__submit_feathr_job( job_name="unit-test", main_jar_path="", main_class_name="", + job_tags=job_tags, ) # Assert if the mocked spark process has called once mocked_spark_proc.assert_called_once() + # Assert job tags + assert local_spark_job_launcher.get_job_tags() == job_tags + assert local_spark_job_launcher.get_job_result_uri() == expected_result_uri + @pytest.mark.parametrize( "confs", [{}, {"spark.feathr.outputFormat": "parquet"}] diff --git a/feathr_project/test/unit/test_dtype.py b/feathr_project/test/unit/test_dtype.py new file mode 100644 index 000000000..eb6aaf2ce --- /dev/null +++ b/feathr_project/test/unit/test_dtype.py @@ -0,0 +1,24 @@ +import pytest +from feathr import Feature, TypedKey, ValueType, INT32 + + +def test_key_type(): + key = TypedKey(key_column="key", key_column_type=ValueType.INT32) + assert key.key_column_type == ValueType.INT32 + + with pytest.raises(KeyError): + key = TypedKey(key_column="key", key_column_type=INT32) + +def test_feature_type(): + key = TypedKey(key_column="key", key_column_type=ValueType.INT32) + + feature = Feature(name="name", + key=key, + feature_type=INT32) + + assert feature.feature_type == INT32 + + with pytest.raises(KeyError): + feature = Feature(name="name", + key=key, + feature_type=ValueType.INT32) \ No newline at end of file diff --git a/feathr_project/test/unit/utils/test_config.py b/feathr_project/test/unit/utils/test_config.py new file mode 100644 index 000000000..770980e12 --- /dev/null +++ b/feathr_project/test/unit/utils/test_config.py @@ -0,0 +1,180 @@ +from copy import deepcopy +import os +from pathlib import Path +from unittest.mock import MagicMock +import yaml + +import pytest +from pytest_mock import MockerFixture + +import feathr.utils.config +from feathr.utils.config import generate_config + + +@pytest.mark.parametrize( + "output_filepath", [None, "config.yml"], +) +def test__generate_config__output_filepath( + output_filepath: str, + tmp_path: Path, +): + resource_prefix = "test_prefix" + project_name = "test_project" + + # Use tmp_path so that the test files get cleaned up after the tests + if output_filepath: + output_filepath = str(tmp_path / output_filepath) + + config_filepath = generate_config( + resource_prefix=resource_prefix, + project_name=project_name, + output_filepath=output_filepath, + use_env_vars=False, + ) + + # Assert if the config file was generated in the specified output path. + if output_filepath: + assert output_filepath == config_filepath + + # Assert the generated config string is correct. + with open(config_filepath, "r") as f: + config = yaml.safe_load(f) + + assert config["project_config"]["project_name"] == project_name + assert config["feature_registry"]["api_endpoint"] == f"https://{resource_prefix}webapp.azurewebsites.net/api/v1" + assert config["spark_config"]["spark_cluster"] == "local" + assert config["online_store"]["redis"]["host"] == f"{resource_prefix}redis.redis.cache.windows.net" + + +@pytest.mark.parametrize( + "spark_cluster,env_key,kwargs", + [ + ("local", None, dict()), + ( + "databricks", + "DATABRICKS_WORKSPACE_TOKEN_VALUE", + dict(spark_config__databricks__workspace_instance_url="databricks_url"), + ), + ( + "azure_synapse", + "ADLS_KEY", + dict( + spark_config__azure_synapse__dev_url="synapse_url", + spark_config__azure_synapse__pool_name="pool_name", + ), + ), + ] +) +def test__generate_config__spark_cluster( + mocker: MockerFixture, + spark_cluster: str, + env_key: str, + kwargs: str, +): + """Test if spark cluster specific configs are generated without errors. + TODO - For now, this test doesn't check if the config values are correctly working with the actual Feathr client. + """ + # Mock the os.environ to return the specified env vars + mocker.patch.object(feathr.utils.config.os, "environ", {env_key: "some_value"}) + + generate_config( + resource_prefix="test_prefix", + project_name="test_project", + spark_config__spark_cluster=spark_cluster, + use_env_vars=False, + **kwargs, + ) + + +@pytest.mark.parametrize( + "adls_key,pool_name,expected_error", + [ + ("some_key", "some_name", None), + (None, "some_name", ValueError), + ("some_key", None, ValueError), + ] +) +def test__generate_config__azure_synapse_exceptions( + mocker: MockerFixture, + adls_key: str, + pool_name: str, + expected_error: Exception, +): + """Test if exceptions are raised when databricks url and token are not provided.""" + + # Either env vars or argument should yield the same result + for environ in [{"ADLS_KEY": adls_key}, { + "ADLS_KEY": adls_key, + "SPARK_CONFIG__AZURE_SYNAPSE__POOL_NAME": pool_name, + }]: + # Mock the os.environ to return the specified env vars + mocker.patch.object(feathr.utils.config.os, "environ", environ) + + # Test either using env vars or arguments + if "SPARK_CONFIG__AZURE_SYNAPSE__POOL_NAME" in environ: + kwargs = dict() + else: + kwargs = dict(spark_config__azure_synapse__pool_name=pool_name) + + if expected_error is None: + generate_config( + resource_prefix="test_prefix", + project_name="test_project", + spark_config__spark_cluster="azure_synapse", + **kwargs, + ) + else: + with pytest.raises(ValueError): + generate_config( + resource_prefix="test_prefix", + project_name="test_project", + spark_config__spark_cluster="azure_synapse", + **kwargs, + ) + + +@pytest.mark.parametrize( + "databricks_token,workspace_url,expected_error", + [ + ("some_token", "some_url", None), + (None, "some_url", ValueError), + ("some_token", None, ValueError), + ] +) +def test__generate_config__databricks_exceptions( + mocker: MockerFixture, + databricks_token: str, + workspace_url: str, + expected_error: Exception, +): + """Test if exceptions are raised when databricks url and token are not provided.""" + + # Either env vars or argument should yield the same result + for environ in [{"DATABRICKS_WORKSPACE_TOKEN_VALUE": databricks_token}, { + "DATABRICKS_WORKSPACE_TOKEN_VALUE": databricks_token, + "SPARK_CONFIG__DATABRICKS__WORKSPACE_INSTANCE_URL": workspace_url, + }]: + # Mock the os.environ to return the specified env vars + mocker.patch.object(feathr.utils.config.os, "environ", environ) + + # Test either using env vars or arguments + if "SPARK_CONFIG__DATABRICKS__WORKSPACE_INSTANCE_URL" in environ: + kwargs = dict() + else: + kwargs = dict(spark_config__databricks__workspace_instance_url=workspace_url) + + if expected_error is None: + generate_config( + resource_prefix="test_prefix", + project_name="test_project", + spark_config__spark_cluster="databricks", + **kwargs, + ) + else: + with pytest.raises(ValueError): + generate_config( + resource_prefix="test_prefix", + project_name="test_project", + spark_config__spark_cluster="databricks", + **kwargs, + ) diff --git a/feathr_project/test/unit/utils/test_job_utils.py b/feathr_project/test/unit/utils/test_job_utils.py new file mode 100644 index 000000000..4a0d835e5 --- /dev/null +++ b/feathr_project/test/unit/utils/test_job_utils.py @@ -0,0 +1,277 @@ +# TODO with, without optional args +# TODO test with no data files exception and unsupported format exception +from pathlib import Path +from typing import Type +from unittest.mock import MagicMock + +import pandas as pd +import pytest +from pytest_mock import MockerFixture +from pyspark.sql import DataFrame, SparkSession + +from feathr import FeathrClient +from feathr.constants import OUTPUT_FORMAT, OUTPUT_PATH_TAG +from feathr.utils.job_utils import ( + get_result_df, + get_result_pandas_df, + get_result_spark_df, +) + + +def test__get_result_pandas_df(mocker: MockerFixture): + """Test if the base function, get_result_df, called w/ proper args""" + mocked_get_result_df = mocker.patch("feathr.utils.job_utils.get_result_df") + client = MagicMock() + data_format = "some_data_format" + res_url = "some_res_url" + local_cache_path = "some_local_cache_path" + get_result_pandas_df(client, data_format, res_url, local_cache_path) + mocked_get_result_df.assert_called_once_with( + client=client, + data_format=data_format, + res_url=res_url, + local_cache_path=local_cache_path, + ) + + +def test__get_result_spark_df(mocker: MockerFixture): + """Test if the base function, get_result_df, called w/ proper args""" + mocked_get_result_df = mocker.patch("feathr.utils.job_utils.get_result_df") + client = MagicMock() + spark = MagicMock() + data_format = "some_data_format" + res_url = "some_res_url" + local_cache_path = "some_local_cache_path" + get_result_spark_df(spark, client, data_format, res_url, local_cache_path) + mocked_get_result_df.assert_called_once_with( + client=client, + data_format=data_format, + res_url=res_url, + local_cache_path=local_cache_path, + spark=spark, + ) + + +@pytest.mark.parametrize( + "is_databricks,spark_runtime,res_url,local_cache_path,expected_local_cache_path", [ + # For local spark results, res_url must be a local path and local_cache_path will be ignored. + (False, "local", "some_res_url", None, "some_res_url"), + (False, "local", "some_res_url", "some_local_cache_path", "some_res_url"), + # For databricks results, res_url must be a dbfs path. + # If the function is called in databricks, local_cache_path will be ignored. + (True, "databricks", "dbfs:/some_res_url", None, "/dbfs/some_res_url"), + (True, "databricks", "dbfs:/some_res_url", "some_local_cache_path", "/dbfs/some_res_url"), + (False, "databricks", "dbfs:/some_res_url", None, "mocked_temp_path"), + (False, "databricks", "dbfs:/some_res_url", "some_local_cache_path", "some_local_cache_path"), + ] +) +def test__get_result_df__with_local_cache_path( + mocker: MockerFixture, + is_databricks: bool, + spark_runtime: str, + res_url: str, + local_cache_path: str, + expected_local_cache_path: str, +): + """Test local_cache_path is used if provided""" + # Mock client + client = MagicMock() + client.spark_runtime = spark_runtime + client.feathr_spark_launcher.download_result = MagicMock() + mocked_load_files_to_pandas_df = mocker.patch("feathr.utils.job_utils._load_files_to_pandas_df") + + # Mock is_databricks + mocker.patch("feathr.utils.job_utils.is_databricks", return_value=is_databricks) + + # Mock temporary file module + mocked_named_temporary_dir = MagicMock() + mocked_named_temporary_dir.name = expected_local_cache_path + mocker.patch("feathr.utils.job_utils.TemporaryDirectory", return_value=mocked_named_temporary_dir) + + data_format = "csv" + get_result_df(client, data_format=data_format, res_url=res_url, local_cache_path=local_cache_path) + + mocked_load_files_to_pandas_df.assert_called_once_with( + dir_path=expected_local_cache_path, + data_format=data_format, + ) + + +@pytest.mark.parametrize( + "is_databricks,spark_runtime,res_url,data_format,expected_error", [ + # Test RuntimeError when the function is running at Databricks but client.spark_runtime is not databricks + (True, "local", "some_url", "some_format", RuntimeError), + (True, "azure_synapse", "some_url", "some_format", RuntimeError), + (True, "databricks", "some_url", "some_format", None), + (False, "local", "some_url", "some_format", None), + (False, "azure_synapse", "some_url", "some_format", None), + (False, "databricks", "some_url", "some_format", None), + # Test ValueError when res_url is None + (True, "databricks", None, "some_format", ValueError), + (False, "local", None, "some_format", ValueError), + (False, "azure_synapse", None, "some_format", ValueError), + (False, "databricks", None, "some_format", ValueError), + # Test ValueError when data_format is None + (True, "databricks", "some_url", None, ValueError), + (False, "local", "some_url", None, ValueError), + (False, "azure_synapse", "some_url", None, ValueError), + (False, "databricks", "some_url", None, ValueError), + ] +) +def test__get_result_df__exceptions( + mocker: MockerFixture, + is_databricks: bool, + spark_runtime: str, + res_url: str, + data_format: str, + expected_error: Type[Exception], +): + """Test exceptions""" + + # Mock is_data_bricks + mocker.patch("feathr.utils.job_utils.is_databricks", return_value=is_databricks) + + # Mock _load_files_to_pandas_df + mocker.patch("feathr.utils.job_utils._load_files_to_pandas_df") + + # Either job tags or argument should yield the same result + for job_tag in [None, {OUTPUT_FORMAT: data_format, OUTPUT_PATH_TAG: res_url}]: + # Mock client + client = MagicMock() + client.get_job_result_uri = MagicMock(return_value=res_url) + client.get_job_tags = MagicMock(return_value=job_tag) + client.spark_runtime = spark_runtime + + if expected_error is None: + get_result_df( + client=client, + res_url=None if job_tag else res_url, + data_format=None if job_tag else data_format, + ) + else: + with pytest.raises(expected_error): + get_result_df( + client=client, + res_url=None if job_tag else res_url, + data_format=None if job_tag else data_format, + ) + + +@pytest.mark.parametrize( + "data_format,output_filename,expected_count", [ + ("csv", "output.csv", 5), + ("csv", "output_dir.csv", 4), # TODO add a header to the csv file and change expected_count to 5 after fixing the bug https://github.com/feathr-ai/feathr/issues/811 + ("parquet", "output.parquet", 5), + ("avro", "output.avro", 5), + ("delta", "output-delta", 5), + ] +) +def test__get_result_df( + workspace_dir: str, + data_format: str, + output_filename: str, + expected_count: int, +): + """Test get_result_df returns pandas DataFrame""" + for spark_runtime in ["local", "databricks", "azure_synapse"]: + # Note: make sure the output file exists in the test_user_workspace + res_url = str(Path(workspace_dir, "mock_results", output_filename)) + local_cache_path = res_url + + # Mock client + client = MagicMock() + client.spark_runtime = spark_runtime + + # Mock feathr_spark_launcher.download_result + if client.spark_runtime == "databricks": + res_url = f"dbfs:/{res_url}" + if client.spark_runtime == "azure_synapse" and data_format == "delta": + # TODO currently pass the delta table test on Synapse result due to the delta table package bug. + continue + + df = get_result_df( + client=client, + data_format=data_format, + res_url=res_url, + local_cache_path=local_cache_path, + ) + assert isinstance(df, pd.DataFrame) + assert len(df) == expected_count + + +@pytest.mark.parametrize( + "data_format,output_filename,expected_count", [ + ("csv", "output.csv", 5), + ("csv", "output_dir.csv", 4), # TODO add a header to the csv file and change expected_count = 5 after fixing the bug https://github.com/feathr-ai/feathr/issues/811 + ("parquet", "output.parquet", 5), + ("avro", "output.avro", 5), + ("delta", "output-delta", 5), + ] +) +def test__get_result_df__with_spark_session( + workspace_dir: str, + spark: SparkSession, + data_format: str, + output_filename: str, + expected_count: int, +): + """Test get_result_df returns spark DataFrame""" + for spark_runtime in ["local", "databricks", "azure_synapse"]: + # Note: make sure the output file exists in the test_user_workspace + res_url = str(Path(workspace_dir, "mock_results", output_filename)) + local_cache_path = res_url + + # Mock client + client = MagicMock() + client.spark_runtime = spark_runtime + + if client.spark_runtime == "databricks": + res_url = f"dbfs:/{res_url}" + + df = get_result_df( + client=client, + data_format=data_format, + res_url=res_url, + spark=spark, + local_cache_path=local_cache_path, + ) + assert isinstance(df, DataFrame) + assert df.count() == expected_count + + +@pytest.mark.parametrize( + "format,output_filename,expected_count", [ + ("csv", "output.csv", 5), + ] +) +def test__get_result_df__arg_alias( + workspace_dir: str, + format: str, + output_filename: str, + expected_count: int, +): + """Test get_result_df returns pandas DataFrame with the argument alias `format` instead of using `data_format`""" + for spark_runtime in ["local", "databricks", "azure_synapse"]: + # Note: make sure the output file exists in the test_user_workspace + res_url = str(Path(workspace_dir, "mock_results", output_filename)) + local_cache_path = res_url + + # Mock client + client = MagicMock() + client.spark_runtime = spark_runtime + + # Mock feathr_spark_launcher.download_result + if client.spark_runtime == "databricks": + res_url = f"dbfs:/{res_url}" + if client.spark_runtime == "azure_synapse" and format == "delta": + # TODO currently pass the delta table test on Synapse result due to the delta table package bug. + continue + + df = get_result_df( + client=client, + format=format, + res_url=res_url, + local_cache_path=local_cache_path, + ) + assert isinstance(df, pd.DataFrame) + assert len(df) == expected_count diff --git a/gradle.properties b/gradle.properties new file mode 100644 index 000000000..a79d31dc3 --- /dev/null +++ b/gradle.properties @@ -0,0 +1,3 @@ +version=0.11.1-rc1 +SONATYPE_AUTOMATIC_RELEASE=true +POM_ARTIFACT_ID=feathr_2.12 diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 000000000..41d9927a4 Binary files /dev/null and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 000000000..ffed3a254 --- /dev/null +++ b/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,5 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-7.2-bin.zip +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew new file mode 100755 index 000000000..1b6c78733 --- /dev/null +++ b/gradlew @@ -0,0 +1,234 @@ +#!/bin/sh + +# +# Copyright © 2015-2021 the original authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +############################################################################## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# +############################################################################## + +# Attempt to set APP_HOME + +# Resolve links: $0 may be a link +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac +done + +APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit + +APP_NAME="Gradle" +APP_BASE_NAME=${0##*/} + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD=maximum + +warn () { + echo "$*" +} >&2 + +die () { + echo + echo "$*" + echo + exit 1 +} >&2 + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD=$JAVA_HOME/jre/sh/java + else + JAVACMD=$JAVA_HOME/bin/java + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD=java + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac +fi + +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. + +# For Cygwin or MSYS, switch paths to Windows format before running java +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + + # Now convert the arguments - kludge to limit ourselves to /bin/sh + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) + fi + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg + done +fi + +# Collect all arguments for the java command; +# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of +# shell script including quotes and variable substitutions, so put them in +# double quotes to make sure that they get re-expanded; and +# * put everything else in single quotes, so that it's not re-expanded. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + org.gradle.wrapper.GradleWrapperMain \ + "$@" + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# + +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' + +exec "$JAVACMD" "$@" diff --git a/gradlew.bat b/gradlew.bat new file mode 100644 index 000000000..107acd32c --- /dev/null +++ b/gradlew.bat @@ -0,0 +1,89 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto execute + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/project/Dependencies.scala b/project/Dependencies.scala deleted file mode 100644 index 858a1fe4c..000000000 --- a/project/Dependencies.scala +++ /dev/null @@ -1,5 +0,0 @@ -import sbt._ - -object Dependencies { - lazy val scalaTest = "org.scalatest" %% "scalatest" % "3.0.8" -} diff --git a/project/assembly.sbt b/project/assembly.sbt deleted file mode 100644 index 415991121..000000000 --- a/project/assembly.sbt +++ /dev/null @@ -1 +0,0 @@ -addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "1.1.0") \ No newline at end of file diff --git a/project/build.properties b/project/build.properties deleted file mode 100644 index c8fcab543..000000000 --- a/project/build.properties +++ /dev/null @@ -1 +0,0 @@ -sbt.version=1.6.2 diff --git a/project/plugins.sbt b/project/plugins.sbt deleted file mode 100644 index dd31cefa0..000000000 --- a/project/plugins.sbt +++ /dev/null @@ -1,33 +0,0 @@ -addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "1.1.0") - -/** - * Helps us publish the artifacts to sonatype, which in turn - * pushes to maven central. - * - * https://github.com/xerial/sbt-sonatype/releases - */ -addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "3.9.5") //https://github.com/xerial/sbt-sonatype/releases - -/** - * - * Signs all the jars, used in conjunction with sbt-sonatype. - * - * https://github.com/sbt/sbt-pgp/releases - */ -addSbtPlugin("com.github.sbt" % "sbt-pgp" % "2.1.2") - -/* -This is an sbt plugin to help automate releases to Sonatype and Maven Central from GitHub Actions. -https://github.com/sbt/sbt-ci-release -*/ -addSbtPlugin("com.github.sbt" % "sbt-ci-release" % "1.5.10") - -/** - * - * Supports more advanced dependency tree scripts - * - * ex. - * sbt dependencyTree -java-home /Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home - * https://www.baeldung.com/scala/sbt-dependency-tree - */ -addDependencyTreePlugin diff --git a/registry/access_control/api.py b/registry/access_control/api.py index 8a95d28ad..60c2a107d 100644 --- a/registry/access_control/api.py +++ b/registry/access_control/api.py @@ -1,11 +1,11 @@ import json from typing import Optional -from fastapi import APIRouter, Depends import requests +from fastapi import APIRouter, Depends, Response +from rbac import config from rbac.access import * -from rbac.models import User from rbac.db_rbac import DbRBAC -from rbac import config +from rbac.models import User router = APIRouter() rbac = DbRBAC() @@ -13,100 +13,109 @@ @router.get('/projects', name="Get a list of Project Names [No Auth Required]") -async def get_projects() -> list[str]: - response = requests.get( - url=f"{registry_url}/projects").content.decode('utf-8') - return json.loads(response) +async def get_projects(response: Response) -> list[str]: + response.status_code, res = check( + requests.get(url=f"{registry_url}/projects")) + return res @router.get('/projects/{project}', name="Get My Project [Read Access Required]") -async def get_project(project: str, access: UserAccess = Depends(project_read_access)): - response = requests.get(url=f"{registry_url}/projects/{project}", +async def get_project(project: str, response: Response, access: UserAccess = Depends(project_read_access)): + response.status_code, res = check(requests.get(url=f"{registry_url}/projects/{project}", + headers=get_api_header(access.user_name))) + return res + +@router.get("/dependent/{entity}", name="Get downstream/dependent entitites for a given entity [Read Access Required]") +def get_dependent_entities(entity: str, access: UserAccess = Depends(project_read_access)): + response = requests.get(url=f"{registry_url}/dependent/{entity}", headers=get_api_header(access.user_name)).content.decode('utf-8') return json.loads(response) - @router.get("/projects/{project}/datasources", name="Get data sources of my project [Read Access Required]") -def get_project_datasources(project: str, access: UserAccess = Depends(project_read_access)) -> list: - response = requests.get(url=f"{registry_url}/projects/{project}/datasources", - headers=get_api_header(access.user_name)).content.decode('utf-8') - return json.loads(response) +def get_project_datasources(project: str, response: Response, access: UserAccess = Depends(project_read_access)) -> list: + response.status_code, res = check(requests.get(url=f"{registry_url}/projects/{project}/datasources", + headers=get_api_header(access.user_name))) + return res @router.get("/projects/{project}/datasources/{datasource}", name="Get a single data source by datasource Id [Read Access Required]") -def get_project_datasource(project: str, datasource: str, requestor: UserAccess = Depends(project_read_access)) -> list: - response = requests.get(url=f"{registry_url}/projects/{project}/datasources/{datasource}", - headers=get_api_header(requestor.user_name)).content.decode('utf-8') - return json.loads(response) +def get_project_datasource(project: str, datasource: str, response: Response, requestor: UserAccess = Depends(project_read_access)) -> list: + response.status_code, res = check(requests.get(url=f"{registry_url}/projects/{project}/datasources/{datasource}", + headers=get_api_header(requestor.user_name))) + return res @router.get("/projects/{project}/features", name="Get features under my project [Read Access Required]") -def get_project_features(project: str, keyword: Optional[str] = None, access: UserAccess = Depends(project_read_access)) -> list: - response = requests.get(url=f"{registry_url}/projects/{project}/features", - headers=get_api_header(access.user_name)).content.decode('utf-8') - return json.loads(response) +def get_project_features(project: str, response: Response, keyword: Optional[str] = None, access: UserAccess = Depends(project_read_access)) -> list: + response.status_code, res = check(requests.get(url=f"{registry_url}/projects/{project}/features", + headers=get_api_header(access.user_name))) + return res @router.get("/features/{feature}", name="Get a single feature by feature Id [Read Access Required]") -def get_feature(feature: str, requestor: User = Depends(get_user)) -> dict: - response = requests.get(url=f"{registry_url}/features/{feature}", - headers=get_api_header(requestor.username)).content.decode('utf-8') - ret = json.loads(response) +def get_feature(feature: str, response: Response, requestor: User = Depends(get_user)) -> dict: + response.status_code, res = check(requests.get(url=f"{registry_url}/features/{feature}", + headers=get_api_header(requestor.username))) - feature_qualifiedName = ret['attributes']['qualifiedName'] + feature_qualifiedName = res['attributes']['qualifiedName'] validate_project_access_for_feature( feature_qualifiedName, requestor, AccessType.READ) - return ret + return res +@router.delete("/entity/{entity}", name="Deletes a single entity by qualified name [Write Access Required]") +def delete_entity(entity: str, access: UserAccess = Depends(project_write_access)) -> str: + requests.delete(url=f"{registry_url}/entity/{feature}", + headers=get_api_header(access.user_name)).content.decode('utf-8') @router.get("/features/{feature}/lineage", name="Get Feature Lineage [Read Access Required]") -def get_feature_lineage(feature: str, requestor: User = Depends(get_user)) -> dict: - response = requests.get(url=f"{registry_url}/features/{feature}/lineage", - headers=get_api_header(requestor.username)).content.decode('utf-8') - ret = json.loads(response) +def get_feature_lineage(feature: str, response: Response, requestor: User = Depends(get_user)) -> dict: + response.status_code, res = check(requests.get(url=f"{registry_url}/features/{feature}/lineage", + headers=get_api_header(requestor.username))) - feature_qualifiedName = ret['guidEntityMap'][feature]['attributes']['qualifiedName'] + feature_qualifiedName = res['guidEntityMap'][feature]['attributes']['qualifiedName'] validate_project_access_for_feature( feature_qualifiedName, requestor, AccessType.READ) - return ret + return res @router.post("/projects", name="Create new project with definition [Auth Required]") -def new_project(definition: dict, requestor: User = Depends(get_user)) -> dict: +def new_project(definition: dict, response: Response, requestor: User = Depends(get_user)) -> dict: rbac.init_userrole(requestor.username, definition["name"]) - response = requests.post(url=f"{registry_url}/projects", json=definition, - headers=get_api_header(requestor.username)).content.decode('utf-8') - return json.loads(response) + response.status_code, res = check(requests.post(url=f"{registry_url}/projects", json=definition, + headers=get_api_header(requestor.username))) + return res @router.post("/projects/{project}/datasources", name="Create new data source of my project [Write Access Required]") -def new_project_datasource(project: str, definition: dict, access: UserAccess = Depends(project_write_access)) -> dict: - response = requests.post(url=f"{registry_url}/projects/{project}/datasources", json=definition, headers=get_api_header( - access.user_name)).content.decode('utf-8') - return json.loads(response) +def new_project_datasource(project: str, definition: dict, response: Response, access: UserAccess = Depends(project_write_access)) -> dict: + response.status_code, res = check(requests.post(url=f"{registry_url}/projects/{project}/datasources", json=definition, headers=get_api_header( + access.user_name))) + return res @router.post("/projects/{project}/anchors", name="Create new anchors of my project [Write Access Required]") -def new_project_anchor(project: str, definition: dict, access: UserAccess = Depends(project_write_access)) -> dict: - response = requests.post(url=f"{registry_url}/projects/{project}/anchors", json=definition, headers=get_api_header( - access.user_name)).content.decode('utf-8') - return json.loads(response) +def new_project_anchor(project: str, definition: dict, response: Response, access: UserAccess = Depends(project_write_access)) -> dict: + response.status_code, res = check(requests.post(url=f"{registry_url}/projects/{project}/anchors", json=definition, headers=get_api_header( + access.user_name))) + return res @router.post("/projects/{project}/anchors/{anchor}/features", name="Create new anchor features of my project [Write Access Required]") -def new_project_anchor_feature(project: str, anchor: str, definition: dict, access: UserAccess = Depends(project_write_access)) -> dict: - response = requests.post(url=f"{registry_url}/projects/{project}/anchors/{anchor}/features", json=definition, headers=get_api_header( - access.user_name)).content.decode('utf-8') - return json.loads(response) +def new_project_anchor_feature(project: str, anchor: str, definition: dict, response: Response, access: UserAccess = Depends(project_write_access)) -> dict: + response.status_code, res = check(requests.post(url=f"{registry_url}/projects/{project}/anchors/{anchor}/features", json=definition, headers=get_api_header( + access.user_name))) + return res @router.post("/projects/{project}/derivedfeatures", name="Create new derived features of my project [Write Access Required]") -def new_project_derived_feature(project: str, definition: dict, access: UserAccess = Depends(project_write_access)) -> dict: - response = requests.post(url=f"{registry_url}/projects/{project}/derivedfeatures", - json=definition, headers=get_api_header(access.user_name)).content.decode('utf-8') - return json.loads(response) +def new_project_derived_feature(project: str, definition: dict, response: Response, access: UserAccess = Depends(project_write_access)) -> dict: + response.status_code, res = check(requests.post(url=f"{registry_url}/projects/{project}/derivedfeatures", + json=definition, headers=get_api_header(access.user_name))) + return res # Below are access control management APIs + + @router.get("/userroles", name="List all active user role records [Project Manage Access Required]") def get_userroles(requestor: User = Depends(get_user)) -> list: return rbac.list_userroles(requestor.username) @@ -118,5 +127,9 @@ def add_userrole(project: str, user: str, role: str, reason: str, access: UserAc @router.delete("/users/{user}/userroles/delete", name="Delete a user role [Project Manage Access Required]") -def delete_userrole(user: str, role: str, reason: str, access: UserAccess= Depends(project_manage_access)): +def delete_userrole(user: str, role: str, reason: str, access: UserAccess = Depends(project_manage_access)): return rbac.delete_userrole(access.project_name, user, role, reason, access.user_name) + + +def check(r): + return r.status_code, json.loads(r.content.decode("utf-8")) diff --git a/registry/access_control/rbac/access.py b/registry/access_control/rbac/access.py index a25646813..adee628c2 100644 --- a/registry/access_control/rbac/access.py +++ b/registry/access_control/rbac/access.py @@ -1,3 +1,4 @@ +from time import sleep from typing import Any, Union from uuid import UUID from fastapi import Depends, HTTPException, status @@ -23,6 +24,12 @@ def __init__(self, detail: Any = None) -> None: detail=detail, headers={"WWW-Authenticate": "Bearer"}) +class BadRequest(HTTPException): + def __init__(self, detail: Any = None) -> None: + super().__init__(status_code=status.HTTP_400_BAD_REQUEST, + detail=detail, headers={"WWW-Authenticate": "Bearer"}) + + def get_user(user: User = Depends(authorize)) -> User: return user @@ -72,13 +79,22 @@ def _get_project_name(id_or_name: Union[str, UUID]): _to_uuid(id_or_name) if id_or_name not in rbac.projects_ids: # refresh project id map if id not found - _get_projects_ids() + _get_projects_ids() + if id_or_name not in rbac.projects_ids: + # purview discovery-query api has latency, need retry to avoid new project not included issue. + # TODO: Update purview project-ids API to realtime one and remove below patch. + count = 0 + max = 5 + while id_or_name not in rbac.projects_ids and count < max: + sleep(0.5) + _get_projects_ids() + count += 1 return rbac.projects_ids[id_or_name] except KeyError: - raise RuntimeError(f"Project Id {id_or_name} not found in Registry {config.RBAC_REGISTRY_URL}") + raise BadRequest(f"Project Id {id_or_name} not found in Registry {config.RBAC_REGISTRY_URL}. Please check if the project exists or retry later.") except ValueError: + # It is a name pass - # It is a name return id_or_name @@ -88,4 +104,4 @@ def _get_projects_ids(): response = requests.get(url=f"{config.RBAC_REGISTRY_URL}/projects-ids").content.decode('utf-8') rbac.projects_ids = json.loads(response) except Exception as e: - raise RuntimeError(f"Failed to get projects ids from Registry {config.RBAC_REGISTRY_URL}, {e}") \ No newline at end of file + raise BadRequest(f"Failed to get projects ids from Registry {config.RBAC_REGISTRY_URL}, {e}") \ No newline at end of file diff --git a/registry/data-models/common/__init__.py b/registry/data-models/common/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/registry/data-models/common/models.py b/registry/data-models/common/models.py new file mode 100644 index 000000000..6cd92d21d --- /dev/null +++ b/registry/data-models/common/models.py @@ -0,0 +1,137 @@ +from pydantic import BaseModel +from typing import Dict, Optional, List, Union +import json +from enum import Enum + + +class ValueType(Enum): + """ + Type of the feature. + """ + INT = "int" + LONG = "long" + FLOAT = "float" + DOUBLE = "double" + STRING = "string" + BOOLEAN = "boolean" + BYTES = "bytes" + + +class DimensionType(Enum): + """ + Supported dimension types for tensors in Feathr. + """ + INT = "int" + LONG = "long" + STRING = "string" + BOOLEAN = "boolean" + BYTES = "bytes" + + +class TensorCategory(Enum): + """ + Supported Tensor categories in Feathr. + """ + DENSE = "dense" # Dense tensors store values in a contiguous sequential block of memory where all values are represented. + SPARSE = "sparse" # Sparse tensor represents a dataset in which most of the entries are zero. + RAGGED = "ragged" # Ragged tensors (also known as nested tensors) are similar to dense tensors but have variable-length dimensions. + + +class FeatureValueType(Enum): + """ + The high level types associated with a feature. + This represents the high level semantic types supported by early versions of feathr. + """ + BOOLEAN = "boolean" # Boolean valued feature + NUMERIC = "numeric" # Numerically valued feature + CATEGORICAL = "categorical" # Represent a feature that consists of a single category + CATEGORICAL_SET = "categorical_set" # Represent a feature that consists of multiple categories + DENSE_VECTOR = "dense_vector" # Represent a feature in vector format where the majority of the elements are non-zero + TERM_VECTOR = "term_vector" # Represent features that has string terms and numeric value + TENSOR = "tensor" # Represent tensor based features. + UNSPECIFIED = "unspecified" # Placeholder for when no types are specified + + +class Dimension(BaseModel): + """ + Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. + """ + type: DimensionType # Type of the dimension in the tensor. Each dimension can have a different type. + shape: Optional[int] # Size of the dimension in the tensor. If unset, it means the size is unknown and actual size will be determined at runtime. + + +class TensorFeatureFormat(BaseModel): + """ + Defines the format of feature data. Feature data is produced by applying transformation on source, in a Feature. + Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially + higher dimensions. + """ + tensorCategory: TensorCategory # Type of the tensor. + valueType: ValueType # Type of the value column. + dimensions: List[Dimension] # A feature data can have zero or more dimensions (columns that represent keys). + + +class FeatureType(BaseModel): + """ + Information about a featureName. It defines the type, format and default value. + Tensor is the next generation representation of the features, so using + Tensor type w TensorFeatureFormat would be preferable FeatureType. + """ + type: FeatureValueType # Defines the high level semantic type of feature. + format: Optional[TensorFeatureFormat] # Defines the format of feature data. + defaultValue: Union[bool, int, float, str, bytes] + + +class Clazz(BaseModel): + """ + Reference to a class by fully-qualified name + """ + fullyQualifiedName: str # A fully-qualified class name including paths. + + +class Function(BaseModel): + """ + Base model for all functions + """ + expression: str # Expression in str format + functionType: str # Type of function in str format, will be used in UI + + +class MvelExpression(Function): + """ + An expression in MVEL language. + """ + mvel: str # The MVEL expression + + +class UserDefinedFunction(Function): + """ + User defined function that can be used in feature extraction or derivation. + """ + clazz: Clazz # Reference to the class that implements the user defined function. + parameters: Dict[str, json] = {} # This field defines the custom parameters of the user defined function + + +class SparkSqlExpression(Function): + """ + An expression in Spark SQL. + """ + sql: str # Spark SQl expression + + +class SemanticVersion(BaseModel): + """ + A representation of a semantic version (see https://semver.org/) + """ + majorVersion: int # The major version of this version. This is the x in x.y.z. + minorVersion: int # The minor version of this version. This is the y in x.y.z + patchVersion: int # The patch version of this version. This is the z in x.y.z + metadata: Optional[str] # Optional build metadata attached to this version. + + +class FeathrModel(BaseModel): + """ + Base model for feathr entity which will be displayed in Feathr UI + """ + displayName: str # name of the entity showed on UI + typeName: str # type of entity in str format, will be displayed in UI diff --git a/registry/data-models/data-model-diagram.md b/registry/data-models/data-model-diagram.md index e43ffa0af..d612e005f 100644 --- a/registry/data-models/data-model-diagram.md +++ b/registry/data-models/data-model-diagram.md @@ -6,30 +6,186 @@ This file defines abstract backend data models diagram for feature registry. ```mermaid classDiagram - Project "1" --> "n" FeatureName : contains - Project "1" --> "n" Anchor : contains - FeatureName "1" --> "n" Feature : contains - Anchor "1" --> "n" Feature : contains - Feature <|-- AnchorFeature : extends + Project "1" --> "n" FeatureName: contains + Project "1" --> "n" Anchor: contains + FeatureName "1" --> "n" Feature: contains + FeatureName --> "Optional" SemanticVersion: contains + FeatureName --> "Optional" FeatureType: contains + Anchor "1" --> "n" AnchorFeature: contains + Anchor --> DataSource: contains + Feature <|-- AnchorFeature: extends Feature <|-- DerivedFeature: extends - Feature --> Transformation - Feature --> Transformation : contains + Feature --> Transformation: contains + Feature --> Source: contains + Transformation --> Function: contains Source <|-- DataSource: extends + DataSource --> "Optional" Clazz: contains + DataSource --> "Optional" Function: contains Source <|-- MultiFeatureSource: extends MultiFeatureSource "1" --> "1..n" FeatureSource: contains - AnchorFeature --> DataSource : contains + AnchorFeature --> DataSource: contains DerivedFeature --> MultiFeatureSource: contains - + FeathrModel <|-- Project: extends + FeathrModel <|-- FeatureName: extends + FeathrModel <|-- Anchor: extends + FeathrModel <|-- Feature: extends + FeathrModel <|-- Source: extends + Dimension --> DimensionType: contains + TensorFeatureFormat --> TensorCategory: contains + TensorFeatureFormat --> ValueType: contains + TensorFeatureFormat "1" --> "1..n" Dimension: contains + FeatureType --> FeatureValueType: contains + FeatureType --> "Optional" TensorFeatureFormat: contains + Window --> WindowTimeUnit: contains + Function <|-- MvelExpression: extends + Function <|-- UserDefinedFunction: extends + Function <|-- SparkSqlExpression: extends + SlidingWindowAggregation --> SparkSqlExpression: contains + SlidingWindowAggregation --> SlidingWindowAggregationType: contains + SlidingWindowAggregation --> Window: contains + SlidingWindowEmbeddingAggregation --> SparkSqlExpression: contains + SlidingWindowEmbeddingAggregation --> SlidingWindowEmbeddingAggregationType: contains + SlidingWindowEmbeddingAggregation --> Window: contains + SlidingWindowLatestAvailable --> SparkSqlExpression: contains + SlidingWindowLatestAvailable --> Window: contains + Function <|-- SlidingWindowAggregation: extends + Function <|-- SlidingWindowEmbeddingAggregation: extends + Function <|-- SlidingWindowLatestAvailable: extends + + class ValueType{ + <> + INT + LONG + FLOAT + DOUBLE + STRING + BOOLEAN + BYTES + } + class DimensionType{ + <> + INT + LONG + STRING + BOOLEAN + BYTES + } + class TensorCategory{ + <> + DENSE + SPARSE + RAGGED + } + class FeatureValueType{ + <> + BOOLEAN + NUMERIC + CATEGORICAL + CATEGORICAL_SET + DENSE_VECTOR + TERM_VECTOR + TENSOR + UNSPECIFIED + } + class Dimension{ + +DimensionType type + +Optional[str] shape + } + class TensorFeatureFormat{ + +TensorCategory tensorCategory + +ValueType valueType + +List[Dimension] dimensions + } + class FeatureType{ + +FeatureValueType type + +Optional[TensorFeatureFormat] format + +Union[bool, int, float, str, types] defaultValue + } + class Clazz{ + +str fullyQualifiedName + } + class Function{ + +str expression + } + class MvelExpression{ + +str mvel + } + class UserDefinedFunction{ + +str sql + } + class SemanticVersion{ + +int majorVersion + +int minorVersion + +int patchVersion + +Optional[str] metadata + } + class FeathrModel{ + +str displayName + +str typeName + } + class SlidingWindowAggregationType{ + <> + SUM + COUNT + MAX + MIN + AVG + } + class SlidingWindowEmbeddingAggregationType{ + <> + MAX_POOLING + MIN_POOLING + AVG_POOLING + } + class WindowTimeUnit{ + <> + DAY + HOUR + MINUTE + SECOND + } + class Window{ + +int size + +WindowTimeUnit unit + } + class SlidingWindowAggregation{ + +SlidingWindowAggregationType aggregationType + +Window window + +SparkSqlExpression targetColumn + +Optional[SparkSqlExpression] filter + +Optional[SparkSqlExpression] groupBy + +Optional[int] limit + } + class SlidingWindowEmbeddingAggregation{ + +SlidingWindowEmbeddingAggregationType aggregationType + +Window window + +SparkSqlExpression targetColumn + +Optional[SparkSqlExpression] filter + +Optional[SparkSqlExpression] groupBy + } + class SlidingWindowLatestAvailable{ + +Optional[Window] window + +SparkSqlExpression targetColumn + +Optional[SparkSqlExpression] filter + +Optional[SparkSqlExpression] groupBy + +Optional[int] limit + } class Source{ } class DataSource{ + +Optional[Clazz] clazz + +Optional[Function] keyFunction } class FeatureSource{ - +FeatureNameId feature_name_id + +FeatureNameId input_feature_name_id + +Optional[str] alias } class MultiFeatureSource{ +List[FeatureSource] sources } + class Transformation{ + +Function transformationFunction + } class Feature{ +FeatureId id +FeatureNameId feature_namme_id @@ -37,6 +193,7 @@ classDiagram +Transformation transformation } class AnchorFeature{ + +AnchorId anchor_id +DataSource source } class DerivedFeature{ @@ -46,6 +203,8 @@ classDiagram +FeatureNameId id +ProjectId project_id +List[FeatureId] feature_ids + +Optional[SemanticVersion] semanticVersion + +Optional[FeatureType] featureType } class Project{ +ProjectId id diff --git a/registry/data-models/models.py b/registry/data-models/models.py index c4ae31f68..c230240ab 100644 --- a/registry/data-models/models.py +++ b/registry/data-models/models.py @@ -1,12 +1,13 @@ +from registry.data-models.transformation.models import * +from registry.data-models.common.models import SemanticVersion, FeathrModel, Function +from typing import Optional from pydantic import BaseModel -from typing import List + """ This file defines abstract backend data models for feature registry. Backend data models will be used by backend API server to talk to feature registry backend. Purpose of this is to decouple backend data models from API specific data models. -For each feature registry provider/implementation, they will extend this abstract -data models and backend API. Diagram of the data models: ./data-model-diagram.md """ @@ -43,11 +44,7 @@ class ProjectId(BaseModel): id: str # id of a project -class Source(BaseModel): - """ - Source of the feature. - It defines where the feature is extracted or derived from. - """ +class Source(FeathrModel): pass @@ -56,7 +53,8 @@ class DataSource(Source): Data source of the feature. It defines the raw data source the feature is extracted from. """ - pass + clazz: Optional[Clazz] # Fully qualified Java class name for data model + keyFunction: Optional[Function] class FeatureSource(BaseModel): @@ -65,6 +63,7 @@ class FeatureSource(BaseModel): creating other derived features. """ input_feature_name_id: FeatureNameId # Input feature name Key + alias: Optional[str] # A feature's alias to be used in transformation function. class MultiFeatureSource(Source): @@ -73,7 +72,6 @@ class MultiFeatureSource(Source): It defines one to many features where the feature is derived from. """ sources: List[FeatureSource] # All source features which the feature is derived from - pass class Transformation(BaseModel): @@ -81,10 +79,10 @@ class Transformation(BaseModel): The transformation of a Feature. A transformation function represents the transformation logic to produce feature value from the source of FeatureAnchor """ - pass + transformationFunction: Function -class Feature(BaseModel): +class Feature(FeathrModel): """ Actual implementation of FeatureName. An implementation defines where a feature is extracted from (Source) and how it is computed (Transformation). @@ -100,6 +98,7 @@ class AnchorFeature(Feature): """ Feature implementation of FeatureName which anchored to a data source. """ + anchor_id: AnchorId # ID of the anchor this feature belongs to source: DataSource # Raw data source where the feature is extracted from @@ -110,7 +109,7 @@ class DerivedFeature(Feature): source: MultiFeatureSource # Source features where the feature is derived from -class FeatureName(BaseModel): +class FeatureName(FeathrModel): """ Named Feature Interface that can be backed by multiple Feature implementations across different environments accessing different sources (data lake access for batch training, @@ -122,9 +121,11 @@ class FeatureName(BaseModel): id: FeatureNameId # unique ID for FeatureName, used to extract data for current FeatureName project_id: ProjectId # ID of the project the FeatureName belongs to feature_ids: List[FeatureId] # List of ids of feature that the FeatureName has + semanticVersion: Optional[SemanticVersion] # Semantic version associated with this FeatureName + featureType: Optional[FeatureType] # Information about featureName, like feature type, format and value. -class Project(BaseModel): +class Project(FeathrModel): """ Group of FeatureNames. It can be a project the team is working on, or a namespace which related FeatureNames have. @@ -134,7 +135,7 @@ class Project(BaseModel): anchor_ids: List[AnchorId] # List of Anchor ids that the project has -class Anchor(BaseModel): +class Anchor(FeathrModel): """ Group of AnchorFeatures which anchored on same DataSource. This is mainly used by feature producer gather information about DataSource diff --git a/registry/data-models/transformation/__init__.py b/registry/data-models/transformation/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/registry/data-models/transformation/models.py b/registry/data-models/transformation/models.py new file mode 100644 index 000000000..98e7f1e5e --- /dev/null +++ b/registry/data-models/transformation/models.py @@ -0,0 +1,84 @@ +from registry.data-models.common.models import * +from typing import Optional + + +class SlidingWindowAggregationType(Enum): + """ + Represents supported types of aggregation. + """ + SUM = "sum" + COUNT = "count" + MAX = "maximum" + MIN = "minium" + AVG = "average" + + +class SlidingWindowEmbeddingAggregationType(Enum): + """ + Represents supported types for embedding aggregation. + Pooling is a sample-based discretization process. The objective is to down-sample an input + representation and reduce its dimensionality. + """ + MAX_POOLING = "max_pooling" # Max pooling is done by applying a max filter to (usually) non-overlapping subregions of the initial representation + MIN_POOLING = "min_pooling" # Min pooling is done by applying a min filter to (usually) non-overlapping subregions of the initial representation. + AVG_POOLING = "avg_pooling" # Average pooling is done by applying a average filter to (usually) non-overlapping subregions of the initial representation + + +class WindowTimeUnit(Enum): + """ + Represents a unit of time. + """ + DAY = "day" + HOUR = "hour" + MINUTE = "minute" + SECOND = "second" + + +class Window(BaseModel): + """ + Represents a time window used in sliding window algorithms. + """ + size: int # Represents the duration of the window. + unit: WindowTimeUnit + + +class SlidingWindowAggregation(Function): + """ + Sliding window aggregation produces feature data by aggregating a collection of data within a given time + interval into an aggregate value. It ensures point-in-time correctness, when joining with label data, + it looks back the configurable time window from each entry's timestamp and compute the aggregate value. + This class can be extended to support LateralView in aggregation. + """ + aggregationType: SlidingWindowAggregationType # Represents supported types of aggregation. + window: Window # Represents the time window to look back from label data's timestamp. + targetColumn: SparkSqlExpression # The target column to perform aggregation against. + filter: Optional[SparkSqlExpression] # Represents the filter statement before the aggregation. + groupBy: Optional[SparkSqlExpression] # Represents the target to be grouped by before aggregation. + limit: Optional[int] # Represents the max number of groups (with aggregation results) to return. + + +class SlidingWindowEmbeddingAggregation(Function): + """ + Sliding window embedding aggregation produces a single embedding by performing element-wise operations or + discretion on a collection of embeddings within a given time interval. It ensures point-in-time correctness, + when joining with label data, feathr looks back the configurable time window from each entry's timestamp and produce + the aggregated embedding. + """ + aggregationType: SlidingWindowEmbeddingAggregationType # Represents supported types for embedding aggregation. + window: Window # Represents the time window to look back from label data's timestamp. + targetColumn: SparkSqlExpression # The target column to perform aggregation against. + filter: Optional[SparkSqlExpression] # Represents the filter statement before the aggregation. + groupBy: Optional[SparkSqlExpression] # Represents the target to be grouped by before aggregation. + + +class SlidingWindowLatestAvailable(Function): + """ + This sliding window algorithm picks the latest available feature data from the source data. + Note the latest here means event time instead of processing time. + This class can be extended to support LateralView in aggregation. + """ + window: Optional[Window] # Represents the time window to look back from label data's timestamp. + targetColumn: SparkSqlExpression # The target column to perform aggregation against. + filter: Optional[SparkSqlExpression] # Represents the filter statement before the aggregation. + groupBy: Optional[SparkSqlExpression] # Represents the target to be grouped by before aggregation. + limit: Optional[int] # Represents the max number of groups (with aggregation results) to return. diff --git a/registry/purview-registry/api-spec.md b/registry/purview-registry/api-spec.md index d2e82a878..52172f6df 100644 --- a/registry/purview-registry/api-spec.md +++ b/registry/purview-registry/api-spec.md @@ -287,6 +287,9 @@ Get everything defined in the project Response Type: [`EntitiesAndRelationships`](#entitiesandrelationships) +### `GET /dependent/{entity}` +Gets downstream/dependent entities for given entity + ### `GET /projects/{project}/datasources` Get all sources defined in the project. @@ -320,6 +323,9 @@ Response Type: Object | entity | [`Entity`](#entity) | | | referredEntities| `map` | For compatibility, not used | +### `DELETE /entity/{entity}` +Deletes entity + ### `POST /projects` Create new project diff --git a/registry/purview-registry/main.py b/registry/purview-registry/main.py index 5d38adf74..8044a0ef8 100644 --- a/registry/purview-registry/main.py +++ b/registry/purview-registry/main.py @@ -1,11 +1,13 @@ import os +import traceback from re import sub from typing import Optional from uuid import UUID from fastapi import APIRouter, FastAPI, HTTPException +from fastapi.responses import JSONResponse from starlette.middleware.cors import CORSMiddleware from registry import * -from registry.purview_registry import PurviewRegistry +from registry.purview_registry import PreconditionError, PurviewRegistry, ConflictError from registry.models import AnchorDef, AnchorFeatureDef, DerivedFeatureDef, EntityType, ProjectDef, SourceDef, to_snake rp = "/v1" @@ -44,6 +46,56 @@ def to_camel(s): ) +def exc_to_content(e: Exception) -> dict: + content={"message": str(e)} + if os.environ.get("REGISTRY_DEBUGGING"): + content["traceback"] = "".join(traceback.TracebackException.from_exception(e).format()) + return content + +@app.exception_handler(ConflictError) +async def conflict_error_handler(_, exc: ConflictError): + return JSONResponse( + status_code=409, + content=exc_to_content(exc), + ) + +@app.exception_handler(PreconditionError) +async def precondition_error_handler(_, exc: ConflictError): + return JSONResponse( + status_code=412, + content=exc_to_content(exc), + ) + + +@app.exception_handler(ValueError) +async def value_error_handler(_, exc: ValueError): + return JSONResponse( + status_code=400, + content=exc_to_content(exc), + ) + +@app.exception_handler(TypeError) +async def type_error_handler(_, exc: ValueError): + return JSONResponse( + status_code=400, + content=exc_to_content(exc), + ) + + +@app.exception_handler(KeyError) +async def key_error_handler(_, exc: KeyError): + return JSONResponse( + status_code=404, + content=exc_to_content(exc), + ) + +@app.exception_handler(IndexError) +async def index_error_handler(_, exc: IndexError): + return JSONResponse( + status_code=404, + content=exc_to_content(exc), + ) + @router.get("/projects",tags=["Project"]) def get_projects() -> list[str]: return registry.get_projects() @@ -56,6 +108,22 @@ def get_projects_ids() -> dict: def get_projects(project: str) -> dict: return to_camel(registry.get_project(project).to_dict()) +@router.get("/dependent/{entity}") +def get_dependent_entities(entity: str) -> list: + entity_id = registry.get_entity_id(entity) + downstream_entities = registry.get_dependent_entities(entity_id) + return list([e.to_dict() for e in downstream_entities]) + +@router.delete("/entity/{entity}") +def delete_entity(entity: str): + entity_id = registry.get_entity_id(entity) + downstream_entities = registry.get_dependent_entities(entity_id) + if len(downstream_entities) > 0: + raise HTTPException( + status_code=412, detail=f"""Entity cannot be deleted as it has downstream/dependent entities. + Entities: {list([e.qualified_name for e in downstream_entities])}""" + ) + registry.delete_entity(entity_id) @router.get("/projects/{project}/datasources",tags=["Project"]) def get_project_datasources(project: str) -> list: @@ -90,7 +158,6 @@ def get_feature(feature: str) -> dict: status_code=404, detail=f"Feature {feature} not found") return to_camel(e.to_dict()) - @router.get("/features/{feature}/lineage",tags=["Feature"]) def get_feature_lineage(feature: str) -> dict: lineage = registry.get_lineage(feature) diff --git a/registry/purview-registry/registry/interface.py b/registry/purview-registry/registry/interface.py index 7559a3f27..2e60cc32d 100644 --- a/registry/purview-registry/registry/interface.py +++ b/registry/purview-registry/registry/interface.py @@ -92,3 +92,17 @@ def create_project_anchor_feature(self, project_id: UUID, anchor_id: UUID, defin @abstractmethod def create_project_derived_feature(self, project_id: UUID, definition: DerivedFeatureDef) -> UUID: pass + + @abstractmethod + def get_dependent_entities(self, entity_id: Union[str, UUID]) -> list[Entity]: + """ + Given entity id, returns list of all entities that are downstream/dependent on given entity + """ + pass + + @abstractmethod + def delete_entity(self, entity_id: Union[str, UUID]): + """ + Deletes given entity + """ + pass diff --git a/registry/purview-registry/registry/purview_registry.py b/registry/purview-registry/registry/purview_registry.py index 15a650167..97aa2f654 100644 --- a/registry/purview-registry/registry/purview_registry.py +++ b/registry/purview-registry/registry/purview_registry.py @@ -29,6 +29,13 @@ TYPEDEF_ARRAY_ANCHOR=f"array" TYPEDEF_ARRAY_DERIVED_FEATURE=f"array" TYPEDEF_ARRAY_ANCHOR_FEATURE=f"array" + +class ConflictError(Exception): + pass + +class PreconditionError(Exception): + pass + class PurviewRegistry(Registry): def __init__(self,azure_purview_name: str, registry_delimiter: str = "__", credential=None,register_types = True): self.registry_delimiter = registry_delimiter @@ -191,6 +198,35 @@ def get_lineage(self, id_or_name: Union[str, UUID]) -> EntitiesAndRelations: return EntitiesAndRelations( upstream_entities + downstream_entities, upstream_edges + downstream_edges) + + def get_dependent_entities(self, entity_id: Union[str, UUID]) -> list[Entity]: + """ + Given entity id, returns list of all entities that are downstream/dependent on given entity + """ + entity_id = self.get_entity_id(entity_id) + entity = self.get_entity(entity_id) + downstream_entities = [] + if entity.entity_type == EntityType.Project: + downstream_entities, _ = self._bfs(entity_id, RelationshipType.Contains) + if entity.entity_type == EntityType.Source: + downstream_entities, _ = self._bfs(entity_id, RelationshipType.Produces) + if entity.entity_type == EntityType.Anchor: + downstream_entities, _ = self._bfs(entity_id, RelationshipType.Contains) + if entity.entity_type in (EntityType.AnchorFeature, EntityType.DerivedFeature): + downstream_entities, _ = self._bfs(entity_id, RelationshipType.Produces) + return [e for e in downstream_entities if str(e.id) != str(entity_id)] + + def delete_entity(self, entity_id: Union[str, UUID]): + """ + Deletes given entity + """ + entity_id = self.get_entity_id(entity_id) + neighbors = self.get_all_neighbours(entity_id) + edge_guids = [str(x.id) for x in neighbors] + # Delete all edges associated with entity + self.purview_client.delete_entity(edge_guids) + #Delete entity + self.purview_client.delete_entity(str(entity_id)) def _get_edges(self, ids: list[UUID]) -> list[Edge]: all_edges = set() @@ -201,7 +237,7 @@ def _get_edges(self, ids: list[UUID]) -> list[Edge]: and neighbour.to_id in ids: all_edges.add(neighbour) return list(all_edges) - + def _create_edge_from_process(self, name:str, guid: str) -> Edge: names = name.split(self.registry_delimiter) return Edge(guid, names[1], names[2], RelationshipType.new(names[0])) @@ -583,13 +619,12 @@ def _upload_single_entity(self, entity:AtlasEntity): """ Try to find existing entity/process first, if found, return the existing entity's GUID """ - id = self.get_entity_id(entity.qualifiedName) - response = self.purview_client.get_entity(id)['entities'][0] + response = self.purview_client.get_entity(qualifiedName=entity.qualifiedName, typeName=entity.typeName)['entities'][0] j = entity.to_json() if j["typeName"] == response["typeName"]: if j["typeName"] == "Process": if response["attributes"]["qualifiedName"] != j["attributes"]["qualifiedName"]: - raise RuntimeError("The requested entity %s conflicts with the existing entity in PurView" % j["attributes"]["qualifiedName"]) + raise ConflictError("The requested entity %s conflicts with the existing entity in PurView" % j["attributes"]["qualifiedName"]) else: if "type" in response['attributes'] and response["typeName"] in (TYPEDEF_ANCHOR_FEATURE, TYPEDEF_DERIVED_FEATURE): conf = ConfigFactory.parse_string(response['attributes']['type']) @@ -598,23 +633,30 @@ def _upload_single_entity(self, entity:AtlasEntity): keys.add("qualifiedName") for k in keys: if response["attributes"][k] != j["attributes"][k]: - raise RuntimeError("The requested entity %s conflicts with the existing entity in PurView" % j["attributes"]["qualifiedName"]) + raise ConflictError("The requested entity %s conflicts with the existing entity in PurView" % j["attributes"]["qualifiedName"]) entity.guid = response["guid"] return else: - raise RuntimeError("The requested entity %s conflicts with the existing entity in PurView" % j["attributes"]["qualifiedName"]) + raise ConflictError("The requested entity %s conflicts with the existing entity in PurView" % j["attributes"]["qualifiedName"]) except AtlasException as e: pass + except KeyError as e: + # This is because the response is empty when the entity is not found + pass entity.lastModifiedTS="0" - results = self.purview_client.upload_entities( - batch=entity) + results = None + try: + results = self.purview_client.upload_entities( + batch=entity) + except AtlasException as e: + raise PreconditionError("Feature registration failed.", e) if results: d = {x.guid: x for x in [entity]} for k, v in results['guidAssignments'].items(): d[k].guid = v else: - raise RuntimeError("Feature registration failed.", results) + raise PreconditionError("Feature registration failed.", results) def _generate_fully_qualified_name(self, segments): return self.registry_delimiter.join(segments) diff --git a/registry/purview-registry/test/test_creation.py b/registry/purview-registry/test/test_creation.py index d99364cfc..71696fc9e 100644 --- a/registry/purview-registry/test/test_creation.py +++ b/registry/purview-registry/test/test_creation.py @@ -21,3 +21,24 @@ name="df1", feature_type=ft1, transformation=t1, key=[k], input_anchor_features=[feature1], input_derived_features=[])) print(proj_id,source_id,anchor1_id,feature1,derived) + +derived_downstream_entities = registry.get_dependent_entities(derived) +assert len(derived_downstream_entities) == 0 + +feature1_downstream_entities = registry.get_dependent_entities(feature1) +assert len(feature1_downstream_entities) == 1 + +registry.delete_entity(derived) + +# Try getting derived feature but KeyError exception should be thrown +derived_exists = 1 +try: + df1 = registry.get_entity(derived) +except KeyError: + derived_exists = 0 +assert derived_exists == 0 + +feature1_downstream_entities = registry.get_dependent_entities(feature1) +assert len(feature1_downstream_entities) == 0 + +# cleanup() diff --git a/registry/sql-registry/api-spec.md b/registry/sql-registry/api-spec.md index d2e82a878..b4ec243dc 100644 --- a/registry/sql-registry/api-spec.md +++ b/registry/sql-registry/api-spec.md @@ -285,6 +285,9 @@ Response Type: `dict` ### `GET /projects/{project}` Get everything defined in the project +### `GET /dependent/{entity}` +Gets downstream/dependent entities for given entity + Response Type: [`EntitiesAndRelationships`](#entitiesandrelationships) ### `GET /projects/{project}/datasources` @@ -320,6 +323,9 @@ Response Type: Object | entity | [`Entity`](#entity) | | | referredEntities| `map` | For compatibility, not used | +### `DELETE /entity/{entity}` +Deletes entity + ### `POST /projects` Create new project diff --git a/registry/sql-registry/main.py b/registry/sql-registry/main.py index 46cefbb34..dcb4d79cb 100644 --- a/registry/sql-registry/main.py +++ b/registry/sql-registry/main.py @@ -86,6 +86,22 @@ def get_projects_ids() -> dict: def get_projects(project: str) -> dict: return registry.get_project(project).to_dict() +@router.get("/dependent/{entity}") +def get_dependent_entities(entity: str) -> list: + entity_id = registry.get_entity_id(entity) + downstream_entities = registry.get_dependent_entities(entity_id) + return list([e.to_dict() for e in downstream_entities]) + +@router.delete("/entity/{entity}") +def delete_entity(entity: str): + entity_id = registry.get_entity_id(entity) + downstream_entities = registry.get_dependent_entities(entity_id) + if len(downstream_entities) > 0: + raise HTTPException( + status_code=412, detail=f"""Entity cannot be deleted as it has downstream/dependent entities. + Entities: {list([e.qualified_name for e in downstream_entities])}""" + ) + registry.delete_entity(entity_id) @router.get("/projects/{project}/datasources") def get_project_datasources(project: str) -> list: @@ -135,13 +151,11 @@ def get_feature(feature: str) -> dict: status_code=404, detail=f"Feature {feature} not found") return e.to_dict() - @router.get("/features/{feature}/lineage") def get_feature_lineage(feature: str) -> dict: lineage = registry.get_lineage(feature) return lineage.to_dict() - @router.post("/projects") def new_project(definition: dict) -> dict: id = registry.create_project(ProjectDef(**to_snake(definition))) diff --git a/registry/sql-registry/registry/db_registry.py b/registry/sql-registry/registry/db_registry.py index 1553508d8..d0b4c75c5 100644 --- a/registry/sql-registry/registry/db_registry.py +++ b/registry/sql-registry/registry/db_registry.py @@ -105,6 +105,32 @@ def get_project(self, id_or_name: Union[str, UUID]) -> EntitiesAndRelations: df.attributes.input_features = features all_edges = self._get_edges(ids) return EntitiesAndRelations([project] + children, list(edges.union(all_edges))) + + def get_dependent_entities(self, entity_id: Union[str, UUID]) -> list[Entity]: + """ + Given entity id, returns list of all entities that are downstream/dependant on the given entity + """ + entity_id = self.get_entity_id(entity_id) + entity = self.get_entity(entity_id) + downstream_entities = [] + if entity.entity_type == EntityType.Project: + downstream_entities, _ = self._bfs(entity_id, RelationshipType.Contains) + if entity.entity_type == EntityType.Source: + downstream_entities, _ = self._bfs(entity_id, RelationshipType.Produces) + if entity.entity_type == EntityType.Anchor: + downstream_entities, _ = self._bfs(entity_id, RelationshipType.Contains) + if entity.entity_type in (EntityType.AnchorFeature, EntityType.DerivedFeature): + downstream_entities, _ = self._bfs(entity_id, RelationshipType.Produces) + return [e for e in downstream_entities if str(e.id) != str(entity_id)] + + def delete_entity(self, entity_id: Union[str, UUID]): + """ + Deletes given entity + """ + entity_id = self.get_entity_id(entity_id) + with self.conn.transaction() as c: + self._delete_all_entity_edges(c, entity_id) + self._delete_entity(c, entity_id) def search_entity(self, keyword: str, @@ -386,6 +412,20 @@ def _create_edge(self, cursor, from_id: UUID, to_id: UUID, type: RelationshipTyp "to_id": str(to_id), "type": type.name }) + + def _delete_all_entity_edges(self, cursor, entity_id: UUID): + """ + Deletes all edges associated with an entity + """ + sql = fr'''DELETE FROM edges WHERE from_id = %s OR to_id = %s''' + cursor.execute(sql, (str(entity_id), str(entity_id))) + + def _delete_entity(self, cursor, entity_id: UUID): + """ + Deletes entity from entities table + """ + sql = fr'''DELETE FROM entities WHERE entity_id = %s''' + cursor.execute(sql, str(entity_id)) def _fill_entity(self, e: Entity) -> Entity: """ diff --git a/registry/sql-registry/registry/interface.py b/registry/sql-registry/registry/interface.py index 7f1439079..62f6071cd 100644 --- a/registry/sql-registry/registry/interface.py +++ b/registry/sql-registry/registry/interface.py @@ -111,3 +111,17 @@ def create_project_derived_feature(self, project_id: UUID, definition: DerivedFe Create a new derived feature under the project """ pass + + @abstractmethod + def get_dependent_entities(self, entity_id: Union[str, UUID]) -> list[Entity]: + """ + Given entity id, returns list of all entities that are downstream/dependant on the given entity + """ + pass + + @abstractmethod + def delete_entity(self, entity_id: Union[str, UUID]): + """ + Deletes given entity + """ + pass \ No newline at end of file diff --git a/registry/sql-registry/test/test_create.py b/registry/sql-registry/test/test_create.py index d3077698b..fd6ba74df 100644 --- a/registry/sql-registry/test/test_create.py +++ b/registry/sql-registry/test/test_create.py @@ -55,4 +55,24 @@ def cleanup(): # df1 has only 1 input anchor feature "af1" assert df1.attributes.input_anchor_features[0].id == af1_id +df1_downstream_entities = r.get_dependent_entities(df1_id) +assert len(df1_downstream_entities) == 0 + +af1_downstream_entities = r.get_dependent_entities(af1_id) +assert len(af1_downstream_entities) == 1 + +#Delete derived feature +r.delete_entity(df1_id) + +# Try getting derived feature but KeyError exception should be thrown +derived_exists = 1 +try: + df1 = r.get_entity(df1_id) +except KeyError: + derived_exists = 0 +assert derived_exists == 0 + +af1_downstream_entities = r.get_dependent_entities(af1_id) +assert len(af1_downstream_entities) == 0 + # cleanup() diff --git a/repositories.gradle b/repositories.gradle new file mode 100644 index 000000000..e7701fb50 --- /dev/null +++ b/repositories.gradle @@ -0,0 +1,21 @@ +repositories { + gradlePluginPortal() + mavenLocal() + mavenCentral() + maven { + url "https://packages.confluent.io/maven/" + } + maven { + url "https://plugins.gradle.org/m2/" + } + maven { + url "https://linkedin.jfrog.io/artifactory/open-source/" // GMA, pegasus + } +} + +try { + subprojects { + project.repositories.addAll(rootProject.repositories) + } +} catch (Throwable t) { +} diff --git a/settings.gradle b/settings.gradle new file mode 100644 index 000000000..28f78ba0d --- /dev/null +++ b/settings.gradle @@ -0,0 +1,14 @@ +/* + * This file was generated by the Gradle 'init' task. + * + * The settings file is used to specify which projects to include in your build. + * + * Detailed information about configuring a multi-project build in Gradle can be found + * in the user manual at https://docs.gradle.org/7.4.2/userguide/multi_project_builds.html + */ + +rootProject.name = 'feathr' +include 'feathr-impl' +include 'feathr-config' +include 'feathr-data-models' +include 'feathr-compute' \ No newline at end of file diff --git a/sonatype.sbt b/sonatype.sbt deleted file mode 100644 index 624344cc9..000000000 --- a/sonatype.sbt +++ /dev/null @@ -1,27 +0,0 @@ -publishTo := sonatypePublishToBundle.value - -// Feathr Sonatype account was created before Feb 2021, hence this host. -sonatypeCredentialHost := "oss.sonatype.org" - - -// Your profile name of the sonatype account. The default is the same with the organization value -sonatypeProfileName := "com.linkedin.feathr" - -// To sync with Maven central, you need to supply the following information: -publishMavenStyle := true - -// Open-source license of your choice -licenses := Seq("APL2" -> url("http://www.apache.org/licenses/LICENSE-2.0.txt")) - - -// Project metadata -homepage := Some(url("https://github.com/linkedin/feathr")) -scmInfo := Some( - ScmInfo( - url("https://github.com/linkedin/feathr"), - "scm:git@github.com:linkedin/feathr.git" - ) -) -developers := List( - Developer(id="feathr_dev", name="Feathr Dev", email="feathrai@gmail.com", url=url("https://github.com/linkedin/feathr")) -) \ No newline at end of file diff --git a/src/META-INF/MANIFEST.MF b/src/META-INF/MANIFEST.MF deleted file mode 100644 index f211793ea..000000000 --- a/src/META-INF/MANIFEST.MF +++ /dev/null @@ -1 +0,0 @@ -Main-Class: com.linkedin.feathr.cli.FeatureExperimentEntryPoint diff --git a/src/main/scala/com/linkedin/feathr/common/package.scala b/src/main/scala/com/linkedin/feathr/common/package.scala deleted file mode 100644 index 925d8720b..000000000 --- a/src/main/scala/com/linkedin/feathr/common/package.scala +++ /dev/null @@ -1,89 +0,0 @@ -package com.linkedin.feathr - -import com.typesafe.config.Config -import scala.collection.JavaConverters._ - -/** - * parameter map(config) utility class, help user to get parameter value with a default value, - * example usage: - * - * import com.linkedin.feathr.common.RichConfig._ - * val batchValue = _params.map(_.getBooleanWithDefault(batchPath, true)).get - * - */ -package object common { - - val SELECTED_FEATURES = "selectedFeatures" - implicit class RichConfig(val config: Config) { - /* - get a parameter at 'path' with default value - */ - def getStringWithDefault(path: String, default: String): String = if (config.hasPath(path)) { - config.getString(path) - } else { - default - } - - /* - get a parameter at 'path' with default value - */ - def getBooleanWithDefault(path: String, default: Boolean): Boolean = if (config.hasPath(path)) { - config.getBoolean(path) - } else { - default - } - - /* - get a parameter at 'path' with default value - */ - def getIntWithDefault(path: String, default: Int): Int = if (config.hasPath(path)) { - config.getInt(path) - } else { - default - } - - /* - get a parameter at 'path' with default value - */ - def getDoubleWithDefault(path: String, default: Double): Double = if (config.hasPath(path)) { - config.getDouble(path) - } else { - default - } - /* - get a parameter at 'path' with default value - */ - def getMapWithDefault(path: String, default: Map[String, Object]): Map[String, Object] = if (config.hasPath(path)) { - config.getObject(path).unwrapped().asScala.toMap - } else { - default - } - - /* - get a parameter with optional string list - */ - def getStringListOpt(path: String): Option[Seq[String]] = if (config.hasPath(path)) { - Some(config.getStringList(path).asScala.toSeq) - } else { - None - } - - /* - get a parameter with optional string - */ - def getStringOpt(path: String): Option[String] = if (config.hasPath(path)) { - Some(config.getString(path)) - } else { - None - } - - /* - get a parameter with optional number - */ - def getNumberOpt(path: String): Option[Number] = if (config.hasPath(path)) { - Some(config.getNumber(path)) - } else { - None - } - } -} diff --git a/src/main/scala/com/linkedin/feathr/offline/source/dataloader/jdbc/SnowflakeSqlDataLoader.scala b/src/main/scala/com/linkedin/feathr/offline/source/dataloader/jdbc/SnowflakeSqlDataLoader.scala deleted file mode 100644 index 312caad7c..000000000 --- a/src/main/scala/com/linkedin/feathr/offline/source/dataloader/jdbc/SnowflakeSqlDataLoader.scala +++ /dev/null @@ -1,67 +0,0 @@ -package com.linkedin.feathr.offline.source.dataloader.jdbc - -import org.apache.commons.httpclient.URI -import org.apache.http.client.utils.URLEncodedUtils -import org.apache.spark.sql.{DataFrame, DataFrameReader, SparkSession} - -import scala.collection.JavaConverters.asScalaBufferConverter -import java.nio.charset.Charset - -/** - * This is used for Snowflake data source JDBC connector - * - */ -class SnowflakeSqlDataLoader(ss: SparkSession) extends JdbcConnector(ss) { - val SNOWFLAKE_SOURCE_NAME = "net.snowflake.spark.snowflake" - - override def getDFReader(jdbcOptions: Map[String, String], url: String): DataFrameReader = { - val dfReader = _ss.read - .format(SNOWFLAKE_SOURCE_NAME) - .options(jdbcOptions) - - val uri = new URI(url) - val charset = Charset.forName("UTF-8") - val params = URLEncodedUtils.parse(uri.getQuery, charset).asScala - params.foreach(x => { - dfReader.option(x.getName, x.getValue) - }) - dfReader - } - - override def extractJdbcOptions(ss: SparkSession, url: String): Map[String, String] = { - val jdbcOptions1 = getJdbcParams(ss) - val jdbcOptions2 = getJdbcAuth(ss) - jdbcOptions1 ++ jdbcOptions2 - } - - def getJdbcParams(ss: SparkSession): Map[String, String] = { - Map[String, String]( - "sfURL" -> ss.conf.get("sfURL"), - "sfUser" -> ss.conf.get("sfUser"), - "sfRole" -> ss.conf.get("sfRole"), - ) - } - - def getJdbcAuth(ss: SparkSession): Map[String, String] = { - // If user set password, then we use password to auth - ss.conf.getOption("sfPassword") match { - case Some(_) => - Map[String, String]( - "sfUser" -> ss.conf.get("sfUser"), - "sfRole" -> ss.conf.get("sfRole"), - "sfPassword" -> ss.conf.get("sfPassword"), - ) - case _ => { - // TODO Add token support - Map[String, String]() - } - } - } - - override def loadDataFrame(url: String, jdbcOptions: Map[String, String] = Map[String, String]()): DataFrame = { - val sparkReader = getDFReader(jdbcOptions, url) - sparkReader - .option("url", url) - .load() - } -} \ No newline at end of file diff --git a/src/test/scala/com/linkedin/feathr/offline/TestFeathrUdfPlugins.scala b/src/test/scala/com/linkedin/feathr/offline/TestFeathrUdfPlugins.scala deleted file mode 100644 index 63637a989..000000000 --- a/src/test/scala/com/linkedin/feathr/offline/TestFeathrUdfPlugins.scala +++ /dev/null @@ -1,139 +0,0 @@ -package com.linkedin.feathr.offline - -import com.linkedin.feathr.common.FeatureTypes -import com.linkedin.feathr.offline.anchored.keyExtractor.AlienSourceKeyExtractorAdaptor -import com.linkedin.feathr.offline.client.plugins.FeathrUdfPluginContext -import com.linkedin.feathr.offline.derived.AlienDerivationFunctionAdaptor -import com.linkedin.feathr.offline.mvel.plugins.FeathrExpressionExecutionContext -import com.linkedin.feathr.offline.plugins.{AlienFeatureValue, AlienFeatureValueTypeAdaptor} -import com.linkedin.feathr.offline.util.FeathrTestUtils -import org.apache.spark.sql.Row -import org.apache.spark.sql.types.{FloatType, StringType, StructField, StructType} -import org.testng.Assert.assertEquals -import org.testng.annotations.Test - -class TestFeathrUdfPlugins extends FeathrIntegTest { - - val MULTILINE_QUOTE = "\"\"\"" - - private val mvelContext = new FeathrExpressionExecutionContext() - @Test - def testMvelUdfPluginSupport: Unit = { - mvelContext.setupExecutorMvelContext(classOf[AlienFeatureValue], new AlienFeatureValueTypeAdaptor(), ss.sparkContext) - FeathrUdfPluginContext.registerUdfAdaptor(new AlienDerivationFunctionAdaptor(), ss.sparkContext) - FeathrUdfPluginContext.registerUdfAdaptor(new AlienSourceKeyExtractorAdaptor(), ss.sparkContext) - val df = runLocalFeatureJoinForTest( - joinConfigAsString = """ - | features: { - | key: a_id - | featureList: ["f1", "f2", "f3", "f4", "f5", "f6", "f7", "f8", "fA"] - | } - """.stripMargin, - featureDefAsString = s""" - |anchors: { - | anchor1: { - | source: "anchor1-source.csv" - | key: "mId" - | features: { - | // create an alien-type feature value, and expect Feathr to consume it via plugin - | f1: $MULTILINE_QUOTE - | import com.linkedin.feathr.offline.plugins.AlienFeatureValueMvelUDFs; - | AlienFeatureValueMvelUDFs.sqrt_float(gamma) - | $MULTILINE_QUOTE - | - | // create an alien-type feature value, and pass it to a UDF that expects Feathr feature value - | f2: $MULTILINE_QUOTE - | import com.linkedin.feathr.offline.plugins.AlienFeatureValueMvelUDFs; - | import com.linkedin.feathr.offline.plugins.FeathrFeatureValueMvelUDFs; - | FeathrFeatureValueMvelUDFs.inverse_ffv(AlienFeatureValueMvelUDFs.sqrt_float(gamma)) - | $MULTILINE_QUOTE - | - | // create a Feathr feature value, and pass it to a UDF that expects the alien feature value - | f3: $MULTILINE_QUOTE - | import com.linkedin.feathr.offline.plugins.AlienFeatureValueMvelUDFs; - | import com.linkedin.feathr.offline.plugins.FeathrFeatureValueMvelUDFs; - | AlienFeatureValueMvelUDFs.sqrt_afv(FeathrFeatureValueMvelUDFs.inverse_float(gamma)) - | $MULTILINE_QUOTE - | - | f4: { - | type: CATEGORICAL - | def: $MULTILINE_QUOTE - | import com.linkedin.feathr.offline.plugins.AlienFeatureValueMvelUDFs; - | AlienFeatureValueMvelUDFs.uppercase_string(alpha); - | $MULTILINE_QUOTE - | } - | } - | } - | anchor2: { - | source: "anchor1-source.csv" - | keyExtractor: "com.linkedin.feathr.offline.anchored.keyExtractor.AlienSampleKeyExtractor" - | features: { - | fA: { - | def: cast_float(beta) - | type: NUMERIC - | default: 0 - | } - | } - | } - |} - | - |derivations: { - | // use an UDF that expects/returns alien-valued feature value - | f5: { - | type: NUMERIC - | definition: $MULTILINE_QUOTE - | import com.linkedin.feathr.offline.plugins.AlienFeatureValueMvelUDFs; - | AlienFeatureValueMvelUDFs.sqrt_float(f3) - | $MULTILINE_QUOTE - | } - | f6: { - | type: NUMERIC - | definition: $MULTILINE_QUOTE - | import com.linkedin.feathr.offline.plugins.AlienFeatureValueMvelUDFs; - | AlienFeatureValueMvelUDFs.sqrt_float(f2) - | $MULTILINE_QUOTE - | } - | f7: { - | type: CATEGORICAL - | definition: $MULTILINE_QUOTE - | import com.linkedin.feathr.offline.plugins.AlienFeatureValueMvelUDFs; - | AlienFeatureValueMvelUDFs.lowercase_string_afv(f4); - | $MULTILINE_QUOTE - | } - | f8: { - | key: ["mId"] - | inputs: [{ key: "mId", feature: "f6" }] - | class: "com.linkedin.feathr.offline.derived.SampleAlienFeatureDerivationFunction" - | type: NUMERIC - | } - |} - """.stripMargin, - observationDataPath = "anchorAndDerivations/testMVELLoopExpFeature-observations.csv", - mvelContext = Some(mvelContext)) - - val f8Type = df.fdsMetadata.header.get.featureInfoMap.filter(_._1.getFeatureName == "f8").head._2.featureType.getFeatureType - assertEquals(f8Type, FeatureTypes.NUMERIC) - - val selectedColumns = Seq("a_id", "fA") - val filteredDf = df.data.select(selectedColumns.head, selectedColumns.tail: _*) - - val expectedDf = ss.createDataFrame( - ss.sparkContext.parallelize( - Seq( - Row( - "1", - 10.0f), - Row( - "2", - 10.0f), - Row( - "3", - 10.0f))), - StructType( - List( - StructField("a_id", StringType, true), - StructField("fA", FloatType, true)))) - def cmpFunc(row: Row): String = row.get(0).toString - FeathrTestUtils.assertDataFrameApproximatelyEquals(filteredDf, expectedDf, cmpFunc) - } -} diff --git a/ui/.editorconfig b/ui/.editorconfig new file mode 100644 index 000000000..b5e435a15 --- /dev/null +++ b/ui/.editorconfig @@ -0,0 +1,10 @@ +# http://editorconfig.org +root = true + +[*] +charset=utf-8 +end_of_line=lf +insert_final_newline=false +indent_style=space +indent_size=2 + diff --git a/ui/.eslintrc b/ui/.eslintrc index 2a16ad386..c271bfa24 100644 --- a/ui/.eslintrc +++ b/ui/.eslintrc @@ -4,7 +4,15 @@ "es6": true, "node": true }, - "plugins": ["@typescript-eslint/eslint-plugin"], + "plugins": ["react", "@typescript-eslint/eslint-plugin", "prettier"], + "settings": { + "import/resolver": { + "node": { + "extensions": [".tsx", ".ts", ".jsx", ".js", ".json"] + }, + "typescript": {} + } + }, "extends": [ // https://github.com/eslint/eslint/blob/main/conf/eslint-recommended.js "eslint:recommended", @@ -12,7 +20,8 @@ "react-app", // https://reactjs.org/docs/hooks-rules.html "plugin:react-hooks/recommended", - "prettier" + "plugin:prettier/recommended", + "plugin:json/recommended" ], "parser": "@typescript-eslint/parser", "parserOptions": { @@ -20,7 +29,22 @@ "sourceType": "module" }, "rules": { - "dot-notation": "error" + "dot-notation": "error", + "import/extensions": [ + "error", + "ignorePackages", + { + "ts": "never", + "tsx": "never", + "js": "never", + "jsx": "never" + } + ], + "import/no-extraneous-dependencies": ["error", { "devDependencies": true }], + "import/prefer-default-export": "off", + "import/no-unresolved": "error", + "import/no-dynamic-require": "off", + "import/no-mutable-exports": "warn" }, "overrides": [ { diff --git a/ui/.vscode/settings.json b/ui/.vscode/settings.json index c8e624dc0..5fffcb522 100644 --- a/ui/.vscode/settings.json +++ b/ui/.vscode/settings.json @@ -11,5 +11,8 @@ ], "[css]": { "editor.defaultFormatter": "esbenp.prettier-vscode" + }, + "[javascript]": { + "editor.defaultFormatter": "esbenp.prettier-vscode" } } diff --git a/ui/craco.config.js b/ui/craco.config.js new file mode 100644 index 000000000..e44884899 --- /dev/null +++ b/ui/craco.config.js @@ -0,0 +1,79 @@ +const path = require("path"); + +const { loaderByName } = require("@craco/craco"); +const CracoLessPlugin = require("craco-less"); + +const webpack = require("webpack"); + +const packageJson = require("./package.json"); + +const resolve = (dir) => path.resolve(__dirname, dir); + +const currentTime = new Date(); + +module.exports = { + babel: { + plugins: [ + [ + "import", + { + libraryName: "antd", + libraryDirectory: "es", + style: true, + }, + ], + ], + }, + webpack: { + alias: { + "@": resolve("src"), + }, + configure: (webpackConfig, { env, paths }) => { + const index = webpackConfig.plugins.findIndex( + (itme) => itme instanceof webpack.DefinePlugin + ); + + if (index > -1) { + const definePlugin = webpackConfig.plugins[index]; + webpackConfig.plugins.splice( + index, + 1, + new webpack.DefinePlugin({ + "process.env": { + ...definePlugin.definitions["process.env"], + FEATHR_VERSION: JSON.stringify(packageJson.version), + FEATHR_GENERATED_TIME: JSON.stringify(currentTime.toISOString()), + }, + }) + ); + } + + return webpackConfig; + }, + }, + plugins: [ + { + plugin: CracoLessPlugin, + options: { + lessLoaderOptions: { + lessOptions: { + modifyVars: {}, + javascriptEnabled: true, + }, + }, + modifyLessModuleRule(lessModuleRule, context) { + // Configure the file suffix + lessModuleRule.test = /\.module\.less$/; + + // Configure the generated local ident name. + const cssLoader = lessModuleRule.use.find(loaderByName("css-loader")); + cssLoader.options.modules = { + localIdentName: "[local]_[hash:base64:5]", + }; + + return lessModuleRule; + }, + }, + }, + ], +}; diff --git a/ui/package-lock.json b/ui/package-lock.json index 28bd6553c..347f393c5 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -1,26 +1,30 @@ { "name": "feathr-ui", - "version": "0.1.0", + "version": "0.9.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "feathr-ui", - "version": "0.1.0", + "version": "0.9.0", "dependencies": { + "@ant-design/icons": "^4.7.0", "@azure/msal-browser": "^2.24.0", "@azure/msal-react": "^1.4.0", - "antd": "^4.20.2", + "antd": "^4.23.6", "axios": "^0.27.2", + "classnames": "^2.3.2", "dagre": "^0.8.5", "dayjs": "^1.11.5", "react": "^17.0.2", "react-dom": "^17.0.2", "react-flow-renderer": "^9.7.4", "react-query": "^3.38.0", + "react-resizable": "^3.0.4", "react-router-dom": "^6.3.0" }, "devDependencies": { + "@craco/craco": "^7.0.0-alpha.8", "@testing-library/jest-dom": "^5.16.3", "@testing-library/react": "^12.1.4", "@testing-library/user-event": "^13.5.0", @@ -29,17 +33,25 @@ "@types/node": "^16.11.26", "@types/react": "^17.0.43", "@types/react-dom": "^17.0.14", + "@types/react-resizable": "^3.0.3", "@typescript-eslint/eslint-plugin": "^5.30.7", "@typescript-eslint/parser": "^5.30.7", + "babel-plugin-import": "^1.13.5", + "craco-less": "^2.1.0-alpha.0", "eslint": "^8.20.0", "eslint-config-prettier": "^8.5.0", + "eslint-import-resolver-typescript": "^3.5.1", + "eslint-plugin-import": "^2.26.0", + "eslint-plugin-json": "^3.1.0", + "eslint-plugin-prettier": "^4.2.1", "eslint-plugin-react-hooks": "^4.6.0", "husky": "^8.0.1", "lint-staged": "^13.0.3", "prettier": "2.7.1", "react-scripts": "5.0.0", "typescript": "^4.6.3", - "web-vitals": "^2.1.4" + "web-vitals": "^2.1.4", + "webpack": "^5.72.0" } }, "node_modules/@ampproject/remapping": { @@ -63,7 +75,8 @@ }, "node_modules/@ant-design/icons": { "version": "4.7.0", - "license": "MIT", + "resolved": "https://registry.npmjs.org/@ant-design/icons/-/icons-4.7.0.tgz", + "integrity": "sha512-aoB4Z7JA431rt6d4u+8xcNPPCrdufSRMUOpxa1ab6mz1JCQZOEVolj2WVs/tDFmN62zzK30mNelEsprLYsSF3g==", "dependencies": { "@ant-design/colors": "^6.0.0", "@ant-design/icons-svg": "^4.2.1", @@ -84,14 +97,15 @@ "license": "MIT" }, "node_modules/@ant-design/react-slick": { - "version": "0.28.4", - "license": "MIT", + "version": "0.29.2", + "resolved": "https://registry.npmjs.org/@ant-design/react-slick/-/react-slick-0.29.2.tgz", + "integrity": "sha512-kgjtKmkGHa19FW21lHnAfyyH9AAoh35pBdcJ53rHmQ3O+cfFHGHnUbj/HFrRNJ5vIts09FKJVAD8RpaC+RaWfA==", "dependencies": { "@babel/runtime": "^7.10.4", "classnames": "^2.2.5", "json2mq": "^0.2.0", "lodash": "^4.17.21", - "resize-observer-polyfill": "^1.5.0" + "resize-observer-polyfill": "^1.5.1" }, "peerDependencies": { "react": ">=16.9.0" @@ -1936,10 +1950,11 @@ } }, "node_modules/@babel/runtime": { - "version": "7.17.9", - "license": "MIT", + "version": "7.20.0", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.20.0.tgz", + "integrity": "sha512-NDYdls71fTXoU8TZHfbBWg7DiZfNzClcKui/+kyi6ppD2L1qnWW3VV6CjtaBXSUGGhiTWJ6ereOIkUvenif66Q==", "dependencies": { - "regenerator-runtime": "^0.13.4" + "regenerator-runtime": "^0.13.10" }, "engines": { "node": ">=6.9.0" @@ -2007,6 +2022,43 @@ "dev": true, "license": "MIT" }, + "node_modules/@craco/craco": { + "version": "7.0.0-alpha.8", + "resolved": "https://registry.npmjs.org/@craco/craco/-/craco-7.0.0-alpha.8.tgz", + "integrity": "sha512-IN3/ldPaktGflPu342cg7n8LYa2c3x9H2XzngUkDzTjro25ig1GyVcUdnG1U0X6wrRTF9K1AxZ5su9jLbdyFUw==", + "dev": true, + "dependencies": { + "autoprefixer": "^10.4.12", + "cosmiconfig": "^7.0.1", + "cosmiconfig-typescript-loader": "^4.1.1", + "cross-spawn": "^7.0.3", + "lodash": "^4.17.21", + "semver": "^7.3.7", + "webpack-merge": "^5.8.0" + }, + "bin": { + "craco": "dist/bin/craco.js" + }, + "engines": { + "node": ">=6" + }, + "peerDependencies": { + "react-scripts": "^5.0.0" + } + }, + "node_modules/@cspotcode/source-map-support": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", + "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==", + "dev": true, + "peer": true, + "dependencies": { + "@jridgewell/trace-mapping": "0.3.9" + }, + "engines": { + "node": ">=12" + } + }, "node_modules/@csstools/normalize.css": { "version": "12.0.0", "dev": true, @@ -2769,6 +2821,32 @@ "node": ">= 8" } }, + "node_modules/@pkgr/utils": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/@pkgr/utils/-/utils-2.3.1.tgz", + "integrity": "sha512-wfzX8kc1PMyUILA+1Z/EqoE4UCXGy0iRGMhPwdfae1+f0OXlLqCk+By+aMzgJBzR9AzS4CDizioG6Ss1gvAFJw==", + "dev": true, + "dependencies": { + "cross-spawn": "^7.0.3", + "is-glob": "^4.0.3", + "open": "^8.4.0", + "picocolors": "^1.0.0", + "tiny-glob": "^0.2.9", + "tslib": "^2.4.0" + }, + "engines": { + "node": "^12.20.0 || ^14.18.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/unts" + } + }, + "node_modules/@pkgr/utils/node_modules/tslib": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.0.tgz", + "integrity": "sha512-d6xOpEDfsi2CZVlPQzGeux8XMwLT9hssAsaPYExaQMuYskwb+x1x7J371tWlbBdWHroy99KnVB6qIkUbs5X3UQ==", + "dev": true + }, "node_modules/@pmmmwh/react-refresh-webpack-plugin": { "version": "0.5.5", "dev": true, @@ -3246,6 +3324,34 @@ "node": ">=10.13.0" } }, + "node_modules/@tsconfig/node10": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.9.tgz", + "integrity": "sha512-jNsYVVxU8v5g43Erja32laIDHXeoNvFEpX33OK4d6hljo3jDhCBDhx5dhCCTMWUojscpAagGiRkBKxpdl9fxqA==", + "dev": true, + "peer": true + }, + "node_modules/@tsconfig/node12": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/@tsconfig/node12/-/node12-1.0.11.tgz", + "integrity": "sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag==", + "dev": true, + "peer": true + }, + "node_modules/@tsconfig/node14": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@tsconfig/node14/-/node14-1.0.3.tgz", + "integrity": "sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow==", + "dev": true, + "peer": true + }, + "node_modules/@tsconfig/node16": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@tsconfig/node16/-/node16-1.0.3.tgz", + "integrity": "sha512-yOlFc+7UtL/89t2ZhjPvvB/DeAr3r+Dq58IgzsFkOAvVC6NMJXmCGjbptdXdR9qsX7pKcTL+s87FtYREi2dEEQ==", + "dev": true, + "peer": true + }, "node_modules/@types/aria-query": { "version": "4.2.2", "dev": true, @@ -3506,6 +3612,15 @@ "redux": "^4.0.0" } }, + "node_modules/@types/react-resizable": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/react-resizable/-/react-resizable-3.0.3.tgz", + "integrity": "sha512-W/QsUOZoXBAIBQNhNm95A5ohoaiUA874lWQytO2UP9dOjp5JHO9+a0cwYNabea7sA12ZDJnGVUFZxcNaNksAWA==", + "dev": true, + "dependencies": { + "@types/react": "*" + } + }, "node_modules/@types/resolve": { "version": "1.17.1", "dev": true, @@ -4465,52 +4580,53 @@ } }, "node_modules/antd": { - "version": "4.20.2", - "license": "MIT", + "version": "4.23.6", + "resolved": "https://registry.npmjs.org/antd/-/antd-4.23.6.tgz", + "integrity": "sha512-AYH57cWBDe1ChtbnvG8i9dpKG4WnjE3AG0zIKpXByFNnxsr4saV6/19ihE8/ImSGpohN4E2zTXmo7R5/MyVRKQ==", "dependencies": { "@ant-design/colors": "^6.0.0", "@ant-design/icons": "^4.7.0", - "@ant-design/react-slick": "~0.28.1", - "@babel/runtime": "^7.12.5", + "@ant-design/react-slick": "~0.29.1", + "@babel/runtime": "^7.18.3", "@ctrl/tinycolor": "^3.4.0", "classnames": "^2.2.6", "copy-to-clipboard": "^3.2.0", "lodash": "^4.17.21", "memoize-one": "^6.0.0", "moment": "^2.29.2", - "rc-cascader": "~3.5.0", + "rc-cascader": "~3.7.0", "rc-checkbox": "~2.3.0", - "rc-collapse": "~3.1.0", - "rc-dialog": "~8.8.1", - "rc-drawer": "~4.4.2", - "rc-dropdown": "~3.5.0", - "rc-field-form": "~1.26.1", - "rc-image": "~5.6.0", - "rc-input": "~0.0.1-alpha.5", - "rc-input-number": "~7.3.0", - "rc-mentions": "~1.7.0", - "rc-menu": "~9.5.5", - "rc-motion": "^2.5.1", + "rc-collapse": "~3.3.0", + "rc-dialog": "~8.9.0", + "rc-drawer": "~5.1.0", + "rc-dropdown": "~4.0.0", + "rc-field-form": "~1.27.0", + "rc-image": "~5.7.0", + "rc-input": "~0.1.2", + "rc-input-number": "~7.3.9", + "rc-mentions": "~1.10.0", + "rc-menu": "~9.6.3", + "rc-motion": "^2.6.1", "rc-notification": "~4.6.0", - "rc-pagination": "~3.1.9", - "rc-picker": "~2.6.4", - "rc-progress": "~3.2.1", + "rc-pagination": "~3.1.17", + "rc-picker": "~2.6.11", + "rc-progress": "~3.3.2", "rc-rate": "~2.9.0", "rc-resize-observer": "^1.2.0", - "rc-segmented": "~2.1.0 ", - "rc-select": "~14.1.1", + "rc-segmented": "~2.1.0", + "rc-select": "~14.1.13", "rc-slider": "~10.0.0", "rc-steps": "~4.1.0", "rc-switch": "~3.2.0", - "rc-table": "~7.24.0", - "rc-tabs": "~11.13.0", - "rc-textarea": "~0.3.0", - "rc-tooltip": "~5.1.1", - "rc-tree": "~5.5.0", - "rc-tree-select": "~5.3.0", + "rc-table": "~7.26.0", + "rc-tabs": "~12.2.0", + "rc-textarea": "~0.4.5", + "rc-tooltip": "~5.2.0", + "rc-tree": "~5.7.0", + "rc-tree-select": "~5.5.0", "rc-trigger": "^5.2.10", "rc-upload": "~4.3.0", - "rc-util": "^5.20.0", + "rc-util": "^5.22.5", "scroll-into-view-if-needed": "^2.2.25" }, "funding": { @@ -4580,7 +4696,8 @@ }, "node_modules/array-tree-filter": { "version": "2.1.0", - "license": "MIT" + "resolved": "https://registry.npmjs.org/array-tree-filter/-/array-tree-filter-2.1.0.tgz", + "integrity": "sha512-4ROwICNlNw/Hqa9v+rk5h22KjmzB1JGTMVKP2AKJBOCgb0yL0ASf0+YvCcLNNwquOHNX48jkeZIJ3a+oOQqKcw==" }, "node_modules/array-union": { "version": "2.1.0", @@ -4648,8 +4765,9 @@ "license": "MIT" }, "node_modules/async-validator": { - "version": "4.1.1", - "license": "MIT" + "version": "4.2.5", + "resolved": "https://registry.npmjs.org/async-validator/-/async-validator-4.2.5.tgz", + "integrity": "sha512-7HhHjtERjqlNbZtqNqy2rckN/SpOOlmDliet+lP7k+eKZEjPk3DgyeU9lIXLdeLz0uBbbVp+9Qdow9wJWgwwfg==" }, "node_modules/asynckit": { "version": "0.4.0", @@ -4675,7 +4793,9 @@ } }, "node_modules/autoprefixer": { - "version": "10.4.7", + "version": "10.4.12", + "resolved": "https://registry.npmjs.org/autoprefixer/-/autoprefixer-10.4.12.tgz", + "integrity": "sha512-WrCGV9/b97Pa+jtwf5UGaRjgQIg7OK3D06GnoYoZNcG1Xb8Gt3EfuKjlhh9i/VtT16g6PYjZ69jdJ2g8FxSC4Q==", "dev": true, "funding": [ { @@ -4687,10 +4807,9 @@ "url": "https://tidelift.com/funding/github/npm/autoprefixer" } ], - "license": "MIT", "dependencies": { - "browserslist": "^4.20.3", - "caniuse-lite": "^1.0.30001335", + "browserslist": "^4.21.4", + "caniuse-lite": "^1.0.30001407", "fraction.js": "^4.2.0", "normalize-range": "^0.1.2", "picocolors": "^1.0.0", @@ -4837,6 +4956,15 @@ "object.assign": "^4.1.0" } }, + "node_modules/babel-plugin-import": { + "version": "1.13.5", + "resolved": "https://registry.npmjs.org/babel-plugin-import/-/babel-plugin-import-1.13.5.tgz", + "integrity": "sha512-IkqnoV+ov1hdJVofly9pXRJmeDm9EtROfrc5i6eII0Hix2xMs5FEm8FG3ExMvazbnZBbgHIt6qdO8And6lCloQ==", + "dev": true, + "dependencies": { + "@babel/helper-module-imports": "^7.0.0" + } + }, "node_modules/babel-plugin-istanbul": { "version": "6.1.1", "dev": true, @@ -5147,7 +5275,9 @@ "license": "BSD-2-Clause" }, "node_modules/browserslist": { - "version": "4.20.3", + "version": "4.21.4", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.21.4.tgz", + "integrity": "sha512-CBHJJdDmgjl3daYjN5Cp5kbTf1mUhZoS+beLklHIvkOWscs83YAhLlF3Wsh/lciQYAcbBJgTOD44VtG31ZM4Hw==", "dev": true, "funding": [ { @@ -5159,13 +5289,11 @@ "url": "https://tidelift.com/funding/github/npm/browserslist" } ], - "license": "MIT", "dependencies": { - "caniuse-lite": "^1.0.30001332", - "electron-to-chromium": "^1.4.118", - "escalade": "^3.1.1", - "node-releases": "^2.0.3", - "picocolors": "^1.0.0" + "caniuse-lite": "^1.0.30001400", + "electron-to-chromium": "^1.4.251", + "node-releases": "^2.0.6", + "update-browserslist-db": "^1.0.9" }, "bin": { "browserslist": "cli.js" @@ -5271,7 +5399,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001336", + "version": "1.0.30001422", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001422.tgz", + "integrity": "sha512-hSesn02u1QacQHhaxl/kNMZwqVG35Sz/8DgvmgedxSH8z9UUpcDYSPYgsj3x5dQNRcNp6BwpSfQfVzYUTm+fog==", "dev": true, "funding": [ { @@ -5282,8 +5412,7 @@ "type": "tidelift", "url": "https://tidelift.com/funding/github/npm/caniuse-lite" } - ], - "license": "CC-BY-4.0" + ] }, "node_modules/case-sensitive-paths-webpack-plugin": { "version": "2.4.0", @@ -5386,8 +5515,9 @@ "license": "MIT" }, "node_modules/classnames": { - "version": "2.3.1", - "license": "MIT" + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/classnames/-/classnames-2.3.2.tgz", + "integrity": "sha512-CSbhY4cFEJRe6/GQzIk5qXZ4Jeg5pcsP7b5peFSDpffpe1cqjASH/n9UTjBwOp6XpMSTwQ8Za2K5V02ueA7Tmw==" }, "node_modules/clean-css": { "version": "5.3.0", @@ -5485,6 +5615,20 @@ "wrap-ansi": "^7.0.0" } }, + "node_modules/clone-deep": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/clone-deep/-/clone-deep-4.0.1.tgz", + "integrity": "sha512-neHB9xuzh/wk0dIHweyAXv2aPGZIVk3pLMe+/RNzINf17fe0OG96QroktYAUm7SM1PBnzTabaLboqqxDyMU+SQ==", + "dev": true, + "dependencies": { + "is-plain-object": "^2.0.4", + "kind-of": "^6.0.2", + "shallow-clone": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/clsx": { "version": "1.1.1", "license": "MIT", @@ -5592,9 +5736,10 @@ "license": "MIT" }, "node_modules/colord": { - "version": "2.9.2", - "dev": true, - "license": "MIT" + "version": "2.9.3", + "resolved": "https://registry.npmjs.org/colord/-/colord-2.9.3.tgz", + "integrity": "sha512-jeC1axXpnb0/2nn/Y1LPuLdgXBLH7aDcHu4KEKfqw3CUhX7ZpfBSlPKyqXE6btIgEzfWtrX3/tyBCaCvXvMkOw==", + "dev": true }, "node_modules/colorette": { "version": "2.0.16", @@ -5758,6 +5903,18 @@ "dev": true, "license": "MIT" }, + "node_modules/copy-anything": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/copy-anything/-/copy-anything-2.0.6.tgz", + "integrity": "sha512-1j20GZTsvKNkc4BY3NpMOM8tt///wY3FpIzozTOFO2ffuZcV61nojHXVKIy3WM+7ADCy5FVhdZYHYDdgTU0yJw==", + "dev": true, + "dependencies": { + "is-what": "^3.14.1" + }, + "funding": { + "url": "https://github.com/sponsors/mesqueeb" + } + }, "node_modules/copy-to-clipboard": { "version": "3.3.1", "license": "MIT", @@ -5826,6 +5983,43 @@ "node": ">=10" } }, + "node_modules/cosmiconfig-typescript-loader": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/cosmiconfig-typescript-loader/-/cosmiconfig-typescript-loader-4.1.1.tgz", + "integrity": "sha512-9DHpa379Gp0o0Zefii35fcmuuin6q92FnLDffzdZ0l9tVd3nEobG3O+MZ06+kuBvFTSVScvNb/oHA13Nd4iipg==", + "dev": true, + "engines": { + "node": ">=12", + "npm": ">=6" + }, + "peerDependencies": { + "@types/node": "*", + "cosmiconfig": ">=7", + "ts-node": ">=10", + "typescript": ">=3" + } + }, + "node_modules/craco-less": { + "version": "2.1.0-alpha.0", + "resolved": "https://registry.npmjs.org/craco-less/-/craco-less-2.1.0-alpha.0.tgz", + "integrity": "sha512-1kj9Y7Y06Fbae3SJJtz1OvXsaKxjh0jTOwnvzKWOqrojQZbwC2K/d0dxDRUpHTDkIUmxbdzqMmI4LM9JfthQ6Q==", + "dev": true, + "dependencies": { + "less": "^4.1.1", + "less-loader": "^7.3.0" + }, + "peerDependencies": { + "@craco/craco": ">7.0.0-alpha", + "react-scripts": "^5.0.0" + } + }, + "node_modules/create-require": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz", + "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==", + "dev": true, + "peer": true + }, "node_modules/cross-spawn": { "version": "7.0.3", "dev": true, @@ -6343,8 +6537,9 @@ } }, "node_modules/date-fns": { - "version": "2.28.0", - "license": "MIT", + "version": "2.29.3", + "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.29.3.tgz", + "integrity": "sha512-dDCnyH2WnnKusqvZZ6+jA1O51Ibt8ZMRNkDZdyAyK4YfbDwa/cEmuztzG5pk6hqlp9aSBPYcjOlktquahGwGeA==", "engines": { "node": ">=0.11" }, @@ -6530,6 +6725,16 @@ "dev": true, "license": "Apache-2.0" }, + "node_modules/diff": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", + "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==", + "dev": true, + "peer": true, + "engines": { + "node": ">=0.3.1" + } + }, "node_modules/diff-sequences": { "version": "27.5.1", "dev": true, @@ -6733,9 +6938,10 @@ } }, "node_modules/electron-to-chromium": { - "version": "1.4.134", - "dev": true, - "license": "ISC" + "version": "1.4.284", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.284.tgz", + "integrity": "sha512-M8WEXFuKXMYMVr45fo8mq0wUrrJHheiKZf6BArTKk9ZBYCKJEOU5H8cdWgDT+qCVZf7Na4lVUaZsA+h6uA9+PA==", + "dev": true }, "node_modules/emittery": { "version": "0.8.1", @@ -6770,9 +6976,10 @@ } }, "node_modules/enhanced-resolve": { - "version": "5.9.3", + "version": "5.10.0", + "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.10.0.tgz", + "integrity": "sha512-T0yTFjdpldGY8PmuXXR0PyQ1ufZpEGiHVrp7zHKB7jdR4qlmZHhONVM5AQOAWXuF/w3dnHbEQVrNptJgt7F+cQ==", "dev": true, - "license": "MIT", "dependencies": { "graceful-fs": "^4.2.4", "tapable": "^2.2.0" @@ -6789,6 +6996,19 @@ "url": "https://github.com/fb55/entities?sponsor=1" } }, + "node_modules/errno": { + "version": "0.1.8", + "resolved": "https://registry.npmjs.org/errno/-/errno-0.1.8.tgz", + "integrity": "sha512-dJ6oBr5SQ1VSd9qkk7ByRgb/1SH4JZjCHSW/mr63/QcXO9zLVxvJ6Oy13nio03rxpSnVDDjFor75SjVeZWPW/A==", + "dev": true, + "optional": true, + "dependencies": { + "prr": "~1.0.1" + }, + "bin": { + "errno": "cli.js" + } + }, "node_modules/error-ex": { "version": "1.3.2", "dev": true, @@ -7061,6 +7281,62 @@ "ms": "^2.1.1" } }, + "node_modules/eslint-import-resolver-typescript": { + "version": "3.5.1", + "resolved": "https://registry.npmjs.org/eslint-import-resolver-typescript/-/eslint-import-resolver-typescript-3.5.1.tgz", + "integrity": "sha512-U7LUjNJPYjNsHvAUAkt/RU3fcTSpbllA0//35B4eLYTX74frmOepbt7F7J3D1IGtj9k21buOpaqtDd4ZlS/BYQ==", + "dev": true, + "dependencies": { + "debug": "^4.3.4", + "enhanced-resolve": "^5.10.0", + "get-tsconfig": "^4.2.0", + "globby": "^13.1.2", + "is-core-module": "^2.10.0", + "is-glob": "^4.0.3", + "synckit": "^0.8.3" + }, + "engines": { + "node": "^12.20.0 || ^14.18.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/unts" + }, + "peerDependencies": { + "eslint": "*", + "eslint-plugin-import": "*" + } + }, + "node_modules/eslint-import-resolver-typescript/node_modules/globby": { + "version": "13.1.2", + "resolved": "https://registry.npmjs.org/globby/-/globby-13.1.2.tgz", + "integrity": "sha512-LKSDZXToac40u8Q1PQtZihbNdTYSNMuWe+K5l+oa6KgDzSvVrHXlJy40hUP522RjAIoNLJYBJi7ow+rbFpIhHQ==", + "dev": true, + "dependencies": { + "dir-glob": "^3.0.1", + "fast-glob": "^3.2.11", + "ignore": "^5.2.0", + "merge2": "^1.4.1", + "slash": "^4.0.0" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/eslint-import-resolver-typescript/node_modules/slash": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/slash/-/slash-4.0.0.tgz", + "integrity": "sha512-3dOsAHXXUkQTpOYcoAxLIorMTp4gIQr5IW3iVb7A7lFIp0VHhnynm9izx6TssdrIcVIESAlVjtnO2K8bg+Coew==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/eslint-module-utils": { "version": "2.7.3", "dev": true, @@ -7161,8 +7437,9 @@ }, "node_modules/eslint-plugin-import": { "version": "2.26.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.26.0.tgz", + "integrity": "sha512-hYfi3FXaM8WPLf4S1cikh/r4IxnO6zrhZbEGz2b660EJRbuxgpDS5gkCuYgGWg2xxh2rBuIr4Pvhve/7c31koA==", "dev": true, - "license": "MIT", "dependencies": { "array-includes": "^3.1.4", "array.prototype.flat": "^1.2.5", @@ -7232,6 +7509,19 @@ } } }, + "node_modules/eslint-plugin-json": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-json/-/eslint-plugin-json-3.1.0.tgz", + "integrity": "sha512-MrlG2ynFEHe7wDGwbUuFPsaT2b1uhuEFhJ+W1f1u+1C2EkXmTYJp4B1aAdQQ8M+CC3t//N/oRKiIVw14L2HR1g==", + "dev": true, + "dependencies": { + "lodash": "^4.17.21", + "vscode-json-languageservice": "^4.1.6" + }, + "engines": { + "node": ">=12.0" + } + }, "node_modules/eslint-plugin-jsx-a11y": { "version": "6.5.1", "dev": true, @@ -7269,6 +7559,27 @@ "node": ">=6.0" } }, + "node_modules/eslint-plugin-prettier": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/eslint-plugin-prettier/-/eslint-plugin-prettier-4.2.1.tgz", + "integrity": "sha512-f/0rXLXUt0oFYs8ra4w49wYZBG5GKZpAYsJSm6rnYL5uVDjd+zowwMwVZHnAjf4edNrKpCDYfXDgmRE/Ak7QyQ==", + "dev": true, + "dependencies": { + "prettier-linter-helpers": "^1.0.0" + }, + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "eslint": ">=7.28.0", + "prettier": ">=2.0.0" + }, + "peerDependenciesMeta": { + "eslint-config-prettier": { + "optional": true + } + } + }, "node_modules/eslint-plugin-react": { "version": "7.29.4", "dev": true, @@ -7718,10 +8029,17 @@ "version": "3.1.3", "license": "MIT" }, + "node_modules/fast-diff": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/fast-diff/-/fast-diff-1.2.0.tgz", + "integrity": "sha512-xJuoT5+L99XlZ8twedaRf6Ax2TgQVxvgZOYoPKqZufmJib0tL2tegPBOZb1pVNgIhlqDlA0eO0c3wBvQcmzx4w==", + "dev": true + }, "node_modules/fast-glob": { - "version": "3.2.11", + "version": "3.2.12", + "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.2.12.tgz", + "integrity": "sha512-DVj4CQIYYow0BlaelwK1pHl5n5cRSJfM60UA0zK891sVInoPri2Ekj7+e1CT3/3qxXenpI+nBBmQAcJPJgaj4w==", "dev": true, - "license": "MIT", "dependencies": { "@nodelib/fs.stat": "^2.0.2", "@nodelib/fs.walk": "^1.2.3", @@ -8251,6 +8569,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/get-tsconfig": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.2.0.tgz", + "integrity": "sha512-X8u8fREiYOE6S8hLbq99PeykTDoLVnxvF4DjWKJmz9xy2nNRdUcV8ZN9tniJFeKyTU3qnC9lL8n4Chd6LmVKHg==", + "dev": true, + "funding": { + "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1" + } + }, "node_modules/glob": { "version": "7.2.0", "license": "ISC", @@ -8328,6 +8655,12 @@ "node": ">=4" } }, + "node_modules/globalyzer": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/globalyzer/-/globalyzer-0.1.0.tgz", + "integrity": "sha512-40oNTM9UfG6aBmuKxk/giHn5nQ8RVz/SS4Ir6zgzOv9/qC3kKZ9v4etGTcJbEl/NyVQH7FGU7d+X1egr57Md2Q==", + "dev": true + }, "node_modules/globby": { "version": "11.1.0", "dev": true, @@ -8347,6 +8680,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/globrex": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/globrex/-/globrex-0.1.2.tgz", + "integrity": "sha512-uHJgbwAMwNFf5mLst7IWLNg14x1CkeqglJb/K3doi4dw6q2IvAAmM/Y81kevy83wP+Sst+nutFTYOGg3d1lsxg==", + "dev": true + }, "node_modules/graceful-fs": { "version": "4.2.10", "dev": true, @@ -8487,11 +8826,6 @@ "wbuf": "^1.1.0" } }, - "node_modules/hpack.js/node_modules/isarray": { - "version": "1.0.0", - "dev": true, - "license": "MIT" - }, "node_modules/hpack.js/node_modules/readable-stream": { "version": "2.3.7", "dev": true, @@ -8786,6 +9120,19 @@ "node": ">= 4" } }, + "node_modules/image-size": { + "version": "0.5.5", + "resolved": "https://registry.npmjs.org/image-size/-/image-size-0.5.5.tgz", + "integrity": "sha512-6TDAlDPZxUFCv+fuOkIoXT/V/f3Qbq8e37p+YOiYrUv3v9cc3/6x78VdfPgFVaB9dZYeLUfKgHRebpkm/oP2VQ==", + "dev": true, + "optional": true, + "bin": { + "image-size": "bin/image-size.js" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/immer": { "version": "9.0.12", "dev": true, @@ -8936,9 +9283,10 @@ } }, "node_modules/is-core-module": { - "version": "2.9.0", + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.11.0.tgz", + "integrity": "sha512-RRjxlvLDkD1YJwDbroBHMb+cukurkDWNyHx7D3oNB5x9rb5ogcksMC5wHCadcXoo67gVr/+3GFySh3134zi6rw==", "dev": true, - "license": "MIT", "dependencies": { "has": "^1.0.3" }, @@ -9066,6 +9414,18 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/is-plain-object": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz", + "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==", + "dev": true, + "dependencies": { + "isobject": "^3.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/is-potential-custom-element-name": { "version": "1.0.1", "dev": true, @@ -9168,6 +9528,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-what": { + "version": "3.14.1", + "resolved": "https://registry.npmjs.org/is-what/-/is-what-3.14.1.tgz", + "integrity": "sha512-sNxgpk9793nzSs7bA6JQJGeIuRBQhAaNGG77kzYQgMkrID+lS6SlK07K5LaptscDlSaIgH+GPFzf+d75FVxozA==", + "dev": true + }, "node_modules/is-wsl": { "version": "2.2.0", "dev": true, @@ -9179,11 +9545,26 @@ "node": ">=8" } }, + "node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "dev": true + }, "node_modules/isexe": { "version": "2.0.0", "dev": true, "license": "ISC" }, + "node_modules/isobject": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz", + "integrity": "sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/istanbul-lib-coverage": { "version": "3.2.0", "dev": true, @@ -10527,7 +10908,8 @@ }, "node_modules/json2mq": { "version": "0.2.0", - "license": "MIT", + "resolved": "https://registry.npmjs.org/json2mq/-/json2mq-0.2.0.tgz", + "integrity": "sha512-SzoRg7ux5DWTII9J2qkrZrqV1gt+rTaoufMxEzXbS26Uid0NwaJd123HcoB80TgubEppxxIGdNxCx50fEoEWQA==", "dependencies": { "string-convert": "^0.2.0" } @@ -10543,6 +10925,12 @@ "node": ">=6" } }, + "node_modules/jsonc-parser": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/jsonc-parser/-/jsonc-parser-3.2.0.tgz", + "integrity": "sha512-gfFQZrcTc8CnKXp6Y4/CBT3fTc0OVuDofpre4aEeEpSBPV5X5v4+Vmx+8snU7RLPrNHPKSgLxGo9YuQzz20o+w==", + "dev": true + }, "node_modules/jsonfile": { "version": "6.1.0", "dev": true, @@ -10611,14 +10999,92 @@ "language-subtag-registry": "~0.3.2" } }, - "node_modules/leven": { - "version": "3.1.0", + "node_modules/less": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/less/-/less-4.1.3.tgz", + "integrity": "sha512-w16Xk/Ta9Hhyei0Gpz9m7VS8F28nieJaL/VyShID7cYvP6IL5oHeL6p4TXSDJqZE/lNv0oJ2pGVjJsRkfwm5FA==", "dev": true, - "license": "MIT", + "dependencies": { + "copy-anything": "^2.0.1", + "parse-node-version": "^1.0.1", + "tslib": "^2.3.0" + }, + "bin": { + "lessc": "bin/lessc" + }, "engines": { "node": ">=6" - } - }, + }, + "optionalDependencies": { + "errno": "^0.1.1", + "graceful-fs": "^4.1.2", + "image-size": "~0.5.0", + "make-dir": "^2.1.0", + "mime": "^1.4.1", + "needle": "^3.1.0", + "source-map": "~0.6.0" + } + }, + "node_modules/less-loader": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/less-loader/-/less-loader-7.3.0.tgz", + "integrity": "sha512-Mi8915g7NMaLlgi77mgTTQvK022xKRQBIVDSyfl3ErTuBhmZBQab0mjeJjNNqGbdR+qrfTleKXqbGI4uEFavxg==", + "dev": true, + "dependencies": { + "klona": "^2.0.4", + "loader-utils": "^2.0.0", + "schema-utils": "^3.0.0" + }, + "engines": { + "node": ">= 10.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + }, + "peerDependencies": { + "less": "^3.5.0 || ^4.0.0", + "webpack": "^4.0.0 || ^5.0.0" + } + }, + "node_modules/less/node_modules/make-dir": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-2.1.0.tgz", + "integrity": "sha512-LS9X+dc8KLxXCb8dni79fLIIUA5VyZoyjSMCwTluaXA0o27cCK0bhXkpgw+sTXVpPy/lSO57ilRixqk0vDmtRA==", + "dev": true, + "optional": true, + "dependencies": { + "pify": "^4.0.1", + "semver": "^5.6.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/less/node_modules/semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", + "dev": true, + "optional": true, + "bin": { + "semver": "bin/semver" + } + }, + "node_modules/less/node_modules/tslib": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.0.tgz", + "integrity": "sha512-d6xOpEDfsi2CZVlPQzGeux8XMwLT9hssAsaPYExaQMuYskwb+x1x7J371tWlbBdWHroy99KnVB6qIkUbs5X3UQ==", + "dev": true + }, + "node_modules/leven": { + "version": "3.1.0", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/levn": { "version": "0.4.1", "dev": true, @@ -10867,9 +11333,10 @@ } }, "node_modules/loader-utils": { - "version": "2.0.2", + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/loader-utils/-/loader-utils-2.0.4.tgz", + "integrity": "sha512-xXqpXoINfFhgua9xiqD8fPFHgkoq1mmmpE92WlDbm9rNRd/EbRb+Gqf908T2DMfuHjjJlksiK2RbHVOdD/MqSw==", "dev": true, - "license": "MIT", "dependencies": { "big.js": "^5.2.2", "emojis-list": "^3.0.0", @@ -11040,6 +11507,13 @@ "semver": "bin/semver.js" } }, + "node_modules/make-error": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", + "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==", + "dev": true, + "peer": true + }, "node_modules/makeerror": { "version": "1.0.12", "dev": true, @@ -11296,6 +11770,47 @@ "dev": true, "license": "MIT" }, + "node_modules/needle": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/needle/-/needle-3.1.0.tgz", + "integrity": "sha512-gCE9weDhjVGCRqS8dwDR/D3GTAeyXLXuqp7I8EzH6DllZGXSUyxuqqLh+YX9rMAWaaTFyVAg6rHGL25dqvczKw==", + "dev": true, + "optional": true, + "dependencies": { + "debug": "^3.2.6", + "iconv-lite": "^0.6.3", + "sax": "^1.2.4" + }, + "bin": { + "needle": "bin/needle" + }, + "engines": { + "node": ">= 4.4.x" + } + }, + "node_modules/needle/node_modules/debug": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", + "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", + "dev": true, + "optional": true, + "dependencies": { + "ms": "^2.1.1" + } + }, + "node_modules/needle/node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "dev": true, + "optional": true, + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/negotiator": { "version": "0.6.3", "dev": true, @@ -11337,9 +11852,10 @@ "license": "MIT" }, "node_modules/node-releases": { - "version": "2.0.4", - "dev": true, - "license": "MIT" + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.6.tgz", + "integrity": "sha512-PiVXnNuFm5+iYkLBNeq5211hvO38y63T0i2KKh2KnUs3RpzJ+JtODFjkD8yjLwnDkTYF1eKXheUwdssR+NRZdg==", + "dev": true }, "node_modules/normalize-path": { "version": "3.0.0", @@ -11698,6 +12214,15 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/parse-node-version": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/parse-node-version/-/parse-node-version-1.0.1.tgz", + "integrity": "sha512-3YHlOa/JgH6Mnpr05jP9eDG254US9ek25LyIxZlDItp2iJtwyaXQb57lBYLdT3MowkUFYEV2XXNAYIPlESvJlA==", + "dev": true, + "engines": { + "node": ">= 0.10" + } + }, "node_modules/parse5": { "version": "6.0.1", "dev": true, @@ -11793,6 +12318,16 @@ "node": ">=0.10" } }, + "node_modules/pify": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz", + "integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==", + "dev": true, + "optional": true, + "engines": { + "node": ">=6" + } + }, "node_modules/pirates": { "version": "4.0.5", "dev": true, @@ -11928,7 +12463,9 @@ } }, "node_modules/postcss": { - "version": "8.4.13", + "version": "8.4.18", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.18.tgz", + "integrity": "sha512-Wi8mWhncLJm11GATDaQKobXSNEYGUHeQLiQqDFG1qQ5UTDPTEvKw0Xt5NsTpktGTwLps3ByrWsBrG0rB8YQ9oA==", "dev": true, "funding": [ { @@ -11940,9 +12477,8 @@ "url": "https://tidelift.com/funding/github/npm/postcss" } ], - "license": "MIT", "dependencies": { - "nanoid": "^3.3.3", + "nanoid": "^3.3.4", "picocolors": "^1.0.0", "source-map-js": "^1.0.2" }, @@ -13085,6 +13621,18 @@ "url": "https://github.com/prettier/prettier?sponsor=1" } }, + "node_modules/prettier-linter-helpers": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/prettier-linter-helpers/-/prettier-linter-helpers-1.0.0.tgz", + "integrity": "sha512-GbK2cP9nraSSUF9N2XwUwqfzlAFlMNYYl+ShE/V+H8a9uNl/oUqB1w2EL54Jh0OlyRSd8RfWYJ3coVS4TROP2w==", + "dev": true, + "dependencies": { + "fast-diff": "^1.1.2" + }, + "engines": { + "node": ">=6.0.0" + } + }, "node_modules/pretty-bytes": { "version": "5.6.0", "dev": true, @@ -13188,6 +13736,13 @@ "node": ">= 0.10" } }, + "node_modules/prr": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/prr/-/prr-1.0.1.tgz", + "integrity": "sha512-yPw4Sng1gWghHQWj0B3ZggWUm4qVbPwPFcRG8KyxiU7J2OHFSoEHKS+EZ3fv5l1t9CyCiop6l/ZYeWbrgoQejw==", + "dev": true, + "optional": true + }, "node_modules/psl": { "version": "1.8.0", "dev": true, @@ -13317,14 +13872,15 @@ } }, "node_modules/rc-cascader": { - "version": "3.5.0", - "license": "MIT", + "version": "3.7.0", + "resolved": "https://registry.npmjs.org/rc-cascader/-/rc-cascader-3.7.0.tgz", + "integrity": "sha512-SFtGpwmYN7RaWEAGTS4Rkc62ZV/qmQGg/tajr/7mfIkleuu8ro9Hlk6J+aA0x1YS4zlaZBtTcSaXM01QMiEV/A==", "dependencies": { "@babel/runtime": "^7.12.5", "array-tree-filter": "^2.1.0", "classnames": "^2.3.1", "rc-select": "~14.1.0", - "rc-tree": "~5.5.0", + "rc-tree": "~5.7.0", "rc-util": "^5.6.1" }, "peerDependencies": { @@ -13345,8 +13901,9 @@ } }, "node_modules/rc-collapse": { - "version": "3.1.4", - "license": "MIT", + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/rc-collapse/-/rc-collapse-3.3.1.tgz", + "integrity": "sha512-cOJfcSe3R8vocrF8T+PgaHDrgeA1tX+lwfhwSj60NX9QVRidsILIbRNDLD6nAzmcvVC5PWiIRiR4S1OobxdhCg==", "dependencies": { "@babel/runtime": "^7.10.1", "classnames": "2.x", @@ -13360,8 +13917,9 @@ } }, "node_modules/rc-dialog": { - "version": "8.8.1", - "license": "MIT", + "version": "8.9.0", + "resolved": "https://registry.npmjs.org/rc-dialog/-/rc-dialog-8.9.0.tgz", + "integrity": "sha512-Cp0tbJnrvPchJfnwIvOMWmJ4yjX3HWFatO6oBFD1jx8QkgsQCR0p8nUWAKdd3seLJhEC39/v56kZaEjwp9muoQ==", "dependencies": { "@babel/runtime": "^7.10.1", "classnames": "^2.2.6", @@ -13374,11 +13932,14 @@ } }, "node_modules/rc-drawer": { - "version": "4.4.3", + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/rc-drawer/-/rc-drawer-5.1.0.tgz", + "integrity": "sha512-pU3Tsn99pxGdYowXehzZbdDVE+4lDXSGb7p8vA9mSmr569oc2Izh4Zw5vLKSe/Xxn2p5MSNbLVqD4tz+pK6SOw==", "dependencies": { "@babel/runtime": "^7.10.1", "classnames": "^2.2.6", - "rc-util": "^5.7.0" + "rc-motion": "^2.6.1", + "rc-util": "^5.21.2" }, "peerDependencies": { "react": ">=16.9.0", @@ -13386,12 +13947,13 @@ } }, "node_modules/rc-dropdown": { - "version": "3.5.2", - "license": "MIT", + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/rc-dropdown/-/rc-dropdown-4.0.1.tgz", + "integrity": "sha512-OdpXuOcme1rm45cR0Jzgfl1otzmU4vuBVb+etXM8vcaULGokAKVpKlw8p6xzspG7jGd/XxShvq+N3VNEfk/l5g==", "dependencies": { - "@babel/runtime": "^7.10.1", + "@babel/runtime": "^7.18.3", "classnames": "^2.2.6", - "rc-trigger": "^5.0.4", + "rc-trigger": "^5.3.1", "rc-util": "^5.17.0" }, "peerDependencies": { @@ -13400,10 +13962,11 @@ } }, "node_modules/rc-field-form": { - "version": "1.26.3", - "license": "MIT", + "version": "1.27.3", + "resolved": "https://registry.npmjs.org/rc-field-form/-/rc-field-form-1.27.3.tgz", + "integrity": "sha512-HGqxHnmGQgkPApEcikV4qTg3BLPC82uB/cwBDftDt1pYaqitJfSl5TFTTUMKVEJVT5RqJ2Zi68ME1HmIMX2HAw==", "dependencies": { - "@babel/runtime": "^7.8.4", + "@babel/runtime": "^7.18.0", "async-validator": "^4.1.0", "rc-util": "^5.8.0" }, @@ -13416,12 +13979,13 @@ } }, "node_modules/rc-image": { - "version": "5.6.2", - "license": "MIT", + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/rc-image/-/rc-image-5.7.1.tgz", + "integrity": "sha512-QyMfdhoUfb5W14plqXSisaYwpdstcLYnB0MjX5ccIK2rydQM9sDPuekQWu500DDGR2dBaIF5vx9XbWkNFK17Fg==", "dependencies": { "@babel/runtime": "^7.11.2", "classnames": "^2.2.6", - "rc-dialog": "~8.8.0", + "rc-dialog": "~8.9.0", "rc-util": "^5.0.6" }, "peerDependencies": { @@ -13430,8 +13994,9 @@ } }, "node_modules/rc-input": { - "version": "0.0.1-alpha.7", - "license": "MIT", + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/rc-input/-/rc-input-0.1.4.tgz", + "integrity": "sha512-FqDdNz+fV2dKNgfXzcSLKvC+jEs1709t7nD+WdfjrdSaOcefpgc7BUJYadc3usaING+b7ediMTfKxuJBsEFbXA==", "dependencies": { "@babel/runtime": "^7.11.1", "classnames": "^2.2.1", @@ -13443,12 +14008,13 @@ } }, "node_modules/rc-input-number": { - "version": "7.3.4", - "license": "MIT", + "version": "7.3.9", + "resolved": "https://registry.npmjs.org/rc-input-number/-/rc-input-number-7.3.9.tgz", + "integrity": "sha512-u0+miS+SATdb6DtssYei2JJ1WuZME+nXaG6XGtR8maNyW5uGDytfDu60OTWLQEb0Anv/AcCzehldV8CKmKyQfA==", "dependencies": { "@babel/runtime": "^7.10.1", "classnames": "^2.2.5", - "rc-util": "^5.9.8" + "rc-util": "^5.23.0" }, "peerDependencies": { "react": ">=16.9.0", @@ -13456,15 +14022,16 @@ } }, "node_modules/rc-mentions": { - "version": "1.7.1", - "license": "MIT", + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/rc-mentions/-/rc-mentions-1.10.0.tgz", + "integrity": "sha512-oMlYWnwXSxP2NQVlgxOTzuG/u9BUc3ySY78K3/t7MNhJWpZzXTao+/Bic6tyZLuNCO89//hVQJBdaR2rnFQl6Q==", "dependencies": { "@babel/runtime": "^7.10.1", "classnames": "^2.2.6", - "rc-menu": "~9.5.1", - "rc-textarea": "^0.3.0", + "rc-menu": "~9.6.0", + "rc-textarea": "^0.4.0", "rc-trigger": "^5.0.4", - "rc-util": "^5.0.1" + "rc-util": "^5.22.5" }, "peerDependencies": { "react": ">=16.9.0", @@ -13472,8 +14039,9 @@ } }, "node_modules/rc-menu": { - "version": "9.5.5", - "license": "MIT", + "version": "9.6.4", + "resolved": "https://registry.npmjs.org/rc-menu/-/rc-menu-9.6.4.tgz", + "integrity": "sha512-6DiNAjxjVIPLZXHffXxxcyE15d4isRL7iQ1ru4MqYDH2Cqc5bW96wZOdMydFtGLyDdnmEQ9jVvdCE9yliGvzkw==", "dependencies": { "@babel/runtime": "^7.10.1", "classnames": "2.x", @@ -13489,8 +14057,9 @@ } }, "node_modules/rc-motion": { - "version": "2.6.0", - "license": "MIT", + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/rc-motion/-/rc-motion-2.6.2.tgz", + "integrity": "sha512-4w1FaX3dtV749P8GwfS4fYnFG4Rb9pxvCYPc/b2fw1cmlHJWNNgOFIz7ysiD+eOrzJSvnLJWlNQQncpNMXwwpg==", "dependencies": { "@babel/runtime": "^7.11.1", "classnames": "^2.2.1", @@ -13519,8 +14088,9 @@ } }, "node_modules/rc-overflow": { - "version": "1.2.5", - "license": "MIT", + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/rc-overflow/-/rc-overflow-1.2.8.tgz", + "integrity": "sha512-QJ0UItckWPQ37ZL1dMEBAdY1dhfTXFL9k6oTTcyydVwoUNMnMqCGqnRNA98axSr/OeDKqR6DVFyi8eA5RQI/uQ==", "dependencies": { "@babel/runtime": "^7.11.1", "classnames": "^2.2.1", @@ -13533,8 +14103,9 @@ } }, "node_modules/rc-pagination": { - "version": "3.1.16", - "license": "MIT", + "version": "3.1.17", + "resolved": "https://registry.npmjs.org/rc-pagination/-/rc-pagination-3.1.17.tgz", + "integrity": "sha512-/BQ5UxcBnW28vFAcP2hfh+Xg15W0QZn8TWYwdCApchMH1H0CxiaUUcULP8uXcFM1TygcdKWdt3JqsL9cTAfdkQ==", "dependencies": { "@babel/runtime": "^7.10.1", "classnames": "^2.2.1" @@ -13545,8 +14116,9 @@ } }, "node_modules/rc-picker": { - "version": "2.6.8", - "license": "MIT", + "version": "2.6.11", + "resolved": "https://registry.npmjs.org/rc-picker/-/rc-picker-2.6.11.tgz", + "integrity": "sha512-INJ7ULu+Kj4UgqbcqE8Q+QpMw55xFf9kkyLBHJFk0ihjJpAV4glialRfqHE7k4KX2BWYPQfpILwhwR14x2EiRQ==", "dependencies": { "@babel/runtime": "^7.10.1", "classnames": "^2.2.1", @@ -13566,8 +14138,9 @@ } }, "node_modules/rc-progress": { - "version": "3.2.4", - "license": "MIT", + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/rc-progress/-/rc-progress-3.3.3.tgz", + "integrity": "sha512-MDVNVHzGanYtRy2KKraEaWeZLri2ZHWIRyaE1a9MQ2MuJ09m+Wxj5cfcaoaR6z5iRpHpA59YeUxAlpML8N4PJw==", "dependencies": { "@babel/runtime": "^7.10.1", "classnames": "^2.2.6", @@ -13596,7 +14169,8 @@ }, "node_modules/rc-resize-observer": { "version": "1.2.0", - "license": "MIT", + "resolved": "https://registry.npmjs.org/rc-resize-observer/-/rc-resize-observer-1.2.0.tgz", + "integrity": "sha512-6W+UzT3PyDM0wVCEHfoW3qTHPTvbdSgiA43buiy8PzmeMnfgnDeb9NjdimMXMl3/TcrvvWl5RRVdp+NqcR47pQ==", "dependencies": { "@babel/runtime": "^7.10.1", "classnames": "^2.2.1", @@ -13623,8 +14197,9 @@ } }, "node_modules/rc-select": { - "version": "14.1.2", - "license": "MIT", + "version": "14.1.13", + "resolved": "https://registry.npmjs.org/rc-select/-/rc-select-14.1.13.tgz", + "integrity": "sha512-WMEsC3gTwA1dbzWOdVIXDmWyidYNLq68AwvvUlRROw790uGUly0/vmqDozXrIr0QvN/A3CEULx12o+WtLCAefg==", "dependencies": { "@babel/runtime": "^7.10.1", "classnames": "2.x", @@ -13690,13 +14265,14 @@ } }, "node_modules/rc-table": { - "version": "7.24.1", - "license": "MIT", + "version": "7.26.0", + "resolved": "https://registry.npmjs.org/rc-table/-/rc-table-7.26.0.tgz", + "integrity": "sha512-0cD8e6S+DTGAt5nBZQIPFYEaIukn17sfa5uFL98faHlH/whZzD8ii3dbFL4wmUDEL4BLybhYop+QUfZJ4CPvNQ==", "dependencies": { "@babel/runtime": "^7.10.1", "classnames": "^2.2.5", "rc-resize-observer": "^1.1.0", - "rc-util": "^5.14.0", + "rc-util": "^5.22.5", "shallowequal": "^1.1.0" }, "engines": { @@ -13708,13 +14284,15 @@ } }, "node_modules/rc-tabs": { - "version": "11.13.0", - "license": "MIT", + "version": "12.2.1", + "resolved": "https://registry.npmjs.org/rc-tabs/-/rc-tabs-12.2.1.tgz", + "integrity": "sha512-09pVv4kN8VFqp6THceEmxOW8PAShQC08hrroeVYP4Y8YBFaP1PIWdyFL01czcbyz5YZFj9flZ7aljMaAl0jLVg==", "dependencies": { "@babel/runtime": "^7.11.2", "classnames": "2.x", - "rc-dropdown": "~3.5.0", - "rc-menu": "~9.5.1", + "rc-dropdown": "~4.0.0", + "rc-menu": "~9.6.0", + "rc-motion": "^2.6.2", "rc-resize-observer": "^1.0.0", "rc-util": "^5.5.0" }, @@ -13727,13 +14305,14 @@ } }, "node_modules/rc-textarea": { - "version": "0.3.7", - "license": "MIT", + "version": "0.4.6", + "resolved": "https://registry.npmjs.org/rc-textarea/-/rc-textarea-0.4.6.tgz", + "integrity": "sha512-HEKCu8nouXXayqYelQnhQm8fdH7v92pAQvfVCz+jhIPv2PHTyBxVrmoZJMn3B8cU+wdyuvRGkshngO3/TzBn4w==", "dependencies": { "@babel/runtime": "^7.10.1", "classnames": "^2.2.1", "rc-resize-observer": "^1.0.0", - "rc-util": "^5.7.0", + "rc-util": "^5.24.4", "shallowequal": "^1.1.0" }, "peerDependencies": { @@ -13742,10 +14321,12 @@ } }, "node_modules/rc-tooltip": { - "version": "5.1.1", - "license": "MIT", + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/rc-tooltip/-/rc-tooltip-5.2.2.tgz", + "integrity": "sha512-jtQzU/18S6EI3lhSGoDYhPqNpWajMtS5VV/ld1LwyfrDByQpYmw/LW6U7oFXXLukjfDHQ7Ju705A82PRNFWYhg==", "dependencies": { "@babel/runtime": "^7.11.2", + "classnames": "^2.3.1", "rc-trigger": "^5.0.0" }, "peerDependencies": { @@ -13754,14 +14335,15 @@ } }, "node_modules/rc-tree": { - "version": "5.5.0", - "license": "MIT", + "version": "5.7.0", + "resolved": "https://registry.npmjs.org/rc-tree/-/rc-tree-5.7.0.tgz", + "integrity": "sha512-F+Ewkv/UcutshnVBMISP+lPdHDlcsL+YH/MQDVWbk+QdkfID7vXiwrHMEZn31+2Rbbm21z/HPceGS8PXGMmnQg==", "dependencies": { "@babel/runtime": "^7.10.1", "classnames": "2.x", "rc-motion": "^2.0.1", "rc-util": "^5.16.1", - "rc-virtual-list": "^3.4.2" + "rc-virtual-list": "^3.4.8" }, "engines": { "node": ">=10.x" @@ -13772,13 +14354,14 @@ } }, "node_modules/rc-tree-select": { - "version": "5.3.0", - "license": "MIT", + "version": "5.5.3", + "resolved": "https://registry.npmjs.org/rc-tree-select/-/rc-tree-select-5.5.3.tgz", + "integrity": "sha512-gv8KyC6J7f9e50OkGk1ibF7v8vL+iaBnA8Ep/EVlMma2/tGdBQXO9xIvPjX8eQrZL5PjoeTUndNPM3cY3721ng==", "dependencies": { "@babel/runtime": "^7.10.1", "classnames": "2.x", "rc-select": "~14.1.0", - "rc-tree": "~5.5.0", + "rc-tree": "~5.7.0", "rc-util": "^5.16.1" }, "peerDependencies": { @@ -13787,10 +14370,11 @@ } }, "node_modules/rc-trigger": { - "version": "5.2.18", - "license": "MIT", + "version": "5.3.3", + "resolved": "https://registry.npmjs.org/rc-trigger/-/rc-trigger-5.3.3.tgz", + "integrity": "sha512-IC4nuTSAME7RJSgwvHCNDQrIzhvGMKf6NDu5veX+zk1MG7i1UnwTWWthcP9WHw3+FZfP3oZGvkrHFPu/EGkFKw==", "dependencies": { - "@babel/runtime": "^7.11.2", + "@babel/runtime": "^7.18.3", "classnames": "^2.2.6", "rc-align": "^4.0.0", "rc-motion": "^2.0.0", @@ -13818,10 +14402,11 @@ } }, "node_modules/rc-util": { - "version": "5.21.2", - "license": "MIT", + "version": "5.24.4", + "resolved": "https://registry.npmjs.org/rc-util/-/rc-util-5.24.4.tgz", + "integrity": "sha512-2a4RQnycV9eV7lVZPEJ7QwJRPlZNc06J7CwcwZo4vIHr3PfUqtYgl1EkUV9ETAc6VRRi8XZOMFhYG63whlIC9Q==", "dependencies": { - "@babel/runtime": "^7.12.5", + "@babel/runtime": "^7.18.3", "react-is": "^16.12.0", "shallowequal": "^1.1.0" }, @@ -13831,9 +14416,11 @@ } }, "node_modules/rc-virtual-list": { - "version": "3.4.7", - "license": "MIT", + "version": "3.4.11", + "resolved": "https://registry.npmjs.org/rc-virtual-list/-/rc-virtual-list-3.4.11.tgz", + "integrity": "sha512-BvUUH60kkeTBPigN5F89HtGaA5jSP4y2aM6cJ4dk9Y42I9yY+h6i08wF6UKeDcxdfOU8j3I5HxkSS/xA77J3wA==", "dependencies": { + "@babel/runtime": "^7.20.0", "classnames": "^2.2.6", "rc-resize-observer": "^1.0.0", "rc-util": "^5.15.0" @@ -13934,9 +14521,10 @@ } }, "node_modules/react-dev-utils/node_modules/loader-utils": { - "version": "3.2.0", + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/loader-utils/-/loader-utils-3.2.1.tgz", + "integrity": "sha512-ZvFw1KWS3GVyYBYb7qkmRM/WwL2TQQBxgCK62rlvm4WpVQ23Nb4tYjApUlfjrEGvOs7KHEsmyUn75OHZrJMWPw==", "dev": true, - "license": "MIT", "engines": { "node": ">= 12.13.0" } @@ -14054,6 +14642,18 @@ "node": ">=0.10.0" } }, + "node_modules/react-resizable": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/react-resizable/-/react-resizable-3.0.4.tgz", + "integrity": "sha512-StnwmiESiamNzdRHbSSvA65b0ZQJ7eVQpPusrSmcpyGKzC0gojhtO62xxH6YOBmepk9dQTBi9yxidL3W4s3EBA==", + "dependencies": { + "prop-types": "15.x", + "react-draggable": "^4.0.3" + }, + "peerDependencies": { + "react": ">= 16.3" + } + }, "node_modules/react-router": { "version": "6.3.0", "license": "MIT", @@ -14173,25 +14773,15 @@ } }, "node_modules/recursive-readdir": { - "version": "2.2.2", - "dev": true, - "license": "MIT", - "dependencies": { - "minimatch": "3.0.4" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/recursive-readdir/node_modules/minimatch": { - "version": "3.0.4", + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/recursive-readdir/-/recursive-readdir-2.2.3.tgz", + "integrity": "sha512-8HrF5ZsXk5FAH9dgsx3BlUer73nIhuj+9OrQwEbLTPOBzGkL1lsFCR01am+v+0m2Cmbs1nP12hLDl5FA7EszKA==", "dev": true, - "license": "ISC", "dependencies": { - "brace-expansion": "^1.1.7" + "minimatch": "^3.0.5" }, "engines": { - "node": "*" + "node": ">=6.0.0" } }, "node_modules/redent": { @@ -14230,8 +14820,9 @@ } }, "node_modules/regenerator-runtime": { - "version": "0.13.9", - "license": "MIT" + "version": "0.13.10", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.10.tgz", + "integrity": "sha512-KepLsg4dU12hryUO7bp/axHAKvwGOCV0sGloQtpagJ12ai+ojVDqkeGSiRX1zlq+kjIMZ1t7gpze+26QqtdGqw==" }, "node_modules/regenerator-transform": { "version": "0.15.0", @@ -14964,6 +15555,18 @@ "dev": true, "license": "ISC" }, + "node_modules/shallow-clone": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/shallow-clone/-/shallow-clone-3.0.1.tgz", + "integrity": "sha512-/6KqX+GVUdqPuPPd2LxDDxzX6CAbjJehAAOKlNpqqUpAqPM6HeL8f+o3a+JsyGjn2lv0WY8UsTgUJjU9Ok55NA==", + "dev": true, + "dependencies": { + "kind-of": "^6.0.2" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/shallowequal": { "version": "1.1.0", "license": "MIT" @@ -15233,7 +15836,8 @@ }, "node_modules/string-convert": { "version": "0.2.1", - "license": "MIT" + "resolved": "https://registry.npmjs.org/string-convert/-/string-convert-0.2.1.tgz", + "integrity": "sha512-u/1tdPl4yQnPBjnVrmdLo9gtuLvELKsAoRapekWggdiQNvvvum+jYF329d84NAa660KQw7pB2n36KrIKVoXa3A==" }, "node_modules/string-length": { "version": "4.0.2", @@ -15426,9 +16030,10 @@ } }, "node_modules/supports-hyperlinks": { - "version": "2.2.0", + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/supports-hyperlinks/-/supports-hyperlinks-2.3.0.tgz", + "integrity": "sha512-RpsAZlpWcDwOPQA22aCH4J0t7L8JmAvsCxfOSEwm7cQs3LshN36QaTkwd70DnBOXDWGssw2eUoc8CaRWT0XunA==", "dev": true, - "license": "MIT", "dependencies": { "has-flag": "^4.0.0", "supports-color": "^7.0.0" @@ -15540,6 +16145,28 @@ "dev": true, "license": "MIT" }, + "node_modules/synckit": { + "version": "0.8.4", + "resolved": "https://registry.npmjs.org/synckit/-/synckit-0.8.4.tgz", + "integrity": "sha512-Dn2ZkzMdSX827QbowGbU/4yjWuvNaCoScLLoMo/yKbu+P4GBR6cRGKZH27k6a9bRzdqcyd1DE96pQtQ6uNkmyw==", + "dev": true, + "dependencies": { + "@pkgr/utils": "^2.3.1", + "tslib": "^2.4.0" + }, + "engines": { + "node": "^14.18.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/unts" + } + }, + "node_modules/synckit/node_modules/tslib": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.0.tgz", + "integrity": "sha512-d6xOpEDfsi2CZVlPQzGeux8XMwLT9hssAsaPYExaQMuYskwb+x1x7J371tWlbBdWHroy99KnVB6qIkUbs5X3UQ==", + "dev": true + }, "node_modules/tailwindcss": { "version": "3.0.24", "dev": true, @@ -15726,6 +16353,16 @@ "dev": true, "license": "MIT" }, + "node_modules/tiny-glob": { + "version": "0.2.9", + "resolved": "https://registry.npmjs.org/tiny-glob/-/tiny-glob-0.2.9.tgz", + "integrity": "sha512-g/55ssRPUjShh+xkfx9UPDXqhckHEsHr4Vd9zX55oSdGZc/MD0m3sferOkwWtp98bv+kcVfEHtRJgBVJzelrzg==", + "dev": true, + "dependencies": { + "globalyzer": "0.1.0", + "globrex": "^0.1.2" + } + }, "node_modules/tmpl": { "version": "1.0.5", "dev": true, @@ -15796,6 +16433,67 @@ "dev": true, "license": "MIT" }, + "node_modules/ts-node": { + "version": "10.9.1", + "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.1.tgz", + "integrity": "sha512-NtVysVPkxxrwFGUUxGYhfux8k78pQB3JqYBXlLRZgdGUqTO5wU/UyHop5p70iEbGhB7q5KmiZiU0Y3KlJrScEw==", + "dev": true, + "peer": true, + "dependencies": { + "@cspotcode/source-map-support": "^0.8.0", + "@tsconfig/node10": "^1.0.7", + "@tsconfig/node12": "^1.0.7", + "@tsconfig/node14": "^1.0.0", + "@tsconfig/node16": "^1.0.2", + "acorn": "^8.4.1", + "acorn-walk": "^8.1.1", + "arg": "^4.1.0", + "create-require": "^1.1.0", + "diff": "^4.0.1", + "make-error": "^1.1.1", + "v8-compile-cache-lib": "^3.0.1", + "yn": "3.1.1" + }, + "bin": { + "ts-node": "dist/bin.js", + "ts-node-cwd": "dist/bin-cwd.js", + "ts-node-esm": "dist/bin-esm.js", + "ts-node-script": "dist/bin-script.js", + "ts-node-transpile-only": "dist/bin-transpile.js", + "ts-script": "dist/bin-script-deprecated.js" + }, + "peerDependencies": { + "@swc/core": ">=1.2.50", + "@swc/wasm": ">=1.2.50", + "@types/node": "*", + "typescript": ">=2.7" + }, + "peerDependenciesMeta": { + "@swc/core": { + "optional": true + }, + "@swc/wasm": { + "optional": true + } + } + }, + "node_modules/ts-node/node_modules/acorn-walk": { + "version": "8.2.0", + "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.2.0.tgz", + "integrity": "sha512-k+iyHEuPgSw6SbuDpGQM+06HQUa04DZ3o+F6CSzXMvvI5KMvnaEqXe+YVe555R9nn6GPt404fos4wcgpw12SDA==", + "dev": true, + "peer": true, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/ts-node/node_modules/arg": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz", + "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==", + "dev": true, + "peer": true + }, "node_modules/tsconfig-paths": { "version": "3.14.1", "dev": true, @@ -15998,6 +16696,32 @@ "yarn": "*" } }, + "node_modules/update-browserslist-db": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.10.tgz", + "integrity": "sha512-OztqDenkfFkbSG+tRxBeAnCVPckDBcvibKd35yDONx6OU8N7sqgwc7rCbkJ/WcYtVRZ4ba68d6byhC21GFh7sQ==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + } + ], + "dependencies": { + "escalade": "^3.1.1", + "picocolors": "^1.0.0" + }, + "bin": { + "browserslist-lint": "cli.js" + }, + "peerDependencies": { + "browserslist": ">= 4.21.0" + } + }, "node_modules/uri-js": { "version": "4.4.1", "dev": true, @@ -16051,6 +16775,13 @@ "dev": true, "license": "MIT" }, + "node_modules/v8-compile-cache-lib": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", + "integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==", + "dev": true, + "peer": true + }, "node_modules/v8-to-istanbul": { "version": "8.1.1", "dev": true, @@ -16080,6 +16811,43 @@ "node": ">= 0.8" } }, + "node_modules/vscode-json-languageservice": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/vscode-json-languageservice/-/vscode-json-languageservice-4.2.1.tgz", + "integrity": "sha512-xGmv9QIWs2H8obGbWg+sIPI/3/pFgj/5OWBhNzs00BkYQ9UaB2F6JJaGB/2/YOZJ3BvLXQTC4Q7muqU25QgAhA==", + "dev": true, + "dependencies": { + "jsonc-parser": "^3.0.0", + "vscode-languageserver-textdocument": "^1.0.3", + "vscode-languageserver-types": "^3.16.0", + "vscode-nls": "^5.0.0", + "vscode-uri": "^3.0.3" + } + }, + "node_modules/vscode-languageserver-textdocument": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/vscode-languageserver-textdocument/-/vscode-languageserver-textdocument-1.0.7.tgz", + "integrity": "sha512-bFJH7UQxlXT8kKeyiyu41r22jCZXG8kuuVVA33OEJn1diWOZK5n8zBSPZFHVBOu8kXZ6h0LIRhf5UnCo61J4Hg==", + "dev": true + }, + "node_modules/vscode-languageserver-types": { + "version": "3.17.2", + "resolved": "https://registry.npmjs.org/vscode-languageserver-types/-/vscode-languageserver-types-3.17.2.tgz", + "integrity": "sha512-zHhCWatviizPIq9B7Vh9uvrH6x3sK8itC84HkamnBWoDFJtzBf7SWlpLCZUit72b3os45h6RWQNC9xHRDF8dRA==", + "dev": true + }, + "node_modules/vscode-nls": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/vscode-nls/-/vscode-nls-5.2.0.tgz", + "integrity": "sha512-RAaHx7B14ZU04EU31pT+rKz2/zSl7xMsfIZuo8pd+KZO6PXtQmpevpq3vxvWNcrGbdmhM/rr5Uw5Mz+NBfhVng==", + "dev": true + }, + "node_modules/vscode-uri": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/vscode-uri/-/vscode-uri-3.0.6.tgz", + "integrity": "sha512-fmL7V1eiDBFRRnu+gfRWTzyPpNIHJTc4mWnFkwBUmO9U3KPgJAmTx7oxi2bl/Rh6HLdU7+4C9wlj0k2E4AdKFQ==", + "dev": true + }, "node_modules/w3c-hr-time": { "version": "1.0.2", "dev": true, @@ -16139,8 +16907,9 @@ }, "node_modules/webpack": { "version": "5.72.0", + "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.72.0.tgz", + "integrity": "sha512-qmSmbspI0Qo5ld49htys8GY9XhS9CGqFoHTsOVAnjBdg0Zn79y135R+k4IR4rKK6+eKaabMhJwiVB7xw0SJu5w==", "dev": true, - "license": "MIT", "dependencies": { "@types/eslint-scope": "^3.7.3", "@types/estree": "^0.0.51", @@ -16359,6 +17128,19 @@ "node": ">=10.13.0" } }, + "node_modules/webpack-merge": { + "version": "5.8.0", + "resolved": "https://registry.npmjs.org/webpack-merge/-/webpack-merge-5.8.0.tgz", + "integrity": "sha512-/SaI7xY0831XwP6kzuwhKWVKDP9t1QY1h65lAFLbZqMPIuYcD9QAW4u9STIbU9kaJbPBB/geU/gLr1wDjOhQ+Q==", + "dev": true, + "dependencies": { + "clone-deep": "^4.0.1", + "wildcard": "^2.0.0" + }, + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/webpack-sources": { "version": "3.2.3", "dev": true, @@ -16465,6 +17247,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/wildcard": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/wildcard/-/wildcard-2.0.0.tgz", + "integrity": "sha512-JcKqAHLPxcdb9KM49dufGXn2x3ssnfjbcaQdLlfZsL9rH9wgDQjUtDxbo8NE0F6SFvydeu1VhZe7hZuHsB2/pw==", + "dev": true + }, "node_modules/word-wrap": { "version": "1.2.3", "dev": true, @@ -16815,6 +17603,16 @@ "node": ">=10" } }, + "node_modules/yn": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", + "integrity": "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==", + "dev": true, + "peer": true, + "engines": { + "node": ">=6" + } + }, "node_modules/yocto-queue": { "version": "0.1.0", "dev": true, @@ -16844,6 +17642,8 @@ }, "@ant-design/icons": { "version": "4.7.0", + "resolved": "https://registry.npmjs.org/@ant-design/icons/-/icons-4.7.0.tgz", + "integrity": "sha512-aoB4Z7JA431rt6d4u+8xcNPPCrdufSRMUOpxa1ab6mz1JCQZOEVolj2WVs/tDFmN62zzK30mNelEsprLYsSF3g==", "requires": { "@ant-design/colors": "^6.0.0", "@ant-design/icons-svg": "^4.2.1", @@ -16856,13 +17656,15 @@ "version": "4.2.1" }, "@ant-design/react-slick": { - "version": "0.28.4", + "version": "0.29.2", + "resolved": "https://registry.npmjs.org/@ant-design/react-slick/-/react-slick-0.29.2.tgz", + "integrity": "sha512-kgjtKmkGHa19FW21lHnAfyyH9AAoh35pBdcJ53rHmQ3O+cfFHGHnUbj/HFrRNJ5vIts09FKJVAD8RpaC+RaWfA==", "requires": { "@babel/runtime": "^7.10.4", "classnames": "^2.2.5", "json2mq": "^0.2.0", "lodash": "^4.17.21", - "resize-observer-polyfill": "^1.5.0" + "resize-observer-polyfill": "^1.5.1" } }, "@apideck/better-ajv-errors": { @@ -17966,9 +18768,11 @@ } }, "@babel/runtime": { - "version": "7.17.9", + "version": "7.20.0", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.20.0.tgz", + "integrity": "sha512-NDYdls71fTXoU8TZHfbBWg7DiZfNzClcKui/+kyi6ppD2L1qnWW3VV6CjtaBXSUGGhiTWJ6ereOIkUvenif66Q==", "requires": { - "regenerator-runtime": "^0.13.4" + "regenerator-runtime": "^0.13.10" } }, "@babel/runtime-corejs3": { @@ -18016,6 +18820,31 @@ "version": "0.2.3", "dev": true }, + "@craco/craco": { + "version": "7.0.0-alpha.8", + "resolved": "https://registry.npmjs.org/@craco/craco/-/craco-7.0.0-alpha.8.tgz", + "integrity": "sha512-IN3/ldPaktGflPu342cg7n8LYa2c3x9H2XzngUkDzTjro25ig1GyVcUdnG1U0X6wrRTF9K1AxZ5su9jLbdyFUw==", + "dev": true, + "requires": { + "autoprefixer": "^10.4.12", + "cosmiconfig": "^7.0.1", + "cosmiconfig-typescript-loader": "^4.1.1", + "cross-spawn": "^7.0.3", + "lodash": "^4.17.21", + "semver": "^7.3.7", + "webpack-merge": "^5.8.0" + } + }, + "@cspotcode/source-map-support": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", + "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==", + "dev": true, + "peer": true, + "requires": { + "@jridgewell/trace-mapping": "0.3.9" + } + }, "@csstools/normalize.css": { "version": "12.0.0", "dev": true @@ -18515,6 +19344,28 @@ "fastq": "^1.6.0" } }, + "@pkgr/utils": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/@pkgr/utils/-/utils-2.3.1.tgz", + "integrity": "sha512-wfzX8kc1PMyUILA+1Z/EqoE4UCXGy0iRGMhPwdfae1+f0OXlLqCk+By+aMzgJBzR9AzS4CDizioG6Ss1gvAFJw==", + "dev": true, + "requires": { + "cross-spawn": "^7.0.3", + "is-glob": "^4.0.3", + "open": "^8.4.0", + "picocolors": "^1.0.0", + "tiny-glob": "^0.2.9", + "tslib": "^2.4.0" + }, + "dependencies": { + "tslib": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.0.tgz", + "integrity": "sha512-d6xOpEDfsi2CZVlPQzGeux8XMwLT9hssAsaPYExaQMuYskwb+x1x7J371tWlbBdWHroy99KnVB6qIkUbs5X3UQ==", + "dev": true + } + } + }, "@pmmmwh/react-refresh-webpack-plugin": { "version": "0.5.5", "dev": true, @@ -18769,6 +19620,34 @@ "version": "0.2.0", "dev": true }, + "@tsconfig/node10": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.9.tgz", + "integrity": "sha512-jNsYVVxU8v5g43Erja32laIDHXeoNvFEpX33OK4d6hljo3jDhCBDhx5dhCCTMWUojscpAagGiRkBKxpdl9fxqA==", + "dev": true, + "peer": true + }, + "@tsconfig/node12": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/@tsconfig/node12/-/node12-1.0.11.tgz", + "integrity": "sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag==", + "dev": true, + "peer": true + }, + "@tsconfig/node14": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@tsconfig/node14/-/node14-1.0.3.tgz", + "integrity": "sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow==", + "dev": true, + "peer": true + }, + "@tsconfig/node16": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@tsconfig/node16/-/node16-1.0.3.tgz", + "integrity": "sha512-yOlFc+7UtL/89t2ZhjPvvB/DeAr3r+Dq58IgzsFkOAvVC6NMJXmCGjbptdXdR9qsX7pKcTL+s87FtYREi2dEEQ==", + "dev": true, + "peer": true + }, "@types/aria-query": { "version": "4.2.2", "dev": true @@ -18993,6 +19872,15 @@ "redux": "^4.0.0" } }, + "@types/react-resizable": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/react-resizable/-/react-resizable-3.0.3.tgz", + "integrity": "sha512-W/QsUOZoXBAIBQNhNm95A5ohoaiUA874lWQytO2UP9dOjp5JHO9+a0cwYNabea7sA12ZDJnGVUFZxcNaNksAWA==", + "dev": true, + "requires": { + "@types/react": "*" + } + }, "@types/resolve": { "version": "1.17.1", "dev": true, @@ -19573,51 +20461,53 @@ } }, "antd": { - "version": "4.20.2", + "version": "4.23.6", + "resolved": "https://registry.npmjs.org/antd/-/antd-4.23.6.tgz", + "integrity": "sha512-AYH57cWBDe1ChtbnvG8i9dpKG4WnjE3AG0zIKpXByFNnxsr4saV6/19ihE8/ImSGpohN4E2zTXmo7R5/MyVRKQ==", "requires": { "@ant-design/colors": "^6.0.0", "@ant-design/icons": "^4.7.0", - "@ant-design/react-slick": "~0.28.1", - "@babel/runtime": "^7.12.5", + "@ant-design/react-slick": "~0.29.1", + "@babel/runtime": "^7.18.3", "@ctrl/tinycolor": "^3.4.0", "classnames": "^2.2.6", "copy-to-clipboard": "^3.2.0", "lodash": "^4.17.21", "memoize-one": "^6.0.0", "moment": "^2.29.2", - "rc-cascader": "~3.5.0", + "rc-cascader": "~3.7.0", "rc-checkbox": "~2.3.0", - "rc-collapse": "~3.1.0", - "rc-dialog": "~8.8.1", - "rc-drawer": "~4.4.2", - "rc-dropdown": "~3.5.0", - "rc-field-form": "~1.26.1", - "rc-image": "~5.6.0", - "rc-input": "~0.0.1-alpha.5", - "rc-input-number": "~7.3.0", - "rc-mentions": "~1.7.0", - "rc-menu": "~9.5.5", - "rc-motion": "^2.5.1", + "rc-collapse": "~3.3.0", + "rc-dialog": "~8.9.0", + "rc-drawer": "~5.1.0", + "rc-dropdown": "~4.0.0", + "rc-field-form": "~1.27.0", + "rc-image": "~5.7.0", + "rc-input": "~0.1.2", + "rc-input-number": "~7.3.9", + "rc-mentions": "~1.10.0", + "rc-menu": "~9.6.3", + "rc-motion": "^2.6.1", "rc-notification": "~4.6.0", - "rc-pagination": "~3.1.9", - "rc-picker": "~2.6.4", - "rc-progress": "~3.2.1", + "rc-pagination": "~3.1.17", + "rc-picker": "~2.6.11", + "rc-progress": "~3.3.2", "rc-rate": "~2.9.0", "rc-resize-observer": "^1.2.0", - "rc-segmented": "~2.1.0 ", - "rc-select": "~14.1.1", + "rc-segmented": "~2.1.0", + "rc-select": "~14.1.13", "rc-slider": "~10.0.0", "rc-steps": "~4.1.0", "rc-switch": "~3.2.0", - "rc-table": "~7.24.0", - "rc-tabs": "~11.13.0", - "rc-textarea": "~0.3.0", - "rc-tooltip": "~5.1.1", - "rc-tree": "~5.5.0", - "rc-tree-select": "~5.3.0", + "rc-table": "~7.26.0", + "rc-tabs": "~12.2.0", + "rc-textarea": "~0.4.5", + "rc-tooltip": "~5.2.0", + "rc-tree": "~5.7.0", + "rc-tree-select": "~5.5.0", "rc-trigger": "^5.2.10", "rc-upload": "~4.3.0", - "rc-util": "^5.20.0", + "rc-util": "^5.22.5", "scroll-into-view-if-needed": "^2.2.25" } }, @@ -19660,7 +20550,9 @@ } }, "array-tree-filter": { - "version": "2.1.0" + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/array-tree-filter/-/array-tree-filter-2.1.0.tgz", + "integrity": "sha512-4ROwICNlNw/Hqa9v+rk5h22KjmzB1JGTMVKP2AKJBOCgb0yL0ASf0+YvCcLNNwquOHNX48jkeZIJ3a+oOQqKcw==" }, "array-union": { "version": "2.1.0", @@ -19703,7 +20595,9 @@ "dev": true }, "async-validator": { - "version": "4.1.1" + "version": "4.2.5", + "resolved": "https://registry.npmjs.org/async-validator/-/async-validator-4.2.5.tgz", + "integrity": "sha512-7HhHjtERjqlNbZtqNqy2rckN/SpOOlmDliet+lP7k+eKZEjPk3DgyeU9lIXLdeLz0uBbbVp+9Qdow9wJWgwwfg==" }, "asynckit": { "version": "0.4.0" @@ -19717,11 +20611,13 @@ "dev": true }, "autoprefixer": { - "version": "10.4.7", + "version": "10.4.12", + "resolved": "https://registry.npmjs.org/autoprefixer/-/autoprefixer-10.4.12.tgz", + "integrity": "sha512-WrCGV9/b97Pa+jtwf5UGaRjgQIg7OK3D06GnoYoZNcG1Xb8Gt3EfuKjlhh9i/VtT16g6PYjZ69jdJ2g8FxSC4Q==", "dev": true, "requires": { - "browserslist": "^4.20.3", - "caniuse-lite": "^1.0.30001335", + "browserslist": "^4.21.4", + "caniuse-lite": "^1.0.30001407", "fraction.js": "^4.2.0", "normalize-range": "^0.1.2", "picocolors": "^1.0.0", @@ -19820,6 +20716,15 @@ "object.assign": "^4.1.0" } }, + "babel-plugin-import": { + "version": "1.13.5", + "resolved": "https://registry.npmjs.org/babel-plugin-import/-/babel-plugin-import-1.13.5.tgz", + "integrity": "sha512-IkqnoV+ov1hdJVofly9pXRJmeDm9EtROfrc5i6eII0Hix2xMs5FEm8FG3ExMvazbnZBbgHIt6qdO8And6lCloQ==", + "dev": true, + "requires": { + "@babel/helper-module-imports": "^7.0.0" + } + }, "babel-plugin-istanbul": { "version": "6.1.1", "dev": true, @@ -20050,14 +20955,15 @@ "dev": true }, "browserslist": { - "version": "4.20.3", + "version": "4.21.4", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.21.4.tgz", + "integrity": "sha512-CBHJJdDmgjl3daYjN5Cp5kbTf1mUhZoS+beLklHIvkOWscs83YAhLlF3Wsh/lciQYAcbBJgTOD44VtG31ZM4Hw==", "dev": true, "requires": { - "caniuse-lite": "^1.0.30001332", - "electron-to-chromium": "^1.4.118", - "escalade": "^3.1.1", - "node-releases": "^2.0.3", - "picocolors": "^1.0.0" + "caniuse-lite": "^1.0.30001400", + "electron-to-chromium": "^1.4.251", + "node-releases": "^2.0.6", + "update-browserslist-db": "^1.0.9" } }, "bser": { @@ -20124,7 +21030,9 @@ } }, "caniuse-lite": { - "version": "1.0.30001336", + "version": "1.0.30001422", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001422.tgz", + "integrity": "sha512-hSesn02u1QacQHhaxl/kNMZwqVG35Sz/8DgvmgedxSH8z9UUpcDYSPYgsj3x5dQNRcNp6BwpSfQfVzYUTm+fog==", "dev": true }, "case-sensitive-paths-webpack-plugin": { @@ -20190,7 +21098,9 @@ "version": "5.0.3" }, "classnames": { - "version": "2.3.1" + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/classnames/-/classnames-2.3.2.tgz", + "integrity": "sha512-CSbhY4cFEJRe6/GQzIk5qXZ4Jeg5pcsP7b5peFSDpffpe1cqjASH/n9UTjBwOp6XpMSTwQ8Za2K5V02ueA7Tmw==" }, "clean-css": { "version": "5.3.0", @@ -20249,6 +21159,17 @@ "wrap-ansi": "^7.0.0" } }, + "clone-deep": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/clone-deep/-/clone-deep-4.0.1.tgz", + "integrity": "sha512-neHB9xuzh/wk0dIHweyAXv2aPGZIVk3pLMe+/RNzINf17fe0OG96QroktYAUm7SM1PBnzTabaLboqqxDyMU+SQ==", + "dev": true, + "requires": { + "is-plain-object": "^2.0.4", + "kind-of": "^6.0.2", + "shallow-clone": "^3.0.0" + } + }, "clsx": { "version": "1.1.1" }, @@ -20321,7 +21242,9 @@ "dev": true }, "colord": { - "version": "2.9.2", + "version": "2.9.3", + "resolved": "https://registry.npmjs.org/colord/-/colord-2.9.3.tgz", + "integrity": "sha512-jeC1axXpnb0/2nn/Y1LPuLdgXBLH7aDcHu4KEKfqw3CUhX7ZpfBSlPKyqXE6btIgEzfWtrX3/tyBCaCvXvMkOw==", "dev": true }, "colorette": { @@ -20429,6 +21352,15 @@ "version": "1.0.6", "dev": true }, + "copy-anything": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/copy-anything/-/copy-anything-2.0.6.tgz", + "integrity": "sha512-1j20GZTsvKNkc4BY3NpMOM8tt///wY3FpIzozTOFO2ffuZcV61nojHXVKIy3WM+7ADCy5FVhdZYHYDdgTU0yJw==", + "dev": true, + "requires": { + "is-what": "^3.14.1" + } + }, "copy-to-clipboard": { "version": "3.3.1", "requires": { @@ -20472,6 +21404,30 @@ "yaml": "^1.10.0" } }, + "cosmiconfig-typescript-loader": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/cosmiconfig-typescript-loader/-/cosmiconfig-typescript-loader-4.1.1.tgz", + "integrity": "sha512-9DHpa379Gp0o0Zefii35fcmuuin6q92FnLDffzdZ0l9tVd3nEobG3O+MZ06+kuBvFTSVScvNb/oHA13Nd4iipg==", + "dev": true, + "requires": {} + }, + "craco-less": { + "version": "2.1.0-alpha.0", + "resolved": "https://registry.npmjs.org/craco-less/-/craco-less-2.1.0-alpha.0.tgz", + "integrity": "sha512-1kj9Y7Y06Fbae3SJJtz1OvXsaKxjh0jTOwnvzKWOqrojQZbwC2K/d0dxDRUpHTDkIUmxbdzqMmI4LM9JfthQ6Q==", + "dev": true, + "requires": { + "less": "^4.1.1", + "less-loader": "^7.3.0" + } + }, + "create-require": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz", + "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==", + "dev": true, + "peer": true + }, "cross-spawn": { "version": "7.0.3", "dev": true, @@ -20782,7 +21738,9 @@ } }, "date-fns": { - "version": "2.28.0" + "version": "2.29.3", + "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.29.3.tgz", + "integrity": "sha512-dDCnyH2WnnKusqvZZ6+jA1O51Ibt8ZMRNkDZdyAyK4YfbDwa/cEmuztzG5pk6hqlp9aSBPYcjOlktquahGwGeA==" }, "dayjs": { "version": "1.11.5", @@ -20891,6 +21849,13 @@ "version": "1.2.2", "dev": true }, + "diff": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", + "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==", + "dev": true, + "peer": true + }, "diff-sequences": { "version": "27.5.1", "dev": true @@ -21032,7 +21997,9 @@ } }, "electron-to-chromium": { - "version": "1.4.134", + "version": "1.4.284", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.284.tgz", + "integrity": "sha512-M8WEXFuKXMYMVr45fo8mq0wUrrJHheiKZf6BArTKk9ZBYCKJEOU5H8cdWgDT+qCVZf7Na4lVUaZsA+h6uA9+PA==", "dev": true }, "emittery": { @@ -21052,7 +22019,9 @@ "dev": true }, "enhanced-resolve": { - "version": "5.9.3", + "version": "5.10.0", + "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.10.0.tgz", + "integrity": "sha512-T0yTFjdpldGY8PmuXXR0PyQ1ufZpEGiHVrp7zHKB7jdR4qlmZHhONVM5AQOAWXuF/w3dnHbEQVrNptJgt7F+cQ==", "dev": true, "requires": { "graceful-fs": "^4.2.4", @@ -21063,6 +22032,16 @@ "version": "2.2.0", "dev": true }, + "errno": { + "version": "0.1.8", + "resolved": "https://registry.npmjs.org/errno/-/errno-0.1.8.tgz", + "integrity": "sha512-dJ6oBr5SQ1VSd9qkk7ByRgb/1SH4JZjCHSW/mr63/QcXO9zLVxvJ6Oy13nio03rxpSnVDDjFor75SjVeZWPW/A==", + "dev": true, + "optional": true, + "requires": { + "prr": "~1.0.1" + } + }, "error-ex": { "version": "1.3.2", "dev": true, @@ -21312,6 +22291,42 @@ } } }, + "eslint-import-resolver-typescript": { + "version": "3.5.1", + "resolved": "https://registry.npmjs.org/eslint-import-resolver-typescript/-/eslint-import-resolver-typescript-3.5.1.tgz", + "integrity": "sha512-U7LUjNJPYjNsHvAUAkt/RU3fcTSpbllA0//35B4eLYTX74frmOepbt7F7J3D1IGtj9k21buOpaqtDd4ZlS/BYQ==", + "dev": true, + "requires": { + "debug": "^4.3.4", + "enhanced-resolve": "^5.10.0", + "get-tsconfig": "^4.2.0", + "globby": "^13.1.2", + "is-core-module": "^2.10.0", + "is-glob": "^4.0.3", + "synckit": "^0.8.3" + }, + "dependencies": { + "globby": { + "version": "13.1.2", + "resolved": "https://registry.npmjs.org/globby/-/globby-13.1.2.tgz", + "integrity": "sha512-LKSDZXToac40u8Q1PQtZihbNdTYSNMuWe+K5l+oa6KgDzSvVrHXlJy40hUP522RjAIoNLJYBJi7ow+rbFpIhHQ==", + "dev": true, + "requires": { + "dir-glob": "^3.0.1", + "fast-glob": "^3.2.11", + "ignore": "^5.2.0", + "merge2": "^1.4.1", + "slash": "^4.0.0" + } + }, + "slash": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/slash/-/slash-4.0.0.tgz", + "integrity": "sha512-3dOsAHXXUkQTpOYcoAxLIorMTp4gIQr5IW3iVb7A7lFIp0VHhnynm9izx6TssdrIcVIESAlVjtnO2K8bg+Coew==", + "dev": true + } + } + }, "eslint-module-utils": { "version": "2.7.3", "dev": true, @@ -21376,6 +22391,8 @@ }, "eslint-plugin-import": { "version": "2.26.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.26.0.tgz", + "integrity": "sha512-hYfi3FXaM8WPLf4S1cikh/r4IxnO6zrhZbEGz2b660EJRbuxgpDS5gkCuYgGWg2xxh2rBuIr4Pvhve/7c31koA==", "dev": true, "requires": { "array-includes": "^3.1.4", @@ -21420,6 +22437,16 @@ "@typescript-eslint/experimental-utils": "^5.0.0" } }, + "eslint-plugin-json": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-json/-/eslint-plugin-json-3.1.0.tgz", + "integrity": "sha512-MrlG2ynFEHe7wDGwbUuFPsaT2b1uhuEFhJ+W1f1u+1C2EkXmTYJp4B1aAdQQ8M+CC3t//N/oRKiIVw14L2HR1g==", + "dev": true, + "requires": { + "lodash": "^4.17.21", + "vscode-json-languageservice": "^4.1.6" + } + }, "eslint-plugin-jsx-a11y": { "version": "6.5.1", "dev": true, @@ -21448,6 +22475,15 @@ } } }, + "eslint-plugin-prettier": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/eslint-plugin-prettier/-/eslint-plugin-prettier-4.2.1.tgz", + "integrity": "sha512-f/0rXLXUt0oFYs8ra4w49wYZBG5GKZpAYsJSm6rnYL5uVDjd+zowwMwVZHnAjf4edNrKpCDYfXDgmRE/Ak7QyQ==", + "dev": true, + "requires": { + "prettier-linter-helpers": "^1.0.0" + } + }, "eslint-plugin-react": { "version": "7.29.4", "dev": true, @@ -21682,8 +22718,16 @@ "fast-deep-equal": { "version": "3.1.3" }, + "fast-diff": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/fast-diff/-/fast-diff-1.2.0.tgz", + "integrity": "sha512-xJuoT5+L99XlZ8twedaRf6Ax2TgQVxvgZOYoPKqZufmJib0tL2tegPBOZb1pVNgIhlqDlA0eO0c3wBvQcmzx4w==", + "dev": true + }, "fast-glob": { - "version": "3.2.11", + "version": "3.2.12", + "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.2.12.tgz", + "integrity": "sha512-DVj4CQIYYow0BlaelwK1pHl5n5cRSJfM60UA0zK891sVInoPri2Ekj7+e1CT3/3qxXenpI+nBBmQAcJPJgaj4w==", "dev": true, "requires": { "@nodelib/fs.stat": "^2.0.2", @@ -22016,6 +23060,12 @@ "get-intrinsic": "^1.1.1" } }, + "get-tsconfig": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.2.0.tgz", + "integrity": "sha512-X8u8fREiYOE6S8hLbq99PeykTDoLVnxvF4DjWKJmz9xy2nNRdUcV8ZN9tniJFeKyTU3qnC9lL8n4Chd6LmVKHg==", + "dev": true + }, "glob": { "version": "7.2.0", "requires": { @@ -22067,6 +23117,12 @@ "version": "11.12.0", "dev": true }, + "globalyzer": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/globalyzer/-/globalyzer-0.1.0.tgz", + "integrity": "sha512-40oNTM9UfG6aBmuKxk/giHn5nQ8RVz/SS4Ir6zgzOv9/qC3kKZ9v4etGTcJbEl/NyVQH7FGU7d+X1egr57Md2Q==", + "dev": true + }, "globby": { "version": "11.1.0", "dev": true, @@ -22079,6 +23135,12 @@ "slash": "^3.0.0" } }, + "globrex": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/globrex/-/globrex-0.1.2.tgz", + "integrity": "sha512-uHJgbwAMwNFf5mLst7IWLNg14x1CkeqglJb/K3doi4dw6q2IvAAmM/Y81kevy83wP+Sst+nutFTYOGg3d1lsxg==", + "dev": true + }, "graceful-fs": { "version": "4.2.10", "dev": true @@ -22167,10 +23229,6 @@ "wbuf": "^1.1.0" }, "dependencies": { - "isarray": { - "version": "1.0.0", - "dev": true - }, "readable-stream": { "version": "2.3.7", "dev": true, @@ -22356,6 +23414,13 @@ "version": "5.2.0", "dev": true }, + "image-size": { + "version": "0.5.5", + "resolved": "https://registry.npmjs.org/image-size/-/image-size-0.5.5.tgz", + "integrity": "sha512-6TDAlDPZxUFCv+fuOkIoXT/V/f3Qbq8e37p+YOiYrUv3v9cc3/6x78VdfPgFVaB9dZYeLUfKgHRebpkm/oP2VQ==", + "dev": true, + "optional": true + }, "immer": { "version": "9.0.12", "dev": true @@ -22442,7 +23507,9 @@ "dev": true }, "is-core-module": { - "version": "2.9.0", + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.11.0.tgz", + "integrity": "sha512-RRjxlvLDkD1YJwDbroBHMb+cukurkDWNyHx7D3oNB5x9rb5ogcksMC5wHCadcXoo67gVr/+3GFySh3134zi6rw==", "dev": true, "requires": { "has": "^1.0.3" @@ -22505,6 +23572,15 @@ "version": "3.0.0", "dev": true }, + "is-plain-object": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz", + "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==", + "dev": true, + "requires": { + "isobject": "^3.0.1" + } + }, "is-potential-custom-element-name": { "version": "1.0.1", "dev": true @@ -22561,6 +23637,12 @@ "call-bind": "^1.0.2" } }, + "is-what": { + "version": "3.14.1", + "resolved": "https://registry.npmjs.org/is-what/-/is-what-3.14.1.tgz", + "integrity": "sha512-sNxgpk9793nzSs7bA6JQJGeIuRBQhAaNGG77kzYQgMkrID+lS6SlK07K5LaptscDlSaIgH+GPFzf+d75FVxozA==", + "dev": true + }, "is-wsl": { "version": "2.2.0", "dev": true, @@ -22568,10 +23650,22 @@ "is-docker": "^2.0.0" } }, + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "dev": true + }, "isexe": { "version": "2.0.0", "dev": true }, + "isobject": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz", + "integrity": "sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg==", + "dev": true + }, "istanbul-lib-coverage": { "version": "3.2.0", "dev": true @@ -23503,6 +24597,8 @@ }, "json2mq": { "version": "0.2.0", + "resolved": "https://registry.npmjs.org/json2mq/-/json2mq-0.2.0.tgz", + "integrity": "sha512-SzoRg7ux5DWTII9J2qkrZrqV1gt+rTaoufMxEzXbS26Uid0NwaJd123HcoB80TgubEppxxIGdNxCx50fEoEWQA==", "requires": { "string-convert": "^0.2.0" } @@ -23511,6 +24607,12 @@ "version": "2.2.1", "dev": true }, + "jsonc-parser": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/jsonc-parser/-/jsonc-parser-3.2.0.tgz", + "integrity": "sha512-gfFQZrcTc8CnKXp6Y4/CBT3fTc0OVuDofpre4aEeEpSBPV5X5v4+Vmx+8snU7RLPrNHPKSgLxGo9YuQzz20o+w==", + "dev": true + }, "jsonfile": { "version": "6.1.0", "dev": true, @@ -23554,6 +24656,61 @@ "language-subtag-registry": "~0.3.2" } }, + "less": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/less/-/less-4.1.3.tgz", + "integrity": "sha512-w16Xk/Ta9Hhyei0Gpz9m7VS8F28nieJaL/VyShID7cYvP6IL5oHeL6p4TXSDJqZE/lNv0oJ2pGVjJsRkfwm5FA==", + "dev": true, + "requires": { + "copy-anything": "^2.0.1", + "errno": "^0.1.1", + "graceful-fs": "^4.1.2", + "image-size": "~0.5.0", + "make-dir": "^2.1.0", + "mime": "^1.4.1", + "needle": "^3.1.0", + "parse-node-version": "^1.0.1", + "source-map": "~0.6.0", + "tslib": "^2.3.0" + }, + "dependencies": { + "make-dir": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-2.1.0.tgz", + "integrity": "sha512-LS9X+dc8KLxXCb8dni79fLIIUA5VyZoyjSMCwTluaXA0o27cCK0bhXkpgw+sTXVpPy/lSO57ilRixqk0vDmtRA==", + "dev": true, + "optional": true, + "requires": { + "pify": "^4.0.1", + "semver": "^5.6.0" + } + }, + "semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", + "dev": true, + "optional": true + }, + "tslib": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.0.tgz", + "integrity": "sha512-d6xOpEDfsi2CZVlPQzGeux8XMwLT9hssAsaPYExaQMuYskwb+x1x7J371tWlbBdWHroy99KnVB6qIkUbs5X3UQ==", + "dev": true + } + } + }, + "less-loader": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/less-loader/-/less-loader-7.3.0.tgz", + "integrity": "sha512-Mi8915g7NMaLlgi77mgTTQvK022xKRQBIVDSyfl3ErTuBhmZBQab0mjeJjNNqGbdR+qrfTleKXqbGI4uEFavxg==", + "dev": true, + "requires": { + "klona": "^2.0.4", + "loader-utils": "^2.0.0", + "schema-utils": "^3.0.0" + } + }, "leven": { "version": "3.1.0", "dev": true @@ -23698,7 +24855,9 @@ "dev": true }, "loader-utils": { - "version": "2.0.2", + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/loader-utils/-/loader-utils-2.0.4.tgz", + "integrity": "sha512-xXqpXoINfFhgua9xiqD8fPFHgkoq1mmmpE92WlDbm9rNRd/EbRb+Gqf908T2DMfuHjjJlksiK2RbHVOdD/MqSw==", "dev": true, "requires": { "big.js": "^5.2.2", @@ -23816,6 +24975,13 @@ } } }, + "make-error": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", + "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==", + "dev": true, + "peer": true + }, "makeerror": { "version": "1.0.12", "dev": true, @@ -23974,6 +25140,40 @@ "version": "1.4.0", "dev": true }, + "needle": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/needle/-/needle-3.1.0.tgz", + "integrity": "sha512-gCE9weDhjVGCRqS8dwDR/D3GTAeyXLXuqp7I8EzH6DllZGXSUyxuqqLh+YX9rMAWaaTFyVAg6rHGL25dqvczKw==", + "dev": true, + "optional": true, + "requires": { + "debug": "^3.2.6", + "iconv-lite": "^0.6.3", + "sax": "^1.2.4" + }, + "dependencies": { + "debug": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", + "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", + "dev": true, + "optional": true, + "requires": { + "ms": "^2.1.1" + } + }, + "iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "dev": true, + "optional": true, + "requires": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + } + } + } + }, "negotiator": { "version": "0.6.3", "dev": true @@ -24005,7 +25205,9 @@ "dev": true }, "node-releases": { - "version": "2.0.4", + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.6.tgz", + "integrity": "sha512-PiVXnNuFm5+iYkLBNeq5211hvO38y63T0i2KKh2KnUs3RpzJ+JtODFjkD8yjLwnDkTYF1eKXheUwdssR+NRZdg==", "dev": true }, "normalize-path": { @@ -24223,6 +25425,12 @@ "lines-and-columns": "^1.1.6" } }, + "parse-node-version": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/parse-node-version/-/parse-node-version-1.0.1.tgz", + "integrity": "sha512-3YHlOa/JgH6Mnpr05jP9eDG254US9ek25LyIxZlDItp2iJtwyaXQb57lBYLdT3MowkUFYEV2XXNAYIPlESvJlA==", + "dev": true + }, "parse5": { "version": "6.0.1", "dev": true @@ -24280,6 +25488,13 @@ "version": "0.6.0", "dev": true }, + "pify": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz", + "integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==", + "dev": true, + "optional": true + }, "pirates": { "version": "4.0.5", "dev": true @@ -24365,10 +25580,12 @@ } }, "postcss": { - "version": "8.4.13", + "version": "8.4.18", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.18.tgz", + "integrity": "sha512-Wi8mWhncLJm11GATDaQKobXSNEYGUHeQLiQqDFG1qQ5UTDPTEvKw0Xt5NsTpktGTwLps3ByrWsBrG0rB8YQ9oA==", "dev": true, "requires": { - "nanoid": "^3.3.3", + "nanoid": "^3.3.4", "picocolors": "^1.0.0", "source-map-js": "^1.0.2" } @@ -24966,6 +26183,15 @@ "version": "2.7.1", "dev": true }, + "prettier-linter-helpers": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/prettier-linter-helpers/-/prettier-linter-helpers-1.0.0.tgz", + "integrity": "sha512-GbK2cP9nraSSUF9N2XwUwqfzlAFlMNYYl+ShE/V+H8a9uNl/oUqB1w2EL54Jh0OlyRSd8RfWYJ3coVS4TROP2w==", + "dev": true, + "requires": { + "fast-diff": "^1.1.2" + } + }, "pretty-bytes": { "version": "5.6.0", "dev": true @@ -25038,6 +26264,13 @@ } } }, + "prr": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/prr/-/prr-1.0.1.tgz", + "integrity": "sha512-yPw4Sng1gWghHQWj0B3ZggWUm4qVbPwPFcRG8KyxiU7J2OHFSoEHKS+EZ3fv5l1t9CyCiop6l/ZYeWbrgoQejw==", + "dev": true, + "optional": true + }, "psl": { "version": "1.8.0", "dev": true @@ -25111,13 +26344,15 @@ } }, "rc-cascader": { - "version": "3.5.0", + "version": "3.7.0", + "resolved": "https://registry.npmjs.org/rc-cascader/-/rc-cascader-3.7.0.tgz", + "integrity": "sha512-SFtGpwmYN7RaWEAGTS4Rkc62ZV/qmQGg/tajr/7mfIkleuu8ro9Hlk6J+aA0x1YS4zlaZBtTcSaXM01QMiEV/A==", "requires": { "@babel/runtime": "^7.12.5", "array-tree-filter": "^2.1.0", "classnames": "^2.3.1", "rc-select": "~14.1.0", - "rc-tree": "~5.5.0", + "rc-tree": "~5.7.0", "rc-util": "^5.6.1" } }, @@ -25129,7 +26364,9 @@ } }, "rc-collapse": { - "version": "3.1.4", + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/rc-collapse/-/rc-collapse-3.3.1.tgz", + "integrity": "sha512-cOJfcSe3R8vocrF8T+PgaHDrgeA1tX+lwfhwSj60NX9QVRidsILIbRNDLD6nAzmcvVC5PWiIRiR4S1OobxdhCg==", "requires": { "@babel/runtime": "^7.10.1", "classnames": "2.x", @@ -25139,7 +26376,9 @@ } }, "rc-dialog": { - "version": "8.8.1", + "version": "8.9.0", + "resolved": "https://registry.npmjs.org/rc-dialog/-/rc-dialog-8.9.0.tgz", + "integrity": "sha512-Cp0tbJnrvPchJfnwIvOMWmJ4yjX3HWFatO6oBFD1jx8QkgsQCR0p8nUWAKdd3seLJhEC39/v56kZaEjwp9muoQ==", "requires": { "@babel/runtime": "^7.10.1", "classnames": "^2.2.6", @@ -25148,41 +26387,52 @@ } }, "rc-drawer": { - "version": "4.4.3", + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/rc-drawer/-/rc-drawer-5.1.0.tgz", + "integrity": "sha512-pU3Tsn99pxGdYowXehzZbdDVE+4lDXSGb7p8vA9mSmr569oc2Izh4Zw5vLKSe/Xxn2p5MSNbLVqD4tz+pK6SOw==", "requires": { "@babel/runtime": "^7.10.1", "classnames": "^2.2.6", - "rc-util": "^5.7.0" + "rc-motion": "^2.6.1", + "rc-util": "^5.21.2" } }, "rc-dropdown": { - "version": "3.5.2", + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/rc-dropdown/-/rc-dropdown-4.0.1.tgz", + "integrity": "sha512-OdpXuOcme1rm45cR0Jzgfl1otzmU4vuBVb+etXM8vcaULGokAKVpKlw8p6xzspG7jGd/XxShvq+N3VNEfk/l5g==", "requires": { - "@babel/runtime": "^7.10.1", + "@babel/runtime": "^7.18.3", "classnames": "^2.2.6", - "rc-trigger": "^5.0.4", + "rc-trigger": "^5.3.1", "rc-util": "^5.17.0" } }, "rc-field-form": { - "version": "1.26.3", + "version": "1.27.3", + "resolved": "https://registry.npmjs.org/rc-field-form/-/rc-field-form-1.27.3.tgz", + "integrity": "sha512-HGqxHnmGQgkPApEcikV4qTg3BLPC82uB/cwBDftDt1pYaqitJfSl5TFTTUMKVEJVT5RqJ2Zi68ME1HmIMX2HAw==", "requires": { - "@babel/runtime": "^7.8.4", + "@babel/runtime": "^7.18.0", "async-validator": "^4.1.0", "rc-util": "^5.8.0" } }, "rc-image": { - "version": "5.6.2", + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/rc-image/-/rc-image-5.7.1.tgz", + "integrity": "sha512-QyMfdhoUfb5W14plqXSisaYwpdstcLYnB0MjX5ccIK2rydQM9sDPuekQWu500DDGR2dBaIF5vx9XbWkNFK17Fg==", "requires": { "@babel/runtime": "^7.11.2", "classnames": "^2.2.6", - "rc-dialog": "~8.8.0", + "rc-dialog": "~8.9.0", "rc-util": "^5.0.6" } }, "rc-input": { - "version": "0.0.1-alpha.7", + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/rc-input/-/rc-input-0.1.4.tgz", + "integrity": "sha512-FqDdNz+fV2dKNgfXzcSLKvC+jEs1709t7nD+WdfjrdSaOcefpgc7BUJYadc3usaING+b7ediMTfKxuJBsEFbXA==", "requires": { "@babel/runtime": "^7.11.1", "classnames": "^2.2.1", @@ -25190,26 +26440,32 @@ } }, "rc-input-number": { - "version": "7.3.4", + "version": "7.3.9", + "resolved": "https://registry.npmjs.org/rc-input-number/-/rc-input-number-7.3.9.tgz", + "integrity": "sha512-u0+miS+SATdb6DtssYei2JJ1WuZME+nXaG6XGtR8maNyW5uGDytfDu60OTWLQEb0Anv/AcCzehldV8CKmKyQfA==", "requires": { "@babel/runtime": "^7.10.1", "classnames": "^2.2.5", - "rc-util": "^5.9.8" + "rc-util": "^5.23.0" } }, "rc-mentions": { - "version": "1.7.1", + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/rc-mentions/-/rc-mentions-1.10.0.tgz", + "integrity": "sha512-oMlYWnwXSxP2NQVlgxOTzuG/u9BUc3ySY78K3/t7MNhJWpZzXTao+/Bic6tyZLuNCO89//hVQJBdaR2rnFQl6Q==", "requires": { "@babel/runtime": "^7.10.1", "classnames": "^2.2.6", - "rc-menu": "~9.5.1", - "rc-textarea": "^0.3.0", + "rc-menu": "~9.6.0", + "rc-textarea": "^0.4.0", "rc-trigger": "^5.0.4", - "rc-util": "^5.0.1" + "rc-util": "^5.22.5" } }, "rc-menu": { - "version": "9.5.5", + "version": "9.6.4", + "resolved": "https://registry.npmjs.org/rc-menu/-/rc-menu-9.6.4.tgz", + "integrity": "sha512-6DiNAjxjVIPLZXHffXxxcyE15d4isRL7iQ1ru4MqYDH2Cqc5bW96wZOdMydFtGLyDdnmEQ9jVvdCE9yliGvzkw==", "requires": { "@babel/runtime": "^7.10.1", "classnames": "2.x", @@ -25221,7 +26477,9 @@ } }, "rc-motion": { - "version": "2.6.0", + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/rc-motion/-/rc-motion-2.6.2.tgz", + "integrity": "sha512-4w1FaX3dtV749P8GwfS4fYnFG4Rb9pxvCYPc/b2fw1cmlHJWNNgOFIz7ysiD+eOrzJSvnLJWlNQQncpNMXwwpg==", "requires": { "@babel/runtime": "^7.11.1", "classnames": "^2.2.1", @@ -25238,7 +26496,9 @@ } }, "rc-overflow": { - "version": "1.2.5", + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/rc-overflow/-/rc-overflow-1.2.8.tgz", + "integrity": "sha512-QJ0UItckWPQ37ZL1dMEBAdY1dhfTXFL9k6oTTcyydVwoUNMnMqCGqnRNA98axSr/OeDKqR6DVFyi8eA5RQI/uQ==", "requires": { "@babel/runtime": "^7.11.1", "classnames": "^2.2.1", @@ -25247,14 +26507,18 @@ } }, "rc-pagination": { - "version": "3.1.16", + "version": "3.1.17", + "resolved": "https://registry.npmjs.org/rc-pagination/-/rc-pagination-3.1.17.tgz", + "integrity": "sha512-/BQ5UxcBnW28vFAcP2hfh+Xg15W0QZn8TWYwdCApchMH1H0CxiaUUcULP8uXcFM1TygcdKWdt3JqsL9cTAfdkQ==", "requires": { "@babel/runtime": "^7.10.1", "classnames": "^2.2.1" } }, "rc-picker": { - "version": "2.6.8", + "version": "2.6.11", + "resolved": "https://registry.npmjs.org/rc-picker/-/rc-picker-2.6.11.tgz", + "integrity": "sha512-INJ7ULu+Kj4UgqbcqE8Q+QpMw55xFf9kkyLBHJFk0ihjJpAV4glialRfqHE7k4KX2BWYPQfpILwhwR14x2EiRQ==", "requires": { "@babel/runtime": "^7.10.1", "classnames": "^2.2.1", @@ -25267,7 +26531,9 @@ } }, "rc-progress": { - "version": "3.2.4", + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/rc-progress/-/rc-progress-3.3.3.tgz", + "integrity": "sha512-MDVNVHzGanYtRy2KKraEaWeZLri2ZHWIRyaE1a9MQ2MuJ09m+Wxj5cfcaoaR6z5iRpHpA59YeUxAlpML8N4PJw==", "requires": { "@babel/runtime": "^7.10.1", "classnames": "^2.2.6", @@ -25284,6 +26550,8 @@ }, "rc-resize-observer": { "version": "1.2.0", + "resolved": "https://registry.npmjs.org/rc-resize-observer/-/rc-resize-observer-1.2.0.tgz", + "integrity": "sha512-6W+UzT3PyDM0wVCEHfoW3qTHPTvbdSgiA43buiy8PzmeMnfgnDeb9NjdimMXMl3/TcrvvWl5RRVdp+NqcR47pQ==", "requires": { "@babel/runtime": "^7.10.1", "classnames": "^2.2.1", @@ -25301,7 +26569,9 @@ } }, "rc-select": { - "version": "14.1.2", + "version": "14.1.13", + "resolved": "https://registry.npmjs.org/rc-select/-/rc-select-14.1.13.tgz", + "integrity": "sha512-WMEsC3gTwA1dbzWOdVIXDmWyidYNLq68AwvvUlRROw790uGUly0/vmqDozXrIr0QvN/A3CEULx12o+WtLCAefg==", "requires": { "@babel/runtime": "^7.10.1", "classnames": "2.x", @@ -25339,67 +26609,83 @@ } }, "rc-table": { - "version": "7.24.1", + "version": "7.26.0", + "resolved": "https://registry.npmjs.org/rc-table/-/rc-table-7.26.0.tgz", + "integrity": "sha512-0cD8e6S+DTGAt5nBZQIPFYEaIukn17sfa5uFL98faHlH/whZzD8ii3dbFL4wmUDEL4BLybhYop+QUfZJ4CPvNQ==", "requires": { "@babel/runtime": "^7.10.1", "classnames": "^2.2.5", "rc-resize-observer": "^1.1.0", - "rc-util": "^5.14.0", + "rc-util": "^5.22.5", "shallowequal": "^1.1.0" } }, "rc-tabs": { - "version": "11.13.0", + "version": "12.2.1", + "resolved": "https://registry.npmjs.org/rc-tabs/-/rc-tabs-12.2.1.tgz", + "integrity": "sha512-09pVv4kN8VFqp6THceEmxOW8PAShQC08hrroeVYP4Y8YBFaP1PIWdyFL01czcbyz5YZFj9flZ7aljMaAl0jLVg==", "requires": { "@babel/runtime": "^7.11.2", "classnames": "2.x", - "rc-dropdown": "~3.5.0", - "rc-menu": "~9.5.1", + "rc-dropdown": "~4.0.0", + "rc-menu": "~9.6.0", + "rc-motion": "^2.6.2", "rc-resize-observer": "^1.0.0", "rc-util": "^5.5.0" } }, "rc-textarea": { - "version": "0.3.7", + "version": "0.4.6", + "resolved": "https://registry.npmjs.org/rc-textarea/-/rc-textarea-0.4.6.tgz", + "integrity": "sha512-HEKCu8nouXXayqYelQnhQm8fdH7v92pAQvfVCz+jhIPv2PHTyBxVrmoZJMn3B8cU+wdyuvRGkshngO3/TzBn4w==", "requires": { "@babel/runtime": "^7.10.1", "classnames": "^2.2.1", "rc-resize-observer": "^1.0.0", - "rc-util": "^5.7.0", + "rc-util": "^5.24.4", "shallowequal": "^1.1.0" } }, "rc-tooltip": { - "version": "5.1.1", + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/rc-tooltip/-/rc-tooltip-5.2.2.tgz", + "integrity": "sha512-jtQzU/18S6EI3lhSGoDYhPqNpWajMtS5VV/ld1LwyfrDByQpYmw/LW6U7oFXXLukjfDHQ7Ju705A82PRNFWYhg==", "requires": { "@babel/runtime": "^7.11.2", + "classnames": "^2.3.1", "rc-trigger": "^5.0.0" } }, "rc-tree": { - "version": "5.5.0", + "version": "5.7.0", + "resolved": "https://registry.npmjs.org/rc-tree/-/rc-tree-5.7.0.tgz", + "integrity": "sha512-F+Ewkv/UcutshnVBMISP+lPdHDlcsL+YH/MQDVWbk+QdkfID7vXiwrHMEZn31+2Rbbm21z/HPceGS8PXGMmnQg==", "requires": { "@babel/runtime": "^7.10.1", "classnames": "2.x", "rc-motion": "^2.0.1", "rc-util": "^5.16.1", - "rc-virtual-list": "^3.4.2" + "rc-virtual-list": "^3.4.8" } }, "rc-tree-select": { - "version": "5.3.0", + "version": "5.5.3", + "resolved": "https://registry.npmjs.org/rc-tree-select/-/rc-tree-select-5.5.3.tgz", + "integrity": "sha512-gv8KyC6J7f9e50OkGk1ibF7v8vL+iaBnA8Ep/EVlMma2/tGdBQXO9xIvPjX8eQrZL5PjoeTUndNPM3cY3721ng==", "requires": { "@babel/runtime": "^7.10.1", "classnames": "2.x", "rc-select": "~14.1.0", - "rc-tree": "~5.5.0", + "rc-tree": "~5.7.0", "rc-util": "^5.16.1" } }, "rc-trigger": { - "version": "5.2.18", + "version": "5.3.3", + "resolved": "https://registry.npmjs.org/rc-trigger/-/rc-trigger-5.3.3.tgz", + "integrity": "sha512-IC4nuTSAME7RJSgwvHCNDQrIzhvGMKf6NDu5veX+zk1MG7i1UnwTWWthcP9WHw3+FZfP3oZGvkrHFPu/EGkFKw==", "requires": { - "@babel/runtime": "^7.11.2", + "@babel/runtime": "^7.18.3", "classnames": "^2.2.6", "rc-align": "^4.0.0", "rc-motion": "^2.0.0", @@ -25415,16 +26701,21 @@ } }, "rc-util": { - "version": "5.21.2", + "version": "5.24.4", + "resolved": "https://registry.npmjs.org/rc-util/-/rc-util-5.24.4.tgz", + "integrity": "sha512-2a4RQnycV9eV7lVZPEJ7QwJRPlZNc06J7CwcwZo4vIHr3PfUqtYgl1EkUV9ETAc6VRRi8XZOMFhYG63whlIC9Q==", "requires": { - "@babel/runtime": "^7.12.5", + "@babel/runtime": "^7.18.3", "react-is": "^16.12.0", "shallowequal": "^1.1.0" } }, "rc-virtual-list": { - "version": "3.4.7", + "version": "3.4.11", + "resolved": "https://registry.npmjs.org/rc-virtual-list/-/rc-virtual-list-3.4.11.tgz", + "integrity": "sha512-BvUUH60kkeTBPigN5F89HtGaA5jSP4y2aM6cJ4dk9Y42I9yY+h6i08wF6UKeDcxdfOU8j3I5HxkSS/xA77J3wA==", "requires": { + "@babel/runtime": "^7.20.0", "classnames": "^2.2.6", "rc-resize-observer": "^1.0.0", "rc-util": "^5.15.0" @@ -25492,7 +26783,9 @@ "dev": true }, "loader-utils": { - "version": "3.2.0", + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/loader-utils/-/loader-utils-3.2.1.tgz", + "integrity": "sha512-ZvFw1KWS3GVyYBYb7qkmRM/WwL2TQQBxgCK62rlvm4WpVQ23Nb4tYjApUlfjrEGvOs7KHEsmyUn75OHZrJMWPw==", "dev": true } } @@ -25560,6 +26853,15 @@ "version": "0.11.0", "dev": true }, + "react-resizable": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/react-resizable/-/react-resizable-3.0.4.tgz", + "integrity": "sha512-StnwmiESiamNzdRHbSSvA65b0ZQJ7eVQpPusrSmcpyGKzC0gojhtO62xxH6YOBmepk9dQTBi9yxidL3W4s3EBA==", + "requires": { + "prop-types": "15.x", + "react-draggable": "^4.0.3" + } + }, "react-router": { "version": "6.3.0", "requires": { @@ -25644,19 +26946,12 @@ } }, "recursive-readdir": { - "version": "2.2.2", + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/recursive-readdir/-/recursive-readdir-2.2.3.tgz", + "integrity": "sha512-8HrF5ZsXk5FAH9dgsx3BlUer73nIhuj+9OrQwEbLTPOBzGkL1lsFCR01am+v+0m2Cmbs1nP12hLDl5FA7EszKA==", "dev": true, "requires": { - "minimatch": "3.0.4" - }, - "dependencies": { - "minimatch": { - "version": "3.0.4", - "dev": true, - "requires": { - "brace-expansion": "^1.1.7" - } - } + "minimatch": "^3.0.5" } }, "redent": { @@ -25685,7 +26980,9 @@ } }, "regenerator-runtime": { - "version": "0.13.9" + "version": "0.13.10", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.10.tgz", + "integrity": "sha512-KepLsg4dU12hryUO7bp/axHAKvwGOCV0sGloQtpagJ12ai+ojVDqkeGSiRX1zlq+kjIMZ1t7gpze+26QqtdGqw==" }, "regenerator-transform": { "version": "0.15.0", @@ -26164,6 +27461,15 @@ "version": "1.2.0", "dev": true }, + "shallow-clone": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/shallow-clone/-/shallow-clone-3.0.1.tgz", + "integrity": "sha512-/6KqX+GVUdqPuPPd2LxDDxzX6CAbjJehAAOKlNpqqUpAqPM6HeL8f+o3a+JsyGjn2lv0WY8UsTgUJjU9Ok55NA==", + "dev": true, + "requires": { + "kind-of": "^6.0.2" + } + }, "shallowequal": { "version": "1.1.0" }, @@ -26344,7 +27650,9 @@ "dev": true }, "string-convert": { - "version": "0.2.1" + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/string-convert/-/string-convert-0.2.1.tgz", + "integrity": "sha512-u/1tdPl4yQnPBjnVrmdLo9gtuLvELKsAoRapekWggdiQNvvvum+jYF329d84NAa660KQw7pB2n36KrIKVoXa3A==" }, "string-length": { "version": "4.0.2", @@ -26465,7 +27773,9 @@ } }, "supports-hyperlinks": { - "version": "2.2.0", + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/supports-hyperlinks/-/supports-hyperlinks-2.3.0.tgz", + "integrity": "sha512-RpsAZlpWcDwOPQA22aCH4J0t7L8JmAvsCxfOSEwm7cQs3LshN36QaTkwd70DnBOXDWGssw2eUoc8CaRWT0XunA==", "dev": true, "requires": { "has-flag": "^4.0.0", @@ -26543,6 +27853,24 @@ "version": "3.2.4", "dev": true }, + "synckit": { + "version": "0.8.4", + "resolved": "https://registry.npmjs.org/synckit/-/synckit-0.8.4.tgz", + "integrity": "sha512-Dn2ZkzMdSX827QbowGbU/4yjWuvNaCoScLLoMo/yKbu+P4GBR6cRGKZH27k6a9bRzdqcyd1DE96pQtQ6uNkmyw==", + "dev": true, + "requires": { + "@pkgr/utils": "^2.3.1", + "tslib": "^2.4.0" + }, + "dependencies": { + "tslib": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.0.tgz", + "integrity": "sha512-d6xOpEDfsi2CZVlPQzGeux8XMwLT9hssAsaPYExaQMuYskwb+x1x7J371tWlbBdWHroy99KnVB6qIkUbs5X3UQ==", + "dev": true + } + } + }, "tailwindcss": { "version": "3.0.24", "dev": true, @@ -26656,6 +27984,16 @@ "version": "1.1.0", "dev": true }, + "tiny-glob": { + "version": "0.2.9", + "resolved": "https://registry.npmjs.org/tiny-glob/-/tiny-glob-0.2.9.tgz", + "integrity": "sha512-g/55ssRPUjShh+xkfx9UPDXqhckHEsHr4Vd9zX55oSdGZc/MD0m3sferOkwWtp98bv+kcVfEHtRJgBVJzelrzg==", + "dev": true, + "requires": { + "globalyzer": "0.1.0", + "globrex": "^0.1.2" + } + }, "tmpl": { "version": "1.0.5", "dev": true @@ -26704,6 +28042,44 @@ "version": "1.0.1", "dev": true }, + "ts-node": { + "version": "10.9.1", + "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.1.tgz", + "integrity": "sha512-NtVysVPkxxrwFGUUxGYhfux8k78pQB3JqYBXlLRZgdGUqTO5wU/UyHop5p70iEbGhB7q5KmiZiU0Y3KlJrScEw==", + "dev": true, + "peer": true, + "requires": { + "@cspotcode/source-map-support": "^0.8.0", + "@tsconfig/node10": "^1.0.7", + "@tsconfig/node12": "^1.0.7", + "@tsconfig/node14": "^1.0.0", + "@tsconfig/node16": "^1.0.2", + "acorn": "^8.4.1", + "acorn-walk": "^8.1.1", + "arg": "^4.1.0", + "create-require": "^1.1.0", + "diff": "^4.0.1", + "make-error": "^1.1.1", + "v8-compile-cache-lib": "^3.0.1", + "yn": "3.1.1" + }, + "dependencies": { + "acorn-walk": { + "version": "8.2.0", + "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.2.0.tgz", + "integrity": "sha512-k+iyHEuPgSw6SbuDpGQM+06HQUa04DZ3o+F6CSzXMvvI5KMvnaEqXe+YVe555R9nn6GPt404fos4wcgpw12SDA==", + "dev": true, + "peer": true + }, + "arg": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz", + "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==", + "dev": true, + "peer": true + } + } + }, "tsconfig-paths": { "version": "3.14.1", "dev": true, @@ -26828,6 +28204,16 @@ "version": "1.2.0", "dev": true }, + "update-browserslist-db": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.10.tgz", + "integrity": "sha512-OztqDenkfFkbSG+tRxBeAnCVPckDBcvibKd35yDONx6OU8N7sqgwc7rCbkJ/WcYtVRZ4ba68d6byhC21GFh7sQ==", + "dev": true, + "requires": { + "escalade": "^3.1.1", + "picocolors": "^1.0.0" + } + }, "uri-js": { "version": "4.4.1", "dev": true, @@ -26865,6 +28251,13 @@ "version": "2.3.0", "dev": true }, + "v8-compile-cache-lib": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", + "integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==", + "dev": true, + "peer": true + }, "v8-to-istanbul": { "version": "8.1.1", "dev": true, @@ -26884,6 +28277,43 @@ "version": "1.1.2", "dev": true }, + "vscode-json-languageservice": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/vscode-json-languageservice/-/vscode-json-languageservice-4.2.1.tgz", + "integrity": "sha512-xGmv9QIWs2H8obGbWg+sIPI/3/pFgj/5OWBhNzs00BkYQ9UaB2F6JJaGB/2/YOZJ3BvLXQTC4Q7muqU25QgAhA==", + "dev": true, + "requires": { + "jsonc-parser": "^3.0.0", + "vscode-languageserver-textdocument": "^1.0.3", + "vscode-languageserver-types": "^3.16.0", + "vscode-nls": "^5.0.0", + "vscode-uri": "^3.0.3" + } + }, + "vscode-languageserver-textdocument": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/vscode-languageserver-textdocument/-/vscode-languageserver-textdocument-1.0.7.tgz", + "integrity": "sha512-bFJH7UQxlXT8kKeyiyu41r22jCZXG8kuuVVA33OEJn1diWOZK5n8zBSPZFHVBOu8kXZ6h0LIRhf5UnCo61J4Hg==", + "dev": true + }, + "vscode-languageserver-types": { + "version": "3.17.2", + "resolved": "https://registry.npmjs.org/vscode-languageserver-types/-/vscode-languageserver-types-3.17.2.tgz", + "integrity": "sha512-zHhCWatviizPIq9B7Vh9uvrH6x3sK8itC84HkamnBWoDFJtzBf7SWlpLCZUit72b3os45h6RWQNC9xHRDF8dRA==", + "dev": true + }, + "vscode-nls": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/vscode-nls/-/vscode-nls-5.2.0.tgz", + "integrity": "sha512-RAaHx7B14ZU04EU31pT+rKz2/zSl7xMsfIZuo8pd+KZO6PXtQmpevpq3vxvWNcrGbdmhM/rr5Uw5Mz+NBfhVng==", + "dev": true + }, + "vscode-uri": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/vscode-uri/-/vscode-uri-3.0.6.tgz", + "integrity": "sha512-fmL7V1eiDBFRRnu+gfRWTzyPpNIHJTc4mWnFkwBUmO9U3KPgJAmTx7oxi2bl/Rh6HLdU7+4C9wlj0k2E4AdKFQ==", + "dev": true + }, "w3c-hr-time": { "version": "1.0.2", "dev": true, @@ -26930,6 +28360,8 @@ }, "webpack": { "version": "5.72.0", + "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.72.0.tgz", + "integrity": "sha512-qmSmbspI0Qo5ld49htys8GY9XhS9CGqFoHTsOVAnjBdg0Zn79y135R+k4IR4rKK6+eKaabMhJwiVB7xw0SJu5w==", "dev": true, "requires": { "@types/eslint-scope": "^3.7.3", @@ -27078,6 +28510,16 @@ } } }, + "webpack-merge": { + "version": "5.8.0", + "resolved": "https://registry.npmjs.org/webpack-merge/-/webpack-merge-5.8.0.tgz", + "integrity": "sha512-/SaI7xY0831XwP6kzuwhKWVKDP9t1QY1h65lAFLbZqMPIuYcD9QAW4u9STIbU9kaJbPBB/geU/gLr1wDjOhQ+Q==", + "dev": true, + "requires": { + "clone-deep": "^4.0.1", + "wildcard": "^2.0.0" + } + }, "webpack-sources": { "version": "3.2.3", "dev": true @@ -27137,6 +28579,12 @@ "is-symbol": "^1.0.3" } }, + "wildcard": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/wildcard/-/wildcard-2.0.0.tgz", + "integrity": "sha512-JcKqAHLPxcdb9KM49dufGXn2x3ssnfjbcaQdLlfZsL9rH9wgDQjUtDxbo8NE0F6SFvydeu1VhZe7hZuHsB2/pw==", + "dev": true + }, "word-wrap": { "version": "1.2.3", "dev": true @@ -27405,6 +28853,13 @@ "version": "20.2.9", "dev": true }, + "yn": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", + "integrity": "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==", + "dev": true, + "peer": true + }, "yocto-queue": { "version": "0.1.0", "dev": true diff --git a/ui/package.json b/ui/package.json index dc8ee5e7e..33f8f2ae7 100644 --- a/ui/package.json +++ b/ui/package.json @@ -1,21 +1,25 @@ { "name": "feathr-ui", - "version": "0.1.0", + "version": "0.9.0", "private": true, "dependencies": { + "@ant-design/icons": "^4.7.0", "@azure/msal-browser": "^2.24.0", "@azure/msal-react": "^1.4.0", - "antd": "^4.20.2", + "antd": "^4.23.6", "axios": "^0.27.2", + "classnames": "^2.3.2", "dagre": "^0.8.5", "dayjs": "^1.11.5", "react": "^17.0.2", "react-dom": "^17.0.2", "react-flow-renderer": "^9.7.4", "react-query": "^3.38.0", + "react-resizable": "^3.0.4", "react-router-dom": "^6.3.0" }, "devDependencies": { + "@craco/craco": "^7.0.0-alpha.8", "@testing-library/jest-dom": "^5.16.3", "@testing-library/react": "^12.1.4", "@testing-library/user-event": "^13.5.0", @@ -24,25 +28,34 @@ "@types/node": "^16.11.26", "@types/react": "^17.0.43", "@types/react-dom": "^17.0.14", + "@types/react-resizable": "^3.0.3", "@typescript-eslint/eslint-plugin": "^5.30.7", "@typescript-eslint/parser": "^5.30.7", + "babel-plugin-import": "^1.13.5", + "craco-less": "^2.1.0-alpha.0", "eslint": "^8.20.0", "eslint-config-prettier": "^8.5.0", + "eslint-import-resolver-typescript": "^3.5.1", + "eslint-plugin-import": "^2.26.0", + "eslint-plugin-json": "^3.1.0", + "eslint-plugin-prettier": "^4.2.1", "eslint-plugin-react-hooks": "^4.6.0", "husky": "^8.0.1", "lint-staged": "^13.0.3", "prettier": "2.7.1", "react-scripts": "5.0.0", "typescript": "^4.6.3", - "web-vitals": "^2.1.4" + "web-vitals": "^2.1.4", + "webpack": "^5.72.0" }, "scripts": { - "start": "react-scripts start", - "build": "react-scripts build", - "test": "react-scripts test", + "start": "craco start", + "build": "craco build", + "test": "craco test", "eject": "react-scripts eject", "lint:fix": "npx eslint --fix --ext ts --ext tsx src/ ", - "format": "npx prettier --write src/**" + "format": "npx prettier --write src/**", + "lintStaged": "lint-staged" }, "browserslist": { "production": [ diff --git a/ui/public/favicon.ico b/ui/public/favicon.ico index a11777cc4..fc2f6ca0f 100644 Binary files a/ui/public/favicon.ico and b/ui/public/favicon.ico differ diff --git a/ui/public/index.html b/ui/public/index.html index 0050dcf77..d0bc57b87 100644 --- a/ui/public/index.html +++ b/ui/public/index.html @@ -9,7 +9,7 @@ name="description" content="Feathr Feature Store Web UI" /> - + Feathr Feature Store diff --git a/ui/public/logo192.png b/ui/public/logo192.png deleted file mode 100644 index fc44b0a37..000000000 Binary files a/ui/public/logo192.png and /dev/null differ diff --git a/ui/public/logo200.png b/ui/public/logo200.png new file mode 100644 index 000000000..254621fb0 Binary files /dev/null and b/ui/public/logo200.png differ diff --git a/ui/public/logo512.png b/ui/public/logo512.png deleted file mode 100644 index a4e47a654..000000000 Binary files a/ui/public/logo512.png and /dev/null differ diff --git a/ui/public/manifest.json b/ui/public/manifest.json index 50a99047f..f6d4ea50a 100644 --- a/ui/public/manifest.json +++ b/ui/public/manifest.json @@ -8,14 +8,9 @@ "type": "image/x-icon" }, { - "src": "logo192.png", + "src": "logo200.png", "type": "image/png", - "sizes": "192x192" - }, - { - "src": "logo512.png", - "type": "image/png", - "sizes": "512x512" + "sizes": "200x200" } ], "start_url": ".", diff --git a/ui/src/api/api.tsx b/ui/src/api/api.tsx index a95ab2bd5..6c8b6f665 100644 --- a/ui/src/api/api.tsx +++ b/ui/src/api/api.tsx @@ -38,14 +38,18 @@ export const fetchDataSource = async ( ) => { const axios = await authAxios(msalInstance); return axios - .get( + .get( `${getApiBaseUrl()}/projects/${project}/datasources/${dataSourceId}`, { params: { project: project, datasource: dataSourceId }, } ) .then((response) => { - return response.data; + if (response.data.message || response.data.detail) { + return Promise.reject(response.data.message || response.data.detail); + } else { + return response.data; + } }); }; @@ -109,33 +113,21 @@ export const fetchFeatureLineages = async (featureId: string) => { // Following are place-holder code export const createFeature = async (feature: Feature) => { const axios = await authAxios(msalInstance); - return axios - .post(`${getApiBaseUrl()}/features`, feature, { - headers: { "Content-Type": "application/json;" }, - params: {}, - }) - .then((response) => { - return response; - }) - .catch((error) => { - return error.response; - }); + return axios.post(`${getApiBaseUrl()}/features`, feature, { + headers: { "Content-Type": "application/json;" }, + params: {}, + }); }; -export const updateFeature = async (feature: Feature, id: string) => { +export const updateFeature = async (feature: Feature, id?: string) => { const axios = await authAxios(msalInstance); - feature.guid = id; - return await axios - .put(`${getApiBaseUrl()}/features/${id}`, feature, { - headers: { "Content-Type": "application/json;" }, - params: {}, - }) - .then((response) => { - return response; - }) - .catch((error) => { - return error.response; - }); + if (id) { + feature.guid = id; + } + return axios.put(`${getApiBaseUrl()}/features/${feature.guid}`, feature, { + headers: { "Content-Type": "application/json;" }, + params: {}, + }); }; export const listUserRole = async () => { @@ -245,6 +237,8 @@ export const authAxios = async (msalInstance: PublicClientApplication) => { if (error.response?.status === 403) { const detail = error.response.data.detail; window.location.href = "/responseErrors/403/" + detail; + } else { + return Promise.reject(error.response.data); } //TODO: handle other response errors } diff --git a/ui/src/app.tsx b/ui/src/app.tsx index 5984717f9..b3d2b317a 100644 --- a/ui/src/app.tsx +++ b/ui/src/app.tsx @@ -24,44 +24,50 @@ import { getMsalConfig } from "./utils/utils"; const queryClient = new QueryClient(); const msalClient = getMsalConfig(); + const App = () => { return ( - + - +
- - } /> - } /> - } /> - } /> - } /> - } /> - } - /> - } - /> - } - /> - } /> - } /> - } /> - } /> - } - /> - + + + } /> + } /> + } /> + } /> + } /> + } /> + } + /> + } + /> + } + /> + } /> + } /> + } /> + } + /> + } + /> + + diff --git a/ui/src/components/CardDescriptions/index.tsx b/ui/src/components/CardDescriptions/index.tsx new file mode 100644 index 000000000..dffdec77d --- /dev/null +++ b/ui/src/components/CardDescriptions/index.tsx @@ -0,0 +1,34 @@ +import React from "react"; +import { Card, Descriptions } from "antd"; + +import { isEmpty } from "@/utils/utils"; + +export interface CardDescriptionsProps { + title?: string; + mapping: any[]; + descriptions: any; +} + +const CardDescriptions = (props: CardDescriptionsProps) => { + const { title, mapping, descriptions } = props; + + return !isEmpty(descriptions) ? ( + + + {mapping.reduce((list: any, item) => { + const value = descriptions?.[item.key]; + if (value) { + list.push( + + {typeof value === "string" ? value : JSON.stringify(value)} + + ); + } + return list; + }, [])} + + + ) : null; +}; + +export default CardDescriptions; diff --git a/ui/src/components/FlowGraph/FlowGraph.tsx b/ui/src/components/FlowGraph/FlowGraph.tsx new file mode 100644 index 000000000..ef3f16033 --- /dev/null +++ b/ui/src/components/FlowGraph/FlowGraph.tsx @@ -0,0 +1,236 @@ +import React, { + MouseEvent as ReactMouseEvent, + forwardRef, + useCallback, + useEffect, + useRef, + useState, +} from "react"; +import ReactFlow, { + ConnectionLineType, + Controls, + Edge, + Node, + Elements, + getIncomers, + getOutgoers, + ReactFlowProvider, + isNode, + OnLoadParams, +} from "react-flow-renderer"; +import { Spin } from "antd"; +import { LoadingOutlined } from "@ant-design/icons"; +import { useSearchParams } from "react-router-dom"; +import cs from "classnames"; +import { FeatureLineage } from "@/models/model"; +import { isFeature, FeatureType } from "@/utils/utils"; +import LineageNode from "./LineageNode"; +import { NodeData, FlowGraphProps } from "./interface"; +import { getElements } from "./utils"; + +import styles from "./index.module.less"; + +const FlowGraphNodeTypes = { + "custom-node": LineageNode, +}; + +const defaultProps: FlowGraphProps = { + project: "", + snapGrid: [15, 15], + featureType: FeatureType.AllNodes, +}; + +const FlowGraph = (props: FlowGraphProps, ref: any) => { + const { + className, + style, + data, + loading, + height, + minHeight, + project, + nodeId, + featureType, + snapGrid, + } = { + ...defaultProps, + ...props, + }; + const [, setURLSearchParams] = useSearchParams(); + const flowRef = useRef(); + const hasReadRef = useRef(false); + const elementRef = useRef>(); + const hasHighlight = useRef(false); + const [elements, setElements] = useState>([]); + + // Reset all node highlight status + const resetHighlight = useCallback(() => { + if ( + elementRef.current && + elementRef.current.length > 0 && + hasHighlight.current + ) { + hasHighlight.current = false; + setElements((state) => { + return state.map((element) => { + if (isNode(element)) { + element.style = { + ...element.style, + opacity: 1, + }; + element.data!.active = false; + } else { + element.animated = false; + } + return element; + }); + }); + } + }, [setElements]); + + // Highlight path of selected node, including all linked up and down stream nodes + const highlightPath = useCallback( + (node: Node) => { + if (elementRef.current && elementRef.current.length > 0) { + hasHighlight.current = true; + setElements((elements) => { + const incomerIds = new Set( + getIncomers(node, elements).map((item) => item.id) + ); + const outgoerIds = new Set( + getOutgoers(node, elements).map((item) => item.id) + ); + + return elements.map((element) => { + if (isNode(element)) { + const highlight = + element.id === node.id || + incomerIds.has(element.id) || + outgoerIds.has(element.id); + element.style = { + ...element.style, + opacity: highlight ? 1 : 0.25, + }; + element.data = { + ...element.data, + active: + element.id === node.id && isFeature(element.data!.subtitle), + }; + } else { + const highlight = + element.source === node.id || element.target === node.id; + const animated = + incomerIds.has(element.source) && + (incomerIds.has(element.target) || node.id === element.target); + + element.animated = highlight || animated; + } + return element; + }); + }); + } + }, + [setElements] + ); + + // Fired when panel is clicked, reset all highlighted path, and remove the nodeId query string in url path. + const onPaneClick = useCallback(() => { + resetHighlight(); + setURLSearchParams({}); + }, [resetHighlight, setURLSearchParams]); + + const onElementClick = useCallback( + (e: ReactMouseEvent, element: Node | Edge) => { + e.stopPropagation(); + if (isNode(element)) { + setURLSearchParams({ + nodeId: element.id, + featureType: element.data!.subtitle, + }); + setTimeout(() => { + highlightPath(element); + }, 0); + } + }, + [highlightPath, setURLSearchParams] + ); + + const handleInit = useCallback( + ( + project: string, + data: FeatureLineage, + featureType?: FeatureType, + nodeId?: string + ) => { + const elements = (elementRef.current = getElements( + project, + data, + featureType + )); + setElements(elements); + if (nodeId) { + const node = elements?.find( + (item) => item.id === nodeId + ) as Node; + if (node) { + highlightPath(node); + } + } + }, + [setElements, highlightPath] + ); + + // Fit the graph to the center of layout view when graph is initialized + const onLoad = (reactFlowInstance: OnLoadParams) => { + flowRef.current = reactFlowInstance; + flowRef.current?.fitView(); + }; + + useEffect(() => { + if (data) { + const type = hasHighlight.current ? FeatureType.AllNodes : featureType; + handleInit(project!, data, type, nodeId); + } + }, [data, project, nodeId, featureType, handleInit]); + + useEffect(() => { + if (elements.length > 0 && !hasReadRef.current) { + hasReadRef.current = true; + setTimeout(() => { + flowRef.current?.fitView(); + }, 0); + } + }, [elements]); + + return ( + } + > + + + + + + + ); +}; + +const FlowGraphComponent = forwardRef(FlowGraph); + +FlowGraphComponent.displayName = "FlowGraph"; + +export default FlowGraphComponent; diff --git a/ui/src/components/FlowGraph/LineageNode.tsx b/ui/src/components/FlowGraph/LineageNode.tsx new file mode 100644 index 000000000..27a99cc4f --- /dev/null +++ b/ui/src/components/FlowGraph/LineageNode.tsx @@ -0,0 +1,57 @@ +import React, { forwardRef, memo } from "react"; +import cs from "classnames"; +import { RightCircleOutlined } from "@ant-design/icons"; +import { useNavigate } from "react-router-dom"; +import { Handle, NodeProps, Position } from "react-flow-renderer"; +import { LineageNodeProps } from "./interface"; + +import styles from "./index.module.less"; + +const LineageNode = (props: LineageNodeProps, ref: any) => { + const navigate = useNavigate(); + + const { label, subtitle, version, borderColor, detialUrl, active } = + props.data; + + const nodeTitle = version ? `${label} (v${version})` : label; + const nodeSubtitle = subtitle.replace("feathr_", ""); + const nodeColorStyle = { + border: `2px solid ${borderColor}`, + }; + + const onNodeIconClick = () => { + if (detialUrl) { + navigate(detialUrl); + } + // `/projects/${project}/features/${featureId}`); + }; + + return ( +
+
+ +
+ {nodeTitle} + {active && ( + + )} +
{nodeSubtitle}
+
+ +
+
+ ); +}; + +const LineageNodeComponent = forwardRef(LineageNode); + +LineageNodeComponent.displayName = "LineageNode"; + +export default memo(LineageNodeComponent); diff --git a/ui/src/components/FlowGraph/index.module.less b/ui/src/components/FlowGraph/index.module.less new file mode 100644 index 000000000..9e69f59d7 --- /dev/null +++ b/ui/src/components/FlowGraph/index.module.less @@ -0,0 +1,43 @@ +.flowGraph { + width: 100%; +} + +.lineageNode { + height: 100%; + + &Active { + overflow: hidden; + border-radius: 0.25rem; + border-width: 2px; + border-style: solid; + --tw-border-opacity: 1; + border-color: rgba(57, 35, 150, var(--tw-border-opacity)); + --tw-bg-opacity: 1; + background-color: rgba(57, 35, 150, var(--tw-bg-opacity)); + --tw-text-opacity: 1; + color: rgba(255, 255, 255, var(--tw-text-opacity)); + opacity: 1; + } + + .box { + padding: 4px 12px 7px; + } + + .title { + font-size: 15px; + font-weight: 700; + } + + .subtitle { + font-size: 10px; + font-style: italic; + text-overflow: ellipsis; + max-width: 135px; + overflow: hidden; + white-space: nowrap; + } + + .navigate { + padding: 4px 12px 7px; + } +} diff --git a/ui/src/components/FlowGraph/index.ts b/ui/src/components/FlowGraph/index.ts new file mode 100644 index 000000000..0f6d659d8 --- /dev/null +++ b/ui/src/components/FlowGraph/index.ts @@ -0,0 +1,5 @@ +import FlowGraph from "./FlowGraph"; + +export * from "./interface"; + +export default FlowGraph; diff --git a/ui/src/components/FlowGraph/interface.ts b/ui/src/components/FlowGraph/interface.ts new file mode 100644 index 000000000..0949dbe97 --- /dev/null +++ b/ui/src/components/FlowGraph/interface.ts @@ -0,0 +1,30 @@ +import { CSSProperties } from "react"; +import { FeatureLineage } from "@/models/model"; +import { FeatureType } from "@/utils/utils"; +import { NodeProps, ReactFlowProps } from "react-flow-renderer"; + +export interface NodeData { + id: string; + label: string; + subtitle: string; + featureId: string; + version: string; + borderColor?: string; + active?: boolean; + detialUrl?: string; +} + +export interface FlowGraphProps { + className?: string; + style?: CSSProperties; + minHeight?: string | number; + height?: string | number; + loading?: boolean; + data?: FeatureLineage; + nodeId?: string; + project?: string; + snapGrid?: ReactFlowProps["snapGrid"]; + featureType?: FeatureType; +} + +export interface LineageNodeProps extends NodeProps {} diff --git a/ui/src/components/FlowGraph/utils.ts b/ui/src/components/FlowGraph/utils.ts new file mode 100644 index 000000000..141962895 --- /dev/null +++ b/ui/src/components/FlowGraph/utils.ts @@ -0,0 +1,192 @@ +import { Feature, FeatureLineage, RelationData } from "@/models/model"; +import { FeatureType, getFeatureDetailUrl } from "@/utils/utils"; +import dagre from "dagre"; +import { + Node, + Edge, + ArrowHeadType, + Position, + Elements, +} from "react-flow-renderer"; +import { NodeData } from "./interface"; + +const featureTypeColors: Record = { + feathr_source_v1: "hsl(315, 100%, 50%)", + feathr_anchor_v1: "hsl(270, 100%, 50%)", + feathr_anchor_feature_v1: "hsl(225, 100%, 50%)", + feathr_derived_feature_v1: "hsl(135, 100%, 50%)", +}; + +const DEFAULT_WIDTH = 20; +const DEFAULT_HEIGHT = 36; + +const generateNode = (project: string, data: Feature): Node => { + return { + id: data.guid, + type: "custom-node", + style: { + border: `2px solid featureTypeColors[data.typeName]`, + }, + position: { + x: 0, + y: 0, + }, + data: { + id: data.guid, + label: data.displayText, + subtitle: data.typeName, + featureId: data.guid, + version: data.version, + borderColor: featureTypeColors[data.typeName], + detialUrl: getFeatureDetailUrl(project, data), + }, + }; +}; + +const generateEdge = ( + data: RelationData, + entityMap: Record +): Edge => { + let { fromEntityId: from, toEntityId: to, relationshipType } = data; + + if (relationshipType === "Consumes") { + [from, to] = [to, from]; + } + const sourceNode = entityMap?.[from]; + const targetNode = entityMap?.[to]; + + return { + id: `e-${from}_${to}`, + source: from, + target: to, + arrowHeadType: ArrowHeadType.ArrowClosed, + data: { + sourceTypeName: sourceNode?.typeName, + targetTypeName: targetNode?.typeName, + }, + }; +}; + +export const getLineageNodes = ( + project: string, + lineageData: FeatureLineage, + featureType: FeatureType +): Node[] => { + const { guidEntityMap } = lineageData; + if (!guidEntityMap) { + return []; + } + + return Object.values(guidEntityMap).reduce( + (nodes: Node[], item: Feature) => { + if ( + item.typeName !== "feathr_workspace_v1" && + (featureType === FeatureType.AllNodes || + item.typeName === featureType || + (featureType === FeatureType.AnchorFeature && + item.typeName === FeatureType.Anchor)) + ) { + nodes.push(generateNode(project, item)); + } + return nodes; + }, + [] as Node[] + ); +}; + +export const getLineageEdge = ( + lineageData: FeatureLineage, + featureType: FeatureType +): Edge[] => { + if (!lineageData.relations || !lineageData.guidEntityMap) { + return []; + } + + return lineageData.relations.reduce((edges: Edge[], item) => { + if (["Consumes", "Contains", "Produces"].includes(item.relationshipType)) { + const edge = generateEdge(item, lineageData.guidEntityMap!); + if ( + edges.findIndex((item) => item.id === edge.id) === -1 && + edge.data.sourceTypeName !== "feathr_workspace_v1" && + (featureType === FeatureType.AllNodes || + (featureType === FeatureType.AnchorFeature && + edge.data.sourceTypeName === FeatureType.Anchor && + edge.data.targetTypeName === FeatureType.AnchorFeature)) + ) { + edges.push(edge); + } + } + + return edges; + }, [] as Edge[]); +}; + +export const getElements = ( + project: string, + lineageData: FeatureLineage, + featureType: FeatureType = FeatureType.AllNodes, + direction = "LR" +) => { + const elements: Elements = []; + + const dagreGraph = new dagre.graphlib.Graph({ compound: true }); + + dagreGraph.setDefaultEdgeLabel(() => ({})); + dagreGraph.setGraph({ rankdir: direction }); + + const isHorizontal = direction === "LR"; + + const nodes = getLineageNodes(project, lineageData, featureType); + let edges = getLineageEdge(lineageData, featureType); + + const anchorEdges = edges.filter((item) => { + return ( + item.data.sourceTypeName === FeatureType.Anchor && + item.data.targetTypeName === FeatureType.AnchorFeature + ); + }); + + edges = edges.reduce((data: any, item) => { + const anchorEdge = anchorEdges.find((i: any) => i.target === item.target); + if (anchorEdge) { + if ( + !( + item.data.sourceTypeName === FeatureType.Source && + item.data.targetTypeName === FeatureType.AnchorFeature + ) + ) { + data.push(item); + } + } else { + data.push(item); + } + return data; + }, []); + + nodes.forEach((item) => { + dagreGraph.setNode(item.id, { + label: item.data!.label, + node: item, + width: item.data!.label.length * 8 + DEFAULT_WIDTH, + height: item.style?.height || DEFAULT_HEIGHT, + }); + elements.push(item); + }); + + edges?.forEach((item: any) => { + dagreGraph.setEdge(item.source, item.target); + elements.push(item); + }); + + dagre.layout(dagreGraph); + + nodes.forEach((item) => { + const nodeWithPosition = dagreGraph.node(item.id); + item.targetPosition = isHorizontal ? Position.Left : Position.Top; + item.sourcePosition = isHorizontal ? Position.Right : Position.Bottom; + item.position.x = nodeWithPosition.x; + item.position.y = nodeWithPosition.y - DEFAULT_HEIGHT / 2; + }); + + return elements; +}; diff --git a/ui/src/components/ProjectsSelect/index.tsx b/ui/src/components/ProjectsSelect/index.tsx new file mode 100644 index 000000000..ca5fddf9f --- /dev/null +++ b/ui/src/components/ProjectsSelect/index.tsx @@ -0,0 +1,51 @@ +import React from "react"; +import { Select } from "antd"; +import { fetchProjects } from "@/api"; +import { useQuery } from "react-query"; + +export interface ProjectsSelectProps { + width?: number; + defaultValue?: string; + onChange?: (value: string) => void; +} + +const ProjectsSelect = (props: ProjectsSelectProps) => { + const { width = 350, defaultValue, onChange, ...restProps } = props; + + const { isLoading, data: options } = useQuery< + { value: string; label: string }[] + >( + ["projectsSelect"], + async () => { + try { + const result = await fetchProjects(); + return result.map((item) => ({ + value: item, + label: item, + })); + } catch (e) { + return Promise.reject(e); + } + }, + { + retry: false, + refetchOnWindowFocus: false, + } + ); + + return ( + + + + + + + + + + + + + + + + + ); +}; + +const FeatureFormComponent = forwardRef(FeatureForm); + +FeatureFormComponent.displayName = "FeatureFormComponent"; + +export default FeatureFormComponent; diff --git a/ui/src/pages/feature/components/FeatureTable/index.tsx b/ui/src/pages/feature/components/FeatureTable/index.tsx new file mode 100644 index 000000000..69e9c1ae6 --- /dev/null +++ b/ui/src/pages/feature/components/FeatureTable/index.tsx @@ -0,0 +1,151 @@ +import React, { forwardRef, useRef } from "react"; +import { Button } from "antd"; +import { useQuery } from "react-query"; +import { useNavigate } from "react-router-dom"; +import { Feature } from "@/models/model"; +import { fetchFeatures } from "@/api"; +import ResizeTable, { ResizeColumnType } from "@/components/ResizeTable"; + +export interface DataSourceTableProps { + project?: string; + keyword?: string; +} + +export interface SearchModel { + scope?: string; + roleName?: string; +} + +const DataSourceTable = (props: DataSourceTableProps, ref: any) => { + const navigate = useNavigate(); + + const { project, keyword } = props; + + const projectRef = useRef(project); + + const getDetialUrl = (guid: string) => { + return `/projects/${projectRef.current}/features/${guid}`; + }; + + const columns: ResizeColumnType[] = [ + { + key: "name", + title: "Name", + ellipsis: true, + width: 200, + render: (record: Feature) => { + return ( + + ); + }, + }, + { + key: "type", + title: "Type", + ellipsis: true, + width: 120, + render: (record: Feature) => { + return record.typeName.replace(/feathr_|_v1/gi, ""); + }, + }, + { + key: "transformation", + title: "Transformation", + width: 220, + render: (record: Feature) => { + const { transformExpr, defExpr } = record.attributes.transformation; + return transformExpr || defExpr; + }, + }, + { + key: "entitykey", + title: "Entity Key", + ellipsis: true, + width: 120, + render: (record: Feature) => { + const key = record.attributes.key && record.attributes.key[0]; + if ("NOT_NEEDED" !== key.keyColumn) { + return `${key.keyColumn} (${key.keyColumnType})`; + } else { + return "N/A"; + } + }, + }, + { + key: "aggregation", + title: "Aggregation", + ellipsis: true, + width: 150, + render: (record: Feature) => { + const { transformation } = record.attributes; + return ( + <> + {transformation.aggFunc && `Type: ${transformation.aggFunc}`} +
+ {transformation.aggFunc && `Window: ${transformation.window}`} + + ); + }, + }, + { + title: "Action", + fixed: "right", + width: 100, + resize: false, + render: (record: Feature) => { + return ( + + ); + }, + }, + ]; + + const { isLoading, data: tableData } = useQuery( + ["dataSources", project, keyword], + async () => { + if (project) { + projectRef.current = project; + return await fetchFeatures(project, 1, 10, keyword || ""); + } else { + return []; + } + }, + { + retry: false, + refetchOnWindowFocus: false, + } + ); + + return ( + + ); +}; + +const DataSourceTableComponent = forwardRef( + DataSourceTable +); + +DataSourceTableComponent.displayName = "DataSourceTableComponent"; + +export default DataSourceTableComponent; diff --git a/ui/src/pages/feature/components/NodeDetails/FeatureNodeDetail.tsx b/ui/src/pages/feature/components/NodeDetails/FeatureNodeDetail.tsx new file mode 100644 index 000000000..0224d1d86 --- /dev/null +++ b/ui/src/pages/feature/components/NodeDetails/FeatureNodeDetail.tsx @@ -0,0 +1,52 @@ +import React from "react"; +import { Space } from "antd"; +import { Feature } from "@/models/model"; +import CardDescriptions from "@/components/CardDescriptions"; +import { + TransformationMap, + FeatureKeyMap, + TypeMap, +} from "@/utils/attributesMapping"; +import { getJSONMap } from "@/utils/utils"; + +export interface FeatureNodeDetialProps { + feature: Feature; +} + +const FeatureNodeDetial = (props: FeatureNodeDetialProps) => { + const { feature } = props; + + const { attributes } = feature; + const { transformation, key, type, tags } = attributes; + + const tagsMap = getJSONMap(tags); + + return ( + + + {key?.map((item, index) => { + return ( + + ); + })} + + + + ); +}; + +export default FeatureNodeDetial; diff --git a/ui/src/pages/feature/components/NodeDetails/SourceNodeDetial.tsx b/ui/src/pages/feature/components/NodeDetails/SourceNodeDetial.tsx new file mode 100644 index 000000000..fbf5be158 --- /dev/null +++ b/ui/src/pages/feature/components/NodeDetails/SourceNodeDetial.tsx @@ -0,0 +1,22 @@ +import React from "react"; +import { DataSource } from "@/models/model"; +import { SourceAttributesMap } from "@/utils/attributesMapping"; +import CardDescriptions from "@/components/CardDescriptions"; + +export interface SourceNodeDetialProps { + source: DataSource; +} + +const SourceNodeDetial = (props: SourceNodeDetialProps) => { + const { source } = props; + const { attributes } = source; + return ( + + ); +}; + +export default SourceNodeDetial; diff --git a/ui/src/pages/feature/components/NodeDetails/index.module.less b/ui/src/pages/feature/components/NodeDetails/index.module.less new file mode 100644 index 000000000..9e9815d75 --- /dev/null +++ b/ui/src/pages/feature/components/NodeDetails/index.module.less @@ -0,0 +1,10 @@ +.wrap { + :global { + .ant-space { + margin-bottom: 16px; + } + .card { + box-shadow: none; + } + } +} diff --git a/ui/src/pages/feature/components/NodeDetails/index.tsx b/ui/src/pages/feature/components/NodeDetails/index.tsx new file mode 100644 index 000000000..edbce587d --- /dev/null +++ b/ui/src/pages/feature/components/NodeDetails/index.tsx @@ -0,0 +1,66 @@ +import React from "react"; +import { useParams, useSearchParams } from "react-router-dom"; +import { fetchFeature, fetchDataSource } from "@/api"; +import { LoadingOutlined } from "@ant-design/icons"; +import { useQuery } from "react-query"; +import { Spin, Typography } from "antd"; +import { FeatureType } from "@/utils/utils"; +import FeatureNodeDetail from "./FeatureNodeDetail"; +import SourceNodeDetial from "./SourceNodeDetial"; + +import styles from "./index.module.less"; + +const { Paragraph } = Typography; + +const NodeDetails = () => { + const [searchParams] = useSearchParams(); + const { project } = useParams(); + const nodeId = searchParams.get("nodeId") as string; + const featureType = searchParams.get("featureType") as string; + + const isSource = featureType === FeatureType.Source; + const isFeature = + featureType === FeatureType.AnchorFeature || + featureType === FeatureType.DerivedFeature; + + const { isLoading, data } = useQuery( + ["nodeDetails", project, nodeId], + async () => { + if (isSource || isFeature) { + const api = isSource ? fetchDataSource : fetchFeature; + return await api(project!, nodeId); + } + }, + { + retry: false, + refetchOnWindowFocus: false, + } + ); + + return ( + } + > +
+ {data ? ( + isSource ? ( + + ) : ( + + ) + ) : ( + !isLoading && ( + + Click on source or feature node to show metadata and metric + details + + ) + )} +
+ + ); +}; + +export default NodeDetails; diff --git a/ui/src/pages/feature/components/SearchBar/index.tsx b/ui/src/pages/feature/components/SearchBar/index.tsx new file mode 100644 index 000000000..1a32f28b2 --- /dev/null +++ b/ui/src/pages/feature/components/SearchBar/index.tsx @@ -0,0 +1,67 @@ +import React, { useRef } from "react"; +import { Form, Input, Button } from "antd"; +import { useNavigate } from "react-router-dom"; +import ProjectsSelect from "@/components/ProjectsSelect"; + +export interface SearchValue { + project?: string; + keyword?: string; +} + +export interface SearchBarProps { + defaultValues?: SearchValue; + onSearch?: (values: SearchValue) => void; +} + +const { Item } = Form; + +const SearchBar = (props: SearchBarProps) => { + const [form] = Form.useForm(); + + const navigate = useNavigate(); + + const { defaultValues, onSearch } = props; + + const timeRef = useRef(null); + + const onChangeKeyword = () => { + clearTimeout(timeRef.current); + timeRef.current = setTimeout(() => { + form.submit(); + }, 350); + }; + + return ( +
+
+ + + + + + +
+ +
+ ); +}; + +export default SearchBar; diff --git a/ui/src/pages/feature/featureDetails.tsx b/ui/src/pages/feature/featureDetails.tsx index 549e5e3f7..a5bef8688 100644 --- a/ui/src/pages/feature/featureDetails.tsx +++ b/ui/src/pages/feature/featureDetails.tsx @@ -1,218 +1,117 @@ -import React, { useEffect, useState } from "react"; -import { Alert, Button, Card, Col, Row, Space, Spin, Typography } from "antd"; +import React, { useEffect, useRef, useState } from "react"; +import { + Alert, + Button, + PageHeader, + Breadcrumb, + Space, + Card, + Spin, + Descriptions, +} from "antd"; import { LoadingOutlined } from "@ant-design/icons"; -import { useNavigate, useParams } from "react-router-dom"; -import { QueryStatus, useQuery } from "react-query"; +import { Link, useNavigate, useParams } from "react-router-dom"; +import { useQuery } from "react-query"; import { AxiosError } from "axios"; -import { fetchFeature } from "../../api"; -import { Feature, InputFeature } from "../../models/model"; -import { FeatureLineage } from "../../models/model"; -import { fetchFeatureLineages } from "../../api"; -import { Elements } from "react-flow-renderer"; -import Graph from "../../components/graph/graph"; -import { getElements } from "../../components/graph/utils"; - -const { Title } = Typography; - -type FeatureKeyProps = { feature: Feature }; -const FeatureKey = ({ feature }: FeatureKeyProps) => { - const keys = feature.attributes.key; - return ( - <> - {keys && keys.length > 0 && ( - - - Entity Key -
-

Full Name: {keys[0].fullName}

-

Key Column: {keys[0].keyColumn}

-

Description: {keys[0].description}

-

Key Column Alias: {keys[0].keyColumnAlias}

-

Key Column Type: {keys[0].keyColumnType}

-
- - - )} - - ); -}; - -type FeatureTypeProps = { feature: Feature }; -const FeatureType = ({ feature }: FeatureTypeProps) => { - const type = feature.attributes.type; - return ( - <> - {type && ( - - - Type -
-

Dimension Type: {type.dimensionType}

-

Tensor Category: {type.tensorCategory}

-

Type: {type.type}

-

Value Type: {type.valType}

-
- - - )} - - ); -}; - -type FeatureTransformationProps = { feature: Feature }; -const FeatureTransformation = ({ feature }: FeatureTransformationProps) => { - const transformation = feature.attributes.transformation; - return ( - <> - {transformation && ( - - - Transformation -
- {transformation.transformExpr && ( -

Expression: {transformation.transformExpr}

- )} - {transformation.filter &&

Filter: {transformation.filter}

} - {transformation.aggFunc && ( -

Aggregation: {transformation.aggFunc}

- )} - {transformation.limit &&

Limit: {transformation.limit}

} - {transformation.groupBy && ( -

Group By: {transformation.groupBy}

- )} - {transformation.window &&

Window: {transformation.window}

} - {transformation.defExpr && ( -

Expression: {transformation.defExpr}

- )} -
- - - )} - - ); -}; +import { fetchFeature, fetchFeatureLineages } from "@/api"; +import { Feature, InputFeature, FeatureLineage } from "@/models/model"; +import FlowGraph from "@/components/FlowGraph"; +import CardDescriptions from "@/components/CardDescriptions"; +import { + FeatureKeyMap, + TransformationMap, + TypeMap, +} from "@/utils/attributesMapping"; +import { getJSONMap } from "@/utils/utils"; + +const contentStyle = { marginRight: 16 }; type InputAnchorFeaturesProps = { project: string; feature: Feature }; -const InputAnchorFeatures = ({ - project, - feature, -}: InputAnchorFeaturesProps) => { - const navigate = useNavigate(); - const inputAnchorFeatures = feature.attributes.inputAnchorFeatures; - return ( - <> - {inputAnchorFeatures && inputAnchorFeatures.length > 0 && ( - - - Input Anchor Features - {inputAnchorFeatures.map((input_feature) => ( - - ))} - - - )} - - ); + +const InputAnchorFeatures = (props: InputAnchorFeaturesProps) => { + const { project, feature } = props; + + const { inputAnchorFeatures } = feature.attributes; + + return inputAnchorFeatures?.length > 0 ? ( + + + {inputAnchorFeatures.map((input_feature) => ( + + + {input_feature.uniqueAttributes.qualifiedName} + + + ))} + + + ) : null; }; type InputDerivedFeaturesProps = { project: string; feature: Feature }; -const InputDerivedFeatures = ({ - project, - feature, -}: InputDerivedFeaturesProps) => { - const navigate = useNavigate(); - const inputDerivedFeatures = feature.attributes.inputDerivedFeatures; - return ( - <> - {inputDerivedFeatures && inputDerivedFeatures.length > 0 && ( - - - Input Derived Features - {inputDerivedFeatures.map((input_feature: InputFeature) => ( - - ))} - - - )} - - ); + +const InputDerivedFeatures = (props: InputDerivedFeaturesProps) => { + const { project, feature } = props; + + const { inputDerivedFeatures } = feature.attributes; + + return inputDerivedFeatures?.length ? ( + + + {inputDerivedFeatures.map((input_feature: InputFeature) => ( + + + {input_feature.uniqueAttributes.qualifiedName} + + + ))} + + + ) : null; }; const FeatureLineageGraph = () => { - const { featureId } = useParams() as Params; + const { project, featureId } = useParams() as Params; const [lineageData, setLineageData] = useState({ - guidEntityMap: null, - relations: null, + guidEntityMap: {}, + relations: [], }); - const [elements, SetElements] = useState([]); + const [loading, setLoading] = useState(false); + const mountedRef = useRef(true); + useEffect(() => { const fetchLineageData = async () => { setLoading(true); const data = await fetchFeatureLineages(featureId); - setLineageData(data); - setLoading(false); + if (mountedRef.current) { + setLineageData(data); + setLoading(false); + } }; fetchLineageData(); }, [featureId]); - // Generate graph data on client side, invoked after graphData or featureType is changed useEffect(() => { - const generateGraphData = async () => { - SetElements(getElements(lineageData, "all_nodes")!); + mountedRef.current = true; + return () => { + mountedRef.current = false; }; - - generateGraphData(); - }, [lineageData]); - - return ( - <> - {loading ? ( - } /> - ) : ( - - - Lineage - - - - )} - - ); + }, []); + + return !loading ? ( + + + + ) : null; }; type Params = { @@ -222,87 +121,89 @@ type Params = { const FeatureDetails = () => { const { project, featureId } = useParams() as Params; const navigate = useNavigate(); - const loadingIcon = ; - const { status, error, data } = useQuery( + + const { + isLoading, + error, + data = { attributes: {} } as Feature, + } = useQuery( ["featureId", featureId], - () => fetchFeature(project, featureId) + () => fetchFeature(project, featureId), + { + retry: false, + refetchOnWindowFocus: false, + } ); + const { attributes } = data; + const { transformation, key, type, name, tags } = attributes; - const openLineageWindow = () => { - const lineageUrl = `/projects/${project}/lineage`; - navigate(lineageUrl); - }; + const tagsMap = getJSONMap(tags); - const render = (status: QueryStatus): JSX.Element => { - switch (status) { - case "error": - return ( - - - - ); - case "idle": - return ( - - - - ); - case "loading": - return ( - - - - ); - case "success": - if (data === undefined) { - return ( - - - - ); - } else { - return ( - <> - - - {data.attributes.name} -
- - - -
-
- - - - - - - - -
- - - ); + return ( +
+ + + Features + + Feature Details + } - } - }; - - return
{render(status)}
; + extra={[ + , + ]} + > + } + > + + {error && } + + + + {key?.map((item, index) => { + return ( + + ); + })} + + + + + + + +
+ ); }; export default FeatureDetails; diff --git a/ui/src/pages/feature/features.tsx b/ui/src/pages/feature/features.tsx index 275cde11f..9ace6ead6 100644 --- a/ui/src/pages/feature/features.tsx +++ b/ui/src/pages/feature/features.tsx @@ -1,20 +1,27 @@ -import { Button, Card, Space, Typography } from "antd"; -import { useNavigate, useSearchParams } from "react-router-dom"; -import FeatureList from "../../components/featureList"; - -const { Title } = Typography; +import { useState } from "react"; +import { PageHeader } from "antd"; +import { useSearchParams } from "react-router-dom"; +import SearchBar, { SearchValue } from "./components/SearchBar"; +import FeatureTable from "./components/FeatureTable"; const Features = () => { const [searchParams] = useSearchParams(); - const project = (searchParams.get("project") as string) ?? ""; - const keyword = (searchParams.get("keyword") as string) ?? ""; + + const [search, setProject] = useState({ + project: searchParams.get("project") || undefined, + keyword: searchParams.get("keyword") || undefined, + }); + + const onSearch = (values: SearchValue) => { + setProject(values); + }; return (
- - Features - - + + + +
); }; diff --git a/ui/src/pages/feature/lineageGraph.tsx b/ui/src/pages/feature/lineageGraph.tsx index ac75dff91..d8b1473df 100644 --- a/ui/src/pages/feature/lineageGraph.tsx +++ b/ui/src/pages/feature/lineageGraph.tsx @@ -1,17 +1,17 @@ -import React, { useEffect, useState } from "react"; -import { Card, Col, Radio, Row, Spin, Tabs, Typography } from "antd"; +import React, { useEffect, useRef, useState } from "react"; +import { PageHeader, Row, Col, Radio, Tabs } from "antd"; import { useParams, useSearchParams } from "react-router-dom"; -import { Elements } from "react-flow-renderer"; -import Graph from "../../components/graph/graph"; -import { fetchProjectLineages } from "../../api"; -import { FeatureLineage } from "../../models/model"; -import { LoadingOutlined } from "@ant-design/icons"; -import GraphNodeDetails from "../../components/graph/graphNodeDetails"; -import { getElements } from "../../components/graph/utils"; -import { FeatureType } from "../../utils/utils"; +import FlowGraph from "@/components/FlowGraph"; +import { fetchProjectLineages } from "@/api"; +import { FeatureLineage } from "@/models/model"; +import { FeatureType } from "@/utils/utils"; +import NodeDetails from "./components/NodeDetails"; -const { Title } = Typography; -const { TabPane } = Tabs; +const items = [ + { label: "Metadata", key: "1", children: }, + { label: "Metrics", key: "2", children:

Under construction

}, // 务必填写 key + { label: "Jobs", key: "3", children:

Under construction

}, +]; type Params = { project: string; @@ -22,90 +22,75 @@ const LineageGraph = () => { const nodeId = searchParams.get("nodeId") as string; const [lineageData, setLineageData] = useState({ - guidEntityMap: null, - relations: null, + guidEntityMap: {}, + relations: [], }); + const [loading, setLoading] = useState(false); - const [elements, SetElements] = useState([]); - const [featureType, setFeatureType] = useState("all_nodes"); + + const [featureType, setFeatureType] = useState( + FeatureType.AllNodes + ); + + const mountedRef = useRef(true); // Fetch lineage data from server side, invoked immediately after component is mounted useEffect(() => { const fetchLineageData = async () => { setLoading(true); const data = await fetchProjectLineages(project); - setLineageData(data); - setLoading(false); + if (mountedRef.current) { + setLineageData(data); + setLoading(false); + } }; fetchLineageData(); }, [project]); - // Generate graph data on client side, invoked after graphData or featureType is changed + const toggleFeatureType = (type: FeatureType) => { + setFeatureType(type); + }; + useEffect(() => { - const generateGraphData = async () => { - SetElements(getElements(lineageData, featureType)!); + mountedRef.current = true; + return () => { + mountedRef.current = false; }; - - generateGraphData(); - }, [lineageData, featureType]); - - const toggleFeatureType = (type: string) => { - setFeatureType((prevType: string | null) => { - if (prevType === type) { - return null; - } - return type; - }); - }; + }, []); return (
- - Lineage {project} -
- toggleFeatureType(e.target.value)} - > - All Nodes - Source - Anchor - - Anchor Feature - - - Derived Feature - - -
-
- {loading ? ( - } + + toggleFeatureType(e.target.value)} + > + All Nodes + Source + + Anchor Feature + + + Derived Feature + + + + + - ) : ( - - - - - - - - - - -

Under construction

- - -

Under construction

- - - - - )} -
- + + + + + +
); }; diff --git a/ui/src/pages/feature/newFeature.tsx b/ui/src/pages/feature/newFeature.tsx index d51dd2aa0..50afd64c3 100644 --- a/ui/src/pages/feature/newFeature.tsx +++ b/ui/src/pages/feature/newFeature.tsx @@ -1,16 +1,13 @@ import React from "react"; -import { Card, Typography } from "antd"; -import FeatureForm from "../../components/featureForm"; - -const { Title } = Typography; +import { PageHeader } from "antd"; +import FeatureForm from "./components/FeatureForm"; const NewFeature = () => { return (
- - Create Feature + - +
); }; diff --git a/ui/src/pages/home/home.css b/ui/src/pages/home/home.css deleted file mode 100644 index 308e45367..000000000 --- a/ui/src/pages/home/home.css +++ /dev/null @@ -1,23 +0,0 @@ -.home .ant-card { - box-shadow: 5px 8px 15px 5px rgba(208, 216, 243, 0.6); - border-radius: 8px; -} - -.home .card-meta { - display: flex; -} - -.home .card-meta .ant-card-meta-avatar { - max-width: 80px; - flex-basis: 30%; - box-sizing: border-box; -} - -.home .card-meta .ant-card-meta-avatar > span { - width: 100%; -} - -.home .card-meta .ant-card-meta-avatar svg { - width: 100%; - height: auto; -} diff --git a/ui/src/pages/home/home.tsx b/ui/src/pages/home/home.tsx index 88732ffeb..824d5db95 100644 --- a/ui/src/pages/home/home.tsx +++ b/ui/src/pages/home/home.tsx @@ -1,14 +1,16 @@ import React from "react"; -import { Link } from "react-router-dom"; -import { Card, Col, Row, Typography } from "antd"; + import { CopyOutlined, DatabaseOutlined, EyeOutlined, ProjectOutlined, } from "@ant-design/icons"; +import { Card, Col, Row, Typography } from "antd"; +import cs from "classnames"; +import { Link } from "react-router-dom"; -import "./home.css"; +import styles from "./index.module.less"; const { Title } = Typography; const { Meta } = Card; @@ -42,7 +44,7 @@ const features = [ const Home = () => { return ( -
+
Welcome to Feathr Feature Store @@ -71,7 +73,7 @@ const Home = () => { > @@ -94,7 +96,7 @@ const Home = () => {
  • Documentation @@ -104,7 +106,7 @@ const Home = () => {
  • Running Feathr on Cloud @@ -114,7 +116,7 @@ const Home = () => {
  • Cloud Integrations and Architecture on Cloud @@ -124,7 +126,7 @@ const Home = () => {
  • Slack Channel @@ -135,7 +137,7 @@ const Home = () => {
  • Community Guidelines diff --git a/ui/src/pages/home/index.module.less b/ui/src/pages/home/index.module.less new file mode 100644 index 000000000..59354c568 --- /dev/null +++ b/ui/src/pages/home/index.module.less @@ -0,0 +1,28 @@ +.home { + :global { + .ant-card { + box-shadow: 5px 8px 15px 5px rgba(208, 216, 243, 0.6); + border-radius: 8px; + } + } + + .cardMeta { + display: flex; + :global { + .ant-card-meta-avatar { + max-width: 80px; + flex-basis: 30%; + box-sizing: border-box; + + > span { + width: 100%; + } + + svg { + width: 100%; + height: auto; + } + } + } + } +} diff --git a/ui/src/pages/management/components/RoleForm/index.tsx b/ui/src/pages/management/components/RoleForm/index.tsx index 9e073abd8..5cef3d02c 100644 --- a/ui/src/pages/management/components/RoleForm/index.tsx +++ b/ui/src/pages/management/components/RoleForm/index.tsx @@ -1,6 +1,6 @@ import React, { forwardRef, useCallback, useEffect, useState } from "react"; import { Form, Select, Input, Button, message } from "antd"; -import { listUserRole, addUserRole } from "../../../../api"; +import { listUserRole, addUserRole } from "@/api"; export interface RoleFormProps { getRole?: (isAdmin: boolean) => void; @@ -92,7 +92,9 @@ const RoleForm = (props: RoleFormProps, ref: any) => { + + +
    • + ); +}; + +const SearchBarComponent = forwardRef(SearchBar); + +SearchBarComponent.displayName = "SearchBarComponent"; + +export default SearchBarComponent; diff --git a/ui/src/pages/project/projects.tsx b/ui/src/pages/project/projects.tsx index 03cbf3d48..932915089 100644 --- a/ui/src/pages/project/projects.tsx +++ b/ui/src/pages/project/projects.tsx @@ -1,16 +1,21 @@ -import React from "react"; -import { Card, Typography } from "antd"; -import ProjectList from "../../components/projectList"; - -const { Title } = Typography; +import React, { useState } from "react"; +import { PageHeader } from "antd"; +import ProjectTable from "./components/ProjectTable"; +import SearchBar from "./components/SearchBar"; const Projects = () => { + const [project, setProject] = useState(""); + + const onSearch = ({ project }: { project: string }) => { + setProject(project); + }; + return (
    - - Projects - - + + + +
    ); }; diff --git a/ui/src/site.css b/ui/src/site.css index 2d90b28bf..5906d7315 100644 --- a/ui/src/site.css +++ b/ui/src/site.css @@ -1,10 +1,8 @@ .page { - margin: 1%; + margin: 16px; } .card { - margin-top: 15px; - margin-right: 15px; box-shadow: 5px 8px 15px 5px rgba(208, 216, 243, 0.6); border-radius: 8px; } @@ -60,3 +58,7 @@ .dataSource-container { column-count: 1; } + +.display-flex { + display: flex; +} diff --git a/ui/src/typings/file.d.ts b/ui/src/typings/file.d.ts new file mode 100644 index 000000000..ae5426269 --- /dev/null +++ b/ui/src/typings/file.d.ts @@ -0,0 +1,44 @@ +declare module "*.svg" { + const path: string; + export default path; +} + +declare module "*.bmp" { + const path: string; + export default path; +} + +declare module "*.gif" { + const path: string; + export default path; +} + +declare module "*.jpg" { + const path: string; + export default path; +} + +declare module "*.jpeg" { + const path: string; + export default path; +} + +declare module "*.png" { + const path: string; + export default path; +} + +declare module "*.css" { + const classes: { readonly [key: string]: string }; + export default classes; +} + +declare module "*.less" { + const classes: { readonly [key: string]: string }; + export default classes; +} + +declare module "*.module.less" { + const classes: { readonly [key: string]: string }; + export default classes; +} diff --git a/ui/src/utils/attributesMapping.ts b/ui/src/utils/attributesMapping.ts new file mode 100644 index 000000000..9e888d1d0 --- /dev/null +++ b/ui/src/utils/attributesMapping.ts @@ -0,0 +1,48 @@ +import { + FeatureTransformation, + FeatureKey, + FeatureType, + DataSourceAttributes, +} from "@/models/model"; + +export const TransformationMap: Array<{ + label: string; + key: keyof FeatureTransformation; +}> = [ + { label: "Expression", key: "transformExpr" }, + { label: "Filter", key: "filter" }, + { label: "Aggregation", key: "aggFunc" }, + { label: "Limit", key: "limit" }, + { label: "Group By", key: "groupBy" }, + { label: "Window", key: "window" }, + { label: "Expression", key: "defExpr" }, +]; + +export const FeatureKeyMap: Array<{ label: string; key: keyof FeatureKey }> = [ + { label: "Full name", key: "fullName" }, + { label: "Description", key: "description" }, + { label: "Key column", key: "keyColumn" }, + { label: "Key column alias", key: "keyColumnAlias" }, + { label: "Key column type", key: "keyColumnType" }, +]; + +export const TypeMap: Array<{ label: string; key: keyof FeatureType }> = [ + { label: "Dimension Type", key: "dimensionType" }, + { label: "Tensor Category", key: "tensorCategory" }, + { label: "Type", key: "type" }, + { label: "Value Type", key: "valType" }, +]; + +export const SourceAttributesMap: Array<{ + label: string; + key: keyof DataSourceAttributes; +}> = [ + { label: "Name", key: "name" }, + { label: "Type", key: "type" }, + { label: "Path", key: "path" }, + { label: "Preprocessing", key: "preprocessing" }, + { label: "Event Timestamp Column", key: "event_timestamp_column" }, + { label: "Timestamp Format", key: "timestamp_format" }, + { label: "Qualified Name", key: "qualified_name" }, + { label: "Tags", key: "tags" }, +]; diff --git a/ui/src/utils/utils.tsx b/ui/src/utils/utils.tsx index 85bfd8f42..28b066233 100644 --- a/ui/src/utils/utils.tsx +++ b/ui/src/utils/utils.tsx @@ -1,3 +1,4 @@ +import { Feature } from "@/models/model"; import { Configuration, PublicClientApplication } from "@azure/msal-browser"; export const getMsalConfig = () => { @@ -16,8 +17,6 @@ export const getMsalConfig = () => { redirectUri: window.location.origin, }, }; - console.log("clientId = ", clientId); - console.log("authority = ", authority); return new PublicClientApplication(msalConfig); }; @@ -33,6 +32,29 @@ export const enum FeatureType { export const isFeature = (featureType: string) => { return ( featureType === FeatureType.AnchorFeature || - featureType === FeatureType.DerivedFeature + featureType === FeatureType.DerivedFeature || + featureType === FeatureType.Source ); }; + +export const getFeatureDetailUrl = (project: string, feature: Feature) => { + switch (feature.typeName) { + case FeatureType.Source: + return `/projects/${project}/dataSources/${feature.guid}`; + case FeatureType.AnchorFeature: + case FeatureType.DerivedFeature: + return `/projects/${project}/features/${feature.guid}`; + default: + return; + } +}; + +export const getJSONMap = (json: any = {}) => { + return Object.keys(json).map((key) => { + return { label: key, key }; + }); +}; + +export const isEmpty = (obj: any = {}) => { + return !obj || Object.getOwnPropertyNames(obj).length === 0; +}; diff --git a/ui/tsconfig.json b/ui/tsconfig.json index 9d379a3c4..55d8b5695 100644 --- a/ui/tsconfig.json +++ b/ui/tsconfig.json @@ -14,7 +14,20 @@ "resolveJsonModule": true, "isolatedModules": true, "noEmit": true, - "jsx": "react-jsx" + "jsx": "react-jsx", + "baseUrl": "./", + "rootDir": ".", + "paths": { + "@/*": ["src/*"] + }, + "plugins": [ + { + "name": "typescript-plugin-css-modules", + "options": { + "classnameTransform": "camelCaseOnly" + } + } + ] }, - "include": ["src"] + "include": ["src", "src/**/*.d.ts"] }