Skip to content

Commit

Permalink
feat: add metadata glossary and insight funcs
Browse files Browse the repository at this point in the history
  • Loading branch information
ciusji committed Mar 25, 2023
1 parent d9ff8af commit 140b435
Show file tree
Hide file tree
Showing 2,576 changed files with 304,573 additions and 0 deletions.
15 changes: 15 additions & 0 deletions .github/CODEOWNERS
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
# Individual scopes

# Review from UI owners for changes around UI code
/openmetadata-ui/src/main/resources/ui/ @harshach @shahsank3t

# Review from Backend owners for changes around Backend code
/openmetadata-service/ @open-metadata/backend

# Review from Ingestion owners for changes around Ingestion code
/ingestion @open-metadata/ingestion
/ingestion-core @open-metadata/ingestion

# Review from Devops owners for changes around workflows
/.github @akash-jain-10 @harshach
/docker @akash-jain-10 @harshach
7 changes: 7 additions & 0 deletions .github/ISSUE_TEMPLATE/config.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
contact_links:
- name: Read the docs & troubleshoot
url: https://docs.open-metadata.org/
about: You can find information on anything related to OpenMetadata.
- name: Chat on Slack
url: https://slack.open-metadata.org/
about: And get help from the community
60 changes: 60 additions & 0 deletions .github/teams.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
# Add here any member that should belong to either a specific team,
# or that can have the tests automatically validated to run safely.

"safe to test":
- '@ayush-shah'
- '@MilanBariya'
- '@nahuelverdugo'
- '@NiharDoshi99'
- '@OnkarVO7'
- '@TeddyCr'
- '@ulixius9'
- '@pmbrull'
- '@aniketkatkar97'
- '@Ashish8689'
- '@chirag-madlani'
- '@Sachin-chaurasiya'
- '@shahsank3t'
- '@ShaileshParmar11'
- '@karanh37'
- '@harshach'
- '@mohityadav766'
- '@sureshms'
- '@akash-jain-10'
- '@Vj-L'
- '@dhruvinmaniar123'
- '@ShilpaVernekar'
- '@snyk-bot'
- '@dependabot'

ingestion:
- '@ayush-shah'
- '@MilanBariya'
- '@nahuelverdugo'
- '@NiharDoshi99'
- '@OnkarVO7'
- '@TeddyCr'
- '@ulixius9'
- '@pmbrull'

UI:
- '@aniketkatkar97'
- '@Ashish8689'
- '@chirag-madlani'
- '@Sachin-chaurasiya'
- '@shahsank3t'
- '@ShaileshParmar11'
- '@karanh37'

backend:
- '@harshach'
- '@mohityadav766'
- '@sureshms'

devops:
- '@akash-jain-10'
- '@Vj-L'
- '@dhruvinmaniar123'

documentation:
- '@ShilpaVernekar'
133 changes: 133 additions & 0 deletions .github/workflows/airflow-apis-tests-3_9.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,133 @@

# Copyright 2021 Collate
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

name: airflow-apis-tests
on:
push:
branches:
- main
- '0.[0-9]+.[0-9]+'
paths:
- 'openmetadata-airflow-apis/**'
pull_request_target:
types: [labeled, opened, synchronize, reopened]
paths:
- 'openmetadata-airflow-apis/**'

permissions:
contents: read

concurrency:
group: airflow-apis-tests-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
jobs:
airflow-apis-tests:
runs-on: ubuntu-latest
steps:

- name: Wait for the labeler
uses: lewagon/wait-on-check-action@0179dfc359f90a703c41240506f998ee1603f9ea #v1.0.0
if: ${{ github.event_name == 'pull_request_target' }}
with:
ref: ${{ github.event.pull_request.head.sha }}
check-name: Team Label
repo-token: ${{ secrets.GITHUB_TOKEN }}
wait-interval: 30

- name: Verify PR labels
uses: jesusvasquez333/[email protected]
if: ${{ github.event_name == 'pull_request_target' }}
with:
github-token: '${{ secrets.GITHUB_TOKEN }}'
valid-labels: 'safe to test'
pull-request-number: '${{ github.event.pull_request.number }}'
disable-reviews: true # To not auto approve changes

- name: Checkout
uses: actions/checkout@v3
with:
ref: ${{ github.event.pull_request.head.sha }}
fetch-depth: 0

- name: Set up JDK 11
uses: actions/setup-java@v3
with:
java-version: '11'
distribution: 'adopt'

- name: Set up Python 3.9
uses: actions/setup-python@v4
with:
python-version: 3.9

- name: Install Ubuntu dependencies
run: |
# stop relying on apt cache of GitHub runners
sudo apt-get update
sudo apt-get install -y unixodbc-dev python3-venv librdkafka-dev gcc libsasl2-dev build-essential libssl-dev libffi-dev \
librdkafka-dev unixodbc-dev libevent-dev
- name: Generate models
run: |
python3 -m venv env
source env/bin/activate
sudo make install_antlr_cli
make install_dev generate
- name: Install open-metadata dependencies
run: |
source env/bin/activate
make install_all install_test
- name: Start Server and Ingest Sample Data
uses: nick-fields/[email protected]
env:
INGESTION_DEPENDENCY: "mysql,elasticsearch"
with:
timeout_minutes: 30
max_attempts: 2
retry_on: error
command: ./docker/run_local_docker.sh -m no-ui

- name: Run Python Tests & Record Coverage
run: |
source env/bin/activate
make install_apis
make coverage_apis
rm pom.xml
# fix coverage xml report for github
sed -i 's/openmetadata_managed_apis/\/github\/workspace\/openmetadata-airflow-apis\/openmetadata_managed_apis/g' openmetadata-airflow-apis/ci-coverage.xml
- name: Push Results in PR to Sonar
uses: sonarsource/sonarcloud-github-action@master
if: ${{ github.event_name == 'pull_request_target' }}
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
SONAR_TOKEN: ${{ secrets.AIRFLOW_APIS_SONAR_TOKEN }}
with:
projectBaseDir: openmetadata-airflow-apis/
args: >
-Dproject.settings=openmetadata-airflow-apis/sonar-project.properties
-Dsonar.pullrequest.key=${{ github.event.pull_request.number }}
-Dsonar.pullrequest.branch=${{ github.head_ref }}
-Dsonar.pullrequest.github.repository=OpenMetadata
-Dsonar.scm.revision=${{ github.event.pull_request.head.sha }}
-Dsonar.pullrequest.provider=github
- name: Push Results to Sonar
uses: sonarsource/sonarcloud-github-action@master
if: ${{ github.event_name == 'push' }}
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
SONAR_TOKEN: ${{ secrets.AIRFLOW_APIS_SONAR_TOKEN }}
with:
projectBaseDir: openmetadata-airflow-apis/
166 changes: 166 additions & 0 deletions .github/workflows/cypress-integration-tests-mysql.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,166 @@
# Copyright 2021 Collate
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This workflow will build a package using Maven and then publish it to GitHub packages when a release is created
# For more information see: https://github.com/actions/setup-java/blob/main/docs/advanced-usage.md#apache-maven-with-a-settings-path

name: MySQL Cypress Integration Tests

on:
push:
branches:
- main
- '0.[0-9]+.[0-9]+'
paths-ignore:
- 'openmetadata-docs/**'
pull_request_target:
types: [labeled, opened, synchronize, reopened]
paths-ignore:
- 'openmetadata-docs/**'

permissions:
contents: read

concurrency:
group: cypress-integration-tests-mysql-${{ github.head_ref || github.run_id }}
cancel-in-progress: true

jobs:
install:
runs-on: ubuntu-latest
strategy:
matrix:
job: [0, 1]
environment: test
steps:

- name: Wait for the labeler
uses: lewagon/wait-on-check-action@0179dfc359f90a703c41240506f998ee1603f9ea #v1.0.0
if: ${{ github.event_name == 'pull_request_target' }}
with:
ref: ${{ github.event.pull_request.head.sha }}
check-name: Team Label
repo-token: ${{ secrets.GITHUB_TOKEN }}
wait-interval: 30

- name: Verify PR labels
uses: jesusvasquez333/[email protected]
if: ${{ github.event_name == 'pull_request_target' }}
with:
github-token: '${{ secrets.GITHUB_TOKEN }}'
valid-labels: 'safe to test'
pull-request-number: '${{ github.event.pull_request.number }}'
disable-reviews: true # To not auto approve changes

- name: Checkout
uses: actions/checkout@v3
with:
ref: ${{ github.event.pull_request.head.sha }}

- name: Cache Maven Dependencies
id: cache-output
uses: actions/cache@v3
with:
path: ~/.m2
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-maven-
- name: Get yarn cache directory path
if: steps.cache-output.outputs.exit-code == 0
id: yarn-cache-dir-path
run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT

- uses: actions/cache@v3
if: steps.yarn-cache-dir-path.outputs.exit-code == 0
id: yarn-cache
with:
path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
restore-keys: |
${{ runner.os }}-yarn-
- name: Set up JDK 11
if: steps.cache-output.outputs.exit-code == 0
uses: actions/setup-java@v3
with:
java-version: '11'
distribution: 'adopt'

- name: Generating Data Models
run: |
sudo make install_antlr_cli
make install_dev generate
- name: Start Server and Ingest Sample Data
env:
INGESTION_DEPENDENCY: "all"
run: ./docker/run_local_docker.sh -d mysql
timeout-minutes: 30

- name: Run Cypress Tests
uses: cypress-io/github-action@v4
with:
install: false
record: true
parallel: true
working-directory: openmetadata-ui/src/main/resources/ui/
wait-on: 'http://localhost:8585'
group: Tests with database mysql
browser: chrome
env:
CYPRESS_RECORD_KEY: ${{ secrets.TEST_RECORD_KEY }}
CYPRESS_SNOWFLAKE_USERNAME: ${{ secrets.TEST_SNOWFLAKE_USERNAME }}
CYPRESS_SNOWFLAKE_PASSWORD: ${{ secrets.TEST_SNOWFLAKE_PASSWORD }}
CYPRESS_SNOWFLAKE_ACCOUNT: ${{ secrets.TEST_SNOWFLAKE_ACCOUNT }}
CYPRESS_SNOWFLAKE_DATABASE: ${{ secrets.TEST_SNOWFLAKE_DATABASE }}
CYPRESS_SNOWFLAKE_WAREHOUSE: ${{ secrets.TEST_SNOWFLAKE_WAREHOUSE }}
CYPRESS_BQ_PRIVATE_KEY: ${{ secrets.TEST_BQ_PRIVATE_KEY }}
CYPRESS_BQ_PROJECT_ID: ${{ secrets.TEST_BQ_PROJECT_ID }}
CYPRESS_BQ_PRIVATE_KEY_ID: ${{ secrets.TEST_BQ_PRIVATE_KEY_ID }}
CYPRESS_BQ_CLIENT_EMAIL: ${{ secrets.TEST_BQ_CLIENT_EMAIL }}
CYPRESS_BQ_CLIENT_ID: ${{ secrets.TEST_BQ_CLIENT_ID }}
CYPRESS_REDSHIFT_HOST: ${{ secrets.TEST_REDSHIFT_HOST }}
CYPRESS_REDSHIFT_USERNAME: ${{ secrets.TEST_REDSHIFT_USERNAME }}
CYPRESS_REDSHIFT_PASSWORD: ${{ secrets.TEST_REDSHIFT_PASSWORD }}
CYPRESS_REDSHIFT_DATABASE: ${{ secrets.TEST_REDSHIFT_DATABASE }}
CYPRESS_METABASE_USERNAME: ${{ secrets.TEST_METABASE_USERNAME }}
CYPRESS_METABASE_PASSWORD: ${{ secrets.TEST_METABASE_PASSWORD }}
CYPRESS_METABASE_DB_SERVICE_NAME: ${{ secrets.TEST_METABASE_DB_SERVICE_NAME }}
CYPRESS_METABASE_HOST_PORT: ${{ secrets.TEST_METABASE_HOST_PORT }}
CYPRESS_SUPERSET_USERNAME: ${{ secrets.TEST_SUPERSET_USERNAME }}
CYPRESS_SUPERSET_PASSWORD: ${{ secrets.TEST_SUPERSET_PASSWORD }}
CYPRESS_SUPERSET_HOST_PORT: ${{ secrets.TEST_SUPERSET_HOST_PORT }}
CYPRESS_KAFKA_BOOTSTRAP_SERVERS: ${{ secrets.TEST_KAFKA_BOOTSTRAP_SERVERS }}
CYPRESS_KAFKA_SCHEMA_REGISTRY_URL: ${{ secrets.TEST_KAFKA_SCHEMA_REGISTRY_URL }}
CYPRESS_GLUE_ACCESS_KEY: ${{ secrets.TEST_GLUE_ACCESS_KEY }}
CYPRESS_GLUE_SECRET_KEY: ${{ secrets.TEST_GLUE_SECRET_KEY }}
CYPRESS_GLUE_AWS_REGION: ${{ secrets.TEST_GLUE_AWS_REGION }}
CYPRESS_GLUE_ENDPOINT: ${{ secrets.TEST_GLUE_ENDPOINT }}
CYPRESS_GLUE_STORAGE_SERVICE: ${{ secrets.TEST_GLUE_STORAGE_SERVICE }}
CYPRESS_MYSQL_USERNAME: ${{ secrets.TEST_MYSQL_USERNAME }}
CYPRESS_MYSQL_PASSWORD: ${{ secrets.TEST_MYSQL_PASSWORD }}
CYPRESS_MYSQL_HOST_PORT: ${{ secrets.TEST_MYSQL_HOST_PORT }}
CYPRESS_MYSQL_DATABASE_SCHEMA: ${{ secrets.TEST_MYSQL_DATABASE_SCHEMA }}
CYPRESS_POSTGRES_USERNAME: ${{ secrets.TEST_POSTGRES_USERNAME }}
CYPRESS_POSTGRES_PASSWORD: ${{ secrets.TEST_POSTGRES_PASSWORD }}
CYPRESS_POSTGRES_HOST_PORT: ${{ secrets.TEST_POSTGRES_HOST_PORT }}
CYPRESS_POSTGRES_DATABASE: ${{ secrets.TEST_POSTGRES_DATABASE }}

# Recommended: pass the GitHub token lets this action correctly
# determine the unique run id necessary to re-run the checks
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

- name: Clean Up
run: |
cd ./docker/development
docker compose down --remove-orphans
sudo rm -rf ${PWD}/docker-volume
Loading

0 comments on commit 140b435

Please sign in to comment.