-
Notifications
You must be signed in to change notification settings - Fork 8
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Chore: merge 0.13.2.beta tests and docs
- Loading branch information
Showing
980 changed files
with
88,533 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,25 @@ | ||
# Copyright 2021 Collate | ||
# Licensed under the Apache License, Version 2.0 (the "License"); | ||
# you may not use this file except in compliance with the License. | ||
# You may obtain a copy of the License at | ||
# http://www.apache.org/licenses/LICENSE-2.0 | ||
# Unless required by applicable law or agreed to in writing, software | ||
# distributed under the License is distributed on an "AS IS" BASIS, | ||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
# See the License for the specific language governing permissions and | ||
# limitations under the License. | ||
|
||
FROM alpine:3.15 | ||
|
||
EXPOSE 8585 | ||
|
||
RUN apk update \ | ||
&& apk upgrade \ | ||
&& apk add --update wget curl bash openjdk11 \ | ||
&& rm -rf /var/cache/apk/* | ||
COPY docker/metadata/openmetadata-start.sh ./ | ||
COPY openmetadata-dist/target/openmetadata-*.tar.gz / | ||
RUN tar zxvf openmetadata-*.tar.gz && \ | ||
rm openmetadata-*.tar.gz | ||
RUN chmod 777 openmetadata-start.sh | ||
CMD ["./openmetadata-start.sh"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,4 @@ | ||
FROM mysql/mysql-server:latest | ||
WORKDIR /docker-entrypoint-initdb.d | ||
COPY docker/local-metadata/mysql-script.sql . | ||
RUN chmod -R 775 /docker-entrypoint-initdb.d |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,4 @@ | ||
FROM postgres:14 | ||
WORKDIR /docker-entrypoint-initdb.d | ||
COPY docker/local-metadata/postgres-script.sql . | ||
RUN chmod -R 775 /docker-entrypoint-initdb.d |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,173 @@ | ||
# Copyright 2021 Collate | ||
# Licensed under the Apache License, Version 2.0 (the "License"); | ||
# you may not use this file except in compliance with the License. | ||
# You may obtain a copy of the License at | ||
# http://www.apache.org/licenses/LICENSE-2.0 | ||
# Unless required by applicable law or agreed to in writing, software | ||
# distributed under the License is distributed on an "AS IS" BASIS, | ||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
# See the License for the specific language governing permissions and | ||
# limitations under the License. | ||
|
||
version: "3.9" | ||
volumes: | ||
ingestion-volume-dag-airflow: | ||
ingestion-volume-dags: | ||
ingestion-volume-tmp: | ||
es-data: | ||
services: | ||
postgresql: | ||
build: | ||
context: ../../. | ||
dockerfile: docker/local-metadata/Dockerfile_postgres | ||
container_name: openmetadata_postgresql | ||
restart: always | ||
depends_on: | ||
- elasticsearch | ||
environment: | ||
POSTGRES_USER: postgres | ||
POSTGRES_PASSWORD: password | ||
expose: | ||
- 5432 | ||
ports: | ||
- "5432:5432" | ||
volumes: | ||
- ./docker-volume/db-data-postgres:/var/lib/postgresql/data | ||
networks: | ||
- local_app_net | ||
healthcheck: | ||
test: psql -U postgres -tAc 'select 1' -d openmetadata_db | ||
interval: 15s | ||
timeout: 10s | ||
retries: 10 | ||
|
||
elasticsearch: | ||
image: docker.elastic.co/elasticsearch/elasticsearch:7.10.2 | ||
container_name: openmetadata_elasticsearch | ||
environment: | ||
- discovery.type=single-node | ||
- ES_JAVA_OPTS=-Xms1024m -Xmx1024m | ||
networks: | ||
- local_app_net | ||
expose: | ||
- 9200 | ||
- 9300 | ||
ports: | ||
- "9200:9200" | ||
- "9300:9300" | ||
volumes: | ||
- es-data:/usr/share/elasticsearch/data | ||
|
||
openmetadata-server: | ||
build: | ||
context: ../../. | ||
dockerfile: docker/local-metadata/Dockerfile | ||
container_name: openmetadata_server | ||
environment: | ||
# OpenMetadata Server Authentication Configuration | ||
AUTHORIZER_CLASS_NAME: ${AUTHORIZER_CLASS_NAME:-org.openmetadata.service.security.DefaultAuthorizer} | ||
AUTHORIZER_REQUEST_FILTER: ${AUTHORIZER_REQUEST_FILTER:-org.openmetadata.service.security.JwtFilter} | ||
AUTHORIZER_ADMIN_PRINCIPALS: ${AUTHORIZER_ADMIN_PRINCIPALS:-[admin]} | ||
AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN: ${AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN:-["all"]} | ||
AUTHORIZER_INGESTION_PRINCIPALS: ${AUTHORIZER_INGESTION_PRINCIPALS:-[ingestion-bot]} | ||
AUTHORIZER_PRINCIPAL_DOMAIN: ${AUTHORIZER_PRINCIPAL_DOMAIN:-"openmetadata.org"} | ||
AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN: ${AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN:-false} | ||
AUTHORIZER_ENABLE_SECURE_SOCKET: ${AUTHORIZER_ENABLE_SECURE_SOCKET:-false} | ||
AUTHENTICATION_PROVIDER: ${AUTHENTICATION_PROVIDER:-basic} | ||
CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME: ${CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME:-""} | ||
AUTHENTICATION_PUBLIC_KEYS: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/config/jwks]} | ||
AUTHENTICATION_AUTHORITY: ${AUTHENTICATION_AUTHORITY:-https://accounts.google.com} | ||
AUTHENTICATION_CLIENT_ID: ${AUTHENTICATION_CLIENT_ID:-""} | ||
AUTHENTICATION_CALLBACK_URL: ${AUTHENTICATION_CALLBACK_URL:-""} | ||
AUTHENTICATION_JWT_PRINCIPAL_CLAIMS: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS:-[email,preferred_username,sub]} | ||
AUTHENTICATION_ENABLE_SELF_SIGNUP : ${AUTHENTICATION_ENABLE_SELF_SIGNUP:-true} | ||
# JWT Configuration | ||
RSA_PUBLIC_KEY_FILE_PATH: ${RSA_PUBLIC_KEY_FILE_PATH:-"./conf/public_key.der"} | ||
RSA_PRIVATE_KEY_FILE_PATH: ${RSA_PRIVATE_KEY_FILE_PATH:-"./conf/private_key.der"} | ||
JWT_ISSUER: ${JWT_ISSUER:-"open-metadata.org"} | ||
JWT_KEY_ID: ${JWT_KEY_ID:-"Gb389a-9f76-gdjs-a92j-0242bk94356"} | ||
# OpenMetadata Server Airflow Configuration | ||
AIRFLOW_HOST: ${AIRFLOW_HOST:-http://ingestion:8080} | ||
SERVER_HOST_API_URL: ${SERVER_HOST_API_URL:-http://openmetadata-server:8585/api} | ||
# Database configuration for Postgres | ||
DB_DRIVER_CLASS: ${DB_DRIVER_CLASS:-org.postgresql.Driver} | ||
DB_SCHEME: ${DB_SCHEME:-postgresql} | ||
DB_USE_SSL: ${DB_USE_SSL:-false} | ||
DB_USER: ${DB_USER:-openmetadata_user} | ||
DB_USER_PASSWORD: ${DB_USER_PASSWORD:-openmetadata_password} | ||
DB_HOST: ${DB_HOST:-postgresql} | ||
DB_PORT: ${DB_PORT:-5432} | ||
OM_DATABASE: ${OM_DATABASE:-openmetadata_db} | ||
# Airflow SSL Configurations | ||
AIRFLOW_VERIFY_SSL: ${AIRFLOW_VERIFY_SSL:-"no-ssl"} | ||
AIRFLOW_SSL_CERT_PATH: ${AIRFLOW_SSL_CERT_PATH:-""} | ||
# ElasticSearch Configurations | ||
ELASTICSEARCH_HOST: ${ELASTICSEARCH_HOST:- elasticsearch} | ||
ELASTICSEARCH_PORT: ${ELASTICSEARCH_PORT:-9200} | ||
ELASTICSEARCH_SCHEME: ${ELASTICSEARCH_SCHEME:-http} | ||
ELASTICSEARCH_USER: ${ELASTICSEARCH_USER:-""} | ||
ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_PASSWORD:-""} | ||
# Heap OPTS Configurations | ||
OPENMETADATA_HEAP_OPTS: ${OPENMETADATA_HEAP_OPTS:--Xmx1G -Xms1G} | ||
expose: | ||
- 8585 | ||
- 8586 | ||
ports: | ||
- "8585:8585" | ||
- "8586:8586" | ||
depends_on: | ||
elasticsearch: | ||
condition: service_started | ||
postgresql: | ||
condition: service_healthy | ||
networks: | ||
- local_app_net | ||
healthcheck: | ||
test: [ "CMD", "curl", "-f", "http://localhost:8586/healthcheck" ] | ||
|
||
ingestion: | ||
build: | ||
context: ../../. | ||
dockerfile: ingestion/Dockerfile.ci | ||
args: | ||
INGESTION_DEPENDENCY: ${INGESTION_DEPENDENCY:-all} | ||
container_name: openmetadata_ingestion | ||
depends_on: | ||
elasticsearch: | ||
condition: service_started | ||
postgresql: | ||
condition: service_healthy | ||
openmetadata-server: | ||
condition: service_started | ||
environment: | ||
AIRFLOW__API__AUTH_BACKENDS: "airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session" | ||
AIRFLOW__CORE__EXECUTOR: LocalExecutor | ||
AIRFLOW__OPENMETADATA_AIRFLOW_APIS__DAG_GENERATED_CONFIGS: "/opt/airflow/dag_generated_configs" | ||
DB_HOST: ${AIRFLOW_DB_HOST:-postgresql} | ||
DB_PORT: ${AIRFLOW_DB_PORT:-5432} | ||
AIRFLOW_DB: ${AIRFLOW_DB:-airflow_db} | ||
DB_USER: ${AIRFLOW_DB_USER:-airflow_user} | ||
DB_SCHEME: ${AIRFLOW_DB_SCHEME:-postgresql+psycopg2} | ||
DB_PASSWORD: ${AIRFLOW_DB_PASSWORD:-airflow_pass} | ||
entrypoint: /bin/bash | ||
command: | ||
- "/opt/airflow/ingestion_dependency.sh" | ||
expose: | ||
- 8080 | ||
ports: | ||
- "8080:8080" | ||
networks: | ||
- local_app_net | ||
volumes: | ||
- ingestion-volume-dag-airflow:/opt/airflow/dag_generated_configs | ||
- ingestion-volume-dags:/opt/airflow/dags | ||
- ingestion-volume-tmp:/tmp | ||
- /var/run/docker.sock:/var/run/docker.sock:z # Need 600 permissions to run DockerOperator | ||
|
||
networks: | ||
local_app_net: | ||
name: ometa_network | ||
ipam: | ||
driver: default | ||
config: | ||
- subnet: "172.16.239.0/24" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,176 @@ | ||
# Copyright 2021 Collate | ||
# Licensed under the Apache License, Version 2.0 (the "License"); | ||
# you may not use this file except in compliance with the License. | ||
# You may obtain a copy of the License at | ||
# http://www.apache.org/licenses/LICENSE-2.0 | ||
# Unless required by applicable law or agreed to in writing, software | ||
# distributed under the License is distributed on an "AS IS" BASIS, | ||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
# See the License for the specific language governing permissions and | ||
# limitations under the License. | ||
|
||
version: "3.9" | ||
volumes: | ||
ingestion-volume-dag-airflow: | ||
ingestion-volume-dags: | ||
ingestion-volume-tmp: | ||
es-data: | ||
services: | ||
mysql: | ||
build: | ||
context: ../../. | ||
dockerfile: docker/local-metadata/Dockerfile_mysql | ||
container_name: openmetadata_mysql | ||
restart: always | ||
depends_on: | ||
- elasticsearch | ||
environment: | ||
MYSQL_ROOT_PASSWORD: password | ||
expose: | ||
- 3306 | ||
ports: | ||
- "3306:3306" | ||
volumes: | ||
- ./docker-volume/db-data:/var/lib/mysql | ||
networks: | ||
- local_app_net | ||
healthcheck: | ||
test: mysql --user=root --password=$$MYSQL_ROOT_PASSWORD --silent --execute "use openmetadata_db" | ||
interval: 15s | ||
timeout: 10s | ||
retries: 10 | ||
|
||
elasticsearch: | ||
image: docker.elastic.co/elasticsearch/elasticsearch:7.10.2 | ||
container_name: openmetadata_elasticsearch | ||
environment: | ||
- discovery.type=single-node | ||
- ES_JAVA_OPTS=-Xms1024m -Xmx1024m | ||
networks: | ||
- local_app_net | ||
expose: | ||
- 9200 | ||
- 9300 | ||
ports: | ||
- "9200:9200" | ||
- "9300:9300" | ||
volumes: | ||
- es-data:/usr/share/elasticsearch/data | ||
|
||
openmetadata-server: | ||
build: | ||
context: ../../. | ||
dockerfile: docker/local-metadata/Dockerfile | ||
container_name: openmetadata_server | ||
environment: | ||
OPENMETADATA_DEBUG: ${OPENMETADATA_DEBUG:-false} | ||
# OpenMetadata Server Authentication Configuration | ||
AUTHORIZER_CLASS_NAME: ${AUTHORIZER_CLASS_NAME:-org.openmetadata.service.security.DefaultAuthorizer} | ||
AUTHORIZER_REQUEST_FILTER: ${AUTHORIZER_REQUEST_FILTER:-org.openmetadata.service.security.JwtFilter} | ||
AUTHORIZER_ADMIN_PRINCIPALS: ${AUTHORIZER_ADMIN_PRINCIPALS:-[admin]} | ||
AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN: ${AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN:-["all"]} | ||
AUTHORIZER_INGESTION_PRINCIPALS: ${AUTHORIZER_INGESTION_PRINCIPALS:-[ingestion-bot]} | ||
AUTHORIZER_PRINCIPAL_DOMAIN: ${AUTHORIZER_PRINCIPAL_DOMAIN:-"openmetadata.org"} | ||
AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN: ${AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN:-false} | ||
AUTHORIZER_ENABLE_SECURE_SOCKET: ${AUTHORIZER_ENABLE_SECURE_SOCKET:-false} | ||
AUTHENTICATION_PROVIDER: ${AUTHENTICATION_PROVIDER:-basic} | ||
CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME: ${CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME:-""} | ||
AUTHENTICATION_PUBLIC_KEYS: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/config/jwks]} | ||
AUTHENTICATION_AUTHORITY: ${AUTHENTICATION_AUTHORITY:-https://accounts.google.com} | ||
AUTHENTICATION_CLIENT_ID: ${AUTHENTICATION_CLIENT_ID:-""} | ||
AUTHENTICATION_CALLBACK_URL: ${AUTHENTICATION_CALLBACK_URL:-""} | ||
AUTHENTICATION_JWT_PRINCIPAL_CLAIMS: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS:-[email,preferred_username,sub]} | ||
AUTHENTICATION_ENABLE_SELF_SIGNUP : ${AUTHENTICATION_ENABLE_SELF_SIGNUP:-true} | ||
# JWT Configuration | ||
RSA_PUBLIC_KEY_FILE_PATH: ${RSA_PUBLIC_KEY_FILE_PATH:-"./conf/public_key.der"} | ||
RSA_PRIVATE_KEY_FILE_PATH: ${RSA_PRIVATE_KEY_FILE_PATH:-"./conf/private_key.der"} | ||
JWT_ISSUER: ${JWT_ISSUER:-"open-metadata.org"} | ||
JWT_KEY_ID: ${JWT_KEY_ID:-"Gb389a-9f76-gdjs-a92j-0242bk94356"} | ||
# OpenMetadata Server Airflow Configuration | ||
AIRFLOW_HOST: ${AIRFLOW_HOST:-http://ingestion:8080} | ||
SERVER_HOST_API_URL: ${SERVER_HOST_API_URL:-http://openmetadata-server:8585/api} | ||
# Database configuration for MySQL | ||
DB_DRIVER_CLASS: ${DB_DRIVER_CLASS:-com.mysql.cj.jdbc.Driver} | ||
DB_SCHEME: ${DB_SCHEME:-mysql} | ||
DB_USE_SSL: ${DB_USE_SSL:-false} | ||
DB_USER: ${DB_USER:-openmetadata_user} | ||
DB_USER_PASSWORD: ${DB_USER_PASSWORD:-openmetadata_password} | ||
DB_HOST: ${DB_HOST:-mysql} | ||
DB_PORT: ${DB_PORT:-3306} | ||
OM_DATABASE: ${OM_DATABASE:-openmetadata_db} | ||
# Airflow SSL Configurations | ||
AIRFLOW_VERIFY_SSL: ${AIRFLOW_VERIFY_SSL:-"no-ssl"} | ||
AIRFLOW_SSL_CERT_PATH: ${AIRFLOW_SSL_CERT_PATH:-""} | ||
# ElasticSearch Configurations | ||
ELASTICSEARCH_HOST: ${ELASTICSEARCH_HOST:- elasticsearch} | ||
ELASTICSEARCH_PORT: ${ELASTICSEARCH_PORT:-9200} | ||
ELASTICSEARCH_SCHEME: ${ELASTICSEARCH_SCHEME:-http} | ||
ELASTICSEARCH_USER: ${ELASTICSEARCH_USER:-""} | ||
ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_PASSWORD:-""} | ||
# Heap OPTS Configurations | ||
OPENMETADATA_HEAP_OPTS: ${OPENMETADATA_HEAP_OPTS:--Xmx1G -Xms1G} | ||
expose: | ||
- 8585 | ||
- 8586 | ||
- 5005 | ||
ports: | ||
- "8585:8585" | ||
- "8586:8586" | ||
- "5005:5005" | ||
depends_on: | ||
elasticsearch: | ||
condition: service_started | ||
mysql: | ||
condition: service_healthy | ||
networks: | ||
- local_app_net | ||
healthcheck: | ||
test: [ "CMD", "curl", "-f", "http://localhost:8586/healthcheck" ] | ||
|
||
ingestion: | ||
build: | ||
context: ../../. | ||
dockerfile: ingestion/Dockerfile.ci | ||
args: | ||
INGESTION_DEPENDENCY: ${INGESTION_DEPENDENCY:-all} | ||
container_name: openmetadata_ingestion | ||
environment: | ||
AIRFLOW__API__AUTH_BACKENDS: "airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session" | ||
AIRFLOW__CORE__EXECUTOR: LocalExecutor | ||
AIRFLOW__OPENMETADATA_AIRFLOW_APIS__DAG_GENERATED_CONFIGS: "/opt/airflow/dag_generated_configs" | ||
DB_HOST: ${AIRFLOW_DB_HOST:-mysql} | ||
DB_PORT: ${AIRFLOW_DB_PORT:-3306} | ||
AIRFLOW_DB: ${AIRFLOW_DB:-airflow_db} | ||
AIRFLOW_DB_SCHEME: ${AIRFLOW_DB_SCHEME:-mysql+pymysql} | ||
DB_USER: ${AIRFLOW_DB_USER:-airflow_user} | ||
DB_PASSWORD: ${AIRFLOW_DB_PASSWORD:-airflow_pass} | ||
entrypoint: /bin/bash | ||
command: | ||
- "/opt/airflow/ingestion_dependency.sh" | ||
depends_on: | ||
elasticsearch: | ||
condition: service_started | ||
mysql: | ||
condition: service_healthy | ||
openmetadata-server: | ||
condition: service_started | ||
expose: | ||
- 8080 | ||
ports: | ||
- "8080:8080" | ||
networks: | ||
- local_app_net | ||
volumes: | ||
- ingestion-volume-dag-airflow:/opt/airflow/dag_generated_configs | ||
- ingestion-volume-dags:/opt/airflow/dags | ||
- ingestion-volume-tmp:/tmp | ||
- /var/run/docker.sock:/var/run/docker.sock:z # Need 600 permissions to run DockerOperator | ||
|
||
|
||
networks: | ||
local_app_net: | ||
name: ometa_network | ||
ipam: | ||
driver: default | ||
config: | ||
- subnet: "172.16.239.0/24" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,7 @@ | ||
CREATE DATABASE openmetadata_db; | ||
CREATE DATABASE airflow_db CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; | ||
CREATE USER 'openmetadata_user'@'%' IDENTIFIED BY 'openmetadata_password'; | ||
CREATE USER 'airflow_user'@'%' IDENTIFIED BY 'airflow_pass'; | ||
GRANT ALL PRIVILEGES ON openmetadata_db.* TO 'openmetadata_user'@'%' WITH GRANT OPTION; | ||
GRANT ALL PRIVILEGES ON airflow_db.* TO 'airflow_user'@'%' WITH GRANT OPTION; | ||
commit; |
Oops, something went wrong.