Skip to content

Commit

Permalink
Chore: merge 0.13.2.beta tests and docs
Browse files Browse the repository at this point in the history
  • Loading branch information
ciusji committed Mar 30, 2023
1 parent 1186e7f commit 71c668e
Show file tree
Hide file tree
Showing 980 changed files with 88,533 additions and 0 deletions.
25 changes: 25 additions & 0 deletions docker/local-metadata/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
# Copyright 2021 Collate
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

FROM alpine:3.15

EXPOSE 8585

RUN apk update \
&& apk upgrade \
&& apk add --update wget curl bash openjdk11 \
&& rm -rf /var/cache/apk/*
COPY docker/metadata/openmetadata-start.sh ./
COPY openmetadata-dist/target/openmetadata-*.tar.gz /
RUN tar zxvf openmetadata-*.tar.gz && \
rm openmetadata-*.tar.gz
RUN chmod 777 openmetadata-start.sh
CMD ["./openmetadata-start.sh"]
4 changes: 4 additions & 0 deletions docker/local-metadata/Dockerfile_mysql
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
FROM mysql/mysql-server:latest
WORKDIR /docker-entrypoint-initdb.d
COPY docker/local-metadata/mysql-script.sql .
RUN chmod -R 775 /docker-entrypoint-initdb.d
4 changes: 4 additions & 0 deletions docker/local-metadata/Dockerfile_postgres
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
FROM postgres:14
WORKDIR /docker-entrypoint-initdb.d
COPY docker/local-metadata/postgres-script.sql .
RUN chmod -R 775 /docker-entrypoint-initdb.d
173 changes: 173 additions & 0 deletions docker/local-metadata/docker-compose-postgres.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,173 @@
# Copyright 2021 Collate
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

version: "3.9"
volumes:
ingestion-volume-dag-airflow:
ingestion-volume-dags:
ingestion-volume-tmp:
es-data:
services:
postgresql:
build:
context: ../../.
dockerfile: docker/local-metadata/Dockerfile_postgres
container_name: openmetadata_postgresql
restart: always
depends_on:
- elasticsearch
environment:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: password
expose:
- 5432
ports:
- "5432:5432"
volumes:
- ./docker-volume/db-data-postgres:/var/lib/postgresql/data
networks:
- local_app_net
healthcheck:
test: psql -U postgres -tAc 'select 1' -d openmetadata_db
interval: 15s
timeout: 10s
retries: 10

elasticsearch:
image: docker.elastic.co/elasticsearch/elasticsearch:7.10.2
container_name: openmetadata_elasticsearch
environment:
- discovery.type=single-node
- ES_JAVA_OPTS=-Xms1024m -Xmx1024m
networks:
- local_app_net
expose:
- 9200
- 9300
ports:
- "9200:9200"
- "9300:9300"
volumes:
- es-data:/usr/share/elasticsearch/data

openmetadata-server:
build:
context: ../../.
dockerfile: docker/local-metadata/Dockerfile
container_name: openmetadata_server
environment:
# OpenMetadata Server Authentication Configuration
AUTHORIZER_CLASS_NAME: ${AUTHORIZER_CLASS_NAME:-org.openmetadata.service.security.DefaultAuthorizer}
AUTHORIZER_REQUEST_FILTER: ${AUTHORIZER_REQUEST_FILTER:-org.openmetadata.service.security.JwtFilter}
AUTHORIZER_ADMIN_PRINCIPALS: ${AUTHORIZER_ADMIN_PRINCIPALS:-[admin]}
AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN: ${AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN:-["all"]}
AUTHORIZER_INGESTION_PRINCIPALS: ${AUTHORIZER_INGESTION_PRINCIPALS:-[ingestion-bot]}
AUTHORIZER_PRINCIPAL_DOMAIN: ${AUTHORIZER_PRINCIPAL_DOMAIN:-"openmetadata.org"}
AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN: ${AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN:-false}
AUTHORIZER_ENABLE_SECURE_SOCKET: ${AUTHORIZER_ENABLE_SECURE_SOCKET:-false}
AUTHENTICATION_PROVIDER: ${AUTHENTICATION_PROVIDER:-basic}
CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME: ${CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME:-""}
AUTHENTICATION_PUBLIC_KEYS: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/config/jwks]}
AUTHENTICATION_AUTHORITY: ${AUTHENTICATION_AUTHORITY:-https://accounts.google.com}
AUTHENTICATION_CLIENT_ID: ${AUTHENTICATION_CLIENT_ID:-""}
AUTHENTICATION_CALLBACK_URL: ${AUTHENTICATION_CALLBACK_URL:-""}
AUTHENTICATION_JWT_PRINCIPAL_CLAIMS: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS:-[email,preferred_username,sub]}
AUTHENTICATION_ENABLE_SELF_SIGNUP : ${AUTHENTICATION_ENABLE_SELF_SIGNUP:-true}
# JWT Configuration
RSA_PUBLIC_KEY_FILE_PATH: ${RSA_PUBLIC_KEY_FILE_PATH:-"./conf/public_key.der"}
RSA_PRIVATE_KEY_FILE_PATH: ${RSA_PRIVATE_KEY_FILE_PATH:-"./conf/private_key.der"}
JWT_ISSUER: ${JWT_ISSUER:-"open-metadata.org"}
JWT_KEY_ID: ${JWT_KEY_ID:-"Gb389a-9f76-gdjs-a92j-0242bk94356"}
# OpenMetadata Server Airflow Configuration
AIRFLOW_HOST: ${AIRFLOW_HOST:-http://ingestion:8080}
SERVER_HOST_API_URL: ${SERVER_HOST_API_URL:-http://openmetadata-server:8585/api}
# Database configuration for Postgres
DB_DRIVER_CLASS: ${DB_DRIVER_CLASS:-org.postgresql.Driver}
DB_SCHEME: ${DB_SCHEME:-postgresql}
DB_USE_SSL: ${DB_USE_SSL:-false}
DB_USER: ${DB_USER:-openmetadata_user}
DB_USER_PASSWORD: ${DB_USER_PASSWORD:-openmetadata_password}
DB_HOST: ${DB_HOST:-postgresql}
DB_PORT: ${DB_PORT:-5432}
OM_DATABASE: ${OM_DATABASE:-openmetadata_db}
# Airflow SSL Configurations
AIRFLOW_VERIFY_SSL: ${AIRFLOW_VERIFY_SSL:-"no-ssl"}
AIRFLOW_SSL_CERT_PATH: ${AIRFLOW_SSL_CERT_PATH:-""}
# ElasticSearch Configurations
ELASTICSEARCH_HOST: ${ELASTICSEARCH_HOST:- elasticsearch}
ELASTICSEARCH_PORT: ${ELASTICSEARCH_PORT:-9200}
ELASTICSEARCH_SCHEME: ${ELASTICSEARCH_SCHEME:-http}
ELASTICSEARCH_USER: ${ELASTICSEARCH_USER:-""}
ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_PASSWORD:-""}
# Heap OPTS Configurations
OPENMETADATA_HEAP_OPTS: ${OPENMETADATA_HEAP_OPTS:--Xmx1G -Xms1G}
expose:
- 8585
- 8586
ports:
- "8585:8585"
- "8586:8586"
depends_on:
elasticsearch:
condition: service_started
postgresql:
condition: service_healthy
networks:
- local_app_net
healthcheck:
test: [ "CMD", "curl", "-f", "http://localhost:8586/healthcheck" ]

ingestion:
build:
context: ../../.
dockerfile: ingestion/Dockerfile.ci
args:
INGESTION_DEPENDENCY: ${INGESTION_DEPENDENCY:-all}
container_name: openmetadata_ingestion
depends_on:
elasticsearch:
condition: service_started
postgresql:
condition: service_healthy
openmetadata-server:
condition: service_started
environment:
AIRFLOW__API__AUTH_BACKENDS: "airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session"
AIRFLOW__CORE__EXECUTOR: LocalExecutor
AIRFLOW__OPENMETADATA_AIRFLOW_APIS__DAG_GENERATED_CONFIGS: "/opt/airflow/dag_generated_configs"
DB_HOST: ${AIRFLOW_DB_HOST:-postgresql}
DB_PORT: ${AIRFLOW_DB_PORT:-5432}
AIRFLOW_DB: ${AIRFLOW_DB:-airflow_db}
DB_USER: ${AIRFLOW_DB_USER:-airflow_user}
DB_SCHEME: ${AIRFLOW_DB_SCHEME:-postgresql+psycopg2}
DB_PASSWORD: ${AIRFLOW_DB_PASSWORD:-airflow_pass}
entrypoint: /bin/bash
command:
- "/opt/airflow/ingestion_dependency.sh"
expose:
- 8080
ports:
- "8080:8080"
networks:
- local_app_net
volumes:
- ingestion-volume-dag-airflow:/opt/airflow/dag_generated_configs
- ingestion-volume-dags:/opt/airflow/dags
- ingestion-volume-tmp:/tmp
- /var/run/docker.sock:/var/run/docker.sock:z # Need 600 permissions to run DockerOperator

networks:
local_app_net:
name: ometa_network
ipam:
driver: default
config:
- subnet: "172.16.239.0/24"
176 changes: 176 additions & 0 deletions docker/local-metadata/docker-compose.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,176 @@
# Copyright 2021 Collate
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

version: "3.9"
volumes:
ingestion-volume-dag-airflow:
ingestion-volume-dags:
ingestion-volume-tmp:
es-data:
services:
mysql:
build:
context: ../../.
dockerfile: docker/local-metadata/Dockerfile_mysql
container_name: openmetadata_mysql
restart: always
depends_on:
- elasticsearch
environment:
MYSQL_ROOT_PASSWORD: password
expose:
- 3306
ports:
- "3306:3306"
volumes:
- ./docker-volume/db-data:/var/lib/mysql
networks:
- local_app_net
healthcheck:
test: mysql --user=root --password=$$MYSQL_ROOT_PASSWORD --silent --execute "use openmetadata_db"
interval: 15s
timeout: 10s
retries: 10

elasticsearch:
image: docker.elastic.co/elasticsearch/elasticsearch:7.10.2
container_name: openmetadata_elasticsearch
environment:
- discovery.type=single-node
- ES_JAVA_OPTS=-Xms1024m -Xmx1024m
networks:
- local_app_net
expose:
- 9200
- 9300
ports:
- "9200:9200"
- "9300:9300"
volumes:
- es-data:/usr/share/elasticsearch/data

openmetadata-server:
build:
context: ../../.
dockerfile: docker/local-metadata/Dockerfile
container_name: openmetadata_server
environment:
OPENMETADATA_DEBUG: ${OPENMETADATA_DEBUG:-false}
# OpenMetadata Server Authentication Configuration
AUTHORIZER_CLASS_NAME: ${AUTHORIZER_CLASS_NAME:-org.openmetadata.service.security.DefaultAuthorizer}
AUTHORIZER_REQUEST_FILTER: ${AUTHORIZER_REQUEST_FILTER:-org.openmetadata.service.security.JwtFilter}
AUTHORIZER_ADMIN_PRINCIPALS: ${AUTHORIZER_ADMIN_PRINCIPALS:-[admin]}
AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN: ${AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN:-["all"]}
AUTHORIZER_INGESTION_PRINCIPALS: ${AUTHORIZER_INGESTION_PRINCIPALS:-[ingestion-bot]}
AUTHORIZER_PRINCIPAL_DOMAIN: ${AUTHORIZER_PRINCIPAL_DOMAIN:-"openmetadata.org"}
AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN: ${AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN:-false}
AUTHORIZER_ENABLE_SECURE_SOCKET: ${AUTHORIZER_ENABLE_SECURE_SOCKET:-false}
AUTHENTICATION_PROVIDER: ${AUTHENTICATION_PROVIDER:-basic}
CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME: ${CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME:-""}
AUTHENTICATION_PUBLIC_KEYS: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/config/jwks]}
AUTHENTICATION_AUTHORITY: ${AUTHENTICATION_AUTHORITY:-https://accounts.google.com}
AUTHENTICATION_CLIENT_ID: ${AUTHENTICATION_CLIENT_ID:-""}
AUTHENTICATION_CALLBACK_URL: ${AUTHENTICATION_CALLBACK_URL:-""}
AUTHENTICATION_JWT_PRINCIPAL_CLAIMS: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS:-[email,preferred_username,sub]}
AUTHENTICATION_ENABLE_SELF_SIGNUP : ${AUTHENTICATION_ENABLE_SELF_SIGNUP:-true}
# JWT Configuration
RSA_PUBLIC_KEY_FILE_PATH: ${RSA_PUBLIC_KEY_FILE_PATH:-"./conf/public_key.der"}
RSA_PRIVATE_KEY_FILE_PATH: ${RSA_PRIVATE_KEY_FILE_PATH:-"./conf/private_key.der"}
JWT_ISSUER: ${JWT_ISSUER:-"open-metadata.org"}
JWT_KEY_ID: ${JWT_KEY_ID:-"Gb389a-9f76-gdjs-a92j-0242bk94356"}
# OpenMetadata Server Airflow Configuration
AIRFLOW_HOST: ${AIRFLOW_HOST:-http://ingestion:8080}
SERVER_HOST_API_URL: ${SERVER_HOST_API_URL:-http://openmetadata-server:8585/api}
# Database configuration for MySQL
DB_DRIVER_CLASS: ${DB_DRIVER_CLASS:-com.mysql.cj.jdbc.Driver}
DB_SCHEME: ${DB_SCHEME:-mysql}
DB_USE_SSL: ${DB_USE_SSL:-false}
DB_USER: ${DB_USER:-openmetadata_user}
DB_USER_PASSWORD: ${DB_USER_PASSWORD:-openmetadata_password}
DB_HOST: ${DB_HOST:-mysql}
DB_PORT: ${DB_PORT:-3306}
OM_DATABASE: ${OM_DATABASE:-openmetadata_db}
# Airflow SSL Configurations
AIRFLOW_VERIFY_SSL: ${AIRFLOW_VERIFY_SSL:-"no-ssl"}
AIRFLOW_SSL_CERT_PATH: ${AIRFLOW_SSL_CERT_PATH:-""}
# ElasticSearch Configurations
ELASTICSEARCH_HOST: ${ELASTICSEARCH_HOST:- elasticsearch}
ELASTICSEARCH_PORT: ${ELASTICSEARCH_PORT:-9200}
ELASTICSEARCH_SCHEME: ${ELASTICSEARCH_SCHEME:-http}
ELASTICSEARCH_USER: ${ELASTICSEARCH_USER:-""}
ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_PASSWORD:-""}
# Heap OPTS Configurations
OPENMETADATA_HEAP_OPTS: ${OPENMETADATA_HEAP_OPTS:--Xmx1G -Xms1G}
expose:
- 8585
- 8586
- 5005
ports:
- "8585:8585"
- "8586:8586"
- "5005:5005"
depends_on:
elasticsearch:
condition: service_started
mysql:
condition: service_healthy
networks:
- local_app_net
healthcheck:
test: [ "CMD", "curl", "-f", "http://localhost:8586/healthcheck" ]

ingestion:
build:
context: ../../.
dockerfile: ingestion/Dockerfile.ci
args:
INGESTION_DEPENDENCY: ${INGESTION_DEPENDENCY:-all}
container_name: openmetadata_ingestion
environment:
AIRFLOW__API__AUTH_BACKENDS: "airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session"
AIRFLOW__CORE__EXECUTOR: LocalExecutor
AIRFLOW__OPENMETADATA_AIRFLOW_APIS__DAG_GENERATED_CONFIGS: "/opt/airflow/dag_generated_configs"
DB_HOST: ${AIRFLOW_DB_HOST:-mysql}
DB_PORT: ${AIRFLOW_DB_PORT:-3306}
AIRFLOW_DB: ${AIRFLOW_DB:-airflow_db}
AIRFLOW_DB_SCHEME: ${AIRFLOW_DB_SCHEME:-mysql+pymysql}
DB_USER: ${AIRFLOW_DB_USER:-airflow_user}
DB_PASSWORD: ${AIRFLOW_DB_PASSWORD:-airflow_pass}
entrypoint: /bin/bash
command:
- "/opt/airflow/ingestion_dependency.sh"
depends_on:
elasticsearch:
condition: service_started
mysql:
condition: service_healthy
openmetadata-server:
condition: service_started
expose:
- 8080
ports:
- "8080:8080"
networks:
- local_app_net
volumes:
- ingestion-volume-dag-airflow:/opt/airflow/dag_generated_configs
- ingestion-volume-dags:/opt/airflow/dags
- ingestion-volume-tmp:/tmp
- /var/run/docker.sock:/var/run/docker.sock:z # Need 600 permissions to run DockerOperator


networks:
local_app_net:
name: ometa_network
ipam:
driver: default
config:
- subnet: "172.16.239.0/24"
7 changes: 7 additions & 0 deletions docker/local-metadata/mysql-script.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
CREATE DATABASE openmetadata_db;
CREATE DATABASE airflow_db CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
CREATE USER 'openmetadata_user'@'%' IDENTIFIED BY 'openmetadata_password';
CREATE USER 'airflow_user'@'%' IDENTIFIED BY 'airflow_pass';
GRANT ALL PRIVILEGES ON openmetadata_db.* TO 'openmetadata_user'@'%' WITH GRANT OPTION;
GRANT ALL PRIVILEGES ON airflow_db.* TO 'airflow_user'@'%' WITH GRANT OPTION;
commit;
Loading

0 comments on commit 71c668e

Please sign in to comment.