From 9bb09671f7ceb0c41ab4047476c912d543cbb6ef Mon Sep 17 00:00:00 2001 From: Bot-targa Date: Mon, 23 Sep 2024 09:00:52 +0200 Subject: [PATCH 01/11] chore(docs): update Data Catalog documentation --- .../data-catalog/20_configuration.md | 97 ++--- .../data-catalog/changelog.md | 30 ++ ...el.schema.json => agent.model.schema.json} | 0 .../data-catalog/catalog.model.schema.json | 367 ++++++++++++++++++ 4 files changed, 440 insertions(+), 54 deletions(-) rename static/docs_files_to_download/data-catalog/{model.schema.json => agent.model.schema.json} (100%) create mode 100644 static/docs_files_to_download/data-catalog/catalog.model.schema.json diff --git a/docs/runtime_suite_templates/data-catalog/20_configuration.md b/docs/runtime_suite_templates/data-catalog/20_configuration.md index 8e59639c09..0157aff189 100644 --- a/docs/runtime_suite_templates/data-catalog/20_configuration.md +++ b/docs/runtime_suite_templates/data-catalog/20_configuration.md @@ -186,49 +186,6 @@ The configuration has the following main sections: } ``` -### Secret support - -In k8s environments secrets can be injected in a running workload as an environment variable, -as a standalone file or a INI key in a standalone file. Such secrets may be base64 encoded. - -_Data Catalog Agent_ configuration supports referencing such secrets inline in selected fields of the -JSON configuration file. When the field supports secrets you may write a plain string or objects. - -In case of a string the secret is considered `plain` and written in the config file. -In case of an object with `env` guard like: - -```json -{ - "type": "env", - "key": "MY_SECRET_ENV_VAR" -} -``` - -the agent will use the content of the env var `MY_SECRET_ENV_VAR`. An extra `encoding` field equal -to `base64` can be used to specify a pre-read decoded to use. - -In case of an object with `file` guard like: - -```json -{ - "type": "file", - "path": "/path/to/secret" -} -``` - -it will use the content of the file on such `path`. If the file is formatted as an `ini` file a `key` may -be specified - -```json -{ - "type": "file", - "path": "/path/to/secret", - "key": "CONNECTION_STRING" -} -``` - -An extra `encoding` field equal to `base64` can be used to specify a pre-read decoded to use. - Secretable fields are marked in the following sections. ## Connections @@ -305,7 +262,7 @@ Other keys are `host` and `port` which for a **PostgreSQL** connection are defau #### Secretable fields -`uid`, `pwd` or `params` support secrets +`uid`, `pwd` or `params` support [secrets resolution](/fast_data/configuration/secrets_resolution.md). ### Oracle @@ -461,7 +418,7 @@ Also the environment variable must be set: #### Secretable fields -`uid`, `pwd` or `params` support secrets +`uid`, `pwd` or `params` support [secrets resolution](/fast_data/configuration/secrets_resolution.md). ### MS SQL server @@ -505,7 +462,7 @@ Other keys are `host` and `port` which for a **PostgreSQL** connection are defau #### Secretable fields -`uid`, `pwd` or `params` support secrets +`uid`, `pwd` or `params` support [secrets resolution](/fast_data/configuration/secrets_resolution.md). ### MySQL @@ -549,7 +506,7 @@ Other keys are `host` and `port` which for a **PostgreSQL** connection are defau #### Secretable fields -`uid`, `pwd` or `params` support secrets +`uid`, `pwd` or `params` support [secrets resolution](/fast_data/configuration/secrets_resolution.md). ### Mia CRUD Service @@ -714,22 +671,27 @@ Now you should have everything you need to fill out the configuration parameters #### Secretable fields -`clientId`, `username`, `clientSecret`, `password`, `securityToken` or `privateKey` support secrets +`clientId`, `username`, `clientSecret`, `password`, `securityToken` or `privateKey` support [secrets resolution](/fast_data/configuration/secrets_resolution.md). ## Targets There are 4 targets available: -1. [**default**] stdout -2. file -3. mia-console +1. [**default**] `stdout` +2. `mongodb` +2. `file` +3. `mia-console` For each listed connection, after metadata is retrieved, `agent` **sequentially** sends data to the target as: -- `json` for `stdout` and `file` -- [`ndjson`](https://github.com/ndjson/ndjson-spec) for `mia-console` +- `json` for `stdout` and `file`; +- [`ndjson`](https://github.com/ndjson/ndjson-spec) for `mia-console`. +- [`BSON`](https://bsonspec.org/) for `mongodb` -The final content is an `array` of models. Model spec is given in the form of a JSON schema. +The final content is an `array` of models, where the format of its records changes accordingly to the target: + +- `stdout`, `file` and `mia-console`: the models are written in the native agent format, which is defined in the following JSON schema; +- `mongodb`: the models are written in a format that is supported by the [Data Catalog](/data_catalog/overview.mdx) application, as defined in the following JSON schema; ### Standard Output @@ -744,6 +706,29 @@ To explicitly configure the `stdout` target use: } ``` +### MongoDB + +The MongoDB target enables Data Catalog Agent to feed data from external sources to the [Data Catalog](/data_catalog/overview.mdx) application. + +To configure the `mongodb` target use: + +```js +{ + // ... + "target": { + "type": "mongodb", + "url": "mongodb://test:27017/?replicaSet=rs", // 👈 mongodb connection string: the database must be a replica set + "database": "test_database", // 👈 if defined, it will be used as default database to store the models + } +} +``` + +The target will write the content of the connections to a MongoDB replica set database, in a collection named `open-lineage-datasets`. + +:::tip +To enforce document validation on that collection, be sure to run [Data Catalog Configuration Scripts](/data_catalog/database_setup.mdx) before executing the agent. +::: + ### File To configure the `file` target use: @@ -771,6 +756,10 @@ which will save output files in the folder `./output`. To override this use: ### MIA Console +:::caution +This target has been **deprecated** in favour of [`mongodb`](#mongodb) to support [Data Catalog](/data_catalog/overview.mdx) solution. +::: + To configure the `mia-console` target use: ```json diff --git a/docs/runtime_suite_templates/data-catalog/changelog.md b/docs/runtime_suite_templates/data-catalog/changelog.md index 6a83a01975..5d0cc0fea5 100644 --- a/docs/runtime_suite_templates/data-catalog/changelog.md +++ b/docs/runtime_suite_templates/data-catalog/changelog.md @@ -15,6 +15,36 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.3.2] - 2024-09-20 + +## [1.3.2] - 2024-09-20 + +### Added + +#### Targets + +- `mongodb` target. Models will be stored on a dedicated collection with the following target configuration: + ```json + { + // ... + "target": { + "type": "mongodb", + "url": "mongodb://test:27017/?replicaSet=rs", // 👈 mongodb connection string: the database must be a replica set + "database": "test_database", // 👈 if defined, it will be used as default database to store the models + } + } + ``` + + The record will be stored in a collection named `open-lineage-datasets`. + + > **NOTE:** + > + > To use MongoDB as a target, the database must be configured as a replica set. + +### Updated + +- _Data Catalog Agent_ bumped to version `0.6.4` + ## [1.3.1] - 2024-07-31 ### Updated diff --git a/static/docs_files_to_download/data-catalog/model.schema.json b/static/docs_files_to_download/data-catalog/agent.model.schema.json similarity index 100% rename from static/docs_files_to_download/data-catalog/model.schema.json rename to static/docs_files_to_download/data-catalog/agent.model.schema.json diff --git a/static/docs_files_to_download/data-catalog/catalog.model.schema.json b/static/docs_files_to_download/data-catalog/catalog.model.schema.json new file mode 100644 index 0000000000..8576fbdaa6 --- /dev/null +++ b/static/docs_files_to_download/data-catalog/catalog.model.schema.json @@ -0,0 +1,367 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "OpenLineage", + "examples": [ + { + "producer": "my_producer", + "schemaURL": "https://openlineage.io/spec/2-0-2/OpenLineage.json", + "eventTime": "2024-06-27 10:43:30.889698872 UTC", + "dataset": { + "datasetId": "mock-id", + "name": "books", + "namespace": "test", + "schema": "my-schema", + "catalogName": "test", + "facets": { + "schema": { + "_producer": "my_producer", + "_schemaURL": "customSchemaDatasetFacet.json", + "jsonSchema": { + "type": "object", + "additionalProperties": false, + "properties": { + "FIRST_NAME": { + "type": "string" + }, + "ID": { + "type": "number" + }, + "INSERTED_ON": {}, + "LAST_NAME": { + "type": "string" + } + } + }, + "fields": [ + { + "name": "FIRST_NAME", + "type": "VARCHAR2", + "required": false, + "unique": false, + "primaryKey": false, + "nullable": false, + "sourceDescription": "size=255, precision=0" + }, + { + "name": "ID", + "type": "NUMBER", + "required": true, + "unique": true, + "primaryKey": true, + "nullable": false, + "sourceDescription": "size=0, precision=0" + }, + { + "name": "INSERTED_ON", + "type": "TIMESTAMP(6)", + "required": true, + "unique": true, + "primaryKey": true, + "nullable": false, + "sourceDescription": "size=0, precision=0" + }, + { + "name": "LAST_NAME", + "type": "VARCHAR2", + "required": false, + "unique": false, + "primaryKey": false, + "nullable": false, + "sourceDescription": "size=255, precision=0" + } + ] + }, + "version": { + "_producer": "my_producer", + "_schemaURL": "https://openlineage.io/spec/facets/1-0-1/DatasetVersionDatasetFacet.json", + "datasetVersion": "1" + }, + "storage": { + "_producer": "my_producer", + "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/StorageDatasetFacet.json", + "storageLayer": "oracle19c_test", + "version": "Oracle Database 19c Enterprise Edition Release 19.0.0.0.0 - Production", + "vendor": "mysql" + } + } + } + } + ], + "type": "object", + "required": [ + "dataset", + "eventTime", + "producer" + ], + "properties": { + "dataset": { + "$ref": "#/definitions/Dataset" + }, + "eventTime": { + "type": "string" + }, + "producer": { + "type": "string" + }, + "schemaURL": { + "default": "https://openlineage.io/spec/2-0-2/OpenLineage.json", + "type": "string" + } + }, + "definitions": { + "Dataset": { + "type": "object", + "required": [ + "datasetId", + "facets", + "name", + "namespace" + ], + "properties": { + "catalogName": { + "type": [ + "string", + "null" + ] + }, + "datasetId": { + "type": "string" + }, + "facets": { + "$ref": "#/definitions/Facets" + }, + "name": { + "type": "string" + }, + "namespace": { + "type": "string" + }, + "schema": { + "type": [ + "string", + "null" + ] + } + } + }, + "FacetSchema": { + "type": "object", + "required": [ + "_producer", + "_schemaURL", + "fields" + ], + "properties": { + "_producer": { + "type": "string" + }, + "_schemaURL": { + "type": "string" + }, + "fields": { + "type": "array", + "items": { + "$ref": "#/definitions/Field" + } + }, + "jsonSchema": { + "default": { + "type": "object", + "additionalProperties": false, + "properties": {} + }, + "allOf": [ + { + "$ref": "#/definitions/json-schema" + } + ] + } + } + }, + "FacetStorage": { + "type": "object", + "required": [ + "_producer", + "_schemaURL", + "storageLayer", + "vendor", + "version" + ], + "properties": { + "_producer": { + "type": "string" + }, + "_schemaURL": { + "type": [ + "string", + "null" + ] + }, + "storageLayer": { + "type": "string" + }, + "vendor": { + "type": "string" + }, + "version": { + "type": "string" + } + } + }, + "FacetVersion": { + "type": "object", + "required": [ + "_producer", + "_schemaURL", + "datasetVersion" + ], + "properties": { + "_producer": { + "type": "string" + }, + "_schemaURL": { + "type": "string" + }, + "datasetVersion": { + "type": "string" + } + } + }, + "Facets": { + "type": "object", + "required": [ + "schema", + "storage", + "version" + ], + "properties": { + "schema": { + "$ref": "#/definitions/FacetSchema" + }, + "storage": { + "$ref": "#/definitions/FacetStorage" + }, + "version": { + "$ref": "#/definitions/FacetVersion" + } + } + }, + "Field": { + "type": "object", + "required": [ + "name", + "nullable", + "primaryKey", + "required", + "sourceDescription", + "type", + "unique" + ], + "properties": { + "default": { + "type": [ + "string", + "null" + ] + }, + "name": { + "type": "string" + }, + "nullable": { + "type": "boolean" + }, + "primaryKey": { + "type": "boolean" + }, + "required": { + "type": "boolean" + }, + "sourceDescription": { + "description": "field description extracted from the source system", + "type": "string" + }, + "type": { + "type": "string" + }, + "unique": { + "type": "boolean" + } + } + }, + "json-primitive-type": { + "type": "string", + "enum": [ + "null", + "boolean", + "number", + "integer", + "string", + "array", + "object" + ] + }, + "json-property": { + "type": "object", + "properties": { + "default": true, + "nullable": { + "writeOnly": true, + "type": [ + "boolean", + "null" + ] + }, + "type": { + "anyOf": [ + { + "$ref": "#/definitions/json-type" + }, + { + "type": "null" + } + ] + } + } + }, + "json-schema": { + "type": "object", + "required": [ + "properties", + "type" + ], + "properties": { + "additionalProperties": { + "type": "boolean" + }, + "properties": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/json-property" + } + }, + "required": { + "type": "array", + "items": { + "type": "string" + } + }, + "type": { + "$ref": "#/definitions/json-primitive-type" + } + } + }, + "json-type": { + "anyOf": [ + { + "$ref": "#/definitions/json-primitive-type" + }, + { + "type": "array", + "items": { + "$ref": "#/definitions/json-primitive-type" + } + } + ] + } + } +} \ No newline at end of file From bc7757830d1e76201275efeb5e457564fe98cef1 Mon Sep 17 00:00:00 2001 From: Bot-targa Date: Tue, 24 Sep 2024 09:00:51 +0200 Subject: [PATCH 02/11] chore(docs): update Data Catalog documentation From 06a1f879e9a2fc0fa2126e3780685e3644e9b7f6 Mon Sep 17 00:00:00 2001 From: Bot-targa Date: Wed, 25 Sep 2024 09:00:54 +0200 Subject: [PATCH 03/11] chore(docs): update Data Catalog documentation From 7043d73d8238ce87c55d587d622f68f133486c26 Mon Sep 17 00:00:00 2001 From: Bot-targa Date: Thu, 26 Sep 2024 09:00:55 +0200 Subject: [PATCH 04/11] chore(docs): update Data Catalog documentation From dd3538ab0d7ab89bb3ef3c811344d3ccf2a5cebd Mon Sep 17 00:00:00 2001 From: Bot-targa Date: Fri, 27 Sep 2024 09:00:55 +0200 Subject: [PATCH 05/11] chore(docs): update Data Catalog documentation From 02de2da0f010a7b3da44cf6e2af06cd3f5eee725 Mon Sep 17 00:00:00 2001 From: Bot-targa Date: Fri, 27 Sep 2024 17:10:54 +0200 Subject: [PATCH 06/11] chore(docs): update Data Catalog documentation --- docs/runtime_suite_templates/data-catalog/changelog.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/runtime_suite_templates/data-catalog/changelog.md b/docs/runtime_suite_templates/data-catalog/changelog.md index 5d0cc0fea5..68387bce7f 100644 --- a/docs/runtime_suite_templates/data-catalog/changelog.md +++ b/docs/runtime_suite_templates/data-catalog/changelog.md @@ -17,8 +17,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [1.3.2] - 2024-09-20 -## [1.3.2] - 2024-09-20 - ### Added #### Targets From 11eb3c40b4e0ffb6733030c747e71b1163c6a93b Mon Sep 17 00:00:00 2001 From: Bot-targa Date: Fri, 27 Sep 2024 17:26:13 +0200 Subject: [PATCH 07/11] chore(docs): update Data Catalog documentation --- .../data-catalog/10_overview.md | 19 ++++++++++++++++++ .../data-catalog/_category_.json | 4 ++++ .../data-catalog/changelog.md | 20 +++++++++++++++++++ 3 files changed, 43 insertions(+) create mode 100644 docs/runtime_suite_applications/data-catalog/10_overview.md create mode 100644 docs/runtime_suite_applications/data-catalog/_category_.json create mode 100644 docs/runtime_suite_applications/data-catalog/changelog.md diff --git a/docs/runtime_suite_applications/data-catalog/10_overview.md b/docs/runtime_suite_applications/data-catalog/10_overview.md new file mode 100644 index 0000000000..be142be357 --- /dev/null +++ b/docs/runtime_suite_applications/data-catalog/10_overview.md @@ -0,0 +1,19 @@ +--- +id: overview +title: Overview +sidebar_label: Overview +--- + + + +_Data Catalog_ is a Mia-Platform Marketplace application designed to configure in your Console project the +components of [Data Catalog](/docs/data-catalog/overview.mdx) solution. +It streamlines adding the necessary microservices, endpoints and configuration maps providing blueprint +from which further customization can be executed to build and deploy an ad-hoc Data Catalog solution. + +An in depth explanation of what is Mia-Platform Data Catalog, which are its components and how to configure them +can be found in the documentation section dedicated to the [product](/docs/data-catalog/overview.mdx). diff --git a/docs/runtime_suite_applications/data-catalog/_category_.json b/docs/runtime_suite_applications/data-catalog/_category_.json new file mode 100644 index 0000000000..8632b78793 --- /dev/null +++ b/docs/runtime_suite_applications/data-catalog/_category_.json @@ -0,0 +1,4 @@ +{ + "label": "Data Catalog", + "position": 10 +} \ No newline at end of file diff --git a/docs/runtime_suite_applications/data-catalog/changelog.md b/docs/runtime_suite_applications/data-catalog/changelog.md new file mode 100644 index 0000000000..e73b5b1200 --- /dev/null +++ b/docs/runtime_suite_applications/data-catalog/changelog.md @@ -0,0 +1,20 @@ +--- +id: changelog +title: Changelog +sidebar_label: CHANGELOG +--- + + + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## Unreleased + +Initial release of Data Catalog application From 1cbc5aace0bec67c947b84265fb1637fc6e7f553 Mon Sep 17 00:00:00 2001 From: Bot-targa Date: Fri, 27 Sep 2024 17:35:41 +0200 Subject: [PATCH 08/11] chore(docs): update Data Catalog documentation From ac8b1dece2babc58acb9f399d959311a815ae106 Mon Sep 17 00:00:00 2001 From: Ian Date: Fri, 27 Sep 2024 17:47:29 +0200 Subject: [PATCH 09/11] remove files related to the template --- .../data-catalog/10_overview.md | 25 - .../data-catalog/20_configuration.md | 803 ---------------- .../data-catalog/_category_.json | 4 - .../data-catalog/changelog.md | 99 -- .../data-catalog/img/architecture.png | Bin 19519 -> 0 bytes .../data-catalog/agent.model.schema.json | 265 ------ .../data-catalog/catalog.model.schema.json | 367 -------- .../data-catalog/configuration.schema.json | 877 ------------------ 8 files changed, 2440 deletions(-) delete mode 100644 docs/runtime_suite_templates/data-catalog/10_overview.md delete mode 100644 docs/runtime_suite_templates/data-catalog/20_configuration.md delete mode 100644 docs/runtime_suite_templates/data-catalog/_category_.json delete mode 100644 docs/runtime_suite_templates/data-catalog/changelog.md delete mode 100644 docs/runtime_suite_templates/data-catalog/img/architecture.png delete mode 100644 static/docs_files_to_download/data-catalog/agent.model.schema.json delete mode 100644 static/docs_files_to_download/data-catalog/catalog.model.schema.json delete mode 100644 static/docs_files_to_download/data-catalog/configuration.schema.json diff --git a/docs/runtime_suite_templates/data-catalog/10_overview.md b/docs/runtime_suite_templates/data-catalog/10_overview.md deleted file mode 100644 index 43e1f1db6c..0000000000 --- a/docs/runtime_suite_templates/data-catalog/10_overview.md +++ /dev/null @@ -1,25 +0,0 @@ ---- -id: overview -title: Overview -sidebar_label: Overview ---- - - - -_Data Catalog Agent_ is command line interface binary that queries datasources for the data schemas of the resources they own and sends them to a target. In the context of databases, it provides facilities to: - -1. query tables schemas -2. aggregate them in a unique asset (JSON format) -3. send it to an HTTP server or store it locally on file - -_Data Catalog Agent_ serves as a driver for collecting and browsing, in a unified view, tables/collections persisted in different sources (System of Record, a.k.a SoR). - -![architecture](img/architecture.png) - -_Mia Console_ allows to intercept the _Data Catalog Agent_ output assets and visualize them in a dedicated web view. _Mia Console_ is one of the configurable targets of _Data Catalog Agent_. - -_Data Catalog Agent_ is well suited for running locally or configured in _Mia Console_ as a [CronJob](/development_suite/api-console/api-design/jobs-cronjob.md). diff --git a/docs/runtime_suite_templates/data-catalog/20_configuration.md b/docs/runtime_suite_templates/data-catalog/20_configuration.md deleted file mode 100644 index 0157aff189..0000000000 --- a/docs/runtime_suite_templates/data-catalog/20_configuration.md +++ /dev/null @@ -1,803 +0,0 @@ ---- -id: configuration -title: Configuration -sidebar_label: Configuration ---- - - - -Due to licensing _Data Catalog Agent_ requires packaging with third-party software, unless your datasources are PostgreSQL databases whose support is already included in the docker image of _Data Catalog Agent_. - -In case only PostgreSQL is needed you can skip to the [cronjob configuration](#cronjob-configuration) section. - -## Datasource support - -Mia's _Data Catalog_ marketplace item upon creation, creates also a git repository containing: - -1. a CI config file -2. a docker `Dockerfile` - -the `Dockerfile` provides several build steps to include the datasource support you need. Supported datasources are: - -- Oracle Database from 11 to 21 -- PostgreSQL from 9.6 to 16 -- Mysql from 5.7 to 8.2 -- Mssql from 2017 to 2022 - -Upon inspection you'll notice that the `Dockerfile` contains targets for each database: - -```Dockerfile -FROM ... AS data-catalog-agent-oracle - -# rest of the file - -FROM ... AS data-catalog-agent-mysql - -# and so on... -``` - -Each push on the default branch or any tag starting with `v` will create a new docker image including the selected support. The default support is `postgres` but it can be changed: - -1. by editing the `.gitlab-ci.yml` file overriding the initial value of the `DOCKER_TARGET` from `postgres` to the database you need to support -2. one-of change upon manually triggering the CI overriding the same variable `DOCKER_TARGET` - -Your repository contains a `README.md` file which also contains a lists of operations needed to include drivers for `oracle`, `mysql`, `mssql` databases. - -### Oracle Database: add support - -**In the template repository**, to enable Oracle Database support actions are needed: - -1. edit the value of `DOCKER_TARGET` variable in `.gitlab-ci.yml` to `oracle` or alternatively select it on manual run -2. embed oracle drivers (see [embed drivers](#embed-drivers)) -3. [optional] add `tnsnames.ora` and/or `Oracle Wallet` (see [add tns/wallet](#add-tns-support)). - -#### Embed drivers - -**In the template repository**, download drivers from the [official page](https://www.oracle.com/database/technologies/instant-client/linux-x86-64-downloads.html) according with the version you need. - -The following assets are needed: - -1. ODBC driver -2. instantclient basiclite - -```shell -# on unix -export ORACLE_CLIENT_VERSION="" -unzip -j instantclient-basiclite-linux.x64-${ORACLE_CLIENT_VERSION}dbru.zip "instantclient_*/lib*" -d lib/x86_64-unknown-linux-gnu/oracle -unzip -j instantclient-odbc-linux.x64-${ORACLE_CLIENT_VERSION}dbru.zip "instantclient_*/lib*" -d lib/x86_64-unknown-linux-gnu/oracle -``` - -create a symlink - -```shell -export LIBSQORA_NAME=`ls lib/x86_64-unknown-linux-gnu/oracle | grep -E -i '^libsqora\.so\.'` -ln -s "${LIBSQORA_NAME}" lib/x86_64-unknown-linux-gnu/oracle/libsqora.so -``` - -#### Add TNS support - -Oracle connectivity can be handled with `tnsnames.ora` files and/or wallets. The container expects such assets in the `/home/agent/oracle/admin` folder. The `TNS_ADMIN` is already set to `/home/agent/oracle/admin` and the container will copy the content of the `oracle/admin` folder in this repository. - -Drop your wallet assets or `tnsnames.ora` file in `oracle/admin` to include them. - -In case you want to use a wallet remind to [tune] the `sqlora.net` file registering the `/home/agent/oracle/admin` path: - -> WALLET_LOCATION = (SOURCE = (METHOD = file) (METHOD_DATA = (DIRECTORY="/home/agent/oracle/admin"))) -> SSL_SERVER_DN_MATCH=yes - -### MySQL: add support - -**In the template repository**, ODBC driver for MySQL must be downloaded from the [official website](https://dev.mysql.com/downloads/connector/odbc/) -selecting: - -1. version `8.3.0` -2. OS `Linux - Generic` -3. OS version `glibc 2.28 x86_64` - -then run: - -```shell -export MYSQL_NAME=mysql-connector-odbc-8.3.0-linux-glibc2.28-x86-64bit -tar -xvf ${MYSQL_NAME}.tar.gz ${MYSQL_NAME}/lib/libmyodbc8w.so -mv ${MYSQL_NAME}/lib/* lib/x86_64-unknown-linux-gnu/mysql -``` - -### MS SQL Server: add support - -No further actions beside editing `.gitlab-ci.yml` are required. - -### Mia CRUD Service: add support - -By default, running the CI of this project produces an artifact with Mia CRUD Service support since it does not require any specific drivers. - -### Salesforce SObjects API: add support - -By default, running the CI of this project produces an artifact with Salesforce SObjects support since it does not require any specific drivers. - -## Cronjob Configuration - -From your project select `CronJobs` > `Create new CronJob`, insert a unique name for the job, an optional description, the full name of the docker image created by your CI job. It will come in the form `/data-catalog/agent:`. Also insert a `CronJob schedule` using the [k8s syntax](https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/#writing-a-cronjob-spec). - -By clicking on create, you need to configure: - -1. the config map containing the configuration of the _Data Catalog Agent_ -2. the cronjob yaml file - -Create a new config map giving it a name of your choice (e.g., `data-catalog-agent-configuration`) and mount path `/home/agent/.config/dc/agent/`. We can now add it to the `CronJob` yaml manifest. - -The prompted yaml for the `CronJob` should be edited as follows: - -```yaml -apiVersion: batch/v1 -kind: CronJob -metadata: - ... -spec: - ... - jobTemplate: - spec: - ... - template: - ... - spec: - ... - containers: - containers: - - name: ... - ... - env: - - name: TNS_ADMIN # not required if it defaults to /home/agent/oracle/admin or no oracle support is needed - value: /path/to/tns/admin/folder - args: ["scan"] # add args here - volumeMounts: - - name: data-catalog-agent-configuration # reference a volume mount - mountPath: /home/agent/.config/dc/agent/ - volumes: - - name: data-catalog-agent-configuration # use the name given to the config map - configMap: - name: data-catalog-agent-configuration -``` - -Now create a new file in the config map called `configuration.json`. This file must be a valid _Data Catalog Agent_ configuration. - -## Data Catalog Agent Configuration - -:::caution -The configuration is read from a file which must be located ad `~/.config/dc/agent/configuration.json` where `~` is the user's home folder (`$HOME`). Such location can be overridden via the environment variable `DC_AGENT_CONFIGURATION_FILEPATH` which must contain a filepath to a valid configuration file. -::: - -The configuration has the following main sections: - -1. connections -2. target - -```json -{ - "connections": { - ... - }, - "target": { - ... - } -} -``` - -Secretable fields are marked in the following sections. - -## Connections - -A list of entries to connect to datasources via supported integration methods. The general recipe is: - -```json -{ - "connections": { - "": { - "type": "", - "configuration": { - // depends on the integration method - }, - "settings": { - // depends on the integration method - } - } - } -} -``` - -Available integration methods are: - -- [`odbc`](https://en.wikipedia.org/wiki/Open_Database_Connectivity) -- `http` - -Such drivers come often in the form of a [`dynamic shared object`](https://en.wikipedia.org/wiki/Shared_library) and must be embedded with the binary and available at runtime (see the template repository README file). - -A non-required object `settings` can be added to the connection configuration to define the following additional properties: - - `namespace`: string that overrides the default connection name; - - `batchSize`: number that defines the number of tables that can be paginated in a odbc query. if not specified, 1000 tables for query will be retrieved. - - `columnBatchSize`: number that defines the number of columns that are expected to be processed by each batch iteration. if not specified, 20 columns for batch will be used. - -### PostgreSQL - -To configure a **PostgreSQL** ODBC connection use: - -```json -{ - "connections": { - "": { // 👈 pick a name - "type": "odbc", - "configuration": { - "vendor": "postgres", - "params": { - "uid": "test_user", - "pwd": "password", - "database": "" - } - } - } - } -} -``` - -or use an inline ODBC connection string - -```json -{ - "connections": { - "": { // 👈 pick a name - "type": "odbc", - "configuration": { - "vendor": "postgres", - "params": "DRIVER=postgres;SERVER=0.0.0.0;PORT=5432;DATABASE=db;UID=user;PWD=pwd;" - } - } - } -} -``` - -Other keys are `host` and `port` which for a **PostgreSQL** connection are defaulted to `0.0.0.0` and `5432`. Any other configuration parameter can be appended using the key `flags` which must be a semicolon separated string. - -#### Secretable fields - -`uid`, `pwd` or `params` support [secrets resolution](/fast_data/configuration/secrets_resolution.md). - -### Oracle - -To configure an **Oracle** ODBC connection use: - -```json -{ - "connections": { - "": { // 👈 pick a name - "type": "odbc", - "configuration": { - "vendor": "oracle", - "params": { - "uid": "test_user", - "pwd": "password", - "dbq": "" - } - } - } - } -} -``` - -or use an inline ODBC connection string and also set the TNS_ADMIN environment variable -to access a `tnsnames.ora` file where the DBQ name can be resolved - -```json -{ - "connections": { - "": { // 👈 pick a name - "type": "odbc", - "configuration": { - "vendor": "oracle", - "params": { - "uid": "test_user", - "pwd": "password", - // 👇 this must be defined in your tnsnames.ora file - "dbq": "DRIVER=oracle;UID=user;PWD=p4ssw0rd;DBQ=DATABASE_NAME" - } - } - } - } -} -``` - -`SERVICE_NAME` is an entry of your [`tnsnames.ora`](https://docs.oracle.com/en/database/oracle/oracle-database/18/netrf/local-naming-parameters-in-tnsnames-ora-file.html#GUID-12C94B15-2CE1-4B98-9D0C-8226A9DDF4CB) file. Such file **MUST** be available when running the binary and its location **MUST** be exported to an environment variable unless defaulted to `/home/agent/oracle/admin`. - -in case you don't want to add the environment variable, the content of one entry of the `tnsnames.ora` file can be inlined such as: - -```json -{ - "connections": { - "": { - "type": "odbc", - "configuration": { - "vendor": "oracle", - "params": { - ..., - "dbq": "(DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)(HOST=0.0.0.0)(PORT=5041))(CONN ..." - } - } - } - } -} -``` - -Other keys are: - -- `version` which can be neglected **UNLESS USING ORACLE 11g** since it [does not support](https://docs.oracle.com/database/121/LNPLS/release_changes.htm#LNPLS113) pagination via `OFFSET ? ROWS FETCH FIRST ? ROWS ONLY`. - - :::caution - - `version` MUST be explicit at `configuration` level. - ```json - { - "type": "odbc", - "configuration": { - "vendor": "oracle", - "version": 11, - "params": { /** ... */ } - } - } - ``` - ::: - -- `flags` which will be added, as a string, to the connection like `"flags": "DBA=R;"`, for the available flags check [oracle documentation: 25.4.1 Format of the Connection String](https://docs.oracle.com/en/database/oracle/oracle-database/21/adfns/odbc-driver.html#GUID-618B141E-DD46-4907-99C2-486E801CA878). - -- `options` field for _oracle_ odbc configurations: this field defines the tables that will be used for retrieve the metadata. - If not specified, `user` option will be used as default. Three options available: - - - `user`: tables `user_tables`, `user_views` and `user_mviews` will be used. No filter needs to be specified since they default to the user's schema. - ```json - { - "type": "odbc", - "configuration": { - "vendor": "oracle", - "params": { /** ... */ }, - "options": { - "user" - } - } - } - ``` - - `all`: tables `all_tables`, `all_views` and `all_mviews` will be used. Optionally, a list of schemas needs to be provided in `owners` field to filter the desired schemas. - ```json - { - "type": "odbc", - "configuration": { - "vendor": "oracle", - "params": { /** ... */ }, - "options": { - "all": { - "owners": ["some_schema_name"] - } - } - } - } - ``` - - `dba`: tables `dba_tables`, `dba_views` and `dba_mviews` will be used. Optionally, a list of schemas needs to be provided in `owners` field to filter the desired schemas. - ```json - { - "type": "odbc", - "configuration": { - "vendor": "oracle", - "params": { /** ... */ }, - "options": { - "dba": { - "owners": ["some_schema_name"] - } - } - } - } - ``` - - :::caution - - In this case, remember that password needs to be written in the form of ` AS SYSDBA`. - - ::: - -Connection with an [oracle wallet](https://docs.oracle.com/en/database/oracle/oracle-database/19/dbimi/using-oracle-wallet-manager.html#GUID-D0AA8373-B0AC-4DD8-9FA9-403E345E5A71) is also [supported](https://docs.oracle.com/en/cloud/paas/autonomous-database/serverless/adbsb/connect-prepare-oci-wallets.html#GUID-EFAFA00E-54CC-47C7-8C71-E7868279EF3B): - -Preparation must be done on the template repository: - -1. The whole wallet content must be available under the same path of the `tnsnames.ora` (they usually come together) -2. The file `sqlora.net` inside the wallet must be updated to reflect the `TNS_ADMIN` path - > WALLET_LOCATION = (SOURCE = (METHOD = file) (METHOD_DATA = (DIRECTORY="/path/to/folder"))) - > SSL_SERVER_DN_MATCH=yes - -Also the environment variable must be set: - -1. The `TNS_ADMIN` environment variable must be set **explicitly** unless defaulted to `/home/agent/oracle/admin` - -#### Secretable fields - -`uid`, `pwd` or `params` support [secrets resolution](/fast_data/configuration/secrets_resolution.md). - -### MS SQL server - -To configure a **MS SQL Server** ODBC connection use: - -```json -{ - "connections": { - "": { - "type": "odbc", - "configuration": { - "vendor": "mssql", - "params": { - "uid": "test_user", - "pwd": "password", - "database": "" - } - } - } - } -} -``` - -or use an inline ODBC connection string: - -```json -{ - "connections": { - "": { - "type": "odbc", - "configuration": { - "vendor": "mssql", - "params": "Driver=mssql;Server=0.0.0.0,1433;Database=db;Uid=user;Pwd=p4ssw0rd;TrustServerCertificate=yes;" - } - } - } -} -``` - -Other keys are `host` and `port` which for a **PostgreSQL** connection are defaulted to `0.0.0.0` and `1433`. Any [extra connection property](https://learn.microsoft.com/en-us/sql/connect/odbc/dsn-connection-string-attribute?view=sql-server-ver16) can be added via the key `flags` which will be added, as a string, to the connection, as a semicolon separated string. It is quite useful, for local development purposes to add the flag `"TrustServerCertificate=yes"`. - -#### Secretable fields - -`uid`, `pwd` or `params` support [secrets resolution](/fast_data/configuration/secrets_resolution.md). - -### MySQL - -To configure a **MySQL** ODBC connection use: - -```json -{ - "connections": { - "": { - "type": "odbc", - "configuration": { - "vendor": "mysql", - "params": { - "uid": "test_user", - "pwd": "password", - "database": "" - } - } - } - } -} -``` - -or use an inline ODBC connection string: - -```json -{ - "connections": { - "": { - "type": "odbc", - "configuration": { - "vendor": "mysql", - "params": "DRIVER=mysql;SERVER=0.0.0.0;PORT=3306;DATABASE=db;UID=user;PWD=p4ssw0rd;" - } - } - } -} -``` - -Other keys are `host` and `port` which for a **PostgreSQL** connection are defaulted to `0.0.0.0` and `3306`. Any [extra connection property](https://dev.mysql.com/doc/connector-odbc/en/connector-odbc-configuration-connection-parameters.html) can be added via the key `flags` which will be added, as a string, to the connection, as a semicolon separated string. - -#### Secretable fields - -`uid`, `pwd` or `params` support [secrets resolution](/fast_data/configuration/secrets_resolution.md). - -### Mia CRUD Service - -To configure a **Mia CRUD Service** connection use: - -```json -{ - "connections": { - "": { - "type": "http", - "configuration": { - "vendor": "mia-crud-service", - "params": { - "baseUrl": "http://mia-crud-service:3000", - "endpoint": "/-/schemas", - "healthcheck": "/-/healthz", - "headers": { - "accept": "application/x-ndjson" - } - } - } - } - } -} -``` - -The driver basically calls the `/-/schemas` URL to extract all the data models from MongoDB. Since the response is, of course, a JSON Schema, some information in the `ddl` object may be missing, like the `size` of the properties. - -Additionally, the version of the CRUD Service is obtained from the `/-/healthz` endpoint. - -In case custom `tls` parameters are required such as a custom root CA or an insecure HTTPS connection, -use the parameter `tls` with keys `insecure` and/or `certificate`: - -```json -{ - "connections": { - "": { - "type": "http", - "configuration": { - ..., - "params": { - "tls": { - "insecure": true - } - } - } - } - } -} -``` - -or - -```json -{ - "connections": { - "": { - "type": "http", - "configuration": { - ..., - "params": { - "tls": { - "certificate": { - "type": "file", - "path": "/path/to/root/CA" - } - } - } - } - } - } -} -``` - -### Salesforce SObjects - -To configure a **Salesforce SObjects** HTTP connection you can use two authentication methods: - -* [jwt-bearer](https://help.salesforce.com/s/articleView?id=sf.remoteaccess_oauth_jwt_flow.htm&type=5): - ```json - { - "connections": { - "": { - "type": "http", - "configuration": { - "vendor": "salesforce-sobjects", - "params": { - "authenticationFlow": "jwt-bearer", - "baseUrl": "https://my-subdomain.my.salesforce.com", - "clientId": "XXXXXXXXXXXXXXXXXXXXXXXXXX", - "privateKey": "/path/to/private-key/key.pem", - "username": "my-user@email.com", - "apiVersion": "59.0" - } - } - } - } - } - ``` -* [username-password](https://help.salesforce.com/s/articleView?id=sf.remoteaccess_oauth_username_password_flow.htm&type=5): (**deprecated**) - ```json - { - "connections": { - "": { - "type": "http", - "configuration": { - "vendor": "salesforce-sobjects", - "params": { - "authenticationFlow": "username-password", - "baseUrl": "https://my-subdomain.my.salesforce.com", - "apiVersion": "31.0", - "clientId": "XXXXXXXXXXXXXXXXXXXXXXXXXX", - "clientSecret": "XXXXXXXXXXXXXXXXXXXXXXXXXX", - "username": "my-user@email.com", - "password": "XXXXXXXXXXXXXXXXXXXXXXXXXX", - "securityToken": "XXXXXXXXXXXXXXXXXXXXXXXXXX" - } - } - } - } - } - ``` - -> The range of supported `apiVersion` goes from `31.0` to `59.0` - -The driver uses the [Salesforce SObjects API](https://developer.salesforce.com/docs/atlas.en-us.api_rest.meta/api_rest/intro_what_is_rest_api.htm) to retrieve the schema from all the `SObject`s you have. Make sure the user you configure has the right permissions to retrieve all the `SObject`s you need. - -If not all the `SObjects` are retrieved you'll have to follow these steps to expose them through the API: - -**For custom `SObjects`** - -- Go to your Salesforce dashboard -- Access Setup -- Access the "Permission Sets" section -- Create a new Permission Set (e.g. API for Custom Objects) with license type "Salesforce API Integration" -- Click on "Object Settings" inside the new Permission Set, then, for each custom SObject, update the visibility in the desired way (e.g. View All, Modify All) -- Assign the new Permission Set to the user with "Salesforce Integration" user license and "Salesforce API Only System Integrations" profile - -**For standard `SObjects`** - -- Go to your Salesforce dashboard -- Access Setup -- Access the "Permission Sets" section -- Create a Permission Set (e.g. API for Standard Objects) without specifying a license type -- Click on "Object Settings" inside the new Permission Set, then, for each standard SObject, update the visibility in the desired way (e.g. View All, Modify All) -- Assign the new Permission Set to the user with "Salesforce Integration" user license and "Salesforce API Only System Integrations" profile - -#### JWT Authorization - -To be able to connect to the Salesforce API using the JWT Bearer flow you will need to follow the next steps. - -1. Follow the steps one and two from this guide: https://help.salesforce.com/s/articleView?id=sf.connected_app_create_api_integration.htm&type=5 -2. Then you'll need to upload your self-signed certificate in PEM format (step 5 from the guide). If you don't have a certificate follow these instructions: - 1. Generate a rsa key with the command `openssl genrsa -out tls/key.pem` - 2. Then create a request to sign the certificate with this command `openssl req -new -key tls/key.pem -out tls/sf_connect.csr` - 3. Sign the request with you private key with `openssl x509 -signkey tls/key.pem -in tls/sf_connect.csr -req -out tls/sf_connect.crt` - 4. Upload the generated certificate (`tls/sf_connect.crt`) -3. Now you need to enable the OAuth2 mechanism following the 10th step from the guide, creating a connected app and all. -4. Finally you need to enable the JWT access with the step 14 of the guide. - -Now you should have everything you need to fill out the configuration parameters. If you are using a testing instance you'll need to set the param `loginUrl` to `https://test.salesforce.com/`. - -#### Secretable fields - -`clientId`, `username`, `clientSecret`, `password`, `securityToken` or `privateKey` support [secrets resolution](/fast_data/configuration/secrets_resolution.md). - -## Targets - -There are 4 targets available: - -1. [**default**] `stdout` -2. `mongodb` -2. `file` -3. `mia-console` - -For each listed connection, after metadata is retrieved, `agent` **sequentially** sends data to the target as: - -- `json` for `stdout` and `file`; -- [`ndjson`](https://github.com/ndjson/ndjson-spec) for `mia-console`. -- [`BSON`](https://bsonspec.org/) for `mongodb` - -The final content is an `array` of models, where the format of its records changes accordingly to the target: - -- `stdout`, `file` and `mia-console`: the models are written in the native agent format, which is defined in the following JSON schema; -- `mongodb`: the models are written in a format that is supported by the [Data Catalog](/data_catalog/overview.mdx) application, as defined in the following JSON schema; - -### Standard Output - -To explicitly configure the `stdout` target use: - -```json -{ - ..., - "target": { - "type": "stdout" - } -} -``` - -### MongoDB - -The MongoDB target enables Data Catalog Agent to feed data from external sources to the [Data Catalog](/data_catalog/overview.mdx) application. - -To configure the `mongodb` target use: - -```js -{ - // ... - "target": { - "type": "mongodb", - "url": "mongodb://test:27017/?replicaSet=rs", // 👈 mongodb connection string: the database must be a replica set - "database": "test_database", // 👈 if defined, it will be used as default database to store the models - } -} -``` - -The target will write the content of the connections to a MongoDB replica set database, in a collection named `open-lineage-datasets`. - -:::tip -To enforce document validation on that collection, be sure to run [Data Catalog Configuration Scripts](/data_catalog/database_setup.mdx) before executing the agent. -::: - -### File - -To configure the `file` target use: - -```json -{ - ..., - "target": { - "type": "file" - } -} -``` - -which will save output files in the folder `./output`. To override this use: - -```json -{ - ..., - "target": { - "type": "file", - "dir": "/path/to/dir" - } -} -``` - -### MIA Console - -:::caution -This target has been **deprecated** in favour of [`mongodb`](#mongodb) to support [Data Catalog](/data_catalog/overview.mdx) solution. -::: - -To configure the `mia-console` target use: - -```json -{ - ..., - "target": { - "type": "mia-console", - "baseUrl": "https://my-server-url", // 👈 mia console base url - "projectId": "1234", // 👈 models are pushed towards the project with this id - "apiKey": "1234", // 👈 mia console api key, may vary across test/preview/prod environments - "credentials": { - // machine to machine credentials - } - } -} -``` - -credentials are used to [obtain](https://mia-platform.eu/blog/client-credentials-m2m-authentication-oauth/) an access token. In order to do that you must provide: - -```json -{ - ..., - "target": { - ..., - "credentials": { - "clientId": "1234", - "clientKeyId": "123", - "privateKey": {"type": "file", "path": "tls/key.pem"} // 👈 either a file system path or an rsa private key inlined with `\n` - } - } -} -``` - -on `type` `mia-console` the auth endpoint can be customized using `oauthTokenEndpoint`. The service endpoint hosted on `mia-console` can be overridden in 2 ways: - -1. customizing the revision using the field `revision` -2. or customizing the overall url using the field `dataCatalogEndpoint` - -#### Secretable fields - -`clientId`, `clientKeyId`, or `privateKey` support secrets diff --git a/docs/runtime_suite_templates/data-catalog/_category_.json b/docs/runtime_suite_templates/data-catalog/_category_.json deleted file mode 100644 index 8632b78793..0000000000 --- a/docs/runtime_suite_templates/data-catalog/_category_.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "label": "Data Catalog", - "position": 10 -} \ No newline at end of file diff --git a/docs/runtime_suite_templates/data-catalog/changelog.md b/docs/runtime_suite_templates/data-catalog/changelog.md deleted file mode 100644 index 68387bce7f..0000000000 --- a/docs/runtime_suite_templates/data-catalog/changelog.md +++ /dev/null @@ -1,99 +0,0 @@ ---- -id: changelog -title: Changelog -sidebar_label: CHANGELOG ---- - - - -All notable changes to this project will be documented in this file. - -The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), -and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - -## [1.3.2] - 2024-09-20 - -### Added - -#### Targets - -- `mongodb` target. Models will be stored on a dedicated collection with the following target configuration: - ```json - { - // ... - "target": { - "type": "mongodb", - "url": "mongodb://test:27017/?replicaSet=rs", // 👈 mongodb connection string: the database must be a replica set - "database": "test_database", // 👈 if defined, it will be used as default database to store the models - } - } - ``` - - The record will be stored in a collection named `open-lineage-datasets`. - - > **NOTE:** - > - > To use MongoDB as a target, the database must be configured as a replica set. - -### Updated - -- _Data Catalog Agent_ bumped to version `0.6.4` - -## [1.3.1] - 2024-07-31 - -### Updated - -- _Data Catalog Agent_ bumped to version `0.6.3` - -### Fixed - -- bug introduced by previous release which prevented MySQL queries - to return records. - -## [1.3.0] - 2024-07-12 - -### Updated - -- _Data Catalog Agent_ bumped to version `0.6.2` -- rolling out of support for Oracle `11c` -- Crud Service http connection now has an additional parameter, `healthcheck`, which contains the path used to retrieve the version of the service. - -### Added - -- `settings` field to connection configuration object; -- `options` optional field for _oracle_ odbc configurations: this field defines the tables that will be used for retrieve the metadata. - If not specified, `user` option will be used as default; - -### Changed - -- `version` field in the `oracle` odbc configuration has been moved outside of the `params` object, so it can be used along with odbc connection string. - ```json title={example.json} - { - "type": "odbc", - "configuration": { - "vendor": "oracle", - "version": "11", - "params": {/** ... */} - } - } - ``` - - -## [1.2.0] - 2024-05-07 - -### Added - -- Support for `salesforce_objects` -- _Data Catalog Agent_ bumped to version `0.5.1` - -## [1.1.0] 2024-02-08 - -- Add Mia CRUD Service support to the data-catalog agent - -## [1.0.0] 2024-01-31 - -- First release diff --git a/docs/runtime_suite_templates/data-catalog/img/architecture.png b/docs/runtime_suite_templates/data-catalog/img/architecture.png deleted file mode 100644 index 457b0bb4a6f2384e4b3d5e95135edb8ef46ae6ba..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 19519 zcma%jbyS?swk3^2aCbULa0n3GgG-Rc-3jil4J3GQ_u%fX!3jZvySuyFG{1ZAnm2D| zy*2*;>+7#iRh>F@s%r0j2vwAqL`5b-hJu1Zm6j3%K|uiqA%Fitgope*_zZIf`2}zS zNq&Z^7$w<m6qxXvC@G54>iFgrT_MD(N(p6WgD*e_i%ERprWVs^3_t zMJt0jTnW<>ud@~J`wr9R&sK`qeVZY>`bii}^)33_rJzDM)vdto(rf$gD5n$mAHVP! zej!&umq9T^4^k%81pD2T>4R#U#z)WztrvvStw_yvo@&eLy(6-y^Bf||=xhEKiG zUe{=zZBdNJ2+L5XZM*(B&Ve%;o-*?BW}fsBEI6aXs>O)kQcR-BL}-7lv`_=8<@ zyP*oN&nX383aDdyId>b|Dj4+eC(Ik!hw}0Ewg|EFS5>2$ezy?fnhfw6@BSM2e6nQO zF6-Eke@hrWMBlfq0Z;?u8o=SuV-S<}bEl=zFM zEXKfYA)6mriKfEzrL;-LaQ`$1<L=rSFuReaUB%Z*(fz2Z`o2t zQK%889I^WKUvG!xLUpmikMGKAFXpQFdIc&=LYty|j007Yh!E-EI|cq@@^2e77Kv5T zQ8vDXiVJS`)!_*K*XM4^&ktVTL4zl`kVNtONT%4H* z7vus)znH!?C#(x+MoNY=`SXC*#csCs@kU=C4L)zhg_v*npON0+0)>eVmW;hQ@|&gL z^KJ$RX9xfJV1*6{h*k3NSO}%=VBD&XhOpWVpnDM3@*9y6wHz4%%q zgAp%G@b1@Ktv)V5Ne`46C>IFmSFL%AtL#eTJQFk!StjUl8}I?0RrM+cOapV?8JsNqsLb-yXON^qBi6 z_8cbt}THvhnptQUETd<(>uS6itAPoLAp&B#w z0|>JMZ}8iii{zS{KGjTn3h%hJLG%^#InMeZihP{bPFGN#u-4kFNUNQ`ph;fB|1%uU z3OP15%>@ZPWCFVU@jG!9=cq?Kn>y<{ju6F!Uo!;-0nRmvEWM#|s z@bw3?pw>8oXiM1vu7Aac0~`ugbp6nbL}Q08tMij$iouDtTdqn{F;bf%C;q-LDX zy3;Je_i)_)EzLN^YAtao@KT){C5a`^;EC?#BGJ)uHTte?#+SbUtOx}X-rwMnztF#xz;@VOo*$&s_hY zH;4vDgoJggUJl=Jc%sTYu)Q8)@&iJ65aKg@JdIs*`?^852GdfDd&3wTg z7Ll>U7PhgrA&4d-^|g-p$I`OuuQe!Xi;1(Uj~W_Uh|`@u&p}Q!qg|#Oa?VHNm<{5{ zh6DW>+|K9OW$K=FC!zgJ#JOItC(zMGXcN`+wBNjUU(tN{u0;Q!F|F183I!Cm8m9>S zlbM8bkqJnE`uvxoe~6eICLYDoRSV@e(=qP$*&omj0}Ta&dJe1c*-SpdmkZakoPe)S z4izgvko0ip6kSJgvs0fo=U~nU1Iq1*`ceJQ%5Hr#m`jnXSZU&XqGJtTHf?TCp&lqZ<5C>g-1_Z)T75nb?lRoiU zW39A#Q>J~jp31>Y$LD#kTp(w^x)C|Ayc2TU-P65(utR?%EmNvt!58fF#Y+=`Y{gB? z8k@)EIdAUd=blWxI4|YW=xqIPqNy^{;u0sPN4Z?trN%+upb^a`{o3n$g346{alO{t zAy|o`Mn`J_g_#gY7o4>Kfv{9`FO&xePx_C6YUOxzda(Wd{SLeopSW=Vr_ct8mI45=7s_0x{0^SkYVBqw-YoG&pyRhsVigyWc4 zL-ym43+Aa9qAB3CY(r$iv29TBqUjDIKDF=wfOO?r!ynXzf*rrhO$^dhaN^=Ld7heX zP0nU<9%{8*?|~ymO;gcae(M*=)7PbbsewMAv$y=sVj%osN`5(aXX@~KT_*zoSXgm+ zVY6kfo<3P(vC5Nr1)eA`Z1>I@COtAQly0S~7Y8$f+H+-<1|sP=`XEOGB4se3SYyZ; zT}q}h@}S^l-i3(xr?u23GUKWX62ri>YOHaw1jFdIa~B*&d<RTgIUc$bJ?%|U(Xd4sFafABV7+P+7LC6Wl zV~639kwGyrF?TF4E{{Q6yp2VGRWNORC&i$k?s=3fXfYelRt`QIR^lrZ{>y1tINlPb z>PKLG{-@d8u(yuo>A|pZ-@Qwnp$(LoF3I&cls{{BBPLd33V7^#S7faZ^TR5>zlpBG ze4$3P1j=z{YP`E*a>=yB`(&cdngUGqu?0|cR_Ua*O!FHoH~GFRfhOl|XvMSOwjCEt zfraVd+2_W=6o#n15!S$#?uCB5=8LVq3vu@GB2D6_38!fAUeZ|BTD2GPFcYJA-TY~i zSq)+=#rS1m)JD&vTai-hPg;f9^PWe=!3F}Eq;S*V0rEw~>M$6x`=y@5`d*yu+h0oN z;|<|@-s8P6Cf*-GTT;cU<>FD0b46d3bdo$95ZA5saJ_5SD#2QxtFd{5#++cnz~gR) z_%uMzIvksI?~e#zwZrGT@%$4-Ij{Aw%GL<}qTyB5$d6KdsT`zPA+{aXN zy;lD2&!S@Y&#jFVXOjp3sQ}{vv+0=C0$>;plfTc2c9rF&;3vx!Jb^_n;mH}&@%AxA z!?c`ise2V+KTk_i;Bfb&jT-5%{2wf6$TcjdD+ zOCu>#<8p-VaSkEA`&(zGxpan#gSn-jS!jy671md4f3jsr!tJ9Q5yEts*!@^TGxPHD zH3w`xQ6q50+Q<9b1rDu@NOt53#-MbKQWzth4naEXL%wpa@&zvp+wyyfPU2y(T-&P| z8QGQ*EO$~vgJkj3t3a+&r`MCvDV^ih`7NG(`nY9zp`JwaMaKEs3e(Y2o0ga#u^Xw# z@xzzRVUd%i7WhY3+e8sX-xm_}?>Be)@HIL?q$^ z86j)pniTyWHPB-8thf>#9xaRZ@VBAQLJwma4_^5a9zt~H*GR*mWPnjeEVG9&bEbf* z^?h=ub)0+D5!%T-ykKQ#j{fY9`QN;R5V`$WP#HXwlyvP~b{=iaE->!Zb`1-@0tyDy zjOHr9pI4EfSs@6yI9xF&HU{VyL=-eM55E&8pQwZtYdLA}z`3G+3QWL^;0NTUU=dKm zU13G*C5`@cE9}>3FyuPmG!Fv1+5K%b#;&tm#l`+QEE-&ZX1QAK+_J)N%)C_O zOPgZI;SS{DL3f?n!x+WX6?>A1@ZmW#@o&7r9o-cwQyZr6u$kpHJ{jgwA!67?`K+uk z2nOwBkxp3LJ};qcM9Zyv?ssi3KV`n^S9!9~E=rovtNoiV4C}l$5wx>akfmPj69`4c zbnuflwjt!NYN?(vqh>8kq8g8|Qxdl16{b^0T|)2U5M0x+J{^>_lKCq07o|K6q$42* z)$8$icsE)HMKF#)!KW~l1vD5oRCs9hlrm_PK@)131}H!rvMJ~9i6CDPZ=$am*B|V7 zcI_8n7$-F)j*CoE4KN{_<`fYST@m&EB_!;H42SSqfR>SBFwsD&c+xILADyUYj@>st zS;hDYFB+8lSu`MKWQjtA3ZD!wY&fMBDB2&9Lz|H;ihz)uY$B5_sbWG!LD@+ut5yqp zssvlI=TdXKIe>2RJC#Ci8k8)Gxm-K4+`5OURDv&K#yxOI$gd3b*GkWM?EXl`r$e+l2!r59F2K_=F;4t)O)@{XL z0IOsvX~C2*y14w~qPisHI>K8*xFg^IwO_OS@L|zgjkHvq*4i+6s;)ar(<+Y?cO_HhpL{HP&cC7+o2<3>b=FrTLHdpS3m2x#bOxf^SSv- z-!f9Fp{%6Cnep&8i51n(c_e`7Uo;t^ma(jHsuG)WYRybWk2BX5rxcJ0r^qbJr? z30q7j*$D;ndtB3fJdhmz3nyBdH5E&aBE^I>GMyI*napXIA3xBH(;<6-4N{OZt%qP~ zw;C2y!uQMr4jRAUr%8Uq|BgI_3R1Zpcvfz+vYKNNy^Jt>;5U7%pfj*doSz*;q}fGt zjPS|MzhRcIwp6gHpu>fnd+2FOVEYJqr>KF6wZ+sF4RzmsI%dMaehK5H{x?MN<-#L+ zh-lZb4EzCSO`JDXF~-JR3jl925r##;o|4?6bWjIchi`a*>fP!e>}gy%gX{?VbJAau zRO5$eh(tbojfTJ#pb}wA0NdGbt97mHO6yU<{ECZbiwZES-m&EF(kdW)gnc* zh6w5}ERf}2EEe7u{jGz+qBRDm#Cw_wTWT3LB$?5vRpz&1?md$59Gi2!^ZA*$vGS1s z&In`g5yQVD%3!tzaFklOlEe-JI?s+-(`Yq8XqU<@;k%-H_Pp%V>^uvJ?m&@pQQdmP zQWic8cZSYt`<}F@Hdk-`t~HUg=_`s3V3m^WOu!M!mFeJDg*fVu#ze?5asqd^z;5KL zYE$ETiO6$px#+g5!Jark@p+Vb3s;e9uI+Vf%`zi&ih67V7B<7yW#A2%_nw`@bhB3YWeO%MVz%JXU2jFOM|a z6!M-@~Cg3|kJKF!cW<_MfOof(8IBU3JTb&Ch>jpEaTnb$9WWS8p}{eV1(V{7m-8Ck7Zc3*n(o2J}BmJ8(e1Z|Te@QSoSqz+bXT z6!%{g^JG`Peg~7RyLQ;xGO%ApW%V!=ZjNQ>aV_^!1%~Ep;@Ks=sG{eKh}Jbo?G9Eo zn>V|95q*j&ouY<|MA;8bEG@JgriP4&Zl*U!97zzG|5YM6mfFKzrm{~cSd}zeW?)d7 zb9>Ar0Z8lr5f5Dbt_26cDtJeiz?c@8-tr=YO-WEbWNjJCWejgz3+++v?h7&|=Q$?! zyuWdB7d9x;L<=uFG()W-&k;u_h}ObMD}^SAULb_43rhf%91KHHC*=z@GJ^{Y<$MmT zGZWy;Er#=hF4?c4ZuNDNtt(W$Qg$NZ#!BUS4L!khorII|=bB7EA+H<3k5`q1?WI@a zv5i`?)sC>m`DD}QmJrkXK#MfVmb5^U>Md`%;Gpd;`>9dkNmr(m0XyH9bIO9*GQf>w zy{;SRI^Q`44F;6t$$sn&xv@xki9Z5SebF2*78qoYk+)B@YK_+U7h+PZ#v7oo_DZpn z$R8L9v}z+74$XuQVmE^J-@$M?+0?m`}d zU$aG(n5M?))R;nsRi6qWrvkp*_qwAZ@;cYE#(>30+)MX^<3dHX! z-DBjnTpW4JlK{mTkqSvp(I`^KC>3=T-vKEz0>7>c6CpeG6o|88*_j6cvYCvo$sB6c zAxc3IJ(sVAwGV_4ADnHWF>vMiMCTQ8(?Yh7jIM;4dUMQZ8LkBNYI-JIe7Vn%?ZGuj z5`u6Iz56%9mFL=@AV1X&o(i=!paF?MQW1Sf$uAQbsA5ud4sFO_YdsI@IGCVgmCLk9 zkXTt+5z$b48c#6?69o9!%m%aj7F=@aAyyy!0mY;llWmFQN=k@`XOg~$0Uy~W`nyLz z*|EjLQgw~6e_uJcqMup&tLUE2|9#eOwQFh$hZ-4Qq8ZDTR^=D(^cof+?)S7?a)Y!1 z0KexAi4Sj`@lVUoX%PSIrI*x`@Kh1wk7a}?ORi*2fs$5qg8z8=`XU2Xo^{E9-zsF$|7cT9tk>xk%+?pazH!}o z{{ACIsTpssLwTz7=ZlU?mI7w2fM)E0pJf>tpa<`$h~Ch5_^b{up8@^D1qE|=1A14M zt9U)0&VkJ-DWCa-e4l=WzDu%SyRsP0_(gOrphr!t}Q6ufvppnyf6W z_rtf&)0Lj!Ut>iY1vo!oKIN(`CshgE$tU;gK2&y9ymzwIU$fV7?ihjjZ6! zvkP+O-M!)QQOe#*0f!vBDc0uJaxi++nvd=rgS9dg1f|(PhnKyyM!dVjoIP3UONz!& zsZ_!2L&p6qrNNE(?W)=9{DA`!+93#HuBx@Xp@7(5do65uk*nk<$`9e; z5JZ6r4#g2I7V7@w*s7^~IVC&k_Gk`qK>3a3{ z#IUE0XE2G6i}QFs*yW^^wa5zp%61lw%g6iUgejiZROP$XUEiWzbM(R(N|SN2uG^!8 zQjG>vhD5$GjD~$=q<7eg!$~w;dTpLR0Lq2A8eIX|DvVub^AQ$^cQ_%_d~V1cJ$MD0iEO*nDn} zbF85xM!EwVomzOcQRnViJIowgOUaqx6igDl4-}WynzcsX{xB*vcr5d}-yIPW&6b7A zkE9u$R%5e$L2%t?q?nF}29ZUWSUsG{%ScA$0p36$X28A%rw|_FD#{{7pDB1absDAT z(WUp%?WIVq9qs05>07fc@!v94$%Kw0m-e$xFcl(*UUgjc^z}08Vm#{;zI0{rgQxeS ziENF?DbgbBpaPFot42}<<fy09X?wx%inf)q1vtPnO@<{d9u{ij|YO9H4V@C>-{Vc%~|} za9zcbH`|i98jcpD?y(0pPakjLybDmt!=7p6H;Pvo)`P`4+0R=Ceoxwy@sO0d9)12k zk^i?)ln=mCNd4#AsjhZo1dLQ_Rz{sY>x1hNn1U1TqsO=%1*uN0qiK4xBb+RkT_0QF zKCM;}P+s@x9r4Rt;C*N0!2Ta~j5}(?&4*iD%>9e<&Tt&Z9-?Vhk98xI{T0=x$Ccax zd%YHtcOSm=M(vT)7nX0LGt&jcBjkD$s^&<-n{2F_KrR~Bb{^DExrh_bw=PvoI&x~B zqp_&8x)snkfQ$7&zP<4Z!3v=lMA<(_bJZTWwmPjjAE8xqJTsUm5Tq`YFsV4;2Jb8u zRVMRf*S+U_HzH(iZ~+&ayJJRrHA*7~kQ9YhvD8MlgG#5B`-wVVk_MIVo4VcVP{;5e z92T@rt`~046=iU24xL%b}L(@=59t(}%>e!?ayOJDFUc91f+P+kTa@_a@&}cWN8JfeD zY@8{N{@n}P=tE}oU}sA(*@(`0G!MblTXNMBhA57k4VFFQ>9szg0g(<0Us@O`x{zQV zD8ITgSifEZ>64hk*9BTP_nI%8jxP}>B~ZR6G&gGu)>s|`kFJjT3Xr%V#p#v_9FrlsGZU1~MQD9;9lYl~b-6enGbvGap z)_T3d-l@={+(BLBv<*3{)Mxb@;cbc_q=uo&zj%Fkgg2|y`H|0F5V{v%@s&m2@M-Cu zV7j$X>N{DdJyKw+tB9+tETHKj5)BE85RZ)6JMX&~&NZgV4VjRCpR#WPAl7@S$qU{3 z9S%k~nAS#Syy{O4ZZf|K2IuK=lpyJ1uIn^?D0J*ZXZVa=Z-5K&MubI))3hAT9lsBJ zWm+RP)P%*>$0J^6SjZwBFA)(idEzpaEw!WzOJ8qboQ6tKflZm8f_>k(toq;$0SKyf z4kV%hb}&^Y4aAY2y)OXsH~NhtsZuTD;TQ%;tv`>pfNF95N!H zzZ-SdZnI>03C^{fazU@OOFn+db4?qCPs@pkJ z0z%?H-|CnZrI?glt_#G}sd6nAH$9RR0nTIsSorOg z>uV`a$?U!;j%H)bpx(L=^Nno4oX1&}`Q$Hs-X0+!)TEzw*v%KBflwehzefPwJ3s6O z`UET?3Yeo?*>qi8lQ@0};%T35xd@n+wZc+>WEzPW@4R3e8TTMX7i< zb6aCH6wcz9RL}kt7FxyDg!K&zz503BLerjRi*gAQwo#gGX2 zP_wwDy*aTkhia4*6wok*a+2omO=U+}hibXf<*7BQKkF2CPjUy0d}F~P_yCuSH4!6` zp$y5WkDfCm)yWbq=C-y{jrwB9ZcUeRtX$IJ-g~}U&-A)=-LA_=n(h}D`PMOLBUT`) zR?h)urHCQM6i&ElUuKr05v^+ERy5MY`8jU8NR()2mV6JoK7Oks^Z78Hg_6&=OHH>> zalvTa`m=Th$#LOj%U`T!yB&A;jr8}sU5KkJy^f%HQti_D4AVn^7G%npKnvXRy#4hb zhkuJCT&fkN6BwFR%St3--ln5QZGK^Irja2ilANCXsKbRzCV?&#S(Rb!K5jRe<2SCL zn~!3XhM`<02NG_n>NnWIFJ4^TKNACwRyaD#yH-da?N4Urxg#@qT=Bxqn7gG<=WKjC z1Ut;tI|U-WNJYE4;ZzAhIb|ZyjxmQgux(y}t7>)%K&dChi=DW!K{H$bz(?0^(+k9J z!QUyqR`6@om<4MbTT0=R<6tbN)!CgH8)VPr2#;fmt28@qh15z>A7%N8sYo*D?8{~e zON|qVi^d0_AS3zbLy~*O{k|*)MTaENDv>cHA_8aI!XfHOMoy+4{(BI`)$6`&ZJB1@ zQbt{Qz)CJ|{*|8DrpwU!nCz3DH>Z&ZJHzG5|S`z!T5avo#7zEpFHy~JNz6#B8Fe7aGC{uH_cYC@z zKGi{ZBr|XLchA4+$b)c^>*!Z#>40I%vP_;vlC867!htFVHcXrQ0||teURV4>#=bKa zvbh%wMn^H%CXVCm&mEOj5AV~xoG~YkKB1yjYH`?HDrnG2V zJZ#>F)QU+uwJ)C($!#!(X?aOLni6?7SWyDaOsuIuzw4T$<`4T0xd`~YDxhx1@@2w! z7Br0|!oPD*2uhpUSd{2Ef>r9t;PcD?(tZY>M;E{LbHA=Vt6Sscdy+~rYPBNX+AR;P z*^1g97jv2W##C7L9j(_45`NL*`tlKh&O-vKzWT>9C+=&G5W(wQ1F=)VXkVUY++Qn2 z1=V0-MuU3`GRysfBraCqbknLxY0s(WC=HOX zBkCNng^AC_8;mLJR#WP#5v)K}CE4ei1ICH>8sCGzcXdyER(U!;-{s)bD3X7cVD4#^Ai<0=cN>}4dr^4fbIBwh2ZS;={b0w0p?AXlS%G=D)m)X! zy^*30h2!0yd;WG#LgCPwqbBvS8`{RyTm}+G6A^Mg7=8m3lRx+BmAnx}E+bRTqNKtQ zWx!9pz6blVW5hx@_UI9Q;>>$NY$d%k`EpkCE1(LCDhzgdf39dFT9e~BazA=1RHdZ4 zaGD7&;Db_vEBl*M$NsPZ zuiIeepT*vofv@&OFj5ysqd)lWT;{0SJnkhp$L5N}DY;o3vDO^_2c-ce`Y9hpLuh)7 zFUa~fx}te<$8fjcn`8u*)Ugc1tv$el8iLpms`DyAAQqYy{|qTO2vblDEP;+YfR2@W z4tW5)&c0-;mWqerOIIb({8w2)-m*6*1W_`U-nBFoKF~?35+wpb;GXyI<5=Ucge?a| zM`pS)^USWNEUZwZGET@`nc?2hcVH%RV}Y!2q`t@gX=1&sdc$a(AQ?$nxBJJU92BX* z19{tsbO=sGe~9C8@BO$F(3bE1mMug4XJwr7ksH_&a^kN`-) zwBt+!KsOn6s-^@%mcjyxHTiX+W9x|i1HvigrBX@`*yZkxPf=|?txC*H^aE;9q)5rp zgLcRuLl8){1vC$D+OE?0`llWL!42@?nMQY+mcO-UD)~o!kz_0QYHecT>@IOi@1xSL zKXQR0#qoV}aq3NxkZD`8l3)vv{`^DN*FoN7DyrDSDK&V0XUz74tHnirBXXPfBM(3D z;}m1C_>4;}{2cnYk?4O#Bln4-V|(B99~^&Ti&fq4ZWWIQN9!SYJa!z)heR{YHr)qx zfjo;vnYl?KV0+>Vpfyo9dh(5D2xoo=hXaD zyNuajg`foWC?RvHz_`vP9iwzJ1!NkzWCz;)~(+9%cP-!MAm*Yj7jvYBt; z|58+7XHhl$lme7P$C8{0coKF$3$ckHgUcE5?0pIPIAmDpmMKD5E zrD~1O^3qP~bD-EJX%Qe0ebsYQ_iwljM92qVofjj4t3#72c3m}|UGv&Paa>3cmT#Sa zQm7^#{)=*T4ZsK#UL6>;P=}2)ZV6je-Zc6jX!=7|pzUzx-H(n?QqCGe{g(YRr{BNL}`5{ zij?j?Vb3|`2*83HWa$2Y${T}C^&PC3FaHW2A7STIi?Da3i%%ZF}H_6~u8lECBT^{VL~UfTqm zhwaQ*YdWve&Dr?liyi~EV3SU>NKR?~9?8t?X&^=AvX?0|<&-v0L*`w$)H{3_&^rnz z+~PNE*~RAnmvMeYkCq)BP+`zM3l_cqOsJ9+eEL@vu42XKe)IfK6w5A(Gi?bqIbpOG zR%)pWDXOd}i6#RjTMh#ta5sz@^@AS-Ke5{9R5=mgS%sdhnW zhA0Sio8Ojr2;s{#?JMN%-e_c-S&JnH$Wo-9{t}2HwS0tsBXbL}d!UzxRQmC)_Zg&Deti5%@gYn|EIMUXbT5l?oF>l<+26bd}ql5P##P zmriO!{O!5uLgbbXC>EZ8h!-dsKfyL%?o5WPr8X2sD!=W+n_d&S5FX%a?l5ezj13q? zGWP?bTQP{e{Jb@sKodEgp{=$#I%1XlyvPfS;NyQ;+4IqF-VCB5q$|IJfMMKCel)gv ztpQ=GB!BBvC2{C=Qa&Hl*n^sFGtZlm_ZGC=E6mfX4$DSj6q)dqGby z#i2{!o1?as6XTqpU-0G z8Kgbw5mS5F2@~MsT<;#iuepVHwhG0{A2QsC@aVot2>&WoD*Uj3wP!=>f)X+QVw?1j{F z3L^cRE(xFAgo({|J=uDXiP~^7Cm^!*bbTmv*IWgMS1}jXkm1>DX@5z?%Tb9|`Do{` zza*_GCeuJB&=fdB+7jv*W=%6gmgUu0ZFNYLF6fJq`++xC{S|XCnl%?BN`oXUV?f2n z{`x9}WG^$Xsu%(&u5k3u8baoGe<2ebTg`QjX`@x9x^c>w2nVEq;BZ$J28N|ir3OZ% zzt&G2e&{ctn3mctq~G-aKZ*;0ky*L2)9mWlr3y^KzE5Xp@$x#g$514ey$#K}xg|v! z^}MdudejtyQKzdve~KbO^i^X05rj&eS`GkUYpiLt%QIWqToVGvA!pd;Y&KbW%-8}v zJAPSg`^)~O*S8?ppZ4Qe8m-ouA(^x?ty>;4VyVHSD>xqymU7^S;!8t_T!@f9-W~~& z^z_U3gnx}1&yyfTA>%cmRm3BXqvQ zr$^h+i^OsUA-@{w2GXW@XP2=uv=mZ=bK=iydO%fKnwc-r=FmDy)Q~HugB>yrFQx)8lt0T17CJwksy5FB0`75GWqweBb8I z0#n$MZ6aJE_+*sZL-3VdSynsXy+A!>mZ~8G5_ssd!gXp~r|64=I_y~Csa-fCN2_M0 z4UWqPIJtVLd%<7iq7n)s;k?^$IBLwU;$ppAtafuPBKfdjs^phuHw1Hk^n6K+jvB+s z8Hg&3QvMLJ2B%AD3;#71l*pCkEy3yZ?)+vDkfGEjMi^5=E&Kp47CK93M2VPneS;fw zH0ZA@wk4%xkdz6d)YtKW)<~ zY4Oiaek7|Ot$ZyS7S~D^9cK|A)vPi3verr={K2$Uu309uK~h&(-ci>drt~F^UEWtC zzvvsNN`M!b>lAZ9svE}ZDyL-XUAxXubvvwkB;cifr+b{Yd8#?DXypXS-fAk7XNr zx<9izUzw;dr67E*5R?#EoRcKR`fJWySt5EY=Kpz-gVyr->OEAjz*4odKRds?E;fTS zoe!Vi{0GDE9SVe7YnZHAeYnAV)v~&?kcpI$Mdw5dJ3~Wi3iJKgiKfm|Axc>u>|`%D z%K(l=e*43gq;4E5qlPR2k30wAAxK@z7>~@C>`zvFPzQo4nw*V5F5$TPKCsHoW|?m@ zgiKz$-GQk|Hv8Pc&tqU+7RuQ2@FFHT0Ob8ZM95D2I;ZKu&5Itv$0Vw>-BIg?`RZ~J zo%6*rYem`A7eY5Vzy4op9f_m=rq(IN6_Ek+KkSkd@a6Hz#4`u??ArR{#n3`;wJ-YR zD(rrs$jPQz!Sg0vRh+MFNbt5YV}PzK}alkc8k z`{54J4R$V;Yd(LsxiC5mP_i%6pirf+SaRiuceefZb2;^crQcZfAC2})Zhp5>wT0kO zW!I1zubn<-a9n3BG*yD^!AdU)M5w;P0Xsk-mc?RY69u|IPB;=#9OXAUczoih{WonMU~kHb$;zIqNXdi&cByIDhmUZ)loPPui0k9vBuNe%r@hv z?=WL~A>of@hM|_?LHIzE0N{urR&Rry9`(*gAq3yvO3KHu_AFG!wC<0z=$3-+K~2p_jZXGG%J@>KR!PoZ^eG zkZWv>iOlL+izWI@b5`$(DYjxRNV*LjrT?k^vHDp-fuFw_%fs;{381uK&IYFp6Aijb z)+qVJBP?^7h$BKD#)iuQsqkpZrRJVto)=-6C_0F=lIdLWc9>z|r^u@u0;@EFTB7h4 zXj5~QP1rwCamy1acgRWEN4vAUaK_|$=Z-waLq~Qp1>`7boyAd?YQ}z{TxB;IjJBSV zP@=clNNPx?%}uIlpw@=HjgAItX!W1)sqTOr$xTS1V5Q4-Afl8Qo+m5V8uf!nRHvX^ zx5w9(%Pc!gd%c55z2k#*N#{>H<;zi)+u_Eu>)}o3yLQ};C%gqr^PyY-|Dr=hyg!BL z;A;h*>h-8e_eVhtmM5XL;cdPml>$W05#rD#9-~=ksU&_Nxqw$-OfBcyPT(%BQZG?B z&8^6_47CX6R{FG;Qn8#839lFGwhT@;eUb>Rvd+2S*D$EKR7p9s=&iK6s$f7$&3VIr zt;y?PgZ(WV#m`@lJQ(kSzM6prk(`>y441mFz<`+MW83$;;?#@gJkvvP!ttJ@4evW` z`Nf{e*cF#Jh5B#pC?_g}8c70`+O@)t`j)02F>r_?FZp!VTO85NDp%eW#^@@xm;)jl zmqXq!ry4y}$s+^iD~(~|t=XWrkC2TfveCGG?m5+-d{yo?_kG9(`dinN66^mqYq~J5 zIAf3t1+R@_a(%yyO@*fNS5XR6i{T~_p(iu?m+Xy$GcAd%;(0bH5RgLVsdY|blG>OK zNa5AB-`=5|Iu<0Yd|e1>Ni_D~viq4knn$8hBZd{3ernItyno9pLlPRra+ zm(bMD$r60-Zzz5BKWV)OhR%2+|B@e|y0@X4y)f33m35!u1qcQn4 zEl6|6Y20gtI`iV6==_uMuuYj3#h+|nYjsN?5HS-u%3vNqx>NuaH$Gm=XYZ5=_Kx5o zh;mTnF>!*m2LenTMyognH-NS(X5RVmefqUQSN$vZ@V*o(=E2+3^#@cYKXA8VKA11+ z>1?0|Dd)&%RlR7+jO6oalW`X>`d#O1giEdQEkrz2H<~62R`F~X9(hS-uRsXH84hm{ z-B}O3(-`iU*vTxJsgpR@>4-(!M~^hflUxmWIoXapkIT zSgq$k$harn(8%;)n>Y&g?^}VZhX4&DZmFEb%17B9IqoV?WW<0lC~0Bmv<38X_%Y1O z5N8+jgOFAHAO6z)C>phguoA7dI1QhBI*p!>4EH8~c2p@~n%oJkp1uF5(dOs={HeNI zbzD5$Ies}OMk9}wlmH@%p09$Oy%Mw%)+m2n+I^#&rbSoSufN-^DuGeiC*b*A@RU-9 z1xt<55B>3`%vxMHUOjjhsLZK$bz=Kg%bOxWQczCn9E+Bb6SqvUUoNMz-in@uN%Zn1 zPdY*1cqjiA!DKX@t}6hj1;?->YIE^HDhot3cRJz)p(dHYsdM?G7Tgz-W5ua@)I!{S zB~4sZ!X@kobm$7dTljx%y)PL0mGu9pqlH&V1Xu`Xq(r7t_z5@HU}m@Rd&frZz=y{h zeQ|S&R-W+?ust&W`ZYuHve3%B!fP(J1^Lbov7e4bf7K(9RCzaom4kHChRhN@-FFHrTRJ~HBfjdBsKOmwLolSRGq{X_q<2PrP>c9am@_+ylWrhz1YJIc1Z71Tn z7k=GXzTNao%ajCs9`kjO?PJamHt$xhyz2TOL~%MG(vlHKC0$_C^iY;(lSy&2jYEE) zw@6NY!^{fTcG@0r+n%JsQ=zvY*e~J_L0QNeOK5@U(ppW*E%tg2Ui%c)Li@#J&eG^0 zwHpKQouEBziaV@yxbws^3Ds{_i;;v6>-v=PDs;NQJjE9hymVXD?c7o?Q@Zkpi{ssB zno2=9iRTc)76bF^xRxDi7|^Vzn9x%Uf~j{@{(u&p*GWkZ9y>4&BCKrXk@DavsJPW4 z!w&rLbA5G#%<3n37U|7}KHTsK+anR-`qq+2B(Ik{1x#$Pl?+a{)cEebmnKf}5IcEX zNfIFC6L5R=3%*k{9buj~3qy7V1YUP>aQ(mp1@&(I?f(KKY(9dNA;|d!IG)8q>JeD1 zTN4Nka;mDfQ3JlAzWkidZ@{`ISgwTS+LS!co0uTFy(U0%UnUhWTTysnn5IUN$Y{E4DdQF1Znp@A(i|T|LS)%8^Ybll8?A zQewx`mntaq$UA+o1=VbF`5xzvj)s@3^8ID&VB zAe>0Oo@w-S3%)!*7rw^2n4)pmiZ9dkEl)|ej9VQhIW^p0d3VHjJ zl^+_chBD9^t7FI(++d^MP;0c3cYMo|oi`d8exLJl?$ih#B|e_6AoHv%4#b1ei5X#h zr`Had*B-uH8%nS5E~M9O{ye3($mB`#r%^R33jB|WJDRAuao1J2|3bdeg_?qPYS_Ri zuMen?k@{UXea2!XE=x-zb2|=t{O`LhOzv0Ly|9HpSsOGa5DBO^(^(lM-inQ`D% z2!XrdE-18ThMVr!2+bn*k$3Y?s%nNI3dop2qg15a$GYCO`bX9ZeJhj7DN+92Xzx8v zS0r0f$DZVQ;>{_S-#!)Yqd5Zyzc)0)V69|u2O5POgtbKvZQCu-A%p^idPs{ZUFR)9 z0n$y>d>jkY4o(EJT_DJCl0|mDBnMZW*_m@$lj*rv3|!bqRyhu^KIwGhUDDi${m!Y5 zs{0vEw(+=A`NiSod9fgn+xw)&z5Cniqwx9GW4fy6#^m*CpJ*VG`_mTwED9% z-*Z2DUF)68N$gVqA^rE#aqBsDC$%GlVRM1HKh}l~=!B)T>)_`Vgf?2<6zBCE{}ObV z#|EaNZTEacvJ>*|uY_6rY0;@kQ7$va$GBooAa9ou`iF}wN4>RW?!WSNvGV6oK$XM5u1K#vm<2VxNT^}Otpg$EpklCrCf@Y5;l@TZq@HQ@_YQw`TZWh z-}nE|`}_S|zMt>q^?E(g<|vk-_pFu%{8$owLbsPjsSPq#)TB>9(eo(%=UYhQWb@nl z{OCM_SePCy;Eh~tRH)85422f{$@Ks`atr@Lo>FhU8$@eAG<}DrTRT zVBS-KGj)%!JZUxkxrI?#7dvvKYnK_*MKJ|Cu|Is8_)&S?oJ6SAL}`P z>6lNgSm{+I#L~dlx_iVFBtBy7^g{&-X>v#{tLA-f3nI`3uba{)2XAh_xNVYzTFkD&_F&;* zeG0aZWIcw0Nt(|xul?Ff7yT0iV>Lh@Q;cbm`Dq_|ZIh+navAfsxT(;^xO12>A3J(Y zX%{D%*Zl38RsEj8K*^fIln1Hm)a$qUgBtC@eJ@Maw09z-8%Vs7kp1Uq->4gf5btw-}LXyI11)-7uU_N|WW{>z7rx zBTp|O-$Oz`V@1_r9HMo-(OFYa}rMaa0bnYs=cXoEY9M3cenT24`qSme+r z82i`3K)@ zh`HCgn=Jg_GIjb@kpKuH;7uO9G)W7DcB9t(e#Z;=Xg!Yj+0L%T|AvIqE^D#b`;GWH zev}$3WJdet{it0j_hee{mX8)KBO59dZ>Mp!gW;hg@K!bYTNO}$`}3cRY4Lx|;<*ng zW7a@g`b7{{21pQWbQvhEzk7(JC)C~{ERE)FE- z9GFVf?yhhvk969)Hm{PHw$NIm!FjzfD6IlhPDe&AwyD>QCaQOO7%>iVk%5w0BGaS_WhwKV9}Ii1z4_k*c_2r zYPtXYtAW$kKfD{?Ib`KeQzE`m9 z&9`Wry)vjLe+afs64KgYm~!^pVxnt~=L1rK6>2)qeZmh>K~v??jKN#U-$?|d_V8LQ zPt8(5cQKW=uU*_=CZx-(0nUbG7ArCzbvVUV?t`I-89uz9x-aWm56Y8T4mUQoDJVp< zZ{&g~z0UHyLTxY!0w0o3$sLpwYxV}Iql?A~P#g)<_|+vQFAWk^iK;fz^vFz?UMIYe zalis9<26s@Yyw}3fkb_p#3nJR2)Kp*jVS}*0fJc{%Y}dDwhrUc51EgjP!PyAI zjRr~QfWgS_`-j6iLjwTVm;mp3Q@1$v_1_zG3N_HbT)w1attd8=4KcXz*Wes?p zTGNXcHIbD)XDph($vO%%L6Q!B9Tg>+V5$}X8pFSMNA|v8rT-NZmx5`!#i+5O*6VIV zgHZJt$~gbi?ls$4zqp^40y{&ndE7>kd95!f1YIyHaIgWg-rg#$9ovnl`pK?*qb+nB zh;PKkNDjLtp_%4g9|z_4Fg)W<_0EP$I!Y1>5A&Wr*a>>S_EaPilO8E zuY~ytr%TfpifEw|g8plISqsX~G7PjauyHmbSr=MSr!6OK;)271t6J*9)}F%kieqev&a>cgil{}Y-a)>Hrh diff --git a/static/docs_files_to_download/data-catalog/agent.model.schema.json b/static/docs_files_to_download/data-catalog/agent.model.schema.json deleted file mode 100644 index ef9d881511..0000000000 --- a/static/docs_files_to_download/data-catalog/agent.model.schema.json +++ /dev/null @@ -1,265 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "dc-agent-item", - "examples": [ - { - "ddlSchema": { - "properties": { - "FIRST_NAME": { - "nullable": true, - "size": 255, - "type": "VARCHAR2" - }, - "ID": { - "primary": true, - "type": "NUMBER", - "unique": true - }, - "INSERTED_ON": { - "primary": true, - "type": "TIMESTAMP(6)", - "unique": true - }, - "LAST_NAME": { - "nullable": true, - "size": 255, - "type": "VARCHAR2" - } - } - }, - "jsonSchema": { - "additionalProperties": false, - "properties": { - "FIRST_NAME": { - "type": "string" - }, - "ID": { - "type": "number" - }, - "INSERTED_ON": {}, - "LAST_NAME": { - "type": "string" - } - }, - "type": "object" - }, - "name": "books", - "source": { - "connection": { - "type": "odbc", - "vendor": "oracle", - "version": "Oracle Database 19c Enterprise Edition Release 19.0.0.0.0 - Production" - }, - "name": "oracle19c_test" - }, - "type": "model" - } - ], - "oneOf": [ - { - "type": "object", - "required": [ - "ddlSchema", - "jsonSchema", - "name", - "source", - "type" - ], - "properties": { - "ddlSchema": { - "$ref": "#/definitions/ddl-schema" - }, - "jsonSchema": { - "$ref": "#/definitions/json-schema" - }, - "name": { - "type": "string" - }, - "source": { - "$ref": "#/definitions/source-metadata" - }, - "type": { - "type": "string", - "enum": [ - "model" - ] - } - } - } - ], - "definitions": { - "ddl-property": { - "type": "object", - "required": [ - "type" - ], - "properties": { - "default": true, - "nullable": { - "type": "boolean" - }, - "precision": { - "type": [ - "integer", - "null" - ], - "format": "int32" - }, - "primary": { - "type": "boolean" - }, - "size": { - "type": [ - "integer", - "null" - ], - "format": "int32" - }, - "type": { - "$ref": "#/definitions/raw-type" - }, - "unique": { - "type": "boolean" - } - } - }, - "ddl-schema": { - "type": "object", - "required": [ - "properties" - ], - "properties": { - "properties": { - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/ddl-property" - } - } - } - }, - "json-primitive-type": { - "type": "string", - "enum": [ - "null", - "boolean", - "number", - "integer", - "string", - "array", - "object" - ] - }, - "json-property": { - "type": "object", - "properties": { - "default": true, - "nullable": { - "writeOnly": true, - "type": [ - "boolean", - "null" - ] - }, - "type": { - "anyOf": [ - { - "$ref": "#/definitions/json-type" - }, - { - "type": "null" - } - ] - } - } - }, - "json-schema": { - "type": "object", - "required": [ - "properties", - "type" - ], - "properties": { - "additionalProperties": { - "type": "boolean" - }, - "properties": { - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/json-property" - } - }, - "required": { - "type": "array", - "items": { - "type": "string" - } - }, - "type": { - "$ref": "#/definitions/json-primitive-type" - } - } - }, - "json-type": { - "anyOf": [ - { - "$ref": "#/definitions/json-primitive-type" - }, - { - "type": "array", - "items": { - "$ref": "#/definitions/json-primitive-type" - } - } - ] - }, - "raw-type": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - } - ] - }, - "source-connection-metadata": { - "type": "object", - "required": [ - "type", - "vendor" - ], - "properties": { - "type": { - "type": "string" - }, - "vendor": { - "type": "string" - }, - "version": { - "type": [ - "string", - "null" - ] - } - } - }, - "source-metadata": { - "type": "object", - "required": [ - "connection", - "name" - ], - "properties": { - "connection": { - "$ref": "#/definitions/source-connection-metadata" - }, - "name": { - "type": "string" - } - } - } - } -} \ No newline at end of file diff --git a/static/docs_files_to_download/data-catalog/catalog.model.schema.json b/static/docs_files_to_download/data-catalog/catalog.model.schema.json deleted file mode 100644 index 8576fbdaa6..0000000000 --- a/static/docs_files_to_download/data-catalog/catalog.model.schema.json +++ /dev/null @@ -1,367 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "OpenLineage", - "examples": [ - { - "producer": "my_producer", - "schemaURL": "https://openlineage.io/spec/2-0-2/OpenLineage.json", - "eventTime": "2024-06-27 10:43:30.889698872 UTC", - "dataset": { - "datasetId": "mock-id", - "name": "books", - "namespace": "test", - "schema": "my-schema", - "catalogName": "test", - "facets": { - "schema": { - "_producer": "my_producer", - "_schemaURL": "customSchemaDatasetFacet.json", - "jsonSchema": { - "type": "object", - "additionalProperties": false, - "properties": { - "FIRST_NAME": { - "type": "string" - }, - "ID": { - "type": "number" - }, - "INSERTED_ON": {}, - "LAST_NAME": { - "type": "string" - } - } - }, - "fields": [ - { - "name": "FIRST_NAME", - "type": "VARCHAR2", - "required": false, - "unique": false, - "primaryKey": false, - "nullable": false, - "sourceDescription": "size=255, precision=0" - }, - { - "name": "ID", - "type": "NUMBER", - "required": true, - "unique": true, - "primaryKey": true, - "nullable": false, - "sourceDescription": "size=0, precision=0" - }, - { - "name": "INSERTED_ON", - "type": "TIMESTAMP(6)", - "required": true, - "unique": true, - "primaryKey": true, - "nullable": false, - "sourceDescription": "size=0, precision=0" - }, - { - "name": "LAST_NAME", - "type": "VARCHAR2", - "required": false, - "unique": false, - "primaryKey": false, - "nullable": false, - "sourceDescription": "size=255, precision=0" - } - ] - }, - "version": { - "_producer": "my_producer", - "_schemaURL": "https://openlineage.io/spec/facets/1-0-1/DatasetVersionDatasetFacet.json", - "datasetVersion": "1" - }, - "storage": { - "_producer": "my_producer", - "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/StorageDatasetFacet.json", - "storageLayer": "oracle19c_test", - "version": "Oracle Database 19c Enterprise Edition Release 19.0.0.0.0 - Production", - "vendor": "mysql" - } - } - } - } - ], - "type": "object", - "required": [ - "dataset", - "eventTime", - "producer" - ], - "properties": { - "dataset": { - "$ref": "#/definitions/Dataset" - }, - "eventTime": { - "type": "string" - }, - "producer": { - "type": "string" - }, - "schemaURL": { - "default": "https://openlineage.io/spec/2-0-2/OpenLineage.json", - "type": "string" - } - }, - "definitions": { - "Dataset": { - "type": "object", - "required": [ - "datasetId", - "facets", - "name", - "namespace" - ], - "properties": { - "catalogName": { - "type": [ - "string", - "null" - ] - }, - "datasetId": { - "type": "string" - }, - "facets": { - "$ref": "#/definitions/Facets" - }, - "name": { - "type": "string" - }, - "namespace": { - "type": "string" - }, - "schema": { - "type": [ - "string", - "null" - ] - } - } - }, - "FacetSchema": { - "type": "object", - "required": [ - "_producer", - "_schemaURL", - "fields" - ], - "properties": { - "_producer": { - "type": "string" - }, - "_schemaURL": { - "type": "string" - }, - "fields": { - "type": "array", - "items": { - "$ref": "#/definitions/Field" - } - }, - "jsonSchema": { - "default": { - "type": "object", - "additionalProperties": false, - "properties": {} - }, - "allOf": [ - { - "$ref": "#/definitions/json-schema" - } - ] - } - } - }, - "FacetStorage": { - "type": "object", - "required": [ - "_producer", - "_schemaURL", - "storageLayer", - "vendor", - "version" - ], - "properties": { - "_producer": { - "type": "string" - }, - "_schemaURL": { - "type": [ - "string", - "null" - ] - }, - "storageLayer": { - "type": "string" - }, - "vendor": { - "type": "string" - }, - "version": { - "type": "string" - } - } - }, - "FacetVersion": { - "type": "object", - "required": [ - "_producer", - "_schemaURL", - "datasetVersion" - ], - "properties": { - "_producer": { - "type": "string" - }, - "_schemaURL": { - "type": "string" - }, - "datasetVersion": { - "type": "string" - } - } - }, - "Facets": { - "type": "object", - "required": [ - "schema", - "storage", - "version" - ], - "properties": { - "schema": { - "$ref": "#/definitions/FacetSchema" - }, - "storage": { - "$ref": "#/definitions/FacetStorage" - }, - "version": { - "$ref": "#/definitions/FacetVersion" - } - } - }, - "Field": { - "type": "object", - "required": [ - "name", - "nullable", - "primaryKey", - "required", - "sourceDescription", - "type", - "unique" - ], - "properties": { - "default": { - "type": [ - "string", - "null" - ] - }, - "name": { - "type": "string" - }, - "nullable": { - "type": "boolean" - }, - "primaryKey": { - "type": "boolean" - }, - "required": { - "type": "boolean" - }, - "sourceDescription": { - "description": "field description extracted from the source system", - "type": "string" - }, - "type": { - "type": "string" - }, - "unique": { - "type": "boolean" - } - } - }, - "json-primitive-type": { - "type": "string", - "enum": [ - "null", - "boolean", - "number", - "integer", - "string", - "array", - "object" - ] - }, - "json-property": { - "type": "object", - "properties": { - "default": true, - "nullable": { - "writeOnly": true, - "type": [ - "boolean", - "null" - ] - }, - "type": { - "anyOf": [ - { - "$ref": "#/definitions/json-type" - }, - { - "type": "null" - } - ] - } - } - }, - "json-schema": { - "type": "object", - "required": [ - "properties", - "type" - ], - "properties": { - "additionalProperties": { - "type": "boolean" - }, - "properties": { - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/json-property" - } - }, - "required": { - "type": "array", - "items": { - "type": "string" - } - }, - "type": { - "$ref": "#/definitions/json-primitive-type" - } - } - }, - "json-type": { - "anyOf": [ - { - "$ref": "#/definitions/json-primitive-type" - }, - { - "type": "array", - "items": { - "$ref": "#/definitions/json-primitive-type" - } - } - ] - } - } -} \ No newline at end of file diff --git a/static/docs_files_to_download/data-catalog/configuration.schema.json b/static/docs_files_to_download/data-catalog/configuration.schema.json deleted file mode 100644 index 7e2488eaf8..0000000000 --- a/static/docs_files_to_download/data-catalog/configuration.schema.json +++ /dev/null @@ -1,877 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "configuration", - "examples": [ - { - "connections": { - "postgres-16": { - "configuration": { - "params": { - "database": "my-database", - "host": "0.0.0.0", - "port": 5432, - "pwd": "*******", - "uid": "my-user" - }, - "vendor": "postgres" - }, - "type": "odbc" - } - }, - "target": { - "baseUrl": "https://console.cloud.mia-platform.eu", - "credentials": { - "clientId": "", - "clientKeyId": "", - "privateKey": "tls/private-key.pem" - }, - "oauthTokenEndpoint": "/api/m2m/oauth/token", - "projectId": "1234", - "revision": "main", - "type": "mia-console" - } - }, - { - "connections": { - "crud-service": { - "configuration": { - "params": { - "baseUrl": "http://mia-crud-service:3000", - "endpoint": "/-/schemas", - "headers": { - "accept": "application/x-ndjson" - } - }, - "vendor": "mia-crud-service" - }, - "type": "http" - } - }, - "target": { - "baseUrl": "https://console.cloud.mia-platform.eu", - "credentials": { - "clientId": "", - "clientKeyId": "", - "privateKey": "tls/private-key.pem" - }, - "oauthTokenEndpoint": "/api/m2m/oauth/token", - "projectId": "1234", - "revision": "main", - "type": "mia-console" - } - } - ], - "type": "object", - "properties": { - "connections": { - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/connection" - } - }, - "target": { - "default": { - "type": "stdout" - }, - "allOf": [ - { - "$ref": "#/definitions/target" - } - ] - } - }, - "definitions": { - "OracleMode": { - "oneOf": [ - { - "type": "string", - "enum": [ - "user" - ] - }, - { - "type": "object", - "required": [ - "all" - ], - "properties": { - "all": { - "type": "object", - "properties": { - "owners": { - "type": [ - "array", - "null" - ], - "items": { - "type": "string" - } - } - } - } - }, - "additionalProperties": false - }, - { - "type": "object", - "required": [ - "dba" - ], - "properties": { - "dba": { - "type": "object", - "properties": { - "owners": { - "type": [ - "array", - "null" - ], - "items": { - "type": "string" - } - } - } - } - }, - "additionalProperties": false - } - ] - }, - "Settings": { - "type": "object", - "properties": { - "batchSize": { - "type": [ - "integer", - "null" - ], - "format": "uint32", - "minimum": 0.0 - }, - "columnBatchSize": { - "type": [ - "integer", - "null" - ], - "format": "uint32", - "minimum": 0.0 - }, - "connectionName": { - "type": [ - "string", - "null" - ] - } - } - }, - "connection": { - "oneOf": [ - { - "type": "object", - "required": [ - "configuration", - "type" - ], - "properties": { - "configuration": { - "$ref": "#/definitions/odbc-configuration" - }, - "type": { - "type": "string", - "enum": [ - "odbc" - ] - } - } - }, - { - "type": "object", - "required": [ - "configuration", - "type" - ], - "properties": { - "configuration": { - "$ref": "#/definitions/http-configuration" - }, - "type": { - "type": "string", - "enum": [ - "http" - ] - } - } - } - ] - }, - "credentials": { - "type": "object", - "required": [ - "clientId", - "clientKeyId", - "privateKey" - ], - "properties": { - "clientId": { - "type": "string" - }, - "clientKeyId": { - "$ref": "#/definitions/secret" - }, - "privateKey": { - "$ref": "#/definitions/secret" - } - } - }, - "http-configuration": { - "oneOf": [ - { - "type": "object", - "required": [ - "params", - "vendor" - ], - "properties": { - "params": { - "$ref": "#/definitions/mia-crud-service-params" - }, - "vendor": { - "type": "string", - "enum": [ - "mia-crud-service" - ] - } - } - }, - { - "type": "object", - "required": [ - "params", - "vendor" - ], - "properties": { - "params": { - "$ref": "#/definitions/salesforce-params" - }, - "vendor": { - "type": "string", - "enum": [ - "salesforce-sobjects" - ] - } - } - } - ] - }, - "mia-crud-service-params": { - "type": "object", - "required": [ - "baseUrl" - ], - "properties": { - "baseUrl": { - "type": "string" - }, - "endpoint": { - "default": "/-/schemas", - "type": "string" - }, - "headers": { - "default": { - "accept": "application/x-ndjson" - }, - "type": "object", - "additionalProperties": { - "type": "string" - } - }, - "healthcheck": { - "default": "/-/healthz", - "type": "string" - }, - "tls": { - "default": { - "insecure": false - }, - "allOf": [ - { - "$ref": "#/definitions/tls-params" - } - ] - } - } - }, - "mssql-connection-params": { - "examples": [ - { - "database": "db", - "flags": "TrustServerCertificate=yes;", - "host": "0.0.0.0", - "port": 1433, - "pwd": "*****", - "uid": "user" - }, - "Driver=mssql;Server=0.0.0.0,1433;Database=db;Uid=user;Pwd=p4ssw0rd;TrustServerCertificate=yes;" - ], - "anyOf": [ - { - "$ref": "#/definitions/secret" - }, - { - "type": "object", - "required": [ - "database", - "pwd", - "uid" - ], - "properties": { - "database": { - "type": "string" - }, - "driver": { - "type": "string" - }, - "flags": { - "type": [ - "string", - "null" - ] - }, - "host": { - "default": "0.0.0.0", - "type": "string" - }, - "port": { - "default": 1433, - "type": "integer", - "format": "uint16", - "minimum": 0.0 - }, - "pwd": { - "$ref": "#/definitions/secret" - }, - "uid": { - "$ref": "#/definitions/secret" - } - } - } - ] - }, - "mysql-connection-params": { - "examples": [ - { - "database": "db", - "host": "0.0.0.0", - "port": 3306, - "pwd": "*****", - "uid": "user" - }, - "DRIVER=mysql;SERVER=0.0.0.0;PORT=3306;DATABASE=db;UID=user;PWD=p4ssw0rd;" - ], - "anyOf": [ - { - "$ref": "#/definitions/secret" - }, - { - "type": "object", - "required": [ - "database", - "pwd", - "uid" - ], - "properties": { - "database": { - "type": "string" - }, - "driver": { - "type": "string" - }, - "flags": { - "type": [ - "string", - "null" - ] - }, - "host": { - "default": "0.0.0.0", - "type": "string" - }, - "port": { - "default": 3306, - "type": "integer", - "format": "uint16", - "minimum": 0.0 - }, - "pwd": { - "$ref": "#/definitions/secret" - }, - "uid": { - "$ref": "#/definitions/secret" - } - } - } - ] - }, - "odbc-configuration": { - "oneOf": [ - { - "type": "object", - "required": [ - "params", - "vendor" - ], - "properties": { - "params": { - "$ref": "#/definitions/mssql-connection-params" - }, - "vendor": { - "type": "string", - "enum": [ - "mssql" - ] - } - } - }, - { - "type": "object", - "required": [ - "params", - "vendor" - ], - "properties": { - "params": { - "$ref": "#/definitions/mysql-connection-params" - }, - "vendor": { - "type": "string", - "enum": [ - "mysql" - ] - } - } - }, - { - "type": "object", - "required": [ - "params", - "vendor" - ], - "properties": { - "options": { - "default": "user", - "allOf": [ - { - "$ref": "#/definitions/OracleMode" - } - ] - }, - "params": { - "$ref": "#/definitions/oracle-connection-params" - }, - "vendor": { - "type": "string", - "enum": [ - "oracle" - ] - }, - "version": { - "anyOf": [ - { - "$ref": "#/definitions/version" - }, - { - "type": "null" - } - ] - } - } - }, - { - "type": "object", - "required": [ - "params", - "vendor" - ], - "properties": { - "params": { - "$ref": "#/definitions/postgres-connection-params" - }, - "vendor": { - "type": "string", - "enum": [ - "postgres" - ] - } - } - } - ] - }, - "oracle-connection-params": { - "examples": [ - { - "dbq": "ORACLE_TEST", - "uid": "user", - "pwd": "****" - }, - { - "dbq": "(DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)(HOST=0.0.0.0)(PORT=1521))(CONNECT_DATA=(SERVER=DEDICATED)(SERVICE_NAME=ORACLE19C_TEST)))", - "flags": "DBA=R;", - "uid": "user", - "pwd": "****" - }, - "DRIVER=oracle;UID=user;PWD=p4ssw0rd;DBQ=(DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)(HOST=0.0.0.0)(PORT=1521))(CONNECT_DATA=(SERVER=DEDICATED)(SERVICE_NAME=ORACLE19C_TEST)))" - ], - "anyOf": [ - { - "$ref": "#/definitions/secret" - }, - { - "type": "object", - "required": [ - "dbq", - "pwd", - "uid" - ], - "properties": { - "dbq": { - "type": "string" - }, - "driver": { - "type": "string" - }, - "flags": { - "type": [ - "string", - "null" - ] - }, - "pwd": { - "$ref": "#/definitions/secret" - }, - "uid": { - "$ref": "#/definitions/secret" - } - } - } - ] - }, - "postgres-connection-params": { - "examples": [ - { - "database": "db", - "host": "0.0.0.0", - "port": 5432, - "pwd": "*****", - "uid": "user" - }, - "DRIVER=postgres;SERVER=0.0.0.0;PORT=5432;DATABASE=db;UID=user;PWD=p4ssw0rd;" - ], - "anyOf": [ - { - "$ref": "#/definitions/secret" - }, - { - "type": "object", - "required": [ - "database", - "pwd", - "uid" - ], - "properties": { - "database": { - "type": "string" - }, - "driver": { - "type": "string" - }, - "flags": { - "type": [ - "string", - "null" - ] - }, - "host": { - "default": "0.0.0.0", - "type": "string" - }, - "port": { - "default": 5432, - "type": "integer", - "format": "uint16", - "minimum": 0.0 - }, - "pwd": { - "$ref": "#/definitions/secret" - }, - "uid": { - "$ref": "#/definitions/secret" - } - } - } - ] - }, - "salesforce-params": { - "oneOf": [ - { - "type": "object", - "required": [ - "apiVersion", - "authenticationFlow", - "baseUrl", - "clientId", - "clientSecret", - "password", - "securityToken", - "username" - ], - "properties": { - "apiVersion": { - "type": "string" - }, - "authenticationFlow": { - "type": "string", - "enum": [ - "username-password" - ] - }, - "baseUrl": { - "type": "string" - }, - "clientId": { - "$ref": "#/definitions/secret" - }, - "clientSecret": { - "$ref": "#/definitions/secret" - }, - "password": { - "$ref": "#/definitions/secret" - }, - "securityToken": { - "$ref": "#/definitions/secret" - }, - "username": { - "$ref": "#/definitions/secret" - } - } - }, - { - "type": "object", - "required": [ - "apiVersion", - "authenticationFlow", - "baseUrl", - "clientId", - "privateKey", - "username" - ], - "properties": { - "apiVersion": { - "type": "string" - }, - "authenticationFlow": { - "type": "string", - "enum": [ - "jwt-bearer" - ] - }, - "baseUrl": { - "type": "string" - }, - "clientId": { - "$ref": "#/definitions/secret" - }, - "loginUrl": { - "default": "https://login.salesforce.com", - "type": "string" - }, - "privateKey": { - "$ref": "#/definitions/secret" - }, - "username": { - "$ref": "#/definitions/secret" - } - } - } - ] - }, - "secret": { - "oneOf": [ - { - "type": "string" - }, - { - "type": "object", - "required": [ - "key", - "type" - ], - "properties": { - "encoding": { - "type": "string", - "enum": [ - "base64" - ] - }, - "key": { - "type": "string" - }, - "type": { - "const": "env" - } - } - }, - { - "type": "object", - "required": [ - "path", - "type" - ], - "properties": { - "encoding": { - "type": "string", - "enum": [ - "base64" - ] - }, - "key": { - "type": "string" - }, - "path": { - "type": "string" - }, - "type": { - "const": "file" - } - } - } - ] - }, - "target": { - "examples": [ - { - "type": "stdout" - }, - { - "dir": "/path/to/dir", - "type": "file" - }, - { - "baseUrl": "https://console.cloud.mia-platform.eu", - "credentials": { - "clientId": "", - "clientKeyId": "", - "privateKey": "tls/private-key.pem" - }, - "dataCatalogEndpoint": "/custom", - "oauthTokenEndpoint": "/api/m2m/oauth/token", - "type": "mia-console" - } - ], - "oneOf": [ - { - "type": "object", - "anyOf": [ - { - "type": "object", - "required": [ - "projectId" - ], - "properties": { - "projectId": { - "type": "string" - }, - "revision": { - "default": "main", - "type": "string" - } - } - }, - { - "type": "object", - "required": [ - "dataCatalogEndpoint" - ], - "properties": { - "dataCatalogEndpoint": { - "type": "string" - } - } - } - ], - "required": [ - "baseUrl", - "credentials", - "type" - ], - "properties": { - "baseUrl": { - "type": "string" - }, - "credentials": { - "$ref": "#/definitions/credentials" - }, - "headers": { - "type": "object", - "additionalProperties": { - "type": "string" - } - }, - "oauthTokenEndpoint": { - "default": "/api/m2m/oauth/token", - "type": "string" - }, - "type": { - "type": "string", - "enum": [ - "mia-console" - ] - } - } - }, - { - "type": "object", - "required": [ - "type" - ], - "properties": { - "dir": { - "type": [ - "string", - "null" - ] - }, - "type": { - "type": "string", - "enum": [ - "file" - ] - } - } - }, - { - "type": "object", - "required": [ - "type" - ], - "properties": { - "type": { - "type": "string", - "enum": [ - "stdout" - ] - } - } - } - ] - }, - "version": { - "type": "string", - "enum": [ - "21", - "19", - "12", - "11" - ] - } - } -} \ No newline at end of file From b7ab6e2b103f2e62b07b15c80271fe1c53d8ebcd Mon Sep 17 00:00:00 2001 From: Bot-targa Date: Fri, 27 Sep 2024 18:11:34 +0200 Subject: [PATCH 10/11] chore(docs): update Data Catalog documentation From 28b467c6097ac172399291ba7f5c0133265e8f4b Mon Sep 17 00:00:00 2001 From: Bot-targa Date: Fri, 27 Sep 2024 18:11:38 +0200 Subject: [PATCH 11/11] chore(docs): update Data Catalog documentation --- .../data-catalog/10_overview.md | 25 + .../data-catalog/20_configuration.md | 803 ++++++++++++++++ .../data-catalog/_category_.json | 4 + .../data-catalog/changelog.md | 99 ++ .../data-catalog/img/architecture.png | Bin 0 -> 19519 bytes .../data-catalog/agent.model.schema.json | 265 ++++++ .../data-catalog/catalog.model.schema.json | 367 ++++++++ .../data-catalog/configuration.schema.json | 877 ++++++++++++++++++ 8 files changed, 2440 insertions(+) create mode 100644 docs/runtime_suite_templates/data-catalog/10_overview.md create mode 100644 docs/runtime_suite_templates/data-catalog/20_configuration.md create mode 100644 docs/runtime_suite_templates/data-catalog/_category_.json create mode 100644 docs/runtime_suite_templates/data-catalog/changelog.md create mode 100644 docs/runtime_suite_templates/data-catalog/img/architecture.png create mode 100644 static/docs_files_to_download/data-catalog/agent.model.schema.json create mode 100644 static/docs_files_to_download/data-catalog/catalog.model.schema.json create mode 100644 static/docs_files_to_download/data-catalog/configuration.schema.json diff --git a/docs/runtime_suite_templates/data-catalog/10_overview.md b/docs/runtime_suite_templates/data-catalog/10_overview.md new file mode 100644 index 0000000000..43e1f1db6c --- /dev/null +++ b/docs/runtime_suite_templates/data-catalog/10_overview.md @@ -0,0 +1,25 @@ +--- +id: overview +title: Overview +sidebar_label: Overview +--- + + + +_Data Catalog Agent_ is command line interface binary that queries datasources for the data schemas of the resources they own and sends them to a target. In the context of databases, it provides facilities to: + +1. query tables schemas +2. aggregate them in a unique asset (JSON format) +3. send it to an HTTP server or store it locally on file + +_Data Catalog Agent_ serves as a driver for collecting and browsing, in a unified view, tables/collections persisted in different sources (System of Record, a.k.a SoR). + +![architecture](img/architecture.png) + +_Mia Console_ allows to intercept the _Data Catalog Agent_ output assets and visualize them in a dedicated web view. _Mia Console_ is one of the configurable targets of _Data Catalog Agent_. + +_Data Catalog Agent_ is well suited for running locally or configured in _Mia Console_ as a [CronJob](/development_suite/api-console/api-design/jobs-cronjob.md). diff --git a/docs/runtime_suite_templates/data-catalog/20_configuration.md b/docs/runtime_suite_templates/data-catalog/20_configuration.md new file mode 100644 index 0000000000..0157aff189 --- /dev/null +++ b/docs/runtime_suite_templates/data-catalog/20_configuration.md @@ -0,0 +1,803 @@ +--- +id: configuration +title: Configuration +sidebar_label: Configuration +--- + + + +Due to licensing _Data Catalog Agent_ requires packaging with third-party software, unless your datasources are PostgreSQL databases whose support is already included in the docker image of _Data Catalog Agent_. + +In case only PostgreSQL is needed you can skip to the [cronjob configuration](#cronjob-configuration) section. + +## Datasource support + +Mia's _Data Catalog_ marketplace item upon creation, creates also a git repository containing: + +1. a CI config file +2. a docker `Dockerfile` + +the `Dockerfile` provides several build steps to include the datasource support you need. Supported datasources are: + +- Oracle Database from 11 to 21 +- PostgreSQL from 9.6 to 16 +- Mysql from 5.7 to 8.2 +- Mssql from 2017 to 2022 + +Upon inspection you'll notice that the `Dockerfile` contains targets for each database: + +```Dockerfile +FROM ... AS data-catalog-agent-oracle + +# rest of the file + +FROM ... AS data-catalog-agent-mysql + +# and so on... +``` + +Each push on the default branch or any tag starting with `v` will create a new docker image including the selected support. The default support is `postgres` but it can be changed: + +1. by editing the `.gitlab-ci.yml` file overriding the initial value of the `DOCKER_TARGET` from `postgres` to the database you need to support +2. one-of change upon manually triggering the CI overriding the same variable `DOCKER_TARGET` + +Your repository contains a `README.md` file which also contains a lists of operations needed to include drivers for `oracle`, `mysql`, `mssql` databases. + +### Oracle Database: add support + +**In the template repository**, to enable Oracle Database support actions are needed: + +1. edit the value of `DOCKER_TARGET` variable in `.gitlab-ci.yml` to `oracle` or alternatively select it on manual run +2. embed oracle drivers (see [embed drivers](#embed-drivers)) +3. [optional] add `tnsnames.ora` and/or `Oracle Wallet` (see [add tns/wallet](#add-tns-support)). + +#### Embed drivers + +**In the template repository**, download drivers from the [official page](https://www.oracle.com/database/technologies/instant-client/linux-x86-64-downloads.html) according with the version you need. + +The following assets are needed: + +1. ODBC driver +2. instantclient basiclite + +```shell +# on unix +export ORACLE_CLIENT_VERSION="" +unzip -j instantclient-basiclite-linux.x64-${ORACLE_CLIENT_VERSION}dbru.zip "instantclient_*/lib*" -d lib/x86_64-unknown-linux-gnu/oracle +unzip -j instantclient-odbc-linux.x64-${ORACLE_CLIENT_VERSION}dbru.zip "instantclient_*/lib*" -d lib/x86_64-unknown-linux-gnu/oracle +``` + +create a symlink + +```shell +export LIBSQORA_NAME=`ls lib/x86_64-unknown-linux-gnu/oracle | grep -E -i '^libsqora\.so\.'` +ln -s "${LIBSQORA_NAME}" lib/x86_64-unknown-linux-gnu/oracle/libsqora.so +``` + +#### Add TNS support + +Oracle connectivity can be handled with `tnsnames.ora` files and/or wallets. The container expects such assets in the `/home/agent/oracle/admin` folder. The `TNS_ADMIN` is already set to `/home/agent/oracle/admin` and the container will copy the content of the `oracle/admin` folder in this repository. + +Drop your wallet assets or `tnsnames.ora` file in `oracle/admin` to include them. + +In case you want to use a wallet remind to [tune] the `sqlora.net` file registering the `/home/agent/oracle/admin` path: + +> WALLET_LOCATION = (SOURCE = (METHOD = file) (METHOD_DATA = (DIRECTORY="/home/agent/oracle/admin"))) +> SSL_SERVER_DN_MATCH=yes + +### MySQL: add support + +**In the template repository**, ODBC driver for MySQL must be downloaded from the [official website](https://dev.mysql.com/downloads/connector/odbc/) +selecting: + +1. version `8.3.0` +2. OS `Linux - Generic` +3. OS version `glibc 2.28 x86_64` + +then run: + +```shell +export MYSQL_NAME=mysql-connector-odbc-8.3.0-linux-glibc2.28-x86-64bit +tar -xvf ${MYSQL_NAME}.tar.gz ${MYSQL_NAME}/lib/libmyodbc8w.so +mv ${MYSQL_NAME}/lib/* lib/x86_64-unknown-linux-gnu/mysql +``` + +### MS SQL Server: add support + +No further actions beside editing `.gitlab-ci.yml` are required. + +### Mia CRUD Service: add support + +By default, running the CI of this project produces an artifact with Mia CRUD Service support since it does not require any specific drivers. + +### Salesforce SObjects API: add support + +By default, running the CI of this project produces an artifact with Salesforce SObjects support since it does not require any specific drivers. + +## Cronjob Configuration + +From your project select `CronJobs` > `Create new CronJob`, insert a unique name for the job, an optional description, the full name of the docker image created by your CI job. It will come in the form `/data-catalog/agent:`. Also insert a `CronJob schedule` using the [k8s syntax](https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/#writing-a-cronjob-spec). + +By clicking on create, you need to configure: + +1. the config map containing the configuration of the _Data Catalog Agent_ +2. the cronjob yaml file + +Create a new config map giving it a name of your choice (e.g., `data-catalog-agent-configuration`) and mount path `/home/agent/.config/dc/agent/`. We can now add it to the `CronJob` yaml manifest. + +The prompted yaml for the `CronJob` should be edited as follows: + +```yaml +apiVersion: batch/v1 +kind: CronJob +metadata: + ... +spec: + ... + jobTemplate: + spec: + ... + template: + ... + spec: + ... + containers: + containers: + - name: ... + ... + env: + - name: TNS_ADMIN # not required if it defaults to /home/agent/oracle/admin or no oracle support is needed + value: /path/to/tns/admin/folder + args: ["scan"] # add args here + volumeMounts: + - name: data-catalog-agent-configuration # reference a volume mount + mountPath: /home/agent/.config/dc/agent/ + volumes: + - name: data-catalog-agent-configuration # use the name given to the config map + configMap: + name: data-catalog-agent-configuration +``` + +Now create a new file in the config map called `configuration.json`. This file must be a valid _Data Catalog Agent_ configuration. + +## Data Catalog Agent Configuration + +:::caution +The configuration is read from a file which must be located ad `~/.config/dc/agent/configuration.json` where `~` is the user's home folder (`$HOME`). Such location can be overridden via the environment variable `DC_AGENT_CONFIGURATION_FILEPATH` which must contain a filepath to a valid configuration file. +::: + +The configuration has the following main sections: + +1. connections +2. target + +```json +{ + "connections": { + ... + }, + "target": { + ... + } +} +``` + +Secretable fields are marked in the following sections. + +## Connections + +A list of entries to connect to datasources via supported integration methods. The general recipe is: + +```json +{ + "connections": { + "": { + "type": "", + "configuration": { + // depends on the integration method + }, + "settings": { + // depends on the integration method + } + } + } +} +``` + +Available integration methods are: + +- [`odbc`](https://en.wikipedia.org/wiki/Open_Database_Connectivity) +- `http` + +Such drivers come often in the form of a [`dynamic shared object`](https://en.wikipedia.org/wiki/Shared_library) and must be embedded with the binary and available at runtime (see the template repository README file). + +A non-required object `settings` can be added to the connection configuration to define the following additional properties: + - `namespace`: string that overrides the default connection name; + - `batchSize`: number that defines the number of tables that can be paginated in a odbc query. if not specified, 1000 tables for query will be retrieved. + - `columnBatchSize`: number that defines the number of columns that are expected to be processed by each batch iteration. if not specified, 20 columns for batch will be used. + +### PostgreSQL + +To configure a **PostgreSQL** ODBC connection use: + +```json +{ + "connections": { + "": { // 👈 pick a name + "type": "odbc", + "configuration": { + "vendor": "postgres", + "params": { + "uid": "test_user", + "pwd": "password", + "database": "" + } + } + } + } +} +``` + +or use an inline ODBC connection string + +```json +{ + "connections": { + "": { // 👈 pick a name + "type": "odbc", + "configuration": { + "vendor": "postgres", + "params": "DRIVER=postgres;SERVER=0.0.0.0;PORT=5432;DATABASE=db;UID=user;PWD=pwd;" + } + } + } +} +``` + +Other keys are `host` and `port` which for a **PostgreSQL** connection are defaulted to `0.0.0.0` and `5432`. Any other configuration parameter can be appended using the key `flags` which must be a semicolon separated string. + +#### Secretable fields + +`uid`, `pwd` or `params` support [secrets resolution](/fast_data/configuration/secrets_resolution.md). + +### Oracle + +To configure an **Oracle** ODBC connection use: + +```json +{ + "connections": { + "": { // 👈 pick a name + "type": "odbc", + "configuration": { + "vendor": "oracle", + "params": { + "uid": "test_user", + "pwd": "password", + "dbq": "" + } + } + } + } +} +``` + +or use an inline ODBC connection string and also set the TNS_ADMIN environment variable +to access a `tnsnames.ora` file where the DBQ name can be resolved + +```json +{ + "connections": { + "": { // 👈 pick a name + "type": "odbc", + "configuration": { + "vendor": "oracle", + "params": { + "uid": "test_user", + "pwd": "password", + // 👇 this must be defined in your tnsnames.ora file + "dbq": "DRIVER=oracle;UID=user;PWD=p4ssw0rd;DBQ=DATABASE_NAME" + } + } + } + } +} +``` + +`SERVICE_NAME` is an entry of your [`tnsnames.ora`](https://docs.oracle.com/en/database/oracle/oracle-database/18/netrf/local-naming-parameters-in-tnsnames-ora-file.html#GUID-12C94B15-2CE1-4B98-9D0C-8226A9DDF4CB) file. Such file **MUST** be available when running the binary and its location **MUST** be exported to an environment variable unless defaulted to `/home/agent/oracle/admin`. + +in case you don't want to add the environment variable, the content of one entry of the `tnsnames.ora` file can be inlined such as: + +```json +{ + "connections": { + "": { + "type": "odbc", + "configuration": { + "vendor": "oracle", + "params": { + ..., + "dbq": "(DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)(HOST=0.0.0.0)(PORT=5041))(CONN ..." + } + } + } + } +} +``` + +Other keys are: + +- `version` which can be neglected **UNLESS USING ORACLE 11g** since it [does not support](https://docs.oracle.com/database/121/LNPLS/release_changes.htm#LNPLS113) pagination via `OFFSET ? ROWS FETCH FIRST ? ROWS ONLY`. + + :::caution + + `version` MUST be explicit at `configuration` level. + ```json + { + "type": "odbc", + "configuration": { + "vendor": "oracle", + "version": 11, + "params": { /** ... */ } + } + } + ``` + ::: + +- `flags` which will be added, as a string, to the connection like `"flags": "DBA=R;"`, for the available flags check [oracle documentation: 25.4.1 Format of the Connection String](https://docs.oracle.com/en/database/oracle/oracle-database/21/adfns/odbc-driver.html#GUID-618B141E-DD46-4907-99C2-486E801CA878). + +- `options` field for _oracle_ odbc configurations: this field defines the tables that will be used for retrieve the metadata. + If not specified, `user` option will be used as default. Three options available: + + - `user`: tables `user_tables`, `user_views` and `user_mviews` will be used. No filter needs to be specified since they default to the user's schema. + ```json + { + "type": "odbc", + "configuration": { + "vendor": "oracle", + "params": { /** ... */ }, + "options": { + "user" + } + } + } + ``` + - `all`: tables `all_tables`, `all_views` and `all_mviews` will be used. Optionally, a list of schemas needs to be provided in `owners` field to filter the desired schemas. + ```json + { + "type": "odbc", + "configuration": { + "vendor": "oracle", + "params": { /** ... */ }, + "options": { + "all": { + "owners": ["some_schema_name"] + } + } + } + } + ``` + - `dba`: tables `dba_tables`, `dba_views` and `dba_mviews` will be used. Optionally, a list of schemas needs to be provided in `owners` field to filter the desired schemas. + ```json + { + "type": "odbc", + "configuration": { + "vendor": "oracle", + "params": { /** ... */ }, + "options": { + "dba": { + "owners": ["some_schema_name"] + } + } + } + } + ``` + + :::caution + + In this case, remember that password needs to be written in the form of ` AS SYSDBA`. + + ::: + +Connection with an [oracle wallet](https://docs.oracle.com/en/database/oracle/oracle-database/19/dbimi/using-oracle-wallet-manager.html#GUID-D0AA8373-B0AC-4DD8-9FA9-403E345E5A71) is also [supported](https://docs.oracle.com/en/cloud/paas/autonomous-database/serverless/adbsb/connect-prepare-oci-wallets.html#GUID-EFAFA00E-54CC-47C7-8C71-E7868279EF3B): + +Preparation must be done on the template repository: + +1. The whole wallet content must be available under the same path of the `tnsnames.ora` (they usually come together) +2. The file `sqlora.net` inside the wallet must be updated to reflect the `TNS_ADMIN` path + > WALLET_LOCATION = (SOURCE = (METHOD = file) (METHOD_DATA = (DIRECTORY="/path/to/folder"))) + > SSL_SERVER_DN_MATCH=yes + +Also the environment variable must be set: + +1. The `TNS_ADMIN` environment variable must be set **explicitly** unless defaulted to `/home/agent/oracle/admin` + +#### Secretable fields + +`uid`, `pwd` or `params` support [secrets resolution](/fast_data/configuration/secrets_resolution.md). + +### MS SQL server + +To configure a **MS SQL Server** ODBC connection use: + +```json +{ + "connections": { + "": { + "type": "odbc", + "configuration": { + "vendor": "mssql", + "params": { + "uid": "test_user", + "pwd": "password", + "database": "" + } + } + } + } +} +``` + +or use an inline ODBC connection string: + +```json +{ + "connections": { + "": { + "type": "odbc", + "configuration": { + "vendor": "mssql", + "params": "Driver=mssql;Server=0.0.0.0,1433;Database=db;Uid=user;Pwd=p4ssw0rd;TrustServerCertificate=yes;" + } + } + } +} +``` + +Other keys are `host` and `port` which for a **PostgreSQL** connection are defaulted to `0.0.0.0` and `1433`. Any [extra connection property](https://learn.microsoft.com/en-us/sql/connect/odbc/dsn-connection-string-attribute?view=sql-server-ver16) can be added via the key `flags` which will be added, as a string, to the connection, as a semicolon separated string. It is quite useful, for local development purposes to add the flag `"TrustServerCertificate=yes"`. + +#### Secretable fields + +`uid`, `pwd` or `params` support [secrets resolution](/fast_data/configuration/secrets_resolution.md). + +### MySQL + +To configure a **MySQL** ODBC connection use: + +```json +{ + "connections": { + "": { + "type": "odbc", + "configuration": { + "vendor": "mysql", + "params": { + "uid": "test_user", + "pwd": "password", + "database": "" + } + } + } + } +} +``` + +or use an inline ODBC connection string: + +```json +{ + "connections": { + "": { + "type": "odbc", + "configuration": { + "vendor": "mysql", + "params": "DRIVER=mysql;SERVER=0.0.0.0;PORT=3306;DATABASE=db;UID=user;PWD=p4ssw0rd;" + } + } + } +} +``` + +Other keys are `host` and `port` which for a **PostgreSQL** connection are defaulted to `0.0.0.0` and `3306`. Any [extra connection property](https://dev.mysql.com/doc/connector-odbc/en/connector-odbc-configuration-connection-parameters.html) can be added via the key `flags` which will be added, as a string, to the connection, as a semicolon separated string. + +#### Secretable fields + +`uid`, `pwd` or `params` support [secrets resolution](/fast_data/configuration/secrets_resolution.md). + +### Mia CRUD Service + +To configure a **Mia CRUD Service** connection use: + +```json +{ + "connections": { + "": { + "type": "http", + "configuration": { + "vendor": "mia-crud-service", + "params": { + "baseUrl": "http://mia-crud-service:3000", + "endpoint": "/-/schemas", + "healthcheck": "/-/healthz", + "headers": { + "accept": "application/x-ndjson" + } + } + } + } + } +} +``` + +The driver basically calls the `/-/schemas` URL to extract all the data models from MongoDB. Since the response is, of course, a JSON Schema, some information in the `ddl` object may be missing, like the `size` of the properties. + +Additionally, the version of the CRUD Service is obtained from the `/-/healthz` endpoint. + +In case custom `tls` parameters are required such as a custom root CA or an insecure HTTPS connection, +use the parameter `tls` with keys `insecure` and/or `certificate`: + +```json +{ + "connections": { + "": { + "type": "http", + "configuration": { + ..., + "params": { + "tls": { + "insecure": true + } + } + } + } + } +} +``` + +or + +```json +{ + "connections": { + "": { + "type": "http", + "configuration": { + ..., + "params": { + "tls": { + "certificate": { + "type": "file", + "path": "/path/to/root/CA" + } + } + } + } + } + } +} +``` + +### Salesforce SObjects + +To configure a **Salesforce SObjects** HTTP connection you can use two authentication methods: + +* [jwt-bearer](https://help.salesforce.com/s/articleView?id=sf.remoteaccess_oauth_jwt_flow.htm&type=5): + ```json + { + "connections": { + "": { + "type": "http", + "configuration": { + "vendor": "salesforce-sobjects", + "params": { + "authenticationFlow": "jwt-bearer", + "baseUrl": "https://my-subdomain.my.salesforce.com", + "clientId": "XXXXXXXXXXXXXXXXXXXXXXXXXX", + "privateKey": "/path/to/private-key/key.pem", + "username": "my-user@email.com", + "apiVersion": "59.0" + } + } + } + } + } + ``` +* [username-password](https://help.salesforce.com/s/articleView?id=sf.remoteaccess_oauth_username_password_flow.htm&type=5): (**deprecated**) + ```json + { + "connections": { + "": { + "type": "http", + "configuration": { + "vendor": "salesforce-sobjects", + "params": { + "authenticationFlow": "username-password", + "baseUrl": "https://my-subdomain.my.salesforce.com", + "apiVersion": "31.0", + "clientId": "XXXXXXXXXXXXXXXXXXXXXXXXXX", + "clientSecret": "XXXXXXXXXXXXXXXXXXXXXXXXXX", + "username": "my-user@email.com", + "password": "XXXXXXXXXXXXXXXXXXXXXXXXXX", + "securityToken": "XXXXXXXXXXXXXXXXXXXXXXXXXX" + } + } + } + } + } + ``` + +> The range of supported `apiVersion` goes from `31.0` to `59.0` + +The driver uses the [Salesforce SObjects API](https://developer.salesforce.com/docs/atlas.en-us.api_rest.meta/api_rest/intro_what_is_rest_api.htm) to retrieve the schema from all the `SObject`s you have. Make sure the user you configure has the right permissions to retrieve all the `SObject`s you need. + +If not all the `SObjects` are retrieved you'll have to follow these steps to expose them through the API: + +**For custom `SObjects`** + +- Go to your Salesforce dashboard +- Access Setup +- Access the "Permission Sets" section +- Create a new Permission Set (e.g. API for Custom Objects) with license type "Salesforce API Integration" +- Click on "Object Settings" inside the new Permission Set, then, for each custom SObject, update the visibility in the desired way (e.g. View All, Modify All) +- Assign the new Permission Set to the user with "Salesforce Integration" user license and "Salesforce API Only System Integrations" profile + +**For standard `SObjects`** + +- Go to your Salesforce dashboard +- Access Setup +- Access the "Permission Sets" section +- Create a Permission Set (e.g. API for Standard Objects) without specifying a license type +- Click on "Object Settings" inside the new Permission Set, then, for each standard SObject, update the visibility in the desired way (e.g. View All, Modify All) +- Assign the new Permission Set to the user with "Salesforce Integration" user license and "Salesforce API Only System Integrations" profile + +#### JWT Authorization + +To be able to connect to the Salesforce API using the JWT Bearer flow you will need to follow the next steps. + +1. Follow the steps one and two from this guide: https://help.salesforce.com/s/articleView?id=sf.connected_app_create_api_integration.htm&type=5 +2. Then you'll need to upload your self-signed certificate in PEM format (step 5 from the guide). If you don't have a certificate follow these instructions: + 1. Generate a rsa key with the command `openssl genrsa -out tls/key.pem` + 2. Then create a request to sign the certificate with this command `openssl req -new -key tls/key.pem -out tls/sf_connect.csr` + 3. Sign the request with you private key with `openssl x509 -signkey tls/key.pem -in tls/sf_connect.csr -req -out tls/sf_connect.crt` + 4. Upload the generated certificate (`tls/sf_connect.crt`) +3. Now you need to enable the OAuth2 mechanism following the 10th step from the guide, creating a connected app and all. +4. Finally you need to enable the JWT access with the step 14 of the guide. + +Now you should have everything you need to fill out the configuration parameters. If you are using a testing instance you'll need to set the param `loginUrl` to `https://test.salesforce.com/`. + +#### Secretable fields + +`clientId`, `username`, `clientSecret`, `password`, `securityToken` or `privateKey` support [secrets resolution](/fast_data/configuration/secrets_resolution.md). + +## Targets + +There are 4 targets available: + +1. [**default**] `stdout` +2. `mongodb` +2. `file` +3. `mia-console` + +For each listed connection, after metadata is retrieved, `agent` **sequentially** sends data to the target as: + +- `json` for `stdout` and `file`; +- [`ndjson`](https://github.com/ndjson/ndjson-spec) for `mia-console`. +- [`BSON`](https://bsonspec.org/) for `mongodb` + +The final content is an `array` of models, where the format of its records changes accordingly to the target: + +- `stdout`, `file` and `mia-console`: the models are written in the native agent format, which is defined in the following JSON schema; +- `mongodb`: the models are written in a format that is supported by the [Data Catalog](/data_catalog/overview.mdx) application, as defined in the following JSON schema; + +### Standard Output + +To explicitly configure the `stdout` target use: + +```json +{ + ..., + "target": { + "type": "stdout" + } +} +``` + +### MongoDB + +The MongoDB target enables Data Catalog Agent to feed data from external sources to the [Data Catalog](/data_catalog/overview.mdx) application. + +To configure the `mongodb` target use: + +```js +{ + // ... + "target": { + "type": "mongodb", + "url": "mongodb://test:27017/?replicaSet=rs", // 👈 mongodb connection string: the database must be a replica set + "database": "test_database", // 👈 if defined, it will be used as default database to store the models + } +} +``` + +The target will write the content of the connections to a MongoDB replica set database, in a collection named `open-lineage-datasets`. + +:::tip +To enforce document validation on that collection, be sure to run [Data Catalog Configuration Scripts](/data_catalog/database_setup.mdx) before executing the agent. +::: + +### File + +To configure the `file` target use: + +```json +{ + ..., + "target": { + "type": "file" + } +} +``` + +which will save output files in the folder `./output`. To override this use: + +```json +{ + ..., + "target": { + "type": "file", + "dir": "/path/to/dir" + } +} +``` + +### MIA Console + +:::caution +This target has been **deprecated** in favour of [`mongodb`](#mongodb) to support [Data Catalog](/data_catalog/overview.mdx) solution. +::: + +To configure the `mia-console` target use: + +```json +{ + ..., + "target": { + "type": "mia-console", + "baseUrl": "https://my-server-url", // 👈 mia console base url + "projectId": "1234", // 👈 models are pushed towards the project with this id + "apiKey": "1234", // 👈 mia console api key, may vary across test/preview/prod environments + "credentials": { + // machine to machine credentials + } + } +} +``` + +credentials are used to [obtain](https://mia-platform.eu/blog/client-credentials-m2m-authentication-oauth/) an access token. In order to do that you must provide: + +```json +{ + ..., + "target": { + ..., + "credentials": { + "clientId": "1234", + "clientKeyId": "123", + "privateKey": {"type": "file", "path": "tls/key.pem"} // 👈 either a file system path or an rsa private key inlined with `\n` + } + } +} +``` + +on `type` `mia-console` the auth endpoint can be customized using `oauthTokenEndpoint`. The service endpoint hosted on `mia-console` can be overridden in 2 ways: + +1. customizing the revision using the field `revision` +2. or customizing the overall url using the field `dataCatalogEndpoint` + +#### Secretable fields + +`clientId`, `clientKeyId`, or `privateKey` support secrets diff --git a/docs/runtime_suite_templates/data-catalog/_category_.json b/docs/runtime_suite_templates/data-catalog/_category_.json new file mode 100644 index 0000000000..8632b78793 --- /dev/null +++ b/docs/runtime_suite_templates/data-catalog/_category_.json @@ -0,0 +1,4 @@ +{ + "label": "Data Catalog", + "position": 10 +} \ No newline at end of file diff --git a/docs/runtime_suite_templates/data-catalog/changelog.md b/docs/runtime_suite_templates/data-catalog/changelog.md new file mode 100644 index 0000000000..68387bce7f --- /dev/null +++ b/docs/runtime_suite_templates/data-catalog/changelog.md @@ -0,0 +1,99 @@ +--- +id: changelog +title: Changelog +sidebar_label: CHANGELOG +--- + + + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [1.3.2] - 2024-09-20 + +### Added + +#### Targets + +- `mongodb` target. Models will be stored on a dedicated collection with the following target configuration: + ```json + { + // ... + "target": { + "type": "mongodb", + "url": "mongodb://test:27017/?replicaSet=rs", // 👈 mongodb connection string: the database must be a replica set + "database": "test_database", // 👈 if defined, it will be used as default database to store the models + } + } + ``` + + The record will be stored in a collection named `open-lineage-datasets`. + + > **NOTE:** + > + > To use MongoDB as a target, the database must be configured as a replica set. + +### Updated + +- _Data Catalog Agent_ bumped to version `0.6.4` + +## [1.3.1] - 2024-07-31 + +### Updated + +- _Data Catalog Agent_ bumped to version `0.6.3` + +### Fixed + +- bug introduced by previous release which prevented MySQL queries + to return records. + +## [1.3.0] - 2024-07-12 + +### Updated + +- _Data Catalog Agent_ bumped to version `0.6.2` +- rolling out of support for Oracle `11c` +- Crud Service http connection now has an additional parameter, `healthcheck`, which contains the path used to retrieve the version of the service. + +### Added + +- `settings` field to connection configuration object; +- `options` optional field for _oracle_ odbc configurations: this field defines the tables that will be used for retrieve the metadata. + If not specified, `user` option will be used as default; + +### Changed + +- `version` field in the `oracle` odbc configuration has been moved outside of the `params` object, so it can be used along with odbc connection string. + ```json title={example.json} + { + "type": "odbc", + "configuration": { + "vendor": "oracle", + "version": "11", + "params": {/** ... */} + } + } + ``` + + +## [1.2.0] - 2024-05-07 + +### Added + +- Support for `salesforce_objects` +- _Data Catalog Agent_ bumped to version `0.5.1` + +## [1.1.0] 2024-02-08 + +- Add Mia CRUD Service support to the data-catalog agent + +## [1.0.0] 2024-01-31 + +- First release diff --git a/docs/runtime_suite_templates/data-catalog/img/architecture.png b/docs/runtime_suite_templates/data-catalog/img/architecture.png new file mode 100644 index 0000000000000000000000000000000000000000..457b0bb4a6f2384e4b3d5e95135edb8ef46ae6ba GIT binary patch literal 19519 zcma%jbyS?swk3^2aCbULa0n3GgG-Rc-3jil4J3GQ_u%fX!3jZvySuyFG{1ZAnm2D| zy*2*;>+7#iRh>F@s%r0j2vwAqL`5b-hJu1Zm6j3%K|uiqA%Fitgope*_zZIf`2}zS zNq&Z^7$w<m6qxXvC@G54>iFgrT_MD(N(p6WgD*e_i%ERprWVs^3_t zMJt0jTnW<>ud@~J`wr9R&sK`qeVZY>`bii}^)33_rJzDM)vdto(rf$gD5n$mAHVP! zej!&umq9T^4^k%81pD2T>4R#U#z)WztrvvStw_yvo@&eLy(6-y^Bf||=xhEKiG zUe{=zZBdNJ2+L5XZM*(B&Ve%;o-*?BW}fsBEI6aXs>O)kQcR-BL}-7lv`_=8<@ zyP*oN&nX383aDdyId>b|Dj4+eC(Ik!hw}0Ewg|EFS5>2$ezy?fnhfw6@BSM2e6nQO zF6-Eke@hrWMBlfq0Z;?u8o=SuV-S<}bEl=zFM zEXKfYA)6mriKfEzrL;-LaQ`$1<L=rSFuReaUB%Z*(fz2Z`o2t zQK%889I^WKUvG!xLUpmikMGKAFXpQFdIc&=LYty|j007Yh!E-EI|cq@@^2e77Kv5T zQ8vDXiVJS`)!_*K*XM4^&ktVTL4zl`kVNtONT%4H* z7vus)znH!?C#(x+MoNY=`SXC*#csCs@kU=C4L)zhg_v*npON0+0)>eVmW;hQ@|&gL z^KJ$RX9xfJV1*6{h*k3NSO}%=VBD&XhOpWVpnDM3@*9y6wHz4%%q zgAp%G@b1@Ktv)V5Ne`46C>IFmSFL%AtL#eTJQFk!StjUl8}I?0RrM+cOapV?8JsNqsLb-yXON^qBi6 z_8cbt}THvhnptQUETd<(>uS6itAPoLAp&B#w z0|>JMZ}8iii{zS{KGjTn3h%hJLG%^#InMeZihP{bPFGN#u-4kFNUNQ`ph;fB|1%uU z3OP15%>@ZPWCFVU@jG!9=cq?Kn>y<{ju6F!Uo!;-0nRmvEWM#|s z@bw3?pw>8oXiM1vu7Aac0~`ugbp6nbL}Q08tMij$iouDtTdqn{F;bf%C;q-LDX zy3;Je_i)_)EzLN^YAtao@KT){C5a`^;EC?#BGJ)uHTte?#+SbUtOx}X-rwMnztF#xz;@VOo*$&s_hY zH;4vDgoJggUJl=Jc%sTYu)Q8)@&iJ65aKg@JdIs*`?^852GdfDd&3wTg z7Ll>U7PhgrA&4d-^|g-p$I`OuuQe!Xi;1(Uj~W_Uh|`@u&p}Q!qg|#Oa?VHNm<{5{ zh6DW>+|K9OW$K=FC!zgJ#JOItC(zMGXcN`+wBNjUU(tN{u0;Q!F|F183I!Cm8m9>S zlbM8bkqJnE`uvxoe~6eICLYDoRSV@e(=qP$*&omj0}Ta&dJe1c*-SpdmkZakoPe)S z4izgvko0ip6kSJgvs0fo=U~nU1Iq1*`ceJQ%5Hr#m`jnXSZU&XqGJtTHf?TCp&lqZ<5C>g-1_Z)T75nb?lRoiU zW39A#Q>J~jp31>Y$LD#kTp(w^x)C|Ayc2TU-P65(utR?%EmNvt!58fF#Y+=`Y{gB? z8k@)EIdAUd=blWxI4|YW=xqIPqNy^{;u0sPN4Z?trN%+upb^a`{o3n$g346{alO{t zAy|o`Mn`J_g_#gY7o4>Kfv{9`FO&xePx_C6YUOxzda(Wd{SLeopSW=Vr_ct8mI45=7s_0x{0^SkYVBqw-YoG&pyRhsVigyWc4 zL-ym43+Aa9qAB3CY(r$iv29TBqUjDIKDF=wfOO?r!ynXzf*rrhO$^dhaN^=Ld7heX zP0nU<9%{8*?|~ymO;gcae(M*=)7PbbsewMAv$y=sVj%osN`5(aXX@~KT_*zoSXgm+ zVY6kfo<3P(vC5Nr1)eA`Z1>I@COtAQly0S~7Y8$f+H+-<1|sP=`XEOGB4se3SYyZ; zT}q}h@}S^l-i3(xr?u23GUKWX62ri>YOHaw1jFdIa~B*&d<RTgIUc$bJ?%|U(Xd4sFafABV7+P+7LC6Wl zV~639kwGyrF?TF4E{{Q6yp2VGRWNORC&i$k?s=3fXfYelRt`QIR^lrZ{>y1tINlPb z>PKLG{-@d8u(yuo>A|pZ-@Qwnp$(LoF3I&cls{{BBPLd33V7^#S7faZ^TR5>zlpBG ze4$3P1j=z{YP`E*a>=yB`(&cdngUGqu?0|cR_Ua*O!FHoH~GFRfhOl|XvMSOwjCEt zfraVd+2_W=6o#n15!S$#?uCB5=8LVq3vu@GB2D6_38!fAUeZ|BTD2GPFcYJA-TY~i zSq)+=#rS1m)JD&vTai-hPg;f9^PWe=!3F}Eq;S*V0rEw~>M$6x`=y@5`d*yu+h0oN z;|<|@-s8P6Cf*-GTT;cU<>FD0b46d3bdo$95ZA5saJ_5SD#2QxtFd{5#++cnz~gR) z_%uMzIvksI?~e#zwZrGT@%$4-Ij{Aw%GL<}qTyB5$d6KdsT`zPA+{aXN zy;lD2&!S@Y&#jFVXOjp3sQ}{vv+0=C0$>;plfTc2c9rF&;3vx!Jb^_n;mH}&@%AxA z!?c`ise2V+KTk_i;Bfb&jT-5%{2wf6$TcjdD+ zOCu>#<8p-VaSkEA`&(zGxpan#gSn-jS!jy671md4f3jsr!tJ9Q5yEts*!@^TGxPHD zH3w`xQ6q50+Q<9b1rDu@NOt53#-MbKQWzth4naEXL%wpa@&zvp+wyyfPU2y(T-&P| z8QGQ*EO$~vgJkj3t3a+&r`MCvDV^ih`7NG(`nY9zp`JwaMaKEs3e(Y2o0ga#u^Xw# z@xzzRVUd%i7WhY3+e8sX-xm_}?>Be)@HIL?q$^ z86j)pniTyWHPB-8thf>#9xaRZ@VBAQLJwma4_^5a9zt~H*GR*mWPnjeEVG9&bEbf* z^?h=ub)0+D5!%T-ykKQ#j{fY9`QN;R5V`$WP#HXwlyvP~b{=iaE->!Zb`1-@0tyDy zjOHr9pI4EfSs@6yI9xF&HU{VyL=-eM55E&8pQwZtYdLA}z`3G+3QWL^;0NTUU=dKm zU13G*C5`@cE9}>3FyuPmG!Fv1+5K%b#;&tm#l`+QEE-&ZX1QAK+_J)N%)C_O zOPgZI;SS{DL3f?n!x+WX6?>A1@ZmW#@o&7r9o-cwQyZr6u$kpHJ{jgwA!67?`K+uk z2nOwBkxp3LJ};qcM9Zyv?ssi3KV`n^S9!9~E=rovtNoiV4C}l$5wx>akfmPj69`4c zbnuflwjt!NYN?(vqh>8kq8g8|Qxdl16{b^0T|)2U5M0x+J{^>_lKCq07o|K6q$42* z)$8$icsE)HMKF#)!KW~l1vD5oRCs9hlrm_PK@)131}H!rvMJ~9i6CDPZ=$am*B|V7 zcI_8n7$-F)j*CoE4KN{_<`fYST@m&EB_!;H42SSqfR>SBFwsD&c+xILADyUYj@>st zS;hDYFB+8lSu`MKWQjtA3ZD!wY&fMBDB2&9Lz|H;ihz)uY$B5_sbWG!LD@+ut5yqp zssvlI=TdXKIe>2RJC#Ci8k8)Gxm-K4+`5OURDv&K#yxOI$gd3b*GkWM?EXl`r$e+l2!r59F2K_=F;4t)O)@{XL z0IOsvX~C2*y14w~qPisHI>K8*xFg^IwO_OS@L|zgjkHvq*4i+6s;)ar(<+Y?cO_HhpL{HP&cC7+o2<3>b=FrTLHdpS3m2x#bOxf^SSv- z-!f9Fp{%6Cnep&8i51n(c_e`7Uo;t^ma(jHsuG)WYRybWk2BX5rxcJ0r^qbJr? z30q7j*$D;ndtB3fJdhmz3nyBdH5E&aBE^I>GMyI*napXIA3xBH(;<6-4N{OZt%qP~ zw;C2y!uQMr4jRAUr%8Uq|BgI_3R1Zpcvfz+vYKNNy^Jt>;5U7%pfj*doSz*;q}fGt zjPS|MzhRcIwp6gHpu>fnd+2FOVEYJqr>KF6wZ+sF4RzmsI%dMaehK5H{x?MN<-#L+ zh-lZb4EzCSO`JDXF~-JR3jl925r##;o|4?6bWjIchi`a*>fP!e>}gy%gX{?VbJAau zRO5$eh(tbojfTJ#pb}wA0NdGbt97mHO6yU<{ECZbiwZES-m&EF(kdW)gnc* zh6w5}ERf}2EEe7u{jGz+qBRDm#Cw_wTWT3LB$?5vRpz&1?md$59Gi2!^ZA*$vGS1s z&In`g5yQVD%3!tzaFklOlEe-JI?s+-(`Yq8XqU<@;k%-H_Pp%V>^uvJ?m&@pQQdmP zQWic8cZSYt`<}F@Hdk-`t~HUg=_`s3V3m^WOu!M!mFeJDg*fVu#ze?5asqd^z;5KL zYE$ETiO6$px#+g5!Jark@p+Vb3s;e9uI+Vf%`zi&ih67V7B<7yW#A2%_nw`@bhB3YWeO%MVz%JXU2jFOM|a z6!M-@~Cg3|kJKF!cW<_MfOof(8IBU3JTb&Ch>jpEaTnb$9WWS8p}{eV1(V{7m-8Ck7Zc3*n(o2J}BmJ8(e1Z|Te@QSoSqz+bXT z6!%{g^JG`Peg~7RyLQ;xGO%ApW%V!=ZjNQ>aV_^!1%~Ep;@Ks=sG{eKh}Jbo?G9Eo zn>V|95q*j&ouY<|MA;8bEG@JgriP4&Zl*U!97zzG|5YM6mfFKzrm{~cSd}zeW?)d7 zb9>Ar0Z8lr5f5Dbt_26cDtJeiz?c@8-tr=YO-WEbWNjJCWejgz3+++v?h7&|=Q$?! zyuWdB7d9x;L<=uFG()W-&k;u_h}ObMD}^SAULb_43rhf%91KHHC*=z@GJ^{Y<$MmT zGZWy;Er#=hF4?c4ZuNDNtt(W$Qg$NZ#!BUS4L!khorII|=bB7EA+H<3k5`q1?WI@a zv5i`?)sC>m`DD}QmJrkXK#MfVmb5^U>Md`%;Gpd;`>9dkNmr(m0XyH9bIO9*GQf>w zy{;SRI^Q`44F;6t$$sn&xv@xki9Z5SebF2*78qoYk+)B@YK_+U7h+PZ#v7oo_DZpn z$R8L9v}z+74$XuQVmE^J-@$M?+0?m`}d zU$aG(n5M?))R;nsRi6qWrvkp*_qwAZ@;cYE#(>30+)MX^<3dHX! z-DBjnTpW4JlK{mTkqSvp(I`^KC>3=T-vKEz0>7>c6CpeG6o|88*_j6cvYCvo$sB6c zAxc3IJ(sVAwGV_4ADnHWF>vMiMCTQ8(?Yh7jIM;4dUMQZ8LkBNYI-JIe7Vn%?ZGuj z5`u6Iz56%9mFL=@AV1X&o(i=!paF?MQW1Sf$uAQbsA5ud4sFO_YdsI@IGCVgmCLk9 zkXTt+5z$b48c#6?69o9!%m%aj7F=@aAyyy!0mY;llWmFQN=k@`XOg~$0Uy~W`nyLz z*|EjLQgw~6e_uJcqMup&tLUE2|9#eOwQFh$hZ-4Qq8ZDTR^=D(^cof+?)S7?a)Y!1 z0KexAi4Sj`@lVUoX%PSIrI*x`@Kh1wk7a}?ORi*2fs$5qg8z8=`XU2Xo^{E9-zsF$|7cT9tk>xk%+?pazH!}o z{{ACIsTpssLwTz7=ZlU?mI7w2fM)E0pJf>tpa<`$h~Ch5_^b{up8@^D1qE|=1A14M zt9U)0&VkJ-DWCa-e4l=WzDu%SyRsP0_(gOrphr!t}Q6ufvppnyf6W z_rtf&)0Lj!Ut>iY1vo!oKIN(`CshgE$tU;gK2&y9ymzwIU$fV7?ihjjZ6! zvkP+O-M!)QQOe#*0f!vBDc0uJaxi++nvd=rgS9dg1f|(PhnKyyM!dVjoIP3UONz!& zsZ_!2L&p6qrNNE(?W)=9{DA`!+93#HuBx@Xp@7(5do65uk*nk<$`9e; z5JZ6r4#g2I7V7@w*s7^~IVC&k_Gk`qK>3a3{ z#IUE0XE2G6i}QFs*yW^^wa5zp%61lw%g6iUgejiZROP$XUEiWzbM(R(N|SN2uG^!8 zQjG>vhD5$GjD~$=q<7eg!$~w;dTpLR0Lq2A8eIX|DvVub^AQ$^cQ_%_d~V1cJ$MD0iEO*nDn} zbF85xM!EwVomzOcQRnViJIowgOUaqx6igDl4-}WynzcsX{xB*vcr5d}-yIPW&6b7A zkE9u$R%5e$L2%t?q?nF}29ZUWSUsG{%ScA$0p36$X28A%rw|_FD#{{7pDB1absDAT z(WUp%?WIVq9qs05>07fc@!v94$%Kw0m-e$xFcl(*UUgjc^z}08Vm#{;zI0{rgQxeS ziENF?DbgbBpaPFot42}<<fy09X?wx%inf)q1vtPnO@<{d9u{ij|YO9H4V@C>-{Vc%~|} za9zcbH`|i98jcpD?y(0pPakjLybDmt!=7p6H;Pvo)`P`4+0R=Ceoxwy@sO0d9)12k zk^i?)ln=mCNd4#AsjhZo1dLQ_Rz{sY>x1hNn1U1TqsO=%1*uN0qiK4xBb+RkT_0QF zKCM;}P+s@x9r4Rt;C*N0!2Ta~j5}(?&4*iD%>9e<&Tt&Z9-?Vhk98xI{T0=x$Ccax zd%YHtcOSm=M(vT)7nX0LGt&jcBjkD$s^&<-n{2F_KrR~Bb{^DExrh_bw=PvoI&x~B zqp_&8x)snkfQ$7&zP<4Z!3v=lMA<(_bJZTWwmPjjAE8xqJTsUm5Tq`YFsV4;2Jb8u zRVMRf*S+U_HzH(iZ~+&ayJJRrHA*7~kQ9YhvD8MlgG#5B`-wVVk_MIVo4VcVP{;5e z92T@rt`~046=iU24xL%b}L(@=59t(}%>e!?ayOJDFUc91f+P+kTa@_a@&}cWN8JfeD zY@8{N{@n}P=tE}oU}sA(*@(`0G!MblTXNMBhA57k4VFFQ>9szg0g(<0Us@O`x{zQV zD8ITgSifEZ>64hk*9BTP_nI%8jxP}>B~ZR6G&gGu)>s|`kFJjT3Xr%V#p#v_9FrlsGZU1~MQD9;9lYl~b-6enGbvGap z)_T3d-l@={+(BLBv<*3{)Mxb@;cbc_q=uo&zj%Fkgg2|y`H|0F5V{v%@s&m2@M-Cu zV7j$X>N{DdJyKw+tB9+tETHKj5)BE85RZ)6JMX&~&NZgV4VjRCpR#WPAl7@S$qU{3 z9S%k~nAS#Syy{O4ZZf|K2IuK=lpyJ1uIn^?D0J*ZXZVa=Z-5K&MubI))3hAT9lsBJ zWm+RP)P%*>$0J^6SjZwBFA)(idEzpaEw!WzOJ8qboQ6tKflZm8f_>k(toq;$0SKyf z4kV%hb}&^Y4aAY2y)OXsH~NhtsZuTD;TQ%;tv`>pfNF95N!H zzZ-SdZnI>03C^{fazU@OOFn+db4?qCPs@pkJ z0z%?H-|CnZrI?glt_#G}sd6nAH$9RR0nTIsSorOg z>uV`a$?U!;j%H)bpx(L=^Nno4oX1&}`Q$Hs-X0+!)TEzw*v%KBflwehzefPwJ3s6O z`UET?3Yeo?*>qi8lQ@0};%T35xd@n+wZc+>WEzPW@4R3e8TTMX7i< zb6aCH6wcz9RL}kt7FxyDg!K&zz503BLerjRi*gAQwo#gGX2 zP_wwDy*aTkhia4*6wok*a+2omO=U+}hibXf<*7BQKkF2CPjUy0d}F~P_yCuSH4!6` zp$y5WkDfCm)yWbq=C-y{jrwB9ZcUeRtX$IJ-g~}U&-A)=-LA_=n(h}D`PMOLBUT`) zR?h)urHCQM6i&ElUuKr05v^+ERy5MY`8jU8NR()2mV6JoK7Oks^Z78Hg_6&=OHH>> zalvTa`m=Th$#LOj%U`T!yB&A;jr8}sU5KkJy^f%HQti_D4AVn^7G%npKnvXRy#4hb zhkuJCT&fkN6BwFR%St3--ln5QZGK^Irja2ilANCXsKbRzCV?&#S(Rb!K5jRe<2SCL zn~!3XhM`<02NG_n>NnWIFJ4^TKNACwRyaD#yH-da?N4Urxg#@qT=Bxqn7gG<=WKjC z1Ut;tI|U-WNJYE4;ZzAhIb|ZyjxmQgux(y}t7>)%K&dChi=DW!K{H$bz(?0^(+k9J z!QUyqR`6@om<4MbTT0=R<6tbN)!CgH8)VPr2#;fmt28@qh15z>A7%N8sYo*D?8{~e zON|qVi^d0_AS3zbLy~*O{k|*)MTaENDv>cHA_8aI!XfHOMoy+4{(BI`)$6`&ZJB1@ zQbt{Qz)CJ|{*|8DrpwU!nCz3DH>Z&ZJHzG5|S`z!T5avo#7zEpFHy~JNz6#B8Fe7aGC{uH_cYC@z zKGi{ZBr|XLchA4+$b)c^>*!Z#>40I%vP_;vlC867!htFVHcXrQ0||teURV4>#=bKa zvbh%wMn^H%CXVCm&mEOj5AV~xoG~YkKB1yjYH`?HDrnG2V zJZ#>F)QU+uwJ)C($!#!(X?aOLni6?7SWyDaOsuIuzw4T$<`4T0xd`~YDxhx1@@2w! z7Br0|!oPD*2uhpUSd{2Ef>r9t;PcD?(tZY>M;E{LbHA=Vt6Sscdy+~rYPBNX+AR;P z*^1g97jv2W##C7L9j(_45`NL*`tlKh&O-vKzWT>9C+=&G5W(wQ1F=)VXkVUY++Qn2 z1=V0-MuU3`GRysfBraCqbknLxY0s(WC=HOX zBkCNng^AC_8;mLJR#WP#5v)K}CE4ei1ICH>8sCGzcXdyER(U!;-{s)bD3X7cVD4#^Ai<0=cN>}4dr^4fbIBwh2ZS;={b0w0p?AXlS%G=D)m)X! zy^*30h2!0yd;WG#LgCPwqbBvS8`{RyTm}+G6A^Mg7=8m3lRx+BmAnx}E+bRTqNKtQ zWx!9pz6blVW5hx@_UI9Q;>>$NY$d%k`EpkCE1(LCDhzgdf39dFT9e~BazA=1RHdZ4 zaGD7&;Db_vEBl*M$NsPZ zuiIeepT*vofv@&OFj5ysqd)lWT;{0SJnkhp$L5N}DY;o3vDO^_2c-ce`Y9hpLuh)7 zFUa~fx}te<$8fjcn`8u*)Ugc1tv$el8iLpms`DyAAQqYy{|qTO2vblDEP;+YfR2@W z4tW5)&c0-;mWqerOIIb({8w2)-m*6*1W_`U-nBFoKF~?35+wpb;GXyI<5=Ucge?a| zM`pS)^USWNEUZwZGET@`nc?2hcVH%RV}Y!2q`t@gX=1&sdc$a(AQ?$nxBJJU92BX* z19{tsbO=sGe~9C8@BO$F(3bE1mMug4XJwr7ksH_&a^kN`-) zwBt+!KsOn6s-^@%mcjyxHTiX+W9x|i1HvigrBX@`*yZkxPf=|?txC*H^aE;9q)5rp zgLcRuLl8){1vC$D+OE?0`llWL!42@?nMQY+mcO-UD)~o!kz_0QYHecT>@IOi@1xSL zKXQR0#qoV}aq3NxkZD`8l3)vv{`^DN*FoN7DyrDSDK&V0XUz74tHnirBXXPfBM(3D z;}m1C_>4;}{2cnYk?4O#Bln4-V|(B99~^&Ti&fq4ZWWIQN9!SYJa!z)heR{YHr)qx zfjo;vnYl?KV0+>Vpfyo9dh(5D2xoo=hXaD zyNuajg`foWC?RvHz_`vP9iwzJ1!NkzWCz;)~(+9%cP-!MAm*Yj7jvYBt; z|58+7XHhl$lme7P$C8{0coKF$3$ckHgUcE5?0pIPIAmDpmMKD5E zrD~1O^3qP~bD-EJX%Qe0ebsYQ_iwljM92qVofjj4t3#72c3m}|UGv&Paa>3cmT#Sa zQm7^#{)=*T4ZsK#UL6>;P=}2)ZV6je-Zc6jX!=7|pzUzx-H(n?QqCGe{g(YRr{BNL}`5{ zij?j?Vb3|`2*83HWa$2Y${T}C^&PC3FaHW2A7STIi?Da3i%%ZF}H_6~u8lECBT^{VL~UfTqm zhwaQ*YdWve&Dr?liyi~EV3SU>NKR?~9?8t?X&^=AvX?0|<&-v0L*`w$)H{3_&^rnz z+~PNE*~RAnmvMeYkCq)BP+`zM3l_cqOsJ9+eEL@vu42XKe)IfK6w5A(Gi?bqIbpOG zR%)pWDXOd}i6#RjTMh#ta5sz@^@AS-Ke5{9R5=mgS%sdhnW zhA0Sio8Ojr2;s{#?JMN%-e_c-S&JnH$Wo-9{t}2HwS0tsBXbL}d!UzxRQmC)_Zg&Deti5%@gYn|EIMUXbT5l?oF>l<+26bd}ql5P##P zmriO!{O!5uLgbbXC>EZ8h!-dsKfyL%?o5WPr8X2sD!=W+n_d&S5FX%a?l5ezj13q? zGWP?bTQP{e{Jb@sKodEgp{=$#I%1XlyvPfS;NyQ;+4IqF-VCB5q$|IJfMMKCel)gv ztpQ=GB!BBvC2{C=Qa&Hl*n^sFGtZlm_ZGC=E6mfX4$DSj6q)dqGby z#i2{!o1?as6XTqpU-0G z8Kgbw5mS5F2@~MsT<;#iuepVHwhG0{A2QsC@aVot2>&WoD*Uj3wP!=>f)X+QVw?1j{F z3L^cRE(xFAgo({|J=uDXiP~^7Cm^!*bbTmv*IWgMS1}jXkm1>DX@5z?%Tb9|`Do{` zza*_GCeuJB&=fdB+7jv*W=%6gmgUu0ZFNYLF6fJq`++xC{S|XCnl%?BN`oXUV?f2n z{`x9}WG^$Xsu%(&u5k3u8baoGe<2ebTg`QjX`@x9x^c>w2nVEq;BZ$J28N|ir3OZ% zzt&G2e&{ctn3mctq~G-aKZ*;0ky*L2)9mWlr3y^KzE5Xp@$x#g$514ey$#K}xg|v! z^}MdudejtyQKzdve~KbO^i^X05rj&eS`GkUYpiLt%QIWqToVGvA!pd;Y&KbW%-8}v zJAPSg`^)~O*S8?ppZ4Qe8m-ouA(^x?ty>;4VyVHSD>xqymU7^S;!8t_T!@f9-W~~& z^z_U3gnx}1&yyfTA>%cmRm3BXqvQ zr$^h+i^OsUA-@{w2GXW@XP2=uv=mZ=bK=iydO%fKnwc-r=FmDy)Q~HugB>yrFQx)8lt0T17CJwksy5FB0`75GWqweBb8I z0#n$MZ6aJE_+*sZL-3VdSynsXy+A!>mZ~8G5_ssd!gXp~r|64=I_y~Csa-fCN2_M0 z4UWqPIJtVLd%<7iq7n)s;k?^$IBLwU;$ppAtafuPBKfdjs^phuHw1Hk^n6K+jvB+s z8Hg&3QvMLJ2B%AD3;#71l*pCkEy3yZ?)+vDkfGEjMi^5=E&Kp47CK93M2VPneS;fw zH0ZA@wk4%xkdz6d)YtKW)<~ zY4Oiaek7|Ot$ZyS7S~D^9cK|A)vPi3verr={K2$Uu309uK~h&(-ci>drt~F^UEWtC zzvvsNN`M!b>lAZ9svE}ZDyL-XUAxXubvvwkB;cifr+b{Yd8#?DXypXS-fAk7XNr zx<9izUzw;dr67E*5R?#EoRcKR`fJWySt5EY=Kpz-gVyr->OEAjz*4odKRds?E;fTS zoe!Vi{0GDE9SVe7YnZHAeYnAV)v~&?kcpI$Mdw5dJ3~Wi3iJKgiKfm|Axc>u>|`%D z%K(l=e*43gq;4E5qlPR2k30wAAxK@z7>~@C>`zvFPzQo4nw*V5F5$TPKCsHoW|?m@ zgiKz$-GQk|Hv8Pc&tqU+7RuQ2@FFHT0Ob8ZM95D2I;ZKu&5Itv$0Vw>-BIg?`RZ~J zo%6*rYem`A7eY5Vzy4op9f_m=rq(IN6_Ek+KkSkd@a6Hz#4`u??ArR{#n3`;wJ-YR zD(rrs$jPQz!Sg0vRh+MFNbt5YV}PzK}alkc8k z`{54J4R$V;Yd(LsxiC5mP_i%6pirf+SaRiuceefZb2;^crQcZfAC2})Zhp5>wT0kO zW!I1zubn<-a9n3BG*yD^!AdU)M5w;P0Xsk-mc?RY69u|IPB;=#9OXAUczoih{WonMU~kHb$;zIqNXdi&cByIDhmUZ)loPPui0k9vBuNe%r@hv z?=WL~A>of@hM|_?LHIzE0N{urR&Rry9`(*gAq3yvO3KHu_AFG!wC<0z=$3-+K~2p_jZXGG%J@>KR!PoZ^eG zkZWv>iOlL+izWI@b5`$(DYjxRNV*LjrT?k^vHDp-fuFw_%fs;{381uK&IYFp6Aijb z)+qVJBP?^7h$BKD#)iuQsqkpZrRJVto)=-6C_0F=lIdLWc9>z|r^u@u0;@EFTB7h4 zXj5~QP1rwCamy1acgRWEN4vAUaK_|$=Z-waLq~Qp1>`7boyAd?YQ}z{TxB;IjJBSV zP@=clNNPx?%}uIlpw@=HjgAItX!W1)sqTOr$xTS1V5Q4-Afl8Qo+m5V8uf!nRHvX^ zx5w9(%Pc!gd%c55z2k#*N#{>H<;zi)+u_Eu>)}o3yLQ};C%gqr^PyY-|Dr=hyg!BL z;A;h*>h-8e_eVhtmM5XL;cdPml>$W05#rD#9-~=ksU&_Nxqw$-OfBcyPT(%BQZG?B z&8^6_47CX6R{FG;Qn8#839lFGwhT@;eUb>Rvd+2S*D$EKR7p9s=&iK6s$f7$&3VIr zt;y?PgZ(WV#m`@lJQ(kSzM6prk(`>y441mFz<`+MW83$;;?#@gJkvvP!ttJ@4evW` z`Nf{e*cF#Jh5B#pC?_g}8c70`+O@)t`j)02F>r_?FZp!VTO85NDp%eW#^@@xm;)jl zmqXq!ry4y}$s+^iD~(~|t=XWrkC2TfveCGG?m5+-d{yo?_kG9(`dinN66^mqYq~J5 zIAf3t1+R@_a(%yyO@*fNS5XR6i{T~_p(iu?m+Xy$GcAd%;(0bH5RgLVsdY|blG>OK zNa5AB-`=5|Iu<0Yd|e1>Ni_D~viq4knn$8hBZd{3ernItyno9pLlPRra+ zm(bMD$r60-Zzz5BKWV)OhR%2+|B@e|y0@X4y)f33m35!u1qcQn4 zEl6|6Y20gtI`iV6==_uMuuYj3#h+|nYjsN?5HS-u%3vNqx>NuaH$Gm=XYZ5=_Kx5o zh;mTnF>!*m2LenTMyognH-NS(X5RVmefqUQSN$vZ@V*o(=E2+3^#@cYKXA8VKA11+ z>1?0|Dd)&%RlR7+jO6oalW`X>`d#O1giEdQEkrz2H<~62R`F~X9(hS-uRsXH84hm{ z-B}O3(-`iU*vTxJsgpR@>4-(!M~^hflUxmWIoXapkIT zSgq$k$harn(8%;)n>Y&g?^}VZhX4&DZmFEb%17B9IqoV?WW<0lC~0Bmv<38X_%Y1O z5N8+jgOFAHAO6z)C>phguoA7dI1QhBI*p!>4EH8~c2p@~n%oJkp1uF5(dOs={HeNI zbzD5$Ies}OMk9}wlmH@%p09$Oy%Mw%)+m2n+I^#&rbSoSufN-^DuGeiC*b*A@RU-9 z1xt<55B>3`%vxMHUOjjhsLZK$bz=Kg%bOxWQczCn9E+Bb6SqvUUoNMz-in@uN%Zn1 zPdY*1cqjiA!DKX@t}6hj1;?->YIE^HDhot3cRJz)p(dHYsdM?G7Tgz-W5ua@)I!{S zB~4sZ!X@kobm$7dTljx%y)PL0mGu9pqlH&V1Xu`Xq(r7t_z5@HU}m@Rd&frZz=y{h zeQ|S&R-W+?ust&W`ZYuHve3%B!fP(J1^Lbov7e4bf7K(9RCzaom4kHChRhN@-FFHrTRJ~HBfjdBsKOmwLolSRGq{X_q<2PrP>c9am@_+ylWrhz1YJIc1Z71Tn z7k=GXzTNao%ajCs9`kjO?PJamHt$xhyz2TOL~%MG(vlHKC0$_C^iY;(lSy&2jYEE) zw@6NY!^{fTcG@0r+n%JsQ=zvY*e~J_L0QNeOK5@U(ppW*E%tg2Ui%c)Li@#J&eG^0 zwHpKQouEBziaV@yxbws^3Ds{_i;;v6>-v=PDs;NQJjE9hymVXD?c7o?Q@Zkpi{ssB zno2=9iRTc)76bF^xRxDi7|^Vzn9x%Uf~j{@{(u&p*GWkZ9y>4&BCKrXk@DavsJPW4 z!w&rLbA5G#%<3n37U|7}KHTsK+anR-`qq+2B(Ik{1x#$Pl?+a{)cEebmnKf}5IcEX zNfIFC6L5R=3%*k{9buj~3qy7V1YUP>aQ(mp1@&(I?f(KKY(9dNA;|d!IG)8q>JeD1 zTN4Nka;mDfQ3JlAzWkidZ@{`ISgwTS+LS!co0uTFy(U0%UnUhWTTysnn5IUN$Y{E4DdQF1Znp@A(i|T|LS)%8^Ybll8?A zQewx`mntaq$UA+o1=VbF`5xzvj)s@3^8ID&VB zAe>0Oo@w-S3%)!*7rw^2n4)pmiZ9dkEl)|ej9VQhIW^p0d3VHjJ zl^+_chBD9^t7FI(++d^MP;0c3cYMo|oi`d8exLJl?$ih#B|e_6AoHv%4#b1ei5X#h zr`Had*B-uH8%nS5E~M9O{ye3($mB`#r%^R33jB|WJDRAuao1J2|3bdeg_?qPYS_Ri zuMen?k@{UXea2!XE=x-zb2|=t{O`LhOzv0Ly|9HpSsOGa5DBO^(^(lM-inQ`D% z2!XrdE-18ThMVr!2+bn*k$3Y?s%nNI3dop2qg15a$GYCO`bX9ZeJhj7DN+92Xzx8v zS0r0f$DZVQ;>{_S-#!)Yqd5Zyzc)0)V69|u2O5POgtbKvZQCu-A%p^idPs{ZUFR)9 z0n$y>d>jkY4o(EJT_DJCl0|mDBnMZW*_m@$lj*rv3|!bqRyhu^KIwGhUDDi${m!Y5 zs{0vEw(+=A`NiSod9fgn+xw)&z5Cniqwx9GW4fy6#^m*CpJ*VG`_mTwED9% z-*Z2DUF)68N$gVqA^rE#aqBsDC$%GlVRM1HKh}l~=!B)T>)_`Vgf?2<6zBCE{}ObV z#|EaNZTEacvJ>*|uY_6rY0;@kQ7$va$GBooAa9ou`iF}wN4>RW?!WSNvGV6oK$XM5u1K#vm<2VxNT^}Otpg$EpklCrCf@Y5;l@TZq@HQ@_YQw`TZWh z-}nE|`}_S|zMt>q^?E(g<|vk-_pFu%{8$owLbsPjsSPq#)TB>9(eo(%=UYhQWb@nl z{OCM_SePCy;Eh~tRH)85422f{$@Ks`atr@Lo>FhU8$@eAG<}DrTRT zVBS-KGj)%!JZUxkxrI?#7dvvKYnK_*MKJ|Cu|Is8_)&S?oJ6SAL}`P z>6lNgSm{+I#L~dlx_iVFBtBy7^g{&-X>v#{tLA-f3nI`3uba{)2XAh_xNVYzTFkD&_F&;* zeG0aZWIcw0Nt(|xul?Ff7yT0iV>Lh@Q;cbm`Dq_|ZIh+navAfsxT(;^xO12>A3J(Y zX%{D%*Zl38RsEj8K*^fIln1Hm)a$qUgBtC@eJ@Maw09z-8%Vs7kp1Uq->4gf5btw-}LXyI11)-7uU_N|WW{>z7rx zBTp|O-$Oz`V@1_r9HMo-(OFYa}rMaa0bnYs=cXoEY9M3cenT24`qSme+r z82i`3K)@ zh`HCgn=Jg_GIjb@kpKuH;7uO9G)W7DcB9t(e#Z;=Xg!Yj+0L%T|AvIqE^D#b`;GWH zev}$3WJdet{it0j_hee{mX8)KBO59dZ>Mp!gW;hg@K!bYTNO}$`}3cRY4Lx|;<*ng zW7a@g`b7{{21pQWbQvhEzk7(JC)C~{ERE)FE- z9GFVf?yhhvk969)Hm{PHw$NIm!FjzfD6IlhPDe&AwyD>QCaQOO7%>iVk%5w0BGaS_WhwKV9}Ii1z4_k*c_2r zYPtXYtAW$kKfD{?Ib`KeQzE`m9 z&9`Wry)vjLe+afs64KgYm~!^pVxnt~=L1rK6>2)qeZmh>K~v??jKN#U-$?|d_V8LQ zPt8(5cQKW=uU*_=CZx-(0nUbG7ArCzbvVUV?t`I-89uz9x-aWm56Y8T4mUQoDJVp< zZ{&g~z0UHyLTxY!0w0o3$sLpwYxV}Iql?A~P#g)<_|+vQFAWk^iK;fz^vFz?UMIYe zalis9<26s@Yyw}3fkb_p#3nJR2)Kp*jVS}*0fJc{%Y}dDwhrUc51EgjP!PyAI zjRr~QfWgS_`-j6iLjwTVm;mp3Q@1$v_1_zG3N_HbT)w1attd8=4KcXz*Wes?p zTGNXcHIbD)XDph($vO%%L6Q!B9Tg>+V5$}X8pFSMNA|v8rT-NZmx5`!#i+5O*6VIV zgHZJt$~gbi?ls$4zqp^40y{&ndE7>kd95!f1YIyHaIgWg-rg#$9ovnl`pK?*qb+nB zh;PKkNDjLtp_%4g9|z_4Fg)W<_0EP$I!Y1>5A&Wr*a>>S_EaPilO8E zuY~ytr%TfpifEw|g8plISqsX~G7PjauyHmbSr=MSr!6OK;)271t6J*9)}F%kieqev&a>cgil{}Y-a)>Hrh literal 0 HcmV?d00001 diff --git a/static/docs_files_to_download/data-catalog/agent.model.schema.json b/static/docs_files_to_download/data-catalog/agent.model.schema.json new file mode 100644 index 0000000000..ef9d881511 --- /dev/null +++ b/static/docs_files_to_download/data-catalog/agent.model.schema.json @@ -0,0 +1,265 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "dc-agent-item", + "examples": [ + { + "ddlSchema": { + "properties": { + "FIRST_NAME": { + "nullable": true, + "size": 255, + "type": "VARCHAR2" + }, + "ID": { + "primary": true, + "type": "NUMBER", + "unique": true + }, + "INSERTED_ON": { + "primary": true, + "type": "TIMESTAMP(6)", + "unique": true + }, + "LAST_NAME": { + "nullable": true, + "size": 255, + "type": "VARCHAR2" + } + } + }, + "jsonSchema": { + "additionalProperties": false, + "properties": { + "FIRST_NAME": { + "type": "string" + }, + "ID": { + "type": "number" + }, + "INSERTED_ON": {}, + "LAST_NAME": { + "type": "string" + } + }, + "type": "object" + }, + "name": "books", + "source": { + "connection": { + "type": "odbc", + "vendor": "oracle", + "version": "Oracle Database 19c Enterprise Edition Release 19.0.0.0.0 - Production" + }, + "name": "oracle19c_test" + }, + "type": "model" + } + ], + "oneOf": [ + { + "type": "object", + "required": [ + "ddlSchema", + "jsonSchema", + "name", + "source", + "type" + ], + "properties": { + "ddlSchema": { + "$ref": "#/definitions/ddl-schema" + }, + "jsonSchema": { + "$ref": "#/definitions/json-schema" + }, + "name": { + "type": "string" + }, + "source": { + "$ref": "#/definitions/source-metadata" + }, + "type": { + "type": "string", + "enum": [ + "model" + ] + } + } + } + ], + "definitions": { + "ddl-property": { + "type": "object", + "required": [ + "type" + ], + "properties": { + "default": true, + "nullable": { + "type": "boolean" + }, + "precision": { + "type": [ + "integer", + "null" + ], + "format": "int32" + }, + "primary": { + "type": "boolean" + }, + "size": { + "type": [ + "integer", + "null" + ], + "format": "int32" + }, + "type": { + "$ref": "#/definitions/raw-type" + }, + "unique": { + "type": "boolean" + } + } + }, + "ddl-schema": { + "type": "object", + "required": [ + "properties" + ], + "properties": { + "properties": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/ddl-property" + } + } + } + }, + "json-primitive-type": { + "type": "string", + "enum": [ + "null", + "boolean", + "number", + "integer", + "string", + "array", + "object" + ] + }, + "json-property": { + "type": "object", + "properties": { + "default": true, + "nullable": { + "writeOnly": true, + "type": [ + "boolean", + "null" + ] + }, + "type": { + "anyOf": [ + { + "$ref": "#/definitions/json-type" + }, + { + "type": "null" + } + ] + } + } + }, + "json-schema": { + "type": "object", + "required": [ + "properties", + "type" + ], + "properties": { + "additionalProperties": { + "type": "boolean" + }, + "properties": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/json-property" + } + }, + "required": { + "type": "array", + "items": { + "type": "string" + } + }, + "type": { + "$ref": "#/definitions/json-primitive-type" + } + } + }, + "json-type": { + "anyOf": [ + { + "$ref": "#/definitions/json-primitive-type" + }, + { + "type": "array", + "items": { + "$ref": "#/definitions/json-primitive-type" + } + } + ] + }, + "raw-type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "source-connection-metadata": { + "type": "object", + "required": [ + "type", + "vendor" + ], + "properties": { + "type": { + "type": "string" + }, + "vendor": { + "type": "string" + }, + "version": { + "type": [ + "string", + "null" + ] + } + } + }, + "source-metadata": { + "type": "object", + "required": [ + "connection", + "name" + ], + "properties": { + "connection": { + "$ref": "#/definitions/source-connection-metadata" + }, + "name": { + "type": "string" + } + } + } + } +} \ No newline at end of file diff --git a/static/docs_files_to_download/data-catalog/catalog.model.schema.json b/static/docs_files_to_download/data-catalog/catalog.model.schema.json new file mode 100644 index 0000000000..8576fbdaa6 --- /dev/null +++ b/static/docs_files_to_download/data-catalog/catalog.model.schema.json @@ -0,0 +1,367 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "OpenLineage", + "examples": [ + { + "producer": "my_producer", + "schemaURL": "https://openlineage.io/spec/2-0-2/OpenLineage.json", + "eventTime": "2024-06-27 10:43:30.889698872 UTC", + "dataset": { + "datasetId": "mock-id", + "name": "books", + "namespace": "test", + "schema": "my-schema", + "catalogName": "test", + "facets": { + "schema": { + "_producer": "my_producer", + "_schemaURL": "customSchemaDatasetFacet.json", + "jsonSchema": { + "type": "object", + "additionalProperties": false, + "properties": { + "FIRST_NAME": { + "type": "string" + }, + "ID": { + "type": "number" + }, + "INSERTED_ON": {}, + "LAST_NAME": { + "type": "string" + } + } + }, + "fields": [ + { + "name": "FIRST_NAME", + "type": "VARCHAR2", + "required": false, + "unique": false, + "primaryKey": false, + "nullable": false, + "sourceDescription": "size=255, precision=0" + }, + { + "name": "ID", + "type": "NUMBER", + "required": true, + "unique": true, + "primaryKey": true, + "nullable": false, + "sourceDescription": "size=0, precision=0" + }, + { + "name": "INSERTED_ON", + "type": "TIMESTAMP(6)", + "required": true, + "unique": true, + "primaryKey": true, + "nullable": false, + "sourceDescription": "size=0, precision=0" + }, + { + "name": "LAST_NAME", + "type": "VARCHAR2", + "required": false, + "unique": false, + "primaryKey": false, + "nullable": false, + "sourceDescription": "size=255, precision=0" + } + ] + }, + "version": { + "_producer": "my_producer", + "_schemaURL": "https://openlineage.io/spec/facets/1-0-1/DatasetVersionDatasetFacet.json", + "datasetVersion": "1" + }, + "storage": { + "_producer": "my_producer", + "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/StorageDatasetFacet.json", + "storageLayer": "oracle19c_test", + "version": "Oracle Database 19c Enterprise Edition Release 19.0.0.0.0 - Production", + "vendor": "mysql" + } + } + } + } + ], + "type": "object", + "required": [ + "dataset", + "eventTime", + "producer" + ], + "properties": { + "dataset": { + "$ref": "#/definitions/Dataset" + }, + "eventTime": { + "type": "string" + }, + "producer": { + "type": "string" + }, + "schemaURL": { + "default": "https://openlineage.io/spec/2-0-2/OpenLineage.json", + "type": "string" + } + }, + "definitions": { + "Dataset": { + "type": "object", + "required": [ + "datasetId", + "facets", + "name", + "namespace" + ], + "properties": { + "catalogName": { + "type": [ + "string", + "null" + ] + }, + "datasetId": { + "type": "string" + }, + "facets": { + "$ref": "#/definitions/Facets" + }, + "name": { + "type": "string" + }, + "namespace": { + "type": "string" + }, + "schema": { + "type": [ + "string", + "null" + ] + } + } + }, + "FacetSchema": { + "type": "object", + "required": [ + "_producer", + "_schemaURL", + "fields" + ], + "properties": { + "_producer": { + "type": "string" + }, + "_schemaURL": { + "type": "string" + }, + "fields": { + "type": "array", + "items": { + "$ref": "#/definitions/Field" + } + }, + "jsonSchema": { + "default": { + "type": "object", + "additionalProperties": false, + "properties": {} + }, + "allOf": [ + { + "$ref": "#/definitions/json-schema" + } + ] + } + } + }, + "FacetStorage": { + "type": "object", + "required": [ + "_producer", + "_schemaURL", + "storageLayer", + "vendor", + "version" + ], + "properties": { + "_producer": { + "type": "string" + }, + "_schemaURL": { + "type": [ + "string", + "null" + ] + }, + "storageLayer": { + "type": "string" + }, + "vendor": { + "type": "string" + }, + "version": { + "type": "string" + } + } + }, + "FacetVersion": { + "type": "object", + "required": [ + "_producer", + "_schemaURL", + "datasetVersion" + ], + "properties": { + "_producer": { + "type": "string" + }, + "_schemaURL": { + "type": "string" + }, + "datasetVersion": { + "type": "string" + } + } + }, + "Facets": { + "type": "object", + "required": [ + "schema", + "storage", + "version" + ], + "properties": { + "schema": { + "$ref": "#/definitions/FacetSchema" + }, + "storage": { + "$ref": "#/definitions/FacetStorage" + }, + "version": { + "$ref": "#/definitions/FacetVersion" + } + } + }, + "Field": { + "type": "object", + "required": [ + "name", + "nullable", + "primaryKey", + "required", + "sourceDescription", + "type", + "unique" + ], + "properties": { + "default": { + "type": [ + "string", + "null" + ] + }, + "name": { + "type": "string" + }, + "nullable": { + "type": "boolean" + }, + "primaryKey": { + "type": "boolean" + }, + "required": { + "type": "boolean" + }, + "sourceDescription": { + "description": "field description extracted from the source system", + "type": "string" + }, + "type": { + "type": "string" + }, + "unique": { + "type": "boolean" + } + } + }, + "json-primitive-type": { + "type": "string", + "enum": [ + "null", + "boolean", + "number", + "integer", + "string", + "array", + "object" + ] + }, + "json-property": { + "type": "object", + "properties": { + "default": true, + "nullable": { + "writeOnly": true, + "type": [ + "boolean", + "null" + ] + }, + "type": { + "anyOf": [ + { + "$ref": "#/definitions/json-type" + }, + { + "type": "null" + } + ] + } + } + }, + "json-schema": { + "type": "object", + "required": [ + "properties", + "type" + ], + "properties": { + "additionalProperties": { + "type": "boolean" + }, + "properties": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/json-property" + } + }, + "required": { + "type": "array", + "items": { + "type": "string" + } + }, + "type": { + "$ref": "#/definitions/json-primitive-type" + } + } + }, + "json-type": { + "anyOf": [ + { + "$ref": "#/definitions/json-primitive-type" + }, + { + "type": "array", + "items": { + "$ref": "#/definitions/json-primitive-type" + } + } + ] + } + } +} \ No newline at end of file diff --git a/static/docs_files_to_download/data-catalog/configuration.schema.json b/static/docs_files_to_download/data-catalog/configuration.schema.json new file mode 100644 index 0000000000..7e2488eaf8 --- /dev/null +++ b/static/docs_files_to_download/data-catalog/configuration.schema.json @@ -0,0 +1,877 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "configuration", + "examples": [ + { + "connections": { + "postgres-16": { + "configuration": { + "params": { + "database": "my-database", + "host": "0.0.0.0", + "port": 5432, + "pwd": "*******", + "uid": "my-user" + }, + "vendor": "postgres" + }, + "type": "odbc" + } + }, + "target": { + "baseUrl": "https://console.cloud.mia-platform.eu", + "credentials": { + "clientId": "", + "clientKeyId": "", + "privateKey": "tls/private-key.pem" + }, + "oauthTokenEndpoint": "/api/m2m/oauth/token", + "projectId": "1234", + "revision": "main", + "type": "mia-console" + } + }, + { + "connections": { + "crud-service": { + "configuration": { + "params": { + "baseUrl": "http://mia-crud-service:3000", + "endpoint": "/-/schemas", + "headers": { + "accept": "application/x-ndjson" + } + }, + "vendor": "mia-crud-service" + }, + "type": "http" + } + }, + "target": { + "baseUrl": "https://console.cloud.mia-platform.eu", + "credentials": { + "clientId": "", + "clientKeyId": "", + "privateKey": "tls/private-key.pem" + }, + "oauthTokenEndpoint": "/api/m2m/oauth/token", + "projectId": "1234", + "revision": "main", + "type": "mia-console" + } + } + ], + "type": "object", + "properties": { + "connections": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/connection" + } + }, + "target": { + "default": { + "type": "stdout" + }, + "allOf": [ + { + "$ref": "#/definitions/target" + } + ] + } + }, + "definitions": { + "OracleMode": { + "oneOf": [ + { + "type": "string", + "enum": [ + "user" + ] + }, + { + "type": "object", + "required": [ + "all" + ], + "properties": { + "all": { + "type": "object", + "properties": { + "owners": { + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + } + } + } + } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": [ + "dba" + ], + "properties": { + "dba": { + "type": "object", + "properties": { + "owners": { + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + } + } + } + } + }, + "additionalProperties": false + } + ] + }, + "Settings": { + "type": "object", + "properties": { + "batchSize": { + "type": [ + "integer", + "null" + ], + "format": "uint32", + "minimum": 0.0 + }, + "columnBatchSize": { + "type": [ + "integer", + "null" + ], + "format": "uint32", + "minimum": 0.0 + }, + "connectionName": { + "type": [ + "string", + "null" + ] + } + } + }, + "connection": { + "oneOf": [ + { + "type": "object", + "required": [ + "configuration", + "type" + ], + "properties": { + "configuration": { + "$ref": "#/definitions/odbc-configuration" + }, + "type": { + "type": "string", + "enum": [ + "odbc" + ] + } + } + }, + { + "type": "object", + "required": [ + "configuration", + "type" + ], + "properties": { + "configuration": { + "$ref": "#/definitions/http-configuration" + }, + "type": { + "type": "string", + "enum": [ + "http" + ] + } + } + } + ] + }, + "credentials": { + "type": "object", + "required": [ + "clientId", + "clientKeyId", + "privateKey" + ], + "properties": { + "clientId": { + "type": "string" + }, + "clientKeyId": { + "$ref": "#/definitions/secret" + }, + "privateKey": { + "$ref": "#/definitions/secret" + } + } + }, + "http-configuration": { + "oneOf": [ + { + "type": "object", + "required": [ + "params", + "vendor" + ], + "properties": { + "params": { + "$ref": "#/definitions/mia-crud-service-params" + }, + "vendor": { + "type": "string", + "enum": [ + "mia-crud-service" + ] + } + } + }, + { + "type": "object", + "required": [ + "params", + "vendor" + ], + "properties": { + "params": { + "$ref": "#/definitions/salesforce-params" + }, + "vendor": { + "type": "string", + "enum": [ + "salesforce-sobjects" + ] + } + } + } + ] + }, + "mia-crud-service-params": { + "type": "object", + "required": [ + "baseUrl" + ], + "properties": { + "baseUrl": { + "type": "string" + }, + "endpoint": { + "default": "/-/schemas", + "type": "string" + }, + "headers": { + "default": { + "accept": "application/x-ndjson" + }, + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "healthcheck": { + "default": "/-/healthz", + "type": "string" + }, + "tls": { + "default": { + "insecure": false + }, + "allOf": [ + { + "$ref": "#/definitions/tls-params" + } + ] + } + } + }, + "mssql-connection-params": { + "examples": [ + { + "database": "db", + "flags": "TrustServerCertificate=yes;", + "host": "0.0.0.0", + "port": 1433, + "pwd": "*****", + "uid": "user" + }, + "Driver=mssql;Server=0.0.0.0,1433;Database=db;Uid=user;Pwd=p4ssw0rd;TrustServerCertificate=yes;" + ], + "anyOf": [ + { + "$ref": "#/definitions/secret" + }, + { + "type": "object", + "required": [ + "database", + "pwd", + "uid" + ], + "properties": { + "database": { + "type": "string" + }, + "driver": { + "type": "string" + }, + "flags": { + "type": [ + "string", + "null" + ] + }, + "host": { + "default": "0.0.0.0", + "type": "string" + }, + "port": { + "default": 1433, + "type": "integer", + "format": "uint16", + "minimum": 0.0 + }, + "pwd": { + "$ref": "#/definitions/secret" + }, + "uid": { + "$ref": "#/definitions/secret" + } + } + } + ] + }, + "mysql-connection-params": { + "examples": [ + { + "database": "db", + "host": "0.0.0.0", + "port": 3306, + "pwd": "*****", + "uid": "user" + }, + "DRIVER=mysql;SERVER=0.0.0.0;PORT=3306;DATABASE=db;UID=user;PWD=p4ssw0rd;" + ], + "anyOf": [ + { + "$ref": "#/definitions/secret" + }, + { + "type": "object", + "required": [ + "database", + "pwd", + "uid" + ], + "properties": { + "database": { + "type": "string" + }, + "driver": { + "type": "string" + }, + "flags": { + "type": [ + "string", + "null" + ] + }, + "host": { + "default": "0.0.0.0", + "type": "string" + }, + "port": { + "default": 3306, + "type": "integer", + "format": "uint16", + "minimum": 0.0 + }, + "pwd": { + "$ref": "#/definitions/secret" + }, + "uid": { + "$ref": "#/definitions/secret" + } + } + } + ] + }, + "odbc-configuration": { + "oneOf": [ + { + "type": "object", + "required": [ + "params", + "vendor" + ], + "properties": { + "params": { + "$ref": "#/definitions/mssql-connection-params" + }, + "vendor": { + "type": "string", + "enum": [ + "mssql" + ] + } + } + }, + { + "type": "object", + "required": [ + "params", + "vendor" + ], + "properties": { + "params": { + "$ref": "#/definitions/mysql-connection-params" + }, + "vendor": { + "type": "string", + "enum": [ + "mysql" + ] + } + } + }, + { + "type": "object", + "required": [ + "params", + "vendor" + ], + "properties": { + "options": { + "default": "user", + "allOf": [ + { + "$ref": "#/definitions/OracleMode" + } + ] + }, + "params": { + "$ref": "#/definitions/oracle-connection-params" + }, + "vendor": { + "type": "string", + "enum": [ + "oracle" + ] + }, + "version": { + "anyOf": [ + { + "$ref": "#/definitions/version" + }, + { + "type": "null" + } + ] + } + } + }, + { + "type": "object", + "required": [ + "params", + "vendor" + ], + "properties": { + "params": { + "$ref": "#/definitions/postgres-connection-params" + }, + "vendor": { + "type": "string", + "enum": [ + "postgres" + ] + } + } + } + ] + }, + "oracle-connection-params": { + "examples": [ + { + "dbq": "ORACLE_TEST", + "uid": "user", + "pwd": "****" + }, + { + "dbq": "(DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)(HOST=0.0.0.0)(PORT=1521))(CONNECT_DATA=(SERVER=DEDICATED)(SERVICE_NAME=ORACLE19C_TEST)))", + "flags": "DBA=R;", + "uid": "user", + "pwd": "****" + }, + "DRIVER=oracle;UID=user;PWD=p4ssw0rd;DBQ=(DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)(HOST=0.0.0.0)(PORT=1521))(CONNECT_DATA=(SERVER=DEDICATED)(SERVICE_NAME=ORACLE19C_TEST)))" + ], + "anyOf": [ + { + "$ref": "#/definitions/secret" + }, + { + "type": "object", + "required": [ + "dbq", + "pwd", + "uid" + ], + "properties": { + "dbq": { + "type": "string" + }, + "driver": { + "type": "string" + }, + "flags": { + "type": [ + "string", + "null" + ] + }, + "pwd": { + "$ref": "#/definitions/secret" + }, + "uid": { + "$ref": "#/definitions/secret" + } + } + } + ] + }, + "postgres-connection-params": { + "examples": [ + { + "database": "db", + "host": "0.0.0.0", + "port": 5432, + "pwd": "*****", + "uid": "user" + }, + "DRIVER=postgres;SERVER=0.0.0.0;PORT=5432;DATABASE=db;UID=user;PWD=p4ssw0rd;" + ], + "anyOf": [ + { + "$ref": "#/definitions/secret" + }, + { + "type": "object", + "required": [ + "database", + "pwd", + "uid" + ], + "properties": { + "database": { + "type": "string" + }, + "driver": { + "type": "string" + }, + "flags": { + "type": [ + "string", + "null" + ] + }, + "host": { + "default": "0.0.0.0", + "type": "string" + }, + "port": { + "default": 5432, + "type": "integer", + "format": "uint16", + "minimum": 0.0 + }, + "pwd": { + "$ref": "#/definitions/secret" + }, + "uid": { + "$ref": "#/definitions/secret" + } + } + } + ] + }, + "salesforce-params": { + "oneOf": [ + { + "type": "object", + "required": [ + "apiVersion", + "authenticationFlow", + "baseUrl", + "clientId", + "clientSecret", + "password", + "securityToken", + "username" + ], + "properties": { + "apiVersion": { + "type": "string" + }, + "authenticationFlow": { + "type": "string", + "enum": [ + "username-password" + ] + }, + "baseUrl": { + "type": "string" + }, + "clientId": { + "$ref": "#/definitions/secret" + }, + "clientSecret": { + "$ref": "#/definitions/secret" + }, + "password": { + "$ref": "#/definitions/secret" + }, + "securityToken": { + "$ref": "#/definitions/secret" + }, + "username": { + "$ref": "#/definitions/secret" + } + } + }, + { + "type": "object", + "required": [ + "apiVersion", + "authenticationFlow", + "baseUrl", + "clientId", + "privateKey", + "username" + ], + "properties": { + "apiVersion": { + "type": "string" + }, + "authenticationFlow": { + "type": "string", + "enum": [ + "jwt-bearer" + ] + }, + "baseUrl": { + "type": "string" + }, + "clientId": { + "$ref": "#/definitions/secret" + }, + "loginUrl": { + "default": "https://login.salesforce.com", + "type": "string" + }, + "privateKey": { + "$ref": "#/definitions/secret" + }, + "username": { + "$ref": "#/definitions/secret" + } + } + } + ] + }, + "secret": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "object", + "required": [ + "key", + "type" + ], + "properties": { + "encoding": { + "type": "string", + "enum": [ + "base64" + ] + }, + "key": { + "type": "string" + }, + "type": { + "const": "env" + } + } + }, + { + "type": "object", + "required": [ + "path", + "type" + ], + "properties": { + "encoding": { + "type": "string", + "enum": [ + "base64" + ] + }, + "key": { + "type": "string" + }, + "path": { + "type": "string" + }, + "type": { + "const": "file" + } + } + } + ] + }, + "target": { + "examples": [ + { + "type": "stdout" + }, + { + "dir": "/path/to/dir", + "type": "file" + }, + { + "baseUrl": "https://console.cloud.mia-platform.eu", + "credentials": { + "clientId": "", + "clientKeyId": "", + "privateKey": "tls/private-key.pem" + }, + "dataCatalogEndpoint": "/custom", + "oauthTokenEndpoint": "/api/m2m/oauth/token", + "type": "mia-console" + } + ], + "oneOf": [ + { + "type": "object", + "anyOf": [ + { + "type": "object", + "required": [ + "projectId" + ], + "properties": { + "projectId": { + "type": "string" + }, + "revision": { + "default": "main", + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "dataCatalogEndpoint" + ], + "properties": { + "dataCatalogEndpoint": { + "type": "string" + } + } + } + ], + "required": [ + "baseUrl", + "credentials", + "type" + ], + "properties": { + "baseUrl": { + "type": "string" + }, + "credentials": { + "$ref": "#/definitions/credentials" + }, + "headers": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "oauthTokenEndpoint": { + "default": "/api/m2m/oauth/token", + "type": "string" + }, + "type": { + "type": "string", + "enum": [ + "mia-console" + ] + } + } + }, + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "dir": { + "type": [ + "string", + "null" + ] + }, + "type": { + "type": "string", + "enum": [ + "file" + ] + } + } + }, + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "stdout" + ] + } + } + } + ] + }, + "version": { + "type": "string", + "enum": [ + "21", + "19", + "12", + "11" + ] + } + } +} \ No newline at end of file