From 13ededd0a5ac7fe400bb992b9d439a52b9b72384 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Haris=20Osmanagi=C4=87?= Date: Wed, 31 Jul 2024 16:01:55 +0200 Subject: [PATCH] Add pages about storage, configuration (#107) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add pages about storage, configuration * sort alphabetically (#110) --------- Co-authored-by: Raúl Barroso --- docs/features/configuration.mdx | 32 +++++++++++++++++++++++++++++ docs/features/dead-letter-queue.mdx | 2 +- docs/features/metrics.mdx | 2 +- docs/features/opencdc-record.mdx | 2 +- docs/features/record-routing.mdx | 2 +- docs/features/storage.mdx | 28 +++++++++++++++++++++++++ docs/features/stream-inspector.mdx | 2 +- docs/features/web-ui.mdx | 2 +- 8 files changed, 66 insertions(+), 6 deletions(-) create mode 100644 docs/features/configuration.mdx create mode 100644 docs/features/storage.mdx diff --git a/docs/features/configuration.mdx b/docs/features/configuration.mdx new file mode 100644 index 00000000..c302a387 --- /dev/null +++ b/docs/features/configuration.mdx @@ -0,0 +1,32 @@ +--- +title: 'Configuration' +sidebar_position: 1 +--- + + +Conduit accepts CLI flags, environment variables and a configuration file to +configure its behavior. Each CLI flag has a corresponding environment variable +and a corresponding field in the configuration file. Conduit uses the value for +each configuration option based on the following priorities: + +- **CLI flags** (highest priority) - if a CLI flag is provided it will always be + respected, regardless of the environment variable or configuration file. To + see a full list of available flags run `conduit --help`. +- **Environment variables** (lower priority) - an environment variable is only + used if no CLI flag is provided for the same option. Environment variables + have the prefix `CONDUIT` and contain underscores instead of dots and + hyphens (e.g. the flag `-db.postgres.connection-string` corresponds + to `CONDUIT_DB_POSTGRES_CONNECTION_STRING`). +- **Configuration file** (lowest priority) - Conduit by default loads the + file `conduit.yaml` placed in the same folder as Conduit. The path to the file + can be customized using the CLI flag `-config`. It is not required to provide + a configuration file and any value in the configuration file can be overridden + by an environment variable or a flag. The file content should be a YAML + document where keys can be hierarchically split on `.`. For example: + + ```yaml + db: + type: postgres # corresponds to flag -db.type and env variable CONDUIT_DB_TYPE + postgres: + connection-string: postgres://localhost:5432/conduitdb # -db.postgres.connection-string or CONDUIT_DB_POSTGRES_CONNECTION_STRING + ``` diff --git a/docs/features/dead-letter-queue.mdx b/docs/features/dead-letter-queue.mdx index 12f617ee..cce21b86 100644 --- a/docs/features/dead-letter-queue.mdx +++ b/docs/features/dead-letter-queue.mdx @@ -1,6 +1,6 @@ --- title: 'Dead-letter queue' -sidebar_position: 1 +sidebar_position: 2 --- A dead-letter queue is a connector to which Conduit redirects records that can't diff --git a/docs/features/metrics.mdx b/docs/features/metrics.mdx index b9ad6fea..a21074b4 100644 --- a/docs/features/metrics.mdx +++ b/docs/features/metrics.mdx @@ -1,6 +1,6 @@ --- title: 'Metrics' -sidebar_position: 2 +sidebar_position: 3 --- Conduit comes with a number of already defined metrics. The metrics available diff --git a/docs/features/opencdc-record.mdx b/docs/features/opencdc-record.mdx index a781cbc7..7c62d0cc 100644 --- a/docs/features/opencdc-record.mdx +++ b/docs/features/opencdc-record.mdx @@ -1,6 +1,6 @@ --- title: 'OpenCDC record' -sidebar_position: 3 +sidebar_position: 4 --- An OpenCDC record in Conduit aims to standardize the format of data records exchanged between different connectors within a data processing pipeline. The primary objective is to ensure compatibility between various combinations of source and destination connectors. diff --git a/docs/features/record-routing.mdx b/docs/features/record-routing.mdx index bb06405a..d8727d95 100644 --- a/docs/features/record-routing.mdx +++ b/docs/features/record-routing.mdx @@ -1,6 +1,6 @@ --- title: 'Record routing' -sidebar_position: 4 +sidebar_position: 6 keywords: [ 'synchronization', 'collections', 'routing' ] --- diff --git a/docs/features/storage.mdx b/docs/features/storage.mdx new file mode 100644 index 00000000..d95b353c --- /dev/null +++ b/docs/features/storage.mdx @@ -0,0 +1,28 @@ +--- +title: 'Storage' +sidebar_position: 7 +--- + +Conduit's own data (information about pipelines, connectors, etc.) can be stored +in the following databases: +* BadgerDB (default) +* PostgreSQL +* SQLite + +It's also possible to store all the data in memory, which is sometimes useful +for development purposes. + +The database type used can be configured with the `db.type` parameter (through +any of the [configuration](/docs/features/configuration) options in Conduit). +For example, the CLI flag to use a PostgresSQL database with Conduit is as +follows: `-db.type=postgres`. + +Changing database parameters (e.g. the PostgreSQL connection string) is done +through parameters of the following form: `db..`. For +example, the CLI flag to use a PostgreSQL instance listening on `localhost:5432` +would be: `-db.postgres.connection-string=postgres://localhost:5432/conduitdb`. + +The full example in our case would be: +```shell +./conduit -db.type=postgres -db.postgres.connection-string="postgresql://localhost:5432/conduitdb" +``` diff --git a/docs/features/stream-inspector.mdx b/docs/features/stream-inspector.mdx index ec219b65..849c8f65 100644 --- a/docs/features/stream-inspector.mdx +++ b/docs/features/stream-inspector.mdx @@ -1,6 +1,6 @@ --- title: 'Stream Inspector' -sidebar_position: 6 +sidebar_position: 8 tag: ['Stream Inspector', 'Connector'] keywords: ['stream', 'inspector'] --- diff --git a/docs/features/web-ui.mdx b/docs/features/web-ui.mdx index 687c4989..c2f6c587 100644 --- a/docs/features/web-ui.mdx +++ b/docs/features/web-ui.mdx @@ -1,6 +1,6 @@ --- title: 'Web UI' -sidebar_position: 7 +sidebar_position: 9 --- Conduit comes with a web UI that allows you to build a data pipeline using a