From f5599d66fc3f5eda03fb4486ff3b47d22ae207d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Thu, 8 Aug 2024 16:36:02 +0200 Subject: [PATCH 1/4] doc: add rate limit config param --- .../rate-limiting.mdx | 47 +++++++++++++++++++ .../schema-extraction.mdx | 2 +- 2 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 docs/connectors/configuration-parameters/rate-limiting.mdx diff --git a/docs/connectors/configuration-parameters/rate-limiting.mdx b/docs/connectors/configuration-parameters/rate-limiting.mdx new file mode 100644 index 00000000..4cbb51fd --- /dev/null +++ b/docs/connectors/configuration-parameters/rate-limiting.mdx @@ -0,0 +1,47 @@ +--- +title: "Rate Limiting" +sidebar_position: 2 +--- + +Destination connectors can be configured to limit the rate at which records can be written. This is especially useful when the destination resource has a rate limit to ensure that the connector does not exceed it. By default, Conduit does not limit the rate at which records are written. + +## Configuration parameters + +* `sdk.rate.perSecond`: Maximum times records can be written per second (0 means no rate limit). +* `sdk.rate.burst`: Allow bursts of at most X writes (1 or less means that bursts are not allowed). Only takes effect if a rate limit per second is set. + +## Example + +The pipeline will generate an infinite number of structured records in bursts during 5 seconds at a maximum rate, and write them to the log. When you run it, you'll notice that by using both `sdk.rate.perSecond` and `sdk.rate.burst`, the log destination connector will limit the rate of processed records to 1 burst per second. + +```yaml +version: 2.2 +pipelines: + - id: generator-to-log + status: running + description: > + Example pipeline using the generator source connector and the log destination connector. + Showing how to limit the rate of processing records. + connectors: + - id: example + type: source + plugin: generator + settings: + rate: 0 + recordCount: 0 + burst.generateTime: 5s + burst.sleepTime: 0s + format.type: structured + format.options.id: int + format.options.name: string + operations: create + - id: log + type: destination + plugin: log + settings: + level: info + + # limit the rate of proccessing records + sdk.rate.perSecond: 1 + sdk.rate.burst: 1 +``` diff --git a/docs/connectors/configuration-parameters/schema-extraction.mdx b/docs/connectors/configuration-parameters/schema-extraction.mdx index 7a92c69e..f39b0305 100644 --- a/docs/connectors/configuration-parameters/schema-extraction.mdx +++ b/docs/connectors/configuration-parameters/schema-extraction.mdx @@ -1,6 +1,6 @@ --- title: "Schema Extraction" -sidebar_position: 2 +sidebar_position: 3 --- Source and destination connectors can be configured to automatically extract the From 86a71d072c255b121d361c1a70d308e70a09145c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Fri, 9 Aug 2024 18:39:09 +0200 Subject: [PATCH 2/4] update description --- docs/connectors/configuration-parameters/rate-limiting.mdx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/connectors/configuration-parameters/rate-limiting.mdx b/docs/connectors/configuration-parameters/rate-limiting.mdx index 4cbb51fd..f7a1a175 100644 --- a/docs/connectors/configuration-parameters/rate-limiting.mdx +++ b/docs/connectors/configuration-parameters/rate-limiting.mdx @@ -7,8 +7,8 @@ Destination connectors can be configured to limit the rate at which records can ## Configuration parameters -* `sdk.rate.perSecond`: Maximum times records can be written per second (0 means no rate limit). -* `sdk.rate.burst`: Allow bursts of at most X writes (1 or less means that bursts are not allowed). Only takes effect if a rate limit per second is set. +* `sdk.rate.perSecond`: Maximum number of records written per second (0 means no rate limit). +* `sdk.rate.burst`: Allow bursts of at most X records (0 or less means that bursts are not limited). Only takes effect if a rate limit per second is set. Note that if `sdk.batch.size` is bigger than `sdk.rate.burst`, the effective batch size will be equal to `sdk.rate.burst`. ## Example From 3aefc99c4895f5ce6b1d00961945286956d9da26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Mon, 12 Aug 2024 16:14:17 +0200 Subject: [PATCH 3/4] update example --- docs/connectors/configuration-parameters/rate-limiting.mdx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/connectors/configuration-parameters/rate-limiting.mdx b/docs/connectors/configuration-parameters/rate-limiting.mdx index f7a1a175..e4e54146 100644 --- a/docs/connectors/configuration-parameters/rate-limiting.mdx +++ b/docs/connectors/configuration-parameters/rate-limiting.mdx @@ -12,7 +12,7 @@ Destination connectors can be configured to limit the rate at which records can ## Example -The pipeline will generate an infinite number of structured records in bursts during 5 seconds at a maximum rate, and write them to the log. When you run it, you'll notice that by using both `sdk.rate.perSecond` and `sdk.rate.burst`, the log destination connector will limit the rate of processed records to 1 burst per second. +The pipeline will generate an infinite number of structured records in bursts during 5 seconds at a maximum rate, and write them to the log. When you run it, you'll notice that by using both `sdk.rate.perSecond` and `sdk.rate.burst`, the log destination connector will limit the rate of processed records 8 records per second in batches of 2 (i.e. 2 records every 250ms). ```yaml version: 2.2 @@ -42,6 +42,6 @@ pipelines: level: info # limit the rate of proccessing records - sdk.rate.perSecond: 1 - sdk.rate.burst: 1 + sdk.rate.perSecond: 8 + sdk.rate.burst: 2 ``` From 7284a5307c2677956455e741956f5ec69ce9b181 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Mon, 12 Aug 2024 16:55:00 +0200 Subject: [PATCH 4/4] tweaks --- docs/connectors/configuration-parameters/rate-limiting.mdx | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/docs/connectors/configuration-parameters/rate-limiting.mdx b/docs/connectors/configuration-parameters/rate-limiting.mdx index e4e54146..a821aa54 100644 --- a/docs/connectors/configuration-parameters/rate-limiting.mdx +++ b/docs/connectors/configuration-parameters/rate-limiting.mdx @@ -12,7 +12,7 @@ Destination connectors can be configured to limit the rate at which records can ## Example -The pipeline will generate an infinite number of structured records in bursts during 5 seconds at a maximum rate, and write them to the log. When you run it, you'll notice that by using both `sdk.rate.perSecond` and `sdk.rate.burst`, the log destination connector will limit the rate of processed records 8 records per second in batches of 2 (i.e. 2 records every 250ms). +The pipeline will generate structured records as fast as possible, and write them to the log. When you run it, you'll notice that by using both `sdk.rate.perSecond` and `sdk.rate.burst`, the log destination connector will limit the rate of processed records 8 records per second in batches of 2 (i.e. 2 records every 250ms). ```yaml version: 2.2 @@ -27,10 +27,7 @@ pipelines: type: source plugin: generator settings: - rate: 0 - recordCount: 0 - burst.generateTime: 5s - burst.sleepTime: 0s + rate: 0 # generating source records as fast as possible format.type: structured format.options.id: int format.options.name: string