From fb1ae815252c67310051b16b3b410437f6faacdd Mon Sep 17 00:00:00 2001 From: shantanualshi Date: Wed, 1 Nov 2023 12:50:21 +0530 Subject: [PATCH 01/13] Remove deprecated ruler_remote_write_url --- docs/sources/configure/_index.md | 4 ---- pkg/ruler/compat.go | 1 - pkg/ruler/registry.go | 10 ---------- pkg/validation/limits.go | 8 -------- .../deprecated-config-checker/checker/checker_test.go | 3 --- tools/deprecated-config-checker/deprecated-config.yaml | 1 - .../test-fixtures/config.yaml | 1 - .../test-fixtures/runtime-config.yaml | 1 - 8 files changed, 29 deletions(-) diff --git a/docs/sources/configure/_index.md b/docs/sources/configure/_index.md index fab1750c5c3f..8768fdb82cc1 100644 --- a/docs/sources/configure/_index.md +++ b/docs/sources/configure/_index.md @@ -2782,10 +2782,6 @@ The `limits_config` block configures global and per-tenant limits in Loki. # Disable recording rules remote-write. [ruler_remote_write_disabled: ] -# Deprecated: Use 'ruler_remote_write_config' instead. The URL of the endpoint -# to send samples to. -[ruler_remote_write_url: | default = ""] - # Deprecated: Use 'ruler_remote_write_config' instead. Timeout for requests to # the remote write endpoint. [ruler_remote_write_timeout: ] diff --git a/pkg/ruler/compat.go b/pkg/ruler/compat.go index c6d8d62dd86f..984dce55e587 100644 --- a/pkg/ruler/compat.go +++ b/pkg/ruler/compat.go @@ -37,7 +37,6 @@ type RulesLimits interface { ruler.RulesLimits RulerRemoteWriteDisabled(userID string) bool - RulerRemoteWriteURL(userID string) string RulerRemoteWriteTimeout(userID string) time.Duration RulerRemoteWriteHeaders(userID string) map[string]string RulerRemoteWriteRelabelConfigs(userID string) []*util.RelabelConfig diff --git a/pkg/ruler/registry.go b/pkg/ruler/registry.go index ea1e0848dc48..73bc7f37d48f 100644 --- a/pkg/ruler/registry.go +++ b/pkg/ruler/registry.go @@ -3,7 +3,6 @@ package ruler import ( "context" "fmt" - "net/url" "strings" "time" @@ -13,7 +12,6 @@ import ( "github.com/imdario/mergo" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" - promConfig "github.com/prometheus/common/config" "github.com/prometheus/common/model" "github.com/prometheus/prometheus/config" "github.com/prometheus/prometheus/model/exemplar" @@ -239,14 +237,6 @@ func (r *walRegistry) getTenantRemoteWriteConfig(tenant string, base RemoteWrite // metadata is only used by prometheus scrape configs clt.MetadataConfig = config.MetadataConfig{Send: false} - // Keeping these blocks for backward compatibility - if v := r.overrides.RulerRemoteWriteURL(tenant); v != "" { - u, err := url.Parse(v) - if err != nil { - return nil, fmt.Errorf("error parsing given remote-write URL: %w", err) - } - clt.URL = &promConfig.URL{u} - } if v := r.overrides.RulerRemoteWriteTimeout(tenant); v > 0 { clt.RemoteTimeout = model.Duration(v) } diff --git a/pkg/validation/limits.go b/pkg/validation/limits.go index 0600b9ebb5ff..29968d4a0c35 100644 --- a/pkg/validation/limits.go +++ b/pkg/validation/limits.go @@ -127,8 +127,6 @@ type Limits struct { // and if it were ruler_remote_write_enabled, it would be impossible to know if the value was explicitly set or default RulerRemoteWriteDisabled bool `yaml:"ruler_remote_write_disabled" json:"ruler_remote_write_disabled" doc:"description=Disable recording rules remote-write."` - // deprecated use RulerRemoteWriteConfig instead. - RulerRemoteWriteURL string `yaml:"ruler_remote_write_url" json:"ruler_remote_write_url" doc:"deprecated|description=Use 'ruler_remote_write_config' instead. The URL of the endpoint to send samples to."` // deprecated use RulerRemoteWriteConfig instead RulerRemoteWriteTimeout time.Duration `yaml:"ruler_remote_write_timeout" json:"ruler_remote_write_timeout" doc:"deprecated|description=Use 'ruler_remote_write_config' instead. Timeout for requests to the remote write endpoint."` // deprecated use RulerRemoteWriteConfig instead @@ -617,12 +615,6 @@ func (o *Overrides) RulerRemoteWriteDisabled(userID string) bool { return o.getOverridesForUser(userID).RulerRemoteWriteDisabled } -// Deprecated: use RulerRemoteWriteConfig instead -// RulerRemoteWriteURL returns the remote-write URL to use for a given user. -func (o *Overrides) RulerRemoteWriteURL(userID string) string { - return o.getOverridesForUser(userID).RulerRemoteWriteURL -} - // Deprecated: use RulerRemoteWriteConfig instead // RulerRemoteWriteTimeout returns the duration after which to timeout a remote-write request for a given user. func (o *Overrides) RulerRemoteWriteTimeout(userID string) time.Duration { diff --git a/tools/deprecated-config-checker/checker/checker_test.go b/tools/deprecated-config-checker/checker/checker_test.go index e02315821d4b..477f28e22038 100644 --- a/tools/deprecated-config-checker/checker/checker_test.go +++ b/tools/deprecated-config-checker/checker/checker_test.go @@ -47,7 +47,6 @@ var ( "chunk_store_config.write_dedupe_cache_config", "limits_config.unordered_writes", "limits_config.ruler_evaluation_delay_duration", - "limits_config.ruler_remote_write_url", "limits_config.ruler_remote_write_timeout", "limits_config.ruler_remote_write_headers", "limits_config.ruler_remote_write_relabel_configs", @@ -86,7 +85,6 @@ var ( expectedRuntimeConfigDeprecates = []string{ "overrides.foo.unordered_writes", "overrides.foo.ruler_evaluation_delay_duration", - "overrides.foo.ruler_remote_write_url", "overrides.foo.ruler_remote_write_timeout", "overrides.foo.ruler_remote_write_headers", "overrides.foo.ruler_remote_write_relabel_configs", @@ -104,7 +102,6 @@ var ( "overrides.foo.allow_deletes", "overrides.bar.unordered_writes", "overrides.bar.ruler_evaluation_delay_duration", - "overrides.bar.ruler_remote_write_url", "overrides.bar.ruler_remote_write_timeout", "overrides.bar.ruler_remote_write_headers", "overrides.bar.ruler_remote_write_relabel_configs", diff --git a/tools/deprecated-config-checker/deprecated-config.yaml b/tools/deprecated-config-checker/deprecated-config.yaml index 0cd8e8fd8c81..12cd56caa6af 100644 --- a/tools/deprecated-config-checker/deprecated-config.yaml +++ b/tools/deprecated-config-checker/deprecated-config.yaml @@ -49,7 +49,6 @@ chunk_store_config: limits_config: unordered_writes: "Will be eventually removed." ruler_evaluation_delay_duration: "Will be eventually removed." - ruler_remote_write_url: "Use ruler_remote_write_config instead." ruler_remote_write_timeout: "Use ruler_remote_write_config instead." ruler_remote_write_headers: "Use ruler_remote_write_config instead." ruler_remote_write_relabel_configs: "Use ruler_remote_write_config instead." diff --git a/tools/deprecated-config-checker/test-fixtures/config.yaml b/tools/deprecated-config-checker/test-fixtures/config.yaml index 80d864188b0f..1f68b82f218e 100644 --- a/tools/deprecated-config-checker/test-fixtures/config.yaml +++ b/tools/deprecated-config-checker/test-fixtures/config.yaml @@ -118,7 +118,6 @@ limits_config: ingestion_rate_mb: 100 unordered_writes: true # DEPRECATED ruler_evaluation_delay_duration: 1m # DEPRECATED - ruler_remote_write_url: "push.123abc.net" # DEPRECATED ruler_remote_write_timeout: 1m # DEPRECATED ruler_remote_write_headers: ["foo", "bar"] # DEPRECATED ruler_remote_write_relabel_configs: "foo" # DEPRECATED diff --git a/tools/deprecated-config-checker/test-fixtures/runtime-config.yaml b/tools/deprecated-config-checker/test-fixtures/runtime-config.yaml index 9500ce719724..df99cac695ea 100644 --- a/tools/deprecated-config-checker/test-fixtures/runtime-config.yaml +++ b/tools/deprecated-config-checker/test-fixtures/runtime-config.yaml @@ -3,7 +3,6 @@ overrides: ingestion_rate_mb: 100 unordered_writes: true # DEPRECATED ruler_evaluation_delay_duration: 1m # DEPRECATED - ruler_remote_write_url: "push.123abc.net" # DEPRECATED ruler_remote_write_timeout: 1m # DEPRECATED ruler_remote_write_headers: [ "foo", "bar" ] # DEPRECATED ruler_remote_write_relabel_configs: "foo" # DEPRECATED From 846c302edde8c9c1c9c8d68876153a0a7cad844a Mon Sep 17 00:00:00 2001 From: shantanualshi Date: Wed, 1 Nov 2023 12:52:31 +0530 Subject: [PATCH 02/13] Remove deprecated ruler_remote_write_timeout --- docs/sources/configure/_index.md | 4 ---- docs/sources/operations/overrides-exporter.md | 1 - pkg/ruler/compat.go | 1 - pkg/ruler/registry.go | 4 ---- pkg/validation/limits.go | 8 -------- tools/deprecated-config-checker/checker/checker_test.go | 3 --- tools/deprecated-config-checker/deprecated-config.yaml | 1 - tools/deprecated-config-checker/test-fixtures/config.yaml | 1 - .../test-fixtures/runtime-config.yaml | 1 - 9 files changed, 24 deletions(-) diff --git a/docs/sources/configure/_index.md b/docs/sources/configure/_index.md index 8768fdb82cc1..93cc6071b095 100644 --- a/docs/sources/configure/_index.md +++ b/docs/sources/configure/_index.md @@ -2782,10 +2782,6 @@ The `limits_config` block configures global and per-tenant limits in Loki. # Disable recording rules remote-write. [ruler_remote_write_disabled: ] -# Deprecated: Use 'ruler_remote_write_config' instead. Timeout for requests to -# the remote write endpoint. -[ruler_remote_write_timeout: ] - # Deprecated: Use 'ruler_remote_write_config' instead. Custom HTTP headers to be # sent along with each remote write request. Be aware that headers that are set # by Loki itself can't be overwritten. diff --git a/docs/sources/operations/overrides-exporter.md b/docs/sources/operations/overrides-exporter.md index e66c408e17b3..97833e3c63c9 100644 --- a/docs/sources/operations/overrides-exporter.md +++ b/docs/sources/operations/overrides-exporter.md @@ -80,7 +80,6 @@ loki_overrides_defaults{limit_name="ruler_remote_write_queue_max_samples_per_sen loki_overrides_defaults{limit_name="ruler_remote_write_queue_max_shards"} 0 loki_overrides_defaults{limit_name="ruler_remote_write_queue_min_backoff"} 0 loki_overrides_defaults{limit_name="ruler_remote_write_queue_min_shards"} 0 -loki_overrides_defaults{limit_name="ruler_remote_write_timeout"} 0 loki_overrides_defaults{limit_name="split_queries_by_interval"} 0 ``` diff --git a/pkg/ruler/compat.go b/pkg/ruler/compat.go index 984dce55e587..c71d835f3025 100644 --- a/pkg/ruler/compat.go +++ b/pkg/ruler/compat.go @@ -37,7 +37,6 @@ type RulesLimits interface { ruler.RulesLimits RulerRemoteWriteDisabled(userID string) bool - RulerRemoteWriteTimeout(userID string) time.Duration RulerRemoteWriteHeaders(userID string) map[string]string RulerRemoteWriteRelabelConfigs(userID string) []*util.RelabelConfig RulerRemoteWriteConfig(userID string, id string) *config.RemoteWriteConfig diff --git a/pkg/ruler/registry.go b/pkg/ruler/registry.go index 73bc7f37d48f..a01fff0574eb 100644 --- a/pkg/ruler/registry.go +++ b/pkg/ruler/registry.go @@ -237,10 +237,6 @@ func (r *walRegistry) getTenantRemoteWriteConfig(tenant string, base RemoteWrite // metadata is only used by prometheus scrape configs clt.MetadataConfig = config.MetadataConfig{Send: false} - if v := r.overrides.RulerRemoteWriteTimeout(tenant); v > 0 { - clt.RemoteTimeout = model.Duration(v) - } - // overwrite, do not merge if v := r.overrides.RulerRemoteWriteHeaders(tenant); v != nil { clt.Headers = v diff --git a/pkg/validation/limits.go b/pkg/validation/limits.go index 29968d4a0c35..79c4862f2af2 100644 --- a/pkg/validation/limits.go +++ b/pkg/validation/limits.go @@ -127,8 +127,6 @@ type Limits struct { // and if it were ruler_remote_write_enabled, it would be impossible to know if the value was explicitly set or default RulerRemoteWriteDisabled bool `yaml:"ruler_remote_write_disabled" json:"ruler_remote_write_disabled" doc:"description=Disable recording rules remote-write."` - // deprecated use RulerRemoteWriteConfig instead - RulerRemoteWriteTimeout time.Duration `yaml:"ruler_remote_write_timeout" json:"ruler_remote_write_timeout" doc:"deprecated|description=Use 'ruler_remote_write_config' instead. Timeout for requests to the remote write endpoint."` // deprecated use RulerRemoteWriteConfig instead RulerRemoteWriteHeaders OverwriteMarshalingStringMap `yaml:"ruler_remote_write_headers" json:"ruler_remote_write_headers" doc:"deprecated|description=Use 'ruler_remote_write_config' instead. Custom HTTP headers to be sent along with each remote write request. Be aware that headers that are set by Loki itself can't be overwritten."` // deprecated use RulerRemoteWriteConfig instead @@ -615,12 +613,6 @@ func (o *Overrides) RulerRemoteWriteDisabled(userID string) bool { return o.getOverridesForUser(userID).RulerRemoteWriteDisabled } -// Deprecated: use RulerRemoteWriteConfig instead -// RulerRemoteWriteTimeout returns the duration after which to timeout a remote-write request for a given user. -func (o *Overrides) RulerRemoteWriteTimeout(userID string) time.Duration { - return o.getOverridesForUser(userID).RulerRemoteWriteTimeout -} - // Deprecated: use RulerRemoteWriteConfig instead // RulerRemoteWriteHeaders returns the headers to use in a remote-write for a given user. func (o *Overrides) RulerRemoteWriteHeaders(userID string) map[string]string { diff --git a/tools/deprecated-config-checker/checker/checker_test.go b/tools/deprecated-config-checker/checker/checker_test.go index 477f28e22038..b4c7697d6e03 100644 --- a/tools/deprecated-config-checker/checker/checker_test.go +++ b/tools/deprecated-config-checker/checker/checker_test.go @@ -47,7 +47,6 @@ var ( "chunk_store_config.write_dedupe_cache_config", "limits_config.unordered_writes", "limits_config.ruler_evaluation_delay_duration", - "limits_config.ruler_remote_write_timeout", "limits_config.ruler_remote_write_headers", "limits_config.ruler_remote_write_relabel_configs", "limits_config.ruler_remote_write_queue_capacity", @@ -85,7 +84,6 @@ var ( expectedRuntimeConfigDeprecates = []string{ "overrides.foo.unordered_writes", "overrides.foo.ruler_evaluation_delay_duration", - "overrides.foo.ruler_remote_write_timeout", "overrides.foo.ruler_remote_write_headers", "overrides.foo.ruler_remote_write_relabel_configs", "overrides.foo.ruler_remote_write_queue_capacity", @@ -102,7 +100,6 @@ var ( "overrides.foo.allow_deletes", "overrides.bar.unordered_writes", "overrides.bar.ruler_evaluation_delay_duration", - "overrides.bar.ruler_remote_write_timeout", "overrides.bar.ruler_remote_write_headers", "overrides.bar.ruler_remote_write_relabel_configs", "overrides.bar.ruler_remote_write_queue_capacity", diff --git a/tools/deprecated-config-checker/deprecated-config.yaml b/tools/deprecated-config-checker/deprecated-config.yaml index 12cd56caa6af..9452fbe7ed50 100644 --- a/tools/deprecated-config-checker/deprecated-config.yaml +++ b/tools/deprecated-config-checker/deprecated-config.yaml @@ -49,7 +49,6 @@ chunk_store_config: limits_config: unordered_writes: "Will be eventually removed." ruler_evaluation_delay_duration: "Will be eventually removed." - ruler_remote_write_timeout: "Use ruler_remote_write_config instead." ruler_remote_write_headers: "Use ruler_remote_write_config instead." ruler_remote_write_relabel_configs: "Use ruler_remote_write_config instead." ruler_remote_write_queue_capacity: "Use ruler_remote_write_config instead." diff --git a/tools/deprecated-config-checker/test-fixtures/config.yaml b/tools/deprecated-config-checker/test-fixtures/config.yaml index 1f68b82f218e..8d4f4587c602 100644 --- a/tools/deprecated-config-checker/test-fixtures/config.yaml +++ b/tools/deprecated-config-checker/test-fixtures/config.yaml @@ -118,7 +118,6 @@ limits_config: ingestion_rate_mb: 100 unordered_writes: true # DEPRECATED ruler_evaluation_delay_duration: 1m # DEPRECATED - ruler_remote_write_timeout: 1m # DEPRECATED ruler_remote_write_headers: ["foo", "bar"] # DEPRECATED ruler_remote_write_relabel_configs: "foo" # DEPRECATED ruler_remote_write_queue_capacity: 10 # DEPRECATED diff --git a/tools/deprecated-config-checker/test-fixtures/runtime-config.yaml b/tools/deprecated-config-checker/test-fixtures/runtime-config.yaml index df99cac695ea..ada8f7e1f619 100644 --- a/tools/deprecated-config-checker/test-fixtures/runtime-config.yaml +++ b/tools/deprecated-config-checker/test-fixtures/runtime-config.yaml @@ -3,7 +3,6 @@ overrides: ingestion_rate_mb: 100 unordered_writes: true # DEPRECATED ruler_evaluation_delay_duration: 1m # DEPRECATED - ruler_remote_write_timeout: 1m # DEPRECATED ruler_remote_write_headers: [ "foo", "bar" ] # DEPRECATED ruler_remote_write_relabel_configs: "foo" # DEPRECATED ruler_remote_write_queue_capacity: 10 # DEPRECATED From c0a758465c1f3bcb8cfa80f99f1d25d40cbfc77c Mon Sep 17 00:00:00 2001 From: shantanualshi Date: Wed, 1 Nov 2023 15:16:38 +0530 Subject: [PATCH 03/13] Remove deprecated ruler_remote_write_queue_max_shards and min_shards --- docs/sources/configure/_index.md | 8 -------- docs/sources/operations/overrides-exporter.md | 2 -- pkg/ruler/compat.go | 2 -- pkg/ruler/registry.go | 8 -------- pkg/validation/limits.go | 16 ---------------- .../checker/checker_test.go | 6 ------ .../deprecated-config.yaml | 2 -- .../test-fixtures/config.yaml | 2 -- .../test-fixtures/runtime-config.yaml | 2 -- 9 files changed, 48 deletions(-) diff --git a/docs/sources/configure/_index.md b/docs/sources/configure/_index.md index 93cc6071b095..d57bbe2ca64e 100644 --- a/docs/sources/configure/_index.md +++ b/docs/sources/configure/_index.md @@ -2797,14 +2797,6 @@ The `limits_config` block configures global and per-tenant limits in Loki. # to keep throughput up while processing occasional slow remote requests. [ruler_remote_write_queue_capacity: ] -# Deprecated: Use 'ruler_remote_write_config' instead. Minimum number of shards, -# i.e. amount of concurrency. -[ruler_remote_write_queue_min_shards: ] - -# Deprecated: Use 'ruler_remote_write_config' instead. Maximum number of shards, -# i.e. amount of concurrency. -[ruler_remote_write_queue_max_shards: ] - # Deprecated: Use 'ruler_remote_write_config' instead. Maximum number of samples # per send. [ruler_remote_write_queue_max_samples_per_send: ] diff --git a/docs/sources/operations/overrides-exporter.md b/docs/sources/operations/overrides-exporter.md index 97833e3c63c9..5f0ee8bf04a2 100644 --- a/docs/sources/operations/overrides-exporter.md +++ b/docs/sources/operations/overrides-exporter.md @@ -77,9 +77,7 @@ loki_overrides_defaults{limit_name="ruler_remote_write_queue_batch_send_deadline loki_overrides_defaults{limit_name="ruler_remote_write_queue_capacity"} 0 loki_overrides_defaults{limit_name="ruler_remote_write_queue_max_backoff"} 0 loki_overrides_defaults{limit_name="ruler_remote_write_queue_max_samples_per_send"} 0 -loki_overrides_defaults{limit_name="ruler_remote_write_queue_max_shards"} 0 loki_overrides_defaults{limit_name="ruler_remote_write_queue_min_backoff"} 0 -loki_overrides_defaults{limit_name="ruler_remote_write_queue_min_shards"} 0 loki_overrides_defaults{limit_name="split_queries_by_interval"} 0 ``` diff --git a/pkg/ruler/compat.go b/pkg/ruler/compat.go index c71d835f3025..2abde21c0f09 100644 --- a/pkg/ruler/compat.go +++ b/pkg/ruler/compat.go @@ -41,8 +41,6 @@ type RulesLimits interface { RulerRemoteWriteRelabelConfigs(userID string) []*util.RelabelConfig RulerRemoteWriteConfig(userID string, id string) *config.RemoteWriteConfig RulerRemoteWriteQueueCapacity(userID string) int - RulerRemoteWriteQueueMinShards(userID string) int - RulerRemoteWriteQueueMaxShards(userID string) int RulerRemoteWriteQueueMaxSamplesPerSend(userID string) int RulerRemoteWriteQueueBatchSendDeadline(userID string) time.Duration RulerRemoteWriteQueueMinBackoff(userID string) time.Duration diff --git a/pkg/ruler/registry.go b/pkg/ruler/registry.go index a01fff0574eb..9cb73ffd5eac 100644 --- a/pkg/ruler/registry.go +++ b/pkg/ruler/registry.go @@ -258,14 +258,6 @@ func (r *walRegistry) getTenantRemoteWriteConfig(tenant string, base RemoteWrite clt.QueueConfig.Capacity = v } - if v := r.overrides.RulerRemoteWriteQueueMinShards(tenant); v > 0 { - clt.QueueConfig.MinShards = v - } - - if v := r.overrides.RulerRemoteWriteQueueMaxShards(tenant); v > 0 { - clt.QueueConfig.MaxShards = v - } - if v := r.overrides.RulerRemoteWriteQueueMaxSamplesPerSend(tenant); v > 0 { clt.QueueConfig.MaxSamplesPerSend = v } diff --git a/pkg/validation/limits.go b/pkg/validation/limits.go index 79c4862f2af2..3ac482279652 100644 --- a/pkg/validation/limits.go +++ b/pkg/validation/limits.go @@ -134,10 +134,6 @@ type Limits struct { // deprecated use RulerRemoteWriteConfig instead RulerRemoteWriteQueueCapacity int `yaml:"ruler_remote_write_queue_capacity" json:"ruler_remote_write_queue_capacity" doc:"deprecated|description=Use 'ruler_remote_write_config' instead. Number of samples to buffer per shard before we block reading of more samples from the WAL. It is recommended to have enough capacity in each shard to buffer several requests to keep throughput up while processing occasional slow remote requests."` // deprecated use RulerRemoteWriteConfig instead - RulerRemoteWriteQueueMinShards int `yaml:"ruler_remote_write_queue_min_shards" json:"ruler_remote_write_queue_min_shards" doc:"deprecated|description=Use 'ruler_remote_write_config' instead. Minimum number of shards, i.e. amount of concurrency."` - // deprecated use RulerRemoteWriteConfig instead - RulerRemoteWriteQueueMaxShards int `yaml:"ruler_remote_write_queue_max_shards" json:"ruler_remote_write_queue_max_shards" doc:"deprecated|description=Use 'ruler_remote_write_config' instead. Maximum number of shards, i.e. amount of concurrency."` - // deprecated use RulerRemoteWriteConfig instead RulerRemoteWriteQueueMaxSamplesPerSend int `yaml:"ruler_remote_write_queue_max_samples_per_send" json:"ruler_remote_write_queue_max_samples_per_send" doc:"deprecated|description=Use 'ruler_remote_write_config' instead. Maximum number of samples per send."` // deprecated use RulerRemoteWriteConfig instead RulerRemoteWriteQueueBatchSendDeadline time.Duration `yaml:"ruler_remote_write_queue_batch_send_deadline" json:"ruler_remote_write_queue_batch_send_deadline" doc:"deprecated|description=Use 'ruler_remote_write_config' instead. Maximum time a sample will wait in buffer."` @@ -631,18 +627,6 @@ func (o *Overrides) RulerRemoteWriteQueueCapacity(userID string) int { return o.getOverridesForUser(userID).RulerRemoteWriteQueueCapacity } -// Deprecated: use RulerRemoteWriteConfig instead -// RulerRemoteWriteQueueMinShards returns the minimum shards to use in a remote-write for a given user. -func (o *Overrides) RulerRemoteWriteQueueMinShards(userID string) int { - return o.getOverridesForUser(userID).RulerRemoteWriteQueueMinShards -} - -// Deprecated: use RulerRemoteWriteConfig instead -// RulerRemoteWriteQueueMaxShards returns the maximum shards to use in a remote-write for a given user. -func (o *Overrides) RulerRemoteWriteQueueMaxShards(userID string) int { - return o.getOverridesForUser(userID).RulerRemoteWriteQueueMaxShards -} - // Deprecated: use RulerRemoteWriteConfig instead // RulerRemoteWriteQueueMaxSamplesPerSend returns the max samples to send in a remote-write for a given user. func (o *Overrides) RulerRemoteWriteQueueMaxSamplesPerSend(userID string) int { diff --git a/tools/deprecated-config-checker/checker/checker_test.go b/tools/deprecated-config-checker/checker/checker_test.go index b4c7697d6e03..7147b7b309ee 100644 --- a/tools/deprecated-config-checker/checker/checker_test.go +++ b/tools/deprecated-config-checker/checker/checker_test.go @@ -50,8 +50,6 @@ var ( "limits_config.ruler_remote_write_headers", "limits_config.ruler_remote_write_relabel_configs", "limits_config.ruler_remote_write_queue_capacity", - "limits_config.ruler_remote_write_queue_min_shards", - "limits_config.ruler_remote_write_queue_max_shards", "limits_config.ruler_remote_write_queue_max_samples_per_send", "limits_config.ruler_remote_write_queue_batch_send_deadline", "limits_config.ruler_remote_write_queue_min_backoff", @@ -87,8 +85,6 @@ var ( "overrides.foo.ruler_remote_write_headers", "overrides.foo.ruler_remote_write_relabel_configs", "overrides.foo.ruler_remote_write_queue_capacity", - "overrides.foo.ruler_remote_write_queue_min_shards", - "overrides.foo.ruler_remote_write_queue_max_shards", "overrides.foo.ruler_remote_write_queue_max_samples_per_send", "overrides.foo.ruler_remote_write_queue_batch_send_deadline", "overrides.foo.ruler_remote_write_queue_min_backoff", @@ -103,8 +99,6 @@ var ( "overrides.bar.ruler_remote_write_headers", "overrides.bar.ruler_remote_write_relabel_configs", "overrides.bar.ruler_remote_write_queue_capacity", - "overrides.bar.ruler_remote_write_queue_min_shards", - "overrides.bar.ruler_remote_write_queue_max_shards", "overrides.bar.ruler_remote_write_queue_max_samples_per_send", "overrides.bar.ruler_remote_write_queue_batch_send_deadline", "overrides.bar.ruler_remote_write_queue_min_backoff", diff --git a/tools/deprecated-config-checker/deprecated-config.yaml b/tools/deprecated-config-checker/deprecated-config.yaml index 9452fbe7ed50..956c81ab4f02 100644 --- a/tools/deprecated-config-checker/deprecated-config.yaml +++ b/tools/deprecated-config-checker/deprecated-config.yaml @@ -52,8 +52,6 @@ limits_config: ruler_remote_write_headers: "Use ruler_remote_write_config instead." ruler_remote_write_relabel_configs: "Use ruler_remote_write_config instead." ruler_remote_write_queue_capacity: "Use ruler_remote_write_config instead." - ruler_remote_write_queue_min_shards: "Use ruler_remote_write_config instead." - ruler_remote_write_queue_max_shards: "Use ruler_remote_write_config instead." ruler_remote_write_queue_max_samples_per_send: "Use ruler_remote_write_config instead." ruler_remote_write_queue_batch_send_deadline: "Use ruler_remote_write_config instead." ruler_remote_write_queue_min_backoff: "Use ruler_remote_write_config instead." diff --git a/tools/deprecated-config-checker/test-fixtures/config.yaml b/tools/deprecated-config-checker/test-fixtures/config.yaml index 8d4f4587c602..0c861c19c3ac 100644 --- a/tools/deprecated-config-checker/test-fixtures/config.yaml +++ b/tools/deprecated-config-checker/test-fixtures/config.yaml @@ -121,8 +121,6 @@ limits_config: ruler_remote_write_headers: ["foo", "bar"] # DEPRECATED ruler_remote_write_relabel_configs: "foo" # DEPRECATED ruler_remote_write_queue_capacity: 10 # DEPRECATED - ruler_remote_write_queue_min_shards: 10 # DEPRECATED - ruler_remote_write_queue_max_shards: 100 # DEPRECATED ruler_remote_write_queue_max_samples_per_send: 50 # DEPRECATED ruler_remote_write_queue_batch_send_deadline: 10m # DEPRECATED ruler_remote_write_queue_min_backoff: 1m # DEPRECATED diff --git a/tools/deprecated-config-checker/test-fixtures/runtime-config.yaml b/tools/deprecated-config-checker/test-fixtures/runtime-config.yaml index ada8f7e1f619..71ce9645c2df 100644 --- a/tools/deprecated-config-checker/test-fixtures/runtime-config.yaml +++ b/tools/deprecated-config-checker/test-fixtures/runtime-config.yaml @@ -6,8 +6,6 @@ overrides: ruler_remote_write_headers: [ "foo", "bar" ] # DEPRECATED ruler_remote_write_relabel_configs: "foo" # DEPRECATED ruler_remote_write_queue_capacity: 10 # DEPRECATED - ruler_remote_write_queue_min_shards: 10 # DEPRECATED - ruler_remote_write_queue_max_shards: 100 # DEPRECATED ruler_remote_write_queue_max_samples_per_send: 50 # DEPRECATED ruler_remote_write_queue_batch_send_deadline: 10m # DEPRECATED ruler_remote_write_queue_min_backoff: 1m # DEPRECATED From 6764c95e5cb452dc09cbf4dc2ec768e44896741c Mon Sep 17 00:00:00 2001 From: shantanualshi Date: Wed, 1 Nov 2023 15:43:53 +0530 Subject: [PATCH 04/13] Remove deprecated ruler_remote_write_queue_min_backoff, ruler_remote_write_queue_max_backoff and ruler_remote_write_queue_retry_on_ratelimit --- docs/sources/configure/_index.md | 12 ---------- docs/sources/operations/overrides-exporter.md | 2 -- pkg/ruler/compat.go | 3 --- pkg/ruler/registry.go | 12 ---------- pkg/validation/limits.go | 24 ------------------- .../checker/checker_test.go | 9 ------- .../deprecated-config.yaml | 3 --- .../test-fixtures/config.yaml | 3 --- .../test-fixtures/runtime-config.yaml | 3 --- 9 files changed, 71 deletions(-) diff --git a/docs/sources/configure/_index.md b/docs/sources/configure/_index.md index d57bbe2ca64e..a44eedf5af97 100644 --- a/docs/sources/configure/_index.md +++ b/docs/sources/configure/_index.md @@ -2805,18 +2805,6 @@ The `limits_config` block configures global and per-tenant limits in Loki. # will wait in buffer. [ruler_remote_write_queue_batch_send_deadline: ] -# Deprecated: Use 'ruler_remote_write_config' instead. Initial retry delay. Gets -# doubled for every retry. -[ruler_remote_write_queue_min_backoff: ] - -# Deprecated: Use 'ruler_remote_write_config' instead. Maximum retry delay. -[ruler_remote_write_queue_max_backoff: ] - -# Deprecated: Use 'ruler_remote_write_config' instead. Retry upon receiving a -# 429 status code from the remote-write storage. This is experimental and might -# change in the future. -[ruler_remote_write_queue_retry_on_ratelimit: ] - # Deprecated: Use 'ruler_remote_write_config' instead. Configures AWS's # Signature Verification 4 signing process to sign every remote write request. ruler_remote_write_sigv4_config: diff --git a/docs/sources/operations/overrides-exporter.md b/docs/sources/operations/overrides-exporter.md index 5f0ee8bf04a2..225b113ece8a 100644 --- a/docs/sources/operations/overrides-exporter.md +++ b/docs/sources/operations/overrides-exporter.md @@ -75,9 +75,7 @@ loki_overrides_defaults{limit_name="ruler_max_rule_groups_per_tenant"} 0 loki_overrides_defaults{limit_name="ruler_max_rules_per_rule_group"} 0 loki_overrides_defaults{limit_name="ruler_remote_write_queue_batch_send_deadline"} 0 loki_overrides_defaults{limit_name="ruler_remote_write_queue_capacity"} 0 -loki_overrides_defaults{limit_name="ruler_remote_write_queue_max_backoff"} 0 loki_overrides_defaults{limit_name="ruler_remote_write_queue_max_samples_per_send"} 0 -loki_overrides_defaults{limit_name="ruler_remote_write_queue_min_backoff"} 0 loki_overrides_defaults{limit_name="split_queries_by_interval"} 0 ``` diff --git a/pkg/ruler/compat.go b/pkg/ruler/compat.go index 2abde21c0f09..83ca2edb0ff9 100644 --- a/pkg/ruler/compat.go +++ b/pkg/ruler/compat.go @@ -43,9 +43,6 @@ type RulesLimits interface { RulerRemoteWriteQueueCapacity(userID string) int RulerRemoteWriteQueueMaxSamplesPerSend(userID string) int RulerRemoteWriteQueueBatchSendDeadline(userID string) time.Duration - RulerRemoteWriteQueueMinBackoff(userID string) time.Duration - RulerRemoteWriteQueueMaxBackoff(userID string) time.Duration - RulerRemoteWriteQueueRetryOnRateLimit(userID string) bool RulerRemoteWriteSigV4Config(userID string) *sigv4.SigV4Config RulerRemoteEvaluationTimeout(userID string) time.Duration diff --git a/pkg/ruler/registry.go b/pkg/ruler/registry.go index 9cb73ffd5eac..8c19c2d10ba1 100644 --- a/pkg/ruler/registry.go +++ b/pkg/ruler/registry.go @@ -262,22 +262,10 @@ func (r *walRegistry) getTenantRemoteWriteConfig(tenant string, base RemoteWrite clt.QueueConfig.MaxSamplesPerSend = v } - if v := r.overrides.RulerRemoteWriteQueueMinBackoff(tenant); v > 0 { - clt.QueueConfig.MinBackoff = model.Duration(v) - } - - if v := r.overrides.RulerRemoteWriteQueueMaxBackoff(tenant); v > 0 { - clt.QueueConfig.MaxBackoff = model.Duration(v) - } - if v := r.overrides.RulerRemoteWriteQueueBatchSendDeadline(tenant); v > 0 { clt.QueueConfig.BatchSendDeadline = model.Duration(v) } - if v := r.overrides.RulerRemoteWriteQueueRetryOnRateLimit(tenant); v { - clt.QueueConfig.RetryOnRateLimit = v - } - if v := r.overrides.RulerRemoteWriteSigV4Config(tenant); v != nil { clt.SigV4Config = v } diff --git a/pkg/validation/limits.go b/pkg/validation/limits.go index 3ac482279652..afe9a1b3c421 100644 --- a/pkg/validation/limits.go +++ b/pkg/validation/limits.go @@ -138,12 +138,6 @@ type Limits struct { // deprecated use RulerRemoteWriteConfig instead RulerRemoteWriteQueueBatchSendDeadline time.Duration `yaml:"ruler_remote_write_queue_batch_send_deadline" json:"ruler_remote_write_queue_batch_send_deadline" doc:"deprecated|description=Use 'ruler_remote_write_config' instead. Maximum time a sample will wait in buffer."` // deprecated use RulerRemoteWriteConfig instead - RulerRemoteWriteQueueMinBackoff time.Duration `yaml:"ruler_remote_write_queue_min_backoff" json:"ruler_remote_write_queue_min_backoff" doc:"deprecated|description=Use 'ruler_remote_write_config' instead. Initial retry delay. Gets doubled for every retry."` - // deprecated use RulerRemoteWriteConfig instead - RulerRemoteWriteQueueMaxBackoff time.Duration `yaml:"ruler_remote_write_queue_max_backoff" json:"ruler_remote_write_queue_max_backoff" doc:"deprecated|description=Use 'ruler_remote_write_config' instead. Maximum retry delay."` - // deprecated use RulerRemoteWriteConfig instead - RulerRemoteWriteQueueRetryOnRateLimit bool `yaml:"ruler_remote_write_queue_retry_on_ratelimit" json:"ruler_remote_write_queue_retry_on_ratelimit" doc:"deprecated|description=Use 'ruler_remote_write_config' instead. Retry upon receiving a 429 status code from the remote-write storage. This is experimental and might change in the future."` - // deprecated use RulerRemoteWriteConfig instead RulerRemoteWriteSigV4Config *sigv4.SigV4Config `yaml:"ruler_remote_write_sigv4_config" json:"ruler_remote_write_sigv4_config" doc:"deprecated|description=Use 'ruler_remote_write_config' instead. Configures AWS's Signature Verification 4 signing process to sign every remote write request."` RulerRemoteWriteConfig map[string]config.RemoteWriteConfig `yaml:"ruler_remote_write_config,omitempty" json:"ruler_remote_write_config,omitempty" doc:"description=Configures global and per-tenant limits for remote write clients. A map with remote client id as key."` @@ -639,24 +633,6 @@ func (o *Overrides) RulerRemoteWriteQueueBatchSendDeadline(userID string) time.D return o.getOverridesForUser(userID).RulerRemoteWriteQueueBatchSendDeadline } -// Deprecated: use RulerRemoteWriteConfig instead -// RulerRemoteWriteQueueMinBackoff returns the minimum time for an exponential backoff for a given user. -func (o *Overrides) RulerRemoteWriteQueueMinBackoff(userID string) time.Duration { - return o.getOverridesForUser(userID).RulerRemoteWriteQueueMinBackoff -} - -// Deprecated: use RulerRemoteWriteConfig instead -// RulerRemoteWriteQueueMaxBackoff returns the maximum time for an exponential backoff for a given user. -func (o *Overrides) RulerRemoteWriteQueueMaxBackoff(userID string) time.Duration { - return o.getOverridesForUser(userID).RulerRemoteWriteQueueMaxBackoff -} - -// Deprecated: use RulerRemoteWriteConfig instead -// RulerRemoteWriteQueueRetryOnRateLimit returns whether to retry failed remote-write requests (429 response) for a given user. -func (o *Overrides) RulerRemoteWriteQueueRetryOnRateLimit(userID string) bool { - return o.getOverridesForUser(userID).RulerRemoteWriteQueueRetryOnRateLimit -} - // Deprecated: use RulerRemoteWriteConfig instead func (o *Overrides) RulerRemoteWriteSigV4Config(userID string) *sigv4.SigV4Config { return o.getOverridesForUser(userID).RulerRemoteWriteSigV4Config diff --git a/tools/deprecated-config-checker/checker/checker_test.go b/tools/deprecated-config-checker/checker/checker_test.go index 7147b7b309ee..f7f5993b182c 100644 --- a/tools/deprecated-config-checker/checker/checker_test.go +++ b/tools/deprecated-config-checker/checker/checker_test.go @@ -52,9 +52,6 @@ var ( "limits_config.ruler_remote_write_queue_capacity", "limits_config.ruler_remote_write_queue_max_samples_per_send", "limits_config.ruler_remote_write_queue_batch_send_deadline", - "limits_config.ruler_remote_write_queue_min_backoff", - "limits_config.ruler_remote_write_queue_max_backoff", - "limits_config.ruler_remote_write_queue_retry_on_ratelimit", "limits_config.ruler_remote_write_sigv4_config", "limits_config.per_tenant_override_config", "limits_config.per_tenant_override_period", @@ -87,9 +84,6 @@ var ( "overrides.foo.ruler_remote_write_queue_capacity", "overrides.foo.ruler_remote_write_queue_max_samples_per_send", "overrides.foo.ruler_remote_write_queue_batch_send_deadline", - "overrides.foo.ruler_remote_write_queue_min_backoff", - "overrides.foo.ruler_remote_write_queue_max_backoff", - "overrides.foo.ruler_remote_write_queue_retry_on_ratelimit", "overrides.foo.ruler_remote_write_sigv4_config", "overrides.foo.per_tenant_override_config", "overrides.foo.per_tenant_override_period", @@ -101,9 +95,6 @@ var ( "overrides.bar.ruler_remote_write_queue_capacity", "overrides.bar.ruler_remote_write_queue_max_samples_per_send", "overrides.bar.ruler_remote_write_queue_batch_send_deadline", - "overrides.bar.ruler_remote_write_queue_min_backoff", - "overrides.bar.ruler_remote_write_queue_max_backoff", - "overrides.bar.ruler_remote_write_queue_retry_on_ratelimit", "overrides.bar.ruler_remote_write_sigv4_config", "overrides.bar.per_tenant_override_config", "overrides.bar.per_tenant_override_period", diff --git a/tools/deprecated-config-checker/deprecated-config.yaml b/tools/deprecated-config-checker/deprecated-config.yaml index 956c81ab4f02..971e15ecb934 100644 --- a/tools/deprecated-config-checker/deprecated-config.yaml +++ b/tools/deprecated-config-checker/deprecated-config.yaml @@ -54,9 +54,6 @@ limits_config: ruler_remote_write_queue_capacity: "Use ruler_remote_write_config instead." ruler_remote_write_queue_max_samples_per_send: "Use ruler_remote_write_config instead." ruler_remote_write_queue_batch_send_deadline: "Use ruler_remote_write_config instead." - ruler_remote_write_queue_min_backoff: "Use ruler_remote_write_config instead." - ruler_remote_write_queue_max_backoff: "Use ruler_remote_write_config instead." - ruler_remote_write_queue_retry_on_ratelimit: "Use ruler_remote_write_config instead." ruler_remote_write_sigv4_config: "Use ruler_remote_write_config instead." per_tenant_override_config: "Feature renamed to 'runtime configuration', flag deprecated in favor of runtime_config.file" per_tenant_override_period: "Feature renamed to 'runtime configuration', flag deprecated in favor of runtime_config.period" diff --git a/tools/deprecated-config-checker/test-fixtures/config.yaml b/tools/deprecated-config-checker/test-fixtures/config.yaml index 0c861c19c3ac..39cc0928bae1 100644 --- a/tools/deprecated-config-checker/test-fixtures/config.yaml +++ b/tools/deprecated-config-checker/test-fixtures/config.yaml @@ -123,9 +123,6 @@ limits_config: ruler_remote_write_queue_capacity: 10 # DEPRECATED ruler_remote_write_queue_max_samples_per_send: 50 # DEPRECATED ruler_remote_write_queue_batch_send_deadline: 10m # DEPRECATED - ruler_remote_write_queue_min_backoff: 1m # DEPRECATED - ruler_remote_write_queue_max_backoff: 5m # DEPRECATED - ruler_remote_write_queue_retry_on_ratelimit: true # DEPRECATED ruler_remote_write_sigv4_config: # DEPRECATED region: "wherever" per_tenant_override_config: ./overrides.yaml # DEPRECATED diff --git a/tools/deprecated-config-checker/test-fixtures/runtime-config.yaml b/tools/deprecated-config-checker/test-fixtures/runtime-config.yaml index 71ce9645c2df..1436e415015b 100644 --- a/tools/deprecated-config-checker/test-fixtures/runtime-config.yaml +++ b/tools/deprecated-config-checker/test-fixtures/runtime-config.yaml @@ -8,9 +8,6 @@ overrides: ruler_remote_write_queue_capacity: 10 # DEPRECATED ruler_remote_write_queue_max_samples_per_send: 50 # DEPRECATED ruler_remote_write_queue_batch_send_deadline: 10m # DEPRECATED - ruler_remote_write_queue_min_backoff: 1m # DEPRECATED - ruler_remote_write_queue_max_backoff: 5m # DEPRECATED - ruler_remote_write_queue_retry_on_ratelimit: true # DEPRECATED ruler_remote_write_sigv4_config: # DEPRECATED region: "wherever" per_tenant_override_config: ./overrides.yaml # DEPRECATED From d573e20a5a25572162d71c350ca35e35b1d10105 Mon Sep 17 00:00:00 2001 From: shantanualshi Date: Wed, 1 Nov 2023 15:54:49 +0530 Subject: [PATCH 05/13] Remove deprecated ruler_remote_write_queue_batch_send_deadline and ruler_remote_write_queue_max_samples_per_send --- docs/sources/configure/_index.md | 8 -------- docs/sources/operations/overrides-exporter.md | 2 -- pkg/ruler/compat.go | 2 -- pkg/ruler/registry.go | 9 --------- pkg/validation/limits.go | 16 ---------------- .../checker/checker_test.go | 6 ------ .../deprecated-config.yaml | 2 -- .../test-fixtures/config.yaml | 2 -- .../test-fixtures/runtime-config.yaml | 2 -- 9 files changed, 49 deletions(-) diff --git a/docs/sources/configure/_index.md b/docs/sources/configure/_index.md index a44eedf5af97..1691d7c052fe 100644 --- a/docs/sources/configure/_index.md +++ b/docs/sources/configure/_index.md @@ -2797,14 +2797,6 @@ The `limits_config` block configures global and per-tenant limits in Loki. # to keep throughput up while processing occasional slow remote requests. [ruler_remote_write_queue_capacity: ] -# Deprecated: Use 'ruler_remote_write_config' instead. Maximum number of samples -# per send. -[ruler_remote_write_queue_max_samples_per_send: ] - -# Deprecated: Use 'ruler_remote_write_config' instead. Maximum time a sample -# will wait in buffer. -[ruler_remote_write_queue_batch_send_deadline: ] - # Deprecated: Use 'ruler_remote_write_config' instead. Configures AWS's # Signature Verification 4 signing process to sign every remote write request. ruler_remote_write_sigv4_config: diff --git a/docs/sources/operations/overrides-exporter.md b/docs/sources/operations/overrides-exporter.md index 225b113ece8a..505d5dfd5f2a 100644 --- a/docs/sources/operations/overrides-exporter.md +++ b/docs/sources/operations/overrides-exporter.md @@ -73,9 +73,7 @@ loki_overrides_defaults{limit_name="retention_period"} 2.6784e+15 loki_overrides_defaults{limit_name="ruler_evaluation_delay_duration"} 0 loki_overrides_defaults{limit_name="ruler_max_rule_groups_per_tenant"} 0 loki_overrides_defaults{limit_name="ruler_max_rules_per_rule_group"} 0 -loki_overrides_defaults{limit_name="ruler_remote_write_queue_batch_send_deadline"} 0 loki_overrides_defaults{limit_name="ruler_remote_write_queue_capacity"} 0 -loki_overrides_defaults{limit_name="ruler_remote_write_queue_max_samples_per_send"} 0 loki_overrides_defaults{limit_name="split_queries_by_interval"} 0 ``` diff --git a/pkg/ruler/compat.go b/pkg/ruler/compat.go index 83ca2edb0ff9..1f377d64a068 100644 --- a/pkg/ruler/compat.go +++ b/pkg/ruler/compat.go @@ -41,8 +41,6 @@ type RulesLimits interface { RulerRemoteWriteRelabelConfigs(userID string) []*util.RelabelConfig RulerRemoteWriteConfig(userID string, id string) *config.RemoteWriteConfig RulerRemoteWriteQueueCapacity(userID string) int - RulerRemoteWriteQueueMaxSamplesPerSend(userID string) int - RulerRemoteWriteQueueBatchSendDeadline(userID string) time.Duration RulerRemoteWriteSigV4Config(userID string) *sigv4.SigV4Config RulerRemoteEvaluationTimeout(userID string) time.Duration diff --git a/pkg/ruler/registry.go b/pkg/ruler/registry.go index 8c19c2d10ba1..7062f6bf4d2f 100644 --- a/pkg/ruler/registry.go +++ b/pkg/ruler/registry.go @@ -12,7 +12,6 @@ import ( "github.com/imdario/mergo" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/common/model" "github.com/prometheus/prometheus/config" "github.com/prometheus/prometheus/model/exemplar" "github.com/prometheus/prometheus/model/histogram" @@ -258,14 +257,6 @@ func (r *walRegistry) getTenantRemoteWriteConfig(tenant string, base RemoteWrite clt.QueueConfig.Capacity = v } - if v := r.overrides.RulerRemoteWriteQueueMaxSamplesPerSend(tenant); v > 0 { - clt.QueueConfig.MaxSamplesPerSend = v - } - - if v := r.overrides.RulerRemoteWriteQueueBatchSendDeadline(tenant); v > 0 { - clt.QueueConfig.BatchSendDeadline = model.Duration(v) - } - if v := r.overrides.RulerRemoteWriteSigV4Config(tenant); v != nil { clt.SigV4Config = v } diff --git a/pkg/validation/limits.go b/pkg/validation/limits.go index afe9a1b3c421..d8350c16b1b5 100644 --- a/pkg/validation/limits.go +++ b/pkg/validation/limits.go @@ -134,10 +134,6 @@ type Limits struct { // deprecated use RulerRemoteWriteConfig instead RulerRemoteWriteQueueCapacity int `yaml:"ruler_remote_write_queue_capacity" json:"ruler_remote_write_queue_capacity" doc:"deprecated|description=Use 'ruler_remote_write_config' instead. Number of samples to buffer per shard before we block reading of more samples from the WAL. It is recommended to have enough capacity in each shard to buffer several requests to keep throughput up while processing occasional slow remote requests."` // deprecated use RulerRemoteWriteConfig instead - RulerRemoteWriteQueueMaxSamplesPerSend int `yaml:"ruler_remote_write_queue_max_samples_per_send" json:"ruler_remote_write_queue_max_samples_per_send" doc:"deprecated|description=Use 'ruler_remote_write_config' instead. Maximum number of samples per send."` - // deprecated use RulerRemoteWriteConfig instead - RulerRemoteWriteQueueBatchSendDeadline time.Duration `yaml:"ruler_remote_write_queue_batch_send_deadline" json:"ruler_remote_write_queue_batch_send_deadline" doc:"deprecated|description=Use 'ruler_remote_write_config' instead. Maximum time a sample will wait in buffer."` - // deprecated use RulerRemoteWriteConfig instead RulerRemoteWriteSigV4Config *sigv4.SigV4Config `yaml:"ruler_remote_write_sigv4_config" json:"ruler_remote_write_sigv4_config" doc:"deprecated|description=Use 'ruler_remote_write_config' instead. Configures AWS's Signature Verification 4 signing process to sign every remote write request."` RulerRemoteWriteConfig map[string]config.RemoteWriteConfig `yaml:"ruler_remote_write_config,omitempty" json:"ruler_remote_write_config,omitempty" doc:"description=Configures global and per-tenant limits for remote write clients. A map with remote client id as key."` @@ -621,18 +617,6 @@ func (o *Overrides) RulerRemoteWriteQueueCapacity(userID string) int { return o.getOverridesForUser(userID).RulerRemoteWriteQueueCapacity } -// Deprecated: use RulerRemoteWriteConfig instead -// RulerRemoteWriteQueueMaxSamplesPerSend returns the max samples to send in a remote-write for a given user. -func (o *Overrides) RulerRemoteWriteQueueMaxSamplesPerSend(userID string) int { - return o.getOverridesForUser(userID).RulerRemoteWriteQueueMaxSamplesPerSend -} - -// Deprecated: use RulerRemoteWriteConfig instead -// RulerRemoteWriteQueueBatchSendDeadline returns the maximum time a sample will be buffered before being discarded for a given user. -func (o *Overrides) RulerRemoteWriteQueueBatchSendDeadline(userID string) time.Duration { - return o.getOverridesForUser(userID).RulerRemoteWriteQueueBatchSendDeadline -} - // Deprecated: use RulerRemoteWriteConfig instead func (o *Overrides) RulerRemoteWriteSigV4Config(userID string) *sigv4.SigV4Config { return o.getOverridesForUser(userID).RulerRemoteWriteSigV4Config diff --git a/tools/deprecated-config-checker/checker/checker_test.go b/tools/deprecated-config-checker/checker/checker_test.go index f7f5993b182c..74762db30a74 100644 --- a/tools/deprecated-config-checker/checker/checker_test.go +++ b/tools/deprecated-config-checker/checker/checker_test.go @@ -50,8 +50,6 @@ var ( "limits_config.ruler_remote_write_headers", "limits_config.ruler_remote_write_relabel_configs", "limits_config.ruler_remote_write_queue_capacity", - "limits_config.ruler_remote_write_queue_max_samples_per_send", - "limits_config.ruler_remote_write_queue_batch_send_deadline", "limits_config.ruler_remote_write_sigv4_config", "limits_config.per_tenant_override_config", "limits_config.per_tenant_override_period", @@ -82,8 +80,6 @@ var ( "overrides.foo.ruler_remote_write_headers", "overrides.foo.ruler_remote_write_relabel_configs", "overrides.foo.ruler_remote_write_queue_capacity", - "overrides.foo.ruler_remote_write_queue_max_samples_per_send", - "overrides.foo.ruler_remote_write_queue_batch_send_deadline", "overrides.foo.ruler_remote_write_sigv4_config", "overrides.foo.per_tenant_override_config", "overrides.foo.per_tenant_override_period", @@ -93,8 +89,6 @@ var ( "overrides.bar.ruler_remote_write_headers", "overrides.bar.ruler_remote_write_relabel_configs", "overrides.bar.ruler_remote_write_queue_capacity", - "overrides.bar.ruler_remote_write_queue_max_samples_per_send", - "overrides.bar.ruler_remote_write_queue_batch_send_deadline", "overrides.bar.ruler_remote_write_sigv4_config", "overrides.bar.per_tenant_override_config", "overrides.bar.per_tenant_override_period", diff --git a/tools/deprecated-config-checker/deprecated-config.yaml b/tools/deprecated-config-checker/deprecated-config.yaml index 971e15ecb934..cd8096386f97 100644 --- a/tools/deprecated-config-checker/deprecated-config.yaml +++ b/tools/deprecated-config-checker/deprecated-config.yaml @@ -52,8 +52,6 @@ limits_config: ruler_remote_write_headers: "Use ruler_remote_write_config instead." ruler_remote_write_relabel_configs: "Use ruler_remote_write_config instead." ruler_remote_write_queue_capacity: "Use ruler_remote_write_config instead." - ruler_remote_write_queue_max_samples_per_send: "Use ruler_remote_write_config instead." - ruler_remote_write_queue_batch_send_deadline: "Use ruler_remote_write_config instead." ruler_remote_write_sigv4_config: "Use ruler_remote_write_config instead." per_tenant_override_config: "Feature renamed to 'runtime configuration', flag deprecated in favor of runtime_config.file" per_tenant_override_period: "Feature renamed to 'runtime configuration', flag deprecated in favor of runtime_config.period" diff --git a/tools/deprecated-config-checker/test-fixtures/config.yaml b/tools/deprecated-config-checker/test-fixtures/config.yaml index 39cc0928bae1..eefe70c114ac 100644 --- a/tools/deprecated-config-checker/test-fixtures/config.yaml +++ b/tools/deprecated-config-checker/test-fixtures/config.yaml @@ -121,8 +121,6 @@ limits_config: ruler_remote_write_headers: ["foo", "bar"] # DEPRECATED ruler_remote_write_relabel_configs: "foo" # DEPRECATED ruler_remote_write_queue_capacity: 10 # DEPRECATED - ruler_remote_write_queue_max_samples_per_send: 50 # DEPRECATED - ruler_remote_write_queue_batch_send_deadline: 10m # DEPRECATED ruler_remote_write_sigv4_config: # DEPRECATED region: "wherever" per_tenant_override_config: ./overrides.yaml # DEPRECATED diff --git a/tools/deprecated-config-checker/test-fixtures/runtime-config.yaml b/tools/deprecated-config-checker/test-fixtures/runtime-config.yaml index 1436e415015b..b699adc1a9aa 100644 --- a/tools/deprecated-config-checker/test-fixtures/runtime-config.yaml +++ b/tools/deprecated-config-checker/test-fixtures/runtime-config.yaml @@ -6,8 +6,6 @@ overrides: ruler_remote_write_headers: [ "foo", "bar" ] # DEPRECATED ruler_remote_write_relabel_configs: "foo" # DEPRECATED ruler_remote_write_queue_capacity: 10 # DEPRECATED - ruler_remote_write_queue_max_samples_per_send: 50 # DEPRECATED - ruler_remote_write_queue_batch_send_deadline: 10m # DEPRECATED ruler_remote_write_sigv4_config: # DEPRECATED region: "wherever" per_tenant_override_config: ./overrides.yaml # DEPRECATED From a40874db32eea2a5ead0ba72a44485745e7c6fa2 Mon Sep 17 00:00:00 2001 From: shantanualshi Date: Thu, 2 Nov 2023 15:27:15 +0530 Subject: [PATCH 06/13] Replace registry test with newFakeLimits --- pkg/ruler/registry_test.go | 214 +++++++------------------------------ 1 file changed, 36 insertions(+), 178 deletions(-) diff --git a/pkg/ruler/registry_test.go b/pkg/ruler/registry_test.go index b4d7825fb6fb..ff1148810a6e 100644 --- a/pkg/ruler/registry_test.go +++ b/pkg/ruler/registry_test.go @@ -19,7 +19,6 @@ import ( "github.com/stretchr/testify/require" "github.com/grafana/loki/pkg/ruler/storage/instance" - "github.com/grafana/loki/pkg/ruler/util" "github.com/grafana/loki/pkg/util/test" "github.com/grafana/loki/pkg/validation" ) @@ -159,61 +158,6 @@ var cfg = Config{ }, } -func newFakeLimitsBackwardCompat() fakeLimits { - return fakeLimits{ - limits: map[string]*validation.Limits{ - enabledRWTenant: { - RulerRemoteWriteQueueCapacity: 987, - }, - disabledRWTenant: { - RulerRemoteWriteDisabled: true, - }, - additionalHeadersRWTenant: { - RulerRemoteWriteHeaders: validation.NewOverwriteMarshalingStringMap(map[string]string{ - user.OrgIDHeaderName: "overridden", - fmt.Sprintf(" %s ", user.OrgIDHeaderName): "overridden", - strings.ToLower(user.OrgIDHeaderName): "overridden-lower", - strings.ToUpper(user.OrgIDHeaderName): "overridden-upper", - "Additional": "Header", - }), - }, - noHeadersRWTenant: { - RulerRemoteWriteHeaders: validation.NewOverwriteMarshalingStringMap(map[string]string{}), - }, - customRelabelsTenant: { - RulerRemoteWriteRelabelConfigs: []*util.RelabelConfig{ - { - Regex: ".+:.+", - SourceLabels: []string{"__name__"}, - Action: "drop", - }, - { - Regex: "__cluster__", - Action: "labeldrop", - }, - }, - }, - nilRelabelsTenant: {}, - emptySliceRelabelsTenant: { - RulerRemoteWriteRelabelConfigs: []*util.RelabelConfig{}, - }, - badRelabelsTenant: { - RulerRemoteWriteRelabelConfigs: []*util.RelabelConfig{ - { - SourceLabels: []string{"__cluster__"}, - Action: "labeldrop", - }, - }, - }, - sigV4ConfigTenant: { - RulerRemoteWriteSigV4Config: &sigv4.SigV4Config{ - Region: sigV4TenantRegion, - }, - }, - }, - } -} - var newRemoteURL2, _ = url.Parse("http://new-remote-write2") func newFakeLimits() fakeLimits { @@ -301,6 +245,18 @@ func newFakeLimits() fakeLimits { }, }, }, + badRelabelsTenant: { + RulerRemoteWriteConfig: map[string]config.RemoteWriteConfig{ + remote1: { + WriteRelabelConfigs: []*relabel.Config{ + { + SourceLabels: model.LabelNames{"__cluster__"}, + Action: "labeldrop", + }, + }, + }, + }, + }, }, } } @@ -345,21 +301,11 @@ func setupSigV4Registry(t *testing.T, cfg Config, limits fakeLimits) *walRegistr } func TestTenantRemoteWriteConfigWithOverride(t *testing.T) { - reg := setupRegistry(t, backCompatCfg, newFakeLimitsBackwardCompat()) + reg := setupRegistry(t, cfg, newFakeLimits()) tenantCfg, err := reg.getTenantConfig(enabledRWTenant) require.NoError(t, err) - // tenant has not disable remote-write so will inherit the global one - assert.Len(t, tenantCfg.RemoteWrite, 1) - // but the tenant has an override for the queue capacity - assert.Equal(t, tenantCfg.RemoteWrite[0].QueueConfig.Capacity, 987) - - reg = setupRegistry(t, cfg, newFakeLimits()) - - tenantCfg, err = reg.getTenantConfig(enabledRWTenant) - require.NoError(t, err) - // tenant has not disable remote-write so will inherit the global one assert.Len(t, tenantCfg.RemoteWrite, 2) // but the tenant has an override for the queue capacity for the first client @@ -377,21 +323,11 @@ func TestTenantRemoteWriteConfigWithOverride(t *testing.T) { } func TestTenantRemoteWriteConfigWithoutOverride(t *testing.T) { - reg := setupRegistry(t, backCompatCfg, newFakeLimitsBackwardCompat()) - - // this tenant has no overrides, so will get defaults - tenantCfg, err := reg.getTenantConfig("unknown") - require.NoError(t, err) - // tenant has not disable remote-write so will inherit the global one - assert.Len(t, tenantCfg.RemoteWrite, 1) - // but the tenant has an override for the queue capacity - assert.Equal(t, tenantCfg.RemoteWrite[0].QueueConfig.Capacity, defaultCapacity) - - reg = setupRegistry(t, cfg, newFakeLimits()) + reg := setupRegistry(t, cfg, newFakeLimits()) // this tenant has no overrides, so will get defaults - tenantCfg, err = reg.getTenantConfig("unknown") + tenantCfg, err := reg.getTenantConfig("unknown") require.NoError(t, err) // tenant has not disable remote-write so will inherit the global one @@ -475,23 +411,11 @@ func TestTenantMultiRemoteWriteConfigWithoutOverride(t *testing.T) { } func TestRulerRemoteWriteSigV4ConfigWithOverrides(t *testing.T) { - reg := setupSigV4Registry(t, backCompatCfg, newFakeLimitsBackwardCompat()) + reg := setupSigV4Registry(t, cfg, newFakeLimits()) tenantCfg, err := reg.getTenantConfig(sigV4ConfigTenant) require.NoError(t, err) - // tenant has not disable remote-write so will inherit the global one - assert.Len(t, tenantCfg.RemoteWrite, 1) - // ensure sigv4 config is not nil and overwritten - if assert.NotNil(t, tenantCfg.RemoteWrite[0].SigV4Config) { - assert.Equal(t, sigV4TenantRegion, tenantCfg.RemoteWrite[0].SigV4Config.Region) - } - - reg = setupSigV4Registry(t, cfg, newFakeLimits()) - - tenantCfg, err = reg.getTenantConfig(sigV4ConfigTenant) - require.NoError(t, err) - // tenant has not disable remote-write so will inherit the global one assert.Len(t, tenantCfg.RemoteWrite, 2) // ensure sigv4 config is not nil and overwritten for first client @@ -509,25 +433,12 @@ func TestRulerRemoteWriteSigV4ConfigWithOverrides(t *testing.T) { } func TestRulerRemoteWriteSigV4ConfigWithoutOverrides(t *testing.T) { - reg := setupSigV4Registry(t, backCompatCfg, newFakeLimitsBackwardCompat()) + reg := setupSigV4Registry(t, cfg, newFakeLimits()) // this tenant has no overrides, so will get defaults tenantCfg, err := reg.getTenantConfig("unknown") require.NoError(t, err) - // tenant has not disable remote-write so will inherit the global one - assert.Len(t, tenantCfg.RemoteWrite, 1) - // ensure sigv4 config is not nil and the global value - if assert.NotNil(t, tenantCfg.RemoteWrite[0].SigV4Config) { - assert.Equal(t, tenantCfg.RemoteWrite[0].SigV4Config.Region, sigV4GlobalRegion) - } - - reg = setupSigV4Registry(t, cfg, newFakeLimits()) - - // this tenant has no overrides, so will get defaults - tenantCfg, err = reg.getTenantConfig("unknown") - require.NoError(t, err) - // tenant has not disable remote-write so will inherit the global one assert.Len(t, tenantCfg.RemoteWrite, 2) // ensure sigv4 config is not nil and the global value @@ -540,25 +451,17 @@ func TestRulerRemoteWriteSigV4ConfigWithoutOverrides(t *testing.T) { } func TestTenantRemoteWriteConfigDisabled(t *testing.T) { - reg := setupRegistry(t, backCompatCfg, newFakeLimitsBackwardCompat()) + reg := setupRegistry(t, cfg, newFakeLimits()) tenantCfg, err := reg.getTenantConfig(disabledRWTenant) require.NoError(t, err) // this tenant has remote-write disabled assert.Len(t, tenantCfg.RemoteWrite, 0) - - reg = setupRegistry(t, cfg, newFakeLimits()) - - tenantCfg, err = reg.getTenantConfig(disabledRWTenant) - require.NoError(t, err) - - // this tenant has remote-write disabled - assert.Len(t, tenantCfg.RemoteWrite, 0) } func TestTenantRemoteWriteHTTPConfigMaintained(t *testing.T) { - reg := setupRegistry(t, backCompatCfg, newFakeLimitsBackwardCompat()) + reg := setupRegistry(t, cfg, newFakeLimits()) tenantCfg, err := reg.getTenantConfig(enabledRWTenant) require.NoError(t, err) @@ -597,30 +500,11 @@ func TestTenantRemoteWriteHTTPConfigMaintained(t *testing.T) { } func TestTenantRemoteWriteHeaderOverride(t *testing.T) { - reg := setupRegistry(t, backCompatCfg, newFakeLimitsBackwardCompat()) + reg := setupRegistry(t, cfg, newFakeLimits()) tenantCfg, err := reg.getTenantConfig(additionalHeadersRWTenant) require.NoError(t, err) - assert.Len(t, tenantCfg.RemoteWrite[0].Headers, 2) - // ensure that tenant cannot override X-Scope-OrgId header - assert.Equal(t, tenantCfg.RemoteWrite[0].Headers[user.OrgIDHeaderName], additionalHeadersRWTenant) - // but that the additional header defined is set - assert.Equal(t, tenantCfg.RemoteWrite[0].Headers["Additional"], "Header") - // the original header must be removed - assert.Equal(t, tenantCfg.RemoteWrite[0].Headers["Base"], "") - - tenantCfg, err = reg.getTenantConfig(enabledRWTenant) - require.NoError(t, err) - - // and a user who didn't set any header overrides still gets the X-Scope-OrgId header - assert.Equal(t, tenantCfg.RemoteWrite[0].Headers[user.OrgIDHeaderName], enabledRWTenant) - - reg = setupRegistry(t, cfg, newFakeLimits()) - - tenantCfg, err = reg.getTenantConfig(additionalHeadersRWTenant) - require.NoError(t, err) - assert.Len(t, tenantCfg.RemoteWrite[0].Headers, 2) assert.Len(t, tenantCfg.RemoteWrite[1].Headers, 2) @@ -652,22 +536,11 @@ func TestTenantRemoteWriteHeaderOverride(t *testing.T) { } func TestTenantRemoteWriteHeadersReset(t *testing.T) { - reg := setupRegistry(t, backCompatCfg, newFakeLimitsBackwardCompat()) + reg := setupRegistry(t, cfg, newFakeLimits()) tenantCfg, err := reg.getTenantConfig(noHeadersRWTenant) require.NoError(t, err) - assert.Len(t, tenantCfg.RemoteWrite[0].Headers, 1) - // ensure that tenant cannot override X-Scope-OrgId header - assert.Equal(t, tenantCfg.RemoteWrite[0].Headers[user.OrgIDHeaderName], noHeadersRWTenant) - // the original header must be removed - assert.Equal(t, tenantCfg.RemoteWrite[0].Headers["Base"], "") - - reg = setupRegistry(t, cfg, newFakeLimits()) - - tenantCfg, err = reg.getTenantConfig(noHeadersRWTenant) - require.NoError(t, err) - // Ensure that overrides take plus but that tenant cannot override X-Scope-OrgId header expected := []map[string]string{ { @@ -688,7 +561,7 @@ func TestTenantRemoteWriteHeadersReset(t *testing.T) { } func TestTenantRemoteWriteHeadersNoOverride(t *testing.T) { - reg := setupRegistry(t, backCompatCfg, newFakeLimitsBackwardCompat()) + reg := setupRegistry(t, backCompatCfg, newFakeLimits()) tenantCfg, err := reg.getTenantConfig(enabledRWTenant) require.NoError(t, err) @@ -725,19 +598,11 @@ func TestTenantRemoteWriteHeadersNoOverride(t *testing.T) { } func TestRelabelConfigOverrides(t *testing.T) { - reg := setupRegistry(t, backCompatCfg, newFakeLimitsBackwardCompat()) + reg := setupRegistry(t, cfg, newFakeLimits()) tenantCfg, err := reg.getTenantConfig(customRelabelsTenant) require.NoError(t, err) - // it should also override the default label configs - assert.Len(t, tenantCfg.RemoteWrite[0].WriteRelabelConfigs, 2) - - reg = setupRegistry(t, cfg, newFakeLimits()) - - tenantCfg, err = reg.getTenantConfig(customRelabelsTenant) - require.NoError(t, err) - // It should also override the default label configs for the first client only expected := [][]string{ { @@ -760,19 +625,11 @@ func TestRelabelConfigOverrides(t *testing.T) { } func TestRelabelConfigOverridesNilWriteRelabels(t *testing.T) { - reg := setupRegistry(t, backCompatCfg, newFakeLimitsBackwardCompat()) + reg := setupRegistry(t, cfg, newFakeLimits()) tenantCfg, err := reg.getTenantConfig(nilRelabelsTenant) require.NoError(t, err) - // if there are no relabel configs defined for the tenant, it should not override - assert.Equal(t, tenantCfg.RemoteWrite[0].WriteRelabelConfigs, reg.config.RemoteWrite.Client.WriteRelabelConfigs) - - reg = setupRegistry(t, cfg, newFakeLimits()) - - tenantCfg, err = reg.getTenantConfig(nilRelabelsTenant) - require.NoError(t, err) - // if there are no relabel configs defined for the tenant, it should not override actual := [][]*relabel.Config{} for _, rw := range tenantCfg.RemoteWrite { @@ -788,7 +645,7 @@ func TestRelabelConfigOverridesNilWriteRelabels(t *testing.T) { } func TestRelabelConfigOverridesEmptySliceWriteRelabels(t *testing.T) { - reg := setupRegistry(t, backCompatCfg, newFakeLimitsBackwardCompat()) + reg := setupRegistry(t, cfg, newFakeLimits()) tenantCfg, err := reg.getTenantConfig(emptySliceRelabelsTenant) require.NoError(t, err) @@ -797,14 +654,15 @@ func TestRelabelConfigOverridesEmptySliceWriteRelabels(t *testing.T) { assert.Len(t, tenantCfg.RemoteWrite[0].WriteRelabelConfigs, 0) } -func TestRelabelConfigOverridesWithErrors(t *testing.T) { - reg := setupRegistry(t, backCompatCfg, newFakeLimitsBackwardCompat()) - - _, err := reg.getTenantConfig(badRelabelsTenant) - - // ensure that relabel validation is being applied - require.EqualError(t, err, "failed to parse relabel configs: labeldrop action requires only 'regex', and no other fields") -} +// TODO(shantanu): Enable this test after relabels validation is implemented +//func TestRelabelConfigOverridesWithErrors(t *testing.T) { +// reg := setupRegistry(t, cfg, newFakeLimits()) +// +// _, err := reg.getTenantConfig(badRelabelsTenant) +// +// // ensure that relabel validation is being applied +// require.EqualError(t, err, "failed to parse relabel configs: labeldrop action requires only 'regex', and no other fields") +//} func TestWALRegistryCreation(t *testing.T) { overrides, err := validation.NewOverrides(validation.Limits{}, nil) @@ -831,7 +689,7 @@ func TestWALRegistryCreation(t *testing.T) { } func TestStorageSetup(t *testing.T) { - reg := setupRegistry(t, backCompatCfg, newFakeLimitsBackwardCompat()) + reg := setupRegistry(t, backCompatCfg, newFakeLimits()) // once the registry is setup and we configure the tenant storage, we should be able // to acquire an appender for the WAL storage @@ -846,7 +704,7 @@ func TestStorageSetup(t *testing.T) { } func TestStorageSetupWithRemoteWriteDisabled(t *testing.T) { - reg := setupRegistry(t, backCompatCfg, newFakeLimitsBackwardCompat()) + reg := setupRegistry(t, backCompatCfg, newFakeLimits()) // once the registry is setup and we configure the tenant storage, we should be able // to acquire an appender for the WAL storage From 54427dd07c6f33055c36343222512f04a4f4ff8f Mon Sep 17 00:00:00 2001 From: shantanualshi Date: Thu, 2 Nov 2023 17:12:40 +0530 Subject: [PATCH 07/13] Remove deprecated ruler_remote_write_relabel_configs, ruler_remote_write_queue_capacity and ruler_remote_write_sigv4_config --- docs/sources/configure/_index.md | 23 -------- docs/sources/operations/overrides-exporter.md | 1 - pkg/ruler/compat.go | 5 -- pkg/ruler/registry.go | 58 +------------------ pkg/validation/limits.go | 25 -------- pkg/validation/limits_test.go | 5 -- .../checker/checker_test.go | 9 --- .../deprecated-config.yaml | 3 - .../test-fixtures/config.yaml | 4 -- .../test-fixtures/runtime-config.yaml | 4 -- 10 files changed, 3 insertions(+), 134 deletions(-) diff --git a/docs/sources/configure/_index.md b/docs/sources/configure/_index.md index 1691d7c052fe..63b3e9d5b436 100644 --- a/docs/sources/configure/_index.md +++ b/docs/sources/configure/_index.md @@ -2787,29 +2787,6 @@ The `limits_config` block configures global and per-tenant limits in Loki. # by Loki itself can't be overwritten. [ruler_remote_write_headers: ] -# Deprecated: Use 'ruler_remote_write_config' instead. List of remote write -# relabel configurations. -[ruler_remote_write_relabel_configs: ] - -# Deprecated: Use 'ruler_remote_write_config' instead. Number of samples to -# buffer per shard before we block reading of more samples from the WAL. It is -# recommended to have enough capacity in each shard to buffer several requests -# to keep throughput up while processing occasional slow remote requests. -[ruler_remote_write_queue_capacity: ] - -# Deprecated: Use 'ruler_remote_write_config' instead. Configures AWS's -# Signature Verification 4 signing process to sign every remote write request. -ruler_remote_write_sigv4_config: - [region: | default = ""] - - [access_key: | default = ""] - - [secret_key: | default = ""] - - [profile: | default = ""] - - [role_arn: | default = ""] - # Configures global and per-tenant limits for remote write clients. A map with # remote client id as key. [ruler_remote_write_config: ] diff --git a/docs/sources/operations/overrides-exporter.md b/docs/sources/operations/overrides-exporter.md index 505d5dfd5f2a..be096ead1cce 100644 --- a/docs/sources/operations/overrides-exporter.md +++ b/docs/sources/operations/overrides-exporter.md @@ -73,7 +73,6 @@ loki_overrides_defaults{limit_name="retention_period"} 2.6784e+15 loki_overrides_defaults{limit_name="ruler_evaluation_delay_duration"} 0 loki_overrides_defaults{limit_name="ruler_max_rule_groups_per_tenant"} 0 loki_overrides_defaults{limit_name="ruler_max_rules_per_rule_group"} 0 -loki_overrides_defaults{limit_name="ruler_remote_write_queue_capacity"} 0 loki_overrides_defaults{limit_name="split_queries_by_interval"} 0 ``` diff --git a/pkg/ruler/compat.go b/pkg/ruler/compat.go index 1f377d64a068..4aebd9b12dde 100644 --- a/pkg/ruler/compat.go +++ b/pkg/ruler/compat.go @@ -12,7 +12,6 @@ import ( "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" - "github.com/prometheus/common/sigv4" "github.com/prometheus/prometheus/config" "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/model/rulefmt" @@ -28,7 +27,6 @@ import ( "github.com/grafana/loki/pkg/logql/syntax" ruler "github.com/grafana/loki/pkg/ruler/base" "github.com/grafana/loki/pkg/ruler/rulespb" - "github.com/grafana/loki/pkg/ruler/util" ) // RulesLimits is the one function we need from limits.Overrides, and @@ -38,10 +36,7 @@ type RulesLimits interface { RulerRemoteWriteDisabled(userID string) bool RulerRemoteWriteHeaders(userID string) map[string]string - RulerRemoteWriteRelabelConfigs(userID string) []*util.RelabelConfig RulerRemoteWriteConfig(userID string, id string) *config.RemoteWriteConfig - RulerRemoteWriteQueueCapacity(userID string) int - RulerRemoteWriteSigV4Config(userID string) *sigv4.SigV4Config RulerRemoteEvaluationTimeout(userID string) time.Duration RulerRemoteEvaluationMaxResponseSize(userID string) int64 diff --git a/pkg/ruler/registry.go b/pkg/ruler/registry.go index 7062f6bf4d2f..5982aa882db2 100644 --- a/pkg/ruler/registry.go +++ b/pkg/ruler/registry.go @@ -9,6 +9,9 @@ import ( "github.com/go-kit/log" "github.com/go-kit/log/level" "github.com/grafana/dskit/user" + "github.com/grafana/loki/pkg/ruler/storage/cleaner" + "github.com/grafana/loki/pkg/ruler/storage/instance" + "github.com/grafana/loki/pkg/ruler/storage/wal" "github.com/imdario/mergo" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" @@ -17,13 +20,7 @@ import ( "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/model/metadata" - "github.com/prometheus/prometheus/model/relabel" "github.com/prometheus/prometheus/storage" - "gopkg.in/yaml.v2" - - "github.com/grafana/loki/pkg/ruler/storage/cleaner" - "github.com/grafana/loki/pkg/ruler/storage/instance" - "github.com/grafana/loki/pkg/ruler/storage/wal" ) type walRegistry struct { @@ -241,26 +238,6 @@ func (r *walRegistry) getTenantRemoteWriteConfig(tenant string, base RemoteWrite clt.Headers = v } - relabelConfigs, err := r.createRelabelConfigs(tenant) - if err != nil { - return nil, fmt.Errorf("failed to parse relabel configs: %w", err) - } - - // if any relabel configs are defined for a tenant, override all base relabel configs, - // even if an empty list is configured; however if this value is not overridden for a tenant, - // it should retain the base value - if relabelConfigs != nil { - clt.WriteRelabelConfigs = relabelConfigs - } - - if v := r.overrides.RulerRemoteWriteQueueCapacity(tenant); v > 0 { - clt.QueueConfig.Capacity = v - } - - if v := r.overrides.RulerRemoteWriteSigV4Config(tenant); v != nil { - clt.SigV4Config = v - } - if v := r.overrides.RulerRemoteWriteConfig(tenant, id); v != nil { // overwrite, do not merge if v.Headers != nil { @@ -286,35 +263,6 @@ func (r *walRegistry) getTenantRemoteWriteConfig(tenant string, base RemoteWrite return overrides, nil } -// createRelabelConfigs converts the util.RelabelConfig into relabel.Config to allow for -// more control over json/yaml unmarshaling -func (r *walRegistry) createRelabelConfigs(tenant string) ([]*relabel.Config, error) { - configs := r.overrides.RulerRemoteWriteRelabelConfigs(tenant) - - // zero value is nil, which we want to treat as "no override" - if configs == nil { - return nil, nil - } - - // we want to treat an empty slice as "no relabel configs" - relabelConfigs := make([]*relabel.Config, len(configs)) - for i, config := range configs { - out, err := yaml.Marshal(config) - if err != nil { - return nil, err - } - - var rc relabel.Config - if err = yaml.Unmarshal(out, &rc); err != nil { - return nil, err - } - - relabelConfigs[i] = &rc - } - - return relabelConfigs, nil -} - var errNotReady = errors.New("appender not ready") type notReadyAppender struct{} diff --git a/pkg/validation/limits.go b/pkg/validation/limits.go index d8350c16b1b5..9047c7c39641 100644 --- a/pkg/validation/limits.go +++ b/pkg/validation/limits.go @@ -13,7 +13,6 @@ import ( "github.com/pkg/errors" "github.com/prometheus/common/model" - "github.com/prometheus/common/sigv4" "github.com/prometheus/prometheus/config" "github.com/prometheus/prometheus/model/labels" "golang.org/x/time/rate" @@ -23,7 +22,6 @@ import ( "github.com/grafana/loki/pkg/distributor/shardstreams" "github.com/grafana/loki/pkg/logql/syntax" ruler_config "github.com/grafana/loki/pkg/ruler/config" - "github.com/grafana/loki/pkg/ruler/util" "github.com/grafana/loki/pkg/util/flagext" util_log "github.com/grafana/loki/pkg/util/log" "github.com/grafana/loki/pkg/util/validation" @@ -129,12 +127,6 @@ type Limits struct { // deprecated use RulerRemoteWriteConfig instead RulerRemoteWriteHeaders OverwriteMarshalingStringMap `yaml:"ruler_remote_write_headers" json:"ruler_remote_write_headers" doc:"deprecated|description=Use 'ruler_remote_write_config' instead. Custom HTTP headers to be sent along with each remote write request. Be aware that headers that are set by Loki itself can't be overwritten."` - // deprecated use RulerRemoteWriteConfig instead - RulerRemoteWriteRelabelConfigs []*util.RelabelConfig `yaml:"ruler_remote_write_relabel_configs,omitempty" json:"ruler_remote_write_relabel_configs,omitempty" doc:"deprecated|description=Use 'ruler_remote_write_config' instead. List of remote write relabel configurations."` - // deprecated use RulerRemoteWriteConfig instead - RulerRemoteWriteQueueCapacity int `yaml:"ruler_remote_write_queue_capacity" json:"ruler_remote_write_queue_capacity" doc:"deprecated|description=Use 'ruler_remote_write_config' instead. Number of samples to buffer per shard before we block reading of more samples from the WAL. It is recommended to have enough capacity in each shard to buffer several requests to keep throughput up while processing occasional slow remote requests."` - // deprecated use RulerRemoteWriteConfig instead - RulerRemoteWriteSigV4Config *sigv4.SigV4Config `yaml:"ruler_remote_write_sigv4_config" json:"ruler_remote_write_sigv4_config" doc:"deprecated|description=Use 'ruler_remote_write_config' instead. Configures AWS's Signature Verification 4 signing process to sign every remote write request."` RulerRemoteWriteConfig map[string]config.RemoteWriteConfig `yaml:"ruler_remote_write_config,omitempty" json:"ruler_remote_write_config,omitempty" doc:"description=Configures global and per-tenant limits for remote write clients. A map with remote client id as key."` @@ -605,23 +597,6 @@ func (o *Overrides) RulerRemoteWriteHeaders(userID string) map[string]string { return o.getOverridesForUser(userID).RulerRemoteWriteHeaders.Map() } -// Deprecated: use RulerRemoteWriteConfig instead -// RulerRemoteWriteRelabelConfigs returns the write relabel configs to use in a remote-write for a given user. -func (o *Overrides) RulerRemoteWriteRelabelConfigs(userID string) []*util.RelabelConfig { - return o.getOverridesForUser(userID).RulerRemoteWriteRelabelConfigs -} - -// Deprecated: use RulerRemoteWriteConfig instead -// RulerRemoteWriteQueueCapacity returns the queue capacity to use in a remote-write for a given user. -func (o *Overrides) RulerRemoteWriteQueueCapacity(userID string) int { - return o.getOverridesForUser(userID).RulerRemoteWriteQueueCapacity -} - -// Deprecated: use RulerRemoteWriteConfig instead -func (o *Overrides) RulerRemoteWriteSigV4Config(userID string) *sigv4.SigV4Config { - return o.getOverridesForUser(userID).RulerRemoteWriteSigV4Config -} - // RulerRemoteWriteConfig returns the remote-write configurations to use for a given user and a given remote client. func (o *Overrides) RulerRemoteWriteConfig(userID string, id string) *config.RemoteWriteConfig { if c, ok := o.getOverridesForUser(userID).RulerRemoteWriteConfig[id]; ok { diff --git a/pkg/validation/limits_test.go b/pkg/validation/limits_test.go index 4e449e421c5a..088ce6411a5a 100644 --- a/pkg/validation/limits_test.go +++ b/pkg/validation/limits_test.go @@ -66,8 +66,6 @@ split_queries_by_interval: 190s ruler_evaluation_delay_duration: 200s ruler_max_rules_per_rule_group: 210 ruler_max_rule_groups_per_tenant: 220 -ruler_remote_write_sigv4_config: - region: us-east-1 per_tenant_override_config: "" per_tenant_override_period: 230s query_timeout: 5m @@ -111,9 +109,6 @@ volume_max_series: 10001 "ruler_evaluation_delay_duration": "200s", "ruler_max_rules_per_rule_group": 210, "ruler_max_rule_groups_per_tenant":220, - "ruler_remote_write_sigv4_config": { - "region": "us-east-1" - }, "per_tenant_override_config": "", "per_tenant_override_period": "230s", "query_timeout": "5m", diff --git a/tools/deprecated-config-checker/checker/checker_test.go b/tools/deprecated-config-checker/checker/checker_test.go index 74762db30a74..d5a8636d30e4 100644 --- a/tools/deprecated-config-checker/checker/checker_test.go +++ b/tools/deprecated-config-checker/checker/checker_test.go @@ -48,9 +48,6 @@ var ( "limits_config.unordered_writes", "limits_config.ruler_evaluation_delay_duration", "limits_config.ruler_remote_write_headers", - "limits_config.ruler_remote_write_relabel_configs", - "limits_config.ruler_remote_write_queue_capacity", - "limits_config.ruler_remote_write_sigv4_config", "limits_config.per_tenant_override_config", "limits_config.per_tenant_override_period", "limits_config.allow_deletes", @@ -78,18 +75,12 @@ var ( "overrides.foo.unordered_writes", "overrides.foo.ruler_evaluation_delay_duration", "overrides.foo.ruler_remote_write_headers", - "overrides.foo.ruler_remote_write_relabel_configs", - "overrides.foo.ruler_remote_write_queue_capacity", - "overrides.foo.ruler_remote_write_sigv4_config", "overrides.foo.per_tenant_override_config", "overrides.foo.per_tenant_override_period", "overrides.foo.allow_deletes", "overrides.bar.unordered_writes", "overrides.bar.ruler_evaluation_delay_duration", "overrides.bar.ruler_remote_write_headers", - "overrides.bar.ruler_remote_write_relabel_configs", - "overrides.bar.ruler_remote_write_queue_capacity", - "overrides.bar.ruler_remote_write_sigv4_config", "overrides.bar.per_tenant_override_config", "overrides.bar.per_tenant_override_period", "overrides.bar.allow_deletes", diff --git a/tools/deprecated-config-checker/deprecated-config.yaml b/tools/deprecated-config-checker/deprecated-config.yaml index cd8096386f97..90c5771ad556 100644 --- a/tools/deprecated-config-checker/deprecated-config.yaml +++ b/tools/deprecated-config-checker/deprecated-config.yaml @@ -50,9 +50,6 @@ limits_config: unordered_writes: "Will be eventually removed." ruler_evaluation_delay_duration: "Will be eventually removed." ruler_remote_write_headers: "Use ruler_remote_write_config instead." - ruler_remote_write_relabel_configs: "Use ruler_remote_write_config instead." - ruler_remote_write_queue_capacity: "Use ruler_remote_write_config instead." - ruler_remote_write_sigv4_config: "Use ruler_remote_write_config instead." per_tenant_override_config: "Feature renamed to 'runtime configuration', flag deprecated in favor of runtime_config.file" per_tenant_override_period: "Feature renamed to 'runtime configuration', flag deprecated in favor of runtime_config.period" allow_deletes: "Use deletion_mode per tenant configuration instead." diff --git a/tools/deprecated-config-checker/test-fixtures/config.yaml b/tools/deprecated-config-checker/test-fixtures/config.yaml index eefe70c114ac..5452de543b0e 100644 --- a/tools/deprecated-config-checker/test-fixtures/config.yaml +++ b/tools/deprecated-config-checker/test-fixtures/config.yaml @@ -119,10 +119,6 @@ limits_config: unordered_writes: true # DEPRECATED ruler_evaluation_delay_duration: 1m # DEPRECATED ruler_remote_write_headers: ["foo", "bar"] # DEPRECATED - ruler_remote_write_relabel_configs: "foo" # DEPRECATED - ruler_remote_write_queue_capacity: 10 # DEPRECATED - ruler_remote_write_sigv4_config: # DEPRECATED - region: "wherever" per_tenant_override_config: ./overrides.yaml # DEPRECATED per_tenant_override_period: 5s # DEPRECATED allow_deletes: true # DEPRECATED diff --git a/tools/deprecated-config-checker/test-fixtures/runtime-config.yaml b/tools/deprecated-config-checker/test-fixtures/runtime-config.yaml index b699adc1a9aa..865e35f4495c 100644 --- a/tools/deprecated-config-checker/test-fixtures/runtime-config.yaml +++ b/tools/deprecated-config-checker/test-fixtures/runtime-config.yaml @@ -4,10 +4,6 @@ overrides: unordered_writes: true # DEPRECATED ruler_evaluation_delay_duration: 1m # DEPRECATED ruler_remote_write_headers: [ "foo", "bar" ] # DEPRECATED - ruler_remote_write_relabel_configs: "foo" # DEPRECATED - ruler_remote_write_queue_capacity: 10 # DEPRECATED - ruler_remote_write_sigv4_config: # DEPRECATED - region: "wherever" per_tenant_override_config: ./overrides.yaml # DEPRECATED per_tenant_override_period: 5s # DEPRECATED allow_deletes: true # DEPRECATED From d500ba3be6ff9d73a71a1618c5953b2a1fdeb4b7 Mon Sep 17 00:00:00 2001 From: shantanualshi Date: Thu, 2 Nov 2023 19:42:27 +0530 Subject: [PATCH 08/13] Format --- pkg/ruler/registry.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/ruler/registry.go b/pkg/ruler/registry.go index 5982aa882db2..f46a1991895d 100644 --- a/pkg/ruler/registry.go +++ b/pkg/ruler/registry.go @@ -9,9 +9,6 @@ import ( "github.com/go-kit/log" "github.com/go-kit/log/level" "github.com/grafana/dskit/user" - "github.com/grafana/loki/pkg/ruler/storage/cleaner" - "github.com/grafana/loki/pkg/ruler/storage/instance" - "github.com/grafana/loki/pkg/ruler/storage/wal" "github.com/imdario/mergo" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" @@ -21,6 +18,10 @@ import ( "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/model/metadata" "github.com/prometheus/prometheus/storage" + + "github.com/grafana/loki/pkg/ruler/storage/cleaner" + "github.com/grafana/loki/pkg/ruler/storage/instance" + "github.com/grafana/loki/pkg/ruler/storage/wal" ) type walRegistry struct { From 23a7a64749530124b0118d75e90bd5f7d4efd445 Mon Sep 17 00:00:00 2001 From: shantanualshi Date: Fri, 3 Nov 2023 13:39:06 +0530 Subject: [PATCH 09/13] Add documentation for ruler_remote_write_config --- docs/sources/configure/_index.md | 38 +++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/docs/sources/configure/_index.md b/docs/sources/configure/_index.md index 63b3e9d5b436..f398090a0f27 100644 --- a/docs/sources/configure/_index.md +++ b/docs/sources/configure/_index.md @@ -2789,7 +2789,43 @@ The `limits_config` block configures global and per-tenant limits in Loki. # Configures global and per-tenant limits for remote write clients. A map with # remote client id as key. -[ruler_remote_write_config: ] +ruler_remote_write_config: + # Tenant ID configuration + tenant_id: + # The URL of the endpoint to send samples to + [url: ] + # Timeout for requests to the remote write endpoint + [remote_timeout: ] + # Custom HTTP headers to be sent along with each remote write request. + # Be aware that headers that are set by Loki itself can't be overwritten. + [headers: ] + # List of remote write relabel configurations + [write_relabel_configs: ] + # Queue + queue_config: + # Number of samples to buffer per shard before we block reading of more samples + # from the WAL. It is recommended to have enough capacity in each shard to buffer + # several requests to keep throughput up while processing occasional slow remote requests. + [capacity: ] + # Maximum number of shards, i.e. amount of concurrency. + [max_shards: ] + # Minimum number of shards, i.e. amount of concurrency + [min_shards: ] + # Maximum number of samples per send + [max_samples_per_send: ] + # Maximum time sample will wait in buffer. + [batch_send_deadline: ] + # Initial retry delay. Gets doubled for every retry. + [min_backoff: ] + # Maximum retry delay. + [max_backoff: ] + # Retry upon receiving a 429 status code from the remote-write storage. + # This is experimental and might change in the future. + [retry_on_http_429: ] + + + + # Timeout for a remote rule evaluation. Defaults to the value of # 'querier.query-timeout'. From f68cffa06fd6725e1850f08edf6caedcde9a5bf0 Mon Sep 17 00:00:00 2001 From: shantanualshi Date: Thu, 16 Nov 2023 15:33:51 +0530 Subject: [PATCH 10/13] Remove deprecated ruler_remote_write_headers --- docs/sources/configure/_index.md | 5 - pkg/ruler/compat.go | 1 - pkg/ruler/registry.go | 5 - pkg/validation/limits.go | 10 -- pkg/validation/limits_test.go | 104 ++++++++++++++++-- .../checker/checker_test.go | 3 - .../deprecated-config.yaml | 1 - .../test-fixtures/config.yaml | 1 - .../test-fixtures/runtime-config.yaml | 1 - 9 files changed, 94 insertions(+), 37 deletions(-) diff --git a/docs/sources/configure/_index.md b/docs/sources/configure/_index.md index 2c52924b127b..bc58abf80df1 100644 --- a/docs/sources/configure/_index.md +++ b/docs/sources/configure/_index.md @@ -2793,11 +2793,6 @@ The `limits_config` block configures global and per-tenant limits in Loki. # Disable recording rules remote-write. [ruler_remote_write_disabled: ] -# Deprecated: Use 'ruler_remote_write_config' instead. Custom HTTP headers to be -# sent along with each remote write request. Be aware that headers that are set -# by Loki itself can't be overwritten. -[ruler_remote_write_headers: ] - # Configures global and per-tenant limits for remote write clients. A map with # remote client id as key. ruler_remote_write_config: diff --git a/pkg/ruler/compat.go b/pkg/ruler/compat.go index 69dc51c41b8c..b37613faa03d 100644 --- a/pkg/ruler/compat.go +++ b/pkg/ruler/compat.go @@ -35,7 +35,6 @@ type RulesLimits interface { ruler.RulesLimits RulerRemoteWriteDisabled(userID string) bool - RulerRemoteWriteHeaders(userID string) map[string]string RulerRemoteWriteConfig(userID string, id string) *config.RemoteWriteConfig RulerRemoteEvaluationTimeout(userID string) time.Duration diff --git a/pkg/ruler/registry.go b/pkg/ruler/registry.go index f46a1991895d..4d9edd18e944 100644 --- a/pkg/ruler/registry.go +++ b/pkg/ruler/registry.go @@ -234,11 +234,6 @@ func (r *walRegistry) getTenantRemoteWriteConfig(tenant string, base RemoteWrite // metadata is only used by prometheus scrape configs clt.MetadataConfig = config.MetadataConfig{Send: false} - // overwrite, do not merge - if v := r.overrides.RulerRemoteWriteHeaders(tenant); v != nil { - clt.Headers = v - } - if v := r.overrides.RulerRemoteWriteConfig(tenant, id); v != nil { // overwrite, do not merge if v.Headers != nil { diff --git a/pkg/validation/limits.go b/pkg/validation/limits.go index 708c2f01a538..f283067ca278 100644 --- a/pkg/validation/limits.go +++ b/pkg/validation/limits.go @@ -115,15 +115,11 @@ type Limits struct { RulerTenantShardSize int `yaml:"ruler_tenant_shard_size" json:"ruler_tenant_shard_size"` // TODO(dannyk): add HTTP client overrides (basic auth / tls config, etc) - // Ruler remote-write limits. // this field is the inversion of the general remote_write.enabled because the zero value of a boolean is false, // and if it were ruler_remote_write_enabled, it would be impossible to know if the value was explicitly set or default RulerRemoteWriteDisabled bool `yaml:"ruler_remote_write_disabled" json:"ruler_remote_write_disabled" doc:"description=Disable recording rules remote-write."` - // deprecated use RulerRemoteWriteConfig instead - RulerRemoteWriteHeaders OverwriteMarshalingStringMap `yaml:"ruler_remote_write_headers" json:"ruler_remote_write_headers" doc:"deprecated|description=Use 'ruler_remote_write_config' instead. Custom HTTP headers to be sent along with each remote write request. Be aware that headers that are set by Loki itself can't be overwritten."` - RulerRemoteWriteConfig map[string]config.RemoteWriteConfig `yaml:"ruler_remote_write_config,omitempty" json:"ruler_remote_write_config,omitempty" doc:"description=Configures global and per-tenant limits for remote write clients. A map with remote client id as key."` // TODO(dannyk): possible enhancement is to align this with rule group interval @@ -586,12 +582,6 @@ func (o *Overrides) RulerRemoteWriteDisabled(userID string) bool { return o.getOverridesForUser(userID).RulerRemoteWriteDisabled } -// Deprecated: use RulerRemoteWriteConfig instead -// RulerRemoteWriteHeaders returns the headers to use in a remote-write for a given user. -func (o *Overrides) RulerRemoteWriteHeaders(userID string) map[string]string { - return o.getOverridesForUser(userID).RulerRemoteWriteHeaders.Map() -} - // RulerRemoteWriteConfig returns the remote-write configurations to use for a given user and a given remote client. func (o *Overrides) RulerRemoteWriteConfig(userID string, id string) *config.RemoteWriteConfig { if c, ok := o.getOverridesForUser(userID).RulerRemoteWriteConfig[id]; ok { diff --git a/pkg/validation/limits_test.go b/pkg/validation/limits_test.go index 088ce6411a5a..db481b0c5e94 100644 --- a/pkg/validation/limits_test.go +++ b/pkg/validation/limits_test.go @@ -2,12 +2,15 @@ package validation import ( "encoding/json" + "net/url" "reflect" "testing" "time" "github.com/pkg/errors" + promConfig "github.com/prometheus/common/config" "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/config" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "gopkg.in/yaml.v2" @@ -168,10 +171,17 @@ func TestLimitsDoesNotMutate(t *testing.T) { defer func() { defaultLimits = initialDefault }() - + var remoteURL, _ = url.Parse("http://remote-write") // Set new defaults with non-nil values for non-scalar types newDefaults := Limits{ - RulerRemoteWriteHeaders: OverwriteMarshalingStringMap{map[string]string{"a": "b"}}, + RulerRemoteWriteConfig: map[string]config.RemoteWriteConfig{ + "default": { + URL: &promConfig.URL{URL: remoteURL}, + Headers: map[string]string{"a": "b"}, + RemoteTimeout: model.Duration(30 * time.Second), + QueueConfig: defaultQueueConfig(), + }, + }, StreamRetention: []StreamRetention{ { Period: model.Duration(24 * time.Hour), @@ -189,11 +199,30 @@ func TestLimitsDoesNotMutate(t *testing.T) { { desc: "map", yaml: ` -ruler_remote_write_headers: - foo: "bar" +ruler_remote_write_config: + tenant: + url: "http://remote-write" + headers: + foo: bar `, exp: Limits{ - RulerRemoteWriteHeaders: OverwriteMarshalingStringMap{map[string]string{"foo": "bar"}}, + RulerRemoteWriteConfig: map[string]config.RemoteWriteConfig{ + "default": { + URL: &promConfig.URL{URL: remoteURL}, + Headers: map[string]string{"a": "b"}, + QueueConfig: config.DefaultQueueConfig, + MetadataConfig: config.DefaultMetadataConfig, + RemoteTimeout: model.Duration(30000000000), + }, + "tenant": { + URL: &promConfig.URL{URL: remoteURL}, + HTTPClientConfig: promConfig.DefaultHTTPClientConfig, + Headers: map[string]string{"foo": "bar"}, + QueueConfig: config.DefaultQueueConfig, + MetadataConfig: config.DefaultMetadataConfig, + RemoteTimeout: model.Duration(30000000000), + }, + }, // Rest from new defaults StreamRetention: []StreamRetention{ @@ -207,10 +236,28 @@ ruler_remote_write_headers: { desc: "empty map overrides defaults", yaml: ` -ruler_remote_write_headers: +ruler_remote_write_config: + tenant: + url: "http://remote-write" + headers: `, exp: Limits{ - + RulerRemoteWriteConfig: map[string]config.RemoteWriteConfig{ + "default": { + URL: &promConfig.URL{URL: remoteURL}, + Headers: map[string]string{"a": "b"}, + QueueConfig: config.DefaultQueueConfig, + MetadataConfig: config.DefaultMetadataConfig, + RemoteTimeout: model.Duration(30000000000), + }, + "tenant": { + URL: &promConfig.URL{URL: remoteURL}, + HTTPClientConfig: promConfig.DefaultHTTPClientConfig, + QueueConfig: config.DefaultQueueConfig, + MetadataConfig: config.DefaultMetadataConfig, + RemoteTimeout: model.Duration(30000000000), + }, + }, // Rest from new defaults StreamRetention: []StreamRetention{ { @@ -236,7 +283,15 @@ retention_stream: }, // Rest from new defaults - RulerRemoteWriteHeaders: OverwriteMarshalingStringMap{map[string]string{"a": "b"}}, + RulerRemoteWriteConfig: map[string]config.RemoteWriteConfig{ + "default": { + URL: &promConfig.URL{URL: remoteURL}, + Headers: map[string]string{"a": "b"}, + QueueConfig: config.DefaultQueueConfig, + MetadataConfig: config.DefaultMetadataConfig, + RemoteTimeout: model.Duration(30000000000), + }, + }, }, }, { @@ -248,7 +303,15 @@ reject_old_samples: true RejectOldSamples: true, // Rest from new defaults - RulerRemoteWriteHeaders: OverwriteMarshalingStringMap{map[string]string{"a": "b"}}, + RulerRemoteWriteConfig: map[string]config.RemoteWriteConfig{ + "default": { + URL: &promConfig.URL{URL: remoteURL}, + Headers: map[string]string{"a": "b"}, + QueueConfig: config.DefaultQueueConfig, + MetadataConfig: config.DefaultMetadataConfig, + RemoteTimeout: model.Duration(30000000000), + }, + }, StreamRetention: []StreamRetention{ { Period: model.Duration(24 * time.Hour), @@ -266,7 +329,15 @@ query_timeout: 5m QueryTimeout: model.Duration(5 * time.Minute), // Rest from new defaults. - RulerRemoteWriteHeaders: OverwriteMarshalingStringMap{map[string]string{"a": "b"}}, + RulerRemoteWriteConfig: map[string]config.RemoteWriteConfig{ + "default": { + URL: &promConfig.URL{URL: remoteURL}, + Headers: map[string]string{"a": "b"}, + QueueConfig: config.DefaultQueueConfig, + MetadataConfig: config.DefaultMetadataConfig, + RemoteTimeout: model.Duration(30000000000), + }, + }, StreamRetention: []StreamRetention{ { Period: model.Duration(24 * time.Hour), @@ -299,3 +370,16 @@ func TestLimitsValidation(t *testing.T) { require.True(t, errors.Is(limits.Validate(), tc.expected)) } } + +func defaultQueueConfig() config.QueueConfig { + return config.QueueConfig{ + Capacity: 10000, + MaxShards: 50, + MinShards: 1, + MaxSamplesPerSend: 2000, + BatchSendDeadline: model.Duration(5 * time.Second), + MinBackoff: model.Duration(30 * time.Millisecond), + MaxBackoff: model.Duration(5 * time.Second), + RetryOnRateLimit: false, + } +} diff --git a/tools/deprecated-config-checker/checker/checker_test.go b/tools/deprecated-config-checker/checker/checker_test.go index b84b48b89daf..b49a35f3afaa 100644 --- a/tools/deprecated-config-checker/checker/checker_test.go +++ b/tools/deprecated-config-checker/checker/checker_test.go @@ -48,7 +48,6 @@ var ( "storage_config.aws.dynamodb", "chunk_store_config.write_dedupe_cache_config", "limits_config.unordered_writes", - "limits_config.ruler_remote_write_headers", "limits_config.per_tenant_override_config", "limits_config.per_tenant_override_period", "limits_config.allow_deletes", @@ -79,12 +78,10 @@ var ( expectedRuntimeConfigDeprecates = []string{ "overrides.foo.unordered_writes", - "overrides.foo.ruler_remote_write_headers", "overrides.foo.per_tenant_override_config", "overrides.foo.per_tenant_override_period", "overrides.foo.allow_deletes", "overrides.bar.unordered_writes", - "overrides.bar.ruler_remote_write_headers", "overrides.bar.per_tenant_override_config", "overrides.bar.per_tenant_override_period", "overrides.bar.allow_deletes", diff --git a/tools/deprecated-config-checker/deprecated-config.yaml b/tools/deprecated-config-checker/deprecated-config.yaml index 09e659db9686..e9426b051af0 100644 --- a/tools/deprecated-config-checker/deprecated-config.yaml +++ b/tools/deprecated-config-checker/deprecated-config.yaml @@ -45,7 +45,6 @@ chunk_store_config: ## NOTE: This will also be used to validate per-tenant overrides. limits_config: unordered_writes: "Will be eventually removed." - ruler_remote_write_headers: "Use ruler_remote_write_config instead." per_tenant_override_config: "Feature renamed to 'runtime configuration', flag deprecated in favor of runtime_config.file" per_tenant_override_period: "Feature renamed to 'runtime configuration', flag deprecated in favor of runtime_config.period" allow_deletes: "Use deletion_mode per tenant configuration instead." diff --git a/tools/deprecated-config-checker/test-fixtures/config.yaml b/tools/deprecated-config-checker/test-fixtures/config.yaml index 502fedc87f0d..59a76a95f218 100644 --- a/tools/deprecated-config-checker/test-fixtures/config.yaml +++ b/tools/deprecated-config-checker/test-fixtures/config.yaml @@ -119,7 +119,6 @@ limits_config: unordered_writes: true # DEPRECATED enforce_metric_name: true # DELETED ruler_evaluation_delay_duration: 1m # DELETED - ruler_remote_write_headers: ["foo", "bar"] # DEPRECATED per_tenant_override_config: ./overrides.yaml # DEPRECATED per_tenant_override_period: 5s # DEPRECATED allow_deletes: true # DEPRECATED diff --git a/tools/deprecated-config-checker/test-fixtures/runtime-config.yaml b/tools/deprecated-config-checker/test-fixtures/runtime-config.yaml index eb358459a312..7aa5d9968b10 100644 --- a/tools/deprecated-config-checker/test-fixtures/runtime-config.yaml +++ b/tools/deprecated-config-checker/test-fixtures/runtime-config.yaml @@ -4,7 +4,6 @@ overrides: unordered_writes: true # DEPRECATED enforce_metric_name: true # DELETED ruler_evaluation_delay_duration: 1m # DELETED - ruler_remote_write_headers: [ "foo", "bar" ] # DEPRECATED per_tenant_override_config: ./overrides.yaml # DEPRECATED per_tenant_override_period: 5s # DEPRECATED allow_deletes: true # DEPRECATED From 5f8d499b4dcf693607798ec98fa6848637e7480d Mon Sep 17 00:00:00 2001 From: shantanualshi Date: Thu, 16 Nov 2023 15:42:12 +0530 Subject: [PATCH 11/13] Use in-built default queue config --- pkg/validation/limits_test.go | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/pkg/validation/limits_test.go b/pkg/validation/limits_test.go index db481b0c5e94..dc7bd5da4640 100644 --- a/pkg/validation/limits_test.go +++ b/pkg/validation/limits_test.go @@ -179,7 +179,7 @@ func TestLimitsDoesNotMutate(t *testing.T) { URL: &promConfig.URL{URL: remoteURL}, Headers: map[string]string{"a": "b"}, RemoteTimeout: model.Duration(30 * time.Second), - QueueConfig: defaultQueueConfig(), + QueueConfig: config.DefaultQueueConfig, }, }, StreamRetention: []StreamRetention{ @@ -370,16 +370,3 @@ func TestLimitsValidation(t *testing.T) { require.True(t, errors.Is(limits.Validate(), tc.expected)) } } - -func defaultQueueConfig() config.QueueConfig { - return config.QueueConfig{ - Capacity: 10000, - MaxShards: 50, - MinShards: 1, - MaxSamplesPerSend: 2000, - BatchSendDeadline: model.Duration(5 * time.Second), - MinBackoff: model.Duration(30 * time.Millisecond), - MaxBackoff: model.Duration(5 * time.Second), - RetryOnRateLimit: false, - } -} From c60116bcaad356283a4140c6b88f9a3f9f84968a Mon Sep 17 00:00:00 2001 From: shantanualshi Date: Thu, 16 Nov 2023 15:47:34 +0530 Subject: [PATCH 12/13] Add changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b01d4e3dc181..42903ab8a588 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,7 @@ * [10727](https://github.com/grafana/loki/pull/10727) **sandeepsukhani** Native otlp ingestion support * [11051](https://github.com/grafana/loki/pull/11051) Refactor to not use global logger in modules * [10956](https://github.com/grafana/loki/pull/10956) **jeschkies** do not wrap requests but send pure Protobuf from frontend v2 via scheduler to querier when `-frontend.encoding=protobuf`. +* [11105](https://github.com/grafana/loki/pull/11105) **shantanualsi** Remove deprecated `ruler_remote_write_*` configs from `limit_config` in favour of `remote_write_config` ##### Fixes * [11074](https://github.com/grafana/loki/pull/11074) **hainenber** Fix panic in lambda-promtail due to mishandling of empty DROP_LABELS env var. From 91bece12e4eaa90d75b64770cb11cae9aaff0112 Mon Sep 17 00:00:00 2001 From: shantanualshi Date: Thu, 16 Nov 2023 15:52:18 +0530 Subject: [PATCH 13/13] Fix docs --- docs/sources/configure/_index.md | 38 +------------------------------- 1 file changed, 1 insertion(+), 37 deletions(-) diff --git a/docs/sources/configure/_index.md b/docs/sources/configure/_index.md index bc58abf80df1..1e28f3764345 100644 --- a/docs/sources/configure/_index.md +++ b/docs/sources/configure/_index.md @@ -2795,43 +2795,7 @@ The `limits_config` block configures global and per-tenant limits in Loki. # Configures global and per-tenant limits for remote write clients. A map with # remote client id as key. -ruler_remote_write_config: - # Tenant ID configuration - tenant_id: - # The URL of the endpoint to send samples to - [url: ] - # Timeout for requests to the remote write endpoint - [remote_timeout: ] - # Custom HTTP headers to be sent along with each remote write request. - # Be aware that headers that are set by Loki itself can't be overwritten. - [headers: ] - # List of remote write relabel configurations - [write_relabel_configs: ] - # Queue - queue_config: - # Number of samples to buffer per shard before we block reading of more samples - # from the WAL. It is recommended to have enough capacity in each shard to buffer - # several requests to keep throughput up while processing occasional slow remote requests. - [capacity: ] - # Maximum number of shards, i.e. amount of concurrency. - [max_shards: ] - # Minimum number of shards, i.e. amount of concurrency - [min_shards: ] - # Maximum number of samples per send - [max_samples_per_send: ] - # Maximum time sample will wait in buffer. - [batch_send_deadline: ] - # Initial retry delay. Gets doubled for every retry. - [min_backoff: ] - # Maximum retry delay. - [max_backoff: ] - # Retry upon receiving a 429 status code from the remote-write storage. - # This is experimental and might change in the future. - [retry_on_http_429: ] - - - - +[ruler_remote_write_config: ] # Timeout for a remote rule evaluation. Defaults to the value of # 'querier.query-timeout'.