From 89f35ea3635414062100d73c19f8520feb6da841 Mon Sep 17 00:00:00 2001 From: Dimitar Dimitrov Date: Tue, 10 Dec 2024 14:26:19 +0200 Subject: [PATCH 1/2] Set timeout for k8s HTTP client We had a request to an ingesters take 20m and block the reconiliation loop. We want to avoid that, so adding a timeout. We use this client for both k8s and StatefulSet requests. __I assume the operations that the StatefulSet requests will always be fast and will never have to actually do a lot of work (like uploading blocks). Can the reviewers confirm this assumption?__ --- CHANGELOG.md | 1 + cmd/rollout-operator/main.go | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 54afb43a..ade493e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## main / unreleased * [ENHANCEMENT] New parameter log.format allows to set logging format to logfmt (default) or json (new). #184 +* [ENHANCEMENT] Add a 5 minute timeout to requests to Pods and to the Kubernetes control plane. ## v0.21.0 diff --git a/cmd/rollout-operator/main.go b/cmd/rollout-operator/main.go index 0d1d6d1a..8b3a8e39 100644 --- a/cmd/rollout-operator/main.go +++ b/cmd/rollout-operator/main.go @@ -276,6 +276,10 @@ func buildKubeConfig(apiURL, cfgFile string) (*rest.Config, error) { if err != nil { return nil, err } + // Set a generous timeout. + // We use this client for various HTTP operations against the k8s API and against the StatefulSets. + // We want to not be stuck waiting forever on a TCP timeout, but also not interrupt any process the StatefulSets might eb doing. + config.Timeout = 5 * time.Minute return config, nil } From 4ca68b79ba17ccfa831c09222631c4311444e037 Mon Sep 17 00:00:00 2001 From: Dimitar Dimitrov Date: Tue, 10 Dec 2024 14:26:41 +0200 Subject: [PATCH 2/2] Add CHANGELOG.md entry PR number --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ade493e4..fe009c49 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,8 +3,8 @@ ## main / unreleased * [ENHANCEMENT] New parameter log.format allows to set logging format to logfmt (default) or json (new). #184 -* [ENHANCEMENT] Add a 5 minute timeout to requests to Pods and to the Kubernetes control plane. - +* [ENHANCEMENT] Add a 5 minute timeout to requests to Pods and to the Kubernetes control plane. #186 + ## v0.21.0 * [ENHANCEMENT] Log debug information about StatefulSets as they are created, updated and deleted. #182