-
Notifications
You must be signed in to change notification settings - Fork 3
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Wait for Prometheus to be Ready before trying to take leadership (#34)
* feat(readiness): add minimal implementation to wait for readiness * feat(cmd): wait for prometheus to be ready before starting * chore(*): cleaner logging * feat(helm): wait for prometheus to be ready * chore(README): update configuration reference
- Loading branch information
Showing
11 changed files
with
190 additions
and
16 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,68 @@ | ||
| package readiness | ||
|
|
||
| import ( | ||
| "context" | ||
| "net/http" | ||
| "time" | ||
|
|
||
| "k8s.io/klog/v2" | ||
| ) | ||
|
|
||
| type httpWaiter struct { | ||
| url string | ||
| pollPeriod time.Duration | ||
|
|
||
| httpClient *http.Client | ||
| } | ||
|
|
||
| func NewHTTP(url string, pollPeriod time.Duration) Waiter { | ||
| return &httpWaiter{ | ||
| url: url, | ||
| pollPeriod: pollPeriod, | ||
| httpClient: http.DefaultClient, | ||
| } | ||
| } | ||
|
|
||
| func (w *httpWaiter) Wait(ctx context.Context) error { | ||
| klog.InfoS("Waiting for prometheus to be ready", "poll_period", w.pollPeriod, "url", w.url) | ||
|
|
||
| tick := time.NewTicker(w.pollPeriod) | ||
| defer tick.Stop() | ||
|
|
||
| for { | ||
| select { | ||
| case <-ctx.Done(): | ||
| return ctx.Err() | ||
| case <-tick.C: | ||
| ready, err := w.checkReadiness(ctx) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| if ready { | ||
| klog.Info("Prometheus is ready") | ||
| return nil | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| func (w *httpWaiter) checkReadiness(ctx context.Context) (bool, error) { | ||
| req, err := http.NewRequestWithContext(ctx, http.MethodGet, w.url, http.NoBody) | ||
| if err != nil { | ||
| return false, err | ||
| } | ||
|
|
||
| rsp, err := w.httpClient.Do(req) | ||
| if err != nil { | ||
| klog.ErrorS(err, "Failed to check if Prometheus is ready") | ||
| return false, nil | ||
| } | ||
|
|
||
| if rsp.StatusCode != http.StatusOK { | ||
| klog.Error("Prometheus isn't ready yet", "status", rsp.StatusCode) | ||
| return false, nil | ||
| } | ||
|
|
||
| return true, nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,36 @@ | ||
| package readiness_test | ||
|
|
||
| import ( | ||
| "context" | ||
| "net/http" | ||
| "net/http/httptest" | ||
| "testing" | ||
| "time" | ||
|
|
||
| "github.com/jlevesy/prometheus-elector/readiness" | ||
| "github.com/stretchr/testify/assert" | ||
| "github.com/stretchr/testify/require" | ||
| ) | ||
|
|
||
| func TestHTTPWaiter(t *testing.T) { | ||
| var checkCalled bool | ||
|
|
||
| srv := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { | ||
| assert.Equal(t, "/foo", r.URL.Path) | ||
|
|
||
| if checkCalled { | ||
| rw.WriteHeader(http.StatusOK) | ||
| return | ||
| } | ||
|
|
||
| checkCalled = true | ||
| rw.WriteHeader(http.StatusInsufficientStorage) | ||
| })) | ||
| defer srv.Close() | ||
|
|
||
| waiter := readiness.NewHTTP(srv.URL+"/foo", 200*time.Millisecond) | ||
|
|
||
| err := waiter.Wait(context.Background()) | ||
| require.NoError(t, err) | ||
| assert.True(t, checkCalled) | ||
| } |
Oops, something went wrong.