Skip to content

Commit

Permalink
Extend hostid test backoff (#49193)
Browse files Browse the repository at this point in the history
Attempts to reduce any flakiness caused by contention producing a
host id by allowing tests to use a longer exponential backoff and
retrying more times.
  • Loading branch information
rosstimothy committed Nov 19, 2024

Verified

This commit was signed with the committer’s verified signature.
1 parent a575d66 commit 7b629fc
Show file tree
Hide file tree
Showing 2 changed files with 52 additions and 12 deletions.
13 changes: 12 additions & 1 deletion lib/utils/hostid/hostid_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,12 +24,14 @@ import (
"path/filepath"
"strings"
"testing"
"time"

"github.com/google/uuid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/sync/errgroup"

"github.com/gravitational/teleport/api/utils/retryutils"
"github.com/gravitational/teleport/lib/utils"
"github.com/gravitational/teleport/lib/utils/hostid"
)
Expand All @@ -53,7 +55,16 @@ func TestReadOrCreate(t *testing.T) {
i := i
wg.Go(func() error {
<-barrier
id, err := hostid.ReadOrCreateFile(dir)
id, err := hostid.ReadOrCreateFile(
dir,
hostid.WithBackoff(retryutils.RetryV2Config{
First: 50 * time.Millisecond,
Driver: retryutils.NewExponentialDriver(100 * time.Millisecond),
Max: 15 * time.Second,
Jitter: retryutils.FullJitter,
}),
hostid.WithIterationLimit(10),
)
ids[i] = id
return err
})
Expand Down
51 changes: 40 additions & 11 deletions lib/utils/hostid/hostid_unix.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,23 +44,52 @@ func WriteFile(dataDir string, id string) error {
return nil
}

type options struct {
retryConfig retryutils.RetryV2Config
iterationLimit int
}

// WithBackoff overrides the default backoff configuration of
// [ReadOrCreateFile].
func WithBackoff(cfg retryutils.RetryV2Config) func(*options) {
return func(o *options) {
o.retryConfig = cfg
}
}

// WithIterationLimit overrides the default number of time
// [ReadOrCreateFile] will attempt to produce a hostid.
func WithIterationLimit(limit int) func(*options) {
return func(o *options) {
o.iterationLimit = limit
}
}

// ReadOrCreateFile looks for a hostid file in the data dir. If present,
// returns the UUID from it, otherwise generates one
func ReadOrCreateFile(dataDir string) (string, error) {
// returns the UUID from it, otherwise generates one.
func ReadOrCreateFile(dataDir string, opts ...func(*options)) (string, error) {
o := options{
retryConfig: retryutils.RetryV2Config{
First: 100 * time.Millisecond,
Driver: retryutils.NewLinearDriver(100 * time.Millisecond),
Max: time.Second,
Jitter: retryutils.NewFullJitter,
},
iterationLimit: 3,
}

for _, opt := range opts {
opt(&o)
}

hostUUIDFileLock := GetPath(dataDir) + ".lock"
const iterationLimit = 3

backoff, err := retryutils.NewRetryV2(retryutils.RetryV2Config{
First: 100 * time.Millisecond,
Driver: retryutils.NewLinearDriver(100 * time.Millisecond),
Max: time.Second,
Jitter: retryutils.NewFullJitter(),
})

backoff, err := retryutils.NewRetryV2(o.retryConfig)
if err != nil {
return "", trace.Wrap(err)
}

for i := 0; i < iterationLimit; i++ {
for i := 0; i < o.iterationLimit; i++ {
if read, err := ReadFile(dataDir); err == nil {
return read, nil
} else if !trace.IsNotFound(err) {
Expand Down

0 comments on commit 7b629fc

Please sign in to comment.