-
Notifications
You must be signed in to change notification settings - Fork 153
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
fix: add retry max retry mechanism for backup creation
Prevent the backup CR from being stuck in the pending state forever. Backup creation will only be retried up until max retry longhorn-10090 Signed-off-by: Phan Le <[email protected]>
- Loading branch information
1 parent
6ab8076
commit 157e7e8
Showing
3 changed files
with
195 additions
and
4 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,88 @@ | ||
| package util | ||
|
|
||
| import ( | ||
| "sync" | ||
| "time" | ||
| ) | ||
|
|
||
| type counterEntry struct { | ||
| count int | ||
| lastUpdate time.Time | ||
| } | ||
|
|
||
| type TimedCounter struct { | ||
| sync.RWMutex | ||
| perItemCounter map[string]*counterEntry | ||
|
|
||
| expiredDuration time.Duration | ||
| } | ||
|
|
||
| func NewTimedCounter(expiredDuration time.Duration) *TimedCounter { | ||
| return &TimedCounter{ | ||
| perItemCounter: map[string]*counterEntry{}, | ||
| expiredDuration: expiredDuration, | ||
| } | ||
| } | ||
|
|
||
| func (c *TimedCounter) GetCount(id string) int { | ||
| c.RLock() | ||
| defer c.RUnlock() | ||
| entry, ok := c.perItemCounter[id] | ||
| if ok { | ||
| return entry.count | ||
| } | ||
| return 0 | ||
| } | ||
|
|
||
| func (c *TimedCounter) IncreaseCount(id string) { | ||
| c.Lock() | ||
| defer c.Unlock() | ||
| entry, ok := c.perItemCounter[id] | ||
| if !ok { | ||
| entry = c.initEntryUnsafe(id) | ||
| } | ||
| entry.count += 1 | ||
| entry.lastUpdate = time.Now() | ||
| } | ||
|
|
||
| func (c *TimedCounter) DeleteEntry(id string) { | ||
| c.Lock() | ||
| defer c.Unlock() | ||
| delete(c.perItemCounter, id) | ||
| } | ||
|
|
||
| func (c *TimedCounter) gc() { | ||
| c.Lock() | ||
| defer c.Unlock() | ||
| now := time.Now() | ||
| for id, entry := range c.perItemCounter { | ||
| if now.Sub(entry.lastUpdate) > c.expiredDuration { | ||
| delete(c.perItemCounter, id) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| func (c *TimedCounter) RunGC(gcDuration time.Duration, stopCh <-chan struct{}) { | ||
| ticker := time.NewTicker(gcDuration) | ||
| defer ticker.Stop() | ||
| for { | ||
| select { | ||
| case <-ticker.C: | ||
| c.gc() | ||
| case <-stopCh: | ||
| return | ||
| } | ||
| } | ||
| } | ||
|
|
||
| func (c *TimedCounter) GetTotalEntries() int { | ||
| c.RLock() | ||
| defer c.RUnlock() | ||
| return len(c.perItemCounter) | ||
| } | ||
|
|
||
| func (c *TimedCounter) initEntryUnsafe(id string) *counterEntry { | ||
| entry := &counterEntry{count: 0} | ||
| c.perItemCounter[id] = entry | ||
| return entry | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,72 @@ | ||
| package util | ||
|
|
||
| import ( | ||
| "testing" | ||
| "time" | ||
| ) | ||
|
|
||
| func TestGetCount(t *testing.T) { | ||
| counter := NewTimedCounter(1 * time.Minute) | ||
|
|
||
| counter.IncreaseCount("item1") | ||
| if count := counter.GetCount("item1"); count != 1 { | ||
| t.Errorf("expected count 1, got %d", count) | ||
| } | ||
|
|
||
| counter.IncreaseCount("item1") | ||
| if count := counter.GetCount("item1"); count != 2 { | ||
| t.Errorf("expected count 2, got %d", count) | ||
| } | ||
|
|
||
| if count := counter.GetCount("nonexistent"); count != 0 { | ||
| t.Errorf("expected count 0 for nonexistent item, got %d", count) | ||
| } | ||
| } | ||
|
|
||
| func TestDeleteEntry(t *testing.T) { | ||
| counter := NewTimedCounter(1 * time.Minute) | ||
|
|
||
| counter.IncreaseCount("item1") | ||
| counter.DeleteEntry("item1") | ||
|
|
||
| if numEntries := counter.GetTotalEntries(); numEntries != 0 { | ||
| t.Errorf("expected 0 entry after deletion, got %d", numEntries) | ||
| } | ||
|
|
||
| if count := counter.GetCount("item1"); count != 0 { | ||
| t.Errorf("expected count 0 after deletion, got %d", count) | ||
| } | ||
| } | ||
|
|
||
| func TestGC(t *testing.T) { | ||
| counter := NewTimedCounter(200 * time.Millisecond) | ||
|
|
||
| counter.IncreaseCount("item1") | ||
| counter.IncreaseCount("item2") | ||
|
|
||
| time.Sleep(400 * time.Millisecond) // Wait for entries to expire | ||
|
|
||
| counter.gc() // Manually trigger garbage collection | ||
|
|
||
| if numEntries := counter.GetTotalEntries(); numEntries != 0 { | ||
| t.Errorf("expected 0 entry after gc, got %d", numEntries) | ||
| } | ||
| } | ||
|
|
||
| func TestRunGC(t *testing.T) { | ||
| counter := NewTimedCounter(200 * time.Millisecond) | ||
|
|
||
| counter.IncreaseCount("item1") | ||
| counter.IncreaseCount("item2") | ||
|
|
||
| stopCh := make(chan struct{}) | ||
| go counter.RunGC(300*time.Millisecond, stopCh) | ||
|
|
||
| time.Sleep(400 * time.Millisecond) // Wait for GC to run | ||
|
|
||
| if numEntries := counter.GetTotalEntries(); numEntries != 0 { | ||
| t.Errorf("expected 0 entry, got %d", numEntries) | ||
| } | ||
|
|
||
| close(stopCh) | ||
| } |