Skip to content

Commit

Permalink
refactor(SelfHeal): Improve height drift mitigation logging (#322)
Browse files Browse the repository at this point in the history
  • Loading branch information
DavidNix authored Jul 7, 2023
1 parent 8eb17fe commit 52504cc
Showing 1 changed file with 8 additions and 5 deletions.
13 changes: 8 additions & 5 deletions controllers/selfhealing_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package controllers
import (
"context"
"errors"
"fmt"
"net/http"
"time"

Expand Down Expand Up @@ -118,19 +119,21 @@ func (r *SelfHealingReconciler) mitigateHeightDrift(ctx context.Context, reporte
return
}

const msg = "Height drift mitigation deleted pod"
pods := r.driftDetector.LaggingPods(ctx, crd)
if len(pods) > 0 {
reporter.RecordInfo("HeightDriftMitigation", msg)
}
var deleted int
for _, pod := range pods {
// CosmosFullNodeController will detect missing pod and re-create it.
if err := r.Delete(ctx, pod); kube.IgnoreNotFound(err) != nil {
reporter.Error(err, "Failed to delete pod", "pod", pod)
reporter.RecordError("HeightDriftMitigationDeletePod", err)
continue
}
reporter.Info(msg, "pod", pod)
reporter.Info("Deleted pod for meeting height drift threshold", "pod", pod)
deleted++
}
if deleted > 0 {
msg := fmt.Sprintf("Height lagged behind by %d or more blocks; deleted pod(s)", crd.Spec.SelfHeal.HeightDriftMitigation.Threshold)
reporter.RecordInfo("HeightDriftMitigation", msg)
}
}

Expand Down

0 comments on commit 52504cc

Please sign in to comment.