Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[DRAFT] respect vtops downtime during recoveries #521

Draft
wants to merge 1 commit into
base: slack-19.0
Choose a base branch
from
Draft

[DRAFT] respect vtops downtime during recoveries #521

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
module vitess.io/vitess

go 1.22.5
go 1.22.7

require (
cloud.google.com/go/storage v1.39.0
Expand Down Expand Up @@ -102,7 +102,7 @@ require (
github.com/kr/text v0.2.0
github.com/mitchellh/mapstructure v1.5.0
github.com/nsf/jsondiff v0.0.0-20210926074059-1e845ec5d249
github.com/slackhq/vitess-addons v0.19.1
github.com/slackhq/vitess-addons v0.19.3-downtime2
github.com/slok/noglog v0.2.0
github.com/spf13/afero v1.11.0
github.com/spf13/jwalterweatherman v1.1.0
Expand Down
4 changes: 2 additions & 2 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -450,8 +450,8 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/sjmudd/stopwatch v0.1.1 h1:x45OvxFB5OtCkjvYtzRF5fWB857Jzjjk84Oyd5C5ebw=
github.com/sjmudd/stopwatch v0.1.1/go.mod h1:BLw0oIQJ1YLXBO/q9ufK/SgnKBVIkC2qrm6uy78Zw6U=
github.com/slackhq/vitess-addons v0.19.1 h1:k8f8pAJ2zqtetN+dnehAs7DFcZnI9IQRSL18ZMwNRCw=
github.com/slackhq/vitess-addons v0.19.1/go.mod h1:ZMzBBtadSA1MEuNIfZerztxLMhRFO+tmBZxv5HuV4lE=
github.com/slackhq/vitess-addons v0.19.3-downtime2 h1:3vUHK8QzcwffAb/QtbP5WHRVQoowGC25Y18Vsu/PSpk=
github.com/slackhq/vitess-addons v0.19.3-downtime2/go.mod h1:CP+xpAwBY1IR/+MQzLWyCsCfKMUhfnbHNSvm9KMRlPg=
github.com/slok/noglog v0.2.0 h1:1czu4l2EoJ8L92UwdSXXa1Y+c5TIjFAFm2P+mjej95E=
github.com/slok/noglog v0.2.0/go.mod h1:TfKxwpEZPT+UA83bQ6RME146k0MM4e8mwHLf6bhcGDI=
github.com/smartystreets/assertions v0.0.0-20190116191733-b6c0e53d7304/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
Expand Down
1 change: 1 addition & 0 deletions go/vt/vtorc/inst/analysis.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,7 @@ type ReplicationAnalysis struct {
MaxReplicaGTIDMode string
MaxReplicaGTIDErrant string
IsReadOnly bool
IsDowntimed bool
}

func (replicationAnalysis *ReplicationAnalysis) MarshalJSON() ([]byte, error) {
Expand Down
8 changes: 8 additions & 0 deletions go/vt/vtorc/inst/analysis_dao.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ package inst

import (
"fmt"
"os"
"time"

"vitess.io/vitess/go/vt/external/golib/sqlutils"
Expand All @@ -36,14 +37,18 @@ import (

"github.com/patrickmn/go-cache"
"github.com/rcrowley/go-metrics"
"github.com/slackhq/vitess-addons/go/external"
)

var analysisChangeWriteCounter = metrics.NewCounter()

var recentInstantAnalysis *cache.Cache

var VTOpsExec *external.ExecVTOps

func init() {
_ = metrics.Register("analysis.change.write", analysisChangeWriteCounter)
VTOpsExec = external.NewExecVTOps(os.Getenv("VTOPS_PATH"), os.Getenv("VTOPS_HTTP_PROXY"), "vtorc", os.Getenv("HOSTNAME"))

go initializeAnalysisDaoPostConfiguration()
}
Expand Down Expand Up @@ -434,6 +439,9 @@ func GetReplicationAnalysis(keyspace string, shard string, hints *ReplicationAna
return nil
}
isInvalid := m.GetBool("is_invalid")

a.IsDowntimed = VTOpsExec.IsDowntimedHost(a.AnalyzedInstanceHostname, true)

if a.IsClusterPrimary && isInvalid {
a.Analysis = InvalidPrimary
a.Description = "VTOrc hasn't been able to reach the primary even once since restart/shutdown"
Expand Down
17 changes: 9 additions & 8 deletions go/vt/vtorc/logic/topology_recovery.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@ import (
"time"

"github.com/patrickmn/go-cache"
"github.com/slackhq/vitess-addons/go/external"

"vitess.io/vitess/go/stats"
"vitess.io/vitess/go/vt/log"
Expand Down Expand Up @@ -83,9 +82,7 @@ var (

// recoveriesFailureCounter counts the number of failed recoveries that VTOrc has performed
recoveriesFailureCounter = stats.NewCountersWithSingleLabel("FailedRecoveries", "Count of the different failed recoveries performed", "RecoveryType", actionableRecoveriesNames...)

vtopsExec = external.NewExecVTOps(os.Getenv("VTOPS_PATH"), os.Getenv("VTOPS_HTTP_PROXY"), "vtorc", os.Getenv("HOSTNAME"))
vtopsSlackChannel = os.Getenv("SLACK_CHANNEL")
vtopsSlackChannel = os.Getenv("SLACK_CHANNEL")
)

// recoveryFunction is the code of the recovery function to be used
Expand Down Expand Up @@ -302,7 +299,7 @@ func postErsCompletion(topologyRecovery *TopologyRecovery, analysisEntry *inst.R
_ = AuditTopologyRecovery(topologyRecovery, message)
_ = inst.AuditOperation(recoveryName, analysisEntry.AnalyzedInstanceAlias, message)
_ = AuditTopologyRecovery(topologyRecovery, fmt.Sprintf("%v: successfully promoted %+v", recoveryName, promotedReplica.InstanceAlias))
vtopsExec.RaiseProblem(analysisEntry.AnalyzedInstanceHostname, "orc-dead-tablet", true)
inst.VTOpsExec.RaiseProblem(analysisEntry.AnalyzedInstanceHostname, "orc-dead-tablet", true)
}
}

Expand Down Expand Up @@ -724,12 +721,12 @@ func executeCheckAndRecoverFunction(analysisEntry *inst.ReplicationAnalysis) (er
if err != nil {
message := fmt.Sprintf("Recovery failed on %s for problem %s. Error: %s", analysisEntry.AnalyzedInstanceHostname, analysisEntry.Analysis, err.Error())
log.Info(message)
vtopsExec.SendSlackMessage(message, vtopsSlackChannel, true)
inst.VTOpsExec.SendSlackMessage(message, vtopsSlackChannel, true)
recoveriesFailureCounter.Add(recoveryName, 1)
} else {
message := fmt.Sprintf("Recovery succeeded on %s for problem %s.", analysisEntry.AnalyzedInstanceHostname, analysisEntry.Analysis)
log.Info(message)
vtopsExec.SendSlackMessage(message, vtopsSlackChannel, true)
inst.VTOpsExec.SendSlackMessage(message, vtopsSlackChannel, true)
recoveriesSuccessfulCounter.Add(recoveryName, 1)
}
if topologyRecovery == nil {
Expand Down Expand Up @@ -813,6 +810,10 @@ func CheckAndRecover() {
for _, j := range rand.Perm(len(replicationAnalysis)) {
analysisEntry := replicationAnalysis[j]

if analysisEntry.IsDowntimed {
continue
}

go func() {
if err := executeCheckAndRecoverFunction(analysisEntry); err != nil {
log.Error(err)
Expand All @@ -827,7 +828,7 @@ func postPrsCompletion(topologyRecovery *TopologyRecovery, analysisEntry *inst.R
_ = AuditTopologyRecovery(topologyRecovery, message)
_ = inst.AuditOperation(string(analysisEntry.Analysis), analysisEntry.AnalyzedInstanceAlias, message)
_ = AuditTopologyRecovery(topologyRecovery, fmt.Sprintf("%+v: successfully promoted %+v", analysisEntry.Analysis, promotedReplica.InstanceAlias))
vtopsExec.RaiseProblem(analysisEntry.AnalyzedInstanceHostname, "orc-dead-tablet", true)
inst.VTOpsExec.RaiseProblem(analysisEntry.AnalyzedInstanceHostname, "orc-dead-tablet", true)
}
}

Expand Down
Loading