diff --git a/pkg/clients/cosmovisor/cosmovisor.go b/pkg/clients/cosmovisor/cosmovisor.go index 3390447..b59e342 100644 --- a/pkg/clients/cosmovisor/cosmovisor.go +++ b/pkg/clients/cosmovisor/cosmovisor.go @@ -15,6 +15,8 @@ import ( "os" "strings" + upgradeTypes "cosmossdk.io/x/upgrade/types" + "go.opentelemetry.io/otel/trace" "github.com/rs/zerolog" @@ -195,3 +197,58 @@ func (c *Cosmovisor) GetUpgrades(ctx context.Context) (*types.UpgradesPresent, q return &upgrades, cosmovisorGetUpgradesQueryInfo, nil } + +func (c *Cosmovisor) GetUpgradeInfo(ctx context.Context) (*upgradeTypes.Plan, query_info.QueryInfo, error) { + _, span := c.Tracer.Start( + ctx, + "Fetching cosmovisor upgrade info", + ) + defer span.End() + + queryInfo := query_info.QueryInfo{ + Module: constants.ModuleCosmovisor, + Action: constants.ActionCosmovisorGetCosmovisorUpgradeInfo, + Success: false, + } + + env := append( + os.Environ(), + "DAEMON_NAME="+c.Config.ChainBinaryName, + "DAEMON_HOME="+c.Config.ChainFolder, + ) + + out, err := c.CommandExecutor.RunWithEnv( + c.Config.CosmovisorPath, + []string{"show-upgrade-info"}, + env, + ) + if err != nil { + c.Logger.Error(). + Err(err). + Str("output", utils.DecolorifyString(string(out))). + Msg("Could not get Cosmovisor upgrade info") + span.RecordError(err) + return nil, queryInfo, err + } + + if strings.Contains(string(out), "No upgrade info found") { + c.Logger.Trace().Msg("Cosmovisor reports that there is no upgrade info present") + queryInfo.Success = true + return nil, queryInfo, nil + } + + jsonOutput := getJsonString(string(out)) + + var upgradePlan *upgradeTypes.Plan + if unmarshalErr := json.Unmarshal([]byte(jsonOutput), &upgradePlan); unmarshalErr != nil { + c.Logger.Error(). + Err(unmarshalErr). + Str("output", jsonOutput). + Msg("Could not unmarshall upgrade info") + span.RecordError(unmarshalErr) + return upgradePlan, queryInfo, unmarshalErr + } + + queryInfo.Success = true + return upgradePlan, queryInfo, nil +} diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index d3b0330..05428ee 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -19,27 +19,32 @@ const ( ModuleGit Module = "git" ModuleGrpc Module = "grpc" - ActionCosmovisorGetVersion Action = "get_version" - ActionCosmovisorGetCosmovisorVersion Action = "get_cosmovisor_version" - ActionCosmovisorGetUpgrades Action = "get_upgrades" - ActionGitGetLatestRelease Action = "get_latest_release" - ActionTendermintGetNodeStatus Action = "get_node_status" - ActionTendermintGetUpgradePlan Action = "get_upgrade_plan" - ActionTendermintGetBlockTime Action = "get_block_time" - ActionGrpcGetNodeConfig Action = "get_node_config" - ActionGrpcGetNodeInfo Action = "get_node_info" - - FetcherNameNodeStatus FetcherName = "node_status" - FetcherNameCosmovisorVersion FetcherName = "cosmovisor_version" - FetcherNameNodeConfig FetcherName = "node_config" - FetcherNameNodeInfo FetcherName = "node_info" - FetcherNameUptime FetcherName = "uptime" - FetcherNameAppVersion FetcherName = "app_version" - FetcherNameRemoteVersion FetcherName = "remote_version" - FetcherNameLocalVersion FetcherName = "local_version" - FetcherNameUpgrades FetcherName = "upgrades" - FetcherNameBlockTime FetcherName = "block_time" - FetcherNameCosmovisorUpgrades FetcherName = "cosmovisor_upgrades" + ActionCosmovisorGetVersion Action = "get_version" + ActionCosmovisorGetCosmovisorVersion Action = "get_cosmovisor_version" + ActionCosmovisorGetCosmovisorUpgradeInfo Action = "get_cosmovisor_upgrade_info" + ActionCosmovisorGetUpgrades Action = "get_upgrades" + ActionGitGetLatestRelease Action = "get_latest_release" + ActionTendermintGetNodeStatus Action = "get_node_status" + ActionTendermintGetUpgradePlan Action = "get_upgrade_plan" + ActionTendermintGetBlockTime Action = "get_block_time" + ActionGrpcGetNodeConfig Action = "get_node_config" + ActionGrpcGetNodeInfo Action = "get_node_info" + + FetcherNameNodeStatus FetcherName = "node_status" + FetcherNameCosmovisorVersion FetcherName = "cosmovisor_version" + FetcherNameCosmovisorUpgradeInfo FetcherName = "cosmovisor_upgrade_info" + FetcherNameNodeConfig FetcherName = "node_config" + FetcherNameNodeInfo FetcherName = "node_info" + FetcherNameUptime FetcherName = "uptime" + FetcherNameAppVersion FetcherName = "app_version" + FetcherNameRemoteVersion FetcherName = "remote_version" + FetcherNameLocalVersion FetcherName = "local_version" + FetcherNameUpgrades FetcherName = "upgrades" + FetcherNameBlockTime FetcherName = "block_time" + FetcherNameCosmovisorUpgrades FetcherName = "cosmovisor_upgrades" + + UpgradeSourceGovernance string = "governance" + UpgradeSourceUpgradeInfo string = "upgrade-info" ) var ( diff --git a/pkg/fetchers/block_time_fetcher.go b/pkg/fetchers/block_time_fetcher.go index 7fa5f7a..7e2a36f 100644 --- a/pkg/fetchers/block_time_fetcher.go +++ b/pkg/fetchers/block_time_fetcher.go @@ -39,25 +39,19 @@ func (n *BlockTimeFetcher) Name() constants.FetcherName { func (n *BlockTimeFetcher) Dependencies() []constants.FetcherName { return []constants.FetcherName{ constants.FetcherNameUpgrades, + constants.FetcherNameCosmovisorUpgradeInfo, } } func (n *BlockTimeFetcher) Get(ctx context.Context, data ...interface{}) (interface{}, []query_info.QueryInfo) { - if len(data) < 1 { + if len(data) < 2 { panic("data is empty") } - if data[0] == nil { - n.Logger.Trace().Msg("Upgrade plan is empty, not fetching block time.") - return nil, []query_info.QueryInfo{} - } + _, governanceUpgradePlanConverted := Convert[*types.Plan](data[0]) + _, upgradeInfoJSONConverted := Convert[*types.Plan](data[1]) - // upgrade-info was not fetched - upgradePlan, ok := data[0].(*types.Plan) - if !ok { - panic("expected upgrade plan to be *types.Plan") - } - if upgradePlan == nil { + if !governanceUpgradePlanConverted && !upgradeInfoJSONConverted { n.Logger.Trace().Msg("Upgrade plan is empty, not fetching block time.") return nil, []query_info.QueryInfo{} } diff --git a/pkg/fetchers/controller.go b/pkg/fetchers/controller.go index 9142054..bf1c1d6 100644 --- a/pkg/fetchers/controller.go +++ b/pkg/fetchers/controller.go @@ -43,16 +43,22 @@ func StateGet[T any](state State, fetcherName constants.FetcherName) (T, bool) { return zero, false } - if dataRaw == nil { + return Convert[T](dataRaw) +} + +func Convert[T any](input interface{}) (T, bool) { + var zero T + + if input == nil { return zero, false } - data, converted := dataRaw.(T) + data, converted := input.(T) if !converted { panic(fmt.Sprintf( "Error converting data: expected %s, got %s", reflect.TypeOf(zero).String(), - reflect.TypeOf(dataRaw).String(), + reflect.TypeOf(input).String(), )) } diff --git a/pkg/fetchers/cosmovisor_upgrade_info_fetcher.go b/pkg/fetchers/cosmovisor_upgrade_info_fetcher.go new file mode 100644 index 0000000..ce99743 --- /dev/null +++ b/pkg/fetchers/cosmovisor_upgrade_info_fetcher.go @@ -0,0 +1,82 @@ +package fetchers + +import ( + "context" + cosmovisorPkg "main/pkg/clients/cosmovisor" + "main/pkg/clients/tendermint" + "main/pkg/constants" + "main/pkg/query_info" + + "go.opentelemetry.io/otel/trace" + + "github.com/rs/zerolog" +) + +type CosmovisorUpgradeInfoFetcher struct { + Cosmovisor *cosmovisorPkg.Cosmovisor + QueryUpgrades bool + Logger zerolog.Logger + Tracer trace.Tracer +} + +func NewCosmovisorUpgradeInfoFetcher( + logger zerolog.Logger, + cosmovisor *cosmovisorPkg.Cosmovisor, + tracer trace.Tracer, +) *CosmovisorUpgradeInfoFetcher { + return &CosmovisorUpgradeInfoFetcher{ + Logger: logger.With().Str("component", "cosmovisor_upgrade_info").Logger(), + Cosmovisor: cosmovisor, + Tracer: tracer, + } +} + +func (n *CosmovisorUpgradeInfoFetcher) Enabled() bool { + return n.Cosmovisor != nil +} + +func (n *CosmovisorUpgradeInfoFetcher) Name() constants.FetcherName { + return constants.FetcherNameCosmovisorUpgradeInfo +} + +func (n *CosmovisorUpgradeInfoFetcher) Dependencies() []constants.FetcherName { + return []constants.FetcherName{constants.FetcherNameNodeStatus} +} + +func (n *CosmovisorUpgradeInfoFetcher) Get(ctx context.Context, data ...interface{}) (interface{}, []query_info.QueryInfo) { + childCtx, span := n.Tracer.Start( + ctx, + "Fetcher "+string(n.Name()), + ) + defer span.End() + + if len(data) < 1 { + panic("data is empty") + } + + status, statusConverted := Convert[tendermint.StatusResponse](data[0]) + if !statusConverted { + n.Logger.Trace().Msg("Node status is empty, cannot check if the upgrade-info plan is in the past.") + return nil, []query_info.QueryInfo{} + } + + upgradeInfo, queryInfo, err := n.Cosmovisor.GetUpgradeInfo(childCtx) + if err != nil { + n.Logger.Error().Err(err).Msg("Could not fetch upgrade info") + return nil, []query_info.QueryInfo{queryInfo} + } + + if upgradeInfo == nil { + return nil, []query_info.QueryInfo{} + } + + if upgradeInfo.Height < status.Result.SyncInfo.LatestBlockHeight { + n.Logger.Trace(). + Int64("node_height", status.Result.SyncInfo.LatestBlockHeight). + Int64("upgrade_height", upgradeInfo.Height). + Msg("Cosmovisor upgrade-info is in the past, skipping.") + return nil, []query_info.QueryInfo{queryInfo} + } + + return upgradeInfo, []query_info.QueryInfo{queryInfo} +} diff --git a/pkg/generators/time_till_upgrade_generator.go b/pkg/generators/time_till_upgrade_generator.go index e3fab4f..b62a2d5 100644 --- a/pkg/generators/time_till_upgrade_generator.go +++ b/pkg/generators/time_till_upgrade_generator.go @@ -17,23 +17,48 @@ func NewTimeTillUpgradeGenerator() *TimeTillUpgradeGenerator { } func (g *TimeTillUpgradeGenerator) Get(state fetchers.State) []metrics.MetricInfo { - upgrade, upgradeFound := fetchers.StateGet[*types.Plan](state, constants.FetcherNameUpgrades) + governanceUpgrade, governanceUpgradeFound := fetchers.StateGet[*types.Plan](state, constants.FetcherNameUpgrades) + upgradeInfoJson, upgradeInfoJsonFound := fetchers.StateGet[*types.Plan](state, constants.FetcherNameCosmovisorUpgradeInfo) blocksInfo, blocksInfoFound := fetchers.StateGet[*tendermint.BlocksInfo](state, constants.FetcherNameBlockTime) - if !upgradeFound || !blocksInfoFound { + if !blocksInfoFound { return []metrics.MetricInfo{} } + metricsInfo := []metrics.MetricInfo{} + + if governanceUpgradeFound { + metricsInfo = append(metricsInfo, metrics.MetricInfo{ + MetricName: metrics.MetricNameUpgradeEstimatedTime, + Labels: map[string]string{ + "name": governanceUpgrade.Name, + "source": constants.UpgradeSourceGovernance, + }, + Value: float64(g.calculateBlockTime(governanceUpgrade.Height, blocksInfo).Unix()), + }) + } + + if upgradeInfoJsonFound { + metricsInfo = append(metricsInfo, metrics.MetricInfo{ + MetricName: metrics.MetricNameUpgradeEstimatedTime, + Labels: map[string]string{ + "name": upgradeInfoJson.Name, + "source": constants.UpgradeSourceUpgradeInfo, + }, + Value: float64(g.calculateBlockTime(upgradeInfoJson.Height, blocksInfo).Unix()), + }) + } + + return metricsInfo +} + +func (g *TimeTillUpgradeGenerator) calculateBlockTime(height int64, blocksInfo *tendermint.BlocksInfo) time.Time { blockTime := blocksInfo.BlockTime() - blocksTillEstimatedBlock := upgrade.Height - blocksInfo.NewerBlock.Result.Block.Header.Height + blocksTillEstimatedBlock := height - blocksInfo.NewerBlock.Result.Block.Header.Height secondsTillEstimatedBlock := int64(float64(blocksTillEstimatedBlock) * blockTime) durationTillEstimatedBlock := time.Duration(secondsTillEstimatedBlock * int64(time.Second)) upgradeTime := blocksInfo.NewerBlock.Result.Block.Header.Time.Add(durationTillEstimatedBlock) - return []metrics.MetricInfo{{ - MetricName: metrics.MetricNameUpgradeEstimatedTime, - Labels: map[string]string{"name": upgrade.Name}, - Value: float64(upgradeTime.Unix()), - }} + return upgradeTime } diff --git a/pkg/generators/upgrades_generator.go b/pkg/generators/upgrades_generator.go index b0e0d4b..484ab65 100644 --- a/pkg/generators/upgrades_generator.go +++ b/pkg/generators/upgrades_generator.go @@ -15,28 +15,76 @@ func NewUpgradesGenerator() *UpgradesGenerator { } func (g *UpgradesGenerator) Get(state fetchers.State) []metrics.MetricInfo { - metricInfos := []metrics.MetricInfo{{ - MetricName: metrics.MetricNameUpgradeComing, - Labels: map[string]string{}, - Value: 0, - }} - - upgradeInfo, upgradeInfoFound := fetchers.StateGet[*upgradeTypes.Plan](state, constants.FetcherNameUpgrades) - if !upgradeInfoFound { - return metricInfos + metricInfos := []metrics.MetricInfo{} + + governanceUpgrade, governanceUpgradeFound := fetchers.StateGet[*upgradeTypes.Plan](state, constants.FetcherNameUpgrades) + upgradeInfoJSON, upgradeInfoJSONFound := fetchers.StateGet[*upgradeTypes.Plan](state, constants.FetcherNameCosmovisorUpgradeInfo) + + if governanceUpgradeFound { + metricInfos = append(metricInfos, metrics.MetricInfo{ + MetricName: metrics.MetricNameUpgradeComing, + Labels: map[string]string{ + "source": constants.UpgradeSourceGovernance, + }, + Value: 1, + }, metrics.MetricInfo{ + MetricName: metrics.MetricNameUpgradeInfo, + Labels: map[string]string{ + "name": governanceUpgrade.Name, + "info": governanceUpgrade.Info, + "source": constants.UpgradeSourceGovernance, + }, + Value: 1, + }, metrics.MetricInfo{ + MetricName: metrics.MetricNameUpgradeHeight, + Labels: map[string]string{ + "name": governanceUpgrade.Name, + "source": constants.UpgradeSourceGovernance, + }, + Value: float64(governanceUpgrade.Height), + }) + } else { + metricInfos = append(metricInfos, metrics.MetricInfo{ + MetricName: metrics.MetricNameUpgradeComing, + Labels: map[string]string{ + "source": constants.UpgradeSourceGovernance, + }, + Value: 0, + }) } - metricInfos[0].Value = 1 - - metricInfos = append(metricInfos, metrics.MetricInfo{ - MetricName: metrics.MetricNameUpgradeInfo, - Labels: map[string]string{"name": upgradeInfo.Name, "info": upgradeInfo.Info}, - Value: 1, - }, metrics.MetricInfo{ - MetricName: metrics.MetricNameUpgradeHeight, - Labels: map[string]string{"name": upgradeInfo.Name}, - Value: float64(upgradeInfo.Height), - }) + if upgradeInfoJSONFound { + metricInfos = append(metricInfos, metrics.MetricInfo{ + MetricName: metrics.MetricNameUpgradeComing, + Labels: map[string]string{ + "source": constants.UpgradeSourceUpgradeInfo, + }, + Value: 1, + }, metrics.MetricInfo{ + MetricName: metrics.MetricNameUpgradeInfo, + Labels: map[string]string{ + "name": upgradeInfoJSON.Name, + "info": upgradeInfoJSON.Info, + "source": constants.UpgradeSourceUpgradeInfo, + }, + Value: 1, + }, metrics.MetricInfo{ + MetricName: metrics.MetricNameUpgradeHeight, + Labels: map[string]string{ + "name": upgradeInfoJSON.Name, + "source": constants.UpgradeSourceUpgradeInfo, + }, + Value: float64(upgradeInfoJSON.Height), + }) + } else { + metricInfos = append(metricInfos, metrics.MetricInfo{ + MetricName: metrics.MetricNameUpgradeComing, + Labels: map[string]string{ + "source": constants.UpgradeSourceUpgradeInfo, + }, + Value: 0, + }) + } return metricInfos } diff --git a/pkg/metrics/manager.go b/pkg/metrics/manager.go index 8f5cf54..ebf7ecf 100644 --- a/pkg/metrics/manager.go +++ b/pkg/metrics/manager.go @@ -99,7 +99,7 @@ func NewManager() *Manager { Name: constants.MetricsPrefix + "upgrade_coming", Help: "Is future upgrade planned?", }, - []string{"node"}, + []string{"node", "source"}, ), MetricNameUpgradeInfo: prometheus.NewGaugeVec( @@ -107,7 +107,7 @@ func NewManager() *Manager { Name: constants.MetricsPrefix + "upgrade_info", Help: "Future upgrade info", }, - []string{"node", "name", "info"}, + []string{"node", "name", "info", "source"}, ), MetricNameUpgradeHeight: prometheus.NewGaugeVec( @@ -115,7 +115,7 @@ func NewManager() *Manager { Name: constants.MetricsPrefix + "upgrade_height", Help: "Future upgrade height", }, - []string{"node", "name"}, + []string{"node", "name", "source"}, ), MetricNameUpgradeEstimatedTime: prometheus.NewGaugeVec( @@ -123,7 +123,7 @@ func NewManager() *Manager { Name: constants.MetricsPrefix + "upgrade_estimated_time", Help: "Estimated upgrade time, as Unix timestamp", }, - []string{"node", "name"}, + []string{"node", "name", "source"}, ), MetricNameUpgradeBinaryPresent: prometheus.NewGaugeVec( @@ -131,7 +131,7 @@ func NewManager() *Manager { Name: constants.MetricsPrefix + "upgrade_binary_present", Help: "Is upgrade binary present?", }, - []string{"node", "name"}, + []string{"node", "name", "source"}, ), MetricNameQuerierEnabled: prometheus.NewGaugeVec( diff --git a/pkg/node_handler.go b/pkg/node_handler.go index 6bac93a..76a358c 100644 --- a/pkg/node_handler.go +++ b/pkg/node_handler.go @@ -64,6 +64,7 @@ func NewNodeHandler( fetchersPkg.NewUpgradesFetcher(appLogger, tendermintRPC, config.TendermintConfig.QueryUpgrades.Bool, tracer), fetchersPkg.NewBlockTimeFetcher(appLogger, tendermintRPC, tracer), fetchersPkg.NewCosmovisorUpgradesFetcher(appLogger, cosmovisor, tracer), + fetchersPkg.NewCosmovisorUpgradeInfoFetcher(appLogger, cosmovisor, tracer), } generators := []generatorsPkg.Generator{