From da929e02cf20ef9698e33ae08182a5303fb1c5a0 Mon Sep 17 00:00:00 2001
From: george-aj <35270943+george-aj@users.noreply.github.com>
Date: Tue, 7 Feb 2023 12:54:22 -0600
Subject: [PATCH 1/2] Add check on latest_block_time

---
 td2/rpc.go             | 10 +++++++---
 td2/tenderduty_test.go | 27 +++++++++++++++++++++++++++
 2 files changed, 34 insertions(+), 3 deletions(-)
 create mode 100644 td2/tenderduty_test.go

diff --git a/td2/rpc.go b/td2/rpc.go
index 49559b9..ddad68d 100644
--- a/td2/rpc.go
+++ b/td2/rpc.go
@@ -51,7 +51,7 @@ func (cc *ChainConfig) newRpc() error {
 			l(msg)
 			return
 		}
-		if status.SyncInfo.CatchingUp {
+		if status.SyncInfo.CatchingUp || !isLatestBlockTimeCurrent(status.SyncInfo.LatestBlockTime, cc.Alerts.Stalled) {
 			msg = fmt.Sprint("🐢 node is not synced, skipping ", u)
 			syncing = true
 			down = true
@@ -162,7 +162,7 @@ func (cc *ChainConfig) monitorHealth(ctx context.Context, chainName string) {
 						alert("on the wrong network")
 						return
 					}
-					if status.SyncInfo.CatchingUp {
+					if status.SyncInfo.CatchingUp || !isLatestBlockTimeCurrent(status.SyncInfo.LatestBlockTime, cc.Alerts.Stalled) {
 						alert("not synced")
 						node.syncing = true
 						return
@@ -181,7 +181,6 @@ func (cc *ChainConfig) monitorHealth(ctx context.Context, chainName string) {
 					l(fmt.Sprintf("🟢 %-12s node %s is healthy", chainName, node.Url))
 				}(node)
 			}
-
 			if cc.client == nil {
 				e := cc.newRpc()
 				if e != nil {
@@ -238,3 +237,8 @@ func guessPublicEndpoint(u string) string {
 	}
 	return proto + matches[1] + port
 }
+
+// isLatestBlockTimeCurrent checks to see if the `latest_block_time` is within stalledMinutes minutes of UTC time.
+func isLatestBlockTimeCurrent(blockTime time.Time, stalledMinutes int) bool {
+	return time.Now().UTC().Sub(blockTime) < time.Minute*time.Duration(stalledMinutes)
+}
diff --git a/td2/tenderduty_test.go b/td2/tenderduty_test.go
new file mode 100644
index 0000000..e525c14
--- /dev/null
+++ b/td2/tenderduty_test.go
@@ -0,0 +1,27 @@
+package tenderduty
+
+import (
+	"github.com/stretchr/testify/assert"
+	"testing"
+	"time"
+)
+
+func TestIsLatestBlockTimeCurrent(t *testing.T) {
+	currentTime := time.Now().UTC()
+
+	// Check current time
+	blockTime := currentTime
+	assert.True(t, isLatestBlockTimeCurrent(blockTime, 10))
+
+	// Check time from two minutes ago
+	blockTime = currentTime.Add(-time.Minute * 2)
+	assert.True(t, isLatestBlockTimeCurrent(blockTime, 10))
+
+	// Check time from two hours ago
+	blockTime = currentTime.Add(-time.Hour * 2)
+	assert.False(t, isLatestBlockTimeCurrent(blockTime, 10))
+
+	// Check time two minutes in the future
+	blockTime = currentTime.Add(time.Minute * 2)
+	assert.True(t, isLatestBlockTimeCurrent(blockTime, 10))
+}

From 666a28a67c015cb71e1456ffb214da321b30838e Mon Sep 17 00:00:00 2001
From: george-aj <35270943+george-aj@users.noreply.github.com>
Date: Tue, 7 Feb 2023 13:10:14 -0600
Subject: [PATCH 2/2] go.mod updates

---
 go.mod | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/go.mod b/go.mod
index e2419a1..291ac9a 100644
--- a/go.mod
+++ b/go.mod
@@ -10,6 +10,7 @@ require (
 	github.com/go-yaml/yaml v2.1.0+incompatible
 	github.com/gorilla/websocket v1.5.0
 	github.com/prometheus/client_golang v1.12.2
+	github.com/stretchr/testify v1.8.0
 	github.com/tendermint/tendermint v0.34.24
 	github.com/textileio/go-threads v1.1.5
 	golang.org/x/crypto v0.1.0
@@ -90,7 +91,6 @@ require (
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/spf13/viper v1.13.0 // indirect
-	github.com/stretchr/testify v1.8.0 // indirect
 	github.com/subosito/gotenv v1.4.1 // indirect
 	github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect
 	github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c // indirect