From ff684aef4a8884cf0e52f6b624c9cc0d1ab77622 Mon Sep 17 00:00:00 2001 From: Akshay Aggarwal Date: Wed, 20 Sep 2023 18:00:14 +0100 Subject: [PATCH 1/5] Improve check block too old detection --- .../ocr2/plugins/ocr2keeper/evm21/registry_check_pipeline.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/services/ocr2/plugins/ocr2keeper/evm21/registry_check_pipeline.go b/core/services/ocr2/plugins/ocr2keeper/evm21/registry_check_pipeline.go index d89ae8589ac..95dd6a0dae8 100644 --- a/core/services/ocr2/plugins/ocr2keeper/evm21/registry_check_pipeline.go +++ b/core/services/ocr2/plugins/ocr2keeper/evm21/registry_check_pipeline.go @@ -245,7 +245,7 @@ func (r *EvmRegistry) checkUpkeeps(ctx context.Context, payloads []ocr2keepers.U latestBlock := r.bs.latestBlock.Load() checkBlock, _, _ := r.getBlockAndUpkeepId(payloads[index].UpkeepID, payloads[index].Trigger) // primitive way of checking errors - if strings.Contains(req.Error.Error(), "header not found") && int64(latestBlock.Number)-checkBlock.Int64() > checkBlockTooOldRange { + if (strings.Contains(req.Error.Error(), "header not found") || strings.Contains(req.Error.Error(), "missing trie node")) && int64(latestBlock.Number)-checkBlock.Int64() > checkBlockTooOldRange { // Check block not found in RPC and it is too old, non-retryable error r.lggr.Warnf("header not found error encountered in check result for upkeepId %s, check block %d, latest block %d: %s", results[index].UpkeepID.String(), checkBlock.Int64(), int64(latestBlock.Number), req.Error) results[index].Retryable = false From 570c4a0afed2e70f86bccb4a375d81c9f805415a Mon Sep 17 00:00:00 2001 From: Akshay Aggarwal Date: Wed, 20 Sep 2023 19:17:03 +0100 Subject: [PATCH 2/5] extract var and add exploratory comment --- .../ocr2/plugins/ocr2keeper/evm21/registry_check_pipeline.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/services/ocr2/plugins/ocr2keeper/evm21/registry_check_pipeline.go b/core/services/ocr2/plugins/ocr2keeper/evm21/registry_check_pipeline.go index 95dd6a0dae8..d2462c8f895 100644 --- a/core/services/ocr2/plugins/ocr2keeper/evm21/registry_check_pipeline.go +++ b/core/services/ocr2/plugins/ocr2keeper/evm21/registry_check_pipeline.go @@ -245,7 +245,9 @@ func (r *EvmRegistry) checkUpkeeps(ctx context.Context, payloads []ocr2keepers.U latestBlock := r.bs.latestBlock.Load() checkBlock, _, _ := r.getBlockAndUpkeepId(payloads[index].UpkeepID, payloads[index].Trigger) // primitive way of checking errors - if (strings.Contains(req.Error.Error(), "header not found") || strings.Contains(req.Error.Error(), "missing trie node")) && int64(latestBlock.Number)-checkBlock.Int64() > checkBlockTooOldRange { + errString := req.Error.Error() + // Exploratory: remove reliance on custom error string checks + if (strings.Contains(errString, "header not found") || strings.Contains(errString, "missing trie node")) && int64(latestBlock.Number)-checkBlock.Int64() > checkBlockTooOldRange { // Check block not found in RPC and it is too old, non-retryable error r.lggr.Warnf("header not found error encountered in check result for upkeepId %s, check block %d, latest block %d: %s", results[index].UpkeepID.String(), checkBlock.Int64(), int64(latestBlock.Number), req.Error) results[index].Retryable = false From 40c041091f0fd01d704478bb2b857c19f9524133 Mon Sep 17 00:00:00 2001 From: Akshay Aggarwal Date: Wed, 20 Sep 2023 19:17:39 +0100 Subject: [PATCH 3/5] improve string --- .../ocr2/plugins/ocr2keeper/evm21/registry_check_pipeline.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/services/ocr2/plugins/ocr2keeper/evm21/registry_check_pipeline.go b/core/services/ocr2/plugins/ocr2keeper/evm21/registry_check_pipeline.go index d2462c8f895..f646cd42c7f 100644 --- a/core/services/ocr2/plugins/ocr2keeper/evm21/registry_check_pipeline.go +++ b/core/services/ocr2/plugins/ocr2keeper/evm21/registry_check_pipeline.go @@ -244,9 +244,8 @@ func (r *EvmRegistry) checkUpkeeps(ctx context.Context, payloads []ocr2keepers.U if req.Error != nil { latestBlock := r.bs.latestBlock.Load() checkBlock, _, _ := r.getBlockAndUpkeepId(payloads[index].UpkeepID, payloads[index].Trigger) - // primitive way of checking errors errString := req.Error.Error() - // Exploratory: remove reliance on custom error string checks + // Exploratory: remove reliance on primitive way of checking errors if (strings.Contains(errString, "header not found") || strings.Contains(errString, "missing trie node")) && int64(latestBlock.Number)-checkBlock.Int64() > checkBlockTooOldRange { // Check block not found in RPC and it is too old, non-retryable error r.lggr.Warnf("header not found error encountered in check result for upkeepId %s, check block %d, latest block %d: %s", results[index].UpkeepID.String(), checkBlock.Int64(), int64(latestBlock.Number), req.Error) From 351630dd969f68b4882728731ec526147c5a9910 Mon Sep 17 00:00:00 2001 From: Akshay Aggarwal Date: Wed, 20 Sep 2023 19:25:56 +0100 Subject: [PATCH 4/5] proper extract var --- .../ocr2/plugins/ocr2keeper/evm21/registry_check_pipeline.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/services/ocr2/plugins/ocr2keeper/evm21/registry_check_pipeline.go b/core/services/ocr2/plugins/ocr2keeper/evm21/registry_check_pipeline.go index f646cd42c7f..e04d730b275 100644 --- a/core/services/ocr2/plugins/ocr2keeper/evm21/registry_check_pipeline.go +++ b/core/services/ocr2/plugins/ocr2keeper/evm21/registry_check_pipeline.go @@ -244,9 +244,9 @@ func (r *EvmRegistry) checkUpkeeps(ctx context.Context, payloads []ocr2keepers.U if req.Error != nil { latestBlock := r.bs.latestBlock.Load() checkBlock, _, _ := r.getBlockAndUpkeepId(payloads[index].UpkeepID, payloads[index].Trigger) - errString := req.Error.Error() // Exploratory: remove reliance on primitive way of checking errors - if (strings.Contains(errString, "header not found") || strings.Contains(errString, "missing trie node")) && int64(latestBlock.Number)-checkBlock.Int64() > checkBlockTooOldRange { + blockNotFound := (strings.Contains(req.Error.Error(), "header not found") || strings.Contains(req.Error.Error(), "missing trie node")) + if blockNotFound && int64(latestBlock.Number)-checkBlock.Int64() > checkBlockTooOldRange { // Check block not found in RPC and it is too old, non-retryable error r.lggr.Warnf("header not found error encountered in check result for upkeepId %s, check block %d, latest block %d: %s", results[index].UpkeepID.String(), checkBlock.Int64(), int64(latestBlock.Number), req.Error) results[index].Retryable = false From 0bb7bd3f9c028b1cfc5439c5579a2dbb632eac54 Mon Sep 17 00:00:00 2001 From: Akshay Aggarwal Date: Wed, 20 Sep 2023 19:29:40 +0100 Subject: [PATCH 5/5] update log --- .../ocr2/plugins/ocr2keeper/evm21/registry_check_pipeline.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/services/ocr2/plugins/ocr2keeper/evm21/registry_check_pipeline.go b/core/services/ocr2/plugins/ocr2keeper/evm21/registry_check_pipeline.go index e04d730b275..db50b322147 100644 --- a/core/services/ocr2/plugins/ocr2keeper/evm21/registry_check_pipeline.go +++ b/core/services/ocr2/plugins/ocr2keeper/evm21/registry_check_pipeline.go @@ -248,7 +248,7 @@ func (r *EvmRegistry) checkUpkeeps(ctx context.Context, payloads []ocr2keepers.U blockNotFound := (strings.Contains(req.Error.Error(), "header not found") || strings.Contains(req.Error.Error(), "missing trie node")) if blockNotFound && int64(latestBlock.Number)-checkBlock.Int64() > checkBlockTooOldRange { // Check block not found in RPC and it is too old, non-retryable error - r.lggr.Warnf("header not found error encountered in check result for upkeepId %s, check block %d, latest block %d: %s", results[index].UpkeepID.String(), checkBlock.Int64(), int64(latestBlock.Number), req.Error) + r.lggr.Warnf("block not found error encountered in check result for upkeepId %s, check block %d, latest block %d: %s", results[index].UpkeepID.String(), checkBlock.Int64(), int64(latestBlock.Number), req.Error) results[index].Retryable = false results[index].PipelineExecutionState = uint8(encoding.CheckBlockTooOld) } else {