From fa6a47462c4c167ba35029a7a183ccac8377775c Mon Sep 17 00:00:00 2001 From: pavel-raykov Date: Thu, 16 Jan 2025 12:30:32 +0100 Subject: [PATCH 1/3] Minor --- chains/chain.go | 16 + chains/fees/fees.go | 119 +++ chains/go.mod | 43 ++ chains/go.sum | 116 +++ chains/hashable.go | 12 + chains/head.go | 45 ++ chains/headtracker/head_broadcaster.go | 158 ++++ chains/headtracker/head_listener.go | 251 +++++++ chains/headtracker/head_saver.go | 23 + chains/headtracker/head_tracker.go | 478 ++++++++++++ chains/headtracker/types/client.go | 20 + chains/headtracker/types/config.go | 19 + chains/headtracker/types/head.go | 15 + chains/subscription.go | 16 + chains/txmgr/broadcaster.go | 777 ++++++++++++++++++++ chains/txmgr/confirmer.go | 895 +++++++++++++++++++++++ chains/txmgr/models.go | 15 + chains/txmgr/reaper.go | 116 +++ chains/txmgr/resender.go | 234 ++++++ chains/txmgr/strategies.go | 64 ++ chains/txmgr/test_helpers.go | 55 ++ chains/txmgr/tracker.go | 335 +++++++++ chains/txmgr/txmgr.go | 810 ++++++++++++++++++++ chains/txmgr/types/client.go | 88 +++ chains/txmgr/types/config.go | 72 ++ chains/txmgr/types/finalizer.go | 17 + chains/txmgr/types/forwarder_manager.go | 17 + chains/txmgr/types/keystore.go | 21 + chains/txmgr/types/sequence_tracker.go | 26 + chains/txmgr/types/stuck_tx_detector.go | 26 + chains/txmgr/types/tx.go | 359 +++++++++ chains/txmgr/types/tx_attempt_builder.go | 47 ++ chains/txmgr/types/tx_store.go | 139 ++++ chains/txmgr/types/tx_test.go | 50 ++ 34 files changed, 5494 insertions(+) create mode 100644 chains/chain.go create mode 100644 chains/fees/fees.go create mode 100644 chains/go.mod create mode 100644 chains/go.sum create mode 100644 chains/hashable.go create mode 100644 chains/head.go create mode 100644 chains/headtracker/head_broadcaster.go create mode 100644 chains/headtracker/head_listener.go create mode 100644 chains/headtracker/head_saver.go create mode 100644 chains/headtracker/head_tracker.go create mode 100644 chains/headtracker/types/client.go create mode 100644 chains/headtracker/types/config.go create mode 100644 chains/headtracker/types/head.go create mode 100644 chains/subscription.go create mode 100644 chains/txmgr/broadcaster.go create mode 100644 chains/txmgr/confirmer.go create mode 100644 chains/txmgr/models.go create mode 100644 chains/txmgr/reaper.go create mode 100644 chains/txmgr/resender.go create mode 100644 chains/txmgr/strategies.go create mode 100644 chains/txmgr/test_helpers.go create mode 100644 chains/txmgr/tracker.go create mode 100644 chains/txmgr/txmgr.go create mode 100644 chains/txmgr/types/client.go create mode 100644 chains/txmgr/types/config.go create mode 100644 chains/txmgr/types/finalizer.go create mode 100644 chains/txmgr/types/forwarder_manager.go create mode 100644 chains/txmgr/types/keystore.go create mode 100644 chains/txmgr/types/sequence_tracker.go create mode 100644 chains/txmgr/types/stuck_tx_detector.go create mode 100644 chains/txmgr/types/tx.go create mode 100644 chains/txmgr/types/tx_attempt_builder.go create mode 100644 chains/txmgr/types/tx_store.go create mode 100644 chains/txmgr/types/tx_test.go diff --git a/chains/chain.go b/chains/chain.go new file mode 100644 index 0000000..3308909 --- /dev/null +++ b/chains/chain.go @@ -0,0 +1,16 @@ +package chains + +import ( + "fmt" +) + +// Sequence represents the base type, for any chain's sequence object. +// It should be convertible to a string +type Sequence interface { + fmt.Stringer + Int64() int64 // needed for numeric sequence confirmation - to be removed with confirmation logic generalization: https://smartcontract-it.atlassian.net/browse/BCI-860 +} + +// ID represents the base type, for any chain's ID. +// It should be convertible to a string, that can uniquely identify this chain +type ID fmt.Stringer diff --git a/chains/fees/fees.go b/chains/fees/fees.go new file mode 100644 index 0000000..84504ee --- /dev/null +++ b/chains/fees/fees.go @@ -0,0 +1,119 @@ +package fees + +import ( + "errors" + "fmt" + "math" + "math/big" + + "github.com/shopspring/decimal" + + "github.com/smartcontractkit/chainlink-common/pkg/chains/label" + "github.com/smartcontractkit/chainlink-common/pkg/logger" + bigmath "github.com/smartcontractkit/chainlink-common/pkg/utils/big_math" +) + +// Opt is an option for a gas estimator +type Opt int + +const ( + // OptForceRefetch forces the estimator to bust a cache if necessary + OptForceRefetch Opt = iota +) + +type Fee fmt.Stringer + +func ApplyMultiplier(feeLimit uint64, multiplier float32) (uint64, error) { + result := decimal.NewFromBigInt(big.NewInt(0).SetUint64(feeLimit), 0).Mul(decimal.NewFromFloat32(multiplier)) + + if result.GreaterThan(decimal.NewFromBigInt(big.NewInt(0).SetUint64(math.MaxUint64), 0)) { + return 0, fmt.Errorf("integer overflow when applying multiplier of %f to fee limit of %d", multiplier, feeLimit) + } + return result.BigInt().Uint64(), nil +} + +// AddPercentage returns the input value increased by the given percentage. +func AddPercentage(value *big.Int, percentage uint16) *big.Int { + bumped := new(big.Int) + bumped.Mul(value, big.NewInt(int64(100+percentage))) + bumped.Div(bumped, big.NewInt(100)) + return bumped +} + +// Returns the fee in its chain specific unit. +type feeUnitToChainUnit func(fee *big.Int) string + +var ( + ErrBumpFeeExceedsLimit = errors.New("fee bump exceeds limit") + ErrBump = errors.New("fee bump failed") + ErrConnectivity = errors.New("transaction propagation issue: transactions are not being mined") + ErrFeeLimitTooLow = errors.New("provided fee limit too low") +) + +func IsBumpErr(err error) bool { + return err != nil && (errors.Is(err, ErrBumpFeeExceedsLimit) || errors.Is(err, ErrBump) || errors.Is(err, ErrConnectivity)) +} + +// CalculateFee computes the fee price for a transaction. +// The fee price is the minimum of: +// - max fee price specified, default fee price and max fee price for the node. +func CalculateFee( + maxFeePrice, defaultPrice, maxFeePriceConfigured *big.Int, +) *big.Int { + maxFeePriceAllowed := bigmath.Min(maxFeePrice, maxFeePriceConfigured) + return bigmath.Min(defaultPrice, maxFeePriceAllowed) +} + +// CalculateBumpedFee computes the next fee price to attempt as the largest of: +// - A configured percentage bump (bumpPercent) on top of the baseline price. +// - A configured fixed amount of Unit (bumpMin) on top of the baseline price. +// The baseline price is the maximum of the previous fee price attempt and the node's current fee price. +func CalculateBumpedFee( + lggr logger.SugaredLogger, + currentfeePrice, originalfeePrice, maxFeePriceInput, maxBumpPrice, bumpMin *big.Int, + bumpPercent uint16, + toChainUnit feeUnitToChainUnit, +) (*big.Int, error) { + maxFeePrice := bigmath.Min(maxFeePriceInput, maxBumpPrice) + bumpedFeePrice := MaxBumpedFee(originalfeePrice, bumpPercent, bumpMin) + + // Update bumpedFeePrice if currentfeePrice is higher than bumpedFeePrice and within maxFeePrice + bumpedFeePrice = maxFee(lggr, currentfeePrice, bumpedFeePrice, maxFeePrice, "fee price", toChainUnit) + + if bumpedFeePrice.Cmp(maxFeePrice) > 0 { + return maxFeePrice, fmt.Errorf("bumped fee price of %s would exceed configured max fee price of %s (original price was %s). %s: %w", + toChainUnit(bumpedFeePrice), toChainUnit(maxFeePrice), toChainUnit(originalfeePrice), label.NodeConnectivityProblemWarning, ErrBumpFeeExceedsLimit) + } else if bumpedFeePrice.Cmp(originalfeePrice) == 0 { + // NOTE: This really shouldn't happen since we enforce minimums for + // FeeEstimator.BumpPercent and FeeEstimator.BumpMin in the config validation, + // but it's here anyway for a "belts and braces" approach + return bumpedFeePrice, fmt.Errorf("bumped fee price of %s is equal to original fee price of %s."+ + " ACTION REQUIRED: This is a configuration error, you must increase either "+ + "FeeEstimator.BumpPercent or FeeEstimator.BumpMin: %w", toChainUnit(bumpedFeePrice), toChainUnit(bumpedFeePrice), ErrBump) + } + return bumpedFeePrice, nil +} + +// MaxBumpedFee returns highest bumped fee price of originalFeePrice bumped by fixed units or percentage. +func MaxBumpedFee(originalFeePrice *big.Int, feeBumpPercent uint16, feeBumpUnits *big.Int) *big.Int { + return bigmath.Max( + AddPercentage(originalFeePrice, feeBumpPercent), + new(big.Int).Add(originalFeePrice, feeBumpUnits), + ) +} + +// Returns the max of currentFeePrice, bumpedFeePrice, and maxFeePrice +func maxFee(lggr logger.SugaredLogger, currentFeePrice, bumpedFeePrice, maxFeePrice *big.Int, feeType string, toChainUnit feeUnitToChainUnit) *big.Int { + if currentFeePrice == nil { + return bumpedFeePrice + } + if currentFeePrice.Cmp(maxFeePrice) > 0 { + // Shouldn't happen because the estimator should not be allowed to + // estimate a higher fee than the maximum allowed + lggr.AssumptionViolationf("Ignoring current %s of %s that would exceed max %s of %s", feeType, toChainUnit(currentFeePrice), feeType, toChainUnit(maxFeePrice)) + } else if bumpedFeePrice.Cmp(currentFeePrice) < 0 { + // If the current fee price is higher than the old price bumped, use that instead + return currentFeePrice + } + return bumpedFeePrice +} diff --git a/chains/go.mod b/chains/go.mod new file mode 100644 index 0000000..7e1e2f7 --- /dev/null +++ b/chains/go.mod @@ -0,0 +1,43 @@ +module github.com/smartcontractkit/chainlink-framework/chains + +go 1.23.3 + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/google/go-cmp v0.6.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/jmoiron/sqlx v1.4.0 // indirect + github.com/jpillora/backoff v1.0.0 // indirect + github.com/lib/pq v1.10.9 // indirect + github.com/mr-tron/base58 v1.2.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pelletier/go-toml/v2 v2.2.2 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/prometheus/client_golang v1.20.5 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.60.1 // indirect + github.com/prometheus/procfs v0.15.1 // indirect + github.com/shopspring/decimal v1.4.0 // indirect + github.com/smartcontractkit/chainlink-common v0.4.0 // indirect + github.com/smartcontractkit/chainlink-framework/multinode v0.0.0-20250115203616-a2ea5e50b260 // indirect + github.com/smartcontractkit/libocr v0.0.0-20241223215956-e5b78d8e3919 // indirect + github.com/stretchr/testify v1.10.0 // indirect + go.opentelemetry.io/otel v1.31.0 // indirect + go.opentelemetry.io/otel/metric v1.31.0 // indirect + go.opentelemetry.io/otel/trace v1.31.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.0 // indirect + golang.org/x/crypto v0.28.0 // indirect + golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c // indirect + golang.org/x/sys v0.26.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38 // indirect + google.golang.org/grpc v1.67.1 // indirect + google.golang.org/protobuf v1.35.1 // indirect + gopkg.in/guregu/null.v4 v4.0.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/chains/go.sum b/chains/go.sum new file mode 100644 index 0000000..eb0875f --- /dev/null +++ b/chains/go.sum @@ -0,0 +1,116 @@ +filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= +github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY= +github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= +github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/pelletier/go-toml/v2 v2.2.0 h1:QLgLl2yMN7N+ruc31VynXs1vhMZa7CeHHejIeBAsoHo= +github.com/pelletier/go-toml/v2 v2.2.0/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= +github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= +github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= +github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= +github.com/prometheus/common v0.59.1 h1:LXb1quJHWm1P6wq/U824uxYi4Sg0oGvNeUm1z5dJoX0= +github.com/prometheus/common v0.59.1/go.mod h1:GpWM7dewqmVYcd7SmRaiWVe9SSqjf0UrwnYnpEZNuT0= +github.com/prometheus/common v0.60.1 h1:FUas6GcOw66yB/73KC+BOZoFJmbo/1pojoILArPAaSc= +github.com/prometheus/common v0.60.1/go.mod h1:h0LYf1R1deLSKtD4Vdg8gy4RuOvENW2J/h19V5NADQw= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= +github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= +github.com/smartcontractkit/chainlink-common v0.4.0 h1:GZ9MhHt5QHXSaK/sAZvKDxkEqF4fPiFHWHEPqs/2C2o= +github.com/smartcontractkit/chainlink-common v0.4.0/go.mod h1:yti7e1+G9hhkYhj+L5sVUULn9Bn3bBL5/AxaNqdJ5YQ= +github.com/smartcontractkit/chainlink-framework/multinode v0.0.0-20250115203616-a2ea5e50b260 h1:See2isL6KdrTJDlVKWv8qiyYqWhYUcubU2e5yKXV1oY= +github.com/smartcontractkit/chainlink-framework/multinode v0.0.0-20250115203616-a2ea5e50b260/go.mod h1:4JqpgFy01LaqG1yM2iFTzwX3ZgcAvW9WdstBZQgPHzU= +github.com/smartcontractkit/chainlink/v2 v2.19.0 h1:iw91nBoxfkrF9lh2m1jgc04pkj5pZs6j6PK6cwrnTNU= +github.com/smartcontractkit/chainlink/v2 v2.19.0/go.mod h1:5uc+GGa6R1Ugccq0ol2Wnjt5LStJAXqz9TZxHX55t2Y= +github.com/smartcontractkit/libocr v0.0.0-20241007185508-adbe57025f12 h1:NzZGjaqez21I3DU7objl3xExTH4fxYvzTqar8DC6360= +github.com/smartcontractkit/libocr v0.0.0-20241007185508-adbe57025f12/go.mod h1:fb1ZDVXACvu4frX3APHZaEBp0xi1DIm34DcA0CwTsZM= +github.com/smartcontractkit/libocr v0.0.0-20241223215956-e5b78d8e3919 h1:IpGoPTXpvllN38kT2z2j13sifJMz4nbHglidvop7mfg= +github.com/smartcontractkit/libocr v0.0.0-20241223215956-e5b78d8e3919/go.mod h1:fb1ZDVXACvu4frX3APHZaEBp0xi1DIm34DcA0CwTsZM= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +go.opentelemetry.io/otel v1.30.0 h1:F2t8sK4qf1fAmY9ua4ohFS/K+FUuOPemHUIXHtktrts= +go.opentelemetry.io/otel v1.30.0/go.mod h1:tFw4Br9b7fOS+uEao81PJjVMjW/5fvNCbpsDIXqP0pc= +go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY= +go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE= +go.opentelemetry.io/otel/metric v1.30.0 h1:4xNulvn9gjzo4hjg+wzIKG7iNFEaBMX00Qd4QIZs7+w= +go.opentelemetry.io/otel/metric v1.30.0/go.mod h1:aXTfST94tswhWEb+5QjlSqG+cZlmyXy/u8jFpor3WqQ= +go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE= +go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY= +go.opentelemetry.io/otel/trace v1.30.0 h1:7UBkkYzeg3C7kQX8VAidWh2biiQbtAKjyIML8dQ9wmc= +go.opentelemetry.io/otel/trace v1.30.0/go.mod h1:5EyKqTzzmyqB9bwtCCq6pDLktPK6fmGf/Dph+8VI02o= +go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys= +go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= +golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= +golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk= +golang.org/x/exp v0.0.0-20240909161429-701f63a606c0/go.mod h1:2TbTHSBQa924w8M6Xs1QcRcFwyucIwBGpK1p2f1YFFY= +golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c h1:7dEasQXItcW1xKJ2+gg5VOiBnqWrJc+rq0DPKyvvdbY= +golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c/go.mod h1:NQtJDoLvd6faHhE7m4T/1IY708gDefGGjR/iUW8yQQ8= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38 h1:zciRKQ4kBpFgpfC5QQCVtnnNAcLIqweL7plyZRQHVpI= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI= +google.golang.org/grpc v1.67.1 h1:zWnc1Vrcno+lHZCOofnIMvycFcc0QRGIzm9dhnDX68E= +google.golang.org/grpc v1.67.1/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= +google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/guregu/null.v4 v4.0.0 h1:1Wm3S1WEA2I26Kq+6vcW+w0gcDo44YKYD7YIEJNHDjg= +gopkg.in/guregu/null.v4 v4.0.0/go.mod h1:YoQhUrADuG3i9WqesrCmpNRwm1ypAgSHYqoOcTu/JrI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/chains/hashable.go b/chains/hashable.go new file mode 100644 index 0000000..55f50ba --- /dev/null +++ b/chains/hashable.go @@ -0,0 +1,12 @@ +package chains + +import "fmt" + +// A chain-agnostic generic interface to represent the following native types on various chains: +// PublicKey, Address, Account, BlockHash, TxHash +type Hashable interface { + fmt.Stringer + comparable + + Bytes() []byte +} diff --git a/chains/head.go b/chains/head.go new file mode 100644 index 0000000..7439246 --- /dev/null +++ b/chains/head.go @@ -0,0 +1,45 @@ +package chains + +import ( + "math/big" + "time" +) + +// Head provides access to a chain's head, as needed by the TxManager. +// This is a generic interface which ALL chains will implement. +type Head[BLOCK_HASH Hashable] interface { + // BlockNumber is the head's block number + BlockNumber() int64 + + // Timestamp the time of mining of the block + GetTimestamp() time.Time + + // ChainLength returns the length of the chain followed by recursively looking up parents + ChainLength() uint32 + + // EarliestHeadInChain traverses through parents until it finds the earliest one + EarliestHeadInChain() Head[BLOCK_HASH] + + // Parent is the head's parent block + GetParent() Head[BLOCK_HASH] + + // Hash is the head's block hash + BlockHash() BLOCK_HASH + GetParentHash() BLOCK_HASH + + // HashAtHeight returns the hash of the block at the given height, if it is in the chain. + // If not in chain, returns the zero hash + HashAtHeight(blockNum int64) BLOCK_HASH + + // HeadAtHeight returns head at specified height or an error, if one does not exist in provided chain. + HeadAtHeight(blockNum int64) (Head[BLOCK_HASH], error) + + // Returns the total difficulty of the block. For chains who do not have a concept of block + // difficulty, return 0. + BlockDifficulty() *big.Int + // IsValid returns true if the head is valid. + IsValid() bool + + // Returns the latest finalized based on finality tag or depth + LatestFinalizedHead() Head[BLOCK_HASH] +} diff --git a/chains/headtracker/head_broadcaster.go b/chains/headtracker/head_broadcaster.go new file mode 100644 index 0000000..40d624a --- /dev/null +++ b/chains/headtracker/head_broadcaster.go @@ -0,0 +1,158 @@ +package headtracker + +import ( + "context" + "fmt" + "reflect" + "sync" + "time" + + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/services" + "github.com/smartcontractkit/chainlink-common/pkg/utils/mailbox" + "github.com/smartcontractkit/chainlink-framework/chains" +) + +const TrackableCallbackTimeout = 2 * time.Second + +type callbackSet[H chains.Head[BLOCK_HASH], BLOCK_HASH chains.Hashable] map[int]HeadTrackable[H, BLOCK_HASH] + +func (set callbackSet[H, BLOCK_HASH]) values() []HeadTrackable[H, BLOCK_HASH] { + var values []HeadTrackable[H, BLOCK_HASH] + for _, callback := range set { + values = append(values, callback) + } + return values +} + +// HeadTrackable is implemented by the core txm to be able to receive head events from any chain. +// Chain implementations should notify head events to the core txm via this interface. +type HeadTrackable[H chains.Head[BLOCK_HASH], BLOCK_HASH chains.Hashable] interface { + // OnNewLongestChain sends a new head when it becomes available. Subscribers can recursively trace the parent + // of the head to the finalized block back. + OnNewLongestChain(ctx context.Context, head H) +} + +// HeadBroadcaster relays new Heads to all subscribers. +type HeadBroadcaster[H chains.Head[BLOCK_HASH], BLOCK_HASH chains.Hashable] interface { + services.Service + BroadcastNewLongestChain(H) + Subscribe(callback HeadTrackable[H, BLOCK_HASH]) (currentLongestChain H, unsubscribe func()) +} + +type headBroadcaster[H chains.Head[BLOCK_HASH], BLOCK_HASH chains.Hashable] struct { + services.Service + eng *services.Engine + + callbacks callbackSet[H, BLOCK_HASH] + mailbox *mailbox.Mailbox[H] + mutex sync.Mutex + latest H + lastCallbackID int +} + +// NewHeadBroadcaster creates a new HeadBroadcaster +func NewHeadBroadcaster[ + H chains.Head[BLOCK_HASH], + BLOCK_HASH chains.Hashable, +]( + lggr logger.Logger, +) HeadBroadcaster[H, BLOCK_HASH] { + hb := &headBroadcaster[H, BLOCK_HASH]{ + callbacks: make(callbackSet[H, BLOCK_HASH]), + mailbox: mailbox.NewSingle[H](), + } + hb.Service, hb.eng = services.Config{ + Name: "HeadBroadcaster", + Start: hb.start, + Close: hb.close, + }.NewServiceEngine(lggr) + return hb +} + +func (hb *headBroadcaster[H, BLOCK_HASH]) start(context.Context) error { + hb.eng.Go(hb.run) + return nil +} + +func (hb *headBroadcaster[H, BLOCK_HASH]) close() error { + hb.mutex.Lock() + // clear all callbacks + hb.callbacks = make(callbackSet[H, BLOCK_HASH]) + hb.mutex.Unlock() + return nil +} + +func (hb *headBroadcaster[H, BLOCK_HASH]) BroadcastNewLongestChain(head H) { + hb.mailbox.Deliver(head) +} + +// Subscribe subscribes to OnNewLongestChain and Connect until HeadBroadcaster is closed, +// or unsubscribe callback is called explicitly +func (hb *headBroadcaster[H, BLOCK_HASH]) Subscribe(callback HeadTrackable[H, BLOCK_HASH]) (currentLongestChain H, unsubscribe func()) { + hb.mutex.Lock() + defer hb.mutex.Unlock() + + currentLongestChain = hb.latest + + hb.lastCallbackID++ + callbackID := hb.lastCallbackID + hb.callbacks[callbackID] = callback + unsubscribe = func() { + hb.mutex.Lock() + defer hb.mutex.Unlock() + delete(hb.callbacks, callbackID) + } + + return +} + +func (hb *headBroadcaster[H, BLOCK_HASH]) run(ctx context.Context) { + for { + select { + case <-ctx.Done(): + return + case <-hb.mailbox.Notify(): + hb.executeCallbacks(ctx) + } + } +} + +// DEV: the head relayer makes no promises about head delivery! Subscribing +// Jobs should expect to the relayer to skip heads if there is a large number of listeners +// and all callbacks cannot be completed in the allotted time. +func (hb *headBroadcaster[H, BLOCK_HASH]) executeCallbacks(ctx context.Context) { + head, exists := hb.mailbox.Retrieve() + if !exists { + hb.eng.Info("No head to retrieve. It might have been skipped") + return + } + + hb.mutex.Lock() + callbacks := hb.callbacks.values() + hb.latest = head + hb.mutex.Unlock() + + hb.eng.Debugw("Initiating callbacks", + "headNum", head.BlockNumber(), + "numCallbacks", len(callbacks), + ) + + wg := sync.WaitGroup{} + wg.Add(len(callbacks)) + + for _, callback := range callbacks { + go func(trackable HeadTrackable[H, BLOCK_HASH]) { + defer wg.Done() + start := time.Now() + cctx, cancel := context.WithTimeout(ctx, TrackableCallbackTimeout) + defer cancel() + trackable.OnNewLongestChain(cctx, head) + elapsed := time.Since(start) + hb.eng.Debugw(fmt.Sprintf("Finished callback in %s", elapsed), + "callbackType", reflect.TypeOf(trackable), "blockNumber", head.BlockNumber(), "time", elapsed) + }(callback) + } + + wg.Wait() +} diff --git a/chains/headtracker/head_listener.go b/chains/headtracker/head_listener.go new file mode 100644 index 0000000..a50816b --- /dev/null +++ b/chains/headtracker/head_listener.go @@ -0,0 +1,251 @@ +package headtracker + +import ( + "context" + "errors" + "fmt" + "sync/atomic" + "time" + + "github.com/jpillora/backoff" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/services" + "github.com/smartcontractkit/chainlink-framework/chains" + + htrktypes "github.com/smartcontractkit/chainlink-framework/chains/headtracker/types" +) + +var ( + promNumHeadsReceived = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "head_tracker_heads_received", + Help: "The total number of heads seen", + }, []string{"ChainID"}) + promEthConnectionErrors = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "head_tracker_connection_errors", + Help: "The total number of node connection errors", + }, []string{"ChainID"}) +) + +// HeadHandler is a callback that handles incoming heads +type HeadHandler[H chains.Head[BLOCK_HASH], BLOCK_HASH chains.Hashable] func(ctx context.Context, header H) error + +// HeadListener is a chain agnostic interface that manages connection of Client that receives heads from the blockchain node +type HeadListener[H chains.Head[BLOCK_HASH], BLOCK_HASH chains.Hashable] interface { + services.Service + + // ListenForNewHeads runs the listen loop (not thread safe) + ListenForNewHeads(ctx context.Context) + + // ReceivingHeads returns true if the listener is receiving heads (thread safe) + ReceivingHeads() bool + + // Connected returns true if the listener is connected (thread safe) + Connected() bool + + // HealthReport returns report of errors within HeadListener + HealthReport() map[string]error +} + +type headListener[ + HTH htrktypes.Head[BLOCK_HASH, ID], + S chains.Subscription, + ID chains.ID, + BLOCK_HASH chains.Hashable, +] struct { + services.Service + eng *services.Engine + + config htrktypes.Config + client htrktypes.Client[HTH, S, ID, BLOCK_HASH] + onSubscription func(context.Context) + handleNewHead HeadHandler[HTH, BLOCK_HASH] + chHeaders <-chan HTH + headSubscription chains.Subscription + connected atomic.Bool + receivingHeads atomic.Bool +} + +func NewHeadListener[ + HTH htrktypes.Head[BLOCK_HASH, ID], + S chains.Subscription, + ID chains.ID, + BLOCK_HASH chains.Hashable, + CLIENT htrktypes.Client[HTH, S, ID, BLOCK_HASH], +]( + lggr logger.Logger, + client CLIENT, + config htrktypes.Config, + onSubscription func(context.Context), + handleNewHead HeadHandler[HTH, BLOCK_HASH], +) HeadListener[HTH, BLOCK_HASH] { + hl := &headListener[HTH, S, ID, BLOCK_HASH]{ + config: config, + client: client, + onSubscription: onSubscription, + handleNewHead: handleNewHead, + } + hl.Service, hl.eng = services.Config{ + Name: "HeadListener", + Start: hl.start, + }.NewServiceEngine(lggr) + return hl +} + +func (hl *headListener[HTH, S, ID, BLOCK_HASH]) start(context.Context) error { + hl.eng.Go(hl.ListenForNewHeads) + return nil +} + +func (hl *headListener[HTH, S, ID, BLOCK_HASH]) ListenForNewHeads(ctx context.Context) { + defer hl.unsubscribe() + + for { + if !hl.subscribe(ctx) { + break + } + + if hl.onSubscription != nil { + hl.onSubscription(ctx) + } + err := hl.receiveHeaders(ctx, hl.handleNewHead) + if ctx.Err() != nil { + break + } else if err != nil { + hl.eng.Errorw("Error in new head subscription, unsubscribed", "err", err) + continue + } + break + } +} + +func (hl *headListener[HTH, S, ID, BLOCK_HASH]) ReceivingHeads() bool { + return hl.receivingHeads.Load() +} + +func (hl *headListener[HTH, S, ID, BLOCK_HASH]) Connected() bool { + return hl.connected.Load() +} + +func (hl *headListener[HTH, S, ID, BLOCK_HASH]) HealthReport() map[string]error { + var err error + if !hl.ReceivingHeads() { + err = errors.New("Listener is not receiving heads") + } + if !hl.Connected() { + err = errors.New("Listener is not connected") + } + return map[string]error{hl.Name(): err} +} + +func (hl *headListener[HTH, S, ID, BLOCK_HASH]) receiveHeaders(ctx context.Context, handleNewHead HeadHandler[HTH, BLOCK_HASH]) error { + var noHeadsAlarmC <-chan time.Time + var noHeadsAlarmT *time.Ticker + noHeadsAlarmDuration := hl.config.BlockEmissionIdleWarningThreshold() + if noHeadsAlarmDuration > 0 { + noHeadsAlarmT = time.NewTicker(noHeadsAlarmDuration) + noHeadsAlarmC = noHeadsAlarmT.C + } + + for { + select { + case <-ctx.Done(): + return nil + + case blockHeader, open := <-hl.chHeaders: + chainId := hl.client.ConfiguredChainID() + if noHeadsAlarmT != nil { + // We've received a head, reset the no heads alarm + noHeadsAlarmT.Stop() + noHeadsAlarmT = time.NewTicker(noHeadsAlarmDuration) + noHeadsAlarmC = noHeadsAlarmT.C + } + hl.receivingHeads.Store(true) + if !open { + return errors.New("head listener: chHeaders prematurely closed") + } + if !blockHeader.IsValid() { + hl.eng.Error("got nil block header") + continue + } + + // Compare the chain ID of the block header to the chain ID of the client + if !blockHeader.HasChainID() || blockHeader.ChainID().String() != chainId.String() { + hl.eng.Panicf("head listener for %s received block header for %s", chainId, blockHeader.ChainID()) + } + promNumHeadsReceived.WithLabelValues(chainId.String()).Inc() + + err := handleNewHead(ctx, blockHeader) + if ctx.Err() != nil { + return nil + } else if err != nil { + return err + } + + case err, open := <-hl.headSubscription.Err(): + // err can be nil, because of using chainIDSubForwarder + if !open || err == nil { + return errors.New("head listener: subscription Err channel prematurely closed") + } + return err + + case <-noHeadsAlarmC: + // We haven't received a head on the channel for a long time, log a warning + hl.eng.Warnf("have not received a head for %v", noHeadsAlarmDuration) + hl.receivingHeads.Store(false) + } + } +} + +func (hl *headListener[HTH, S, ID, BLOCK_HASH]) subscribe(ctx context.Context) bool { + subscribeRetryBackoff := backoff.Backoff{ + Min: 1 * time.Second, + Max: 15 * time.Second, + Jitter: true, + } + + chainId := hl.client.ConfiguredChainID() + + for { + hl.unsubscribe() + + hl.eng.Debugf("Subscribing to new heads on chain %s", chainId.String()) + + select { + case <-ctx.Done(): + return false + + case <-time.After(subscribeRetryBackoff.Duration()): + err := hl.subscribeToHead(ctx) + if err != nil { + promEthConnectionErrors.WithLabelValues(chainId.String()).Inc() + hl.eng.Warnw("Failed to subscribe to heads on chain", "chainID", chainId.String(), "err", err) + } else { + hl.eng.Debugf("Subscribed to heads on chain %s", chainId.String()) + return true + } + } + } +} + +func (hl *headListener[HTH, S, ID, BLOCK_HASH]) subscribeToHead(ctx context.Context) error { + var err error + hl.chHeaders, hl.headSubscription, err = hl.client.SubscribeToHeads(ctx) + if err != nil { + return fmt.Errorf("Client#SubscribeToHeads: %w", err) + } + + hl.connected.Store(true) + + return nil +} + +func (hl *headListener[HTH, S, ID, BLOCK_HASH]) unsubscribe() { + if hl.headSubscription != nil { + hl.connected.Store(false) + hl.headSubscription.Unsubscribe() + hl.headSubscription = nil + } +} diff --git a/chains/headtracker/head_saver.go b/chains/headtracker/head_saver.go new file mode 100644 index 0000000..27dbe25 --- /dev/null +++ b/chains/headtracker/head_saver.go @@ -0,0 +1,23 @@ +package headtracker + +import ( + "context" + + "github.com/smartcontractkit/chainlink-framework/chains" +) + +// HeadSaver is an chain agnostic interface for saving and loading heads +// Different chains will instantiate generic HeadSaver type with their native Head and BlockHash types. +type HeadSaver[H chains.Head[BLOCK_HASH], BLOCK_HASH chains.Hashable] interface { + // Save updates the latest block number, if indeed the latest, and persists + // this number in case of reboot. + Save(ctx context.Context, head H) error + // Load loads latest heads up to latestFinalized - historyDepth, returns the latest chain. + Load(ctx context.Context, latestFinalized int64) (H, error) + // LatestChain returns the block header with the highest number that has been seen, or nil. + LatestChain() H + // Chain returns a head for the specified hash, or nil. + Chain(hash BLOCK_HASH) H + // MarkFinalized - marks matching block and all it's direct ancestors as finalized + MarkFinalized(ctx context.Context, latestFinalized H) error +} diff --git a/chains/headtracker/head_tracker.go b/chains/headtracker/head_tracker.go new file mode 100644 index 0000000..2ab8e34 --- /dev/null +++ b/chains/headtracker/head_tracker.go @@ -0,0 +1,478 @@ +package headtracker + +import ( + "context" + "errors" + "fmt" + "math/big" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/smartcontractkit/chainlink-framework/chains" + + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/services" + "github.com/smartcontractkit/chainlink-common/pkg/utils/mailbox" + + htrktypes "github.com/smartcontractkit/chainlink-framework/chains/headtracker/types" +) + +var ( + promCurrentHead = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "head_tracker_current_head", + Help: "The highest seen head number", + }, []string{"evmChainID"}) + + promOldHead = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "head_tracker_very_old_head", + Help: "Counter is incremented every time we get a head that is much lower than the highest seen head ('much lower' is defined as a block that is EVM.FinalityDepth or greater below the highest seen head)", + }, []string{"evmChainID"}) +) + +// HeadsBufferSize - The buffer is used when heads sampling is disabled, to ensure the callback is run for every head +const HeadsBufferSize = 10 + +// HeadTracker holds and stores the block experienced by a particular node in a thread safe manner. +type HeadTracker[H chains.Head[BLOCK_HASH], BLOCK_HASH chains.Hashable] interface { + services.Service + // Backfill given a head will fill in any missing heads up to latestFinalized + Backfill(ctx context.Context, headWithChain H) (err error) + LatestChain() H + // LatestAndFinalizedBlock - returns latest and latest finalized blocks. + // NOTE: Returns latest finalized block as is, ignoring the FinalityTagBypass feature flag. + LatestAndFinalizedBlock(ctx context.Context) (latest, finalized H, err error) +} + +type headTracker[ + HTH htrktypes.Head[BLOCK_HASH, ID], + S chains.Subscription, + ID chains.ID, + BLOCK_HASH chains.Hashable, +] struct { + services.Service + eng *services.Engine + + log logger.SugaredLogger + headBroadcaster HeadBroadcaster[HTH, BLOCK_HASH] + headSaver HeadSaver[HTH, BLOCK_HASH] + mailMon *mailbox.Monitor + client htrktypes.Client[HTH, S, ID, BLOCK_HASH] + chainID chains.ID + config htrktypes.Config + htConfig htrktypes.HeadTrackerConfig + + backfillMB *mailbox.Mailbox[HTH] + broadcastMB *mailbox.Mailbox[HTH] + headListener HeadListener[HTH, BLOCK_HASH] + getNilHead func() HTH +} + +// NewHeadTracker instantiates a new HeadTracker using HeadSaver to persist new block numbers. +func NewHeadTracker[ + HTH htrktypes.Head[BLOCK_HASH, ID], + S chains.Subscription, + ID chains.ID, + BLOCK_HASH chains.Hashable, +]( + lggr logger.Logger, + client htrktypes.Client[HTH, S, ID, BLOCK_HASH], + config htrktypes.Config, + htConfig htrktypes.HeadTrackerConfig, + headBroadcaster HeadBroadcaster[HTH, BLOCK_HASH], + headSaver HeadSaver[HTH, BLOCK_HASH], + mailMon *mailbox.Monitor, + getNilHead func() HTH, +) HeadTracker[HTH, BLOCK_HASH] { + ht := &headTracker[HTH, S, ID, BLOCK_HASH]{ + headBroadcaster: headBroadcaster, + client: client, + chainID: client.ConfiguredChainID(), + config: config, + htConfig: htConfig, + backfillMB: mailbox.NewSingle[HTH](), + broadcastMB: mailbox.New[HTH](HeadsBufferSize), + headSaver: headSaver, + mailMon: mailMon, + getNilHead: getNilHead, + } + ht.Service, ht.eng = services.Config{ + Name: "HeadTracker", + NewSubServices: func(lggr logger.Logger) []services.Service { + ht.headListener = NewHeadListener[HTH, S, ID, BLOCK_HASH](lggr, client, config, + // NOTE: Always try to start the head tracker off with whatever the + // latest head is, without waiting for the subscription to send us one. + // + // In some cases the subscription will send us the most recent head + // anyway when we connect (but we should not rely on this because it is + // not specced). If it happens this is fine, and the head will be + // ignored as a duplicate. + func(ctx context.Context) { + err := ht.handleInitialHead(ctx) + if err != nil { + ht.log.Errorw("Error handling initial head", "err", err.Error()) + } + }, ht.handleNewHead) + return []services.Service{ht.headListener} + }, + Start: ht.start, + Close: ht.close, + }.NewServiceEngine(lggr) + ht.log = logger.Sugared(ht.eng) + return ht +} + +// Start starts HeadTracker service. +func (ht *headTracker[HTH, S, ID, BLOCK_HASH]) start(context.Context) error { + ht.eng.Go(ht.backfillLoop) + ht.eng.Go(ht.broadcastLoop) + + ht.mailMon.Monitor(ht.broadcastMB, "HeadTracker", "Broadcast", ht.chainID.String()) + + return nil +} + +func (ht *headTracker[HTH, S, ID, BLOCK_HASH]) handleInitialHead(ctx context.Context) error { + initialHead, err := ht.client.HeadByNumber(ctx, nil) + if err != nil { + return fmt.Errorf("failed to fetch initial head: %w", err) + } + + if !initialHead.IsValid() { + ht.log.Warnw("Got nil initial head", "head", initialHead) + return nil + } + ht.log.Debugw("Got initial head", "head", initialHead, "blockNumber", initialHead.BlockNumber(), "blockHash", initialHead.BlockHash()) + + latestFinalized, err := ht.calculateLatestFinalized(ctx, initialHead, ht.htConfig.FinalityTagBypass()) + if err != nil { + return fmt.Errorf("failed to calculate latest finalized head: %w", err) + } + + if !latestFinalized.IsValid() { + return fmt.Errorf("latest finalized block is not valid") + } + + latestChain, err := ht.headSaver.Load(ctx, latestFinalized.BlockNumber()) + if err != nil { + return fmt.Errorf("failed to initialized headSaver: %w", err) + } + + if latestChain.IsValid() { + earliest := latestChain.EarliestHeadInChain() + ht.log.Debugw( + "Loaded chain from DB", + "latest_blockNumber", latestChain.BlockNumber(), + "latest_blockHash", latestChain.BlockHash(), + "earliest_blockNumber", earliest.BlockNumber(), + "earliest_blockHash", earliest.BlockHash(), + ) + } + if err := ht.handleNewHead(ctx, initialHead); err != nil { + return fmt.Errorf("error handling initial head: %w", err) + } + + return nil +} + +func (ht *headTracker[HTH, S, ID, BLOCK_HASH]) close() error { + return ht.broadcastMB.Close() +} + +func (ht *headTracker[HTH, S, ID, BLOCK_HASH]) Backfill(ctx context.Context, headWithChain HTH) (err error) { + latestFinalized, err := ht.calculateLatestFinalized(ctx, headWithChain, ht.htConfig.FinalityTagBypass()) + if err != nil { + return fmt.Errorf("failed to calculate finalized block: %w", err) + } + + if !latestFinalized.IsValid() { + return errors.New("can not perform backfill without a valid latestFinalized head") + } + + if headWithChain.BlockNumber() < latestFinalized.BlockNumber() { + const errMsg = "invariant violation: expected head of canonical chain to be ahead of the latestFinalized" + ht.log.With("head_block_num", headWithChain.BlockNumber(), + "latest_finalized_block_number", latestFinalized.BlockNumber()). + Criticalf(errMsg) + return errors.New(errMsg) + } + + if headWithChain.BlockNumber()-latestFinalized.BlockNumber() > int64(ht.htConfig.MaxAllowedFinalityDepth()) { + return fmt.Errorf("gap between latest finalized block (%d) and current head (%d) is too large (> %d)", + latestFinalized.BlockNumber(), headWithChain.BlockNumber(), ht.htConfig.MaxAllowedFinalityDepth()) + } + + return ht.backfill(ctx, headWithChain, latestFinalized) +} + +func (ht *headTracker[HTH, S, ID, BLOCK_HASH]) LatestChain() HTH { + return ht.headSaver.LatestChain() +} + +func (ht *headTracker[HTH, S, ID, BLOCK_HASH]) handleNewHead(ctx context.Context, head HTH) error { + prevHead := ht.headSaver.LatestChain() + + ht.log.Debugw(fmt.Sprintf("Received new head %v", head.BlockNumber()), + "blockHash", head.BlockHash(), + "parentHeadHash", head.GetParentHash(), + "blockTs", head.GetTimestamp(), + "blockTsUnix", head.GetTimestamp().Unix(), + "blockDifficulty", head.BlockDifficulty(), + ) + + if err := ht.headSaver.Save(ctx, head); ctx.Err() != nil { + return nil + } else if err != nil { + return fmt.Errorf("failed to save head: %#v: %w", head, err) + } + + if !prevHead.IsValid() || head.BlockNumber() > prevHead.BlockNumber() { + promCurrentHead.WithLabelValues(ht.chainID.String()).Set(float64(head.BlockNumber())) + + headWithChain := ht.headSaver.Chain(head.BlockHash()) + if !headWithChain.IsValid() { + return fmt.Errorf("HeadTracker#handleNewHighestHead headWithChain was unexpectedly nil") + } + ht.backfillMB.Deliver(headWithChain) + ht.broadcastMB.Deliver(headWithChain) + } else if head.BlockNumber() == prevHead.BlockNumber() { + if head.BlockHash() != prevHead.BlockHash() { + ht.log.Debugw("Got duplicate head", "blockNum", head.BlockNumber(), "head", head.BlockHash(), "prevHead", prevHead.BlockHash()) + } else { + ht.log.Debugw("Head already in the database", "head", head.BlockHash()) + } + } else { + ht.log.Debugw("Got out of order head", "blockNum", head.BlockNumber(), "head", head.BlockHash(), "prevHead", prevHead.BlockNumber()) + prevLatestFinalized := prevHead.LatestFinalizedHead() + + if prevLatestFinalized != nil && head.BlockNumber() <= prevLatestFinalized.BlockNumber() { + promOldHead.WithLabelValues(ht.chainID.String()).Inc() + err := fmt.Errorf("got very old block with number %d (highest seen was %d)", head.BlockNumber(), prevHead.BlockNumber()) + ht.log.Critical("Got very old block. Either a very deep re-org occurred, one of the RPC nodes has gotten far out of sync, or the chain went backwards in block numbers. This node may not function correctly without manual intervention.", "err", err) + ht.eng.EmitHealthErr(err) + } + } + return nil +} + +func (ht *headTracker[HTH, S, ID, BLOCK_HASH]) broadcastLoop(ctx context.Context) { + samplingInterval := ht.htConfig.SamplingInterval() + if samplingInterval > 0 { + ht.log.Debugf("Head sampling is enabled - sampling interval is set to: %v", samplingInterval) + debounceHead := time.NewTicker(samplingInterval) + defer debounceHead.Stop() + for { + select { + case <-ctx.Done(): + return + case <-debounceHead.C: + item := ht.broadcastMB.RetrieveLatestAndClear() + if !item.IsValid() { + continue + } + ht.headBroadcaster.BroadcastNewLongestChain(item) + } + } + } else { + ht.log.Info("Head sampling is disabled - callback will be called on every head") + for { + select { + case <-ctx.Done(): + return + case <-ht.broadcastMB.Notify(): + for { + item, exists := ht.broadcastMB.Retrieve() + if !exists { + break + } + ht.headBroadcaster.BroadcastNewLongestChain(item) + } + } + } + } +} + +func (ht *headTracker[HTH, S, ID, BLOCK_HASH]) backfillLoop(ctx context.Context) { + for { + select { + case <-ctx.Done(): + return + case <-ht.backfillMB.Notify(): + for { + head, exists := ht.backfillMB.Retrieve() + if !exists { + break + } + { + err := ht.Backfill(ctx, head) + if err != nil { + ht.log.Warnw("Unexpected error while backfilling heads", "err", err) + } else if ctx.Err() != nil { + break + } + } + } + } + } +} + +// LatestAndFinalizedBlock - returns latest and latest finalized blocks. +// NOTE: Returns latest finalized block as is, ignoring the FinalityTagBypass feature flag. +// TODO: BCI-3321 use cached values instead of making RPC requests +func (ht *headTracker[HTH, S, ID, BLOCK_HASH]) LatestAndFinalizedBlock(ctx context.Context) (latest, finalized HTH, err error) { + latest, err = ht.client.HeadByNumber(ctx, nil) + if err != nil { + err = fmt.Errorf("failed to get latest block: %w", err) + return + } + + if !latest.IsValid() { + err = fmt.Errorf("expected latest block to be valid") + return + } + + finalized, err = ht.calculateLatestFinalized(ctx, latest, false) + if err != nil { + err = fmt.Errorf("failed to calculate latest finalized block: %w", err) + return + } + if !finalized.IsValid() { + err = fmt.Errorf("expected finalized block to be valid") + return + } + + return +} + +func (ht *headTracker[HTH, S, ID, BLOCK_HASH]) getHeadAtHeight(ctx context.Context, chainHeadHash BLOCK_HASH, blockHeight int64) (HTH, error) { + chainHead := ht.headSaver.Chain(chainHeadHash) + if chainHead.IsValid() { + // check if provided chain contains a block of specified height + headAtHeight, err := chainHead.HeadAtHeight(blockHeight) + if err == nil { + // we are forced to reload the block due to type mismatched caused by generics + hthAtHeight := ht.headSaver.Chain(headAtHeight.BlockHash()) + // ensure that the block was not removed from the chain by another goroutine + if hthAtHeight.IsValid() { + return hthAtHeight, nil + } + } + } + + return ht.client.HeadByNumber(ctx, big.NewInt(blockHeight)) +} + +// calculateLatestFinalized - returns latest finalized block. It's expected that currentHeadNumber - is the head of +// canonical chain. There is no guaranties that returned block belongs to the canonical chain. Additional verification +// must be performed before usage. +func (ht *headTracker[HTH, S, ID, BLOCK_HASH]) calculateLatestFinalized(ctx context.Context, currentHead HTH, finalityTagBypass bool) (HTH, error) { + if ht.config.FinalityTagEnabled() && !finalityTagBypass { + latestFinalized, err := ht.client.LatestFinalizedBlock(ctx) + if err != nil { + return latestFinalized, fmt.Errorf("failed to get latest finalized block: %w", err) + } + + if !latestFinalized.IsValid() { + return latestFinalized, fmt.Errorf("failed to get valid latest finalized block") + } + + if ht.config.FinalizedBlockOffset() == 0 { + return latestFinalized, nil + } + + finalizedBlockNumber := max(latestFinalized.BlockNumber()-int64(ht.config.FinalizedBlockOffset()), 0) + return ht.getHeadAtHeight(ctx, latestFinalized.BlockHash(), finalizedBlockNumber) + } + // no need to make an additional RPC call on chains with instant finality + if ht.config.FinalityDepth() == 0 && ht.config.FinalizedBlockOffset() == 0 { + return currentHead, nil + } + finalizedBlockNumber := currentHead.BlockNumber() - int64(ht.config.FinalityDepth()) - int64(ht.config.FinalizedBlockOffset()) + if finalizedBlockNumber <= 0 { + finalizedBlockNumber = 0 + } + return ht.getHeadAtHeight(ctx, currentHead.BlockHash(), finalizedBlockNumber) +} + +// backfill fetches all missing heads up until the latestFinalizedHead +func (ht *headTracker[HTH, S, ID, BLOCK_HASH]) backfill(ctx context.Context, head, latestFinalizedHead HTH) (err error) { + headBlockNumber := head.BlockNumber() + mark := time.Now() + fetched := 0 + baseHeight := latestFinalizedHead.BlockNumber() + l := ht.log.With("blockNumber", headBlockNumber, + "n", headBlockNumber-baseHeight, + "fromBlockHeight", baseHeight, + "toBlockHeight", headBlockNumber-1) + l.Debug("Starting backfill") + defer func() { + if ctx.Err() != nil { + l.Warnw("Backfill context error", "err", ctx.Err()) + return + } + l.Debugw("Finished backfill", + "fetched", fetched, + "time", time.Since(mark), + "err", err) + }() + + for i := head.BlockNumber() - 1; i >= baseHeight; i-- { + // NOTE: Sequential requests here mean it's a potential performance bottleneck, be aware! + existingHead := ht.headSaver.Chain(head.GetParentHash()) + if existingHead.IsValid() { + head = existingHead + continue + } + head, err = ht.fetchAndSaveHead(ctx, i, head.GetParentHash()) + fetched++ + if ctx.Err() != nil { + ht.log.Debugw("context canceled, aborting backfill", "err", err, "ctx.Err", ctx.Err()) + return fmt.Errorf("fetchAndSaveHead failed: %w", ctx.Err()) + } else if err != nil { + return fmt.Errorf("fetchAndSaveHead failed: %w", err) + } + } + + if head.BlockHash() != latestFinalizedHead.BlockHash() { + ht.log.Criticalw("Finalized block missing from conical chain", + "finalized_block_number", latestFinalizedHead.BlockNumber(), "finalized_hash", latestFinalizedHead.BlockHash(), + "canonical_chain_block_number", head.BlockNumber(), "canonical_chain_hash", head.BlockHash()) + return FinalizedMissingError[BLOCK_HASH]{latestFinalizedHead.BlockHash(), head.BlockHash()} + } + + l = l.With("latest_finalized_block_hash", latestFinalizedHead.BlockHash(), + "latest_finalized_block_number", latestFinalizedHead.BlockNumber()) + + err = ht.headSaver.MarkFinalized(ctx, latestFinalizedHead) + if err != nil { + l.Debugw("failed to mark block as finalized", "err", err) + return nil + } + + l.Debugw("marked block as finalized") + + return +} + +type FinalizedMissingError[BLOCK_HASH chains.Hashable] struct { + Finalized, Canonical BLOCK_HASH +} + +func (e FinalizedMissingError[BLOCK_HASH]) Error() string { + return fmt.Sprintf("finalized block %s missing from canonical chain %s", e.Finalized, e.Canonical) +} + +func (ht *headTracker[HTH, S, ID, BLOCK_HASH]) fetchAndSaveHead(ctx context.Context, n int64, hash BLOCK_HASH) (HTH, error) { + ht.log.Debugw("Fetching head", "blockHeight", n, "blockHash", hash) + head, err := ht.client.HeadByHash(ctx, hash) + if err != nil { + return ht.getNilHead(), err + } else if !head.IsValid() { + return ht.getNilHead(), errors.New("got nil head") + } + err = ht.headSaver.Save(ctx, head) + if err != nil { + return ht.getNilHead(), err + } + return head, nil +} diff --git a/chains/headtracker/types/client.go b/chains/headtracker/types/client.go new file mode 100644 index 0000000..f04cfa6 --- /dev/null +++ b/chains/headtracker/types/client.go @@ -0,0 +1,20 @@ +package types + +import ( + "context" + "math/big" + + "github.com/smartcontractkit/chainlink-framework/chains" +) + +type Client[H chains.Head[BLOCK_HASH], S chains.Subscription, ID chains.ID, BLOCK_HASH chains.Hashable] interface { + HeadByNumber(ctx context.Context, number *big.Int) (head H, err error) + HeadByHash(ctx context.Context, hash BLOCK_HASH) (head H, err error) + // ConfiguredChainID returns the chain ID that the node is configured to connect to + ConfiguredChainID() (id ID) + // SubscribeToHeads is the method in which the client receives new Head. + // It can be implemented differently for each chain i.e websocket, polling, etc + SubscribeToHeads(ctx context.Context) (<-chan H, S, error) + // LatestFinalizedBlock - returns the latest block that was marked as finalized + LatestFinalizedBlock(ctx context.Context) (head H, err error) +} diff --git a/chains/headtracker/types/config.go b/chains/headtracker/types/config.go new file mode 100644 index 0000000..19ec519 --- /dev/null +++ b/chains/headtracker/types/config.go @@ -0,0 +1,19 @@ +package types + +import "time" + +type Config interface { + BlockEmissionIdleWarningThreshold() time.Duration + FinalityDepth() uint32 + FinalityTagEnabled() bool + FinalizedBlockOffset() uint32 +} + +type HeadTrackerConfig interface { + HistoryDepth() uint32 + MaxBufferSize() uint32 + SamplingInterval() time.Duration + FinalityTagBypass() bool + MaxAllowedFinalityDepth() uint32 + PersistenceEnabled() bool +} diff --git a/chains/headtracker/types/head.go b/chains/headtracker/types/head.go new file mode 100644 index 0000000..8acaf36 --- /dev/null +++ b/chains/headtracker/types/head.go @@ -0,0 +1,15 @@ +package types + +import ( + "github.com/smartcontractkit/chainlink-framework/chains" +) + +type Head[BLOCK_HASH chains.Hashable, CHAIN_ID chains.ID] interface { + chains.Head[BLOCK_HASH] + // ChainID returns the chain ID that the head is for + ChainID() CHAIN_ID + // Returns true if the head has a chain Id + HasChainID() bool + // IsValid returns true if the head is valid. + IsValid() bool +} diff --git a/chains/subscription.go b/chains/subscription.go new file mode 100644 index 0000000..4ba3aa2 --- /dev/null +++ b/chains/subscription.go @@ -0,0 +1,16 @@ +package chains + +// Subscription represents an event subscription where events are +// delivered on a data channel. +// This is a generic interface for Subscription to represent used by clients. +type Subscription interface { + // Unsubscribe cancels the sending of events to the data channel + // and closes the error channel. Unsubscribe should be callable multiple + // times without causing an error. + Unsubscribe() + // Err returns the subscription error channel. The error channel receives + // a value if there is an issue with the subscription (e.g. the network connection + // delivering the events has been closed). Only one value will ever be sent. + // The error channel is closed by Unsubscribe. + Err() <-chan error +} diff --git a/chains/txmgr/broadcaster.go b/chains/txmgr/broadcaster.go new file mode 100644 index 0000000..221d330 --- /dev/null +++ b/chains/txmgr/broadcaster.go @@ -0,0 +1,777 @@ +package txmgr + +import ( + "context" + "database/sql" + "errors" + "fmt" + "sync" + "time" + + "github.com/jpillora/backoff" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "go.uber.org/multierr" + "gopkg.in/guregu/null.v4" + + "github.com/smartcontractkit/chainlink-common/pkg/chains/label" + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/services" + "github.com/smartcontractkit/chainlink-common/pkg/utils" + "github.com/smartcontractkit/chainlink-framework/multinode" + + "github.com/smartcontractkit/chainlink-framework/chains" + "github.com/smartcontractkit/chainlink-framework/chains/fees" + txmgrtypes "github.com/smartcontractkit/chainlink-framework/chains/txmgr/types" +) + +const ( + // InFlightTransactionRecheckInterval controls how often the Broadcaster + // will poll the unconfirmed queue to see if it is allowed to send another + // transaction + InFlightTransactionRecheckInterval = 1 * time.Second + + // TransmitCheckTimeout controls the maximum amount of time that will be + // spent on the transmit check. + TransmitCheckTimeout = 2 * time.Second + + // maxBroadcastRetries is the number of times a transaction broadcast is retried when the sequence fails to increment on Hedera + maxHederaBroadcastRetries = 3 + + // hederaChainType is the string representation of the Hedera chain type + // Temporary solution until the Broadcaster is moved to the EVM code base + hederaChainType = "hedera" +) + +var ( + promTimeUntilBroadcast = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "tx_manager_time_until_tx_broadcast", + Help: "The amount of time elapsed from when a transaction is enqueued to until it is broadcast.", + Buckets: []float64{ + float64(500 * time.Millisecond), + float64(time.Second), + float64(5 * time.Second), + float64(15 * time.Second), + float64(30 * time.Second), + float64(time.Minute), + float64(2 * time.Minute), + }, + }, []string{"chainID"}) +) + +var ErrTxRemoved = errors.New("tx removed") + +type ProcessUnstartedTxs[ADDR chains.Hashable] func(ctx context.Context, fromAddress ADDR) (retryable bool, err error) + +// TransmitCheckerFactory creates a transmit checker based on a spec. +type TransmitCheckerFactory[ + CHAIN_ID chains.ID, + ADDR chains.Hashable, + TX_HASH, BLOCK_HASH chains.Hashable, + SEQ chains.Sequence, + FEE fees.Fee, +] interface { + // BuildChecker builds a new TransmitChecker based on the given spec. + BuildChecker(spec txmgrtypes.TransmitCheckerSpec[ADDR]) (TransmitChecker[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) +} + +// TransmitChecker determines whether a transaction should be submitted on-chain. +type TransmitChecker[ + CHAIN_ID chains.ID, + ADDR chains.Hashable, + TX_HASH, BLOCK_HASH chains.Hashable, + SEQ chains.Sequence, + FEE fees.Fee, +] interface { + + // Check the given transaction. If the transaction should not be sent, an error indicating why + // is returned. Errors should only be returned if the checker can confirm that a transaction + // should not be sent, other errors (for example connection or other unexpected errors) should + // be logged and swallowed. + Check(ctx context.Context, l logger.SugaredLogger, tx txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], a txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error +} + +// Broadcaster monitors txes for transactions that need to +// be broadcast, assigns sequences and ensures that at least one node +// somewhere has received the transaction successfully. +// +// This does not guarantee delivery! A whole host of other things can +// subsequently go wrong such as transactions being evicted from the mempool, +// nodes going offline etc. Responsibility for ensuring eventual inclusion +// into the chain falls on the shoulders of the confirmer. +// +// What Broadcaster does guarantee is: +// - a monotonic series of increasing sequences for txes that can all eventually be confirmed if you retry enough times +// - transition of txes out of unstarted into either fatal_error or unconfirmed +// - existence of a saved tx_attempt +type Broadcaster[ + CHAIN_ID chains.ID, + HEAD chains.Head[BLOCK_HASH], + ADDR chains.Hashable, + TX_HASH chains.Hashable, + BLOCK_HASH chains.Hashable, + SEQ chains.Sequence, + FEE fees.Fee, +] struct { + services.StateMachine + lggr logger.SugaredLogger + txStore txmgrtypes.TransactionStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, SEQ, FEE] + client txmgrtypes.TransactionClient[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] + txmgrtypes.TxAttemptBuilder[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] + sequenceTracker txmgrtypes.SequenceTracker[ADDR, SEQ] + resumeCallback ResumeCallback + chainID CHAIN_ID + chainType string + config txmgrtypes.BroadcasterChainConfig + feeConfig txmgrtypes.BroadcasterFeeConfig + txConfig txmgrtypes.BroadcasterTransactionsConfig + listenerConfig txmgrtypes.BroadcasterListenerConfig + + // autoSyncSequence, if set, will cause Broadcaster to fast-forward the sequence + // when Start is called + autoSyncSequence bool + + processUnstartedTxsImpl ProcessUnstartedTxs[ADDR] + + ks txmgrtypes.KeyStore[ADDR, CHAIN_ID, SEQ] + enabledAddresses []ADDR + + checkerFactory TransmitCheckerFactory[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] + + // triggers allow other goroutines to force Broadcaster to rescan the + // database early (before the next poll interval) + // Each key has its own trigger + triggers map[ADDR]chan struct{} + + chStop services.StopChan + wg sync.WaitGroup + + initSync sync.Mutex + isStarted bool +} + +func NewBroadcaster[ + CHAIN_ID chains.ID, + HEAD chains.Head[BLOCK_HASH], + ADDR chains.Hashable, + TX_HASH chains.Hashable, + BLOCK_HASH chains.Hashable, + SEQ chains.Sequence, + FEE fees.Fee, +]( + txStore txmgrtypes.TransactionStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, SEQ, FEE], + client txmgrtypes.TransactionClient[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], + config txmgrtypes.BroadcasterChainConfig, + feeConfig txmgrtypes.BroadcasterFeeConfig, + txConfig txmgrtypes.BroadcasterTransactionsConfig, + listenerConfig txmgrtypes.BroadcasterListenerConfig, + keystore txmgrtypes.KeyStore[ADDR, CHAIN_ID, SEQ], + txAttemptBuilder txmgrtypes.TxAttemptBuilder[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], + sequenceTracker txmgrtypes.SequenceTracker[ADDR, SEQ], + lggr logger.Logger, + checkerFactory TransmitCheckerFactory[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], + autoSyncSequence bool, + chainType string, +) *Broadcaster[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] { + lggr = logger.Named(lggr, "Broadcaster") + b := &Broadcaster[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{ + lggr: logger.Sugared(lggr), + txStore: txStore, + client: client, + TxAttemptBuilder: txAttemptBuilder, + chainID: client.ConfiguredChainID(), + chainType: chainType, + config: config, + feeConfig: feeConfig, + txConfig: txConfig, + listenerConfig: listenerConfig, + ks: keystore, + checkerFactory: checkerFactory, + autoSyncSequence: autoSyncSequence, + sequenceTracker: sequenceTracker, + } + + b.processUnstartedTxsImpl = b.processUnstartedTxs + return b +} + +// Start starts Broadcaster service. +// The provided context can be used to terminate Start sequence. +func (eb *Broadcaster[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) Start(ctx context.Context) error { + return eb.StartOnce("Broadcaster", func() (err error) { + return eb.startInternal(ctx) + }) +} + +// startInternal can be called multiple times, in conjunction with closeInternal. The TxMgr uses this functionality to reset broadcaster multiple times in its own lifetime. +func (eb *Broadcaster[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) startInternal(ctx context.Context) error { + eb.initSync.Lock() + defer eb.initSync.Unlock() + if eb.isStarted { + return errors.New("Broadcaster is already started") + } + var err error + eb.enabledAddresses, err = eb.ks.EnabledAddressesForChain(ctx, eb.chainID) + if err != nil { + return fmt.Errorf("Broadcaster: failed to load EnabledAddressesForChain: %w", err) + } + + if len(eb.enabledAddresses) > 0 { + eb.lggr.Debugw(fmt.Sprintf("Booting with %d keys", len(eb.enabledAddresses)), "keys", eb.enabledAddresses) + } else { + eb.lggr.Warnf("Chain %s does not have any keys, no transactions will be sent on this chain", eb.chainID.String()) + } + eb.chStop = make(chan struct{}) + eb.wg = sync.WaitGroup{} + eb.wg.Add(len(eb.enabledAddresses)) + eb.triggers = make(map[ADDR]chan struct{}) + eb.sequenceTracker.LoadNextSequences(ctx, eb.enabledAddresses) + for _, addr := range eb.enabledAddresses { + triggerCh := make(chan struct{}, 1) + eb.triggers[addr] = triggerCh + go eb.monitorTxs(addr, triggerCh) + } + + eb.isStarted = true + return nil +} + +// Close closes the Broadcaster +func (eb *Broadcaster[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) Close() error { + return eb.StopOnce("Broadcaster", func() error { + return eb.closeInternal() + }) +} + +func (eb *Broadcaster[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) closeInternal() error { + eb.initSync.Lock() + defer eb.initSync.Unlock() + if !eb.isStarted { + return fmt.Errorf("Broadcaster is not started: %w", services.ErrAlreadyStopped) + } + close(eb.chStop) + eb.wg.Wait() + eb.isStarted = false + return nil +} + +func (eb *Broadcaster[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) SetResumeCallback(callback ResumeCallback) { + eb.resumeCallback = callback +} + +func (eb *Broadcaster[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) Name() string { + return eb.lggr.Name() +} + +func (eb *Broadcaster[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) HealthReport() map[string]error { + return map[string]error{eb.Name(): eb.Healthy()} +} + +// Trigger forces the monitor for a particular address to recheck for new txes +// Logs error and does nothing if address was not registered on startup +func (eb *Broadcaster[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) Trigger(addr ADDR) { + if eb.isStarted { + triggerCh, exists := eb.triggers[addr] + if !exists { + // ignoring trigger for address which is not registered with this Broadcaster + return + } + select { + case triggerCh <- struct{}{}: + default: + } + } else { + eb.lggr.Debugf("Unstarted; ignoring trigger for %s", addr) + } +} + +func (eb *Broadcaster[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) newResendBackoff() backoff.Backoff { + return backoff.Backoff{ + Min: 1 * time.Second, + Max: 15 * time.Second, + Jitter: true, + } +} + +func (eb *Broadcaster[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) monitorTxs(addr ADDR, triggerCh chan struct{}) { + defer eb.wg.Done() + + ctx, cancel := eb.chStop.NewCtx() + defer cancel() + + if eb.autoSyncSequence { + eb.lggr.Debugw("Auto-syncing sequence", "address", addr.String()) + eb.sequenceTracker.SyncSequence(ctx, addr, eb.chStop) + if ctx.Err() != nil { + return + } + } else { + eb.lggr.Debugw("Skipping sequence auto-sync", "address", addr.String()) + } + + // errorRetryCh allows retry on exponential backoff in case of timeout or + // other unknown error + var errorRetryCh <-chan time.Time + bf := eb.newResendBackoff() + + for { + pollDBTimer := time.NewTimer(utils.WithJitter(eb.listenerConfig.FallbackPollInterval())) + + retryable, err := eb.processUnstartedTxsImpl(ctx, addr) + if err != nil { + eb.lggr.Errorw("Error occurred while handling tx queue in ProcessUnstartedTxs", "err", err) + } + // On retryable errors we implement exponential backoff retries. This + // handles intermittent connectivity, remote RPC races, timing issues etc + if retryable { + pollDBTimer.Reset(utils.WithJitter(eb.listenerConfig.FallbackPollInterval())) + errorRetryCh = time.After(bf.Duration()) + } else { + bf = eb.newResendBackoff() + errorRetryCh = nil + } + + select { + case <-ctx.Done(): + // NOTE: See: https://godoc.org/time#Timer.Stop for an explanation of this pattern + if !pollDBTimer.Stop() { + <-pollDBTimer.C + } + return + case <-triggerCh: + // tx was inserted + if !pollDBTimer.Stop() { + <-pollDBTimer.C + } + continue + case <-pollDBTimer.C: + // DB poller timed out + continue + case <-errorRetryCh: + // Error backoff period reached + continue + } + } +} + +// ProcessUnstartedTxs picks up and handles all txes in the queue +// revive:disable:error-return +func (eb *Broadcaster[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) ProcessUnstartedTxs(ctx context.Context, addr ADDR) (retryable bool, err error) { + return eb.processUnstartedTxs(ctx, addr) +} + +// NOTE: This MUST NOT be run concurrently for the same address or it could +// result in undefined state or deadlocks. +// First handle any in_progress transactions left over from last time. +// Then keep looking up unstarted transactions and processing them until there are none remaining. +func (eb *Broadcaster[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) processUnstartedTxs(ctx context.Context, fromAddress ADDR) (retryable bool, err error) { + var n uint + mark := time.Now() + defer func() { + if n > 0 { + eb.lggr.Debugw("Finished processUnstartedTxs", "address", fromAddress, "time", time.Since(mark), "n", n, "id", "broadcaster") + } + }() + + err, retryable = eb.handleAnyInProgressTx(ctx, fromAddress) + if err != nil { + return retryable, fmt.Errorf("processUnstartedTxs failed on handleAnyInProgressTx: %w", err) + } + for { + maxInFlightTransactions := eb.txConfig.MaxInFlight() + if maxInFlightTransactions > 0 { + nUnconfirmed, err := eb.txStore.CountUnconfirmedTransactions(ctx, fromAddress, eb.chainID) + if err != nil { + return true, fmt.Errorf("CountUnconfirmedTransactions failed: %w", err) + } + if nUnconfirmed >= maxInFlightTransactions { + nUnstarted, err := eb.txStore.CountUnstartedTransactions(ctx, fromAddress, eb.chainID) + if err != nil { + return true, fmt.Errorf("CountUnstartedTransactions failed: %w", err) + } + eb.lggr.Warnw(fmt.Sprintf(`Transaction throttling; %d transactions in-flight and %d unstarted transactions pending (maximum number of in-flight transactions is %d per key). %s`, nUnconfirmed, nUnstarted, maxInFlightTransactions, label.MaxInFlightTransactionsWarning), "maxInFlightTransactions", maxInFlightTransactions, "nUnconfirmed", nUnconfirmed, "nUnstarted", nUnstarted) + select { + case <-time.After(InFlightTransactionRecheckInterval): + case <-ctx.Done(): + return false, context.Cause(ctx) + } + continue + } + } + etx, err := eb.nextUnstartedTransactionWithSequence(fromAddress) + if err != nil { + return true, fmt.Errorf("processUnstartedTxs failed on nextUnstartedTransactionWithSequence: %w", err) + } + if etx == nil { + return false, nil + } + n++ + + if err, retryable := eb.handleUnstartedTx(ctx, etx); err != nil { + return retryable, fmt.Errorf("processUnstartedTxs failed on handleUnstartedTx: %w", err) + } + } +} + +// handleInProgressTx checks if there is any transaction +// in_progress and if so, finishes the job +func (eb *Broadcaster[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) handleAnyInProgressTx(ctx context.Context, fromAddress ADDR) (err error, retryable bool) { + etx, err := eb.txStore.GetTxInProgress(ctx, fromAddress) + if err != nil { + return fmt.Errorf("handleAnyInProgressTx failed: %w", err), true + } + if etx != nil { + if err, retryable := eb.handleInProgressTx(ctx, *etx, etx.TxAttempts[0], etx.CreatedAt, 0); err != nil { + return fmt.Errorf("handleAnyInProgressTx failed: %w", err), retryable + } + } + return nil, false +} + +func (eb *Broadcaster[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) handleUnstartedTx(ctx context.Context, etx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) (error, bool) { + if etx.State != TxUnstarted { + return fmt.Errorf("invariant violation: expected transaction %v to be unstarted, it was %s", etx.ID, etx.State), false + } + + attempt, _, _, retryable, err := eb.NewTxAttempt(ctx, *etx, eb.lggr) + // Mark transaction as fatal if provided gas limit is set too low + if errors.Is(err, fees.ErrFeeLimitTooLow) { + etx.Error = null.StringFrom(fees.ErrFeeLimitTooLow.Error()) + return eb.saveFatallyErroredTransaction(eb.lggr, etx), false + } else if err != nil { + return fmt.Errorf("processUnstartedTxs failed on NewAttempt: %w", err), retryable + } + + checkerSpec, err := etx.GetChecker() + if err != nil { + return fmt.Errorf("parsing transmit checker: %w", err), false + } + + checker, err := eb.checkerFactory.BuildChecker(checkerSpec) + if err != nil { + return fmt.Errorf("building transmit checker: %w", err), false + } + + lgr := etx.GetLogger(eb.lggr.With("fee", attempt.TxFee)) + + // If the transmit check does not complete within the timeout, the transaction will be sent + // anyway. + // It's intentional that we only run `Check` for unstarted transactions. + // Running it on other states might lead to nonce duplication, as we might mark applied transactions as fatally errored. + + checkCtx, cancel := context.WithTimeout(ctx, TransmitCheckTimeout) + defer cancel() + err = checker.Check(checkCtx, lgr, *etx, attempt) + if errors.Is(err, context.Canceled) { + lgr.Warn("Transmission checker timed out, sending anyway") + } else if err != nil { + etx.Error = null.StringFrom(err.Error()) + lgr.Warnw("Transmission checker failed, fatally erroring transaction.", "err", err) + return eb.saveFatallyErroredTransaction(lgr, etx), true + } + cancel() + + if err = eb.txStore.UpdateTxUnstartedToInProgress(ctx, etx, &attempt); errors.Is(err, ErrTxRemoved) { + eb.lggr.Debugw("tx removed", "txID", etx.ID, "subject", etx.Subject) + return nil, false + } else if err != nil { + return fmt.Errorf("processUnstartedTxs failed on UpdateTxUnstartedToInProgress: %w", err), true + } + + return eb.handleInProgressTx(ctx, *etx, attempt, time.Now(), 0) +} + +// There can be at most one in_progress transaction per address. +// Here we complete the job that we didn't finish last time. +func (eb *Broadcaster[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) handleInProgressTx(ctx context.Context, etx txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], attempt txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], initialBroadcastAt time.Time, retryCount int) (error, bool) { + if etx.State != TxInProgress { + return fmt.Errorf("invariant violation: expected transaction %v to be in_progress, it was %s", etx.ID, etx.State), false + } + + lgr := etx.GetLogger(logger.With(eb.lggr, "fee", attempt.TxFee)) + lgr.Infow("Sending transaction", "txAttemptID", attempt.ID, "txHash", attempt.Hash, "meta", etx.Meta, "feeLimit", attempt.ChainSpecificFeeLimit, "callerProvidedFeeLimit", etx.FeeLimit, "attempt", attempt, "etx", etx) + errType, err := eb.client.SendTransactionReturnCode(ctx, etx, attempt, lgr) + + // The validation below is only applicable to Hedera because it has instant finality and a unique sequence behavior + if eb.chainType == hederaChainType { + errType, err = eb.validateOnChainSequence(ctx, lgr, errType, err, etx, retryCount) + } + + if errType == multinode.Fatal || errType == multinode.TerminallyStuck { + eb.SvcErrBuffer.Append(err) + etx.Error = null.StringFrom(err.Error()) + return eb.saveFatallyErroredTransaction(lgr, &etx), true + } + + etx.InitialBroadcastAt = &initialBroadcastAt + etx.BroadcastAt = &initialBroadcastAt + + switch errType { + case multinode.TransactionAlreadyKnown: + fallthrough + case multinode.Successful: + // Either the transaction was successful or one of the following four scenarios happened: + // + // SCENARIO 1 + // + // This is resuming a previous crashed run. In this scenario, it is + // likely that our previous transaction was the one who was confirmed, + // in which case we hand it off to the confirmer to get the + // receipt. + // + // SCENARIO 2 + // + // It is also possible that an external wallet can have messed with the + // account and sent a transaction on this sequence. + // + // In this case, the onus is on the node operator since this is + // explicitly unsupported. + // + // If it turns out to have been an external wallet, we will never get a + // receipt for this transaction and it will eventually be marked as + // errored. + // + // The end result is that we will NOT SEND a transaction for this + // sequence. + // + // SCENARIO 3 + // + // The network client can be assumed to have at-least-once delivery + // behavior. It is possible that the client could have already + // sent this exact same transaction even if this is our first time + // calling SendTransaction(). + // + // SCENARIO 4 (most likely) + // + // A sendonly node got the transaction in first. + // + // In all scenarios, the correct thing to do is assume success for now + // and hand off to the confirmer to get the receipt (or mark as + // failed). + observeTimeUntilBroadcast(eb.chainID, etx.CreatedAt, time.Now()) + err = eb.txStore.UpdateTxAttemptInProgressToBroadcast(ctx, &etx, attempt, txmgrtypes.TxAttemptBroadcast) + if err != nil { + return err, true + } + // Increment sequence if successfully broadcasted + eb.sequenceTracker.GenerateNextSequence(etx.FromAddress, *etx.Sequence) + return err, true + case multinode.Underpriced: + bumpedAttempt, retryable, replaceErr := eb.replaceAttemptWithBumpedGas(ctx, lgr, err, etx, attempt) + if replaceErr != nil { + return replaceErr, retryable + } + + return eb.handleInProgressTx(ctx, etx, bumpedAttempt, initialBroadcastAt, retryCount+1) + case multinode.InsufficientFunds: + // NOTE: This can occur due to either insufficient funds or a gas spike + // combined with a high gas limit. Regardless of the cause, we need to obtain a new estimate, + // replace the current attempt, and retry after the backoff duration. + // The new attempt must be replaced immediately because of a database constraint. + eb.SvcErrBuffer.Append(err) + if _, _, replaceErr := eb.replaceAttemptWithNewEstimation(ctx, lgr, etx, attempt); replaceErr != nil { + return replaceErr, true + } + return err, true + case multinode.Retryable: + return err, true + case multinode.FeeOutOfValidRange: + replacementAttempt, retryable, replaceErr := eb.replaceAttemptWithNewEstimation(ctx, lgr, etx, attempt) + if replaceErr != nil { + return replaceErr, retryable + } + + lgr.Warnw("L2 rejected transaction due to incorrect fee, re-estimated and will try again", + "etxID", etx.ID, "err", err, "newGasPrice", replacementAttempt.TxFee, "newGasLimit", replacementAttempt.ChainSpecificFeeLimit) + return eb.handleInProgressTx(ctx, etx, *replacementAttempt, initialBroadcastAt, 0) + case multinode.Unsupported: + return err, false + case multinode.ExceedsMaxFee: + // Broadcaster: Note that we may have broadcast to multiple nodes and had it + // accepted by one of them! It is not guaranteed that all nodes share + // the same tx fee cap. That is why we must treat this as an unknown + // error that may have been confirmed. + // If there is only one RPC node, or all RPC nodes have the same + // configured cap, this transaction will get stuck and keep repeating + // forever until the issue is resolved. + lgr.Criticalw(`RPC node rejected this tx as outside Fee Cap`, "attempt", attempt) + fallthrough + default: + // Every error that doesn't fall under one of the above categories will be treated as Unknown. + fallthrough + case multinode.Unknown: + eb.SvcErrBuffer.Append(err) + lgr.Criticalw(`Unknown error occurred while handling tx queue in ProcessUnstartedTxs. This chain/RPC client may not be supported. `+ + `Urgent resolution required, Chainlink is currently operating in a degraded state and may miss transactions`, "attempt", attempt, "err", err) + nextSequence, e := eb.client.PendingSequenceAt(ctx, etx.FromAddress) + if e != nil { + err = multierr.Combine(e, err) + return fmt.Errorf("failed to fetch latest pending sequence after encountering unknown RPC error while sending transaction: %w", err), true + } + if nextSequence.Int64() > (*etx.Sequence).Int64() { + // Despite the error, the RPC node considers the previously sent + // transaction to have been accepted. In this case, the right thing to + // do is assume success and hand off to Confirmer + + err = eb.txStore.UpdateTxAttemptInProgressToBroadcast(ctx, &etx, attempt, txmgrtypes.TxAttemptBroadcast) + if err != nil { + return err, true + } + // Increment sequence if successfully broadcasted + eb.sequenceTracker.GenerateNextSequence(etx.FromAddress, *etx.Sequence) + return err, true + } + // Either the unknown error prevented the transaction from being mined, or + // it has not yet propagated to the mempool, or there is some race on the + // remote RPC. + // + // In all cases, the best thing we can do is go into a retry loop and keep + // trying to send the transaction over again. + return fmt.Errorf("retryable error while sending transaction %s (tx ID %d): %w", attempt.Hash.String(), etx.ID, err), true + } +} + +func (eb *Broadcaster[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) validateOnChainSequence(ctx context.Context, lgr logger.SugaredLogger, errType multinode.SendTxReturnCode, err error, etx txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], retryCount int) (multinode.SendTxReturnCode, error) { + // Only check if sequence was incremented if broadcast was successful, otherwise return the existing err type + if errType != multinode.Successful { + return errType, err + } + // Transaction sequence cannot be nil here since a sequence is required to broadcast + txSeq := *etx.Sequence + // Retrieve the latest mined sequence from on-chain + nextSeqOnChain, err := eb.client.SequenceAt(ctx, etx.FromAddress, nil) + if err != nil { + return errType, err + } + + // Check that the transaction count has incremented on-chain to include the broadcasted transaction + // Insufficient transaction fee is a common scenario in which the sequence is not incremented by the chain even though we got a successful response + // If the sequence failed to increment and hasn't reached the max retries, return the Underpriced error to try again with a bumped attempt + if nextSeqOnChain.Int64() == txSeq.Int64() && retryCount < maxHederaBroadcastRetries { + return multinode.Underpriced, nil + } + + // If the transaction reaches the retry limit and fails to get included, mark it as fatally errored + // Some unknown error other than insufficient tx fee could be the cause + if nextSeqOnChain.Int64() == txSeq.Int64() && retryCount >= maxHederaBroadcastRetries { + err := fmt.Errorf("failed to broadcast transaction on %s after %d retries", hederaChainType, retryCount) + lgr.Error(err.Error()) + return multinode.Fatal, err + } + + // Belts and braces approach to detect and handle sqeuence gaps if the broadcast is considered successful + if nextSeqOnChain.Int64() < txSeq.Int64() { + err := fmt.Errorf("next expected sequence on-chain (%s) is less than the broadcasted transaction's sequence (%s)", nextSeqOnChain.String(), txSeq.String()) + lgr.Criticalw("Sequence gap has been detected and needs to be filled", "error", err) + return multinode.Fatal, err + } + + return multinode.Successful, nil +} + +// Finds next transaction in the queue, assigns a sequence, and moves it to "in_progress" state ready for broadcast. +// Returns nil if no transactions are in queue +func (eb *Broadcaster[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) nextUnstartedTransactionWithSequence(fromAddress ADDR) (*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) { + ctx, cancel := eb.chStop.NewCtx() + defer cancel() + etx, err := eb.txStore.FindNextUnstartedTransactionFromAddress(ctx, fromAddress, eb.chainID) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + // Finish. No more transactions left to process. Hoorah! + return nil, nil + } + return nil, fmt.Errorf("findNextUnstartedTransactionFromAddress failed: %w", err) + } + + sequence, err := eb.sequenceTracker.GetNextSequence(ctx, etx.FromAddress) + if err != nil { + return nil, err + } + etx.Sequence = &sequence + return etx, nil +} + +// replaceAttemptWithBumpedGas performs the replacement of the existing tx attempt with a new bumped fee attempt. +func (eb *Broadcaster[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) replaceAttemptWithBumpedGas(ctx context.Context, lgr logger.Logger, txError error, etx txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], attempt txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) (replacedAttempt txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], retryable bool, err error) { + // This log error is not applicable to Hedera since the action required would not be needed for its gas estimator + if eb.chainType != hederaChainType { + logger.With(lgr, + "sendError", txError, + "attemptFee", attempt.TxFee, + "maxGasPriceConfig", eb.feeConfig.MaxFeePrice(), + ).Errorf("attempt fee %v was rejected by the node for being too low. "+ + "Node returned: '%s'. "+ + "Will bump and retry. ACTION REQUIRED: This is a configuration error. "+ + "Consider increasing FeeEstimator.PriceDefault (current value: %s)", + attempt.TxFee, txError.Error(), eb.feeConfig.FeePriceDefault()) + } + + bumpedAttempt, bumpedFee, bumpedFeeLimit, retryable, err := eb.NewBumpTxAttempt(ctx, etx, attempt, nil, lgr) + if err != nil { + return bumpedAttempt, retryable, err + } + + if err = eb.txStore.SaveReplacementInProgressAttempt(ctx, attempt, &bumpedAttempt); err != nil { + return bumpedAttempt, true, err + } + + lgr.Debugw("Bumped fee on initial send", "oldFee", attempt.TxFee.String(), "newFee", bumpedFee.String(), "newFeeLimit", bumpedFeeLimit) + return bumpedAttempt, true, err +} + +// replaceAttemptWithNewEstimation performs the replacement of the existing tx attempt with a new estimated fee attempt. +func (eb *Broadcaster[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) replaceAttemptWithNewEstimation(ctx context.Context, lgr logger.Logger, etx txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], attempt txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) (updatedAttempt *txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], retryable bool, err error) { + newEstimatedAttempt, fee, feeLimit, retryable, err := eb.NewTxAttemptWithType(ctx, etx, lgr, attempt.TxType, fees.OptForceRefetch) + if err != nil { + return &newEstimatedAttempt, retryable, err + } + + if err = eb.txStore.SaveReplacementInProgressAttempt(ctx, attempt, &newEstimatedAttempt); err != nil { + return &newEstimatedAttempt, true, err + } + + lgr.Debugw("new estimated fee on initial send", "oldFee", attempt.TxFee.String(), "newFee", fee.String(), "newFeeLimit", feeLimit) + return &newEstimatedAttempt, true, err +} + +func (eb *Broadcaster[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) saveFatallyErroredTransaction(lgr logger.Logger, etx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error { + ctx, cancel := eb.chStop.NewCtx() + defer cancel() + if etx.State != TxInProgress && etx.State != TxUnstarted { + return fmt.Errorf("can only transition to fatal_error from in_progress or unstarted, transaction is currently %s", etx.State) + } + if !etx.Error.Valid { + return errors.New("expected error field to be set") + } + // NOTE: It's simpler to not do this transactionally for now (would require + // refactoring pipeline runner resume to use postgres events) + // + // There is a very tiny possibility of the following: + // + // 1. We get a fatal error on the tx, resuming the pipeline with error + // 2. Crash or failure during persist of fatal errored tx + // 3. On the subsequent run the tx somehow succeeds and we save it as successful + // + // Now we have an errored pipeline even though the tx succeeded. This case + // is relatively benign and probably nobody will ever run into it in + // practice, but something to be aware of. + if etx.PipelineTaskRunID.Valid && eb.resumeCallback != nil && etx.SignalCallback && !etx.CallbackCompleted { + err := eb.resumeCallback(ctx, etx.PipelineTaskRunID.UUID, nil, fmt.Errorf("fatal error while sending transaction: %s", etx.Error.String)) + if errors.Is(err, sql.ErrNoRows) { + lgr.Debugw("callback missing or already resumed", "etxID", etx.ID) + } else if err != nil { + return fmt.Errorf("failed to resume pipeline: %w", err) + } else { + // Mark tx as having completed callback + if err := eb.txStore.UpdateTxCallbackCompleted(ctx, etx.PipelineTaskRunID.UUID, eb.chainID); err != nil { + return err + } + } + } + return eb.txStore.UpdateTxFatalErrorAndDeleteAttempts(ctx, etx) +} + +func observeTimeUntilBroadcast[CHAIN_ID chains.ID](chainID CHAIN_ID, createdAt, broadcastAt time.Time) { + duration := float64(broadcastAt.Sub(createdAt)) + promTimeUntilBroadcast.WithLabelValues(chainID.String()).Observe(duration) +} diff --git a/chains/txmgr/confirmer.go b/chains/txmgr/confirmer.go new file mode 100644 index 0000000..ef8823e --- /dev/null +++ b/chains/txmgr/confirmer.go @@ -0,0 +1,895 @@ +package txmgr + +import ( + "context" + "database/sql" + "encoding/hex" + "errors" + "fmt" + "sort" + "sync" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "go.uber.org/multierr" + + commonhex "github.com/smartcontractkit/chainlink-common/pkg/utils/hex" + + "github.com/smartcontractkit/chainlink-common/pkg/chains/label" + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/services" + "github.com/smartcontractkit/chainlink-common/pkg/utils/mailbox" + "github.com/smartcontractkit/chainlink-framework/multinode" + + "github.com/smartcontractkit/chainlink-framework/chains" + "github.com/smartcontractkit/chainlink-framework/chains/fees" + txmgrtypes "github.com/smartcontractkit/chainlink-framework/chains/txmgr/types" +) + +const ( + // processHeadTimeout represents a sanity limit on how long ProcessHead + // should take to complete + processHeadTimeout = 10 * time.Minute +) + +var ( + promNumGasBumps = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "tx_manager_num_gas_bumps", + Help: "Number of gas bumps", + }, []string{"chainID"}) + + promGasBumpExceedsLimit = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "tx_manager_gas_bump_exceeds_limit", + Help: "Number of times gas bumping failed from exceeding the configured limit. Any counts of this type indicate a serious problem.", + }, []string{"chainID"}) + promNumConfirmedTxs = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "tx_manager_num_confirmed_transactions", + Help: "Total number of confirmed transactions. Note that this can err to be too high since transactions are counted on each confirmation, which can happen multiple times per transaction in the case of re-orgs", + }, []string{"chainID"}) + promTimeUntilTxConfirmed = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "tx_manager_time_until_tx_confirmed", + Help: "The amount of time elapsed from a transaction being broadcast to being included in a block.", + Buckets: []float64{ + float64(500 * time.Millisecond), + float64(time.Second), + float64(5 * time.Second), + float64(15 * time.Second), + float64(30 * time.Second), + float64(time.Minute), + float64(2 * time.Minute), + float64(5 * time.Minute), + float64(10 * time.Minute), + }, + }, []string{"chainID"}) + promBlocksUntilTxConfirmed = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "tx_manager_blocks_until_tx_confirmed", + Help: "The amount of blocks that have been mined from a transaction being broadcast to being included in a block.", + Buckets: []float64{ + float64(1), + float64(5), + float64(10), + float64(20), + float64(50), + float64(100), + }, + }, []string{"chainID"}) +) + +// Confirmer is a broad service which performs four different tasks in sequence on every new longest chain +// Step 1: Mark that all currently pending transaction attempts were broadcast before this block +// Step 2: Check pending transactions for confirmation and confirmed transactions for re-org +// Step 3: Check if any pending transaction is stuck in the mempool. If so, mark for purge. +// Step 4: See if any transactions have exceeded the gas bumping block threshold and, if so, bump them +type Confirmer[ + CHAIN_ID chains.ID, + HEAD chains.Head[BLOCK_HASH], + ADDR chains.Hashable, + TX_HASH chains.Hashable, + BLOCK_HASH chains.Hashable, + R txmgrtypes.ChainReceipt[TX_HASH, BLOCK_HASH], + SEQ chains.Sequence, + FEE fees.Fee, +] struct { + services.StateMachine + txStore txmgrtypes.TxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] + lggr logger.SugaredLogger + client txmgrtypes.TxmClient[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE] + txmgrtypes.TxAttemptBuilder[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] + stuckTxDetector txmgrtypes.StuckTxDetector[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] + resumeCallback ResumeCallback + feeConfig txmgrtypes.ConfirmerFeeConfig + txConfig txmgrtypes.ConfirmerTransactionsConfig + dbConfig txmgrtypes.ConfirmerDatabaseConfig + chainID CHAIN_ID + + ks txmgrtypes.KeyStore[ADDR, CHAIN_ID, SEQ] + enabledAddresses []ADDR + + mb *mailbox.Mailbox[HEAD] + stopCh services.StopChan + wg sync.WaitGroup + initSync sync.Mutex + isStarted bool + isReceiptNil func(R) bool +} + +func NewConfirmer[ + CHAIN_ID chains.ID, + HEAD chains.Head[BLOCK_HASH], + ADDR chains.Hashable, + TX_HASH chains.Hashable, + BLOCK_HASH chains.Hashable, + R txmgrtypes.ChainReceipt[TX_HASH, BLOCK_HASH], + SEQ chains.Sequence, + FEE fees.Fee, +]( + txStore txmgrtypes.TxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE], + client txmgrtypes.TxmClient[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE], + feeConfig txmgrtypes.ConfirmerFeeConfig, + txConfig txmgrtypes.ConfirmerTransactionsConfig, + dbConfig txmgrtypes.ConfirmerDatabaseConfig, + keystore txmgrtypes.KeyStore[ADDR, CHAIN_ID, SEQ], + txAttemptBuilder txmgrtypes.TxAttemptBuilder[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], + lggr logger.Logger, + isReceiptNil func(R) bool, + stuckTxDetector txmgrtypes.StuckTxDetector[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], +) *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { + lggr = logger.Named(lggr, "Confirmer") + return &Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]{ + txStore: txStore, + lggr: logger.Sugared(lggr), + client: client, + TxAttemptBuilder: txAttemptBuilder, + resumeCallback: nil, + feeConfig: feeConfig, + txConfig: txConfig, + dbConfig: dbConfig, + chainID: client.ConfiguredChainID(), + ks: keystore, + mb: mailbox.NewSingle[HEAD](), + isReceiptNil: isReceiptNil, + stuckTxDetector: stuckTxDetector, + } +} + +// Start is a comment to appease the linter +func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Start(ctx context.Context) error { + return ec.StartOnce("Confirmer", func() error { + if ec.feeConfig.BumpThreshold() == 0 { + ec.lggr.Infow("Gas bumping is disabled (FeeEstimator.BumpThreshold set to 0)", "feeBumpThreshold", 0) + } else { + ec.lggr.Infow(fmt.Sprintf("Fee bumping is enabled, unconfirmed transactions will have their fee bumped every %d blocks", ec.feeConfig.BumpThreshold()), "feeBumpThreshold", ec.feeConfig.BumpThreshold()) + } + + return ec.startInternal(ctx) + }) +} + +func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) startInternal(ctx context.Context) error { + ec.initSync.Lock() + defer ec.initSync.Unlock() + if ec.isStarted { + return errors.New("Confirmer is already started") + } + var err error + ec.enabledAddresses, err = ec.ks.EnabledAddressesForChain(ctx, ec.chainID) + if err != nil { + return fmt.Errorf("Confirmer: failed to load EnabledAddressesForChain: %w", err) + } + if err = ec.stuckTxDetector.LoadPurgeBlockNumMap(ctx, ec.enabledAddresses); err != nil { + ec.lggr.Debugf("Confirmer: failed to load the last purged block num for enabled addresses. Process can continue as normal but purge rate limiting may be affected.") + } + + ec.stopCh = make(chan struct{}) + ec.wg = sync.WaitGroup{} + ec.wg.Add(1) + go ec.runLoop() + ec.isStarted = true + return nil +} + +// Close is a comment to appease the linter +func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Close() error { + return ec.StopOnce("Confirmer", func() error { + return ec.closeInternal() + }) +} + +func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) closeInternal() error { + ec.initSync.Lock() + defer ec.initSync.Unlock() + if !ec.isStarted { + return fmt.Errorf("Confirmer is not started: %w", services.ErrAlreadyStopped) + } + close(ec.stopCh) + ec.wg.Wait() + ec.isStarted = false + return nil +} + +func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SetResumeCallback(callback ResumeCallback) { + ec.resumeCallback = callback +} + +func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Name() string { + return ec.lggr.Name() +} + +func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) HealthReport() map[string]error { + return map[string]error{ec.Name(): ec.Healthy()} +} + +func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) runLoop() { + defer ec.wg.Done() + ctx, cancel := ec.stopCh.NewCtx() + defer cancel() + for { + select { + case <-ec.mb.Notify(): + for { + if ctx.Err() != nil { + return + } + head, exists := ec.mb.Retrieve() + if !exists { + break + } + if err := ec.ProcessHead(ctx, head); err != nil { + ec.lggr.Errorw("Error processing head", "err", err) + continue + } + } + case <-ctx.Done(): + return + } + } +} + +// ProcessHead takes all required transactions for the confirmer on a new head +func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) ProcessHead(ctx context.Context, head chains.Head[BLOCK_HASH]) error { + ctx, cancel := context.WithTimeout(ctx, processHeadTimeout) + defer cancel() + return ec.processHead(ctx, head) +} + +// NOTE: This SHOULD NOT be run concurrently or it could behave badly +func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) processHead(ctx context.Context, head chains.Head[BLOCK_HASH]) error { + ec.lggr.Debugw("processHead start", "headNum", head.BlockNumber(), "id", "confirmer") + + mark := time.Now() + if err := ec.txStore.SetBroadcastBeforeBlockNum(ctx, head.BlockNumber(), ec.chainID); err != nil { + return err + } + ec.lggr.Debugw("Finished SetBroadcastBeforeBlockNum", "headNum", head.BlockNumber(), "time", time.Since(mark), "id", "confirmer") + + mark = time.Now() + if err := ec.CheckForConfirmation(ctx, head); err != nil { + return err + } + ec.lggr.Debugw("Finished CheckForConfirmation", "headNum", head.BlockNumber(), "time", time.Since(mark), "id", "confirmer") + + mark = time.Now() + if err := ec.ProcessStuckTransactions(ctx, head.BlockNumber()); err != nil { + return err + } + ec.lggr.Debugw("Finished ProcessStuckTransactions", "headNum", head.BlockNumber(), "time", time.Since(mark), "id", "confirmer") + + mark = time.Now() + if err := ec.RebroadcastWhereNecessary(ctx, head.BlockNumber()); err != nil { + return err + } + ec.lggr.Debugw("Finished RebroadcastWhereNecessary", "headNum", head.BlockNumber(), "time", time.Since(mark), "id", "confirmer") + ec.lggr.Debugw("processHead finish", "headNum", head.BlockNumber(), "id", "confirmer") + + return nil +} + +// CheckForConfirmation fetches the mined transaction count for each enabled address and marks transactions with a lower sequence as confirmed and ones with equal or higher sequence as unconfirmed +func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) CheckForConfirmation(ctx context.Context, head chains.Head[BLOCK_HASH]) error { + var errorList []error + for _, fromAddress := range ec.enabledAddresses { + minedTxCount, err := ec.client.SequenceAt(ctx, fromAddress, nil) + if err != nil { + errorList = append(errorList, fmt.Errorf("unable to fetch mined transaction count for address %s: %w", fromAddress.String(), err)) + continue + } + reorgTxs, includedTxs, err := ec.txStore.FindReorgOrIncludedTxs(ctx, fromAddress, minedTxCount, ec.chainID) + if err != nil { + errorList = append(errorList, fmt.Errorf("failed to find re-org'd or included transactions based on the mined transaction count %d: %w", minedTxCount.Int64(), err)) + continue + } + // If re-org'd transactions are identified, process them and mark them for rebroadcast + err = ec.ProcessReorgTxs(ctx, reorgTxs, head) + if err != nil { + errorList = append(errorList, fmt.Errorf("failed to process re-org'd transactions: %w", err)) + continue + } + // If unconfirmed transactions are identified as included, process them and mark them as confirmed or terminally stuck (if purge attempt exists) + err = ec.ProcessIncludedTxs(ctx, includedTxs, head) + if err != nil { + errorList = append(errorList, fmt.Errorf("failed to process confirmed transactions: %w", err)) + continue + } + } + if len(errorList) > 0 { + return errors.Join(errorList...) + } + return nil +} + +func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) ProcessReorgTxs(ctx context.Context, reorgTxs []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], head chains.Head[BLOCK_HASH]) error { + if len(reorgTxs) == 0 { + return nil + } + etxIDs := make([]int64, 0, len(reorgTxs)) + attemptIDs := make([]int64, 0, len(reorgTxs)) + for _, etx := range reorgTxs { + if len(etx.TxAttempts) == 0 { + return fmt.Errorf("invariant violation: expected tx %v to have at least one attempt", etx.ID) + } + + // Rebroadcast the one with the highest gas price + attempt := etx.TxAttempts[0] + + logValues := []interface{}{ + "txhash", attempt.Hash.String(), + "currentBlockNum", head.BlockNumber(), + "currentBlockHash", head.BlockHash().String(), + "txID", etx.ID, + "attemptID", attempt.ID, + "nReceipts", len(attempt.Receipts), + "attemptState", attempt.State, + "id", "confirmer", + } + + if len(attempt.Receipts) > 0 && attempt.Receipts[0] != nil { + receipt := attempt.Receipts[0] + logValues = append(logValues, + "replacementBlockHashAtConfirmedHeight", head.HashAtHeight(receipt.GetBlockNumber().Int64()), + "confirmedInBlockNum", receipt.GetBlockNumber(), + "confirmedInBlockHash", receipt.GetBlockHash(), + "confirmedInTxIndex", receipt.GetTransactionIndex(), + ) + } + + if etx.State == TxFinalized { + ec.lggr.AssumptionViolationw(fmt.Sprintf("Re-org detected for finalized transaction. This should never happen. Rebroadcasting transaction %s which may have been re-org'd out of the main chain", attempt.Hash.String()), logValues...) + } else { + ec.lggr.Infow(fmt.Sprintf("Re-org detected. Rebroadcasting transaction %s which may have been re-org'd out of the main chain", attempt.Hash.String()), logValues...) + } + + etxIDs = append(etxIDs, etx.ID) + attemptIDs = append(attemptIDs, attempt.ID) + } + + // Mark transactions as unconfirmed, mark attempts as in-progress, and delete receipts since they do not apply to the new chain + // This may revert some fatal error transactions to unconfirmed if terminally stuck transactions purge attempts get re-org'd + return ec.txStore.UpdateTxsForRebroadcast(ctx, etxIDs, attemptIDs) +} + +func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) ProcessIncludedTxs(ctx context.Context, includedTxs []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], head chains.Head[BLOCK_HASH]) error { + if len(includedTxs) == 0 { + return nil + } + // Add newly confirmed transactions to the prom metric + promNumConfirmedTxs.WithLabelValues(ec.chainID.String()).Add(float64(len(includedTxs))) + + purgeTxIDs := make([]int64, 0, len(includedTxs)) + confirmedTxIDs := make([]int64, 0, len(includedTxs)) + for _, tx := range includedTxs { + // If any attempt in the transaction is marked for purge, the transaction was terminally stuck and should be marked as fatal error + if tx.HasPurgeAttempt() { + // Setting the purged block num here is ok since we have confirmation the tx has been included + ec.stuckTxDetector.SetPurgeBlockNum(tx.FromAddress, head.BlockNumber()) + purgeTxIDs = append(purgeTxIDs, tx.ID) + continue + } + confirmedTxIDs = append(confirmedTxIDs, tx.ID) + observeUntilTxConfirmed(ec.chainID, tx.TxAttempts, head) + } + // Mark the transactions included on-chain with a purge attempt as fatal error with the terminally stuck error message + if err := ec.txStore.UpdateTxFatalError(ctx, purgeTxIDs, ec.stuckTxDetector.StuckTxFatalError()); err != nil { + return fmt.Errorf("failed to update terminally stuck transactions: %w", err) + } + // Mark the transactions included on-chain as confirmed + if err := ec.txStore.UpdateTxConfirmed(ctx, confirmedTxIDs); err != nil { + return fmt.Errorf("failed to update confirmed transactions: %w", err) + } + return nil +} + +// Determines if any of the unconfirmed transactions are terminally stuck for each enabled address +// If any transaction is found to be terminally stuck, this method sends an empty attempt with bumped gas in an attempt to purge the stuck transaction +func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) ProcessStuckTransactions(ctx context.Context, blockNum int64) error { + // Use the detector to find a stuck tx for each enabled address + stuckTxs, err := ec.stuckTxDetector.DetectStuckTransactions(ctx, ec.enabledAddresses, blockNum) + if err != nil { + return fmt.Errorf("failed to detect stuck transactions: %w", err) + } + if len(stuckTxs) == 0 { + return nil + } + + var wg sync.WaitGroup + wg.Add(len(stuckTxs)) + errorList := []error{} + var errMu sync.Mutex + for _, tx := range stuckTxs { + // All stuck transactions will have unique from addresses. It is safe to process separate keys concurrently + // NOTE: This design will block one key if another takes a really long time to execute + go func(tx txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) { + defer wg.Done() + lggr := tx.GetLogger(ec.lggr) + // Create a purge attempt for tx + purgeAttempt, err := ec.TxAttemptBuilder.NewPurgeTxAttempt(ctx, tx, lggr) + if err != nil { + errMu.Lock() + errorList = append(errorList, fmt.Errorf("failed to create a purge attempt: %w", err)) + errMu.Unlock() + return + } + // Save purge attempt + if err := ec.txStore.SaveInProgressAttempt(ctx, &purgeAttempt); err != nil { + errMu.Lock() + errorList = append(errorList, fmt.Errorf("failed to save purge attempt: %w", err)) + errMu.Unlock() + return + } + lggr.Warnw("marked transaction as terminally stuck", "etx", tx) + // Send purge attempt + if err := ec.handleInProgressAttempt(ctx, lggr, tx, purgeAttempt, blockNum); err != nil { + errMu.Lock() + errorList = append(errorList, fmt.Errorf("failed to send purge attempt: %w", err)) + errMu.Unlock() + return + } + // Resume pending task runs with failure for stuck transactions + if err := ec.resumeFailedTaskRuns(ctx, tx); err != nil { + errMu.Lock() + errorList = append(errorList, fmt.Errorf("failed to resume pending task run for transaction: %w", err)) + errMu.Unlock() + return + } + }(tx) + } + wg.Wait() + return errors.Join(errorList...) +} + +func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) resumeFailedTaskRuns(ctx context.Context, etx txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error { + if !etx.PipelineTaskRunID.Valid || ec.resumeCallback == nil || !etx.SignalCallback || etx.CallbackCompleted { + return nil + } + err := ec.resumeCallback(ctx, etx.PipelineTaskRunID.UUID, nil, errors.New(ec.stuckTxDetector.StuckTxFatalError())) + if errors.Is(err, sql.ErrNoRows) { + ec.lggr.Debugw("callback missing or already resumed", "etxID", etx.ID) + } else if err != nil { + return fmt.Errorf("failed to resume pipeline: %w", err) + } else { + // Mark tx as having completed callback + if err := ec.txStore.UpdateTxCallbackCompleted(ctx, etx.PipelineTaskRunID.UUID, ec.chainID); err != nil { + return err + } + } + return nil +} + +// RebroadcastWhereNecessary bumps gas or resends transactions that were previously out-of-funds +func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) RebroadcastWhereNecessary(ctx context.Context, blockHeight int64) error { + var wg sync.WaitGroup + + // It is safe to process separate keys concurrently + // NOTE: This design will block one key if another takes a really long time to execute + wg.Add(len(ec.enabledAddresses)) + errors := []error{} + var errMu sync.Mutex + for _, address := range ec.enabledAddresses { + go func(fromAddress ADDR) { + if err := ec.rebroadcastWhereNecessary(ctx, fromAddress, blockHeight); err != nil { + errMu.Lock() + errors = append(errors, err) + errMu.Unlock() + ec.lggr.Errorw("Error in RebroadcastWhereNecessary", "err", err, "fromAddress", fromAddress) + } + + wg.Done() + }(address) + } + + wg.Wait() + + return multierr.Combine(errors...) +} + +func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) rebroadcastWhereNecessary(ctx context.Context, address ADDR, blockHeight int64) error { + if err := ec.handleAnyInProgressAttempts(ctx, address, blockHeight); err != nil { + return fmt.Errorf("handleAnyInProgressAttempts failed: %w", err) + } + + threshold := int64(ec.feeConfig.BumpThreshold()) + bumpDepth := int64(ec.feeConfig.BumpTxDepth()) + maxInFlightTransactions := ec.txConfig.MaxInFlight() + etxs, err := ec.FindTxsRequiringRebroadcast(ctx, ec.lggr, address, blockHeight, threshold, bumpDepth, maxInFlightTransactions, ec.chainID) + if err != nil { + return fmt.Errorf("FindTxsRequiringRebroadcast failed: %w", err) + } + for _, etx := range etxs { + lggr := etx.GetLogger(ec.lggr) + + attempt, err := ec.attemptForRebroadcast(ctx, lggr, *etx) + if err != nil { + return fmt.Errorf("attemptForRebroadcast failed: %w", err) + } + + lggr.Debugw("Rebroadcasting transaction", "nPreviousAttempts", len(etx.TxAttempts), "fee", attempt.TxFee) + + if err := ec.txStore.SaveInProgressAttempt(ctx, &attempt); err != nil { + return fmt.Errorf("saveInProgressAttempt failed: %w", err) + } + + if err := ec.handleInProgressAttempt(ctx, lggr, *etx, attempt, blockHeight); err != nil { + return fmt.Errorf("handleInProgressAttempt failed: %w", err) + } + } + return nil +} + +// "in_progress" attempts were left behind after a crash/restart and may or may not have been sent. +// We should try to ensure they get on-chain so we can fetch a receipt for them. +// NOTE: We also use this to mark attempts for rebroadcast in event of a +// re-org, so multiple attempts are allowed to be in in_progress state (but +// only one per tx). +func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) handleAnyInProgressAttempts(ctx context.Context, address ADDR, blockHeight int64) error { + attempts, err := ec.txStore.GetInProgressTxAttempts(ctx, address, ec.chainID) + if ctx.Err() != nil { + return nil + } else if err != nil { + return fmt.Errorf("GetInProgressTxAttempts failed: %w", err) + } + for _, a := range attempts { + err := ec.handleInProgressAttempt(ctx, a.Tx.GetLogger(ec.lggr), a.Tx, a, blockHeight) + if ctx.Err() != nil { + break + } else if err != nil { + return fmt.Errorf("handleInProgressAttempt failed: %w", err) + } + } + return nil +} + +// FindTxsRequiringRebroadcast returns attempts that hit insufficient native tokens, +// and attempts that need bumping, in sequence ASC order +func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxsRequiringRebroadcast(ctx context.Context, lggr logger.Logger, address ADDR, blockNum, gasBumpThreshold, bumpDepth int64, maxInFlightTransactions uint32, chainID CHAIN_ID) (etxs []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) { + // NOTE: These two queries could be combined into one using union but it + // becomes harder to read and difficult to test in isolation. KISS principle + etxInsufficientFunds, err := ec.txStore.FindTxsRequiringResubmissionDueToInsufficientFunds(ctx, address, chainID) + if err != nil { + return nil, err + } + + if len(etxInsufficientFunds) > 0 { + lggr.Infow(fmt.Sprintf("Found %d transactions to be re-sent that were previously rejected due to insufficient native token balance", len(etxInsufficientFunds)), "blockNum", blockNum, "address", address) + } + + etxBumps, err := ec.txStore.FindTxsRequiringGasBump(ctx, address, blockNum, gasBumpThreshold, bumpDepth, chainID) + if ctx.Err() != nil { + return nil, nil + } else if err != nil { + return nil, err + } + + if len(etxBumps) > 0 { + // txes are ordered by sequence asc so the first will always be the oldest + etx := etxBumps[0] + // attempts are ordered by time sent asc so first will always be the oldest + var oldestBlocksBehind int64 = -1 // It should never happen that the oldest attempt has no BroadcastBeforeBlockNum set, but in case it does, we shouldn't crash - log this sentinel value instead + if len(etx.TxAttempts) > 0 { + oldestBlockNum := etx.TxAttempts[0].BroadcastBeforeBlockNum + if oldestBlockNum != nil { + oldestBlocksBehind = blockNum - *oldestBlockNum + } + } else { + logger.Sugared(lggr).AssumptionViolationw("Expected tx for gas bump to have at least one attempt", "etxID", etx.ID, "blockNum", blockNum, "address", address) + } + lggr.Infow(fmt.Sprintf("Found %d transactions to re-sent that have still not been confirmed after at least %d blocks. The oldest of these has not still not been confirmed after %d blocks. These transactions will have their gas price bumped. %s", len(etxBumps), gasBumpThreshold, oldestBlocksBehind, label.NodeConnectivityProblemWarning), "blockNum", blockNum, "address", address, "gasBumpThreshold", gasBumpThreshold) + } + + seen := make(map[int64]struct{}) + + for _, etx := range etxInsufficientFunds { + seen[etx.ID] = struct{}{} + etxs = append(etxs, etx) + } + for _, etx := range etxBumps { + if _, exists := seen[etx.ID]; !exists { + etxs = append(etxs, etx) + } + } + + sort.Slice(etxs, func(i, j int) bool { + return (*etxs[i].Sequence).Int64() < (*etxs[j].Sequence).Int64() + }) + + if maxInFlightTransactions > 0 && len(etxs) > int(maxInFlightTransactions) { + lggr.Warnf("%d transactions to rebroadcast which exceeds limit of %d. %s", len(etxs), maxInFlightTransactions, label.MaxInFlightTransactionsWarning) + etxs = etxs[:maxInFlightTransactions] + } + + return +} + +func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) attemptForRebroadcast(ctx context.Context, lggr logger.Logger, etx txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) (attempt txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) { + if len(etx.TxAttempts) > 0 { + etx.TxAttempts[0].Tx = etx + previousAttempt := etx.TxAttempts[0] + logFields := ec.logFieldsPreviousAttempt(previousAttempt) + if previousAttempt.State == txmgrtypes.TxAttemptInsufficientFunds { + // Do not create a new attempt if we ran out of funds last time since bumping gas is pointless + // Instead try to resubmit the same attempt at the same price, in the hope that the wallet was funded since our last attempt + lggr.Debugw("Rebroadcast InsufficientFunds", logFields...) + previousAttempt.State = txmgrtypes.TxAttemptInProgress + return previousAttempt, nil + } + attempt, err = ec.bumpGas(ctx, etx, etx.TxAttempts) + + if fees.IsBumpErr(err) { + lggr.Errorw("Failed to bump gas", append(logFields, "err", err)...) + // Do not create a new attempt if bumping gas would put us over the limit or cause some other problem + // Instead try to resubmit the previous attempt, and keep resubmitting until its accepted + previousAttempt.BroadcastBeforeBlockNum = nil + previousAttempt.State = txmgrtypes.TxAttemptInProgress + return previousAttempt, nil + } + return attempt, err + } + return attempt, fmt.Errorf("invariant violation: Tx %v was unconfirmed but didn't have any attempts. "+ + "Falling back to default gas price instead."+ + "This is a bug! Please report to https://github.com/smartcontractkit/chainlink/issues", etx.ID) +} + +func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) logFieldsPreviousAttempt(attempt txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) []interface{} { + etx := attempt.Tx + return []interface{}{ + "etxID", etx.ID, + "txHash", attempt.Hash, + "previousAttempt", attempt, + "feeLimit", attempt.ChainSpecificFeeLimit, + "callerProvidedFeeLimit", etx.FeeLimit, + "maxGasPrice", ec.feeConfig.MaxFeePrice(), + "sequence", etx.Sequence, + } +} + +func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) bumpGas(ctx context.Context, etx txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], previousAttempts []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) (bumpedAttempt txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) { + previousAttempt := previousAttempts[0] + logFields := ec.logFieldsPreviousAttempt(previousAttempt) + + var bumpedFee FEE + var bumpedFeeLimit uint64 + bumpedAttempt, bumpedFee, bumpedFeeLimit, _, err = ec.NewBumpTxAttempt(ctx, etx, previousAttempt, previousAttempts, ec.lggr) + + // if no error, return attempt + // if err, continue below + if err == nil { + promNumGasBumps.WithLabelValues(ec.chainID.String()).Inc() + ec.lggr.Debugw("Rebroadcast bumping fee for tx", append(logFields, "bumpedFee", bumpedFee.String(), "bumpedFeeLimit", bumpedFeeLimit)...) + return bumpedAttempt, err + } + + if errors.Is(err, fees.ErrBumpFeeExceedsLimit) { + promGasBumpExceedsLimit.WithLabelValues(ec.chainID.String()).Inc() + } + + return bumpedAttempt, fmt.Errorf("error bumping gas: %w", err) +} + +func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) handleInProgressAttempt(ctx context.Context, lggr logger.SugaredLogger, etx txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], attempt txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], blockHeight int64) error { + if attempt.State != txmgrtypes.TxAttemptInProgress { + return fmt.Errorf("invariant violation: expected tx_attempt %v to be in_progress, it was %s", attempt.ID, attempt.State) + } + + now := time.Now() + lggr.Debugw("Sending transaction", "txAttemptID", attempt.ID, "txHash", attempt.Hash, "meta", etx.Meta, "feeLimit", attempt.ChainSpecificFeeLimit, "callerProvidedFeeLimit", etx.FeeLimit, "attempt", attempt, "etx", etx) + errType, sendError := ec.client.SendTransactionReturnCode(ctx, etx, attempt, lggr) + + switch errType { + case multinode.Underpriced: + // This should really not ever happen in normal operation since we + // already bumped above the required minimum in broadcaster. + ec.lggr.Warnw("Got terminally underpriced error for gas bump, this should never happen unless the remote RPC node changed its configuration on the fly, or you are using multiple RPC nodes with different minimum gas price requirements. This is not recommended", "attempt", attempt) + // "Lazily" load attempts here since the overwhelmingly common case is + // that we don't need them unless we enter this path + if err := ec.txStore.LoadTxAttempts(ctx, &etx); err != nil { + return fmt.Errorf("failed to load TxAttempts while bumping on terminally underpriced error: %w", err) + } + if len(etx.TxAttempts) == 0 { + err := errors.New("expected to find at least 1 attempt") + ec.lggr.AssumptionViolationw(err.Error(), "err", err, "attempt", attempt) + return err + } + if attempt.ID != etx.TxAttempts[0].ID { + err := errors.New("expected highest priced attempt to be the current in_progress attempt") + ec.lggr.AssumptionViolationw(err.Error(), "err", err, "attempt", attempt, "txAttempts", etx.TxAttempts) + return err + } + replacementAttempt, err := ec.bumpGas(ctx, etx, etx.TxAttempts) + if err != nil { + return fmt.Errorf("could not bump gas for terminally underpriced transaction: %w", err) + } + promNumGasBumps.WithLabelValues(ec.chainID.String()).Inc() + lggr.With( + "sendError", sendError, + "maxGasPriceConfig", ec.feeConfig.MaxFeePrice(), + "previousAttempt", attempt, + "replacementAttempt", replacementAttempt, + ).Errorf("gas price was rejected by the node for being too low. Node returned: '%s'", sendError.Error()) + + if err := ec.txStore.SaveReplacementInProgressAttempt(ctx, attempt, &replacementAttempt); err != nil { + return fmt.Errorf("saveReplacementInProgressAttempt failed: %w", err) + } + return ec.handleInProgressAttempt(ctx, lggr, etx, replacementAttempt, blockHeight) + case multinode.ExceedsMaxFee: + // Confirmer: Note it is not guaranteed that all nodes share the same tx fee cap. + // So it is very likely that this attempt was successful on another node since + // it was already successfully broadcasted. So we assume it is successful and + // warn the operator that the RPC node is misconfigured. + // This failure scenario is a strong indication that the RPC node + // is misconfigured. This is a critical error and should be resolved by the + // node operator. + // If there is only one RPC node, or all RPC nodes have the same + // configured cap, this transaction will get stuck and keep repeating + // forever until the issue is resolved. + lggr.Criticalw(`RPC node rejected this tx as outside Fee Cap but it may have been accepted by another Node`, "attempt", attempt) + timeout := ec.dbConfig.DefaultQueryTimeout() + return ec.txStore.SaveSentAttempt(ctx, timeout, &attempt, now) + case multinode.Fatal: + // WARNING: This should never happen! + // Should NEVER be fatal this is an invariant violation. The + // Broadcaster can never create a TxAttempt that will + // fatally error. + lggr.Criticalw("Invariant violation: fatal error while re-attempting transaction", + "fee", attempt.TxFee, + "feeLimit", attempt.ChainSpecificFeeLimit, + "callerProvidedFeeLimit", etx.FeeLimit, + "signedRawTx", commonhex.EnsurePrefix(hex.EncodeToString(attempt.SignedRawTx)), + "blockHeight", blockHeight, + ) + ec.SvcErrBuffer.Append(sendError) + // This will loop continuously on every new head so it must be handled manually by the node operator! + return ec.txStore.DeleteInProgressAttempt(ctx, attempt) + case multinode.TerminallyStuck: + // A transaction could broadcast successfully but then be considered terminally stuck on another attempt + // Even though the transaction can succeed under different circumstances, we want to purge this transaction as soon as we get this error + lggr.Warnw("terminally stuck transaction detected", "err", sendError.Error()) + ec.SvcErrBuffer.Append(sendError) + // Create a purge attempt for tx + purgeAttempt, err := ec.TxAttemptBuilder.NewPurgeTxAttempt(ctx, etx, lggr) + if err != nil { + return fmt.Errorf("NewPurgeTxAttempt failed: %w", err) + } + // Replace the in progress attempt with the purge attempt + if err := ec.txStore.SaveReplacementInProgressAttempt(ctx, attempt, &purgeAttempt); err != nil { + return fmt.Errorf("saveReplacementInProgressAttempt failed: %w", err) + } + return ec.handleInProgressAttempt(ctx, lggr, etx, purgeAttempt, blockHeight) + case multinode.TransactionAlreadyKnown: + // Sequence too low indicated that a transaction at this sequence was confirmed already. + // Mark confirmed_missing_receipt and wait for the next cycle to try to get a receipt + lggr.Debugw("Sequence already used", "txAttemptID", attempt.ID, "txHash", attempt.Hash.String()) + timeout := ec.dbConfig.DefaultQueryTimeout() + return ec.txStore.SaveConfirmedAttempt(ctx, timeout, &attempt, now) + case multinode.InsufficientFunds: + timeout := ec.dbConfig.DefaultQueryTimeout() + return ec.txStore.SaveInsufficientFundsAttempt(ctx, timeout, &attempt, now) + case multinode.Successful: + lggr.Debugw("Successfully broadcast transaction", "txAttemptID", attempt.ID, "txHash", attempt.Hash.String()) + timeout := ec.dbConfig.DefaultQueryTimeout() + return ec.txStore.SaveSentAttempt(ctx, timeout, &attempt, now) + case multinode.Unknown: + // Every error that doesn't fall under one of the above categories will be treated as Unknown. + fallthrough + default: + // Any other type of error is considered temporary or resolvable by the + // node operator. The node may have it in the mempool so we must keep the + // attempt (leave it in_progress). Safest thing to do is bail out and wait + // for the next head. + return fmt.Errorf("unexpected error sending tx %v with hash %s: %w", etx.ID, attempt.Hash.String(), sendError) + } +} + +// ForceRebroadcast sends a transaction for every sequence in the given sequence range at the given gas price. +// If an tx exists for this sequence, we re-send the existing tx with the supplied parameters. +// If an tx doesn't exist for this sequence, we send a zero transaction. +// This operates completely orthogonal to the normal Confirmer and can result in untracked attempts! +// Only for emergency usage. +// This is in case of some unforeseen scenario where the node is refusing to release the lock. KISS. +func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) ForceRebroadcast(ctx context.Context, seqs []SEQ, fee FEE, address ADDR, overrideGasLimit uint64) error { + if len(seqs) == 0 { + ec.lggr.Infof("ForceRebroadcast: No sequences provided. Skipping") + return nil + } + ec.lggr.Infof("ForceRebroadcast: will rebroadcast transactions for all sequences between %v and %v", seqs[0], seqs[len(seqs)-1]) + + for _, seq := range seqs { + etx, err := ec.txStore.FindTxWithSequence(ctx, address, seq) + if err != nil { + return fmt.Errorf("ForceRebroadcast failed: %w", err) + } + if etx == nil { + ec.lggr.Debugf("ForceRebroadcast: no tx found with sequence %s, will rebroadcast empty transaction", seq) + hashStr, err := ec.sendEmptyTransaction(ctx, address, seq, overrideGasLimit, fee) + if err != nil { + ec.lggr.Errorw("ForceRebroadcast: failed to send empty transaction", "sequence", seq, "err", err) + continue + } + ec.lggr.Infow("ForceRebroadcast: successfully rebroadcast empty transaction", "sequence", seq, "hash", hashStr) + } else { + ec.lggr.Debugf("ForceRebroadcast: got tx %v with sequence %v, will rebroadcast this transaction", etx.ID, *etx.Sequence) + if overrideGasLimit != 0 { + etx.FeeLimit = overrideGasLimit + } + attempt, _, err := ec.NewCustomTxAttempt(ctx, *etx, fee, etx.FeeLimit, 0x0, ec.lggr) + if err != nil { + ec.lggr.Errorw("ForceRebroadcast: failed to create new attempt", "txID", etx.ID, "err", err) + continue + } + attempt.Tx = *etx // for logging + ec.lggr.Debugw("Sending transaction", "txAttemptID", attempt.ID, "txHash", attempt.Hash, "err", err, "meta", etx.Meta, "feeLimit", attempt.ChainSpecificFeeLimit, "callerProvidedFeeLimit", etx.FeeLimit, "attempt", attempt) + if errCode, err := ec.client.SendTransactionReturnCode(ctx, *etx, attempt, ec.lggr); errCode != multinode.Successful && err != nil { + ec.lggr.Errorw(fmt.Sprintf("ForceRebroadcast: failed to rebroadcast tx %v with sequence %v, gas limit %v, and caller provided fee Limit %v : %s", etx.ID, *etx.Sequence, attempt.ChainSpecificFeeLimit, etx.FeeLimit, err.Error()), "err", err, "fee", attempt.TxFee) + continue + } + ec.lggr.Infof("ForceRebroadcast: successfully rebroadcast tx %v with hash: 0x%x", etx.ID, attempt.Hash) + } + } + return nil +} + +func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) sendEmptyTransaction(ctx context.Context, fromAddress ADDR, seq SEQ, overrideGasLimit uint64, fee FEE) (string, error) { + gasLimit := overrideGasLimit + if gasLimit == 0 { + gasLimit = ec.feeConfig.LimitDefault() + } + txhash, err := ec.client.SendEmptyTransaction(ctx, ec.TxAttemptBuilder.NewEmptyTxAttempt, seq, gasLimit, fee, fromAddress) + if err != nil { + return "", fmt.Errorf("(Confirmer).sendEmptyTransaction failed: %w", err) + } + return txhash, nil +} + +// observeUntilTxConfirmed observes the promBlocksUntilTxConfirmed metric for each confirmed +// transaction. +func observeUntilTxConfirmed[ + CHAIN_ID chains.ID, + ADDR chains.Hashable, + TX_HASH, BLOCK_HASH chains.Hashable, + SEQ chains.Sequence, + FEE fees.Fee, +](chainID CHAIN_ID, attempts []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], head chains.Head[BLOCK_HASH]) { + for _, attempt := range attempts { + // We estimate the time until confirmation by subtracting from the time the tx (not the attempt) + // was created. We want to measure the amount of time taken from when a transaction is created + // via e.g Txm.CreateTransaction to when it is confirmed on-chain, regardless of how many attempts + // were needed to achieve this. + duration := time.Since(attempt.Tx.CreatedAt) + promTimeUntilTxConfirmed. + WithLabelValues(chainID.String()). + Observe(float64(duration)) + + // Since a tx can have many attempts, we take the number of blocks to confirm as the block number + // of the receipt minus the block number of the first ever broadcast for this transaction. + var minBroadcastBefore int64 + for _, a := range attempt.Tx.TxAttempts { + if b := a.BroadcastBeforeBlockNum; b != nil && *b < minBroadcastBefore { + minBroadcastBefore = *b + } + } + if minBroadcastBefore > 0 { + blocksElapsed := head.BlockNumber() - minBroadcastBefore + promBlocksUntilTxConfirmed. + WithLabelValues(chainID.String()). + Observe(float64(blocksElapsed)) + } + } +} diff --git a/chains/txmgr/models.go b/chains/txmgr/models.go new file mode 100644 index 0000000..6662b6c --- /dev/null +++ b/chains/txmgr/models.go @@ -0,0 +1,15 @@ +package txmgr + +import ( + txmgrtypes "github.com/smartcontractkit/chainlink-framework/chains/txmgr/types" +) + +const ( + TxUnstarted = txmgrtypes.TxState("unstarted") + TxInProgress = txmgrtypes.TxState("in_progress") + TxFatalError = txmgrtypes.TxState("fatal_error") + TxUnconfirmed = txmgrtypes.TxState("unconfirmed") + TxConfirmed = txmgrtypes.TxState("confirmed") + TxConfirmedMissingReceipt = txmgrtypes.TxState("confirmed_missing_receipt") + TxFinalized = txmgrtypes.TxState("finalized") +) diff --git a/chains/txmgr/reaper.go b/chains/txmgr/reaper.go new file mode 100644 index 0000000..1cd3ce3 --- /dev/null +++ b/chains/txmgr/reaper.go @@ -0,0 +1,116 @@ +package txmgr + +import ( + "fmt" + "sync/atomic" + "time" + + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/services" + "github.com/smartcontractkit/chainlink-framework/chains" + txmgrtypes "github.com/smartcontractkit/chainlink-framework/chains/txmgr/types" +) + +// Reaper handles periodic database cleanup for Txm +type Reaper[CHAIN_ID chains.ID] struct { + store txmgrtypes.TxHistoryReaper[CHAIN_ID] + txConfig txmgrtypes.ReaperTransactionsConfig + chainID CHAIN_ID + log logger.Logger + latestBlockNum atomic.Int64 + trigger chan struct{} + chStop services.StopChan + chDone chan struct{} +} + +// NewReaper instantiates a new reaper object +func NewReaper[CHAIN_ID chains.ID](lggr logger.Logger, store txmgrtypes.TxHistoryReaper[CHAIN_ID], txConfig txmgrtypes.ReaperTransactionsConfig, chainID CHAIN_ID) *Reaper[CHAIN_ID] { + r := &Reaper[CHAIN_ID]{ + store, + txConfig, + chainID, + logger.Named(lggr, "Reaper"), + atomic.Int64{}, + make(chan struct{}, 1), + make(services.StopChan), + make(chan struct{}), + } + r.latestBlockNum.Store(-1) + return r +} + +// Start the reaper. Should only be called once. +func (r *Reaper[CHAIN_ID]) Start() { + r.log.Debugf("started with age threshold %v and interval %v", r.txConfig.ReaperThreshold(), r.txConfig.ReaperInterval()) + go r.runLoop() +} + +// Stop the reaper. Should only be called once. +func (r *Reaper[CHAIN_ID]) Stop() { + r.log.Debug("stopping") + close(r.chStop) + <-r.chDone +} + +func (r *Reaper[CHAIN_ID]) runLoop() { + defer close(r.chDone) + ticker := services.NewTicker(r.txConfig.ReaperInterval()) + defer ticker.Stop() + for { + select { + case <-r.chStop: + return + case <-ticker.C: + r.work() + case <-r.trigger: + r.work() + ticker.Reset() + } + } +} + +func (r *Reaper[CHAIN_ID]) work() { + latestBlockNum := r.latestBlockNum.Load() + if latestBlockNum < 0 { + return + } + err := r.ReapTxes(latestBlockNum) + if err != nil { + r.log.Error("unable to reap old txes: ", err) + } +} + +// SetLatestBlockNum should be called on every new highest block number +func (r *Reaper[CHAIN_ID]) SetLatestBlockNum(latestBlockNum int64) { + if latestBlockNum < 0 { + panic(fmt.Sprintf("latestBlockNum must be 0 or greater, got: %d", latestBlockNum)) + } + was := r.latestBlockNum.Swap(latestBlockNum) + if was < 0 { + // Run reaper once on startup + r.trigger <- struct{}{} + } +} + +// ReapTxes deletes old txes +func (r *Reaper[CHAIN_ID]) ReapTxes(headNum int64) error { + ctx, cancel := r.chStop.NewCtx() + defer cancel() + threshold := r.txConfig.ReaperThreshold() + if threshold == 0 { + r.log.Debug("Transactions.ReaperThreshold set to 0; skipping ReapTxes") + return nil + } + mark := time.Now() + timeThreshold := mark.Add(-threshold) + + r.log.Debugw(fmt.Sprintf("reaping old txes created before %s", timeThreshold.Format(time.RFC3339)), "ageThreshold", threshold, "timeThreshold", timeThreshold) + + if err := r.store.ReapTxHistory(ctx, timeThreshold, r.chainID); err != nil { + return err + } + + r.log.Debugf("ReapTxes completed in %v", time.Since(mark)) + + return nil +} diff --git a/chains/txmgr/resender.go b/chains/txmgr/resender.go new file mode 100644 index 0000000..2f0b91e --- /dev/null +++ b/chains/txmgr/resender.go @@ -0,0 +1,234 @@ +package txmgr + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/smartcontractkit/chainlink-common/pkg/chains/label" + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/services" + "github.com/smartcontractkit/chainlink-framework/chains" + "github.com/smartcontractkit/chainlink-framework/multinode" + + "github.com/smartcontractkit/chainlink-framework/chains/fees" + txmgrtypes "github.com/smartcontractkit/chainlink-framework/chains/txmgr/types" +) + +const ( + // pollInterval is the maximum amount of time in addition to + // TxResendAfterThreshold that we will wait before resending an attempt + DefaultResenderPollInterval = 5 * time.Second + + // Alert interval for unconfirmed transaction attempts + unconfirmedTxAlertLogFrequency = 2 * time.Minute + + // timeout value for batchSendTransactions + batchSendTransactionTimeout = 30 * time.Second +) + +// Resender periodically picks up transactions that have been languishing +// unconfirmed for a configured amount of time without being sent, and sends +// their highest priced attempt again. This helps to defend against geth/parity +// silently dropping txes, or txes being ejected from the mempool. +// +// Previously we relied on the bumper to do this for us implicitly but there +// can occasionally be problems with this (e.g. abnormally long block times, or +// if gas bumping is disabled) +type Resender[ + CHAIN_ID chains.ID, + ADDR chains.Hashable, + TX_HASH chains.Hashable, + BLOCK_HASH chains.Hashable, + R txmgrtypes.ChainReceipt[TX_HASH, BLOCK_HASH], + SEQ chains.Sequence, + FEE fees.Fee, +] struct { + txStore txmgrtypes.TransactionStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, SEQ, FEE] + client txmgrtypes.TransactionClient[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] + tracker *Tracker[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE] + ks txmgrtypes.KeyStore[ADDR, CHAIN_ID, SEQ] + chainID CHAIN_ID + interval time.Duration + config txmgrtypes.ResenderChainConfig + txConfig txmgrtypes.ResenderTransactionsConfig + logger logger.SugaredLogger + lastAlertTimestamps map[string]time.Time + + stopCh services.StopChan + chDone chan struct{} +} + +func NewResender[ + CHAIN_ID chains.ID, + ADDR chains.Hashable, + TX_HASH chains.Hashable, + BLOCK_HASH chains.Hashable, + R txmgrtypes.ChainReceipt[TX_HASH, BLOCK_HASH], + SEQ chains.Sequence, + FEE fees.Fee, +]( + lggr logger.Logger, + txStore txmgrtypes.TransactionStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, SEQ, FEE], + client txmgrtypes.TransactionClient[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], + tracker *Tracker[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE], + ks txmgrtypes.KeyStore[ADDR, CHAIN_ID, SEQ], + pollInterval time.Duration, + config txmgrtypes.ResenderChainConfig, + txConfig txmgrtypes.ResenderTransactionsConfig, +) *Resender[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { + if txConfig.ResendAfterThreshold() == 0 { + panic("Resender requires a non-zero threshold") + } + // todo: add context to txStore https://smartcontract-it.atlassian.net/browse/BCI-1585 + return &Resender[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]{ + txStore, + client, + tracker, + ks, + client.ConfiguredChainID(), + pollInterval, + config, + txConfig, + logger.Sugared(logger.Named(lggr, "Resender")), + make(map[string]time.Time), + make(chan struct{}), + make(chan struct{}), + } +} + +// Start is a comment which satisfies the linter +func (er *Resender[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Start(ctx context.Context) { + er.logger.Debugf("Enabled with poll interval of %s and age threshold of %s", er.interval, er.txConfig.ResendAfterThreshold()) + go er.runLoop() +} + +// Stop is a comment which satisfies the linter +func (er *Resender[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Stop() { + close(er.stopCh) + <-er.chDone +} + +func (er *Resender[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) runLoop() { + defer close(er.chDone) + ctx, cancel := er.stopCh.NewCtx() + defer cancel() + + if err := er.resendUnconfirmed(ctx); err != nil { + er.logger.Warnw("Failed to resend unconfirmed transactions", "err", err) + } + + ticker := services.NewTicker(er.interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if err := er.resendUnconfirmed(ctx); err != nil { + er.logger.Warnw("Failed to resend unconfirmed transactions", "err", err) + } + } + } +} + +func (er *Resender[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) resendUnconfirmed(ctx context.Context) error { + var cancel func() + ctx, cancel = er.stopCh.Ctx(ctx) + defer cancel() + resendAddresses, err := er.ks.EnabledAddressesForChain(ctx, er.chainID) + if err != nil { + return fmt.Errorf("Resender failed getting enabled keys for chain %s: %w", er.chainID.String(), err) + } + + ageThreshold := er.txConfig.ResendAfterThreshold() + maxInFlightTransactions := er.txConfig.MaxInFlight() + olderThan := time.Now().Add(-ageThreshold) + var allAttempts []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] + + for _, k := range resendAddresses { + var attempts []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] + attempts, err = er.txStore.FindTxAttemptsRequiringResend(ctx, olderThan, maxInFlightTransactions, er.chainID, k) + if err != nil { + return fmt.Errorf("failed to FindTxAttemptsRequiringResend: %w", err) + } + er.logStuckAttempts(attempts, k) + + allAttempts = append(allAttempts, attempts...) + } + + if len(allAttempts) == 0 { + for k := range er.lastAlertTimestamps { + er.lastAlertTimestamps[k] = time.Now() + } + return nil + } + er.logger.Infow(fmt.Sprintf("Re-sending %d unconfirmed transactions that were last sent over %s ago. These transactions are taking longer than usual to be mined. %s", len(allAttempts), ageThreshold, label.NodeConnectivityProblemWarning), "n", len(allAttempts)) + + batchSize := int(er.config.RPCDefaultBatchSize()) + batchCtx, batchCancel := context.WithTimeout(ctx, batchSendTransactionTimeout) + defer batchCancel() + txErrTypes, _, broadcastTime, txIDs, err := er.client.BatchSendTransactions(batchCtx, allAttempts, batchSize, er.logger) + + // update broadcast times before checking additional errors + if len(txIDs) > 0 { + if updateErr := er.txStore.UpdateBroadcastAts(ctx, broadcastTime, txIDs); updateErr != nil { + err = errors.Join(err, fmt.Errorf("failed to update broadcast time: %w", updateErr)) + } + } + if err != nil { + return fmt.Errorf("failed to re-send transactions: %w", err) + } + logResendResult(er.logger, txErrTypes) + + return nil +} + +func logResendResult(lggr logger.Logger, codes []multinode.SendTxReturnCode) { + var nNew int + var nFatal int + for _, c := range codes { + if c == multinode.Successful { + nNew++ + } else if c == multinode.Fatal { + nFatal++ + } + } + lggr.Debugw("Completed", "n", len(codes), "nNew", nNew, "nFatal", nFatal) +} + +func (er *Resender[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) logStuckAttempts(attempts []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], fromAddress ADDR) { + if time.Since(er.lastAlertTimestamps[fromAddress.String()]) >= unconfirmedTxAlertLogFrequency { + oldestAttempt, exists := findOldestUnconfirmedAttempt(attempts) + if exists { + // Wait at least 2 times the TxResendAfterThreshold to log critical with an unconfirmedTxAlertDelay + if time.Since(oldestAttempt.CreatedAt) > er.txConfig.ResendAfterThreshold()*2 { + er.lastAlertTimestamps[fromAddress.String()] = time.Now() + er.logger.Errorw("TxAttempt has been unconfirmed for more than max duration", "maxDuration", er.txConfig.ResendAfterThreshold()*2, + "txID", oldestAttempt.TxID, "txFee", oldestAttempt.TxFee, + "BroadcastBeforeBlockNum", oldestAttempt.BroadcastBeforeBlockNum, "Hash", oldestAttempt.Hash, "fromAddress", fromAddress) + } + } + } +} + +func findOldestUnconfirmedAttempt[ + CHAIN_ID chains.ID, + ADDR chains.Hashable, + TX_HASH, BLOCK_HASH chains.Hashable, + SEQ chains.Sequence, + FEE fees.Fee, +](attempts []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) (txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], bool) { + var oldestAttempt txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] + if len(attempts) < 1 { + return oldestAttempt, false + } + oldestAttempt = attempts[0] + for i := 1; i < len(attempts); i++ { + if oldestAttempt.CreatedAt.Sub(attempts[i].CreatedAt) <= 0 { + oldestAttempt = attempts[i] + } + } + return oldestAttempt, true +} diff --git a/chains/txmgr/strategies.go b/chains/txmgr/strategies.go new file mode 100644 index 0000000..d87d709 --- /dev/null +++ b/chains/txmgr/strategies.go @@ -0,0 +1,64 @@ +package txmgr + +import ( + "context" + "fmt" + + "github.com/google/uuid" + + txmgrtypes "github.com/smartcontractkit/chainlink-framework/chains/txmgr/types" +) + +var _ txmgrtypes.TxStrategy = SendEveryStrategy{} + +// NewQueueingTxStrategy creates a new TxStrategy that drops the oldest transactions after the +// queue size is exceeded if a queue size is specified, and otherwise does not drop transactions. +func NewQueueingTxStrategy(subject uuid.UUID, queueSize uint32) (strategy txmgrtypes.TxStrategy) { + if queueSize > 0 { + strategy = NewDropOldestStrategy(subject, queueSize) + } else { + strategy = SendEveryStrategy{} + } + return +} + +// NewSendEveryStrategy creates a new TxStrategy that does not drop transactions. +func NewSendEveryStrategy() txmgrtypes.TxStrategy { + return SendEveryStrategy{} +} + +// SendEveryStrategy will always send the tx +type SendEveryStrategy struct{} + +func (SendEveryStrategy) Subject() uuid.NullUUID { return uuid.NullUUID{} } +func (SendEveryStrategy) PruneQueue(ctx context.Context, pruneService txmgrtypes.UnstartedTxQueuePruner) ([]int64, error) { + return nil, nil +} + +var _ txmgrtypes.TxStrategy = DropOldestStrategy{} + +// DropOldestStrategy will send the newest N transactions, older ones will be +// removed from the queue +type DropOldestStrategy struct { + subject uuid.UUID + queueSize uint32 +} + +// NewDropOldestStrategy creates a new TxStrategy that drops the oldest transactions after the +// queue size is exceeded. +func NewDropOldestStrategy(subject uuid.UUID, queueSize uint32) DropOldestStrategy { + return DropOldestStrategy{subject, queueSize} +} + +func (s DropOldestStrategy) Subject() uuid.NullUUID { + return uuid.NullUUID{UUID: s.subject, Valid: true} +} + +func (s DropOldestStrategy) PruneQueue(ctx context.Context, pruneService txmgrtypes.UnstartedTxQueuePruner) (ids []int64, err error) { + // NOTE: We prune one less than the queue size to prevent the queue from exceeding the max queue size. Which could occur if a new transaction is added to the queue right after we prune. + ids, err = pruneService.PruneUnstartedTxQueue(ctx, s.queueSize-1, s.subject) + if err != nil { + return ids, fmt.Errorf("DropOldestStrategy#PruneQueue failed: %w", err) + } + return +} diff --git a/chains/txmgr/test_helpers.go b/chains/txmgr/test_helpers.go new file mode 100644 index 0000000..617960b --- /dev/null +++ b/chains/txmgr/test_helpers.go @@ -0,0 +1,55 @@ +package txmgr + +import ( + "context" + "time" + + txmgrtypes "github.com/smartcontractkit/chainlink-framework/chains/txmgr/types" +) + +// TEST ONLY FUNCTIONS +// these need to be exported for the txmgr tests to continue to work + +func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) XXXTestSetClient(client txmgrtypes.TxmClient[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) { + ec.client = client +} + +func (tr *Tracker[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) XXXTestSetTTL(ttl time.Duration) { + tr.ttl = ttl +} + +func (tr *Tracker[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) XXXDeliverBlock(blockHeight int64) { + tr.mb.Deliver(blockHeight) +} + +func (eb *Broadcaster[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) XXXTestStartInternal(ctx context.Context) error { + return eb.startInternal(ctx) +} + +func (eb *Broadcaster[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) XXXTestCloseInternal() error { + return eb.closeInternal() +} + +func (eb *Broadcaster[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) XXXTestDisableUnstartedTxAutoProcessing() { + eb.processUnstartedTxsImpl = func(ctx context.Context, fromAddress ADDR) (retryable bool, err error) { return false, nil } +} + +func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) XXXTestStartInternal() error { + ctx, cancel := ec.stopCh.NewCtx() + defer cancel() + return ec.startInternal(ctx) +} + +func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) XXXTestCloseInternal() error { + return ec.closeInternal() +} + +func (er *Resender[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) XXXTestResendUnconfirmed() error { + ctx, cancel := er.stopCh.NewCtx() + defer cancel() + return er.resendUnconfirmed(ctx) +} + +func (b *Txm[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) XXXTestAbandon(addr ADDR) (err error) { + return b.abandon(addr) +} diff --git a/chains/txmgr/tracker.go b/chains/txmgr/tracker.go new file mode 100644 index 0000000..59b0748 --- /dev/null +++ b/chains/txmgr/tracker.go @@ -0,0 +1,335 @@ +package txmgr + +import ( + "context" + "fmt" + "sync" + "time" + + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/services" + "github.com/smartcontractkit/chainlink-common/pkg/sqlutil" + "github.com/smartcontractkit/chainlink-common/pkg/utils/mailbox" + "github.com/smartcontractkit/chainlink-framework/chains" + + "github.com/smartcontractkit/chainlink-framework/chains/fees" + txmgrtypes "github.com/smartcontractkit/chainlink-framework/chains/txmgr/types" +) + +const ( + // defaultTTL is the default time to live for abandoned transactions + // After this TTL, the TXM stops tracking abandoned Txs. + defaultTTL = 6 * time.Hour + // handleTxesTimeout represents a sanity limit on how long handleTxesByState + // should take to complete + handleTxesTimeout = 10 * time.Minute + // batchSize is the number of txes to fetch from the txStore at once + batchSize = 1000 +) + +// Tracker tracks all transactions which have abandoned fromAddresses. +// The fromAddresses can be deleted by Node Operators from the KeyStore. In such cases, +// existing in-flight transactions for these fromAddresses are considered abandoned too. +// Since such Txs can still have attempts on chain's mempool, these could still be confirmed. +// This tracker just tracks such Txs for some time, in case they get confirmed as-is. +type Tracker[ + CHAIN_ID chains.ID, + ADDR chains.Hashable, + TX_HASH chains.Hashable, + BLOCK_HASH chains.Hashable, + R txmgrtypes.ChainReceipt[TX_HASH, BLOCK_HASH], + SEQ chains.Sequence, + FEE fees.Fee, +] struct { + services.StateMachine + txStore txmgrtypes.TxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] + keyStore txmgrtypes.KeyStore[ADDR, CHAIN_ID, SEQ] + chainID CHAIN_ID + lggr logger.Logger + + lock sync.Mutex + enabledAddrs map[ADDR]bool + txCache map[int64]ADDR // cache tx fromAddress by txID + + ttl time.Duration + mb *mailbox.Mailbox[int64] + + initSync sync.Mutex + wg sync.WaitGroup + chStop services.StopChan + isStarted bool +} + +func NewTracker[ + CHAIN_ID chains.ID, + ADDR chains.Hashable, + TX_HASH chains.Hashable, + BLOCK_HASH chains.Hashable, + R txmgrtypes.ChainReceipt[TX_HASH, BLOCK_HASH], + SEQ chains.Sequence, + FEE fees.Fee, +]( + txStore txmgrtypes.TxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE], + keyStore txmgrtypes.KeyStore[ADDR, CHAIN_ID, SEQ], + chainID CHAIN_ID, + lggr logger.Logger, +) *Tracker[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { + return &Tracker[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]{ + txStore: txStore, + keyStore: keyStore, + chainID: chainID, + lggr: logger.Named(lggr, "TxMgrTracker"), + enabledAddrs: map[ADDR]bool{}, + txCache: map[int64]ADDR{}, + ttl: defaultTTL, + mb: mailbox.NewSingle[int64](), + lock: sync.Mutex{}, + wg: sync.WaitGroup{}, + } +} + +func (tr *Tracker[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Start(ctx context.Context) (err error) { + tr.lggr.Info("Abandoned transaction tracking enabled") + return tr.StartOnce("Tracker", func() error { + return tr.startInternal(ctx) + }) +} + +func (tr *Tracker[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) startInternal(ctx context.Context) (err error) { + tr.initSync.Lock() + defer tr.initSync.Unlock() + + if err := tr.setEnabledAddresses(ctx); err != nil { + return fmt.Errorf("failed to set enabled addresses: %w", err) + } + tr.lggr.Infof("enabled addresses set for chainID %v", tr.chainID) + + tr.chStop = make(chan struct{}) + tr.wg.Add(1) + go tr.runLoop(tr.chStop.NewCtx()) + tr.isStarted = true + return nil +} + +func (tr *Tracker[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Close() error { + return tr.StopOnce("Tracker", func() error { + return tr.closeInternal() + }) +} + +func (tr *Tracker[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) closeInternal() error { + tr.initSync.Lock() + defer tr.initSync.Unlock() + + tr.lggr.Info("stopping tracker") + if !tr.isStarted { + return fmt.Errorf("tracker is not started: %w", services.ErrAlreadyStopped) + } + + close(tr.chStop) + tr.wg.Wait() + tr.isStarted = false + return nil +} + +func (tr *Tracker[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) runLoop(ctx context.Context, cancel context.CancelFunc) { + defer tr.wg.Done() + defer cancel() + + if err := tr.trackAbandonedTxes(ctx); err != nil { + tr.lggr.Errorf("failed to track abandoned txes: %v", err) + return + } + if err := tr.handleTxesByState(ctx); err != nil { + tr.lggr.Errorf("failed to handle txes by state: %v", err) + return + } + if tr.AbandonedTxCount() == 0 { + tr.lggr.Info("no abandoned txes found, skipping runLoop") + return + } + tr.lggr.Infof("%d abandoned txes found, starting runLoop", tr.AbandonedTxCount()) + + ttlExceeded := time.NewTicker(tr.ttl) + defer ttlExceeded.Stop() + for { + select { + case <-tr.mb.Notify(): + for { + blockHeight := tr.mb.RetrieveLatestAndClear() + if blockHeight == 0 { + break + } + if err := tr.handleTxesByState(ctx); err != nil { + tr.lggr.Errorf("failed to handle txes by state: %v", err) + return + } + if tr.AbandonedTxCount() == 0 { + tr.lggr.Info("all abandoned txes handled, stopping runLoop") + return + } + } + case <-ttlExceeded.C: + tr.lggr.Info("ttl exceeded") + tr.markAllTxesFatal(ctx) + return + case <-ctx.Done(): + return + } + } +} + +func (tr *Tracker[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) GetAbandonedAddresses() []ADDR { + tr.lock.Lock() + defer tr.lock.Unlock() + + abandonedAddrs := make([]ADDR, len(tr.txCache)) + for _, fromAddress := range tr.txCache { + abandonedAddrs = append(abandonedAddrs, fromAddress) + } + return abandonedAddrs +} + +// AbandonedTxCount returns the number of abandoned txes currently being tracked +func (tr *Tracker[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) AbandonedTxCount() int { + tr.lock.Lock() + defer tr.lock.Unlock() + return len(tr.txCache) +} + +func (tr *Tracker[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) IsStarted() bool { + tr.initSync.Lock() + defer tr.initSync.Unlock() + return tr.isStarted +} + +// setEnabledAddresses is called on startup to set the enabled addresses for the chain +func (tr *Tracker[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) setEnabledAddresses(ctx context.Context) error { + tr.lock.Lock() + defer tr.lock.Unlock() + + enabledAddrs, err := tr.keyStore.EnabledAddressesForChain(ctx, tr.chainID) + if err != nil { + return fmt.Errorf("failed to get enabled addresses for chain: %w", err) + } + + if len(enabledAddrs) == 0 { + tr.lggr.Warnf("enabled address list is empty") + } + + for _, addr := range enabledAddrs { + tr.enabledAddrs[addr] = true + } + return nil +} + +// trackAbandonedTxes called on startup to find and insert all abandoned txes into the tracker. +func (tr *Tracker[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) trackAbandonedTxes(ctx context.Context) (err error) { + return sqlutil.Batch(func(offset, limit uint) (count uint, err error) { + var enabledAddrs []ADDR + for addr := range tr.enabledAddrs { + enabledAddrs = append(enabledAddrs, addr) + } + + nonFatalTxes, err := tr.txStore.GetAbandonedTransactionsByBatch(ctx, tr.chainID, enabledAddrs, offset, limit) + if err != nil { + return 0, fmt.Errorf("failed to get non fatal txes from txStore: %w", err) + } + // insert abandoned txes + tr.lock.Lock() + for _, tx := range nonFatalTxes { + if !tr.enabledAddrs[tx.FromAddress] { + tr.txCache[tx.ID] = tx.FromAddress + tr.lggr.Debugf("inserted tx %v", tx.ID) + } + } + tr.lock.Unlock() + return uint(len(nonFatalTxes)), nil + }, batchSize) +} + +// handleTxesByState handles all txes in the txCache by their state +// It's called on every new blockHeight and also on startup to handle all txes in the txCache +func (tr *Tracker[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) handleTxesByState(ctx context.Context) error { + tr.lock.Lock() + defer tr.lock.Unlock() + ctx, cancel := context.WithTimeout(ctx, handleTxesTimeout) + defer cancel() + + for id := range tr.txCache { + if ctx.Err() != nil { + return ctx.Err() + } + + tx, err := tr.txStore.GetTxByID(ctx, id) + if err != nil { + tr.lggr.Errorf("failed to get tx by ID: %v", err) + continue + } + if tx == nil { + tr.lggr.Warnf("tx with ID %v no longer exists, removing from tracker", id) + delete(tr.txCache, id) + continue + } + + switch tx.State { + case TxConfirmed: + // TODO: Handle finalized state https://smartcontract-it.atlassian.net/browse/BCI-2920 + case TxConfirmedMissingReceipt, TxUnconfirmed: + // Keep tracking tx + case TxInProgress, TxUnstarted: + // Tx could never be sent on chain even once. That means that we need to sign + // an attempt to even broadcast this Tx to the chain. Since the fromAddress + // is deleted, we can't sign it. + errMsg := "The FromAddress for this Tx was deleted before this Tx could be broadcast to the chain." + if err := tr.markTxFatal(ctx, tx, errMsg); err != nil { + tr.lggr.Errorf("failed to mark tx as fatal: %v", err) + continue + } + delete(tr.txCache, id) + case TxFatalError: + delete(tr.txCache, id) + default: + tr.lggr.Errorf("unhandled transaction state: %v", tx.State) + } + } + + return nil +} + +func (tr *Tracker[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) markTxFatal(ctx context.Context, + tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], + errMsg string) error { + tx.Error.SetValid(errMsg) + + // Set state to TxInProgress so the tracker can attempt to mark it as fatal + tx.State = TxInProgress + if err := tr.txStore.UpdateTxFatalErrorAndDeleteAttempts(ctx, tx); err != nil { + return fmt.Errorf("failed to mark tx %v as abandoned: %w", tx.ID, err) + } + return nil +} + +// markAllTxesFatal tries to mark all txes in the txCache as fatal and removes them from the cache +func (tr *Tracker[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) markAllTxesFatal(ctx context.Context) { + tr.lock.Lock() + defer tr.lock.Unlock() + + errMsg := fmt.Sprintf( + "tx abandoned: fromAddress for this tx was deleted and existing attempts didn't finalize onchain within %d hours", + int(tr.ttl.Hours())) + + for id := range tr.txCache { + tx, err := tr.txStore.GetTxByID(ctx, id) + if err != nil { + tr.lggr.Errorf("failed to get tx by ID: %v", err) + delete(tr.txCache, id) + continue + } + + if err := tr.markTxFatal(ctx, tx, errMsg); err != nil { + tr.lggr.Errorf("failed to mark tx as abandoned: %v", err) + } + delete(tr.txCache, id) + } +} diff --git a/chains/txmgr/txmgr.go b/chains/txmgr/txmgr.go new file mode 100644 index 0000000..7e9fcc7 --- /dev/null +++ b/chains/txmgr/txmgr.go @@ -0,0 +1,810 @@ +package txmgr + +import ( + "context" + "errors" + "fmt" + "math/big" + "sync" + "time" + + "github.com/google/uuid" + "github.com/jpillora/backoff" + nullv4 "gopkg.in/guregu/null.v4" + + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/services" + commontypes "github.com/smartcontractkit/chainlink-common/pkg/types" + "github.com/smartcontractkit/chainlink-common/pkg/utils" + + "github.com/smartcontractkit/chainlink-framework/chains" + "github.com/smartcontractkit/chainlink-framework/chains/fees" + "github.com/smartcontractkit/chainlink-framework/chains/headtracker" + txmgrtypes "github.com/smartcontractkit/chainlink-framework/chains/txmgr/types" +) + +// For more information about the Txm architecture, see the design doc: +// https://www.notion.so/chainlink/Txm-Architecture-Overview-9dc62450cd7a443ba9e7dceffa1a8d6b + +// ResumeCallback is assumed to be idempotent +type ResumeCallback func(ctx context.Context, id uuid.UUID, result interface{}, err error) error + +type NewErrorClassifier func(err error) txmgrtypes.ErrorClassifier + +// TxManager is the main component of the transaction manager. +// It is also the interface to external callers. +type TxManager[ + CHAIN_ID chains.ID, + HEAD chains.Head[BLOCK_HASH], + ADDR chains.Hashable, + TX_HASH chains.Hashable, + BLOCK_HASH chains.Hashable, + SEQ chains.Sequence, + FEE fees.Fee, +] interface { + headtracker.HeadTrackable[HEAD, BLOCK_HASH] + services.Service + Trigger(addr ADDR) + CreateTransaction(ctx context.Context, txRequest txmgrtypes.TxRequest[ADDR, TX_HASH]) (etx txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) + GetForwarderForEOA(ctx context.Context, eoa ADDR) (forwarder ADDR, err error) + GetForwarderForEOAOCR2Feeds(ctx context.Context, eoa, ocr2AggregatorID ADDR) (forwarder ADDR, err error) + RegisterResumeCallback(fn ResumeCallback) + SendNativeToken(ctx context.Context, chainID CHAIN_ID, from, to ADDR, value big.Int, gasLimit uint64) (etx txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) + Reset(addr ADDR, abandon bool) error + // Find transactions by a field in the TxMeta blob and transaction states + FindTxesByMetaFieldAndStates(ctx context.Context, metaField string, metaValue string, states []txmgrtypes.TxState, chainID *big.Int) (txes []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) + // Find transactions with a non-null TxMeta field that was provided by transaction states + FindTxesWithMetaFieldByStates(ctx context.Context, metaField string, states []txmgrtypes.TxState, chainID *big.Int) (txes []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) + // Find transactions with a non-null TxMeta field that was provided and a receipt block number greater than or equal to the one provided + FindTxesWithMetaFieldByReceiptBlockNum(ctx context.Context, metaField string, blockNum int64, chainID *big.Int) (txes []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) + // Find transactions loaded with transaction attempts and receipts by transaction IDs and states + FindTxesWithAttemptsAndReceiptsByIdsAndState(ctx context.Context, ids []int64, states []txmgrtypes.TxState, chainID *big.Int) (txes []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) + FindEarliestUnconfirmedBroadcastTime(ctx context.Context) (nullv4.Time, error) + FindEarliestUnconfirmedTxAttemptBlock(ctx context.Context) (nullv4.Int, error) + CountTransactionsByState(ctx context.Context, state txmgrtypes.TxState) (count uint32, err error) + GetTransactionStatus(ctx context.Context, transactionID string) (state commontypes.TransactionStatus, err error) +} + +type reset struct { + // f is the function to execute between stopping/starting the + // Broadcaster and Confirmer + f func() + // done is either closed after running f, or returns error if f could not + // be run for some reason + done chan error +} + +type Txm[ + CHAIN_ID chains.ID, + HEAD chains.Head[BLOCK_HASH], + ADDR chains.Hashable, + TX_HASH chains.Hashable, + BLOCK_HASH chains.Hashable, + R txmgrtypes.ChainReceipt[TX_HASH, BLOCK_HASH], + SEQ chains.Sequence, + FEE fees.Fee, +] struct { + services.StateMachine + logger logger.SugaredLogger + txStore txmgrtypes.TxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] + config txmgrtypes.TransactionManagerChainConfig + txConfig txmgrtypes.TransactionManagerTransactionsConfig + keyStore txmgrtypes.KeyStore[ADDR, CHAIN_ID, SEQ] + chainID CHAIN_ID + checkerFactory TransmitCheckerFactory[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] + pruneQueueAndCreateLock sync.Mutex + + chHeads chan HEAD + trigger chan ADDR + reset chan reset + resumeCallback ResumeCallback + + chStop services.StopChan + chSubbed chan struct{} + wg sync.WaitGroup + + reaper *Reaper[CHAIN_ID] + resender *Resender[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE] + broadcaster *Broadcaster[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] + confirmer *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE] + tracker *Tracker[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE] + finalizer txmgrtypes.Finalizer[BLOCK_HASH, HEAD] + fwdMgr txmgrtypes.ForwarderManager[ADDR] + txAttemptBuilder txmgrtypes.TxAttemptBuilder[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] + newErrorClassifier NewErrorClassifier +} + +func (b *Txm[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) RegisterResumeCallback(fn ResumeCallback) { + b.resumeCallback = fn + b.broadcaster.SetResumeCallback(fn) + b.confirmer.SetResumeCallback(fn) + b.finalizer.SetResumeCallback(fn) +} + +// NewTxm creates a new Txm with the given configuration. +func NewTxm[ + CHAIN_ID chains.ID, + HEAD chains.Head[BLOCK_HASH], + ADDR chains.Hashable, + TX_HASH chains.Hashable, + BLOCK_HASH chains.Hashable, + R txmgrtypes.ChainReceipt[TX_HASH, BLOCK_HASH], + SEQ chains.Sequence, + FEE fees.Fee, +]( + chainId CHAIN_ID, + cfg txmgrtypes.TransactionManagerChainConfig, + txCfg txmgrtypes.TransactionManagerTransactionsConfig, + keyStore txmgrtypes.KeyStore[ADDR, CHAIN_ID, SEQ], + lggr logger.Logger, + checkerFactory TransmitCheckerFactory[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], + fwdMgr txmgrtypes.ForwarderManager[ADDR], + txAttemptBuilder txmgrtypes.TxAttemptBuilder[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], + txStore txmgrtypes.TxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE], + broadcaster *Broadcaster[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], + confirmer *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE], + resender *Resender[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE], + tracker *Tracker[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE], + finalizer txmgrtypes.Finalizer[BLOCK_HASH, HEAD], + newErrorClassifierFunc NewErrorClassifier, +) *Txm[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { + b := Txm[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]{ + logger: logger.Sugared(lggr), + txStore: txStore, + config: cfg, + txConfig: txCfg, + keyStore: keyStore, + chainID: chainId, + checkerFactory: checkerFactory, + chHeads: make(chan HEAD), + trigger: make(chan ADDR), + chStop: make(chan struct{}), + chSubbed: make(chan struct{}), + reset: make(chan reset), + fwdMgr: fwdMgr, + txAttemptBuilder: txAttemptBuilder, + broadcaster: broadcaster, + confirmer: confirmer, + resender: resender, + tracker: tracker, + newErrorClassifier: newErrorClassifierFunc, + finalizer: finalizer, + } + + if txCfg.ResendAfterThreshold() <= 0 { + b.logger.Info("Resender: Disabled") + } + if txCfg.ReaperThreshold() > 0 && txCfg.ReaperInterval() > 0 { + b.reaper = NewReaper[CHAIN_ID](lggr, b.txStore, txCfg, chainId) + } else { + b.logger.Info("TxReaper: Disabled") + } + + return &b +} + +// Start starts Txm service. +// The provided context can be used to terminate Start sequence. +func (b *Txm[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Start(ctx context.Context) (merr error) { + return b.StartOnce("Txm", func() error { + var ms services.MultiStart + if err := ms.Start(ctx, b.broadcaster); err != nil { + return fmt.Errorf("Txm: Broadcaster failed to start: %w", err) + } + if err := ms.Start(ctx, b.confirmer); err != nil { + return fmt.Errorf("Txm: Confirmer failed to start: %w", err) + } + + if err := ms.Start(ctx, b.txAttemptBuilder); err != nil { + return fmt.Errorf("Txm: Estimator failed to start: %w", err) + } + + if err := ms.Start(ctx, b.tracker); err != nil { + return fmt.Errorf("Txm: Tracker failed to start: %w", err) + } + + if err := ms.Start(ctx, b.finalizer); err != nil { + return fmt.Errorf("Txm: Finalizer failed to start: %w", err) + } + + b.logger.Info("Txm starting runLoop") + b.wg.Add(1) + go b.runLoop() + <-b.chSubbed + + if b.reaper != nil { + b.reaper.Start() + } + + if b.resender != nil { + b.resender.Start(ctx) + } + + if b.fwdMgr != nil { + if err := ms.Start(ctx, b.fwdMgr); err != nil { + return fmt.Errorf("Txm: ForwarderManager failed to start: %w", err) + } + } + + return nil + }) +} + +// Reset stops Broadcaster/Confirmer, executes callback, then starts them again +func (b *Txm[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Reset(addr ADDR, abandon bool) (err error) { + ok := b.IfStarted(func() { + done := make(chan error) + f := func() { + if abandon { + err = b.abandon(addr) + } + } + + b.reset <- reset{f, done} + err = <-done + }) + if !ok { + return errors.New("not started") + } + return err +} + +// abandon, scoped to the key of this txm: +// - marks all pending and inflight transactions fatally errored (note: at this point all transactions are either confirmed or fatally errored) +// this must not be run while Broadcaster or Confirmer are running +func (b *Txm[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) abandon(addr ADDR) (err error) { + ctx, cancel := b.chStop.NewCtx() + defer cancel() + if err = b.txStore.Abandon(ctx, b.chainID, addr); err != nil { + return fmt.Errorf("abandon failed to update txes for key %s: %w", addr.String(), err) + } + return nil +} + +func (b *Txm[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Close() (merr error) { + return b.StopOnce("Txm", func() error { + close(b.chStop) + + b.txStore.Close() + + if b.reaper != nil { + b.reaper.Stop() + } + if b.resender != nil { + b.resender.Stop() + } + if b.fwdMgr != nil { + if err := b.fwdMgr.Close(); err != nil { + merr = errors.Join(merr, fmt.Errorf("Txm: failed to stop ForwarderManager: %w", err)) + } + } + + b.wg.Wait() + + if err := b.txAttemptBuilder.Close(); err != nil { + merr = errors.Join(merr, fmt.Errorf("Txm: failed to close TxAttemptBuilder: %w", err)) + } + + return nil + }) +} + +func (b *Txm[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Name() string { + return b.logger.Name() +} + +func (b *Txm[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) HealthReport() map[string]error { + report := map[string]error{b.Name(): b.Healthy()} + + // only query if txm started properly + b.IfStarted(func() { + services.CopyHealth(report, b.broadcaster.HealthReport()) + services.CopyHealth(report, b.confirmer.HealthReport()) + services.CopyHealth(report, b.txAttemptBuilder.HealthReport()) + services.CopyHealth(report, b.finalizer.HealthReport()) + }) + + if b.txConfig.ForwardersEnabled() { + services.CopyHealth(report, b.fwdMgr.HealthReport()) + } + return report +} + +func (b *Txm[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) runLoop() { + ctx, cancel := b.chStop.NewCtx() + defer cancel() + + // eb, ec and keyStates can all be modified by the runloop. + // This is concurrent-safe because the runloop ensures serial access. + defer b.wg.Done() + keysChanged, unsub := b.keyStore.SubscribeToKeyChanges(ctx) + defer unsub() + + close(b.chSubbed) + + var stopped bool + var stopOnce sync.Once + + // execReset is defined as an inline function here because it closes over + // eb, ec and stopped + execReset := func(ctx context.Context, r *reset) { + // These should always close successfully, since it should be logically + // impossible to enter this code path with ec/eb in a state other than + // "Started" + if err := b.broadcaster.closeInternal(); err != nil { + b.logger.Panicw(fmt.Sprintf("Failed to Close Broadcaster: %v", err), "err", err) + } + if err := b.tracker.closeInternal(); err != nil { + b.logger.Panicw(fmt.Sprintf("Failed to Close Tracker: %v", err), "err", err) + } + if err := b.confirmer.closeInternal(); err != nil { + b.logger.Panicw(fmt.Sprintf("Failed to Close Confirmer: %v", err), "err", err) + } + if r != nil { + r.f() + close(r.done) + } + var wg sync.WaitGroup + // three goroutines to handle independent backoff retries starting: + // - Broadcaster + // - Confirmer + // - Tracker + // If chStop is closed, we mark stopped=true so that the main runloop + // can check and exit early if necessary + // + // execReset will not return until either: + // 1. Broadcaster, Confirmer, and Tracker all started successfully + // 2. chStop was closed (txmgr exit) + wg.Add(3) + go func() { + defer wg.Done() + // Retry indefinitely on failure + backoff := newRedialBackoff() + for { + select { + case <-time.After(backoff.Duration()): + if err := b.broadcaster.startInternal(ctx); err != nil { + b.logger.Criticalw("Failed to start Broadcaster", "err", err) + b.SvcErrBuffer.Append(err) + continue + } + return + case <-b.chStop: + stopOnce.Do(func() { stopped = true }) + return + } + } + }() + go func() { + defer wg.Done() + // Retry indefinitely on failure + backoff := newRedialBackoff() + for { + select { + case <-time.After(backoff.Duration()): + if err := b.tracker.startInternal(ctx); err != nil { + b.logger.Criticalw("Failed to start Tracker", "err", err) + b.SvcErrBuffer.Append(err) + continue + } + return + case <-b.chStop: + stopOnce.Do(func() { stopped = true }) + return + } + } + }() + go func() { + defer wg.Done() + // Retry indefinitely on failure + backoff := newRedialBackoff() + for { + select { + case <-time.After(backoff.Duration()): + if err := b.confirmer.startInternal(ctx); err != nil { + b.logger.Criticalw("Failed to start Confirmer", "err", err) + b.SvcErrBuffer.Append(err) + continue + } + return + case <-b.chStop: + stopOnce.Do(func() { stopped = true }) + return + } + } + }() + + wg.Wait() + } + + for { + select { + case address := <-b.trigger: + b.broadcaster.Trigger(address) + case head := <-b.chHeads: + b.confirmer.mb.Deliver(head) + b.tracker.mb.Deliver(head.BlockNumber()) + b.finalizer.DeliverLatestHead(head) + case reset := <-b.reset: + // This check prevents the weird edge-case where you can select + // into this block after chStop has already been closed and the + // previous reset exited early. + // In this case we do not want to reset again, we would rather go + // around and hit the stop case. + if stopped { + reset.done <- errors.New("Txm was stopped") + continue + } + execReset(ctx, &reset) + case <-b.chStop: + // close and exit + // + // Note that in some cases Broadcaster and/or Confirmer may + // be in an Unstarted state here, if execReset exited early. + // + // In this case, we don't care about stopping them since they are + // already "stopped". + err := b.broadcaster.Close() + if err != nil && (!errors.Is(err, services.ErrAlreadyStopped) || !errors.Is(err, services.ErrCannotStopUnstarted)) { + b.logger.Errorw(fmt.Sprintf("Failed to Close Broadcaster: %v", err), "err", err) + } + err = b.confirmer.Close() + if err != nil && (!errors.Is(err, services.ErrAlreadyStopped) || !errors.Is(err, services.ErrCannotStopUnstarted)) { + b.logger.Errorw(fmt.Sprintf("Failed to Close Confirmer: %v", err), "err", err) + } + err = b.tracker.Close() + if err != nil && (!errors.Is(err, services.ErrAlreadyStopped) || !errors.Is(err, services.ErrCannotStopUnstarted)) { + b.logger.Errorw(fmt.Sprintf("Failed to Close Tracker: %v", err), "err", err) + } + err = b.finalizer.Close() + if err != nil && (!errors.Is(err, services.ErrAlreadyStopped) || !errors.Is(err, services.ErrCannotStopUnstarted)) { + b.logger.Errorw(fmt.Sprintf("Failed to Close Finalizer: %v", err), "err", err) + } + return + case <-keysChanged: + // This check prevents the weird edge-case where you can select + // into this block after chStop has already been closed and the + // previous reset exited early. + // In this case we do not want to reset again, we would rather go + // around and hit the stop case. + if stopped { + continue + } + enabledAddresses, err := b.keyStore.EnabledAddressesForChain(ctx, b.chainID) + if err != nil { + b.logger.Critical("Failed to reload key states after key change") + b.SvcErrBuffer.Append(err) + continue + } + b.logger.Debugw("Keys changed, reloading", "enabledAddresses", enabledAddresses) + + execReset(ctx, nil) + } + } +} + +// OnNewLongestChain conforms to HeadTrackable +func (b *Txm[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) OnNewLongestChain(ctx context.Context, head HEAD) { + ok := b.IfStarted(func() { + if b.reaper != nil { + b.reaper.SetLatestBlockNum(head.BlockNumber()) + } + b.txAttemptBuilder.OnNewLongestChain(ctx, head) + select { + case b.chHeads <- head: + case <-ctx.Done(): + b.logger.Errorw("Timed out handling head", "blockNum", head.BlockNumber(), "ctxErr", ctx.Err()) + } + }) + if !ok { + b.logger.Debugw("Not started; ignoring head", "head", head, "state", b.State()) + } +} + +// Trigger forces the Broadcaster to check early for the given address +func (b *Txm[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Trigger(addr ADDR) { + select { + case b.trigger <- addr: + default: + } +} + +// CreateTransaction inserts a new transaction +func (b *Txm[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) CreateTransaction(ctx context.Context, txRequest txmgrtypes.TxRequest[ADDR, TX_HASH]) (tx txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) { + // Check for existing Tx with IdempotencyKey. If found, return the Tx and do nothing + // Skipping CreateTransaction to avoid double send + if txRequest.IdempotencyKey != nil { + var existingTx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] + existingTx, err = b.txStore.FindTxWithIdempotencyKey(ctx, *txRequest.IdempotencyKey, b.chainID) + if err != nil { + return tx, fmt.Errorf("Failed to search for transaction with IdempotencyKey: %w", err) + } + if existingTx != nil { + b.logger.Infow("Found a Tx with IdempotencyKey. Returning existing Tx without creating a new one.", "IdempotencyKey", *txRequest.IdempotencyKey) + return *existingTx, nil + } + } + + if err = b.checkEnabled(ctx, txRequest.FromAddress); err != nil { + return tx, err + } + + if b.txConfig.ForwardersEnabled() && (!utils.IsZero(txRequest.ForwarderAddress)) { + fwdPayload, fwdErr := b.fwdMgr.ConvertPayload(txRequest.ToAddress, txRequest.EncodedPayload) + if fwdErr == nil { + // Handling meta not set at caller. + if txRequest.Meta != nil { + txRequest.Meta.FwdrDestAddress = &txRequest.ToAddress + } else { + txRequest.Meta = &txmgrtypes.TxMeta[ADDR, TX_HASH]{ + FwdrDestAddress: &txRequest.ToAddress, + } + } + txRequest.ToAddress = txRequest.ForwarderAddress + txRequest.EncodedPayload = fwdPayload + } else { + b.logger.Errorf("Failed to use forwarder set upstream: %v", fwdErr.Error()) + } + } + + err = b.txStore.CheckTxQueueCapacity(ctx, txRequest.FromAddress, b.txConfig.MaxQueued(), b.chainID) + if err != nil { + return tx, fmt.Errorf("Txm#CreateTransaction: %w", err) + } + + tx, err = b.pruneQueueAndCreateTxn(ctx, txRequest, b.chainID) + if err != nil { + return tx, err + } + + // Trigger the Broadcaster to check for new transaction + b.broadcaster.Trigger(txRequest.FromAddress) + + return tx, nil +} + +// Calls forwarderMgr to get a proper forwarder for a given EOA. +func (b *Txm[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) GetForwarderForEOA(ctx context.Context, eoa ADDR) (forwarder ADDR, err error) { + if !b.txConfig.ForwardersEnabled() { + return forwarder, fmt.Errorf("forwarding is not enabled, to enable set Transactions.ForwardersEnabled =true") + } + forwarder, err = b.fwdMgr.ForwarderFor(ctx, eoa) + return +} + +// GetForwarderForEOAOCR2Feeds calls forwarderMgr to get a proper forwarder for a given EOA and checks if its set as a transmitter on the OCR2Aggregator contract. +func (b *Txm[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) GetForwarderForEOAOCR2Feeds(ctx context.Context, eoa, ocr2Aggregator ADDR) (forwarder ADDR, err error) { + if !b.txConfig.ForwardersEnabled() { + return forwarder, fmt.Errorf("forwarding is not enabled, to enable set Transactions.ForwardersEnabled =true") + } + forwarder, err = b.fwdMgr.ForwarderForOCR2Feeds(ctx, eoa, ocr2Aggregator) + return +} + +func (b *Txm[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) checkEnabled(ctx context.Context, addr ADDR) error { + if err := b.keyStore.CheckEnabled(ctx, addr, b.chainID); err != nil { + return fmt.Errorf("cannot send transaction from %s on chain ID %s: %w", addr, b.chainID.String(), err) + } + return nil +} + +// SendNativeToken creates a transaction that transfers the given value of native tokens +func (b *Txm[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SendNativeToken(ctx context.Context, chainID CHAIN_ID, from, to ADDR, value big.Int, gasLimit uint64) (etx txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) { + if utils.IsZero(to) { + return etx, errors.New("cannot send native token to zero address") + } + txRequest := txmgrtypes.TxRequest[ADDR, TX_HASH]{ + FromAddress: from, + ToAddress: to, + EncodedPayload: []byte{}, + Value: value, + FeeLimit: gasLimit, + Strategy: NewSendEveryStrategy(), + } + etx, err = b.pruneQueueAndCreateTxn(ctx, txRequest, chainID) + if err != nil { + return etx, fmt.Errorf("SendNativeToken failed to insert tx: %w", err) + } + + // Trigger the Broadcaster to check for new transaction + b.broadcaster.Trigger(from) + return etx, nil +} + +func (b *Txm[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxesByMetaFieldAndStates(ctx context.Context, metaField string, metaValue string, states []txmgrtypes.TxState, chainID *big.Int) (txes []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) { + txes, err = b.txStore.FindTxesByMetaFieldAndStates(ctx, metaField, metaValue, states, chainID) + return +} + +func (b *Txm[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxesWithMetaFieldByStates(ctx context.Context, metaField string, states []txmgrtypes.TxState, chainID *big.Int) (txes []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) { + txes, err = b.txStore.FindTxesWithMetaFieldByStates(ctx, metaField, states, chainID) + return +} + +func (b *Txm[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxesWithMetaFieldByReceiptBlockNum(ctx context.Context, metaField string, blockNum int64, chainID *big.Int) (txes []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) { + txes, err = b.txStore.FindTxesWithMetaFieldByReceiptBlockNum(ctx, metaField, blockNum, chainID) + return +} + +func (b *Txm[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxesWithAttemptsAndReceiptsByIdsAndState(ctx context.Context, ids []int64, states []txmgrtypes.TxState, chainID *big.Int) (txes []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) { + txes, err = b.txStore.FindTxesWithAttemptsAndReceiptsByIdsAndState(ctx, ids, states, chainID) + return +} + +func (b *Txm[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindEarliestUnconfirmedBroadcastTime(ctx context.Context) (nullv4.Time, error) { + return b.txStore.FindEarliestUnconfirmedBroadcastTime(ctx, b.chainID) +} + +func (b *Txm[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindEarliestUnconfirmedTxAttemptBlock(ctx context.Context) (nullv4.Int, error) { + return b.txStore.FindEarliestUnconfirmedTxAttemptBlock(ctx, b.chainID) +} + +func (b *Txm[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) CountTransactionsByState(ctx context.Context, state txmgrtypes.TxState) (count uint32, err error) { + return b.txStore.CountTransactionsByState(ctx, state, b.chainID) +} + +func (b *Txm[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) GetTransactionStatus(ctx context.Context, transactionID string) (status commontypes.TransactionStatus, err error) { + // Loads attempts and receipts in the transaction + tx, err := b.txStore.FindTxWithIdempotencyKey(ctx, transactionID, b.chainID) + if err != nil { + return status, fmt.Errorf("failed to find transaction with IdempotencyKey %s: %w", transactionID, err) + } + // This check is required since a no-rows error returns nil err + if tx == nil { + return status, fmt.Errorf("failed to find transaction with IdempotencyKey %s", transactionID) + } + switch tx.State { + case TxUnconfirmed, TxConfirmedMissingReceipt: + // Return pending for ConfirmedMissingReceipt since a receipt is required to consider it as unconfirmed + return commontypes.Pending, nil + case TxConfirmed: + // Return unconfirmed for confirmed transactions because they are not yet finalized + return commontypes.Unconfirmed, nil + case TxFinalized: + return commontypes.Finalized, nil + case TxFatalError: + // Use an ErrorClassifier to determine if the transaction is considered Fatal + txErr := b.newErrorClassifier(tx.GetError()) + if txErr != nil && txErr.IsFatal() { + return commontypes.Fatal, tx.GetError() + } + // Return failed for all other tx's marked as FatalError + return commontypes.Failed, tx.GetError() + default: + // Unstarted and InProgress are classified as unknown since they are not supported by the ChainWriter interface + return commontypes.Unknown, nil + } +} + +type NullTxManager[ + CHAIN_ID chains.ID, + HEAD chains.Head[BLOCK_HASH], + ADDR chains.Hashable, + TX_HASH, BLOCK_HASH chains.Hashable, + SEQ chains.Sequence, + FEE fees.Fee, +] struct { + ErrMsg string +} + +func (n *NullTxManager[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) OnNewLongestChain(context.Context, HEAD) { +} + +// Start does noop for NullTxManager. +func (n *NullTxManager[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) Start(context.Context) error { + return nil +} + +// Close does noop for NullTxManager. +func (n *NullTxManager[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) Close() error { + return nil +} + +// Trigger does noop for NullTxManager. +func (n *NullTxManager[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) Trigger(ADDR) { + panic(n.ErrMsg) +} +func (n *NullTxManager[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) CreateTransaction(ctx context.Context, txRequest txmgrtypes.TxRequest[ADDR, TX_HASH]) (etx txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) { + return etx, errors.New(n.ErrMsg) +} +func (n *NullTxManager[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) GetForwarderForEOA(ctx context.Context, addr ADDR) (fwdr ADDR, err error) { + return fwdr, err +} +func (n *NullTxManager[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) GetForwarderForEOAOCR2Feeds(ctx context.Context, _, _ ADDR) (fwdr ADDR, err error) { + return fwdr, err +} + +func (n *NullTxManager[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) Reset(addr ADDR, abandon bool) error { + return nil +} + +// SendNativeToken does nothing, null functionality +func (n *NullTxManager[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) SendNativeToken(ctx context.Context, chainID CHAIN_ID, from, to ADDR, value big.Int, gasLimit uint64) (etx txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) { + return etx, errors.New(n.ErrMsg) +} + +func (n *NullTxManager[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) Ready() error { + return nil +} +func (n *NullTxManager[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) Name() string { + return "NullTxManager" +} +func (n *NullTxManager[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) HealthReport() map[string]error { + return map[string]error{} +} +func (n *NullTxManager[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) RegisterResumeCallback(fn ResumeCallback) { +} +func (n *NullTxManager[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) FindTxesByMetaFieldAndStates(ctx context.Context, metaField string, metaValue string, states []txmgrtypes.TxState, chainID *big.Int) (txes []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) { + return txes, errors.New(n.ErrMsg) +} +func (n *NullTxManager[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) FindTxesWithMetaFieldByStates(ctx context.Context, metaField string, states []txmgrtypes.TxState, chainID *big.Int) (txes []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) { + return txes, errors.New(n.ErrMsg) +} +func (n *NullTxManager[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) FindTxesWithMetaFieldByReceiptBlockNum(ctx context.Context, metaField string, blockNum int64, chainID *big.Int) (txes []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) { + return txes, errors.New(n.ErrMsg) +} +func (n *NullTxManager[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) FindTxesWithAttemptsAndReceiptsByIdsAndState(ctx context.Context, ids []int64, states []txmgrtypes.TxState, chainID *big.Int) (txes []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) { + return txes, errors.New(n.ErrMsg) +} + +func (n *NullTxManager[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) FindEarliestUnconfirmedBroadcastTime(ctx context.Context) (nullv4.Time, error) { + return nullv4.Time{}, errors.New(n.ErrMsg) +} + +func (n *NullTxManager[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) FindEarliestUnconfirmedTxAttemptBlock(ctx context.Context) (nullv4.Int, error) { + return nullv4.Int{}, errors.New(n.ErrMsg) +} + +func (n *NullTxManager[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) CountTransactionsByState(ctx context.Context, state txmgrtypes.TxState) (count uint32, err error) { + return count, errors.New(n.ErrMsg) +} + +func (n *NullTxManager[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) GetTransactionStatus(ctx context.Context, transactionID string) (status commontypes.TransactionStatus, err error) { + return +} + +func (b *Txm[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) pruneQueueAndCreateTxn( + ctx context.Context, + txRequest txmgrtypes.TxRequest[ADDR, TX_HASH], + chainID CHAIN_ID, +) ( + tx txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], + err error, +) { + b.pruneQueueAndCreateLock.Lock() + defer b.pruneQueueAndCreateLock.Unlock() + + pruned, err := txRequest.Strategy.PruneQueue(ctx, b.txStore) + if err != nil { + return tx, err + } + if len(pruned) > 0 { + b.logger.Warnw(fmt.Sprintf("Pruned %d old unstarted transactions", len(pruned)), + "subject", txRequest.Strategy.Subject(), + "pruned-tx-ids", pruned, + ) + } + + tx, err = b.txStore.CreateTransaction(ctx, txRequest, chainID) + if err != nil { + return tx, err + } + b.logger.Debugw("Created transaction", + "fromAddress", txRequest.FromAddress, + "toAddress", txRequest.ToAddress, + "meta", txRequest.Meta, + "transactionID", tx.ID, + ) + + return tx, nil +} + +// newRedialBackoff is a standard backoff to use for redialling or reconnecting to +// unreachable network endpoints +func newRedialBackoff() backoff.Backoff { + return backoff.Backoff{ + Min: 1 * time.Second, + Max: 15 * time.Second, + Jitter: true, + } +} diff --git a/chains/txmgr/types/client.go b/chains/txmgr/types/client.go new file mode 100644 index 0000000..0862746 --- /dev/null +++ b/chains/txmgr/types/client.go @@ -0,0 +1,88 @@ +package types + +import ( + "context" + "fmt" + "math/big" + "time" + + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-framework/chains/fees" + "github.com/smartcontractkit/chainlink-framework/multinode" + + "github.com/smartcontractkit/chainlink-framework/chains" +) + +// TxmClient is a superset of all the methods needed for the txm +type TxmClient[ + CHAIN_ID chains.ID, + ADDR chains.Hashable, + TX_HASH chains.Hashable, + BLOCK_HASH chains.Hashable, + R ChainReceipt[TX_HASH, BLOCK_HASH], + SEQ chains.Sequence, + FEE fees.Fee, +] interface { + ChainClient[CHAIN_ID, ADDR, SEQ] + TransactionClient[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] + + // receipt fetching used by confirmer + BatchGetReceipts( + ctx context.Context, + attempts []TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], + ) (txReceipt []R, txErr []error, err error) +} + +// TransactionClient contains the methods for building, simulating, broadcasting transactions +type TransactionClient[ + CHAIN_ID chains.ID, + ADDR chains.Hashable, + TX_HASH chains.Hashable, + BLOCK_HASH chains.Hashable, + SEQ chains.Sequence, + FEE fees.Fee, +] interface { + ChainClient[CHAIN_ID, ADDR, SEQ] + + BatchSendTransactions( + ctx context.Context, + attempts []TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], + bathSize int, + lggr logger.SugaredLogger, + ) ( + txCodes []multinode.SendTxReturnCode, + txErrs []error, + broadcastTime time.Time, + successfulTxIDs []int64, + err error) + SendTransactionReturnCode( + ctx context.Context, + tx Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], + attempt TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], + lggr logger.SugaredLogger, + ) (multinode.SendTxReturnCode, error) + SendEmptyTransaction( + ctx context.Context, + newTxAttempt func(ctx context.Context, seq SEQ, feeLimit uint64, fee FEE, fromAddress ADDR) (attempt TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error), + seq SEQ, + gasLimit uint64, + fee FEE, + fromAddress ADDR, + ) (txhash string, err error) + CallContract( + ctx context.Context, + attempt TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], + blockNumber *big.Int, + ) (rpcErr fmt.Stringer, extractErr error) +} + +// ChainClient contains the interfaces for reading chain parameters (chain id, sequences, etc) +type ChainClient[ + CHAIN_ID chains.ID, + ADDR chains.Hashable, + SEQ chains.Sequence, +] interface { + ConfiguredChainID() CHAIN_ID + PendingSequenceAt(ctx context.Context, addr ADDR) (SEQ, error) + SequenceAt(ctx context.Context, addr ADDR, blockNum *big.Int) (SEQ, error) +} diff --git a/chains/txmgr/types/config.go b/chains/txmgr/types/config.go new file mode 100644 index 0000000..1ab334b --- /dev/null +++ b/chains/txmgr/types/config.go @@ -0,0 +1,72 @@ +package types + +import "time" + +type TransactionManagerChainConfig interface { + BroadcasterChainConfig +} + +type TransactionManagerFeeConfig interface { + BroadcasterFeeConfig + ConfirmerFeeConfig +} + +type TransactionManagerTransactionsConfig interface { + BroadcasterTransactionsConfig + ConfirmerTransactionsConfig + ResenderTransactionsConfig + ReaperTransactionsConfig + + ForwardersEnabled() bool + MaxQueued() uint64 +} + +type BroadcasterChainConfig interface { + IsL2() bool +} + +type BroadcasterFeeConfig interface { + MaxFeePrice() string // logging value + FeePriceDefault() string // logging value +} + +type BroadcasterTransactionsConfig interface { + MaxInFlight() uint32 +} + +type BroadcasterListenerConfig interface { + FallbackPollInterval() time.Duration +} + +type ConfirmerFeeConfig interface { + BumpTxDepth() uint32 + LimitDefault() uint64 + + // from gas.Config + BumpThreshold() uint64 + MaxFeePrice() string // logging value +} + +type ConfirmerDatabaseConfig interface { + // from pg.QConfig + DefaultQueryTimeout() time.Duration +} + +type ConfirmerTransactionsConfig interface { + MaxInFlight() uint32 + ForwardersEnabled() bool +} + +type ResenderChainConfig interface { + RPCDefaultBatchSize() uint32 +} + +type ResenderTransactionsConfig interface { + ResendAfterThreshold() time.Duration + MaxInFlight() uint32 +} + +type ReaperTransactionsConfig interface { + ReaperInterval() time.Duration + ReaperThreshold() time.Duration +} diff --git a/chains/txmgr/types/finalizer.go b/chains/txmgr/types/finalizer.go new file mode 100644 index 0000000..ceb2d56 --- /dev/null +++ b/chains/txmgr/types/finalizer.go @@ -0,0 +1,17 @@ +package types + +import ( + "context" + + "github.com/google/uuid" + + "github.com/smartcontractkit/chainlink-common/pkg/services" + "github.com/smartcontractkit/chainlink-framework/chains" +) + +type Finalizer[BLOCK_HASH chains.Hashable, HEAD chains.Head[BLOCK_HASH]] interface { + // interfaces for running the underlying estimator + services.Service + DeliverLatestHead(head HEAD) bool + SetResumeCallback(callback func(ctx context.Context, id uuid.UUID, result interface{}, err error) error) +} diff --git a/chains/txmgr/types/forwarder_manager.go b/chains/txmgr/types/forwarder_manager.go new file mode 100644 index 0000000..9b9eef1 --- /dev/null +++ b/chains/txmgr/types/forwarder_manager.go @@ -0,0 +1,17 @@ +package types + +import ( + "context" + + "github.com/smartcontractkit/chainlink-common/pkg/services" + + "github.com/smartcontractkit/chainlink-framework/chains" +) + +type ForwarderManager[ADDR chains.Hashable] interface { + services.Service + ForwarderFor(ctx context.Context, addr ADDR) (forwarder ADDR, err error) + ForwarderForOCR2Feeds(ctx context.Context, eoa, ocr2Aggregator ADDR) (forwarder ADDR, err error) + // Converts payload to be forwarder-friendly + ConvertPayload(dest ADDR, origPayload []byte) ([]byte, error) +} diff --git a/chains/txmgr/types/keystore.go b/chains/txmgr/types/keystore.go new file mode 100644 index 0000000..a37bb11 --- /dev/null +++ b/chains/txmgr/types/keystore.go @@ -0,0 +1,21 @@ +package types + +import ( + "context" + + "github.com/smartcontractkit/chainlink-framework/chains" +) + +// KeyStore encompasses the subset of keystore used by txmgr +type KeyStore[ + // Account Address type. + ADDR chains.Hashable, + // Chain ID type + CHAIN_ID chains.ID, + // Chain's sequence type. For example, EVM chains use nonce, bitcoin uses UTXO. + SEQ chains.Sequence, +] interface { + CheckEnabled(ctx context.Context, address ADDR, chainID CHAIN_ID) error + EnabledAddressesForChain(ctx context.Context, chainId CHAIN_ID) ([]ADDR, error) + SubscribeToKeyChanges(ctx context.Context) (ch chan struct{}, unsub func()) +} diff --git a/chains/txmgr/types/sequence_tracker.go b/chains/txmgr/types/sequence_tracker.go new file mode 100644 index 0000000..6a28ad7 --- /dev/null +++ b/chains/txmgr/types/sequence_tracker.go @@ -0,0 +1,26 @@ +package types + +import ( + "context" + + "github.com/smartcontractkit/chainlink-common/pkg/services" + "github.com/smartcontractkit/chainlink-framework/chains" +) + +type SequenceTracker[ + // Represents an account address, in native chain format. + ADDR chains.Hashable, + // Represents the sequence type for a chain. For example, nonce for EVM. + SEQ chains.Sequence, +] interface { + // Load the next sequence needed for transactions for all enabled addresses + LoadNextSequences(context.Context, []ADDR) + // Get the next sequence to assign to a transaction + GetNextSequence(context.Context, ADDR) (SEQ, error) + // Signals the existing sequence has been used so generates and stores the next sequence + // Can be a no-op depending on the chain + GenerateNextSequence(ADDR, SEQ) + // Syncs the local sequence with the one on-chain in case the address as been used externally + // Can be a no-op depending on the chain + SyncSequence(context.Context, ADDR, services.StopChan) +} diff --git a/chains/txmgr/types/stuck_tx_detector.go b/chains/txmgr/types/stuck_tx_detector.go new file mode 100644 index 0000000..70f96c5 --- /dev/null +++ b/chains/txmgr/types/stuck_tx_detector.go @@ -0,0 +1,26 @@ +package types + +import ( + "context" + + "github.com/smartcontractkit/chainlink-framework/chains" + "github.com/smartcontractkit/chainlink-framework/chains/fees" +) + +// StuckTxDetector is used by the Confirmer to determine if any unconfirmed transactions are terminally stuck +type StuckTxDetector[ +CHAIN_ID chains.ID, // CHAIN_ID - chain id type +ADDR chains.Hashable, // ADDR - chain address type +TX_HASH, BLOCK_HASH chains.Hashable, // various chain hash types +SEQ chains.Sequence, // SEQ - chain sequence type (nonce, utxo, etc) +FEE fees.Fee, // FEE - chain fee type +] interface { + // Uses either a chain specific API or heuristic to determine if any unconfirmed transactions are terminally stuck. Returns only one transaction per enabled address. + DetectStuckTransactions(ctx context.Context, enabledAddresses []ADDR, blockNum int64) ([]Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) + // Loads the internal map that tracks the last block num a transaction was purged at using the DB state + LoadPurgeBlockNumMap(ctx context.Context, addresses []ADDR) error + // Sets the last purged block num after a transaction has been successfully purged with receipt + SetPurgeBlockNum(fromAddress ADDR, blockNum int64) + // Returns the error message to set in the transaction error field to mark it as terminally stuck + StuckTxFatalError() string +} diff --git a/chains/txmgr/types/tx.go b/chains/txmgr/types/tx.go new file mode 100644 index 0000000..e87ae6b --- /dev/null +++ b/chains/txmgr/types/tx.go @@ -0,0 +1,359 @@ +package types + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "math/big" + "slices" + "strings" + "time" + + "github.com/google/uuid" + "gopkg.in/guregu/null.v4" + + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/sqlutil" + clnull "github.com/smartcontractkit/chainlink-common/pkg/utils/null" + + "github.com/smartcontractkit/chainlink-framework/chains" + "github.com/smartcontractkit/chainlink-framework/chains/fees" +) + +// TxStrategy controls how txes are queued and sent +type TxStrategy interface { + // Subject will be saved txes.subject if not null + Subject() uuid.NullUUID + // PruneQueue is called after tx insertion + // It accepts the service responsible for deleting + // unstarted txs and deletion options + PruneQueue(ctx context.Context, pruneService UnstartedTxQueuePruner) (ids []int64, err error) +} + +type TxAttemptState int8 + +type TxState string + +const ( + TxAttemptInProgress TxAttemptState = iota + 1 + TxAttemptInsufficientFunds + TxAttemptBroadcast + txAttemptStateCount // always at end to calculate number of states +) + +var txAttemptStateStrings = []string{ + "unknown_attempt_state", // default 0 value + TxAttemptInProgress: "in_progress", + TxAttemptInsufficientFunds: "insufficient_funds", + TxAttemptBroadcast: "broadcast", +} + +func NewTxAttemptState(state string) (s TxAttemptState) { + if index := slices.Index(txAttemptStateStrings, state); index != -1 { + s = TxAttemptState(index) + } + return s +} + +// String returns string formatted states for logging +func (s TxAttemptState) String() (str string) { + if s < txAttemptStateCount { + return txAttemptStateStrings[s] + } + return txAttemptStateStrings[0] +} + +type TxRequest[ADDR chains.Hashable, TX_HASH chains.Hashable] struct { + // IdempotencyKey is a globally unique ID set by the caller, to prevent accidental creation of duplicated Txs during retries or crash recovery. + // If this field is set, the TXM will first search existing Txs with this field. + // If found, it will return the existing Tx, without creating a new one. TXM will not validate or ensure that existing Tx is same as the incoming TxRequest. + // If not found, TXM will create a new Tx. + // If IdempotencyKey is set to null, TXM will always create a new Tx. + // Since IdempotencyKey has to be globally unique, consider prepending the service or component's name it is being used by + // Such as {service}-{ID}. E.g vrf-12345 + IdempotencyKey *string + FromAddress ADDR + ToAddress ADDR + EncodedPayload []byte + Value big.Int + FeeLimit uint64 + Meta *TxMeta[ADDR, TX_HASH] + ForwarderAddress ADDR + + // Pipeline variables - if you aren't calling this from chain tx task within + // the pipeline, you don't need these variables + MinConfirmations clnull.Uint32 + PipelineTaskRunID *uuid.UUID + + Strategy TxStrategy + + // Checker defines the check that should be run before a transaction is submitted on chain. + Checker TransmitCheckerSpec[ADDR] + + // Mark tx requiring callback + SignalCallback bool +} + +// TransmitCheckerSpec defines the check that should be performed before a transaction is submitted +// on chain. +type TransmitCheckerSpec[ADDR chains.Hashable] struct { + // CheckerType is the type of check that should be performed. Empty indicates no check. + CheckerType TransmitCheckerType `json:",omitempty"` + + // VRFCoordinatorAddress is the address of the VRF coordinator that should be used to perform + // VRF transmit checks. This should be set iff CheckerType is TransmitCheckerTypeVRFV2. + VRFCoordinatorAddress *ADDR `json:",omitempty"` + + // VRFRequestBlockNumber is the block number in which the provided VRF request has been made. + // This should be set iff CheckerType is TransmitCheckerTypeVRFV2. + VRFRequestBlockNumber *big.Int `json:",omitempty"` +} + +// TransmitCheckerType describes the type of check that should be performed before a transaction is +// executed on-chain. +type TransmitCheckerType string + +// TxMeta contains fields of the transaction metadata +// Not all fields are guaranteed to be present +type TxMeta[ADDR chains.Hashable, TX_HASH chains.Hashable] struct { + JobID *int32 `json:"JobID,omitempty"` + + // Pipeline fields + FailOnRevert null.Bool `json:"FailOnRevert,omitempty"` + + // VRF-only fields + RequestID *TX_HASH `json:"RequestID,omitempty"` + RequestTxHash *TX_HASH `json:"RequestTxHash,omitempty"` + // Batch variants of the above + RequestIDs []TX_HASH `json:"RequestIDs,omitempty"` + RequestTxHashes []TX_HASH `json:"RequestTxHashes,omitempty"` + // Used for the VRFv2 - max link this tx will bill + // should it get bumped + MaxLink *string `json:"MaxLink,omitempty"` + // Used for the VRFv2 - the subscription ID of the + // requester of the VRF. + SubID *uint64 `json:"SubId,omitempty"` + // Used for the VRFv2Plus - the uint256 subscription ID of the + // requester of the VRF. + GlobalSubID *string `json:"GlobalSubId,omitempty"` + // Used for VRFv2Plus - max native token this tx will bill + // should it get bumped + MaxEth *string `json:"MaxEth,omitempty"` + + // Used for keepers + UpkeepID *string `json:"UpkeepID,omitempty"` + + // Used for VRF to know if the txn is a ForceFulfilment txn + ForceFulfilled *bool `json:"ForceFulfilled,omitempty"` + ForceFulfillmentAttempt *uint64 `json:"ForceFulfillmentAttempt,omitempty"` + + // Used for Keystone Workflows + WorkflowExecutionID *string `json:"WorkflowExecutionID,omitempty"` + + // Used only for forwarded txs, tracks the original destination address. + // When this is set, it indicates tx is forwarded through To address. + FwdrDestAddress *ADDR `json:"ForwarderDestAddress,omitempty"` + + // MessageIDs is used by CCIP for tx to executed messages correlation in logs + MessageIDs []string `json:"MessageIDs,omitempty"` + // SeqNumbers is used by CCIP for tx to committed sequence numbers correlation in logs + SeqNumbers []uint64 `json:"SeqNumbers,omitempty"` + + // Dual Broadcast + DualBroadcast *bool `json:"DualBroadcast,omitempty"` + DualBroadcastParams *string `json:"DualBroadcastParams,omitempty"` +} + +type TxAttempt[ + CHAIN_ID chains.ID, + ADDR chains.Hashable, + TX_HASH, BLOCK_HASH chains.Hashable, + SEQ chains.Sequence, + FEE fees.Fee, +] struct { + ID int64 + TxID int64 + Tx Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] + TxFee FEE + // ChainSpecificFeeLimit on the TxAttempt is always the same as the on-chain encoded value for fee limit + ChainSpecificFeeLimit uint64 + SignedRawTx []byte + Hash TX_HASH + CreatedAt time.Time + BroadcastBeforeBlockNum *int64 + State TxAttemptState + Receipts []ChainReceipt[TX_HASH, BLOCK_HASH] `json:"-"` + TxType int + IsPurgeAttempt bool +} + +func (a *TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) String() string { + return fmt.Sprintf("TxAttempt(ID:%d,TxID:%d,Fee:%s,TxType:%d", a.ID, a.TxID, a.TxFee, a.TxType) +} + +type Tx[ + CHAIN_ID chains.ID, + ADDR chains.Hashable, + TX_HASH, BLOCK_HASH chains.Hashable, + SEQ chains.Sequence, + FEE fees.Fee, +] struct { + ID int64 + IdempotencyKey *string + Sequence *SEQ + FromAddress ADDR + ToAddress ADDR + EncodedPayload []byte + Value big.Int + // FeeLimit on the Tx is always the conceptual gas limit, which is not + // necessarily the same as the on-chain encoded value (i.e. Optimism) + FeeLimit uint64 + Error null.String + // BroadcastAt is updated every time an attempt for this tx is re-sent + // In almost all cases it will be within a second or so of the actual send time. + BroadcastAt *time.Time + // InitialBroadcastAt is recorded once, the first ever time this tx is sent + InitialBroadcastAt *time.Time + CreatedAt time.Time + State TxState + TxAttempts []TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] `json:"-"` + // Marshalled TxMeta + // Used for additional context around transactions which you want to log + // at send time. + Meta *sqlutil.JSON + Subject uuid.NullUUID + ChainID CHAIN_ID + + PipelineTaskRunID uuid.NullUUID + MinConfirmations clnull.Uint32 + + // TransmitChecker defines the check that should be performed before a transaction is submitted on + // chain. + TransmitChecker *sqlutil.JSON + + // Marks tx requiring callback + SignalCallback bool + // Marks tx callback as signaled + CallbackCompleted bool +} + +func (e *Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) GetError() error { + if e.Error.Valid { + return errors.New(e.Error.String) + } + return nil +} + +// GetID allows Tx to be used as jsonapi.MarshalIdentifier +func (e *Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) GetID() string { + return fmt.Sprintf("%d", e.ID) +} + +// GetMeta returns an Tx's meta in struct form, unmarshalling it from JSON first. +func (e *Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) GetMeta() (*TxMeta[ADDR, TX_HASH], error) { + if e.Meta == nil { + return nil, nil + } + var m TxMeta[ADDR, TX_HASH] + if err := json.Unmarshal(*e.Meta, &m); err != nil { + return nil, fmt.Errorf("unmarshalling meta: %w", err) + } + + return &m, nil +} + +// GetLogger returns a new logger with metadata fields. +func (e *Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) GetLogger(lgr logger.Logger) logger.SugaredLogger { + lgr = logger.With(lgr, + "txID", e.ID, + "sequence", e.Sequence, + "checker", e.TransmitChecker, + "feeLimit", e.FeeLimit, + ) + + meta, err := e.GetMeta() + if err != nil { + lgr.Errorw("failed to get meta of the transaction", "err", err) + return logger.Sugared(lgr) + } + + if meta != nil { + lgr = logger.With(lgr, "jobID", meta.JobID) + + if meta.RequestTxHash != nil { + lgr = logger.With(lgr, "requestTxHash", *meta.RequestTxHash) + } + + if meta.RequestID != nil { + id := *meta.RequestID + lgr = logger.With(lgr, "requestID", new(big.Int).SetBytes(id.Bytes()).String()) + } + + if len(meta.RequestIDs) != 0 { + var ids []string + for _, id := range meta.RequestIDs { + ids = append(ids, new(big.Int).SetBytes(id.Bytes()).String()) + } + lgr = logger.With(lgr, "requestIDs", strings.Join(ids, ",")) + } + + if meta.UpkeepID != nil { + lgr = logger.With(lgr, "upkeepID", *meta.UpkeepID) + } + + if meta.SubID != nil { + lgr = logger.With(lgr, "subID", *meta.SubID) + } + + if meta.MaxLink != nil { + lgr = logger.With(lgr, "maxLink", *meta.MaxLink) + } + + if meta.FwdrDestAddress != nil { + lgr = logger.With(lgr, "FwdrDestAddress", *meta.FwdrDestAddress) + } + + if len(meta.MessageIDs) > 0 { + for _, mid := range meta.MessageIDs { + lgr = logger.With(lgr, "messageID", mid) + } + } + + if len(meta.SeqNumbers) > 0 { + lgr = logger.With(lgr, "SeqNumbers", meta.SeqNumbers) + } + } + + return logger.Sugared(lgr) +} + +// GetChecker returns an Tx's transmit checker spec in struct form, unmarshalling it from JSON +// first. +func (e *Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) GetChecker() (TransmitCheckerSpec[ADDR], error) { + if e.TransmitChecker == nil { + return TransmitCheckerSpec[ADDR]{}, nil + } + var t TransmitCheckerSpec[ADDR] + if err := json.Unmarshal(*e.TransmitChecker, &t); err != nil { + return t, fmt.Errorf("unmarshalling transmit checker: %w", err) + } + + return t, nil +} + +func (e *Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) HasPurgeAttempt() bool { + for _, attempt := range e.TxAttempts { + if attempt.IsPurgeAttempt { + return true + } + } + return false +} + +// Provides error classification to external components in a chain agnostic way +// Only exposes the error types that could be set in the transaction error field +type ErrorClassifier interface { + error + IsFatal() bool +} diff --git a/chains/txmgr/types/tx_attempt_builder.go b/chains/txmgr/types/tx_attempt_builder.go new file mode 100644 index 0000000..eefd482 --- /dev/null +++ b/chains/txmgr/types/tx_attempt_builder.go @@ -0,0 +1,47 @@ +package types + +import ( + "context" + + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/services" + + "github.com/smartcontractkit/chainlink-framework/chains" + "github.com/smartcontractkit/chainlink-framework/chains/fees" + "github.com/smartcontractkit/chainlink-framework/chains/headtracker" +) + +// TxAttemptBuilder takes the base unsigned transaction + optional parameters (tx type, gas parameters) +// and returns a signed TxAttempt +// it is able to estimate fees and sign transactions +type TxAttemptBuilder[ + CHAIN_ID chains.ID, // CHAIN_ID - chain id type + HEAD chains.Head[BLOCK_HASH], // HEAD - chain head type + ADDR chains.Hashable, // ADDR - chain address type + TX_HASH, BLOCK_HASH chains.Hashable, // various chain hash types + SEQ chains.Sequence, // SEQ - chain sequence type (nonce, utxo, etc) + FEE fees.Fee, // FEE - chain fee type +] interface { + // interfaces for running the underlying estimator + services.Service + headtracker.HeadTrackable[HEAD, BLOCK_HASH] + + // NewTxAttempt builds a transaction using the configured transaction type and fee estimator (new estimation) + NewTxAttempt(ctx context.Context, tx Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], lggr logger.Logger, opts ...fees.Opt) (attempt TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], fee FEE, feeLimit uint64, retryable bool, err error) + + // NewTxAttemptWithType builds a transaction using the configured fee estimator (new estimation) + passed in tx type + NewTxAttemptWithType(ctx context.Context, tx Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], lggr logger.Logger, txType int, opts ...fees.Opt) (attempt TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], fee FEE, feeLimit uint64, retryable bool, err error) + + // NewBumpTxAttempt builds a transaction using the configured fee estimator (bumping) + tx type from previous attempt + // this should only be used after an initial attempt has been broadcast and the underlying gas estimator only needs to bump the fee + NewBumpTxAttempt(ctx context.Context, tx Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], previousAttempt TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], priorAttempts []TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], lggr logger.Logger) (attempt TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], bumpedFee FEE, bumpedFeeLimit uint64, retryable bool, err error) + + // NewCustomTxAttempt builds a transaction using the passed in fee + tx type + NewCustomTxAttempt(ctx context.Context, tx Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], fee FEE, gasLimit uint64, txType int, lggr logger.Logger) (attempt TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], retryable bool, err error) + + // NewEmptyTxAttempt is used in ForceRebroadcast to create a signed tx with zero value sent to the zero address + NewEmptyTxAttempt(ctx context.Context, seq SEQ, feeLimit uint64, fee FEE, fromAddress ADDR) (attempt TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) + + // NewPurgeTxAttempt is used to create empty transaction attempts with higher gas than the previous attempt to purge stuck transactions + NewPurgeTxAttempt(ctx context.Context, etx Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], lggr logger.Logger) (attempt TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) +} diff --git a/chains/txmgr/types/tx_store.go b/chains/txmgr/types/tx_store.go new file mode 100644 index 0000000..e4ca368 --- /dev/null +++ b/chains/txmgr/types/tx_store.go @@ -0,0 +1,139 @@ +package types + +import ( + "context" + "math/big" + "time" + + "github.com/google/uuid" + "gopkg.in/guregu/null.v4" + + "github.com/smartcontractkit/chainlink-framework/chains" + "github.com/smartcontractkit/chainlink-framework/chains/fees" +) + +// TxStore is a superset of all the needed persistence layer methods +type TxStore[ + // Represents an account address, in native chain format. + ADDR chains.Hashable, + // Represents a chain id to be used for the chain. + CHAIN_ID chains.ID, + // Represents a unique Tx Hash for a chain + TX_HASH chains.Hashable, + // Represents a unique Block Hash for a chain + BLOCK_HASH chains.Hashable, + // Represents a onchain receipt object that a chain's RPC returns + R ChainReceipt[TX_HASH, BLOCK_HASH], + // Represents the sequence type for a chain. For example, nonce for EVM. + SEQ chains.Sequence, + // Represents the chain specific fee type + FEE fees.Fee, +] interface { + UnstartedTxQueuePruner + TxHistoryReaper[CHAIN_ID] + TransactionStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, SEQ, FEE] + + // Find confirmed txes beyond the minConfirmations param that require callback but have not yet been signaled + FindTxesPendingCallback(ctx context.Context, latest, finalized int64, chainID CHAIN_ID) (receiptsPlus []ReceiptPlus[R], err error) + // Update tx to mark that its callback has been signaled + UpdateTxCallbackCompleted(ctx context.Context, pipelineTaskRunRid uuid.UUID, chainID CHAIN_ID) error + SaveFetchedReceipts(ctx context.Context, r []R) error + + // additional methods for tx store management + CheckTxQueueCapacity(ctx context.Context, fromAddress ADDR, maxQueuedTransactions uint64, chainID CHAIN_ID) (err error) + Close() + Abandon(ctx context.Context, id CHAIN_ID, addr ADDR) error + // Find transactions by a field in the TxMeta blob and transaction states + FindTxesByMetaFieldAndStates(ctx context.Context, metaField string, metaValue string, states []TxState, chainID *big.Int) (tx []*Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) + // Find transactions with a non-null TxMeta field that was provided by transaction states + FindTxesWithMetaFieldByStates(ctx context.Context, metaField string, states []TxState, chainID *big.Int) (tx []*Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) + // Find transactions with a non-null TxMeta field that was provided and a receipt block number greater than or equal to the one provided + FindTxesWithMetaFieldByReceiptBlockNum(ctx context.Context, metaField string, blockNum int64, chainID *big.Int) (tx []*Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) + // Find transactions loaded with transaction attempts and receipts by transaction IDs and states + FindTxesWithAttemptsAndReceiptsByIdsAndState(ctx context.Context, ids []int64, states []TxState, chainID *big.Int) (tx []*Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) + FindTxWithIdempotencyKey(ctx context.Context, idempotencyKey string, chainID CHAIN_ID) (tx *Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) +} + +// TransactionStore contains the persistence layer methods needed to manage Txs and TxAttempts +type TransactionStore[ + ADDR chains.Hashable, + CHAIN_ID chains.ID, + TX_HASH chains.Hashable, + BLOCK_HASH chains.Hashable, + SEQ chains.Sequence, + FEE fees.Fee, +] interface { + CountUnconfirmedTransactions(ctx context.Context, fromAddress ADDR, chainID CHAIN_ID) (count uint32, err error) + CountTransactionsByState(ctx context.Context, state TxState, chainID CHAIN_ID) (count uint32, err error) + CountUnstartedTransactions(ctx context.Context, fromAddress ADDR, chainID CHAIN_ID) (count uint32, err error) + CreateTransaction(ctx context.Context, txRequest TxRequest[ADDR, TX_HASH], chainID CHAIN_ID) (tx Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) + DeleteInProgressAttempt(ctx context.Context, attempt TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error + FindLatestSequence(ctx context.Context, fromAddress ADDR, chainID CHAIN_ID) (SEQ, error) + // FindReorgOrIncludedTxs returns either a list of re-org'd transactions or included transactions based on the provided sequence + FindReorgOrIncludedTxs(ctx context.Context, fromAddress ADDR, nonce SEQ, chainID CHAIN_ID) (reorgTx []*Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], includedTxs []*Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) + FindTxsRequiringGasBump(ctx context.Context, address ADDR, blockNum, gasBumpThreshold, depth int64, chainID CHAIN_ID) (etxs []*Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) + FindTxsRequiringResubmissionDueToInsufficientFunds(ctx context.Context, address ADDR, chainID CHAIN_ID) (etxs []*Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) + FindTxAttemptsConfirmedMissingReceipt(ctx context.Context, chainID CHAIN_ID) (attempts []TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) + FindTxAttemptsRequiringResend(ctx context.Context, olderThan time.Time, maxInFlightTransactions uint32, chainID CHAIN_ID, address ADDR) (attempts []TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) + // Search for Tx using the idempotencyKey and chainID + FindTxWithIdempotencyKey(ctx context.Context, idempotencyKey string, chainID CHAIN_ID) (tx *Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) + // Search for Tx using the fromAddress and sequence + FindTxWithSequence(ctx context.Context, fromAddress ADDR, seq SEQ) (etx *Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) + FindNextUnstartedTransactionFromAddress(ctx context.Context, fromAddress ADDR, chainID CHAIN_ID) (*Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) + + FindEarliestUnconfirmedBroadcastTime(ctx context.Context, chainID CHAIN_ID) (null.Time, error) + FindEarliestUnconfirmedTxAttemptBlock(ctx context.Context, chainID CHAIN_ID) (null.Int, error) + GetTxInProgress(ctx context.Context, fromAddress ADDR) (etx *Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) + GetInProgressTxAttempts(ctx context.Context, address ADDR, chainID CHAIN_ID) (attempts []TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) + GetAbandonedTransactionsByBatch(ctx context.Context, chainID CHAIN_ID, enabledAddrs []ADDR, offset, limit uint) (txs []*Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) + GetTxByID(ctx context.Context, id int64) (tx *Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) + HasInProgressTransaction(ctx context.Context, account ADDR, chainID CHAIN_ID) (exists bool, err error) + LoadTxAttempts(ctx context.Context, etx *Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error + PreloadTxes(ctx context.Context, attempts []TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error + SaveConfirmedAttempt(ctx context.Context, timeout time.Duration, attempt *TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], broadcastAt time.Time) error + SaveInProgressAttempt(ctx context.Context, attempt *TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error + SaveInsufficientFundsAttempt(ctx context.Context, timeout time.Duration, attempt *TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], broadcastAt time.Time) error + SaveReplacementInProgressAttempt(ctx context.Context, oldAttempt TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], replacementAttempt *TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error + SaveSentAttempt(ctx context.Context, timeout time.Duration, attempt *TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], broadcastAt time.Time) error + SetBroadcastBeforeBlockNum(ctx context.Context, blockNum int64, chainID CHAIN_ID) error + UpdateBroadcastAts(ctx context.Context, now time.Time, etxIDs []int64) error + UpdateTxAttemptInProgressToBroadcast(ctx context.Context, etx *Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], attempt TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], NewAttemptState TxAttemptState) error + // UpdateTxCallbackCompleted updates tx to mark that its callback has been signaled + UpdateTxCallbackCompleted(ctx context.Context, pipelineTaskRunRid uuid.UUID, chainID CHAIN_ID) error + // UpdateTxConfirmed updates transaction states to confirmed + UpdateTxConfirmed(ctx context.Context, etxIDs []int64) error + // UpdateTxFatalErrorAndDeleteAttempts updates transaction states to fatal error, deletes attempts, and clears broadcast info and sequence + UpdateTxFatalErrorAndDeleteAttempts(ctx context.Context, etx *Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error + // UpdateTxFatalError updates transaction states to fatal error with error message + UpdateTxFatalError(ctx context.Context, etxIDs []int64, errMsg string) error + UpdateTxsForRebroadcast(ctx context.Context, etxIDs []int64, attemptIDs []int64) error + UpdateTxsUnconfirmed(ctx context.Context, etxIDs []int64) error + UpdateTxUnstartedToInProgress(ctx context.Context, etx *Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], attempt *TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error +} + +type TxHistoryReaper[CHAIN_ID chains.ID] interface { + ReapTxHistory(ctx context.Context, timeThreshold time.Time, chainID CHAIN_ID) error +} + +type UnstartedTxQueuePruner interface { + PruneUnstartedTxQueue(ctx context.Context, queueSize uint32, subject uuid.UUID) (ids []int64, err error) +} + +// R is the raw unparsed transaction receipt +type ReceiptPlus[R any] struct { + ID uuid.UUID `db:"pipeline_run_id"` + Receipt R `db:"receipt"` + FailOnRevert bool `db:"fail_on_revert"` +} + +type ChainReceipt[TX_HASH, BLOCK_HASH chains.Hashable] interface { + GetStatus() uint64 + GetTxHash() TX_HASH + GetBlockNumber() *big.Int + IsZero() bool + IsUnmined() bool + GetFeeUsed() uint64 + GetTransactionIndex() uint + GetBlockHash() BLOCK_HASH + GetRevertReason() *string +} diff --git a/chains/txmgr/types/tx_test.go b/chains/txmgr/types/tx_test.go new file mode 100644 index 0000000..b945017 --- /dev/null +++ b/chains/txmgr/types/tx_test.go @@ -0,0 +1,50 @@ +package types + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestTxAttemptState(t *testing.T) { + type stateCompare struct { + state TxAttemptState + str string + } + + // dynmaically build base states + states := []stateCompare{} + for i, v := range txAttemptStateStrings { + states = append(states, stateCompare{TxAttemptState(i), v}) + } + + t.Run("NewTxAttemptState", func(t *testing.T) { + // string representation + addStates := []stateCompare{ + {TxAttemptState(0), "invalid_state"}, + } + allStates := append(states, addStates...) + for i := range allStates { + s := allStates[i] + t.Run(fmt.Sprintf("%s->%d", s.str, s.state), func(t *testing.T) { + assert.Equal(t, s.state, NewTxAttemptState(s.str)) + }) + } + }) + + t.Run("String", func(t *testing.T) { + // string representation + addStates := []stateCompare{ + {txAttemptStateCount, txAttemptStateStrings[0]}, + {100, txAttemptStateStrings[0]}, + } + allStates := append(states, addStates...) + for i := range allStates { + s := allStates[i] + t.Run(fmt.Sprintf("%d->%s", s.state, s.str), func(t *testing.T) { + assert.Equal(t, s.str, s.state.String()) + }) + } + }) +} From 5f60fb3b2cf932d6d119b00e339cbeb17f3f2221 Mon Sep 17 00:00:00 2001 From: pavel-raykov Date: Thu, 16 Jan 2025 18:04:47 +0100 Subject: [PATCH 2/3] Minor --- chains/go.mod | 21 ++++---- chains/go.sum | 57 +++++++++++--------- chains/headtracker/head_listener.go | 15 +++--- chains/headtracker/head_tracker.go | 21 ++++---- chains/txmgr/broadcaster.go | 63 +++++++++++----------- chains/txmgr/confirmer.go | 69 ++++++++++++------------ chains/txmgr/models.go | 16 +++--- chains/txmgr/reaper.go | 8 +-- chains/txmgr/strategies.go | 14 ++--- chains/txmgr/test_helpers.go | 4 +- chains/txmgr/tracker.go | 17 +++--- chains/txmgr/txmgr.go | 1 - chains/txmgr/types/client.go | 3 +- chains/txmgr/types/forwarder_manager.go | 1 - chains/txmgr/types/tx.go | 1 - chains/txmgr/types/tx_attempt_builder.go | 1 - 16 files changed, 156 insertions(+), 156 deletions(-) diff --git a/chains/go.mod b/chains/go.mod index 7e1e2f7..bf96cc1 100644 --- a/chains/go.mod +++ b/chains/go.mod @@ -2,6 +2,18 @@ module github.com/smartcontractkit/chainlink-framework/chains go 1.23.3 +require ( + github.com/google/uuid v1.6.0 + github.com/jpillora/backoff v1.0.0 + github.com/prometheus/client_golang v1.20.5 + github.com/shopspring/decimal v1.4.0 + github.com/smartcontractkit/chainlink-common v0.4.0 + github.com/smartcontractkit/chainlink-framework/multinode v0.0.0-20250115203616-a2ea5e50b260 + github.com/stretchr/testify v1.10.0 + go.uber.org/multierr v1.11.0 + gopkg.in/guregu/null.v4 v4.0.0 +) + require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect @@ -9,28 +21,20 @@ require ( github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/google/go-cmp v0.6.0 // indirect - github.com/google/uuid v1.6.0 // indirect github.com/jmoiron/sqlx v1.4.0 // indirect - github.com/jpillora/backoff v1.0.0 // indirect github.com/lib/pq v1.10.9 // indirect github.com/mr-tron/base58 v1.2.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/prometheus/client_golang v1.20.5 // indirect github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.60.1 // indirect github.com/prometheus/procfs v0.15.1 // indirect - github.com/shopspring/decimal v1.4.0 // indirect - github.com/smartcontractkit/chainlink-common v0.4.0 // indirect - github.com/smartcontractkit/chainlink-framework/multinode v0.0.0-20250115203616-a2ea5e50b260 // indirect github.com/smartcontractkit/libocr v0.0.0-20241223215956-e5b78d8e3919 // indirect - github.com/stretchr/testify v1.10.0 // indirect go.opentelemetry.io/otel v1.31.0 // indirect go.opentelemetry.io/otel/metric v1.31.0 // indirect go.opentelemetry.io/otel/trace v1.31.0 // indirect - go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect golang.org/x/crypto v0.28.0 // indirect golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c // indirect @@ -38,6 +42,5 @@ require ( google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38 // indirect google.golang.org/grpc v1.67.1 // indirect google.golang.org/protobuf v1.35.1 // indirect - gopkg.in/guregu/null.v4 v4.0.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/chains/go.sum b/chains/go.sum index eb0875f..9b260de 100644 --- a/chains/go.sum +++ b/chains/go.sum @@ -1,18 +1,26 @@ +filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cometbft/cometbft v0.37.5 h1:/U/TlgMh4NdnXNo+YU9T2NMCWyhXNDF34Mx582jlvq0= +github.com/cometbft/cometbft v0.37.5/go.mod h1:QC+mU0lBhKn8r9qvmnq53Dmf3DWBt4VtkcKw2C81wxY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fxamacker/cbor/v2 v2.5.0 h1:oHsG0V/Q6E/wqTS2O1Cozzsy69nqCiguo5Q1a1ADivE= +github.com/fxamacker/cbor/v2 v2.5.0/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= +github.com/go-viper/mapstructure/v2 v2.1.0 h1:gHnMa2Y/pIxElCH2GlZZ1lZSsn6XMtufpGyP1XxdC/w= +github.com/go-viper/mapstructure/v2 v2.1.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -21,17 +29,24 @@ github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY= github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= +github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/pelletier/go-toml/v2 v2.2.0 h1:QLgLl2yMN7N+ruc31VynXs1vhMZa7CeHHejIeBAsoHo= -github.com/pelletier/go-toml/v2 v2.2.0/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/petermattis/goid v0.0.0-20240813172612-4fcff4a6cae7 h1:Dx7Ovyv/SFnMFw3fD4oEoeorXc6saIiQ23LrGLth0Gw= +github.com/petermattis/goid v0.0.0-20240813172612-4fcff4a6cae7/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -41,74 +56,66 @@ github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+ github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= -github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= -github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= -github.com/prometheus/common v0.59.1 h1:LXb1quJHWm1P6wq/U824uxYi4Sg0oGvNeUm1z5dJoX0= -github.com/prometheus/common v0.59.1/go.mod h1:GpWM7dewqmVYcd7SmRaiWVe9SSqjf0UrwnYnpEZNuT0= github.com/prometheus/common v0.60.1 h1:FUas6GcOw66yB/73KC+BOZoFJmbo/1pojoILArPAaSc= github.com/prometheus/common v0.60.1/go.mod h1:h0LYf1R1deLSKtD4Vdg8gy4RuOvENW2J/h19V5NADQw= github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/sasha-s/go-deadlock v0.3.5 h1:tNCOEEDG6tBqrNDOX35j/7hL5FcFViG6awUGROb2NsU= +github.com/sasha-s/go-deadlock v0.3.5/go.mod h1:bugP6EGbdGYObIlx7pUZtWqlvo8k9H6vCBBsiChJQ5U= github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= github.com/smartcontractkit/chainlink-common v0.4.0 h1:GZ9MhHt5QHXSaK/sAZvKDxkEqF4fPiFHWHEPqs/2C2o= github.com/smartcontractkit/chainlink-common v0.4.0/go.mod h1:yti7e1+G9hhkYhj+L5sVUULn9Bn3bBL5/AxaNqdJ5YQ= github.com/smartcontractkit/chainlink-framework/multinode v0.0.0-20250115203616-a2ea5e50b260 h1:See2isL6KdrTJDlVKWv8qiyYqWhYUcubU2e5yKXV1oY= github.com/smartcontractkit/chainlink-framework/multinode v0.0.0-20250115203616-a2ea5e50b260/go.mod h1:4JqpgFy01LaqG1yM2iFTzwX3ZgcAvW9WdstBZQgPHzU= -github.com/smartcontractkit/chainlink/v2 v2.19.0 h1:iw91nBoxfkrF9lh2m1jgc04pkj5pZs6j6PK6cwrnTNU= -github.com/smartcontractkit/chainlink/v2 v2.19.0/go.mod h1:5uc+GGa6R1Ugccq0ol2Wnjt5LStJAXqz9TZxHX55t2Y= -github.com/smartcontractkit/libocr v0.0.0-20241007185508-adbe57025f12 h1:NzZGjaqez21I3DU7objl3xExTH4fxYvzTqar8DC6360= -github.com/smartcontractkit/libocr v0.0.0-20241007185508-adbe57025f12/go.mod h1:fb1ZDVXACvu4frX3APHZaEBp0xi1DIm34DcA0CwTsZM= github.com/smartcontractkit/libocr v0.0.0-20241223215956-e5b78d8e3919 h1:IpGoPTXpvllN38kT2z2j13sifJMz4nbHglidvop7mfg= github.com/smartcontractkit/libocr v0.0.0-20241223215956-e5b78d8e3919/go.mod h1:fb1ZDVXACvu4frX3APHZaEBp0xi1DIm34DcA0CwTsZM= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -go.opentelemetry.io/otel v1.30.0 h1:F2t8sK4qf1fAmY9ua4ohFS/K+FUuOPemHUIXHtktrts= -go.opentelemetry.io/otel v1.30.0/go.mod h1:tFw4Br9b7fOS+uEao81PJjVMjW/5fvNCbpsDIXqP0pc= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY= go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE= -go.opentelemetry.io/otel/metric v1.30.0 h1:4xNulvn9gjzo4hjg+wzIKG7iNFEaBMX00Qd4QIZs7+w= -go.opentelemetry.io/otel/metric v1.30.0/go.mod h1:aXTfST94tswhWEb+5QjlSqG+cZlmyXy/u8jFpor3WqQ= go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE= go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY= -go.opentelemetry.io/otel/trace v1.30.0 h1:7UBkkYzeg3C7kQX8VAidWh2biiQbtAKjyIML8dQ9wmc= -go.opentelemetry.io/otel/trace v1.30.0/go.mod h1:5EyKqTzzmyqB9bwtCCq6pDLktPK6fmGf/Dph+8VI02o= go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys= go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= -golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk= -golang.org/x/exp v0.0.0-20240909161429-701f63a606c0/go.mod h1:2TbTHSBQa924w8M6Xs1QcRcFwyucIwBGpK1p2f1YFFY= golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c h1:7dEasQXItcW1xKJ2+gg5VOiBnqWrJc+rq0DPKyvvdbY= golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c/go.mod h1:NQtJDoLvd6faHhE7m4T/1IY708gDefGGjR/iUW8yQQ8= -golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= -golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= +golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= +golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38 h1:zciRKQ4kBpFgpfC5QQCVtnnNAcLIqweL7plyZRQHVpI= google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI= google.golang.org/grpc v1.67.1 h1:zWnc1Vrcno+lHZCOofnIMvycFcc0QRGIzm9dhnDX68E= google.golang.org/grpc v1.67.1/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA= -google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= -google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/guregu/null.v4 v4.0.0 h1:1Wm3S1WEA2I26Kq+6vcW+w0gcDo44YKYD7YIEJNHDjg= gopkg.in/guregu/null.v4 v4.0.0/go.mod h1:YoQhUrADuG3i9WqesrCmpNRwm1ypAgSHYqoOcTu/JrI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/chains/headtracker/head_listener.go b/chains/headtracker/head_listener.go index a50816b..b8707d6 100644 --- a/chains/headtracker/head_listener.go +++ b/chains/headtracker/head_listener.go @@ -14,8 +14,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/services" "github.com/smartcontractkit/chainlink-framework/chains" - - htrktypes "github.com/smartcontractkit/chainlink-framework/chains/headtracker/types" + "github.com/smartcontractkit/chainlink-framework/chains/headtracker/types" ) var ( @@ -50,7 +49,7 @@ type HeadListener[H chains.Head[BLOCK_HASH], BLOCK_HASH chains.Hashable] interfa } type headListener[ - HTH htrktypes.Head[BLOCK_HASH, ID], + HTH types.Head[BLOCK_HASH, ID], S chains.Subscription, ID chains.ID, BLOCK_HASH chains.Hashable, @@ -58,8 +57,8 @@ type headListener[ services.Service eng *services.Engine - config htrktypes.Config - client htrktypes.Client[HTH, S, ID, BLOCK_HASH] + config types.Config + client types.Client[HTH, S, ID, BLOCK_HASH] onSubscription func(context.Context) handleNewHead HeadHandler[HTH, BLOCK_HASH] chHeaders <-chan HTH @@ -69,15 +68,15 @@ type headListener[ } func NewHeadListener[ - HTH htrktypes.Head[BLOCK_HASH, ID], + HTH types.Head[BLOCK_HASH, ID], S chains.Subscription, ID chains.ID, BLOCK_HASH chains.Hashable, - CLIENT htrktypes.Client[HTH, S, ID, BLOCK_HASH], + CLIENT types.Client[HTH, S, ID, BLOCK_HASH], ]( lggr logger.Logger, client CLIENT, - config htrktypes.Config, + config types.Config, onSubscription func(context.Context), handleNewHead HeadHandler[HTH, BLOCK_HASH], ) HeadListener[HTH, BLOCK_HASH] { diff --git a/chains/headtracker/head_tracker.go b/chains/headtracker/head_tracker.go index 2ab8e34..e890433 100644 --- a/chains/headtracker/head_tracker.go +++ b/chains/headtracker/head_tracker.go @@ -9,13 +9,12 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" - "github.com/smartcontractkit/chainlink-framework/chains" "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/services" "github.com/smartcontractkit/chainlink-common/pkg/utils/mailbox" - - htrktypes "github.com/smartcontractkit/chainlink-framework/chains/headtracker/types" + "github.com/smartcontractkit/chainlink-framework/chains" + "github.com/smartcontractkit/chainlink-framework/chains/headtracker/types" ) var ( @@ -45,7 +44,7 @@ type HeadTracker[H chains.Head[BLOCK_HASH], BLOCK_HASH chains.Hashable] interfac } type headTracker[ - HTH htrktypes.Head[BLOCK_HASH, ID], + HTH types.Head[BLOCK_HASH, ID], S chains.Subscription, ID chains.ID, BLOCK_HASH chains.Hashable, @@ -57,10 +56,10 @@ type headTracker[ headBroadcaster HeadBroadcaster[HTH, BLOCK_HASH] headSaver HeadSaver[HTH, BLOCK_HASH] mailMon *mailbox.Monitor - client htrktypes.Client[HTH, S, ID, BLOCK_HASH] + client types.Client[HTH, S, ID, BLOCK_HASH] chainID chains.ID - config htrktypes.Config - htConfig htrktypes.HeadTrackerConfig + config types.Config + htConfig types.HeadTrackerConfig backfillMB *mailbox.Mailbox[HTH] broadcastMB *mailbox.Mailbox[HTH] @@ -70,15 +69,15 @@ type headTracker[ // NewHeadTracker instantiates a new HeadTracker using HeadSaver to persist new block numbers. func NewHeadTracker[ - HTH htrktypes.Head[BLOCK_HASH, ID], + HTH types.Head[BLOCK_HASH, ID], S chains.Subscription, ID chains.ID, BLOCK_HASH chains.Hashable, ]( lggr logger.Logger, - client htrktypes.Client[HTH, S, ID, BLOCK_HASH], - config htrktypes.Config, - htConfig htrktypes.HeadTrackerConfig, + client types.Client[HTH, S, ID, BLOCK_HASH], + config types.Config, + htConfig types.HeadTrackerConfig, headBroadcaster HeadBroadcaster[HTH, BLOCK_HASH], headSaver HeadSaver[HTH, BLOCK_HASH], mailMon *mailbox.Monitor, diff --git a/chains/txmgr/broadcaster.go b/chains/txmgr/broadcaster.go index 221d330..25f3789 100644 --- a/chains/txmgr/broadcaster.go +++ b/chains/txmgr/broadcaster.go @@ -18,11 +18,10 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/services" "github.com/smartcontractkit/chainlink-common/pkg/utils" - "github.com/smartcontractkit/chainlink-framework/multinode" - "github.com/smartcontractkit/chainlink-framework/chains" "github.com/smartcontractkit/chainlink-framework/chains/fees" - txmgrtypes "github.com/smartcontractkit/chainlink-framework/chains/txmgr/types" + "github.com/smartcontractkit/chainlink-framework/chains/txmgr/types" + "github.com/smartcontractkit/chainlink-framework/multinode" ) const ( @@ -72,7 +71,7 @@ type TransmitCheckerFactory[ FEE fees.Fee, ] interface { // BuildChecker builds a new TransmitChecker based on the given spec. - BuildChecker(spec txmgrtypes.TransmitCheckerSpec[ADDR]) (TransmitChecker[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) + BuildChecker(spec types.TransmitCheckerSpec[ADDR]) (TransmitChecker[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) } // TransmitChecker determines whether a transaction should be submitted on-chain. @@ -88,7 +87,7 @@ type TransmitChecker[ // is returned. Errors should only be returned if the checker can confirm that a transaction // should not be sent, other errors (for example connection or other unexpected errors) should // be logged and swallowed. - Check(ctx context.Context, l logger.SugaredLogger, tx txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], a txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error + Check(ctx context.Context, l logger.SugaredLogger, tx types.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], a types.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error } // Broadcaster monitors txes for transactions that need to @@ -115,17 +114,17 @@ type Broadcaster[ ] struct { services.StateMachine lggr logger.SugaredLogger - txStore txmgrtypes.TransactionStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, SEQ, FEE] - client txmgrtypes.TransactionClient[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] - txmgrtypes.TxAttemptBuilder[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] - sequenceTracker txmgrtypes.SequenceTracker[ADDR, SEQ] + txStore types.TransactionStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, SEQ, FEE] + client types.TransactionClient[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] + types.TxAttemptBuilder[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] + sequenceTracker types.SequenceTracker[ADDR, SEQ] resumeCallback ResumeCallback chainID CHAIN_ID chainType string - config txmgrtypes.BroadcasterChainConfig - feeConfig txmgrtypes.BroadcasterFeeConfig - txConfig txmgrtypes.BroadcasterTransactionsConfig - listenerConfig txmgrtypes.BroadcasterListenerConfig + config types.BroadcasterChainConfig + feeConfig types.BroadcasterFeeConfig + txConfig types.BroadcasterTransactionsConfig + listenerConfig types.BroadcasterListenerConfig // autoSyncSequence, if set, will cause Broadcaster to fast-forward the sequence // when Start is called @@ -133,7 +132,7 @@ type Broadcaster[ processUnstartedTxsImpl ProcessUnstartedTxs[ADDR] - ks txmgrtypes.KeyStore[ADDR, CHAIN_ID, SEQ] + ks types.KeyStore[ADDR, CHAIN_ID, SEQ] enabledAddresses []ADDR checkerFactory TransmitCheckerFactory[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] @@ -159,15 +158,15 @@ func NewBroadcaster[ SEQ chains.Sequence, FEE fees.Fee, ]( - txStore txmgrtypes.TransactionStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, SEQ, FEE], - client txmgrtypes.TransactionClient[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], - config txmgrtypes.BroadcasterChainConfig, - feeConfig txmgrtypes.BroadcasterFeeConfig, - txConfig txmgrtypes.BroadcasterTransactionsConfig, - listenerConfig txmgrtypes.BroadcasterListenerConfig, - keystore txmgrtypes.KeyStore[ADDR, CHAIN_ID, SEQ], - txAttemptBuilder txmgrtypes.TxAttemptBuilder[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], - sequenceTracker txmgrtypes.SequenceTracker[ADDR, SEQ], + txStore types.TransactionStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, SEQ, FEE], + client types.TransactionClient[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], + config types.BroadcasterChainConfig, + feeConfig types.BroadcasterFeeConfig, + txConfig types.BroadcasterTransactionsConfig, + listenerConfig types.BroadcasterListenerConfig, + keystore types.KeyStore[ADDR, CHAIN_ID, SEQ], + txAttemptBuilder types.TxAttemptBuilder[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], + sequenceTracker types.SequenceTracker[ADDR, SEQ], lggr logger.Logger, checkerFactory TransmitCheckerFactory[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], autoSyncSequence bool, @@ -428,7 +427,7 @@ func (eb *Broadcaster[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) hand return nil, false } -func (eb *Broadcaster[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) handleUnstartedTx(ctx context.Context, etx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) (error, bool) { +func (eb *Broadcaster[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) handleUnstartedTx(ctx context.Context, etx *types.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) (error, bool) { if etx.State != TxUnstarted { return fmt.Errorf("invariant violation: expected transaction %v to be unstarted, it was %s", etx.ID, etx.State), false } @@ -483,7 +482,7 @@ func (eb *Broadcaster[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) hand // There can be at most one in_progress transaction per address. // Here we complete the job that we didn't finish last time. -func (eb *Broadcaster[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) handleInProgressTx(ctx context.Context, etx txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], attempt txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], initialBroadcastAt time.Time, retryCount int) (error, bool) { +func (eb *Broadcaster[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) handleInProgressTx(ctx context.Context, etx types.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], attempt types.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], initialBroadcastAt time.Time, retryCount int) (error, bool) { if etx.State != TxInProgress { return fmt.Errorf("invariant violation: expected transaction %v to be in_progress, it was %s", etx.ID, etx.State), false } @@ -549,7 +548,7 @@ func (eb *Broadcaster[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) hand // and hand off to the confirmer to get the receipt (or mark as // failed). observeTimeUntilBroadcast(eb.chainID, etx.CreatedAt, time.Now()) - err = eb.txStore.UpdateTxAttemptInProgressToBroadcast(ctx, &etx, attempt, txmgrtypes.TxAttemptBroadcast) + err = eb.txStore.UpdateTxAttemptInProgressToBroadcast(ctx, &etx, attempt, types.TxAttemptBroadcast) if err != nil { return err, true } @@ -613,7 +612,7 @@ func (eb *Broadcaster[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) hand // transaction to have been accepted. In this case, the right thing to // do is assume success and hand off to Confirmer - err = eb.txStore.UpdateTxAttemptInProgressToBroadcast(ctx, &etx, attempt, txmgrtypes.TxAttemptBroadcast) + err = eb.txStore.UpdateTxAttemptInProgressToBroadcast(ctx, &etx, attempt, types.TxAttemptBroadcast) if err != nil { return err, true } @@ -631,7 +630,7 @@ func (eb *Broadcaster[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) hand } } -func (eb *Broadcaster[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) validateOnChainSequence(ctx context.Context, lgr logger.SugaredLogger, errType multinode.SendTxReturnCode, err error, etx txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], retryCount int) (multinode.SendTxReturnCode, error) { +func (eb *Broadcaster[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) validateOnChainSequence(ctx context.Context, lgr logger.SugaredLogger, errType multinode.SendTxReturnCode, err error, etx types.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], retryCount int) (multinode.SendTxReturnCode, error) { // Only check if sequence was incremented if broadcast was successful, otherwise return the existing err type if errType != multinode.Successful { return errType, err @@ -671,7 +670,7 @@ func (eb *Broadcaster[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) vali // Finds next transaction in the queue, assigns a sequence, and moves it to "in_progress" state ready for broadcast. // Returns nil if no transactions are in queue -func (eb *Broadcaster[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) nextUnstartedTransactionWithSequence(fromAddress ADDR) (*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) { +func (eb *Broadcaster[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) nextUnstartedTransactionWithSequence(fromAddress ADDR) (*types.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) { ctx, cancel := eb.chStop.NewCtx() defer cancel() etx, err := eb.txStore.FindNextUnstartedTransactionFromAddress(ctx, fromAddress, eb.chainID) @@ -692,7 +691,7 @@ func (eb *Broadcaster[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) next } // replaceAttemptWithBumpedGas performs the replacement of the existing tx attempt with a new bumped fee attempt. -func (eb *Broadcaster[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) replaceAttemptWithBumpedGas(ctx context.Context, lgr logger.Logger, txError error, etx txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], attempt txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) (replacedAttempt txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], retryable bool, err error) { +func (eb *Broadcaster[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) replaceAttemptWithBumpedGas(ctx context.Context, lgr logger.Logger, txError error, etx types.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], attempt types.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) (replacedAttempt types.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], retryable bool, err error) { // This log error is not applicable to Hedera since the action required would not be needed for its gas estimator if eb.chainType != hederaChainType { logger.With(lgr, @@ -720,7 +719,7 @@ func (eb *Broadcaster[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) repl } // replaceAttemptWithNewEstimation performs the replacement of the existing tx attempt with a new estimated fee attempt. -func (eb *Broadcaster[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) replaceAttemptWithNewEstimation(ctx context.Context, lgr logger.Logger, etx txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], attempt txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) (updatedAttempt *txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], retryable bool, err error) { +func (eb *Broadcaster[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) replaceAttemptWithNewEstimation(ctx context.Context, lgr logger.Logger, etx types.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], attempt types.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) (updatedAttempt *types.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], retryable bool, err error) { newEstimatedAttempt, fee, feeLimit, retryable, err := eb.NewTxAttemptWithType(ctx, etx, lgr, attempt.TxType, fees.OptForceRefetch) if err != nil { return &newEstimatedAttempt, retryable, err @@ -734,7 +733,7 @@ func (eb *Broadcaster[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) repl return &newEstimatedAttempt, true, err } -func (eb *Broadcaster[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) saveFatallyErroredTransaction(lgr logger.Logger, etx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error { +func (eb *Broadcaster[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) saveFatallyErroredTransaction(lgr logger.Logger, etx *types.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error { ctx, cancel := eb.chStop.NewCtx() defer cancel() if etx.State != TxInProgress && etx.State != TxUnstarted { diff --git a/chains/txmgr/confirmer.go b/chains/txmgr/confirmer.go index ef8823e..454ffa1 100644 --- a/chains/txmgr/confirmer.go +++ b/chains/txmgr/confirmer.go @@ -20,11 +20,10 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/services" "github.com/smartcontractkit/chainlink-common/pkg/utils/mailbox" - "github.com/smartcontractkit/chainlink-framework/multinode" - "github.com/smartcontractkit/chainlink-framework/chains" "github.com/smartcontractkit/chainlink-framework/chains/fees" - txmgrtypes "github.com/smartcontractkit/chainlink-framework/chains/txmgr/types" + "github.com/smartcontractkit/chainlink-framework/chains/txmgr/types" + "github.com/smartcontractkit/chainlink-framework/multinode" ) const ( @@ -87,23 +86,23 @@ type Confirmer[ ADDR chains.Hashable, TX_HASH chains.Hashable, BLOCK_HASH chains.Hashable, - R txmgrtypes.ChainReceipt[TX_HASH, BLOCK_HASH], + R types.ChainReceipt[TX_HASH, BLOCK_HASH], SEQ chains.Sequence, FEE fees.Fee, ] struct { services.StateMachine - txStore txmgrtypes.TxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] + txStore types.TxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] lggr logger.SugaredLogger - client txmgrtypes.TxmClient[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE] - txmgrtypes.TxAttemptBuilder[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] - stuckTxDetector txmgrtypes.StuckTxDetector[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] + client types.TxmClient[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE] + types.TxAttemptBuilder[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] + stuckTxDetector types.StuckTxDetector[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] resumeCallback ResumeCallback - feeConfig txmgrtypes.ConfirmerFeeConfig - txConfig txmgrtypes.ConfirmerTransactionsConfig - dbConfig txmgrtypes.ConfirmerDatabaseConfig + feeConfig types.ConfirmerFeeConfig + txConfig types.ConfirmerTransactionsConfig + dbConfig types.ConfirmerDatabaseConfig chainID CHAIN_ID - ks txmgrtypes.KeyStore[ADDR, CHAIN_ID, SEQ] + ks types.KeyStore[ADDR, CHAIN_ID, SEQ] enabledAddresses []ADDR mb *mailbox.Mailbox[HEAD] @@ -120,20 +119,20 @@ func NewConfirmer[ ADDR chains.Hashable, TX_HASH chains.Hashable, BLOCK_HASH chains.Hashable, - R txmgrtypes.ChainReceipt[TX_HASH, BLOCK_HASH], + R types.ChainReceipt[TX_HASH, BLOCK_HASH], SEQ chains.Sequence, FEE fees.Fee, ]( - txStore txmgrtypes.TxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE], - client txmgrtypes.TxmClient[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE], - feeConfig txmgrtypes.ConfirmerFeeConfig, - txConfig txmgrtypes.ConfirmerTransactionsConfig, - dbConfig txmgrtypes.ConfirmerDatabaseConfig, - keystore txmgrtypes.KeyStore[ADDR, CHAIN_ID, SEQ], - txAttemptBuilder txmgrtypes.TxAttemptBuilder[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], + txStore types.TxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE], + client types.TxmClient[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE], + feeConfig types.ConfirmerFeeConfig, + txConfig types.ConfirmerTransactionsConfig, + dbConfig types.ConfirmerDatabaseConfig, + keystore types.KeyStore[ADDR, CHAIN_ID, SEQ], + txAttemptBuilder types.TxAttemptBuilder[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], lggr logger.Logger, isReceiptNil func(R) bool, - stuckTxDetector txmgrtypes.StuckTxDetector[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], + stuckTxDetector types.StuckTxDetector[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], ) *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { lggr = logger.Named(lggr, "Confirmer") return &Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]{ @@ -318,7 +317,7 @@ func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Che return nil } -func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) ProcessReorgTxs(ctx context.Context, reorgTxs []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], head chains.Head[BLOCK_HASH]) error { +func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) ProcessReorgTxs(ctx context.Context, reorgTxs []*types.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], head chains.Head[BLOCK_HASH]) error { if len(reorgTxs) == 0 { return nil } @@ -368,7 +367,7 @@ func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Pro return ec.txStore.UpdateTxsForRebroadcast(ctx, etxIDs, attemptIDs) } -func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) ProcessIncludedTxs(ctx context.Context, includedTxs []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], head chains.Head[BLOCK_HASH]) error { +func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) ProcessIncludedTxs(ctx context.Context, includedTxs []*types.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], head chains.Head[BLOCK_HASH]) error { if len(includedTxs) == 0 { return nil } @@ -418,7 +417,7 @@ func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Pro for _, tx := range stuckTxs { // All stuck transactions will have unique from addresses. It is safe to process separate keys concurrently // NOTE: This design will block one key if another takes a really long time to execute - go func(tx txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) { + go func(tx types.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) { defer wg.Done() lggr := tx.GetLogger(ec.lggr) // Create a purge attempt for tx @@ -457,7 +456,7 @@ func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Pro return errors.Join(errorList...) } -func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) resumeFailedTaskRuns(ctx context.Context, etx txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error { +func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) resumeFailedTaskRuns(ctx context.Context, etx types.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error { if !etx.PipelineTaskRunID.Valid || ec.resumeCallback == nil || !etx.SignalCallback || etx.CallbackCompleted { return nil } @@ -560,7 +559,7 @@ func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) han // FindTxsRequiringRebroadcast returns attempts that hit insufficient native tokens, // and attempts that need bumping, in sequence ASC order -func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxsRequiringRebroadcast(ctx context.Context, lggr logger.Logger, address ADDR, blockNum, gasBumpThreshold, bumpDepth int64, maxInFlightTransactions uint32, chainID CHAIN_ID) (etxs []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) { +func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxsRequiringRebroadcast(ctx context.Context, lggr logger.Logger, address ADDR, blockNum, gasBumpThreshold, bumpDepth int64, maxInFlightTransactions uint32, chainID CHAIN_ID) (etxs []*types.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) { // NOTE: These two queries could be combined into one using union but it // becomes harder to read and difficult to test in isolation. KISS principle etxInsufficientFunds, err := ec.txStore.FindTxsRequiringResubmissionDueToInsufficientFunds(ctx, address, chainID) @@ -619,16 +618,16 @@ func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Fin return } -func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) attemptForRebroadcast(ctx context.Context, lggr logger.Logger, etx txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) (attempt txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) { +func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) attemptForRebroadcast(ctx context.Context, lggr logger.Logger, etx types.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) (attempt types.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) { if len(etx.TxAttempts) > 0 { etx.TxAttempts[0].Tx = etx previousAttempt := etx.TxAttempts[0] logFields := ec.logFieldsPreviousAttempt(previousAttempt) - if previousAttempt.State == txmgrtypes.TxAttemptInsufficientFunds { + if previousAttempt.State == types.TxAttemptInsufficientFunds { // Do not create a new attempt if we ran out of funds last time since bumping gas is pointless // Instead try to resubmit the same attempt at the same price, in the hope that the wallet was funded since our last attempt lggr.Debugw("Rebroadcast InsufficientFunds", logFields...) - previousAttempt.State = txmgrtypes.TxAttemptInProgress + previousAttempt.State = types.TxAttemptInProgress return previousAttempt, nil } attempt, err = ec.bumpGas(ctx, etx, etx.TxAttempts) @@ -638,7 +637,7 @@ func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) att // Do not create a new attempt if bumping gas would put us over the limit or cause some other problem // Instead try to resubmit the previous attempt, and keep resubmitting until its accepted previousAttempt.BroadcastBeforeBlockNum = nil - previousAttempt.State = txmgrtypes.TxAttemptInProgress + previousAttempt.State = types.TxAttemptInProgress return previousAttempt, nil } return attempt, err @@ -648,7 +647,7 @@ func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) att "This is a bug! Please report to https://github.com/smartcontractkit/chainlink/issues", etx.ID) } -func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) logFieldsPreviousAttempt(attempt txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) []interface{} { +func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) logFieldsPreviousAttempt(attempt types.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) []interface{} { etx := attempt.Tx return []interface{}{ "etxID", etx.ID, @@ -661,7 +660,7 @@ func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) log } } -func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) bumpGas(ctx context.Context, etx txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], previousAttempts []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) (bumpedAttempt txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) { +func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) bumpGas(ctx context.Context, etx types.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], previousAttempts []types.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) (bumpedAttempt types.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) { previousAttempt := previousAttempts[0] logFields := ec.logFieldsPreviousAttempt(previousAttempt) @@ -684,8 +683,8 @@ func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) bum return bumpedAttempt, fmt.Errorf("error bumping gas: %w", err) } -func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) handleInProgressAttempt(ctx context.Context, lggr logger.SugaredLogger, etx txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], attempt txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], blockHeight int64) error { - if attempt.State != txmgrtypes.TxAttemptInProgress { +func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) handleInProgressAttempt(ctx context.Context, lggr logger.SugaredLogger, etx types.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], attempt types.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], blockHeight int64) error { + if attempt.State != types.TxAttemptInProgress { return fmt.Errorf("invariant violation: expected tx_attempt %v to be in_progress, it was %s", attempt.ID, attempt.State) } @@ -866,7 +865,7 @@ func observeUntilTxConfirmed[ TX_HASH, BLOCK_HASH chains.Hashable, SEQ chains.Sequence, FEE fees.Fee, -](chainID CHAIN_ID, attempts []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], head chains.Head[BLOCK_HASH]) { +](chainID CHAIN_ID, attempts []types.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], head chains.Head[BLOCK_HASH]) { for _, attempt := range attempts { // We estimate the time until confirmation by subtracting from the time the tx (not the attempt) // was created. We want to measure the amount of time taken from when a transaction is created diff --git a/chains/txmgr/models.go b/chains/txmgr/models.go index 6662b6c..f385bb5 100644 --- a/chains/txmgr/models.go +++ b/chains/txmgr/models.go @@ -1,15 +1,15 @@ package txmgr import ( - txmgrtypes "github.com/smartcontractkit/chainlink-framework/chains/txmgr/types" + "github.com/smartcontractkit/chainlink-framework/chains/txmgr/types" ) const ( - TxUnstarted = txmgrtypes.TxState("unstarted") - TxInProgress = txmgrtypes.TxState("in_progress") - TxFatalError = txmgrtypes.TxState("fatal_error") - TxUnconfirmed = txmgrtypes.TxState("unconfirmed") - TxConfirmed = txmgrtypes.TxState("confirmed") - TxConfirmedMissingReceipt = txmgrtypes.TxState("confirmed_missing_receipt") - TxFinalized = txmgrtypes.TxState("finalized") + TxUnstarted = types.TxState("unstarted") + TxInProgress = types.TxState("in_progress") + TxFatalError = types.TxState("fatal_error") + TxUnconfirmed = types.TxState("unconfirmed") + TxConfirmed = types.TxState("confirmed") + TxConfirmedMissingReceipt = types.TxState("confirmed_missing_receipt") + TxFinalized = types.TxState("finalized") ) diff --git a/chains/txmgr/reaper.go b/chains/txmgr/reaper.go index 1cd3ce3..885011b 100644 --- a/chains/txmgr/reaper.go +++ b/chains/txmgr/reaper.go @@ -8,13 +8,13 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/services" "github.com/smartcontractkit/chainlink-framework/chains" - txmgrtypes "github.com/smartcontractkit/chainlink-framework/chains/txmgr/types" + "github.com/smartcontractkit/chainlink-framework/chains/txmgr/types" ) // Reaper handles periodic database cleanup for Txm type Reaper[CHAIN_ID chains.ID] struct { - store txmgrtypes.TxHistoryReaper[CHAIN_ID] - txConfig txmgrtypes.ReaperTransactionsConfig + store types.TxHistoryReaper[CHAIN_ID] + txConfig types.ReaperTransactionsConfig chainID CHAIN_ID log logger.Logger latestBlockNum atomic.Int64 @@ -24,7 +24,7 @@ type Reaper[CHAIN_ID chains.ID] struct { } // NewReaper instantiates a new reaper object -func NewReaper[CHAIN_ID chains.ID](lggr logger.Logger, store txmgrtypes.TxHistoryReaper[CHAIN_ID], txConfig txmgrtypes.ReaperTransactionsConfig, chainID CHAIN_ID) *Reaper[CHAIN_ID] { +func NewReaper[CHAIN_ID chains.ID](lggr logger.Logger, store types.TxHistoryReaper[CHAIN_ID], txConfig types.ReaperTransactionsConfig, chainID CHAIN_ID) *Reaper[CHAIN_ID] { r := &Reaper[CHAIN_ID]{ store, txConfig, diff --git a/chains/txmgr/strategies.go b/chains/txmgr/strategies.go index d87d709..d6d8c84 100644 --- a/chains/txmgr/strategies.go +++ b/chains/txmgr/strategies.go @@ -6,14 +6,14 @@ import ( "github.com/google/uuid" - txmgrtypes "github.com/smartcontractkit/chainlink-framework/chains/txmgr/types" + "github.com/smartcontractkit/chainlink-framework/chains/txmgr/types" ) -var _ txmgrtypes.TxStrategy = SendEveryStrategy{} +var _ types.TxStrategy = SendEveryStrategy{} // NewQueueingTxStrategy creates a new TxStrategy that drops the oldest transactions after the // queue size is exceeded if a queue size is specified, and otherwise does not drop transactions. -func NewQueueingTxStrategy(subject uuid.UUID, queueSize uint32) (strategy txmgrtypes.TxStrategy) { +func NewQueueingTxStrategy(subject uuid.UUID, queueSize uint32) (strategy types.TxStrategy) { if queueSize > 0 { strategy = NewDropOldestStrategy(subject, queueSize) } else { @@ -23,7 +23,7 @@ func NewQueueingTxStrategy(subject uuid.UUID, queueSize uint32) (strategy txmgrt } // NewSendEveryStrategy creates a new TxStrategy that does not drop transactions. -func NewSendEveryStrategy() txmgrtypes.TxStrategy { +func NewSendEveryStrategy() types.TxStrategy { return SendEveryStrategy{} } @@ -31,11 +31,11 @@ func NewSendEveryStrategy() txmgrtypes.TxStrategy { type SendEveryStrategy struct{} func (SendEveryStrategy) Subject() uuid.NullUUID { return uuid.NullUUID{} } -func (SendEveryStrategy) PruneQueue(ctx context.Context, pruneService txmgrtypes.UnstartedTxQueuePruner) ([]int64, error) { +func (SendEveryStrategy) PruneQueue(ctx context.Context, pruneService types.UnstartedTxQueuePruner) ([]int64, error) { return nil, nil } -var _ txmgrtypes.TxStrategy = DropOldestStrategy{} +var _ types.TxStrategy = DropOldestStrategy{} // DropOldestStrategy will send the newest N transactions, older ones will be // removed from the queue @@ -54,7 +54,7 @@ func (s DropOldestStrategy) Subject() uuid.NullUUID { return uuid.NullUUID{UUID: s.subject, Valid: true} } -func (s DropOldestStrategy) PruneQueue(ctx context.Context, pruneService txmgrtypes.UnstartedTxQueuePruner) (ids []int64, err error) { +func (s DropOldestStrategy) PruneQueue(ctx context.Context, pruneService types.UnstartedTxQueuePruner) (ids []int64, err error) { // NOTE: We prune one less than the queue size to prevent the queue from exceeding the max queue size. Which could occur if a new transaction is added to the queue right after we prune. ids, err = pruneService.PruneUnstartedTxQueue(ctx, s.queueSize-1, s.subject) if err != nil { diff --git a/chains/txmgr/test_helpers.go b/chains/txmgr/test_helpers.go index 617960b..1709284 100644 --- a/chains/txmgr/test_helpers.go +++ b/chains/txmgr/test_helpers.go @@ -4,13 +4,13 @@ import ( "context" "time" - txmgrtypes "github.com/smartcontractkit/chainlink-framework/chains/txmgr/types" + "github.com/smartcontractkit/chainlink-framework/chains/txmgr/types" ) // TEST ONLY FUNCTIONS // these need to be exported for the txmgr tests to continue to work -func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) XXXTestSetClient(client txmgrtypes.TxmClient[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) { +func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) XXXTestSetClient(client types.TxmClient[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) { ec.client = client } diff --git a/chains/txmgr/tracker.go b/chains/txmgr/tracker.go index 59b0748..7d4729d 100644 --- a/chains/txmgr/tracker.go +++ b/chains/txmgr/tracker.go @@ -11,9 +11,8 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/sqlutil" "github.com/smartcontractkit/chainlink-common/pkg/utils/mailbox" "github.com/smartcontractkit/chainlink-framework/chains" - "github.com/smartcontractkit/chainlink-framework/chains/fees" - txmgrtypes "github.com/smartcontractkit/chainlink-framework/chains/txmgr/types" + "github.com/smartcontractkit/chainlink-framework/chains/txmgr/types" ) const ( @@ -37,13 +36,13 @@ type Tracker[ ADDR chains.Hashable, TX_HASH chains.Hashable, BLOCK_HASH chains.Hashable, - R txmgrtypes.ChainReceipt[TX_HASH, BLOCK_HASH], + R types.ChainReceipt[TX_HASH, BLOCK_HASH], SEQ chains.Sequence, FEE fees.Fee, ] struct { services.StateMachine - txStore txmgrtypes.TxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] - keyStore txmgrtypes.KeyStore[ADDR, CHAIN_ID, SEQ] + txStore types.TxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] + keyStore types.KeyStore[ADDR, CHAIN_ID, SEQ] chainID CHAIN_ID lggr logger.Logger @@ -65,12 +64,12 @@ func NewTracker[ ADDR chains.Hashable, TX_HASH chains.Hashable, BLOCK_HASH chains.Hashable, - R txmgrtypes.ChainReceipt[TX_HASH, BLOCK_HASH], + R types.ChainReceipt[TX_HASH, BLOCK_HASH], SEQ chains.Sequence, FEE fees.Fee, ]( - txStore txmgrtypes.TxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE], - keyStore txmgrtypes.KeyStore[ADDR, CHAIN_ID, SEQ], + txStore types.TxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE], + keyStore types.KeyStore[ADDR, CHAIN_ID, SEQ], chainID CHAIN_ID, lggr logger.Logger, ) *Tracker[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { @@ -298,7 +297,7 @@ func (tr *Tracker[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) handleTxesB } func (tr *Tracker[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) markTxFatal(ctx context.Context, - tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], + tx *types.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], errMsg string) error { tx.Error.SetValid(errMsg) diff --git a/chains/txmgr/txmgr.go b/chains/txmgr/txmgr.go index 7e9fcc7..b51e174 100644 --- a/chains/txmgr/txmgr.go +++ b/chains/txmgr/txmgr.go @@ -16,7 +16,6 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/services" commontypes "github.com/smartcontractkit/chainlink-common/pkg/types" "github.com/smartcontractkit/chainlink-common/pkg/utils" - "github.com/smartcontractkit/chainlink-framework/chains" "github.com/smartcontractkit/chainlink-framework/chains/fees" "github.com/smartcontractkit/chainlink-framework/chains/headtracker" diff --git a/chains/txmgr/types/client.go b/chains/txmgr/types/client.go index 0862746..ef0537d 100644 --- a/chains/txmgr/types/client.go +++ b/chains/txmgr/types/client.go @@ -7,10 +7,9 @@ import ( "time" "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-framework/chains" "github.com/smartcontractkit/chainlink-framework/chains/fees" "github.com/smartcontractkit/chainlink-framework/multinode" - - "github.com/smartcontractkit/chainlink-framework/chains" ) // TxmClient is a superset of all the methods needed for the txm diff --git a/chains/txmgr/types/forwarder_manager.go b/chains/txmgr/types/forwarder_manager.go index 9b9eef1..248c832 100644 --- a/chains/txmgr/types/forwarder_manager.go +++ b/chains/txmgr/types/forwarder_manager.go @@ -4,7 +4,6 @@ import ( "context" "github.com/smartcontractkit/chainlink-common/pkg/services" - "github.com/smartcontractkit/chainlink-framework/chains" ) diff --git a/chains/txmgr/types/tx.go b/chains/txmgr/types/tx.go index e87ae6b..693fca9 100644 --- a/chains/txmgr/types/tx.go +++ b/chains/txmgr/types/tx.go @@ -16,7 +16,6 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/sqlutil" clnull "github.com/smartcontractkit/chainlink-common/pkg/utils/null" - "github.com/smartcontractkit/chainlink-framework/chains" "github.com/smartcontractkit/chainlink-framework/chains/fees" ) diff --git a/chains/txmgr/types/tx_attempt_builder.go b/chains/txmgr/types/tx_attempt_builder.go index eefd482..5e85007 100644 --- a/chains/txmgr/types/tx_attempt_builder.go +++ b/chains/txmgr/types/tx_attempt_builder.go @@ -5,7 +5,6 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/services" - "github.com/smartcontractkit/chainlink-framework/chains" "github.com/smartcontractkit/chainlink-framework/chains/fees" "github.com/smartcontractkit/chainlink-framework/chains/headtracker" From 449ce8337199a51e891758820e1d9f60bd648c45 Mon Sep 17 00:00:00 2001 From: pavel-raykov Date: Thu, 16 Jan 2025 18:24:09 +0100 Subject: [PATCH 3/3] Minor --- chains/headtracker/head_broadcaster.go | 1 + chains/headtracker/head_listener.go | 1 + chains/headtracker/head_tracker.go | 1 + chains/txmgr/broadcaster.go | 3 ++- chains/txmgr/confirmer.go | 3 ++- chains/txmgr/reaper.go | 1 + chains/txmgr/resender.go | 2 +- chains/txmgr/tracker.go | 1 + chains/txmgr/txmgr.go | 1 + chains/txmgr/types/client.go | 1 + chains/txmgr/types/finalizer.go | 1 + chains/txmgr/types/forwarder_manager.go | 1 + chains/txmgr/types/sequence_tracker.go | 1 + chains/txmgr/types/tx.go | 1 + chains/txmgr/types/tx_attempt_builder.go | 1 + 15 files changed, 17 insertions(+), 3 deletions(-) diff --git a/chains/headtracker/head_broadcaster.go b/chains/headtracker/head_broadcaster.go index 40d624a..22f148a 100644 --- a/chains/headtracker/head_broadcaster.go +++ b/chains/headtracker/head_broadcaster.go @@ -10,6 +10,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/services" "github.com/smartcontractkit/chainlink-common/pkg/utils/mailbox" + "github.com/smartcontractkit/chainlink-framework/chains" ) diff --git a/chains/headtracker/head_listener.go b/chains/headtracker/head_listener.go index b8707d6..5a1c6c7 100644 --- a/chains/headtracker/head_listener.go +++ b/chains/headtracker/head_listener.go @@ -13,6 +13,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/services" + "github.com/smartcontractkit/chainlink-framework/chains" "github.com/smartcontractkit/chainlink-framework/chains/headtracker/types" ) diff --git a/chains/headtracker/head_tracker.go b/chains/headtracker/head_tracker.go index e890433..60832ec 100644 --- a/chains/headtracker/head_tracker.go +++ b/chains/headtracker/head_tracker.go @@ -13,6 +13,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/services" "github.com/smartcontractkit/chainlink-common/pkg/utils/mailbox" + "github.com/smartcontractkit/chainlink-framework/chains" "github.com/smartcontractkit/chainlink-framework/chains/headtracker/types" ) diff --git a/chains/txmgr/broadcaster.go b/chains/txmgr/broadcaster.go index 25f3789..70f5d7f 100644 --- a/chains/txmgr/broadcaster.go +++ b/chains/txmgr/broadcaster.go @@ -18,10 +18,11 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/services" "github.com/smartcontractkit/chainlink-common/pkg/utils" + "github.com/smartcontractkit/chainlink-framework/multinode" + "github.com/smartcontractkit/chainlink-framework/chains" "github.com/smartcontractkit/chainlink-framework/chains/fees" "github.com/smartcontractkit/chainlink-framework/chains/txmgr/types" - "github.com/smartcontractkit/chainlink-framework/multinode" ) const ( diff --git a/chains/txmgr/confirmer.go b/chains/txmgr/confirmer.go index 454ffa1..1f4bc7f 100644 --- a/chains/txmgr/confirmer.go +++ b/chains/txmgr/confirmer.go @@ -20,10 +20,11 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/services" "github.com/smartcontractkit/chainlink-common/pkg/utils/mailbox" + "github.com/smartcontractkit/chainlink-framework/multinode" + "github.com/smartcontractkit/chainlink-framework/chains" "github.com/smartcontractkit/chainlink-framework/chains/fees" "github.com/smartcontractkit/chainlink-framework/chains/txmgr/types" - "github.com/smartcontractkit/chainlink-framework/multinode" ) const ( diff --git a/chains/txmgr/reaper.go b/chains/txmgr/reaper.go index 885011b..1fb5000 100644 --- a/chains/txmgr/reaper.go +++ b/chains/txmgr/reaper.go @@ -7,6 +7,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/services" + "github.com/smartcontractkit/chainlink-framework/chains" "github.com/smartcontractkit/chainlink-framework/chains/txmgr/types" ) diff --git a/chains/txmgr/resender.go b/chains/txmgr/resender.go index 2f0b91e..7ae6a4b 100644 --- a/chains/txmgr/resender.go +++ b/chains/txmgr/resender.go @@ -9,9 +9,9 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/chains/label" "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/services" - "github.com/smartcontractkit/chainlink-framework/chains" "github.com/smartcontractkit/chainlink-framework/multinode" + "github.com/smartcontractkit/chainlink-framework/chains" "github.com/smartcontractkit/chainlink-framework/chains/fees" txmgrtypes "github.com/smartcontractkit/chainlink-framework/chains/txmgr/types" ) diff --git a/chains/txmgr/tracker.go b/chains/txmgr/tracker.go index 7d4729d..942c526 100644 --- a/chains/txmgr/tracker.go +++ b/chains/txmgr/tracker.go @@ -10,6 +10,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/services" "github.com/smartcontractkit/chainlink-common/pkg/sqlutil" "github.com/smartcontractkit/chainlink-common/pkg/utils/mailbox" + "github.com/smartcontractkit/chainlink-framework/chains" "github.com/smartcontractkit/chainlink-framework/chains/fees" "github.com/smartcontractkit/chainlink-framework/chains/txmgr/types" diff --git a/chains/txmgr/txmgr.go b/chains/txmgr/txmgr.go index b51e174..7e9fcc7 100644 --- a/chains/txmgr/txmgr.go +++ b/chains/txmgr/txmgr.go @@ -16,6 +16,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/services" commontypes "github.com/smartcontractkit/chainlink-common/pkg/types" "github.com/smartcontractkit/chainlink-common/pkg/utils" + "github.com/smartcontractkit/chainlink-framework/chains" "github.com/smartcontractkit/chainlink-framework/chains/fees" "github.com/smartcontractkit/chainlink-framework/chains/headtracker" diff --git a/chains/txmgr/types/client.go b/chains/txmgr/types/client.go index ef0537d..a94495b 100644 --- a/chains/txmgr/types/client.go +++ b/chains/txmgr/types/client.go @@ -7,6 +7,7 @@ import ( "time" "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-framework/chains" "github.com/smartcontractkit/chainlink-framework/chains/fees" "github.com/smartcontractkit/chainlink-framework/multinode" diff --git a/chains/txmgr/types/finalizer.go b/chains/txmgr/types/finalizer.go index ceb2d56..1dea050 100644 --- a/chains/txmgr/types/finalizer.go +++ b/chains/txmgr/types/finalizer.go @@ -6,6 +6,7 @@ import ( "github.com/google/uuid" "github.com/smartcontractkit/chainlink-common/pkg/services" + "github.com/smartcontractkit/chainlink-framework/chains" ) diff --git a/chains/txmgr/types/forwarder_manager.go b/chains/txmgr/types/forwarder_manager.go index 248c832..9b9eef1 100644 --- a/chains/txmgr/types/forwarder_manager.go +++ b/chains/txmgr/types/forwarder_manager.go @@ -4,6 +4,7 @@ import ( "context" "github.com/smartcontractkit/chainlink-common/pkg/services" + "github.com/smartcontractkit/chainlink-framework/chains" ) diff --git a/chains/txmgr/types/sequence_tracker.go b/chains/txmgr/types/sequence_tracker.go index 6a28ad7..b0cfbce 100644 --- a/chains/txmgr/types/sequence_tracker.go +++ b/chains/txmgr/types/sequence_tracker.go @@ -4,6 +4,7 @@ import ( "context" "github.com/smartcontractkit/chainlink-common/pkg/services" + "github.com/smartcontractkit/chainlink-framework/chains" ) diff --git a/chains/txmgr/types/tx.go b/chains/txmgr/types/tx.go index 693fca9..e87ae6b 100644 --- a/chains/txmgr/types/tx.go +++ b/chains/txmgr/types/tx.go @@ -16,6 +16,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/sqlutil" clnull "github.com/smartcontractkit/chainlink-common/pkg/utils/null" + "github.com/smartcontractkit/chainlink-framework/chains" "github.com/smartcontractkit/chainlink-framework/chains/fees" ) diff --git a/chains/txmgr/types/tx_attempt_builder.go b/chains/txmgr/types/tx_attempt_builder.go index 5e85007..eefd482 100644 --- a/chains/txmgr/types/tx_attempt_builder.go +++ b/chains/txmgr/types/tx_attempt_builder.go @@ -5,6 +5,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/services" + "github.com/smartcontractkit/chainlink-framework/chains" "github.com/smartcontractkit/chainlink-framework/chains/fees" "github.com/smartcontractkit/chainlink-framework/chains/headtracker"