diff --git a/go.mod b/go.mod index c493c7122..5df29d36b 100644 --- a/go.mod +++ b/go.mod @@ -13,12 +13,12 @@ require ( github.com/google/uuid v1.6.0 github.com/hashicorp/go-plugin v1.6.2 github.com/jackc/pgx/v4 v4.18.3 - github.com/jpillora/backoff v1.0.0 - github.com/pelletier/go-toml/v2 v2.2.0 - github.com/prometheus/client_golang v1.17.0 + github.com/pelletier/go-toml/v2 v2.2.2 + github.com/prometheus/client_golang v1.20.5 github.com/smartcontractkit/chainlink-common v0.4.2-0.20250116214855-f49c5c27db51 + github.com/smartcontractkit/chainlink-framework/multinode v0.0.0-20250121205514-f73e2f86c23b github.com/smartcontractkit/libocr v0.0.0-20241007185508-adbe57025f12 - github.com/stretchr/testify v1.9.0 + github.com/stretchr/testify v1.10.0 go.uber.org/zap v1.27.0 golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 golang.org/x/sync v0.10.0 @@ -69,11 +69,13 @@ require ( github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect github.com/jackc/pgtype v1.14.0 // indirect github.com/jmoiron/sqlx v1.4.0 // indirect + github.com/jpillora/backoff v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.17.11 // indirect github.com/klauspost/cpuid/v2 v2.2.8 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect github.com/leodido/go-urn v1.2.0 // indirect github.com/lib/pq v1.10.9 // indirect github.com/linkedin/goavro/v2 v2.12.0 // indirect @@ -82,7 +84,6 @@ require ( github.com/marcboeker/go-duckdb v1.8.3 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/miekg/dns v1.1.35 // indirect github.com/mitchellh/go-testing-interface v1.14.1 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect @@ -90,14 +91,15 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/mostynb/zstdpool-freelist v0.0.0-20201229113212-927304c0c3b1 // indirect github.com/mr-tron/base58 v1.2.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/oklog/run v1.0.0 // indirect github.com/onsi/gomega v1.24.1 // indirect github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect - github.com/prometheus/common v0.44.0 // indirect - github.com/prometheus/procfs v0.11.1 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.60.1 // indirect + github.com/prometheus/procfs v0.15.1 // indirect github.com/riferrei/srclient v0.5.4 // indirect github.com/rogpeppe/go-internal v1.13.1 // indirect github.com/ryanuber/go-glob v1.0.0 // indirect diff --git a/go.sum b/go.sum index f604bc88e..6a5cf6ae7 100644 --- a/go.sum +++ b/go.sum @@ -93,6 +93,8 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= +github.com/cometbft/cometbft v0.37.5 h1:/U/TlgMh4NdnXNo+YU9T2NMCWyhXNDF34Mx582jlvq0= +github.com/cometbft/cometbft v0.37.5/go.mod h1:QC+mU0lBhKn8r9qvmnq53Dmf3DWBt4VtkcKw2C81wxY= github.com/confluentinc/confluent-kafka-go/v2 v2.3.0 h1:icCHutJouWlQREayFwCc7lxDAhws08td+W3/gdqgZts= github.com/confluentinc/confluent-kafka-go/v2 v2.3.0/go.mod h1:/VTy8iEpe6mD9pkCH5BhijlUl8ulUXymKv1Qig5Rgb8= github.com/containerd/cgroups v1.0.4 h1:jN/mbWBEaz+T1pi5OFtnkQ+8qnmEbAr1Oo1FRm5B0dA= @@ -377,6 +379,8 @@ github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= @@ -418,8 +422,6 @@ github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= -github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.35 h1:oTfOaDH+mZkdcgdIjH6yBajRGtIwcwcaR+rt23ZSrJs= github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= @@ -458,6 +460,8 @@ github.com/mostynb/zstdpool-freelist v0.0.0-20201229113212-927304c0c3b1 h1:mPMvm github.com/mostynb/zstdpool-freelist v0.0.0-20201229113212-927304c0c3b1/go.mod h1:ye2e/VUEtE2BHE+G/QcKkcLQVAEJoYRFj5VUOQatCRE= github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229/go.mod h1:0aYXnNPJ8l7uZxf45rWW1a/uME32OF0rhiYGNQ2oF2E= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= @@ -478,8 +482,10 @@ github.com/opencontainers/runc v1.1.3 h1:vIXrkId+0/J2Ymu2m7VjGvbSlAId9XNRPhn2p4b github.com/opencontainers/runc v1.1.3/go.mod h1:1J5XiS+vdZ3wCyZybsuxXZWGrgSr8fFJHLXuG2PsnNg= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml/v2 v2.2.0 h1:QLgLl2yMN7N+ruc31VynXs1vhMZa7CeHHejIeBAsoHo= -github.com/pelletier/go-toml/v2 v2.2.0/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= +github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/petermattis/goid v0.0.0-20240813172612-4fcff4a6cae7 h1:Dx7Ovyv/SFnMFw3fD4oEoeorXc6saIiQ23LrGLth0Gw= +github.com/petermattis/goid v0.0.0-20240813172612-4fcff4a6cae7/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= @@ -493,21 +499,21 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= -github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= -github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= +github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= +github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 h1:v7DLqVdK4VrYkVD5diGdl4sxJurKJEMnODWRJlxV9oM= -github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= -github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= +github.com/prometheus/common v0.60.1 h1:FUas6GcOw66yB/73KC+BOZoFJmbo/1pojoILArPAaSc= +github.com/prometheus/common v0.60.1/go.mod h1:h0LYf1R1deLSKtD4Vdg8gy4RuOvENW2J/h19V5NADQw= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI= -github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/regen-network/protobuf v1.3.3-alpha.regen.1 h1:OHEc+q5iIAXpqiqFKeLpu5NwTIkVXUs48vFMwzqpqY4= github.com/regen-network/protobuf v1.3.3-alpha.regen.1/go.mod h1:2DjTFR1HhMQhiWC5sZ4OhQ3+NtdbZ6oBDKQwq5Ou+FI= @@ -528,6 +534,8 @@ github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIH github.com/santhosh-tekuri/jsonschema/v5 v5.0.0/go.mod h1:FKdcjfQW6rpZSnxxUvEA5H/cDPdvJ/SZJQLWWXWGrZ0= github.com/santhosh-tekuri/jsonschema/v5 v5.2.0 h1:WCcC4vZDS1tYNxjWlwRJZQy28r8CMoggKnxNzxsVDMQ= github.com/santhosh-tekuri/jsonschema/v5 v5.2.0/go.mod h1:FKdcjfQW6rpZSnxxUvEA5H/cDPdvJ/SZJQLWWXWGrZ0= +github.com/sasha-s/go-deadlock v0.3.5 h1:tNCOEEDG6tBqrNDOX35j/7hL5FcFViG6awUGROb2NsU= +github.com/sasha-s/go-deadlock v0.3.5/go.mod h1:bugP6EGbdGYObIlx7pUZtWqlvo8k9H6vCBBsiChJQ5U= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/scylladb/go-reflectx v1.0.1 h1:b917wZM7189pZdlND9PbIJ6NQxfDPfBvUaQ7cjj1iZQ= github.com/scylladb/go-reflectx v1.0.1/go.mod h1:rWnOfDIRWBGN0miMLIcoPt/Dhi2doCMZqwMCJ3KupFc= @@ -545,6 +553,8 @@ github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/smartcontractkit/chainlink-common v0.4.2-0.20250116214855-f49c5c27db51 h1:YdjQiEu5uHWM1ApwdV+nLyJmu1+tt3IeiwPKNGoXwBI= github.com/smartcontractkit/chainlink-common v0.4.2-0.20250116214855-f49c5c27db51/go.mod h1:yti7e1+G9hhkYhj+L5sVUULn9Bn3bBL5/AxaNqdJ5YQ= +github.com/smartcontractkit/chainlink-framework/multinode v0.0.0-20250121205514-f73e2f86c23b h1:TO1pwFeQKDOmv3loFiLJvYhtymuTgQUw9WgtwK1rueg= +github.com/smartcontractkit/chainlink-framework/multinode v0.0.0-20250121205514-f73e2f86c23b/go.mod h1:4JqpgFy01LaqG1yM2iFTzwX3ZgcAvW9WdstBZQgPHzU= github.com/smartcontractkit/grpc-proxy v0.0.0-20240830132753-a7e17fec5ab7 h1:12ijqMM9tvYVEm+nR826WsrNi6zCKpwBhuApq127wHs= github.com/smartcontractkit/grpc-proxy v0.0.0-20240830132753-a7e17fec5ab7/go.mod h1:FX7/bVdoep147QQhsOPkYsPEXhGZjeYx6lBSaSXtZOA= github.com/smartcontractkit/libocr v0.0.0-20241007185508-adbe57025f12 h1:NzZGjaqez21I3DU7objl3xExTH4fxYvzTqar8DC6360= @@ -581,8 +591,9 @@ github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1F github.com/stretchr/testify v1.7.5/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/teris-io/shortid v0.0.0-20171029131806-771a37caa5cf/go.mod h1:M8agBzgqHIhgj7wEn9/0hJUZcrvt9VY+Ln+S1I5Mha0= github.com/teris-io/shortid v0.0.0-20201117134242-e59966efd125 h1:3SNcvBmEPE1YlB1JpVZouslJpI3GBNoiqW7+wb0Rz7w= diff --git a/integration-tests/go.mod b/integration-tests/go.mod index 0cd768e8c..1c35065a7 100644 --- a/integration-tests/go.mod +++ b/integration-tests/go.mod @@ -14,13 +14,13 @@ require ( github.com/lib/pq v1.10.9 github.com/pelletier/go-toml/v2 v2.2.3 github.com/rs/zerolog v1.33.0 - github.com/smartcontractkit/chainlink-common v0.4.2-0.20250116214855-f49c5c27db51 - github.com/smartcontractkit/chainlink-solana v1.1.1-0.20250116220429-efced748f123 + github.com/smartcontractkit/chainlink-common v0.4.2-0.20250121141917-62443f4b3c30 + github.com/smartcontractkit/chainlink-solana v1.1.2-0.20250121212724-40883916c5a2 github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.19 github.com/smartcontractkit/chainlink-testing-framework/seth v1.50.9 - github.com/smartcontractkit/chainlink/deployment v0.0.0-20250116221411-f1017f196025 - github.com/smartcontractkit/chainlink/integration-tests v0.0.0-20250116221411-f1017f196025 - github.com/smartcontractkit/chainlink/v2 v2.14.0-mercury-20240807.0.20250116221411-f1017f196025 + github.com/smartcontractkit/chainlink/deployment v0.0.0-20250121213906-b5b59e7d0dbd + github.com/smartcontractkit/chainlink/integration-tests v0.0.0-20250121213906-b5b59e7d0dbd + github.com/smartcontractkit/chainlink/v2 v2.19.0-ccip1.5.16-alpha.0.0.20250121213906-b5b59e7d0dbd github.com/smartcontractkit/libocr v0.0.0-20241223215956-e5b78d8e3919 github.com/stretchr/testify v1.10.0 github.com/testcontainers/testcontainers-go v0.34.0 @@ -54,9 +54,10 @@ require ( github.com/NethermindEth/juno v0.3.1 // indirect github.com/NethermindEth/starknet.go v0.7.1-0.20240401080518-34a506f3cfdb // indirect github.com/VictoriaMetrics/fastcache v1.12.2 // indirect - github.com/XSAM/otelsql v0.27.0 // indirect + github.com/XSAM/otelsql v0.29.0 // indirect github.com/alecthomas/units v0.0.0-20240626203959-61d1e3462e30 // indirect github.com/andybalholm/brotli v1.1.1 // indirect + github.com/apache/arrow-go/v18 v18.0.0 // indirect github.com/armon/go-metrics v0.4.1 // indirect github.com/atombender/go-jsonschema v0.16.1-0.20240916205339-a74cd4e2851c // indirect github.com/avast/retry-go v3.0.0+incompatible // indirect @@ -193,6 +194,7 @@ require ( github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect github.com/google/btree v1.1.2 // indirect + github.com/google/flatbuffers v24.3.25+incompatible // indirect github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/go-github/v41 v41.0.0 // indirect @@ -270,6 +272,7 @@ require ( github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect + github.com/marcboeker/go-duckdb v1.8.3 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.16 // indirect @@ -308,6 +311,7 @@ require ( github.com/pelletier/go-toml v1.9.5 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect github.com/petermattis/goid v0.0.0-20240813172612-4fcff4a6cae7 // indirect + github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect @@ -338,13 +342,13 @@ require ( github.com/slack-go/slack v0.15.0 // indirect github.com/smartcontractkit/chain-selectors v1.0.36 // indirect github.com/smartcontractkit/chainlink-automation v0.8.1 // indirect - github.com/smartcontractkit/chainlink-ccip v0.0.0-20250116220331-a7ce842532c7 // indirect + github.com/smartcontractkit/chainlink-ccip v0.0.0-20250120130359-cc025272bbff // indirect github.com/smartcontractkit/chainlink-cosmos v0.5.2-0.20241202195413-82468150ac1e // indirect github.com/smartcontractkit/chainlink-data-streams v0.1.1-0.20250115135646-ac859d85e7e3 // indirect github.com/smartcontractkit/chainlink-feeds v0.1.1 // indirect - github.com/smartcontractkit/chainlink-framework/multinode v0.0.0-20250115203616-a2ea5e50b260 // indirect + github.com/smartcontractkit/chainlink-framework/multinode v0.0.0-20250121205514-f73e2f86c23b // indirect github.com/smartcontractkit/chainlink-protos/orchestrator v0.4.0 // indirect - github.com/smartcontractkit/chainlink-starknet/relayer v0.1.1-0.20250116220926-2bc922c6ae8c // indirect + github.com/smartcontractkit/chainlink-starknet/relayer v0.1.1-0.20250117224137-afdcdd75070d // indirect github.com/smartcontractkit/grpc-proxy v0.0.0-20240830132753-a7e17fec5ab7 // indirect github.com/smartcontractkit/tdh2/go/ocr2/decryptionplugin v0.0.0-20241009055228-33d0c0bf38de // indirect github.com/smartcontractkit/tdh2/go/tdh2 v0.0.0-20241009055228-33d0c0bf38de // indirect @@ -383,6 +387,7 @@ require ( github.com/xlab/treeprint v1.2.0 // indirect github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect + github.com/zeebo/xxh3 v1.0.2 // indirect github.com/zondax/hid v0.9.2 // indirect github.com/zondax/ledger-go v0.14.3 // indirect go.dedis.ch/fixbuf v1.0.3 // indirect diff --git a/integration-tests/go.sum b/integration-tests/go.sum index f72967d68..8bf8553c0 100644 --- a/integration-tests/go.sum +++ b/integration-tests/go.sum @@ -125,8 +125,8 @@ github.com/VictoriaMetrics/fastcache v1.12.2 h1:N0y9ASrJ0F6h0QaC3o6uJb3NIZ9VKLjC github.com/VictoriaMetrics/fastcache v1.12.2/go.mod h1:AmC+Nzz1+3G2eCPapF6UcsnkThDcMsQicp4xDukwJYI= github.com/VividCortex/gohistogram v1.0.0 h1:6+hBz+qvs0JOrrNhhmR7lFxo5sINxBCGXrdtl/UvroE= github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= -github.com/XSAM/otelsql v0.27.0 h1:i9xtxtdcqXV768a5C6SoT/RkG+ue3JTOgkYInzlTOqs= -github.com/XSAM/otelsql v0.27.0/go.mod h1:0mFB3TvLa7NCuhm/2nU7/b2wEtsczkj8Rey8ygO7V+A= +github.com/XSAM/otelsql v0.29.0 h1:pEw9YXXs8ZrGRYfDc0cmArIz9lci5b42gmP5+tA1Huc= +github.com/XSAM/otelsql v0.29.0/go.mod h1:d3/0xGIGC5RVEE+Ld7KotwaLy6zDeaF3fLJHOPpdN2w= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -145,6 +145,8 @@ github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOL github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apache/arrow-go/v18 v18.0.0 h1:1dBDaSbH3LtulTyOVYaBCHO3yVRwjV+TZaqn3g6V7ZM= github.com/apache/arrow-go/v18 v18.0.0/go.mod h1:t6+cWRSmKgdQ6HsxisQjok+jBpKGhRDiqcf3p0p/F+A= +github.com/apache/thrift v0.21.0 h1:tdPmh/ptjE1IJnhbhrcl2++TauVjy242rkV/UzJChnE= +github.com/apache/thrift v0.21.0/go.mod h1:W1H8aR/QRtYNvrPeFXBtobyRkd0/YVhTc6i07XIAgDw= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= @@ -884,6 +886,8 @@ github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dv github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4= +github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= github.com/klauspost/compress v1.11.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= @@ -975,6 +979,10 @@ github.com/miekg/dns v1.1.61/go.mod h1:mnAarhS3nWaW+NVP2wTkYVIZyHNJ098SJZUki3eyk github.com/mimoo/StrobeGo v0.0.0-20181016162300-f8f6d4d2b643/go.mod h1:43+3pMjjKimDBf5Kr4ZFNGbLql1zKkbImw+fZbw3geM= github.com/mimoo/StrobeGo v0.0.0-20210601165009-122bf33a46e0 h1:QRUSJEgZn2Snx0EmT/QLXibWjSUDjKWvXIT19NBVp94= github.com/mimoo/StrobeGo v0.0.0-20210601165009-122bf33a46e0/go.mod h1:43+3pMjjKimDBf5Kr4ZFNGbLql1zKkbImw+fZbw3geM= +github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs= +github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY= +github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI= +github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE= github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g= github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= @@ -1219,32 +1227,32 @@ github.com/smartcontractkit/chain-selectors v1.0.36 h1:KSpO8I+JOiuyN4FuXsV471sPo github.com/smartcontractkit/chain-selectors v1.0.36/go.mod h1:xsKM0aN3YGcQKTPRPDDtPx2l4mlTN1Djmg0VVXV40b8= github.com/smartcontractkit/chainlink-automation v0.8.1 h1:sTc9LKpBvcKPc1JDYAmgBc2xpDKBco/Q4h4ydl6+UUU= github.com/smartcontractkit/chainlink-automation v0.8.1/go.mod h1:Iij36PvWZ6blrdC5A/nrQUBuf3MH3JvsBB9sSyc9W08= -github.com/smartcontractkit/chainlink-ccip v0.0.0-20250116220331-a7ce842532c7 h1:Qp7OjFbDz4dOZXi45X3uWlu28dEA1chQZzRYWCQS3lI= -github.com/smartcontractkit/chainlink-ccip v0.0.0-20250116220331-a7ce842532c7/go.mod h1:JJZMCB75aVSAiPNW032F9WUKTlLztTd8bbQB5MEaZa4= -github.com/smartcontractkit/chainlink-common v0.4.2-0.20250116214855-f49c5c27db51 h1:YdjQiEu5uHWM1ApwdV+nLyJmu1+tt3IeiwPKNGoXwBI= -github.com/smartcontractkit/chainlink-common v0.4.2-0.20250116214855-f49c5c27db51/go.mod h1:yti7e1+G9hhkYhj+L5sVUULn9Bn3bBL5/AxaNqdJ5YQ= +github.com/smartcontractkit/chainlink-ccip v0.0.0-20250120130359-cc025272bbff h1:ZEOlcleVdT0/y9V5yjgFJF0j7MpvdrFmKis/xmFNIgE= +github.com/smartcontractkit/chainlink-ccip v0.0.0-20250120130359-cc025272bbff/go.mod h1:JJZMCB75aVSAiPNW032F9WUKTlLztTd8bbQB5MEaZa4= +github.com/smartcontractkit/chainlink-common v0.4.2-0.20250121141917-62443f4b3c30 h1:cLqp7DciHn7MXX3Y2QAiBHjuNPUFfTRreXfjWbUcM6o= +github.com/smartcontractkit/chainlink-common v0.4.2-0.20250121141917-62443f4b3c30/go.mod h1:V3BHfvLnQNBUoZ4bGjD29ZPhyzPE++DkYkhvPb9tcRs= github.com/smartcontractkit/chainlink-cosmos v0.5.2-0.20241202195413-82468150ac1e h1:PRoeby6ZlTuTkv2f+7tVU4+zboTfRzI+beECynF4JQ0= github.com/smartcontractkit/chainlink-cosmos v0.5.2-0.20241202195413-82468150ac1e/go.mod h1:mUh5/woemsVaHgTorA080hrYmO3syBCmPdnWc/5dOqk= github.com/smartcontractkit/chainlink-data-streams v0.1.1-0.20250115135646-ac859d85e7e3 h1:GcPYNVFYjB065CNq0h8nK/VeU08nUkHgBX0cJIEpuHY= github.com/smartcontractkit/chainlink-data-streams v0.1.1-0.20250115135646-ac859d85e7e3/go.mod h1:pDZagSGjs9U+l4YIFhveDznMHqxuuz+5vRxvVgpbdr8= github.com/smartcontractkit/chainlink-feeds v0.1.1 h1:JzvUOM/OgGQA1sOqTXXl52R6AnNt+Wg64sVG+XSA49c= github.com/smartcontractkit/chainlink-feeds v0.1.1/go.mod h1:55EZ94HlKCfAsUiKUTNI7QlE/3d3IwTlsU3YNa/nBb4= -github.com/smartcontractkit/chainlink-framework/multinode v0.0.0-20250115203616-a2ea5e50b260 h1:See2isL6KdrTJDlVKWv8qiyYqWhYUcubU2e5yKXV1oY= -github.com/smartcontractkit/chainlink-framework/multinode v0.0.0-20250115203616-a2ea5e50b260/go.mod h1:4JqpgFy01LaqG1yM2iFTzwX3ZgcAvW9WdstBZQgPHzU= +github.com/smartcontractkit/chainlink-framework/multinode v0.0.0-20250121205514-f73e2f86c23b h1:TO1pwFeQKDOmv3loFiLJvYhtymuTgQUw9WgtwK1rueg= +github.com/smartcontractkit/chainlink-framework/multinode v0.0.0-20250121205514-f73e2f86c23b/go.mod h1:4JqpgFy01LaqG1yM2iFTzwX3ZgcAvW9WdstBZQgPHzU= github.com/smartcontractkit/chainlink-protos/orchestrator v0.4.0 h1:ZBat8EBvE2LpSQR9U1gEbRV6PfAkiFdINmQ8nVnXIAQ= github.com/smartcontractkit/chainlink-protos/orchestrator v0.4.0/go.mod h1:m/A3lqD7ms/RsQ9BT5P2uceYY0QX5mIt4KQxT2G6qEo= -github.com/smartcontractkit/chainlink-starknet/relayer v0.1.1-0.20250116220926-2bc922c6ae8c h1:MMevODDz6f7NdxdnUUC7TRANiYVjP6H7OUc4ZGAMIXc= -github.com/smartcontractkit/chainlink-starknet/relayer v0.1.1-0.20250116220926-2bc922c6ae8c/go.mod h1:lgG9JT2P19KnYuBheKIis5ZeCO+AaSta+RfzvwDQS2Y= +github.com/smartcontractkit/chainlink-starknet/relayer v0.1.1-0.20250117224137-afdcdd75070d h1:hf1Ust1ub9r3+PgRgiry3065QXCXmw6P7YImnue1NEw= +github.com/smartcontractkit/chainlink-starknet/relayer v0.1.1-0.20250117224137-afdcdd75070d/go.mod h1:lgG9JT2P19KnYuBheKIis5ZeCO+AaSta+RfzvwDQS2Y= github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.19 h1:9PMwKNqFKc5FXf4VchyD3CGzZelnSgi13fgVdT2X7T4= github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.19/go.mod h1:ag7LEgejsVtPXaUNkcoFPpAoDkl1J8V2HSbqVUxfEtk= github.com/smartcontractkit/chainlink-testing-framework/seth v1.50.9 h1:yB1x5UXvpZNka+5h57yo1/GrKfXKCqMzChCISpldZx4= github.com/smartcontractkit/chainlink-testing-framework/seth v1.50.9/go.mod h1:lJk0atEJ5Zyo3Tqrmf1Pl9jUEe79EgDb9bD3K5OTUBI= -github.com/smartcontractkit/chainlink/deployment v0.0.0-20250116221411-f1017f196025 h1:NnXn6R6TXFDQBHZshZ1A36R8eYc3aJpDCSm/IxOqqek= -github.com/smartcontractkit/chainlink/deployment v0.0.0-20250116221411-f1017f196025/go.mod h1:iUBNBDqo32gZd3hcenBf2bbOBqY1Y1+GJCNkXQJnZ2w= -github.com/smartcontractkit/chainlink/integration-tests v0.0.0-20250116221411-f1017f196025 h1:iCnm8RGhymDn3vEB89KqmNuFr8EjEGZtBY2AAQWijTg= -github.com/smartcontractkit/chainlink/integration-tests v0.0.0-20250116221411-f1017f196025/go.mod h1:hRrW+Khit75rcWgW79k2fW2B5stK8FOM1Q+5qNIs3sk= -github.com/smartcontractkit/chainlink/v2 v2.14.0-mercury-20240807.0.20250116221411-f1017f196025 h1:9PPgofob8jyEWKoewqNsx7PfGc3yM7Nz+VP3xQ9uSBU= -github.com/smartcontractkit/chainlink/v2 v2.14.0-mercury-20240807.0.20250116221411-f1017f196025/go.mod h1:1mkDfoB1bWDTELMAP23gxPJImG4ycNQMTosda9sfrMA= +github.com/smartcontractkit/chainlink/deployment v0.0.0-20250121213906-b5b59e7d0dbd h1:BXx44Cmb755eBaXY9TrjIKpiRvQxAIVy+xDIZ+ghPa8= +github.com/smartcontractkit/chainlink/deployment v0.0.0-20250121213906-b5b59e7d0dbd/go.mod h1:+1J0gyfyJ4ZDUM2HrTFoZ6jEakHcvgAJNynCIfZxfqE= +github.com/smartcontractkit/chainlink/integration-tests v0.0.0-20250121213906-b5b59e7d0dbd h1:CaxxNuJKGwXZ07IWoHgc4N0QudKgHw9mgyf7azUZEgc= +github.com/smartcontractkit/chainlink/integration-tests v0.0.0-20250121213906-b5b59e7d0dbd/go.mod h1:XPJA1fuzrjd8wQlq2SZyh3sFFhMlRnSfD76aJZMeqfM= +github.com/smartcontractkit/chainlink/v2 v2.19.0-ccip1.5.16-alpha.0.0.20250121213906-b5b59e7d0dbd h1:3WEbSzawSMAWGWaOmjNpXM2hS3wFo8r5Tuf9xm65IDo= +github.com/smartcontractkit/chainlink/v2 v2.19.0-ccip1.5.16-alpha.0.0.20250121213906-b5b59e7d0dbd/go.mod h1:kmlxEGU8POLhXFvgOftaGN1rqYEdSgQJQuA0gS91SfI= github.com/smartcontractkit/grpc-proxy v0.0.0-20240830132753-a7e17fec5ab7 h1:12ijqMM9tvYVEm+nR826WsrNi6zCKpwBhuApq127wHs= github.com/smartcontractkit/grpc-proxy v0.0.0-20240830132753-a7e17fec5ab7/go.mod h1:FX7/bVdoep147QQhsOPkYsPEXhGZjeYx6lBSaSXtZOA= github.com/smartcontractkit/libocr v0.0.0-20241223215956-e5b78d8e3919 h1:IpGoPTXpvllN38kT2z2j13sifJMz4nbHglidvop7mfg= @@ -1384,6 +1392,8 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= diff --git a/integration-tests/relayinterface/chain_components_test.go b/integration-tests/relayinterface/chain_components_test.go index b3e4950ab..d3b85557a 100644 --- a/integration-tests/relayinterface/chain_components_test.go +++ b/integration-tests/relayinterface/chain_components_test.go @@ -384,7 +384,7 @@ func (h *helper) waitForTX(t *testing.T, sig solana.Signature, commitment rpc.Co defer sub.Unsubscribe() - res, err := sub.Recv(context.Background()) + res, err := sub.Recv(tests.Context(t)) require.NoError(t, err) if res.Value.Err != nil { diff --git a/integration-tests/testconfig/testconfig.go b/integration-tests/testconfig/testconfig.go index 1f482b7f5..36a86c4db 100644 --- a/integration-tests/testconfig/testconfig.go +++ b/integration-tests/testconfig/testconfig.go @@ -265,13 +265,9 @@ func (c *TestConfig) GetNodeConfigTOML() (string, error) { url = c.GetURL() } - mnConfig := solcfg.MultiNodeConfig{ - MultiNode: solcfg.MultiNode{ - Enabled: ptr.Ptr(true), - SyncThreshold: ptr.Ptr(uint32(170)), - }, - } - mnConfig.SetDefaults() + mnConfig := solcfg.NewDefaultMultiNodeConfig() + mnConfig.MultiNode.Enabled = ptr.Ptr(true) + mnConfig.MultiNode.SyncThreshold = ptr.Ptr(uint32(170)) var nodes []*solcfg.Node for i, u := range url { diff --git a/pkg/solana/chain.go b/pkg/solana/chain.go index 9316b2981..3d2f03070 100644 --- a/pkg/solana/chain.go +++ b/pkg/solana/chain.go @@ -23,9 +23,9 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/types" "github.com/smartcontractkit/chainlink-common/pkg/types/core" "github.com/smartcontractkit/chainlink-common/pkg/utils" + mn "github.com/smartcontractkit/chainlink-framework/multinode" "github.com/smartcontractkit/chainlink-solana/pkg/solana/client" - mn "github.com/smartcontractkit/chainlink-solana/pkg/solana/client/multinode" "github.com/smartcontractkit/chainlink-solana/pkg/solana/config" "github.com/smartcontractkit/chainlink-solana/pkg/solana/internal" "github.com/smartcontractkit/chainlink-solana/pkg/solana/logpoller" @@ -277,7 +277,7 @@ func newChain(id string, cfg *config.TOMLConfig, ks core.Keystore, lggr logger.L sendOnlyNodes = append(sendOnlyNodes, newSendOnly) } else { newNode := mn.NewNode[mn.StringID, *client.Head, *client.MultiNodeClient]( - mnCfg, mnCfg, lggr, *nodeInfo.URL.URL(), nil, *nodeInfo.Name, + mnCfg, mnCfg, lggr, nodeInfo.URL.URL(), nil, *nodeInfo.Name, i, mn.StringID(id), 0, rpcClient, chainFamily) nodes = append(nodes, newNode) } @@ -550,7 +550,7 @@ func (c *chain) HealthReport() map[string]error { services.CopyHealth(report, c.txm.HealthReport()) services.CopyHealth(report, c.balanceMonitor.HealthReport()) if c.cfg.MultiNode.Enabled() { - report[c.multiNode.Name()] = c.multiNode.Healthy() + services.CopyHealth(report, c.multiNode.HealthReport()) report[c.txSender.Name()] = c.txSender.Healthy() } return report diff --git a/pkg/solana/chain_test.go b/pkg/solana/chain_test.go index c2182e7c1..df51c313f 100644 --- a/pkg/solana/chain_test.go +++ b/pkg/solana/chain_test.go @@ -26,9 +26,9 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/sqlutil/sqltest" "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" + mn "github.com/smartcontractkit/chainlink-framework/multinode" "github.com/smartcontractkit/chainlink-solana/pkg/solana/client" - mn "github.com/smartcontractkit/chainlink-solana/pkg/solana/client/multinode" solcfg "github.com/smartcontractkit/chainlink-solana/pkg/solana/config" "github.com/smartcontractkit/chainlink-solana/pkg/solana/fees" "github.com/smartcontractkit/chainlink-solana/pkg/solana/txm/mocks" @@ -335,12 +335,8 @@ func TestSolanaChain_MultiNode_GetClient(t *testing.T) { ch := solcfg.Chain{} ch.SetDefaults() - mnCfg := solcfg.MultiNodeConfig{ - MultiNode: solcfg.MultiNode{ - Enabled: ptr(true), - }, - } - mnCfg.SetDefaults() + mnCfg := solcfg.NewDefaultMultiNodeConfig() + mnCfg.MultiNode.Enabled = ptr(true) cfg := &solcfg.TOMLConfig{ ChainID: ptr("devnet"), diff --git a/pkg/solana/client/classify_errors.go b/pkg/solana/client/classify_errors.go index ae3402694..deea39a5b 100644 --- a/pkg/solana/client/classify_errors.go +++ b/pkg/solana/client/classify_errors.go @@ -5,7 +5,7 @@ import ( "github.com/gagliardetto/solana-go" - mn "github.com/smartcontractkit/chainlink-solana/pkg/solana/client/multinode" + mn "github.com/smartcontractkit/chainlink-framework/multinode" ) // Solana error patters diff --git a/pkg/solana/client/classify_errors_test.go b/pkg/solana/client/classify_errors_test.go index 29d6a3bc0..2efab4a4a 100644 --- a/pkg/solana/client/classify_errors_test.go +++ b/pkg/solana/client/classify_errors_test.go @@ -6,7 +6,7 @@ import ( "github.com/stretchr/testify/assert" - mn "github.com/smartcontractkit/chainlink-solana/pkg/solana/client/multinode" + mn "github.com/smartcontractkit/chainlink-framework/multinode" ) func TestClassifySendError(t *testing.T) { diff --git a/pkg/solana/client/client.go b/pkg/solana/client/client.go index e3f170783..3ff54f06a 100644 --- a/pkg/solana/client/client.go +++ b/pkg/solana/client/client.go @@ -11,8 +11,8 @@ import ( "golang.org/x/sync/singleflight" "github.com/smartcontractkit/chainlink-common/pkg/logger" + mn "github.com/smartcontractkit/chainlink-framework/multinode" - mn "github.com/smartcontractkit/chainlink-solana/pkg/solana/client/multinode" "github.com/smartcontractkit/chainlink-solana/pkg/solana/config" "github.com/smartcontractkit/chainlink-solana/pkg/solana/monitor" ) diff --git a/pkg/solana/client/client_test.go b/pkg/solana/client/client_test.go index 7d207fa0a..5da07dd3b 100644 --- a/pkg/solana/client/client_test.go +++ b/pkg/solana/client/client_test.go @@ -20,8 +20,8 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" + mn "github.com/smartcontractkit/chainlink-framework/multinode" - mn "github.com/smartcontractkit/chainlink-solana/pkg/solana/client/multinode" "github.com/smartcontractkit/chainlink-solana/pkg/solana/config" "github.com/smartcontractkit/chainlink-solana/pkg/solana/monitor" ) diff --git a/pkg/solana/client/mocks/reader_writer.go b/pkg/solana/client/mocks/reader_writer.go index 232320005..fa5f5adfb 100644 --- a/pkg/solana/client/mocks/reader_writer.go +++ b/pkg/solana/client/mocks/reader_writer.go @@ -5,10 +5,11 @@ package mocks import ( context "context" - rpc "github.com/gagliardetto/solana-go/rpc" - multinode "github.com/smartcontractkit/chainlink-solana/pkg/solana/client/multinode" + multinode "github.com/smartcontractkit/chainlink-framework/multinode" mock "github.com/stretchr/testify/mock" + rpc "github.com/gagliardetto/solana-go/rpc" + solana "github.com/gagliardetto/solana-go" ) diff --git a/pkg/solana/client/multi_client.go b/pkg/solana/client/multi_client.go index 76c6447dc..6eea5343f 100644 --- a/pkg/solana/client/multi_client.go +++ b/pkg/solana/client/multi_client.go @@ -6,7 +6,7 @@ import ( "github.com/gagliardetto/solana-go" "github.com/gagliardetto/solana-go/rpc" - mn "github.com/smartcontractkit/chainlink-solana/pkg/solana/client/multinode" + mn "github.com/smartcontractkit/chainlink-framework/multinode" ) var _ ReaderWriter = (*MultiClient)(nil) diff --git a/pkg/solana/client/multinode/ctx.go b/pkg/solana/client/multinode/ctx.go deleted file mode 100644 index 57b2fc8a8..000000000 --- a/pkg/solana/client/multinode/ctx.go +++ /dev/null @@ -1,17 +0,0 @@ -package client - -import "context" - -type multiNodeContextKey int - -const ( - contextKeyHeathCheckRequest multiNodeContextKey = iota + 1 -) - -func CtxAddHealthCheckFlag(ctx context.Context) context.Context { - return context.WithValue(ctx, contextKeyHeathCheckRequest, struct{}{}) -} - -func CtxIsHeathCheckRequest(ctx context.Context) bool { - return ctx.Value(contextKeyHeathCheckRequest) != nil -} diff --git a/pkg/solana/client/multinode/models.go b/pkg/solana/client/multinode/models.go deleted file mode 100644 index 526bb25c8..000000000 --- a/pkg/solana/client/multinode/models.go +++ /dev/null @@ -1,121 +0,0 @@ -package client - -import ( - "bytes" - "fmt" -) - -type SendTxReturnCode int - -// SendTxReturnCode is a generalized client error that dictates what should be the next action, depending on the RPC error response. -const ( - Successful SendTxReturnCode = iota + 1 - Fatal // Unrecoverable error. Most likely the attempt should be thrown away. - Retryable // The error returned by the RPC indicates that if we retry with the same attempt, the tx will eventually go through. - Underpriced // Attempt was underpriced. New estimation is needed with bumped gas price. - Unknown // Tx failed with an error response that is not recognized by the client. - Unsupported // Attempt failed with an error response that is not supported by the client for the given chain. - TransactionAlreadyKnown // The transaction that was sent has already been received by the RPC. - InsufficientFunds // Tx was rejected due to insufficient funds. - ExceedsMaxFee // Attempt's fee was higher than the node's limit and got rejected. - FeeOutOfValidRange // This error is returned when we use a fee price suggested from an RPC, but the network rejects the attempt due to an invalid range(mostly used by L2 chains). Retry by requesting a new suggested fee price. - TerminallyStuck // The error returned when a transaction is or could get terminally stuck in the mempool without any chance of inclusion. - sendTxReturnCodeLen // tracks the number of errors. Must always be last -) - -// sendTxSevereErrors - error codes which signal that transaction would never be accepted in its current form by the node -var sendTxSevereErrors = []SendTxReturnCode{Fatal, Underpriced, Unsupported, ExceedsMaxFee, FeeOutOfValidRange, Unknown} - -// sendTxSuccessfulCodes - error codes which signal that transaction was accepted by the node -var sendTxSuccessfulCodes = []SendTxReturnCode{Successful, TransactionAlreadyKnown} - -func (c SendTxReturnCode) String() string { - switch c { - case Successful: - return "Successful" - case Fatal: - return "Fatal" - case Retryable: - return "Retryable" - case Underpriced: - return "Underpriced" - case Unknown: - return "Unknown" - case Unsupported: - return "Unsupported" - case TransactionAlreadyKnown: - return "TransactionAlreadyKnown" - case InsufficientFunds: - return "InsufficientFunds" - case ExceedsMaxFee: - return "ExceedsMaxFee" - case FeeOutOfValidRange: - return "FeeOutOfValidRange" - case TerminallyStuck: - return "TerminallyStuck" - default: - return fmt.Sprintf("SendTxReturnCode(%d)", c) - } -} - -type NodeTier int - -const ( - Primary = NodeTier(iota) - Secondary -) - -func (n NodeTier) String() string { - switch n { - case Primary: - return "primary" - case Secondary: - return "secondary" - default: - return fmt.Sprintf("NodeTier(%d)", n) - } -} - -// syncStatus - defines problems related to RPC's state synchronization. Can be used as a bitmask to define multiple issues -type syncStatus int - -const ( - // syncStatusSynced - RPC is fully synced - syncStatusSynced = 0 - // syncStatusNotInSyncWithPool - RPC is lagging behind the highest block observed within the pool of RPCs - syncStatusNotInSyncWithPool syncStatus = 1 << iota - // syncStatusNoNewHead - RPC failed to produce a new head for too long - syncStatusNoNewHead - // syncStatusNoNewFinalizedHead - RPC failed to produce a new finalized head for too long - syncStatusNoNewFinalizedHead - syncStatusLen -) - -func (s syncStatus) String() string { - if s == syncStatusSynced { - return "Synced" - } - var result bytes.Buffer - for i := syncStatusNotInSyncWithPool; i < syncStatusLen; i = i << 1 { - if i&s == 0 { - continue - } - result.WriteString(i.string()) - result.WriteString(",") - } - result.Truncate(result.Len() - 1) - return result.String() -} - -func (s syncStatus) string() string { - switch s { - case syncStatusNotInSyncWithPool: - return "NotInSyncWithRPCPool" - case syncStatusNoNewHead: - return "NoNewHead" - case syncStatusNoNewFinalizedHead: - return "NoNewFinalizedHead" - default: - return fmt.Sprintf("syncStatus(%d)", s) - } -} diff --git a/pkg/solana/client/multinode/multi_node.go b/pkg/solana/client/multinode/multi_node.go deleted file mode 100644 index 02bec62c4..000000000 --- a/pkg/solana/client/multinode/multi_node.go +++ /dev/null @@ -1,381 +0,0 @@ -package client - -import ( - "context" - "errors" - "fmt" - "math/big" - "sync" - "time" - - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" - - "github.com/smartcontractkit/chainlink-common/pkg/logger" - "github.com/smartcontractkit/chainlink-common/pkg/services" -) - -var ( - // PromMultiNodeRPCNodeStates reports current RPC node state - PromMultiNodeRPCNodeStates = promauto.NewGaugeVec(prometheus.GaugeOpts{ - Name: "solana_multi_node_states", - Help: "The number of RPC nodes currently in the given state for the given chain", - }, []string{"network", "chainId", "state"}) - ErroringNodeError = fmt.Errorf("no live nodes available") -) - -// MultiNode is a generalized multi node client interface that includes methods to interact with different chains. -// It also handles multiple node RPC connections simultaneously. -type MultiNode[ - CHAIN_ID ID, - RPC any, -] struct { - services.StateMachine - primaryNodes []Node[CHAIN_ID, RPC] - sendOnlyNodes []SendOnlyNode[CHAIN_ID, RPC] - chainID CHAIN_ID - lggr logger.SugaredLogger - selectionMode string - nodeSelector NodeSelector[CHAIN_ID, RPC] - leaseDuration time.Duration - leaseTicker *time.Ticker - chainFamily string - reportInterval time.Duration - deathDeclarationDelay time.Duration - - activeMu sync.RWMutex - activeNode Node[CHAIN_ID, RPC] - - chStop services.StopChan - wg sync.WaitGroup -} - -func NewMultiNode[ - CHAIN_ID ID, - RPC any, -]( - lggr logger.Logger, - selectionMode string, // type of the "best" RPC selector (e.g HighestHead, RoundRobin, etc.) - leaseDuration time.Duration, // defines interval on which new "best" RPC should be selected - primaryNodes []Node[CHAIN_ID, RPC], - sendOnlyNodes []SendOnlyNode[CHAIN_ID, RPC], - chainID CHAIN_ID, // configured chain ID (used to verify that passed primaryNodes belong to the same chain) - chainFamily string, // name of the chain family - used in the metrics - deathDeclarationDelay time.Duration, -) *MultiNode[CHAIN_ID, RPC] { - nodeSelector := newNodeSelector(selectionMode, primaryNodes) - // Prometheus' default interval is 15s, set this to under 7.5s to avoid - // aliasing (see: https://en.wikipedia.org/wiki/Nyquist_frequency) - const reportInterval = 6500 * time.Millisecond - c := &MultiNode[CHAIN_ID, RPC]{ - primaryNodes: primaryNodes, - sendOnlyNodes: sendOnlyNodes, - chainID: chainID, - lggr: logger.Sugared(lggr).Named("MultiNode").With("chainID", chainID.String()), - selectionMode: selectionMode, - nodeSelector: nodeSelector, - chStop: make(services.StopChan), - leaseDuration: leaseDuration, - chainFamily: chainFamily, - reportInterval: reportInterval, - deathDeclarationDelay: deathDeclarationDelay, - } - - c.lggr.Debugf("The MultiNode is configured to use NodeSelectionMode: %s", selectionMode) - - return c -} - -func (c *MultiNode[CHAIN_ID, RPC]) Name() string { - return c.lggr.Name() -} - -func (c *MultiNode[CHAIN_ID, RPC]) ChainID() CHAIN_ID { - return c.chainID -} - -func (c *MultiNode[CHAIN_ID, RPC]) DoAll(ctx context.Context, do func(ctx context.Context, rpc RPC, isSendOnly bool)) error { - var err error - ok := c.IfNotStopped(func() { - callsCompleted := 0 - for _, n := range c.primaryNodes { - select { - case <-ctx.Done(): - err = ctx.Err() - return - default: - if n.State() != NodeStateAlive { - continue - } - do(ctx, n.RPC(), false) - callsCompleted++ - } - } - if callsCompleted == 0 { - err = ErroringNodeError - } - - for _, n := range c.sendOnlyNodes { - select { - case <-ctx.Done(): - err = ctx.Err() - return - default: - if n.State() != NodeStateAlive { - continue - } - do(ctx, n.RPC(), true) - } - } - }) - if !ok { - return errors.New("MultiNode is stopped") - } - return err -} - -func (c *MultiNode[CHAIN_ID, RPC]) NodeStates() map[string]string { - states := map[string]string{} - for _, n := range c.primaryNodes { - states[n.Name()] = n.State().String() - } - for _, n := range c.sendOnlyNodes { - states[n.Name()] = n.State().String() - } - return states -} - -// Start starts every node in the pool -// -// Nodes handle their own redialing and runloops, so this function does not -// return any error if the nodes aren't available -func (c *MultiNode[CHAIN_ID, RPC]) Start(ctx context.Context) error { - return c.StartOnce("MultiNode", func() (merr error) { - if len(c.primaryNodes) == 0 { - return fmt.Errorf("no available nodes for chain %s", c.chainID.String()) - } - var ms services.MultiStart - for _, n := range c.primaryNodes { - if n.ConfiguredChainID().String() != c.chainID.String() { - return ms.CloseBecause(fmt.Errorf("node %s has configured chain ID %s which does not match multinode configured chain ID of %s", n.String(), n.ConfiguredChainID().String(), c.chainID.String())) - } - n.SetPoolChainInfoProvider(c) - // node will handle its own redialing and automatic recovery - if err := ms.Start(ctx, n); err != nil { - return err - } - } - for _, s := range c.sendOnlyNodes { - if s.ConfiguredChainID().String() != c.chainID.String() { - return ms.CloseBecause(fmt.Errorf("sendonly node %s has configured chain ID %s which does not match multinode configured chain ID of %s", s.String(), s.ConfiguredChainID().String(), c.chainID.String())) - } - if err := ms.Start(ctx, s); err != nil { - return err - } - } - c.wg.Add(1) - go c.runLoop() - - if c.leaseDuration.Seconds() > 0 && c.selectionMode != NodeSelectionModeRoundRobin { - c.lggr.Infof("The MultiNode will switch to best node every %s", c.leaseDuration.String()) - c.wg.Add(1) - go c.checkLeaseLoop() - } else { - c.lggr.Info("Best node switching is disabled") - } - - return nil - }) -} - -// Close tears down the MultiNode and closes all nodes -func (c *MultiNode[CHAIN_ID, RPC]) Close() error { - return c.StopOnce("MultiNode", func() error { - close(c.chStop) - c.wg.Wait() - - return services.CloseAll(services.MultiCloser(c.primaryNodes), services.MultiCloser(c.sendOnlyNodes)) - }) -} - -// SelectRPC returns an RPC of an active node. If there are no active nodes it returns an error. -// Call this method from your chain-specific client implementation to access any chain-specific rpc calls. -func (c *MultiNode[CHAIN_ID, RPC]) SelectRPC() (rpc RPC, err error) { - n, err := c.selectNode() - if err != nil { - return rpc, err - } - return n.RPC(), nil -} - -// selectNode returns the active Node, if it is still NodeStateAlive, otherwise it selects a new one from the NodeSelector. -func (c *MultiNode[CHAIN_ID, RPC]) selectNode() (node Node[CHAIN_ID, RPC], err error) { - c.activeMu.RLock() - node = c.activeNode - c.activeMu.RUnlock() - if node != nil && node.State() == NodeStateAlive { - return // still alive - } - - // select a new one - c.activeMu.Lock() - defer c.activeMu.Unlock() - node = c.activeNode - if node != nil && node.State() == NodeStateAlive { - return // another goroutine beat us here - } - - if c.activeNode != nil { - c.activeNode.UnsubscribeAllExceptAliveLoop() - } - c.activeNode = c.nodeSelector.Select() - - if c.activeNode == nil { - c.lggr.Criticalw("No live RPC nodes available", "NodeSelectionMode", c.nodeSelector.Name()) - errmsg := fmt.Errorf("no live nodes available for chain %s", c.chainID.String()) - c.SvcErrBuffer.Append(errmsg) - err = ErroringNodeError - } - - return c.activeNode, err -} - -// LatestChainInfo - returns number of live nodes available in the pool, so we can prevent the last alive node in a pool from being marked as out-of-sync. -// Return highest ChainInfo most recently received by the alive nodes. -// E.g. If Node A's the most recent block is 10 and highest 15 and for Node B it's - 12 and 14. This method will return 12. -func (c *MultiNode[CHAIN_ID, RPC]) LatestChainInfo() (int, ChainInfo) { - var nLiveNodes int - ch := ChainInfo{ - TotalDifficulty: big.NewInt(0), - } - for _, n := range c.primaryNodes { - if s, nodeChainInfo := n.StateAndLatest(); s == NodeStateAlive { - nLiveNodes++ - ch.BlockNumber = max(ch.BlockNumber, nodeChainInfo.BlockNumber) - ch.FinalizedBlockNumber = max(ch.FinalizedBlockNumber, nodeChainInfo.FinalizedBlockNumber) - ch.TotalDifficulty = MaxTotalDifficulty(ch.TotalDifficulty, nodeChainInfo.TotalDifficulty) - } - } - return nLiveNodes, ch -} - -// HighestUserObservations - returns highest ChainInfo ever observed by any user of the MultiNode -func (c *MultiNode[CHAIN_ID, RPC]) HighestUserObservations() ChainInfo { - ch := ChainInfo{ - TotalDifficulty: big.NewInt(0), - } - for _, n := range c.primaryNodes { - nodeChainInfo := n.HighestUserObservations() - ch.BlockNumber = max(ch.BlockNumber, nodeChainInfo.BlockNumber) - ch.FinalizedBlockNumber = max(ch.FinalizedBlockNumber, nodeChainInfo.FinalizedBlockNumber) - ch.TotalDifficulty = MaxTotalDifficulty(ch.TotalDifficulty, nodeChainInfo.TotalDifficulty) - } - return ch -} - -func (c *MultiNode[CHAIN_ID, RPC]) checkLease() { - bestNode := c.nodeSelector.Select() - for _, n := range c.primaryNodes { - // Terminate client subscriptions. Services are responsible for reconnecting, which will be routed to the new - // best node. Only terminate connections with more than 1 subscription to account for the aliveLoop subscription - if n.State() == NodeStateAlive && n != bestNode { - c.lggr.Infof("Switching to best node from %q to %q", n.String(), bestNode.String()) - n.UnsubscribeAllExceptAliveLoop() - } - } - - c.activeMu.Lock() - defer c.activeMu.Unlock() - if bestNode != c.activeNode { - if c.activeNode != nil { - c.activeNode.UnsubscribeAllExceptAliveLoop() - } - c.activeNode = bestNode - } -} - -func (c *MultiNode[CHAIN_ID, RPC]) checkLeaseLoop() { - defer c.wg.Done() - c.leaseTicker = time.NewTicker(c.leaseDuration) - defer c.leaseTicker.Stop() - - for { - select { - case <-c.leaseTicker.C: - c.checkLease() - case <-c.chStop: - return - } - } -} - -func (c *MultiNode[CHAIN_ID, RPC]) runLoop() { - defer c.wg.Done() - - nodeStates := make([]nodeWithState, len(c.primaryNodes)) - for i, n := range c.primaryNodes { - nodeStates[i] = nodeWithState{ - Node: n.String(), - State: n.State().String(), - DeadSince: nil, - } - } - - c.report(nodeStates) - - monitor := services.NewTicker(c.reportInterval) - defer monitor.Stop() - - for { - select { - case <-monitor.C: - c.report(nodeStates) - case <-c.chStop: - return - } - } -} - -type nodeWithState struct { - Node string - State string - DeadSince *time.Time -} - -func (c *MultiNode[CHAIN_ID, RPC]) report(nodesStateInfo []nodeWithState) { - start := time.Now() - var dead int - counts := make(map[NodeState]int) - for i, n := range c.primaryNodes { - state := n.State() - counts[state]++ - nodesStateInfo[i].State = state.String() - if state == NodeStateAlive { - nodesStateInfo[i].DeadSince = nil - continue - } - - if nodesStateInfo[i].DeadSince == nil { - nodesStateInfo[i].DeadSince = &start - } - - if start.Sub(*nodesStateInfo[i].DeadSince) >= c.deathDeclarationDelay { - dead++ - } - } - for _, state := range allNodeStates { - count := counts[state] - PromMultiNodeRPCNodeStates.WithLabelValues(c.chainFamily, c.chainID.String(), state.String()).Set(float64(count)) - } - - total := len(c.primaryNodes) - live := total - dead - c.lggr.Tracew(fmt.Sprintf("MultiNode state: %d/%d nodes are alive", live, total), "nodeStates", nodesStateInfo) - if total == dead { - rerr := fmt.Errorf("no primary nodes available: 0/%d nodes are alive", total) - c.lggr.Criticalw(rerr.Error(), "nodeStates", nodesStateInfo) - c.SvcErrBuffer.Append(rerr) - } else if dead > 0 { - c.lggr.Warnw(fmt.Sprintf("At least one primary node is dead: %d/%d nodes are alive", live, total), "nodeStates", nodesStateInfo) - } -} diff --git a/pkg/solana/client/multinode/node.go b/pkg/solana/client/multinode/node.go deleted file mode 100644 index afdece741..000000000 --- a/pkg/solana/client/multinode/node.go +++ /dev/null @@ -1,328 +0,0 @@ -package client - -import ( - "context" - "errors" - "fmt" - "net/url" - "sync" - "time" - - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" - - "github.com/smartcontractkit/chainlink-common/pkg/logger" - "github.com/smartcontractkit/chainlink-common/pkg/services" -) - -const QueryTimeout = 10 * time.Second - -var errInvalidChainID = errors.New("invalid chain id") - -var ( - promPoolRPCNodeVerifies = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "solana_pool_rpc_node_verifies", - Help: "The total number of chain ID verifications for the given RPC node", - }, []string{"network", "chainID", "nodeName"}) - promPoolRPCNodeVerifiesFailed = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "solana_pool_rpc_node_verifies_failed", - Help: "The total number of failed chain ID verifications for the given RPC node", - }, []string{"network", "chainID", "nodeName"}) - promPoolRPCNodeVerifiesSuccess = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "solana_pool_rpc_node_verifies_success", - Help: "The total number of successful chain ID verifications for the given RPC node", - }, []string{"network", "chainID", "nodeName"}) -) - -type NodeConfig interface { - PollFailureThreshold() uint32 - PollInterval() time.Duration - SelectionMode() string - SyncThreshold() uint32 - NodeIsSyncingEnabled() bool - FinalizedBlockPollInterval() time.Duration - EnforceRepeatableRead() bool - DeathDeclarationDelay() time.Duration -} - -type ChainConfig interface { - NodeNoNewHeadsThreshold() time.Duration - NoNewFinalizedHeadsThreshold() time.Duration - FinalityDepth() uint32 - FinalityTagEnabled() bool - FinalizedBlockOffset() uint32 -} - -type Node[ - CHAIN_ID ID, - RPC any, -] interface { - // State returns most accurate state of the Node on the moment of call. - // While some of the checks may be performed in the background and State may return cached value, critical, like - // `FinalizedBlockOutOfSync`, must be executed upon every call. - State() NodeState - // StateAndLatest returns nodeState with the latest ChainInfo observed by Node during current lifecycle. - StateAndLatest() (NodeState, ChainInfo) - // HighestUserObservations - returns highest ChainInfo ever observed by underlying RPC excluding results of health check requests - HighestUserObservations() ChainInfo - SetPoolChainInfoProvider(PoolChainInfoProvider) - // Name is a unique identifier for this node. - Name() string - // String - returns string representation of the node, useful for debugging (name + URLS used to connect to the RPC) - String() string - RPC() RPC - // UnsubscribeAllExceptAliveLoop - closes all subscriptions except the aliveLoop subscription - UnsubscribeAllExceptAliveLoop() - ConfiguredChainID() CHAIN_ID - // Order - returns priority order configured for the RPC - Order() int32 - // Start - starts health checks - Start(context.Context) error - Close() error -} - -type node[ - CHAIN_ID ID, - HEAD Head, - RPC RPCClient[CHAIN_ID, HEAD], -] struct { - services.StateMachine - lfcLog logger.Logger - name string - id int - chainID CHAIN_ID - nodePoolCfg NodeConfig - chainCfg ChainConfig - order int32 - chainFamily string - - ws url.URL - http *url.URL - - rpc RPC - - stateMu sync.RWMutex // protects state* fields - state NodeState - - poolInfoProvider PoolChainInfoProvider - - stopCh services.StopChan - // wg waits for subsidiary goroutines - wg sync.WaitGroup - - healthCheckSubs []Subscription -} - -func NewNode[ - CHAIN_ID ID, - HEAD Head, - RPC RPCClient[CHAIN_ID, HEAD], -]( - nodeCfg NodeConfig, - chainCfg ChainConfig, - lggr logger.Logger, - wsuri url.URL, - httpuri *url.URL, - name string, - id int, - chainID CHAIN_ID, - nodeOrder int32, - rpc RPC, - chainFamily string, -) Node[CHAIN_ID, RPC] { - n := new(node[CHAIN_ID, HEAD, RPC]) - n.name = name - n.id = id - n.chainID = chainID - n.nodePoolCfg = nodeCfg - n.chainCfg = chainCfg - n.ws = wsuri - n.order = nodeOrder - if httpuri != nil { - n.http = httpuri - } - n.stopCh = make(services.StopChan) - lggr = logger.Named(lggr, "Node") - lggr = logger.With(lggr, - "nodeTier", Primary.String(), - "nodeName", name, - "node", n.String(), - "chainID", chainID, - "nodeOrder", n.order, - ) - n.lfcLog = logger.Named(lggr, "Lifecycle") - n.rpc = rpc - n.chainFamily = chainFamily - return n -} - -func (n *node[CHAIN_ID, HEAD, RPC]) String() string { - s := fmt.Sprintf("(%s)%s:%s", Primary.String(), n.name, n.ws.String()) - if n.http != nil { - s = s + fmt.Sprintf(":%s", n.http.String()) - } - return s -} - -func (n *node[CHAIN_ID, HEAD, RPC]) ConfiguredChainID() (chainID CHAIN_ID) { - return n.chainID -} - -func (n *node[CHAIN_ID, HEAD, RPC]) Name() string { - return n.name -} - -func (n *node[CHAIN_ID, HEAD, RPC]) RPC() RPC { - return n.rpc -} - -// unsubscribeAllExceptAliveLoop is not thread-safe; it should only be called -// while holding the stateMu lock. -func (n *node[CHAIN_ID, HEAD, RPC]) unsubscribeAllExceptAliveLoop() { - n.rpc.UnsubscribeAllExcept(n.healthCheckSubs...) -} - -func (n *node[CHAIN_ID, HEAD, RPC]) UnsubscribeAllExceptAliveLoop() { - n.stateMu.Lock() - defer n.stateMu.Unlock() - n.unsubscribeAllExceptAliveLoop() -} - -func (n *node[CHAIN_ID, HEAD, RPC]) Close() error { - return n.StopOnce(n.name, n.close) -} - -func (n *node[CHAIN_ID, HEAD, RPC]) close() error { - defer func() { - n.wg.Wait() - n.rpc.Close() - }() - - n.stateMu.Lock() - defer n.stateMu.Unlock() - - close(n.stopCh) - n.state = NodeStateClosed - return nil -} - -// Start dials and verifies the node -// Should only be called once in a node's lifecycle -// Return value is necessary to conform to interface but this will never -// actually return an error. -func (n *node[CHAIN_ID, HEAD, RPC]) Start(startCtx context.Context) error { - return n.StartOnce(n.name, func() error { - n.start(startCtx) - return nil - }) -} - -// start initially dials the node and verifies chain ID -// This spins off lifecycle goroutines. -// Not thread-safe. -// Node lifecycle is synchronous: only one goroutine should be running at a -// time. -func (n *node[CHAIN_ID, HEAD, RPC]) start(startCtx context.Context) { - if n.state != NodeStateUndialed { - panic(fmt.Sprintf("cannot dial node with state %v", n.state)) - } - - if err := n.rpc.Dial(startCtx); err != nil { - n.lfcLog.Errorw("Dial failed: Node is unreachable", "err", err) - n.declareUnreachable() - return - } - n.setState(NodeStateDialed) - - state := n.verifyConn(startCtx, n.lfcLog) - n.declareState(state) -} - -// verifyChainID checks that connection to the node matches the given chain ID -// Not thread-safe -// Pure verifyChainID: does not mutate node "state" field. -func (n *node[CHAIN_ID, HEAD, RPC]) verifyChainID(callerCtx context.Context, lggr logger.Logger) NodeState { - promPoolRPCNodeVerifies.WithLabelValues(n.chainFamily, n.chainID.String(), n.name).Inc() - promFailed := func() { - promPoolRPCNodeVerifiesFailed.WithLabelValues(n.chainFamily, n.chainID.String(), n.name).Inc() - } - - st := n.getCachedState() - switch st { - case NodeStateClosed: - // The node is already closed, and any subsequent transition is invalid. - // To make spotting such transitions a bit easier, return the invalid node state. - return NodeStateLen - case NodeStateDialed, NodeStateOutOfSync, NodeStateInvalidChainID, NodeStateSyncing: - default: - panic(fmt.Sprintf("cannot verify node in state %v", st)) - } - - var chainID CHAIN_ID - var err error - if chainID, err = n.rpc.ChainID(callerCtx); err != nil { - promFailed() - lggr.Errorw("Failed to verify chain ID for node", "err", err, "nodeState", n.getCachedState()) - return NodeStateUnreachable - } else if chainID.String() != n.chainID.String() { - promFailed() - err = fmt.Errorf( - "rpc ChainID doesn't match local chain ID: RPC ID=%s, local ID=%s, node name=%s: %w", - chainID.String(), - n.chainID.String(), - n.name, - errInvalidChainID, - ) - lggr.Errorw("Failed to verify RPC node; remote endpoint returned the wrong chain ID", "err", err, "nodeState", n.getCachedState()) - return NodeStateInvalidChainID - } - - promPoolRPCNodeVerifiesSuccess.WithLabelValues(n.chainFamily, n.chainID.String(), n.name).Inc() - - return NodeStateAlive -} - -// createVerifiedConn - establishes new connection with the RPC and verifies that it's valid: chainID matches, and it's not syncing. -// Returns desired state if one of the verifications fails. Otherwise, returns NodeStateAlive. -func (n *node[CHAIN_ID, HEAD, RPC]) createVerifiedConn(ctx context.Context, lggr logger.Logger) NodeState { - if err := n.rpc.Dial(ctx); err != nil { - n.lfcLog.Errorw("Dial failed: Node is unreachable", "err", err, "nodeState", n.getCachedState()) - return NodeStateUnreachable - } - - return n.verifyConn(ctx, lggr) -} - -// verifyConn - verifies that current connection is valid: chainID matches, and it's not syncing. -// Returns desired state if one of the verifications fails. Otherwise, returns NodeStateAlive. -func (n *node[CHAIN_ID, HEAD, RPC]) verifyConn(ctx context.Context, lggr logger.Logger) NodeState { - state := n.verifyChainID(ctx, lggr) - if state != NodeStateAlive { - return state - } - - if n.nodePoolCfg.NodeIsSyncingEnabled() { - isSyncing, err := n.rpc.IsSyncing(ctx) - if err != nil { - lggr.Errorw("Unexpected error while verifying RPC node synchronization status", "err", err, "nodeState", n.getCachedState()) - return NodeStateUnreachable - } - - if isSyncing { - lggr.Errorw("Verification failed: Node is syncing", "nodeState", n.getCachedState()) - return NodeStateSyncing - } - } - - return NodeStateAlive -} - -func (n *node[CHAIN_ID, HEAD, RPC]) Order() int32 { - return n.order -} - -func (n *node[CHAIN_ID, HEAD, RPC]) newCtx() (context.Context, context.CancelFunc) { - ctx, cancel := n.stopCh.NewCtx() - ctx = CtxAddHealthCheckFlag(ctx) - return ctx, cancel -} diff --git a/pkg/solana/client/multinode/node_fsm.go b/pkg/solana/client/multinode/node_fsm.go deleted file mode 100644 index 136910868..000000000 --- a/pkg/solana/client/multinode/node_fsm.go +++ /dev/null @@ -1,370 +0,0 @@ -package client - -import ( - "fmt" - - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" -) - -var ( - promPoolRPCNodeTransitionsToAlive = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "solana_pool_rpc_node_num_transitions_to_alive", - Help: transitionString(NodeStateAlive), - }, []string{"chainID", "nodeName"}) - promPoolRPCNodeTransitionsToInSync = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "solana_pool_rpc_node_num_transitions_to_in_sync", - Help: fmt.Sprintf("%s to %s", transitionString(NodeStateOutOfSync), NodeStateAlive), - }, []string{"chainID", "nodeName"}) - promPoolRPCNodeTransitionsToOutOfSync = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "solana_pool_rpc_node_num_transitions_to_out_of_sync", - Help: transitionString(NodeStateOutOfSync), - }, []string{"chainID", "nodeName"}) - promPoolRPCNodeTransitionsToUnreachable = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "solana_pool_rpc_node_num_transitions_to_unreachable", - Help: transitionString(NodeStateUnreachable), - }, []string{"chainID", "nodeName"}) - promPoolRPCNodeTransitionsToInvalidChainID = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "solana_pool_rpc_node_num_transitions_to_invalid_chain_id", - Help: transitionString(NodeStateInvalidChainID), - }, []string{"chainID", "nodeName"}) - promPoolRPCNodeTransitionsToUnusable = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "solana_pool_rpc_node_num_transitions_to_unusable", - Help: transitionString(NodeStateUnusable), - }, []string{"chainID", "nodeName"}) - promPoolRPCNodeTransitionsToSyncing = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "solana_pool_rpc_node_num_transitions_to_syncing", - Help: transitionString(NodeStateSyncing), - }, []string{"chainID", "nodeName"}) -) - -// NodeState represents the current state of the node -// Node is a FSM (finite state machine) -type NodeState int - -func (n NodeState) String() string { - switch n { - case NodeStateUndialed: - return "Undialed" - case NodeStateDialed: - return "Dialed" - case NodeStateInvalidChainID: - return "InvalidChainID" - case NodeStateAlive: - return "Alive" - case NodeStateUnreachable: - return "Unreachable" - case NodeStateUnusable: - return "Unusable" - case NodeStateOutOfSync: - return "OutOfSync" - case NodeStateClosed: - return "Closed" - case NodeStateSyncing: - return "Syncing" - case NodeStateFinalizedBlockOutOfSync: - return "FinalizedBlockOutOfSync" - default: - return fmt.Sprintf("NodeState(%d)", n) - } -} - -// GoString prints a prettier state -func (n NodeState) GoString() string { - return fmt.Sprintf("NodeState%s(%d)", n.String(), n) -} - -const ( - // NodeStateUndialed is the first state of a virgin node - NodeStateUndialed = NodeState(iota) - // NodeStateDialed is after a node has successfully dialed but before it has verified the correct chain ID - NodeStateDialed - // NodeStateInvalidChainID is after chain ID verification failed - NodeStateInvalidChainID - // NodeStateAlive is a healthy node after chain ID verification succeeded - NodeStateAlive - // NodeStateUnreachable is a node that cannot be dialed or has disconnected - NodeStateUnreachable - // NodeStateOutOfSync is a node that is accepting connections but exceeded - // the failure threshold without sending any new heads. It will be - // disconnected, then put into a revive loop and re-awakened after redial - // if a new head arrives - NodeStateOutOfSync - // NodeStateUnusable is a sendonly node that has an invalid URL that can never be reached - NodeStateUnusable - // NodeStateClosed is after the connection has been closed and the node is at the end of its lifecycle - NodeStateClosed - // NodeStateSyncing is a node that is actively back-filling blockchain. Usually, it's a newly set up node that is - // still syncing the chain. The main difference from `NodeStateOutOfSync` is that it represents state relative - // to other primary nodes configured in the MultiNode. In contrast, `NodeStateSyncing` represents the internal state of - // the node (RPC). - NodeStateSyncing - // nodeStateFinalizedBlockOutOfSync - node is lagging behind on latest finalized block - NodeStateFinalizedBlockOutOfSync - // nodeStateLen tracks the number of states - NodeStateLen -) - -// allNodeStates represents all possible states a node can be in -var allNodeStates []NodeState - -func init() { - for s := NodeState(0); s < NodeStateLen; s++ { - allNodeStates = append(allNodeStates, s) - } -} - -// FSM methods - -// State allows reading the current state of the node. -func (n *node[CHAIN_ID, HEAD, RPC]) State() NodeState { - n.stateMu.RLock() - defer n.stateMu.RUnlock() - return n.recalculateState() -} - -func (n *node[CHAIN_ID, HEAD, RPC]) getCachedState() NodeState { - n.stateMu.RLock() - defer n.stateMu.RUnlock() - return n.state -} - -func (n *node[CHAIN_ID, HEAD, RPC]) recalculateState() NodeState { - if n.state != NodeStateAlive { - return n.state - } - - // double check that node is not lagging on finalized block - if n.nodePoolCfg.EnforceRepeatableRead() && n.isFinalizedBlockOutOfSync() { - return NodeStateFinalizedBlockOutOfSync - } - - return NodeStateAlive -} - -func (n *node[CHAIN_ID, HEAD, RPC]) isFinalizedBlockOutOfSync() bool { - if n.poolInfoProvider == nil { - return false - } - - highestObservedByCaller := n.poolInfoProvider.HighestUserObservations() - latest, _ := n.rpc.GetInterceptedChainInfo() - if n.chainCfg.FinalityTagEnabled() { - return latest.FinalizedBlockNumber < highestObservedByCaller.FinalizedBlockNumber-int64(n.chainCfg.FinalizedBlockOffset()) - } - - return latest.BlockNumber < highestObservedByCaller.BlockNumber-int64(n.chainCfg.FinalizedBlockOffset()) -} - -// StateAndLatest returns nodeState with the latest ChainInfo observed by Node during current lifecycle. -func (n *node[CHAIN_ID, HEAD, RPC]) StateAndLatest() (NodeState, ChainInfo) { - n.stateMu.RLock() - defer n.stateMu.RUnlock() - latest, _ := n.rpc.GetInterceptedChainInfo() - return n.recalculateState(), latest -} - -// HighestUserObservations - returns highest ChainInfo ever observed by external user of the Node -func (n *node[CHAIN_ID, HEAD, RPC]) HighestUserObservations() ChainInfo { - _, highestUserObservations := n.rpc.GetInterceptedChainInfo() - return highestUserObservations -} -func (n *node[CHAIN_ID, HEAD, RPC]) SetPoolChainInfoProvider(poolInfoProvider PoolChainInfoProvider) { - n.poolInfoProvider = poolInfoProvider -} - -// setState is only used by internal state management methods. -// This is low-level; care should be taken by the caller to ensure the new state is a valid transition. -// State changes should always be synchronous: only one goroutine at a time should change state. -// n.stateMu should not be locked for long periods of time because external clients expect a timely response from n.State() -func (n *node[CHAIN_ID, HEAD, RPC]) setState(s NodeState) { - n.stateMu.Lock() - defer n.stateMu.Unlock() - n.state = s -} - -// declareXXX methods change the state and pass conrol off the new state -// management goroutine - -func (n *node[CHAIN_ID, HEAD, RPC]) declareAlive() { - n.transitionToAlive(func() { - n.lfcLog.Infow("RPC Node is online", "nodeState", n.state) - n.wg.Add(1) - go n.aliveLoop() - }) -} - -func (n *node[CHAIN_ID, HEAD, RPC]) transitionToAlive(fn func()) { - promPoolRPCNodeTransitionsToAlive.WithLabelValues(n.chainID.String(), n.name).Inc() - n.stateMu.Lock() - defer n.stateMu.Unlock() - if n.state == NodeStateClosed { - return - } - switch n.state { - case NodeStateDialed, NodeStateInvalidChainID, NodeStateSyncing: - n.state = NodeStateAlive - default: - panic(transitionFail(n.state, NodeStateAlive)) - } - fn() -} - -// declareInSync puts a node back into Alive state, allowing it to be used by -// pool consumers again -func (n *node[CHAIN_ID, HEAD, RPC]) declareInSync() { - n.transitionToInSync(func() { - n.lfcLog.Infow("RPC Node is back in sync", "nodeState", n.state) - n.wg.Add(1) - go n.aliveLoop() - }) -} - -func (n *node[CHAIN_ID, HEAD, RPC]) transitionToInSync(fn func()) { - promPoolRPCNodeTransitionsToAlive.WithLabelValues(n.chainID.String(), n.name).Inc() - promPoolRPCNodeTransitionsToInSync.WithLabelValues(n.chainID.String(), n.name).Inc() - n.stateMu.Lock() - defer n.stateMu.Unlock() - if n.state == NodeStateClosed { - return - } - switch n.state { - case NodeStateOutOfSync, NodeStateSyncing: - n.state = NodeStateAlive - default: - panic(transitionFail(n.state, NodeStateAlive)) - } - fn() -} - -// declareOutOfSync puts a node into OutOfSync state, disconnecting all current -// clients and making it unavailable for use until back in-sync. -func (n *node[CHAIN_ID, HEAD, RPC]) declareOutOfSync(syncIssues syncStatus) { - n.transitionToOutOfSync(func() { - n.lfcLog.Errorw("RPC Node is out of sync", "nodeState", n.state, "syncIssues", syncIssues) - n.wg.Add(1) - go n.outOfSyncLoop(syncIssues) - }) -} - -func (n *node[CHAIN_ID, HEAD, RPC]) transitionToOutOfSync(fn func()) { - promPoolRPCNodeTransitionsToOutOfSync.WithLabelValues(n.chainID.String(), n.name).Inc() - n.stateMu.Lock() - defer n.stateMu.Unlock() - if n.state == NodeStateClosed { - return - } - switch n.state { - case NodeStateAlive: - n.rpc.Close() - n.state = NodeStateOutOfSync - default: - panic(transitionFail(n.state, NodeStateOutOfSync)) - } - fn() -} - -func (n *node[CHAIN_ID, HEAD, RPC]) declareUnreachable() { - n.transitionToUnreachable(func() { - n.lfcLog.Errorw("RPC Node is unreachable", "nodeState", n.state) - n.wg.Add(1) - go n.unreachableLoop() - }) -} - -func (n *node[CHAIN_ID, HEAD, RPC]) transitionToUnreachable(fn func()) { - promPoolRPCNodeTransitionsToUnreachable.WithLabelValues(n.chainID.String(), n.name).Inc() - n.stateMu.Lock() - defer n.stateMu.Unlock() - if n.state == NodeStateClosed { - return - } - switch n.state { - case NodeStateUndialed, NodeStateDialed, NodeStateAlive, NodeStateOutOfSync, NodeStateInvalidChainID, NodeStateSyncing: - n.rpc.Close() - n.state = NodeStateUnreachable - default: - panic(transitionFail(n.state, NodeStateUnreachable)) - } - fn() -} - -func (n *node[CHAIN_ID, HEAD, RPC]) declareState(state NodeState) { - if n.getCachedState() == NodeStateClosed { - return - } - switch state { - case NodeStateInvalidChainID: - n.declareInvalidChainID() - case NodeStateUnreachable: - n.declareUnreachable() - case NodeStateSyncing: - n.declareSyncing() - case NodeStateAlive: - n.declareAlive() - default: - panic(fmt.Sprintf("%#v state declaration is not implemented", state)) - } -} - -func (n *node[CHAIN_ID, HEAD, RPC]) declareInvalidChainID() { - n.transitionToInvalidChainID(func() { - n.lfcLog.Errorw("RPC Node has the wrong chain ID", "nodeState", n.state) - n.wg.Add(1) - go n.invalidChainIDLoop() - }) -} - -func (n *node[CHAIN_ID, HEAD, RPC]) transitionToInvalidChainID(fn func()) { - promPoolRPCNodeTransitionsToInvalidChainID.WithLabelValues(n.chainID.String(), n.name).Inc() - n.stateMu.Lock() - defer n.stateMu.Unlock() - if n.state == NodeStateClosed { - return - } - switch n.state { - case NodeStateDialed, NodeStateOutOfSync, NodeStateSyncing: - n.rpc.Close() - n.state = NodeStateInvalidChainID - default: - panic(transitionFail(n.state, NodeStateInvalidChainID)) - } - fn() -} - -func (n *node[CHAIN_ID, HEAD, RPC]) declareSyncing() { - n.transitionToSyncing(func() { - n.lfcLog.Errorw("RPC Node is syncing", "nodeState", n.state) - n.wg.Add(1) - go n.syncingLoop() - }) -} - -func (n *node[CHAIN_ID, HEAD, RPC]) transitionToSyncing(fn func()) { - promPoolRPCNodeTransitionsToSyncing.WithLabelValues(n.chainID.String(), n.name).Inc() - n.stateMu.Lock() - defer n.stateMu.Unlock() - if n.state == NodeStateClosed { - return - } - switch n.state { - case NodeStateDialed, NodeStateOutOfSync, NodeStateInvalidChainID: - n.rpc.Close() - n.state = NodeStateSyncing - default: - panic(transitionFail(n.state, NodeStateSyncing)) - } - - if !n.nodePoolCfg.NodeIsSyncingEnabled() { - panic("unexpected transition to NodeStateSyncing, while it's disabled") - } - fn() -} - -func transitionString(state NodeState) string { - return fmt.Sprintf("Total number of times node has transitioned to %s", state) -} - -func transitionFail(from NodeState, to NodeState) string { - return fmt.Sprintf("cannot transition from %#v to %#v", from, to) -} diff --git a/pkg/solana/client/multinode/node_lifecycle.go b/pkg/solana/client/multinode/node_lifecycle.go deleted file mode 100644 index bca637a22..000000000 --- a/pkg/solana/client/multinode/node_lifecycle.go +++ /dev/null @@ -1,697 +0,0 @@ -package client - -import ( - "context" - "fmt" - "math" - "math/big" - "time" - - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" - - "github.com/smartcontractkit/chainlink-common/pkg/logger" - "github.com/smartcontractkit/chainlink-common/pkg/utils" - bigmath "github.com/smartcontractkit/chainlink-common/pkg/utils/big_math" -) - -var ( - promPoolRPCNodeHighestSeenBlock = promauto.NewGaugeVec(prometheus.GaugeOpts{ - Name: "solana_pool_rpc_node_highest_seen_block", - Help: "The highest seen block for the given RPC node", - }, []string{"chainID", "nodeName"}) - promPoolRPCNodeHighestFinalizedBlock = promauto.NewGaugeVec(prometheus.GaugeOpts{ - Name: "solana_pool_rpc_node_highest_finalized_block", - Help: "The highest seen finalized block for the given RPC node", - }, []string{"chainID", "nodeName"}) - promPoolRPCNodeNumSeenBlocks = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "solana_pool_rpc_node_num_seen_blocks", - Help: "The total number of new blocks seen by the given RPC node", - }, []string{"chainID", "nodeName"}) - promPoolRPCNodePolls = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "solana_pool_rpc_node_polls_total", - Help: "The total number of poll checks for the given RPC node", - }, []string{"chainID", "nodeName"}) - promPoolRPCNodePollsFailed = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "solana_pool_rpc_node_polls_failed", - Help: "The total number of failed poll checks for the given RPC node", - }, []string{"chainID", "nodeName"}) - promPoolRPCNodePollsSuccess = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "solana_pool_rpc_node_polls_success", - Help: "The total number of successful poll checks for the given RPC node", - }, []string{"chainID", "nodeName"}) -) - -// zombieNodeCheckInterval controls how often to re-check to see if we need to -// state change in case we have to force a state transition due to no available -// nodes. -// NOTE: This only applies to out-of-sync nodes if they are the last available node -func zombieNodeCheckInterval(noNewHeadsThreshold time.Duration) time.Duration { - interval := noNewHeadsThreshold - if interval <= 0 || interval > QueryTimeout { - interval = QueryTimeout - } - return utils.WithJitter(interval) -} - -const ( - msgCannotDisable = "but cannot disable this connection because there are no other RPC endpoints, or all other RPC endpoints are dead." - msgDegradedState = "Chainlink is now operating in a degraded state and urgent action is required to resolve the issue" -) - -// Node is a FSM -// Each state has a loop that goes with it, which monitors the node and moves it into another state as necessary. -// Only one loop must run at a time. -// Each loop passes control onto the next loop as it exits, except when the node is Closed which terminates the loop permanently. - -// This handles node lifecycle for the ALIVE state -// Should only be run ONCE per node, after a successful Dial -func (n *node[CHAIN_ID, HEAD, RPC]) aliveLoop() { - defer n.wg.Done() - ctx, cancel := n.newCtx() - defer cancel() - - { - // sanity check - state := n.getCachedState() - switch state { - case NodeStateAlive: - case NodeStateClosed: - return - default: - panic(fmt.Sprintf("aliveLoop can only run for node in Alive state, got: %s", state)) - } - } - - noNewHeadsTimeoutThreshold := n.chainCfg.NodeNoNewHeadsThreshold() - noNewFinalizedBlocksTimeoutThreshold := n.chainCfg.NoNewFinalizedHeadsThreshold() - pollFailureThreshold := n.nodePoolCfg.PollFailureThreshold() - pollInterval := n.nodePoolCfg.PollInterval() - - lggr := logger.Sugared(n.lfcLog).Named("Alive").With("noNewHeadsTimeoutThreshold", noNewHeadsTimeoutThreshold, "pollInterval", pollInterval, "pollFailureThreshold", pollFailureThreshold) - lggr.Tracew("Alive loop starting", "nodeState", n.getCachedState()) - - headsSub, err := n.registerNewSubscription(ctx, lggr.With("subscriptionType", "heads"), - n.chainCfg.NodeNoNewHeadsThreshold(), n.rpc.SubscribeToHeads) - if err != nil { - lggr.Errorw("Initial subscribe for heads failed", "nodeState", n.getCachedState(), "err", err) - n.declareUnreachable() - return - } - - defer n.unsubscribeHealthChecks() - - var pollCh <-chan time.Time - if pollInterval > 0 { - lggr.Debug("Polling enabled") - pollT := time.NewTicker(pollInterval) - defer pollT.Stop() - pollCh = pollT.C - if pollFailureThreshold > 0 { - // polling can be enabled with no threshold to enable polling but - // the node will not be marked offline regardless of the number of - // poll failures - lggr.Debug("Polling liveness checking enabled") - } - } else { - lggr.Debug("Polling disabled") - } - - var finalizedHeadsSub headSubscription[HEAD] - if n.chainCfg.FinalityTagEnabled() { - finalizedHeadsSub, err = n.registerNewSubscription(ctx, lggr.With("subscriptionType", "finalizedHeads"), - n.chainCfg.NoNewFinalizedHeadsThreshold(), n.rpc.SubscribeToFinalizedHeads) - if err != nil { - lggr.Errorw("Failed to subscribe to finalized heads", "err", err) - n.declareUnreachable() - return - } - } - - // Get the latest chain info to use as local highest - localHighestChainInfo, _ := n.rpc.GetInterceptedChainInfo() - var pollFailures uint32 - - for { - select { - case <-ctx.Done(): - return - case <-pollCh: - promPoolRPCNodePolls.WithLabelValues(n.chainID.String(), n.name).Inc() - lggr.Tracew("Pinging RPC", "nodeState", n.State(), "pollFailures", pollFailures) - pollCtx, cancel := context.WithTimeout(ctx, pollInterval) - err = n.RPC().Ping(pollCtx) - cancel() - if err != nil { - // prevent overflow - if pollFailures < math.MaxUint32 { - promPoolRPCNodePollsFailed.WithLabelValues(n.chainID.String(), n.name).Inc() - pollFailures++ - } - lggr.Warnw(fmt.Sprintf("Poll failure, RPC endpoint %s failed to respond properly", n.String()), "err", err, "pollFailures", pollFailures, "nodeState", n.getCachedState()) - } else { - lggr.Debugw("Ping successful", "nodeState", n.State()) - promPoolRPCNodePollsSuccess.WithLabelValues(n.chainID.String(), n.name).Inc() - pollFailures = 0 - } - if pollFailureThreshold > 0 && pollFailures >= pollFailureThreshold { - lggr.Errorw(fmt.Sprintf("RPC endpoint failed to respond to %d consecutive polls", pollFailures), "pollFailures", pollFailures, "nodeState", n.getCachedState()) - if n.poolInfoProvider != nil { - if l, _ := n.poolInfoProvider.LatestChainInfo(); l < 2 { - lggr.Criticalf("RPC endpoint failed to respond to polls; %s %s", msgCannotDisable, msgDegradedState) - continue - } - } - n.declareUnreachable() - return - } - if outOfSync, liveNodes := n.isOutOfSyncWithPool(); outOfSync { - // note: there must be another live node for us to be out of sync - if liveNodes < 2 { - lggr.Criticalf("RPC endpoint has fallen behind; %s %s", msgCannotDisable, msgDegradedState) - continue - } - n.declareOutOfSync(syncStatusNotInSyncWithPool) - return - } - case bh, open := <-headsSub.Heads: - if !open { - lggr.Errorw("Subscription channel unexpectedly closed", "nodeState", n.getCachedState()) - n.declareUnreachable() - return - } - receivedNewHead := n.onNewHead(lggr, &localHighestChainInfo, bh) - if receivedNewHead && noNewHeadsTimeoutThreshold > 0 { - headsSub.ResetTimer(noNewHeadsTimeoutThreshold) - } - case err = <-headsSub.Errors: - lggr.Errorw("Subscription was terminated", "err", err, "nodeState", n.getCachedState()) - n.declareUnreachable() - return - case <-headsSub.NoNewHeads: - // We haven't received a head on the channel for at least the - // threshold amount of time, mark it broken - lggr.Errorw(fmt.Sprintf("RPC endpoint detected out of sync; no new heads received for %s (last head received was %v)", noNewHeadsTimeoutThreshold, localHighestChainInfo.BlockNumber), "nodeState", n.getCachedState(), "latestReceivedBlockNumber", localHighestChainInfo.BlockNumber, "noNewHeadsTimeoutThreshold", noNewHeadsTimeoutThreshold) - if n.poolInfoProvider != nil { - if l, _ := n.poolInfoProvider.LatestChainInfo(); l < 2 { - lggr.Criticalf("RPC endpoint detected out of sync; %s %s", msgCannotDisable, msgDegradedState) - // We don't necessarily want to wait the full timeout to check again, we should - // check regularly and log noisily in this state - headsSub.ResetTimer(zombieNodeCheckInterval(noNewHeadsTimeoutThreshold)) - continue - } - } - n.declareOutOfSync(syncStatusNoNewHead) - return - case latestFinalized, open := <-finalizedHeadsSub.Heads: - if !open { - lggr.Errorw("Finalized heads subscription channel unexpectedly closed") - n.declareUnreachable() - return - } - - receivedNewHead := n.onNewFinalizedHead(lggr, &localHighestChainInfo, latestFinalized) - if receivedNewHead && noNewFinalizedBlocksTimeoutThreshold > 0 { - finalizedHeadsSub.ResetTimer(noNewFinalizedBlocksTimeoutThreshold) - } - case <-finalizedHeadsSub.NoNewHeads: - // We haven't received a finalized head on the channel for at least the - // threshold amount of time, mark it broken - lggr.Errorw(fmt.Sprintf("RPC's finalized state is out of sync; no new finalized heads received for %s (last finalized head received was %v)", noNewFinalizedBlocksTimeoutThreshold, localHighestChainInfo.FinalizedBlockNumber), "latestReceivedBlockNumber", localHighestChainInfo.BlockNumber) - if n.poolInfoProvider != nil { - if l, _ := n.poolInfoProvider.LatestChainInfo(); l < 2 { - lggr.Criticalf("RPC's finalized state is out of sync; %s %s", msgCannotDisable, msgDegradedState) - // We don't necessarily want to wait the full timeout to check again, we should - // check regularly and log noisily in this state - finalizedHeadsSub.ResetTimer(zombieNodeCheckInterval(noNewFinalizedBlocksTimeoutThreshold)) - continue - } - } - n.declareOutOfSync(syncStatusNoNewFinalizedHead) - return - case <-finalizedHeadsSub.Errors: - lggr.Errorw("Finalized heads subscription was terminated", "err", err) - n.declareUnreachable() - return - } - } -} - -func (n *node[CHAIN_ID, HEAD, RPC]) unsubscribeHealthChecks() { - n.stateMu.Lock() - for _, sub := range n.healthCheckSubs { - sub.Unsubscribe() - } - n.healthCheckSubs = []Subscription{} - n.stateMu.Unlock() -} - -type headSubscription[HEAD any] struct { - Heads <-chan HEAD - Errors <-chan error - NoNewHeads <-chan time.Time - - noNewHeadsTicker *time.Ticker - sub Subscription - cleanUpTasks []func() -} - -func (sub *headSubscription[HEAD]) ResetTimer(duration time.Duration) { - sub.noNewHeadsTicker.Reset(duration) -} - -func (sub *headSubscription[HEAD]) Unsubscribe() { - for _, doCleanUp := range sub.cleanUpTasks { - doCleanUp() - } -} - -func (n *node[CHAIN_ID, HEAD, PRC]) registerNewSubscription(ctx context.Context, lggr logger.SugaredLogger, - noNewDataThreshold time.Duration, newSub func(ctx context.Context) (<-chan HEAD, Subscription, error)) (headSubscription[HEAD], error) { - result := headSubscription[HEAD]{} - var err error - var sub Subscription - result.Heads, sub, err = newSub(ctx) - if err != nil { - return result, err - } - - result.Errors = sub.Err() - lggr.Debug("Successfully subscribed") - - result.sub = sub - n.stateMu.Lock() - n.healthCheckSubs = append(n.healthCheckSubs, sub) - n.stateMu.Unlock() - - result.cleanUpTasks = append(result.cleanUpTasks, sub.Unsubscribe) - - if noNewDataThreshold > 0 { - lggr.Debugw("Subscription liveness checking enabled") - result.noNewHeadsTicker = time.NewTicker(noNewDataThreshold) - result.NoNewHeads = result.noNewHeadsTicker.C - result.cleanUpTasks = append(result.cleanUpTasks, result.noNewHeadsTicker.Stop) - } else { - lggr.Debug("Subscription liveness checking disabled") - } - - return result, nil -} - -func (n *node[CHAIN_ID, HEAD, RPC]) onNewFinalizedHead(lggr logger.SugaredLogger, chainInfo *ChainInfo, latestFinalized HEAD) bool { - if !latestFinalized.IsValid() { - lggr.Warn("Latest finalized block is not valid") - return false - } - - latestFinalizedBN := latestFinalized.BlockNumber() - lggr.Debugw("Got latest finalized head", "latestFinalized", latestFinalized) - if latestFinalizedBN <= chainInfo.FinalizedBlockNumber { - lggr.Debugw("Ignoring previously seen finalized block number") - return false - } - - promPoolRPCNodeHighestFinalizedBlock.WithLabelValues(n.chainID.String(), n.name).Set(float64(latestFinalizedBN)) - chainInfo.FinalizedBlockNumber = latestFinalizedBN - return true -} - -func (n *node[CHAIN_ID, HEAD, RPC]) onNewHead(lggr logger.SugaredLogger, chainInfo *ChainInfo, head HEAD) bool { - if !head.IsValid() { - lggr.Warn("Latest head is not valid") - return false - } - - promPoolRPCNodeNumSeenBlocks.WithLabelValues(n.chainID.String(), n.name).Inc() - lggr.Debugw("Got head", "head", head) - lggr = lggr.With("latestReceivedBlockNumber", chainInfo.BlockNumber, "blockNumber", head.BlockNumber(), "nodeState", n.getCachedState()) - if head.BlockNumber() <= chainInfo.BlockNumber { - lggr.Debugw("Ignoring previously seen block number") - return false - } - - promPoolRPCNodeHighestSeenBlock.WithLabelValues(n.chainID.String(), n.name).Set(float64(head.BlockNumber())) - chainInfo.BlockNumber = head.BlockNumber() - - if !n.chainCfg.FinalityTagEnabled() { - latestFinalizedBN := max(head.BlockNumber()-int64(n.chainCfg.FinalityDepth()), 0) - if latestFinalizedBN > chainInfo.FinalizedBlockNumber { - promPoolRPCNodeHighestFinalizedBlock.WithLabelValues(n.chainID.String(), n.name).Set(float64(latestFinalizedBN)) - chainInfo.FinalizedBlockNumber = latestFinalizedBN - } - } - - return true -} - -const ( - msgReceivedBlock = "Received block for RPC node, waiting until back in-sync to mark as live again" - msgReceivedFinalizedBlock = "Received new finalized block for RPC node, waiting until back in-sync to mark as live again" - msgInSync = "RPC node back in sync" -) - -// isOutOfSyncWithPool returns outOfSync true if num or td is more than SyncThresold behind the best node. -// Always returns outOfSync false for SyncThreshold 0. -// liveNodes is only included when outOfSync is true. -func (n *node[CHAIN_ID, HEAD, RPC]) isOutOfSyncWithPool() (outOfSync bool, liveNodes int) { - if n.poolInfoProvider == nil { - n.lfcLog.Warn("skipping sync state against the pool - should only occur in tests") - return // skip for tests - } - threshold := n.nodePoolCfg.SyncThreshold() - if threshold == 0 { - return // disabled - } - // Check against best node - ln, ci := n.poolInfoProvider.LatestChainInfo() - localChainInfo, _ := n.rpc.GetInterceptedChainInfo() - mode := n.nodePoolCfg.SelectionMode() - switch mode { - case NodeSelectionModeHighestHead, NodeSelectionModeRoundRobin, NodeSelectionModePriorityLevel: - outOfSync = localChainInfo.BlockNumber < ci.BlockNumber-int64(threshold) - case NodeSelectionModeTotalDifficulty: - bigThreshold := big.NewInt(int64(threshold)) - outOfSync = localChainInfo.TotalDifficulty.Cmp(bigmath.Sub(ci.TotalDifficulty, bigThreshold)) < 0 - default: - panic("unrecognized NodeSelectionMode: " + mode) - } - - if outOfSync && n.getCachedState() == NodeStateAlive { - n.lfcLog.Errorw("RPC endpoint has fallen behind", "blockNumber", localChainInfo.BlockNumber, "bestLatestBlockNumber", ci.BlockNumber, "totalDifficulty", localChainInfo.TotalDifficulty) - } - return outOfSync, ln -} - -// outOfSyncLoop takes an OutOfSync node and waits until isOutOfSync returns false to go back to live status -func (n *node[CHAIN_ID, HEAD, RPC]) outOfSyncLoop(syncIssues syncStatus) { - defer n.wg.Done() - ctx, cancel := n.newCtx() - defer cancel() - - { - // sanity check - state := n.getCachedState() - switch state { - case NodeStateOutOfSync: - case NodeStateClosed: - return - default: - panic(fmt.Sprintf("outOfSyncLoop can only run for node in OutOfSync state, got: %s", state)) - } - } - - outOfSyncAt := time.Now() - - // set logger name to OutOfSync or FinalizedBlockOutOfSync - lggr := logger.Sugared(logger.Named(n.lfcLog, n.getCachedState().String())).With("nodeState", n.getCachedState()) - lggr.Debugw("Trying to revive out-of-sync RPC node") - - // Need to redial since out-of-sync nodes are automatically disconnected - state := n.createVerifiedConn(ctx, lggr) - if state != NodeStateAlive { - n.declareState(state) - return - } - - noNewHeadsTimeoutThreshold := n.chainCfg.NodeNoNewHeadsThreshold() - headsSub, err := n.registerNewSubscription(ctx, lggr.With("subscriptionType", "heads"), - noNewHeadsTimeoutThreshold, n.rpc.SubscribeToHeads) - if err != nil { - lggr.Errorw("Failed to subscribe heads on out-of-sync RPC node", "err", err) - n.declareUnreachable() - return - } - - defer n.unsubscribeHealthChecks() - - lggr.Tracew("Successfully subscribed to heads feed on out-of-sync RPC node") - - noNewFinalizedBlocksTimeoutThreshold := n.chainCfg.NoNewFinalizedHeadsThreshold() - var finalizedHeadsSub headSubscription[HEAD] - if n.chainCfg.FinalityTagEnabled() { - finalizedHeadsSub, err = n.registerNewSubscription(ctx, lggr.With("subscriptionType", "finalizedHeads"), - noNewFinalizedBlocksTimeoutThreshold, n.rpc.SubscribeToFinalizedHeads) - if err != nil { - lggr.Errorw("Subscribe to finalized heads failed on out-of-sync RPC node", "err", err) - n.declareUnreachable() - return - } - - lggr.Tracew("Successfully subscribed to finalized heads feed on out-of-sync RPC node") - } - - _, localHighestChainInfo := n.rpc.GetInterceptedChainInfo() - for { - if syncIssues == syncStatusSynced { - // back in-sync! flip back into alive loop - lggr.Infow(fmt.Sprintf("%s: %s. Node was out-of-sync for %s", msgInSync, n.String(), time.Since(outOfSyncAt))) - n.declareInSync() - return - } - - select { - case <-ctx.Done(): - return - case head, open := <-headsSub.Heads: - if !open { - lggr.Errorw("Subscription channel unexpectedly closed", "nodeState", n.getCachedState()) - n.declareUnreachable() - return - } - - if !n.onNewHead(lggr, &localHighestChainInfo, head) { - continue - } - - // received a new head - clear NoNewHead flag - syncIssues &= ^syncStatusNoNewHead - if outOfSync, _ := n.isOutOfSyncWithPool(); !outOfSync { - // we caught up with the pool - clear NotInSyncWithPool flag - syncIssues &= ^syncStatusNotInSyncWithPool - } else { - // we've received new head, but lagging behind the pool, add NotInSyncWithPool flag to prevent false transition to alive - syncIssues |= syncStatusNotInSyncWithPool - } - - if noNewHeadsTimeoutThreshold > 0 { - headsSub.ResetTimer(noNewHeadsTimeoutThreshold) - } - - lggr.Debugw(msgReceivedBlock, "blockNumber", head.BlockNumber(), "blockDifficulty", head.BlockDifficulty(), "syncIssues", syncIssues) - case <-time.After(zombieNodeCheckInterval(noNewHeadsTimeoutThreshold)): - if n.poolInfoProvider != nil { - if l, _ := n.poolInfoProvider.LatestChainInfo(); l < 1 { - lggr.Criticalw("RPC endpoint is still out of sync, but there are no other available nodes. This RPC node will be forcibly moved back into the live pool in a degraded state", "syncIssues", syncIssues) - n.declareInSync() - return - } - } - case err := <-headsSub.Errors: - lggr.Errorw("Subscription was terminated", "err", err) - n.declareUnreachable() - return - case <-headsSub.NoNewHeads: - // we are not resetting the timer, as there is no need to add syncStatusNoNewHead until it's removed on new head. - syncIssues |= syncStatusNoNewHead - lggr.Debugw(fmt.Sprintf("No new heads received for %s. Node stays out-of-sync due to sync issues: %s", noNewHeadsTimeoutThreshold, syncIssues)) - case latestFinalized, open := <-finalizedHeadsSub.Heads: - if !open { - lggr.Errorw("Finalized heads subscription channel unexpectedly closed") - n.declareUnreachable() - return - } - if !latestFinalized.IsValid() { - lggr.Warn("Latest finalized block is not valid") - continue - } - - receivedNewHead := n.onNewFinalizedHead(lggr, &localHighestChainInfo, latestFinalized) - if !receivedNewHead { - continue - } - - // on new finalized head remove NoNewFinalizedHead flag from the mask - syncIssues &= ^syncStatusNoNewFinalizedHead - if noNewFinalizedBlocksTimeoutThreshold > 0 { - finalizedHeadsSub.ResetTimer(noNewFinalizedBlocksTimeoutThreshold) - } - - var highestSeen ChainInfo - if n.poolInfoProvider != nil { - highestSeen = n.poolInfoProvider.HighestUserObservations() - } - - lggr.Debugw(msgReceivedFinalizedBlock, "blockNumber", latestFinalized.BlockNumber(), "poolHighestBlockNumber", highestSeen.FinalizedBlockNumber, "syncIssues", syncIssues) - case err := <-finalizedHeadsSub.Errors: - lggr.Errorw("Finalized head subscription was terminated", "err", err) - n.declareUnreachable() - return - case <-finalizedHeadsSub.NoNewHeads: - // we are not resetting the timer, as there is no need to add syncStatusNoNewFinalizedHead until it's removed on new finalized head. - syncIssues |= syncStatusNoNewFinalizedHead - lggr.Debugw(fmt.Sprintf("No new finalized heads received for %s. Node stays out-of-sync due to sync issues: %s", noNewFinalizedBlocksTimeoutThreshold, syncIssues)) - } - } -} - -func (n *node[CHAIN_ID, HEAD, RPC]) unreachableLoop() { - defer n.wg.Done() - ctx, cancel := n.newCtx() - defer cancel() - - { - // sanity check - state := n.getCachedState() - switch state { - case NodeStateUnreachable: - case NodeStateClosed: - return - default: - panic(fmt.Sprintf("unreachableLoop can only run for node in Unreachable state, got: %s", state)) - } - } - - unreachableAt := time.Now() - - lggr := logger.Sugared(logger.Named(n.lfcLog, "Unreachable")) - lggr.Debugw("Trying to revive unreachable RPC node", "nodeState", n.getCachedState()) - - dialRetryBackoff := NewRedialBackoff() - - for { - select { - case <-ctx.Done(): - return - case <-time.After(dialRetryBackoff.Duration()): - lggr.Tracew("Trying to re-dial RPC node", "nodeState", n.getCachedState()) - - err := n.rpc.Dial(ctx) - if err != nil { - lggr.Errorw(fmt.Sprintf("Failed to redial RPC node; still unreachable: %v", err), "err", err, "nodeState", n.getCachedState()) - continue - } - - n.setState(NodeStateDialed) - - state := n.verifyConn(ctx, lggr) - switch state { - case NodeStateUnreachable: - n.setState(NodeStateUnreachable) - continue - case NodeStateAlive: - lggr.Infow(fmt.Sprintf("Successfully redialled and verified RPC node %s. Node was offline for %s", n.String(), time.Since(unreachableAt)), "nodeState", n.getCachedState()) - fallthrough - default: - n.declareState(state) - return - } - } - } -} - -func (n *node[CHAIN_ID, HEAD, RPC]) invalidChainIDLoop() { - defer n.wg.Done() - ctx, cancel := n.newCtx() - defer cancel() - - { - // sanity check - state := n.getCachedState() - switch state { - case NodeStateInvalidChainID: - case NodeStateClosed: - return - default: - panic(fmt.Sprintf("invalidChainIDLoop can only run for node in InvalidChainID state, got: %s", state)) - } - } - - invalidAt := time.Now() - - lggr := logger.Named(n.lfcLog, "InvalidChainID") - - // Need to redial since invalid chain ID nodes are automatically disconnected - state := n.createVerifiedConn(ctx, lggr) - if state != NodeStateInvalidChainID { - n.declareState(state) - return - } - - lggr.Debugw(fmt.Sprintf("Periodically re-checking RPC node %s with invalid chain ID", n.String()), "nodeState", n.getCachedState()) - - chainIDRecheckBackoff := NewRedialBackoff() - - for { - select { - case <-ctx.Done(): - return - case <-time.After(chainIDRecheckBackoff.Duration()): - state := n.verifyConn(ctx, lggr) - switch state { - case NodeStateInvalidChainID: - continue - case NodeStateAlive: - lggr.Infow(fmt.Sprintf("Successfully verified RPC node. Node was offline for %s", time.Since(invalidAt)), "nodeState", n.getCachedState()) - fallthrough - default: - n.declareState(state) - return - } - } - } -} - -func (n *node[CHAIN_ID, HEAD, RPC]) syncingLoop() { - defer n.wg.Done() - ctx, cancel := n.newCtx() - defer cancel() - - { - // sanity check - state := n.getCachedState() - switch state { - case NodeStateSyncing: - case NodeStateClosed: - return - default: - panic(fmt.Sprintf("syncingLoop can only run for node in NodeStateSyncing state, got: %s", state)) - } - } - - syncingAt := time.Now() - - lggr := logger.Sugared(logger.Named(n.lfcLog, "Syncing")) - lggr.Debugw(fmt.Sprintf("Periodically re-checking RPC node %s with syncing status", n.String()), "nodeState", n.getCachedState()) - // Need to redial since syncing nodes are automatically disconnected - state := n.createVerifiedConn(ctx, lggr) - if state != NodeStateSyncing { - n.declareState(state) - return - } - - recheckBackoff := NewRedialBackoff() - - for { - select { - case <-ctx.Done(): - return - case <-time.After(recheckBackoff.Duration()): - lggr.Tracew("Trying to recheck if the node is still syncing", "nodeState", n.getCachedState()) - isSyncing, err := n.rpc.IsSyncing(ctx) - if err != nil { - lggr.Errorw("Unexpected error while verifying RPC node synchronization status", "err", err, "nodeState", n.getCachedState()) - n.declareUnreachable() - return - } - - if isSyncing { - lggr.Errorw("Verification failed: Node is syncing", "nodeState", n.getCachedState()) - continue - } - - lggr.Infow(fmt.Sprintf("Successfully verified RPC node. Node was syncing for %s", time.Since(syncingAt)), "nodeState", n.getCachedState()) - n.declareAlive() - return - } - } -} diff --git a/pkg/solana/client/multinode/node_selector.go b/pkg/solana/client/multinode/node_selector.go deleted file mode 100644 index 872026fe2..000000000 --- a/pkg/solana/client/multinode/node_selector.go +++ /dev/null @@ -1,41 +0,0 @@ -package client - -import ( - "fmt" -) - -const ( - NodeSelectionModeHighestHead = "HighestHead" - NodeSelectionModeRoundRobin = "RoundRobin" - NodeSelectionModeTotalDifficulty = "TotalDifficulty" - NodeSelectionModePriorityLevel = "PriorityLevel" -) - -type NodeSelector[ - CHAIN_ID ID, - RPC any, -] interface { - // Select returns a Node, or nil if none can be selected. - // Implementation must be thread-safe. - Select() Node[CHAIN_ID, RPC] - // Name returns the strategy name, e.g. "HighestHead" or "RoundRobin" - Name() string -} - -func newNodeSelector[ - CHAIN_ID ID, - RPC any, -](selectionMode string, nodes []Node[CHAIN_ID, RPC]) NodeSelector[CHAIN_ID, RPC] { - switch selectionMode { - case NodeSelectionModeHighestHead: - return NewHighestHeadNodeSelector[CHAIN_ID, RPC](nodes) - case NodeSelectionModeRoundRobin: - return NewRoundRobinSelector[CHAIN_ID, RPC](nodes) - case NodeSelectionModeTotalDifficulty: - return NewTotalDifficultyNodeSelector[CHAIN_ID, RPC](nodes) - case NodeSelectionModePriorityLevel: - return NewPriorityLevelNodeSelector[CHAIN_ID, RPC](nodes) - default: - panic(fmt.Sprintf("unsupported NodeSelectionMode: %s", selectionMode)) - } -} diff --git a/pkg/solana/client/multinode/node_selector_highest_head.go b/pkg/solana/client/multinode/node_selector_highest_head.go deleted file mode 100644 index 68901cba3..000000000 --- a/pkg/solana/client/multinode/node_selector_highest_head.go +++ /dev/null @@ -1,36 +0,0 @@ -package client - -import "math" - -type highestHeadNodeSelector[ - CHAIN_ID ID, - RPC any, -] []Node[CHAIN_ID, RPC] - -func NewHighestHeadNodeSelector[ - CHAIN_ID ID, - RPC any, -](nodes []Node[CHAIN_ID, RPC]) NodeSelector[CHAIN_ID, RPC] { - return highestHeadNodeSelector[CHAIN_ID, RPC](nodes) -} - -func (s highestHeadNodeSelector[CHAIN_ID, RPC]) Select() Node[CHAIN_ID, RPC] { - var highestHeadNumber int64 = math.MinInt64 - var highestHeadNodes []Node[CHAIN_ID, RPC] - for _, n := range s { - state, currentChainInfo := n.StateAndLatest() - currentHeadNumber := currentChainInfo.BlockNumber - if state == NodeStateAlive && currentHeadNumber >= highestHeadNumber { - if highestHeadNumber < currentHeadNumber { - highestHeadNumber = currentHeadNumber - highestHeadNodes = nil - } - highestHeadNodes = append(highestHeadNodes, n) - } - } - return firstOrHighestPriority(highestHeadNodes) -} - -func (s highestHeadNodeSelector[CHAIN_ID, RPC]) Name() string { - return NodeSelectionModeHighestHead -} diff --git a/pkg/solana/client/multinode/node_selector_priority_level.go b/pkg/solana/client/multinode/node_selector_priority_level.go deleted file mode 100644 index ead720976..000000000 --- a/pkg/solana/client/multinode/node_selector_priority_level.go +++ /dev/null @@ -1,121 +0,0 @@ -package client - -import ( - "math" - "sort" - "sync/atomic" -) - -type priorityLevelNodeSelector[ - CHAIN_ID ID, - RPC any, -] struct { - nodes []Node[CHAIN_ID, RPC] - roundRobinCount []atomic.Uint32 -} - -type nodeWithPriority[ - CHAIN_ID ID, - RPC any, -] struct { - node Node[CHAIN_ID, RPC] - priority int32 -} - -func NewPriorityLevelNodeSelector[ - CHAIN_ID ID, - RPC any, -](nodes []Node[CHAIN_ID, RPC]) NodeSelector[CHAIN_ID, RPC] { - return &priorityLevelNodeSelector[CHAIN_ID, RPC]{ - nodes: nodes, - roundRobinCount: make([]atomic.Uint32, nrOfPriorityTiers(nodes)), - } -} - -func (s priorityLevelNodeSelector[CHAIN_ID, RPC]) Select() Node[CHAIN_ID, RPC] { - nodes := s.getHighestPriorityAliveTier() - - if len(nodes) == 0 { - return nil - } - priorityLevel := nodes[len(nodes)-1].priority - - // NOTE: Inc returns the number after addition, so we must -1 to get the "current" counter - count := int(s.roundRobinCount[priorityLevel].Add(1) - 1) - idx := count % len(nodes) - - return nodes[idx].node -} - -func (s priorityLevelNodeSelector[CHAIN_ID, RPC]) Name() string { - return NodeSelectionModePriorityLevel -} - -// getHighestPriorityAliveTier filters nodes that are not in state NodeStateAlive and -// returns only the highest tier of alive nodes -func (s priorityLevelNodeSelector[CHAIN_ID, RPC]) getHighestPriorityAliveTier() []nodeWithPriority[CHAIN_ID, RPC] { - var nodes []nodeWithPriority[CHAIN_ID, RPC] - for _, n := range s.nodes { - if n.State() == NodeStateAlive { - nodes = append(nodes, nodeWithPriority[CHAIN_ID, RPC]{n, n.Order()}) - } - } - - if len(nodes) == 0 { - return nil - } - - return removeLowerTiers(nodes) -} - -// removeLowerTiers take a slice of nodeWithPriority[CHAIN_ID, BLOCK_HASH, HEAD, RPC] and keeps only the highest tier -func removeLowerTiers[ - CHAIN_ID ID, - RPC any, -](nodes []nodeWithPriority[CHAIN_ID, RPC]) []nodeWithPriority[CHAIN_ID, RPC] { - sort.SliceStable(nodes, func(i, j int) bool { - return nodes[i].priority > nodes[j].priority - }) - - var nodes2 []nodeWithPriority[CHAIN_ID, RPC] - currentPriority := nodes[len(nodes)-1].priority - - for _, n := range nodes { - if n.priority == currentPriority { - nodes2 = append(nodes2, n) - } - } - - return nodes2 -} - -// nrOfPriorityTiers calculates the total number of priority tiers -func nrOfPriorityTiers[ - CHAIN_ID ID, - RPC any, -](nodes []Node[CHAIN_ID, RPC]) int32 { - highestPriority := int32(0) - for _, n := range nodes { - priority := n.Order() - if highestPriority < priority { - highestPriority = priority - } - } - return highestPriority + 1 -} - -// firstOrHighestPriority takes a list of nodes and returns the first one with the highest priority -func firstOrHighestPriority[ - CHAIN_ID ID, - RPC any, -](nodes []Node[CHAIN_ID, RPC]) Node[CHAIN_ID, RPC] { - hp := int32(math.MaxInt32) - var node Node[CHAIN_ID, RPC] - for _, n := range nodes { - if n.Order() < hp { - hp = n.Order() - node = n - } - } - return node -} diff --git a/pkg/solana/client/multinode/node_selector_round_robin.go b/pkg/solana/client/multinode/node_selector_round_robin.go deleted file mode 100644 index c5ed8d853..000000000 --- a/pkg/solana/client/multinode/node_selector_round_robin.go +++ /dev/null @@ -1,46 +0,0 @@ -package client - -import ( - "sync/atomic" -) - -type roundRobinSelector[ - CHAIN_ID ID, - RPC any, -] struct { - nodes []Node[CHAIN_ID, RPC] - roundRobinCount atomic.Uint32 -} - -func NewRoundRobinSelector[ - CHAIN_ID ID, - RPC any, -](nodes []Node[CHAIN_ID, RPC]) NodeSelector[CHAIN_ID, RPC] { - return &roundRobinSelector[CHAIN_ID, RPC]{ - nodes: nodes, - } -} - -func (s *roundRobinSelector[CHAIN_ID, RPC]) Select() Node[CHAIN_ID, RPC] { - var liveNodes []Node[CHAIN_ID, RPC] - for _, n := range s.nodes { - if n.State() == NodeStateAlive { - liveNodes = append(liveNodes, n) - } - } - - nNodes := len(liveNodes) - if nNodes == 0 { - return nil - } - - // NOTE: Inc returns the number after addition, so we must -1 to get the "current" counter - count := int(s.roundRobinCount.Add(1) - 1) - idx := count % nNodes - - return liveNodes[idx] -} - -func (s *roundRobinSelector[CHAIN_ID, RPC]) Name() string { - return NodeSelectionModeRoundRobin -} diff --git a/pkg/solana/client/multinode/node_selector_total_difficulty.go b/pkg/solana/client/multinode/node_selector_total_difficulty.go deleted file mode 100644 index 3f3c79de9..000000000 --- a/pkg/solana/client/multinode/node_selector_total_difficulty.go +++ /dev/null @@ -1,51 +0,0 @@ -package client - -import ( - "math/big" -) - -type totalDifficultyNodeSelector[ - CHAIN_ID ID, - RPC any, -] []Node[CHAIN_ID, RPC] - -func NewTotalDifficultyNodeSelector[ - CHAIN_ID ID, - RPC any, -](nodes []Node[CHAIN_ID, RPC]) NodeSelector[CHAIN_ID, RPC] { - return totalDifficultyNodeSelector[CHAIN_ID, RPC](nodes) -} - -func (s totalDifficultyNodeSelector[CHAIN_ID, RPC]) Select() Node[CHAIN_ID, RPC] { - // NodeNoNewHeadsThreshold may not be enabled, in this case all nodes have td == nil - var highestTD *big.Int - var nodes []Node[CHAIN_ID, RPC] - var aliveNodes []Node[CHAIN_ID, RPC] - - for _, n := range s { - state, currentChainInfo := n.StateAndLatest() - if state != NodeStateAlive { - continue - } - - currentTD := currentChainInfo.TotalDifficulty - aliveNodes = append(aliveNodes, n) - if currentTD != nil && (highestTD == nil || currentTD.Cmp(highestTD) >= 0) { - if highestTD == nil || currentTD.Cmp(highestTD) > 0 { - highestTD = currentTD - nodes = nil - } - nodes = append(nodes, n) - } - } - - //If all nodes have td == nil pick one from the nodes that are alive - if len(nodes) == 0 { - return firstOrHighestPriority(aliveNodes) - } - return firstOrHighestPriority(nodes) -} - -func (s totalDifficultyNodeSelector[CHAIN_ID, RPC]) Name() string { - return NodeSelectionModeTotalDifficulty -} diff --git a/pkg/solana/client/multinode/poller.go b/pkg/solana/client/multinode/poller.go deleted file mode 100644 index 0ce87fade..000000000 --- a/pkg/solana/client/multinode/poller.go +++ /dev/null @@ -1,93 +0,0 @@ -package client - -import ( - "context" - "time" - - "github.com/smartcontractkit/chainlink-common/pkg/logger" - "github.com/smartcontractkit/chainlink-common/pkg/services" -) - -// Poller is a component that polls a function at a given interval -// and delivers the result to a channel. It is used by multinode to poll -// for new heads and implements the Subscription interface. -type Poller[T any] struct { - services.Service - eng *services.Engine - - pollingInterval time.Duration - pollingFunc func(ctx context.Context) (T, error) - pollingTimeout time.Duration - channel chan<- T - errCh chan error -} - -// NewPoller creates a new Poller instance and returns a channel to receive the polled data -func NewPoller[ - T any, -](pollingInterval time.Duration, pollingFunc func(ctx context.Context) (T, error), pollingTimeout time.Duration, lggr logger.Logger) (Poller[T], <-chan T) { - channel := make(chan T) - p := Poller[T]{ - pollingInterval: pollingInterval, - pollingFunc: pollingFunc, - pollingTimeout: pollingTimeout, - channel: channel, - errCh: make(chan error), - } - p.Service, p.eng = services.Config{ - Name: "Poller", - Start: p.start, - Close: p.close, - }.NewServiceEngine(lggr) - return p, channel -} - -var _ Subscription = &Poller[any]{} - -func (p *Poller[T]) start(ctx context.Context) error { - p.eng.Go(p.pollingLoop) - return nil -} - -// Unsubscribe cancels the sending of events to the data channel -func (p *Poller[T]) Unsubscribe() { - _ = p.Close() -} - -func (p *Poller[T]) close() error { - close(p.errCh) - close(p.channel) - return nil -} - -func (p *Poller[T]) Err() <-chan error { - return p.errCh -} - -func (p *Poller[T]) pollingLoop(ctx context.Context) { - ticker := services.NewTicker(p.pollingInterval) - defer ticker.Stop() - - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - // Set polling timeout - pollingCtx, cancelPolling := context.WithTimeout(ctx, p.pollingTimeout) - // Execute polling function - result, err := p.pollingFunc(pollingCtx) - cancelPolling() - if err != nil { - p.eng.Warnf("polling error: %v", err) - continue - } - // Send result to channel or block if channel is full - select { - case p.channel <- result: - case <-ctx.Done(): - return - } - } - } -} diff --git a/pkg/solana/client/multinode/redialbackoff.go b/pkg/solana/client/multinode/redialbackoff.go deleted file mode 100644 index 41be2232d..000000000 --- a/pkg/solana/client/multinode/redialbackoff.go +++ /dev/null @@ -1,17 +0,0 @@ -package client - -import ( - "time" - - "github.com/jpillora/backoff" -) - -// NewRedialBackoff is a standard backoff to use for redialling or reconnecting to -// unreachable network endpoints -func NewRedialBackoff() backoff.Backoff { - return backoff.Backoff{ - Min: 1 * time.Second, - Max: 15 * time.Second, - Jitter: true, - } -} diff --git a/pkg/solana/client/multinode/send_only_node.go b/pkg/solana/client/multinode/send_only_node.go deleted file mode 100644 index 4f60f566d..000000000 --- a/pkg/solana/client/multinode/send_only_node.go +++ /dev/null @@ -1,181 +0,0 @@ -package client - -import ( - "context" - "fmt" - "net/url" - "sync" - - "github.com/smartcontractkit/chainlink-common/pkg/logger" - "github.com/smartcontractkit/chainlink-common/pkg/services" -) - -type sendOnlyClient[ - CHAIN_ID ID, -] interface { - Close() - ChainID(context.Context) (CHAIN_ID, error) - Dial(ctx context.Context) error -} - -// SendOnlyNode represents one node used as a sendonly -type SendOnlyNode[ - CHAIN_ID ID, - RPC any, -] interface { - // Start may attempt to connect to the node, but should only return error for misconfiguration - never for temporary errors. - Start(context.Context) error - Close() error - - ConfiguredChainID() CHAIN_ID - RPC() RPC - - String() string - // State returns NodeState - State() NodeState - // Name is a unique identifier for this node. - Name() string -} - -// It only supports sending transactions -// It must use an http(s) url -type sendOnlyNode[ - CHAIN_ID ID, - RPC sendOnlyClient[CHAIN_ID], -] struct { - services.StateMachine - - stateMu sync.RWMutex // protects state* fields - state NodeState - - rpc RPC - uri url.URL - log logger.Logger - name string - chainID CHAIN_ID - chStop services.StopChan - wg sync.WaitGroup -} - -// NewSendOnlyNode returns a new sendonly node -func NewSendOnlyNode[ - CHAIN_ID ID, - RPC sendOnlyClient[CHAIN_ID], -]( - lggr logger.Logger, - httpuri url.URL, - name string, - chainID CHAIN_ID, - rpc RPC, -) SendOnlyNode[CHAIN_ID, RPC] { - s := new(sendOnlyNode[CHAIN_ID, RPC]) - s.name = name - s.log = logger.Named(logger.Named(lggr, "SendOnlyNode"), name) - s.log = logger.With(s.log, - "nodeTier", "sendonly", - ) - s.rpc = rpc - s.uri = httpuri - s.chainID = chainID - s.chStop = make(chan struct{}) - return s -} - -func (s *sendOnlyNode[CHAIN_ID, RPC]) Start(ctx context.Context) error { - return s.StartOnce(s.name, func() error { - s.start(ctx) - return nil - }) -} - -// Start setups up and verifies the sendonly node -// Should only be called once in a node's lifecycle -func (s *sendOnlyNode[CHAIN_ID, RPC]) start(startCtx context.Context) { - if s.State() != NodeStateUndialed { - panic(fmt.Sprintf("cannot dial node with state %v", s.state)) - } - - err := s.rpc.Dial(startCtx) - if err != nil { - promPoolRPCNodeTransitionsToUnusable.WithLabelValues(s.chainID.String(), s.name).Inc() - s.log.Errorw("Dial failed: SendOnly Node is unusable", "err", err) - s.setState(NodeStateUnusable) - return - } - s.setState(NodeStateDialed) - - if s.chainID.String() == "0" { - // Skip verification if chainID is zero - s.log.Warn("sendonly rpc ChainID verification skipped") - } else { - chainID, err := s.rpc.ChainID(startCtx) - if err != nil || chainID.String() != s.chainID.String() { - promPoolRPCNodeTransitionsToUnreachable.WithLabelValues(s.chainID.String(), s.name).Inc() - if err != nil { - promPoolRPCNodeTransitionsToUnreachable.WithLabelValues(s.chainID.String(), s.name).Inc() - s.log.Errorw(fmt.Sprintf("Verify failed: %v", err), "err", err) - s.setState(NodeStateUnreachable) - } else { - promPoolRPCNodeTransitionsToInvalidChainID.WithLabelValues(s.chainID.String(), s.name).Inc() - s.log.Errorf( - "sendonly rpc ChainID doesn't match local chain ID: RPC ID=%s, local ID=%s, node name=%s", - chainID.String(), - s.chainID.String(), - s.name, - ) - s.setState(NodeStateInvalidChainID) - } - // Since it has failed, spin up the verifyLoop that will keep - // retrying until success - s.wg.Add(1) - go s.verifyLoop() - return - } - } - - promPoolRPCNodeTransitionsToAlive.WithLabelValues(s.chainID.String(), s.name).Inc() - s.setState(NodeStateAlive) - s.log.Infow("Sendonly RPC Node is online", "nodeState", s.state) -} - -func (s *sendOnlyNode[CHAIN_ID, RPC]) Close() error { - return s.StopOnce(s.name, func() error { - s.rpc.Close() - close(s.chStop) - s.wg.Wait() - s.setState(NodeStateClosed) - return nil - }) -} - -func (s *sendOnlyNode[CHAIN_ID, RPC]) ConfiguredChainID() CHAIN_ID { - return s.chainID -} - -func (s *sendOnlyNode[CHAIN_ID, RPC]) RPC() RPC { - return s.rpc -} - -func (s *sendOnlyNode[CHAIN_ID, RPC]) String() string { - return fmt.Sprintf("(%s)%s:%s", Secondary.String(), s.name, s.uri.Redacted()) -} - -func (s *sendOnlyNode[CHAIN_ID, RPC]) setState(state NodeState) (changed bool) { - s.stateMu.Lock() - defer s.stateMu.Unlock() - if s.state == state { - return false - } - s.state = state - return true -} - -func (s *sendOnlyNode[CHAIN_ID, RPC]) State() NodeState { - s.stateMu.RLock() - defer s.stateMu.RUnlock() - return s.state -} - -func (s *sendOnlyNode[CHAIN_ID, RPC]) Name() string { - return s.name -} diff --git a/pkg/solana/client/multinode/send_only_node_lifecycle.go b/pkg/solana/client/multinode/send_only_node_lifecycle.go deleted file mode 100644 index 83642feba..000000000 --- a/pkg/solana/client/multinode/send_only_node_lifecycle.go +++ /dev/null @@ -1,65 +0,0 @@ -package client - -import ( - "fmt" - "time" -) - -// verifyLoop may only be triggered once, on Start, if initial chain ID check -// fails. -// -// It will continue checking until success and then exit permanently. -func (s *sendOnlyNode[CHAIN_ID, RPC]) verifyLoop() { - defer s.wg.Done() - ctx, cancel := s.chStop.NewCtx() - defer cancel() - - backoff := NewRedialBackoff() - for { - select { - case <-ctx.Done(): - return - case <-time.After(backoff.Duration()): - } - chainID, err := s.rpc.ChainID(ctx) - if err != nil { - ok := s.IfStarted(func() { - if changed := s.setState(NodeStateUnreachable); changed { - promPoolRPCNodeTransitionsToUnreachable.WithLabelValues(s.chainID.String(), s.name).Inc() - } - }) - if !ok { - return - } - s.log.Errorw(fmt.Sprintf("Verify failed: %v", err), "err", err) - continue - } else if chainID.String() != s.chainID.String() { - ok := s.IfStarted(func() { - if changed := s.setState(NodeStateInvalidChainID); changed { - promPoolRPCNodeTransitionsToInvalidChainID.WithLabelValues(s.chainID.String(), s.name).Inc() - } - }) - if !ok { - return - } - s.log.Errorf( - "sendonly rpc ChainID doesn't match local chain ID: RPC ID=%s, local ID=%s, node name=%s", - chainID.String(), - s.chainID.String(), - s.name, - ) - - continue - } - ok := s.IfStarted(func() { - if changed := s.setState(NodeStateAlive); changed { - promPoolRPCNodeTransitionsToAlive.WithLabelValues(s.chainID.String(), s.name).Inc() - } - }) - if !ok { - return - } - s.log.Infow("Sendonly RPC Node is online", "NodeState", s.state) - return - } -} diff --git a/pkg/solana/client/multinode/transaction_sender.go b/pkg/solana/client/multinode/transaction_sender.go deleted file mode 100644 index 60d7e74df..000000000 --- a/pkg/solana/client/multinode/transaction_sender.go +++ /dev/null @@ -1,285 +0,0 @@ -package client - -import ( - "context" - "errors" - "fmt" - "math" - "slices" - "sync" - "time" - - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" - - "github.com/smartcontractkit/chainlink-common/pkg/logger" - "github.com/smartcontractkit/chainlink-common/pkg/services" -) - -var ( - // PromMultiNodeInvariantViolations reports violation of our assumptions - PromMultiNodeInvariantViolations = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "solana_multi_node_invariant_violations", - Help: "The number of invariant violations", - }, []string{"network", "chainId", "invariant"}) -) - -type SendTxResult interface { - Code() SendTxReturnCode - Error() error -} - -const sendTxQuorum = 0.7 - -// SendTxRPCClient - defines interface of an RPC used by TransactionSender to broadcast transaction -type SendTxRPCClient[TX any, RESULT SendTxResult] interface { - // SendTransaction errors returned should include name or other unique identifier of the RPC - SendTransaction(ctx context.Context, tx TX) RESULT -} - -func NewTransactionSender[TX any, RESULT SendTxResult, CHAIN_ID ID, RPC SendTxRPCClient[TX, RESULT]]( - lggr logger.Logger, - chainID CHAIN_ID, - chainFamily string, - multiNode *MultiNode[CHAIN_ID, RPC], - newResult func(err error) RESULT, - sendTxSoftTimeout time.Duration, -) *TransactionSender[TX, RESULT, CHAIN_ID, RPC] { - if sendTxSoftTimeout == 0 { - sendTxSoftTimeout = QueryTimeout / 2 - } - return &TransactionSender[TX, RESULT, CHAIN_ID, RPC]{ - chainID: chainID, - chainFamily: chainFamily, - lggr: logger.Sugared(lggr).Named("TransactionSender").With("chainID", chainID.String()), - multiNode: multiNode, - newResult: newResult, - sendTxSoftTimeout: sendTxSoftTimeout, - chStop: make(services.StopChan), - } -} - -type TransactionSender[TX any, RESULT SendTxResult, CHAIN_ID ID, RPC SendTxRPCClient[TX, RESULT]] struct { - services.StateMachine - chainID CHAIN_ID - chainFamily string - lggr logger.SugaredLogger - multiNode *MultiNode[CHAIN_ID, RPC] - newResult func(err error) RESULT - sendTxSoftTimeout time.Duration // defines max waiting time from first response til responses evaluation - - wg sync.WaitGroup // waits for all reporting goroutines to finish - chStop services.StopChan -} - -func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) Name() string { - return txSender.lggr.Name() -} - -// SendTransaction - broadcasts transaction to all the send-only and primary nodes in MultiNode. -// A returned nil or error does not guarantee that the transaction will or won't be included. Additional checks must be -// performed to determine the final state. -// -// Send-only nodes' results are ignored as they tend to return false-positive responses. Broadcast to them is necessary -// to speed up the propagation of TX in the network. -// -// Handling of primary nodes' results consists of collection and aggregation. -// In the collection step, we gather as many results as possible while minimizing waiting time. This operation succeeds -// on one of the following conditions: -// * Received at least one success -// * Received at least one result and `sendTxSoftTimeout` expired -// * Received results from the sufficient number of nodes defined by sendTxQuorum. -// The aggregation is based on the following conditions: -// * If there is at least one success - returns success -// * If there is at least one terminal error - returns terminal error -// * If there is both success and terminal error - returns success and reports invariant violation -// * Otherwise, returns any (effectively random) of the errors. -func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) SendTransaction(ctx context.Context, tx TX) RESULT { - var result RESULT - if !txSender.IfStarted(func() { - txResults := make(chan RESULT) - txResultsToReport := make(chan RESULT) - primaryNodeWg := sync.WaitGroup{} - - healthyNodesNum := 0 - err := txSender.multiNode.DoAll(ctx, func(ctx context.Context, rpc RPC, isSendOnly bool) { - if isSendOnly { - txSender.wg.Add(1) - go func(ctx context.Context) { - ctx, cancel := txSender.chStop.Ctx(context.WithoutCancel(ctx)) - defer cancel() - defer txSender.wg.Done() - // Send-only nodes' results are ignored as they tend to return false-positive responses. - // Broadcast to them is necessary to speed up the propagation of TX in the network. - _ = txSender.broadcastTxAsync(ctx, rpc, tx) - }(ctx) - return - } - - // Primary Nodes - healthyNodesNum++ - primaryNodeWg.Add(1) - go func(ctx context.Context) { - ctx, cancel := txSender.chStop.Ctx(context.WithoutCancel(ctx)) - defer cancel() - defer primaryNodeWg.Done() - r := txSender.broadcastTxAsync(ctx, rpc, tx) - select { - case <-ctx.Done(): - return - case txResults <- r: - } - - select { - case <-ctx.Done(): - return - case txResultsToReport <- r: - } - }(ctx) - }) - - // This needs to be done in parallel so the reporting knows when it's done (when the channel is closed) - txSender.wg.Add(1) - go func() { - defer txSender.wg.Done() - primaryNodeWg.Wait() - close(txResultsToReport) - close(txResults) - }() - - if err != nil { - result = txSender.newResult(err) - return - } - - txSender.wg.Add(1) - go txSender.reportSendTxAnomalies(ctx, tx, txResultsToReport) - - result = txSender.collectTxResults(ctx, tx, healthyNodesNum, txResults) - }) { - result = txSender.newResult(errors.New("TransactionSender not started")) - } - - return result -} - -func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) broadcastTxAsync(ctx context.Context, rpc RPC, tx TX) RESULT { - result := rpc.SendTransaction(ctx, tx) - txSender.lggr.Debugw("Node sent transaction", "tx", tx, "err", result.Error()) - if !slices.Contains(sendTxSuccessfulCodes, result.Code()) && ctx.Err() == nil { - txSender.lggr.Warnw("RPC returned error", "tx", tx, "err", result.Error()) - } - return result -} - -func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) reportSendTxAnomalies(ctx context.Context, tx TX, txResults <-chan RESULT) { - defer txSender.wg.Done() - resultsByCode := sendTxResults[RESULT]{} - // txResults eventually will be closed - for txResult := range txResults { - resultsByCode[txResult.Code()] = append(resultsByCode[txResult.Code()], txResult) - } - - _, criticalErr := aggregateTxResults[RESULT](resultsByCode) - if criticalErr != nil && ctx.Err() == nil { - txSender.lggr.Criticalw("observed invariant violation on SendTransaction", "tx", tx, "resultsByCode", resultsByCode, "err", criticalErr) - PromMultiNodeInvariantViolations.WithLabelValues(txSender.chainFamily, txSender.chainID.String(), criticalErr.Error()).Inc() - } -} - -type sendTxResults[RESULT any] map[SendTxReturnCode][]RESULT - -func aggregateTxResults[RESULT any](resultsByCode sendTxResults[RESULT]) (result RESULT, criticalErr error) { - severeErrors, hasSevereErrors := findFirstIn(resultsByCode, sendTxSevereErrors) - successResults, hasSuccess := findFirstIn(resultsByCode, sendTxSuccessfulCodes) - if hasSuccess { - // We assume that primary node would never report false positive txResult for a transaction. - // Thus, if such case occurs it's probably due to misconfiguration or a bug and requires manual intervention. - if hasSevereErrors { - const errMsg = "found contradictions in nodes replies on SendTransaction: got success and severe error" - // return success, since at least 1 node has accepted our broadcasted Tx, and thus it can now be included onchain - return successResults[0], errors.New(errMsg) - } - - // other errors are temporary - we are safe to return success - return successResults[0], nil - } - - if hasSevereErrors { - return severeErrors[0], nil - } - - // return temporary error - for _, r := range resultsByCode { - return r[0], nil - } - - criticalErr = fmt.Errorf("expected at least one response on SendTransaction") - return result, criticalErr -} - -func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) collectTxResults(ctx context.Context, tx TX, healthyNodesNum int, txResults <-chan RESULT) RESULT { - if healthyNodesNum == 0 { - return txSender.newResult(ErroringNodeError) - } - requiredResults := int(math.Ceil(float64(healthyNodesNum) * sendTxQuorum)) - errorsByCode := sendTxResults[RESULT]{} - var softTimeoutChan <-chan time.Time - var resultsCount int -loop: - for { - select { - case <-ctx.Done(): - txSender.lggr.Debugw("Failed to collect of the results before context was done", "tx", tx, "errorsByCode", errorsByCode) - return txSender.newResult(ctx.Err()) - case r := <-txResults: - errorsByCode[r.Code()] = append(errorsByCode[r.Code()], r) - resultsCount++ - if slices.Contains(sendTxSuccessfulCodes, r.Code()) || resultsCount >= requiredResults { - break loop - } - case <-softTimeoutChan: - txSender.lggr.Debugw("Send Tx soft timeout expired - returning responses we've collected so far", "tx", tx, "resultsCount", resultsCount, "requiredResults", requiredResults) - break loop - } - - if softTimeoutChan == nil { - tm := time.NewTimer(txSender.sendTxSoftTimeout) - softTimeoutChan = tm.C - // we are fine with stopping timer at the end of function - //nolint - defer tm.Stop() - } - } - - // ignore critical error as it's reported in reportSendTxAnomalies - result, _ := aggregateTxResults(errorsByCode) - txSender.lggr.Debugw("Collected results", "errorsByCode", errorsByCode, "result", result) - return result -} - -func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) Start(ctx context.Context) error { - return txSender.StartOnce("TransactionSender", func() error { - return nil - }) -} - -func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) Close() error { - return txSender.StopOnce("TransactionSender", func() error { - txSender.lggr.Debug("Closing TransactionSender") - close(txSender.chStop) - txSender.wg.Wait() - return nil - }) -} - -// findFirstIn - returns the first existing key and value for the slice of keys -func findFirstIn[K comparable, V any](set map[K]V, keys []K) (V, bool) { - for _, k := range keys { - if v, ok := set[k]; ok { - return v, true - } - } - var zeroV V - return zeroV, false -} diff --git a/pkg/solana/client/multinode/types.go b/pkg/solana/client/multinode/types.go deleted file mode 100644 index 51b70e573..000000000 --- a/pkg/solana/client/multinode/types.go +++ /dev/null @@ -1,108 +0,0 @@ -package client - -import ( - "context" - "fmt" - "math/big" -) - -// ID represents the base type, for any chain's ID. -// It should be convertible to a string, that can uniquely identify this chain -type ID fmt.Stringer - -// StringID enables using string directly as a ChainID -type StringID string - -func (s StringID) String() string { - return string(s) -} - -// Subscription represents an event subscription where events are -// delivered on a data channel. -// This is a generic interface for Subscription to represent used by clients. -type Subscription interface { - // Unsubscribe cancels the sending of events to the data channel - // and closes the error channel. Unsubscribe should be callable multiple - // times without causing an error. - Unsubscribe() - // Err returns the subscription error channel. The error channel receives - // a value if there is an issue with the subscription (e.g. the network connection - // delivering the events has been closed). Only one value will ever be sent. - // The error channel is closed by Unsubscribe. - Err() <-chan error -} - -// RPCClient includes all the necessary generalized RPC methods used by Node to perform health checks -type RPCClient[ - CHAIN_ID ID, - HEAD Head, -] interface { - // ChainID - fetches ChainID from the RPC to verify that it matches config - ChainID(ctx context.Context) (CHAIN_ID, error) - // Dial - prepares the RPC for usage. Can be called on fresh or closed RPC - Dial(ctx context.Context) error - // SubscribeToHeads - returns channel and subscription for new heads. - SubscribeToHeads(ctx context.Context) (<-chan HEAD, Subscription, error) - // SubscribeToFinalizedHeads - returns channel and subscription for finalized heads. - SubscribeToFinalizedHeads(ctx context.Context) (<-chan HEAD, Subscription, error) - // Ping - returns error if RPC is not reachable - Ping(context.Context) error - // IsSyncing - returns true if the RPC is in Syncing state and can not process calls - IsSyncing(ctx context.Context) (bool, error) - // UnsubscribeAllExcept - close all subscriptions except `subs` - UnsubscribeAllExcept(subs ...Subscription) - // Close - closes all subscriptions and aborts all RPC calls - Close() - // GetInterceptedChainInfo - returns latest and highest observed by application layer ChainInfo. - // latest ChainInfo is the most recent value received within a NodeClient's current lifecycle between Dial and DisconnectAll. - // highestUserObservations ChainInfo is the highest ChainInfo observed excluding health checks calls. - // Its values must not be reset. - // The results of corresponding calls, to get the most recent head and the latest finalized head, must be - // intercepted and reflected in ChainInfo before being returned to a caller. Otherwise, MultiNode is not able to - // provide repeatable read guarantee. - // DisconnectAll must reset latest ChainInfo to default value. - // Ensure implementation does not have a race condition when values are reset before request completion and as - // a result latest ChainInfo contains information from the previous cycle. - GetInterceptedChainInfo() (latest, highestUserObservations ChainInfo) -} - -// Head is the interface required by the NodeClient -type Head interface { - BlockNumber() int64 - BlockDifficulty() *big.Int - IsValid() bool -} - -// PoolChainInfoProvider - provides aggregation of nodes pool ChainInfo -type PoolChainInfoProvider interface { - // LatestChainInfo - returns number of live nodes available in the pool, so we can prevent the last alive node in a pool from being - // moved to out-of-sync state. It is better to have one out-of-sync node than no nodes at all. - // Returns highest latest ChainInfo within the alive nodes. E.g. most recent block number and highest block number - // observed by Node A are 10 and 15; Node B - 12 and 14. This method will return 12. - LatestChainInfo() (int, ChainInfo) - // HighestUserObservations - returns highest ChainInfo ever observed by any user of MultiNode. - HighestUserObservations() ChainInfo -} - -// ChainInfo - defines RPC's or MultiNode's view on the chain -type ChainInfo struct { - BlockNumber int64 - FinalizedBlockNumber int64 - TotalDifficulty *big.Int -} - -func MaxTotalDifficulty(a, b *big.Int) *big.Int { - if a == nil { - if b == nil { - return nil - } - - return big.NewInt(0).Set(b) - } - - if b == nil || a.Cmp(b) >= 0 { - return big.NewInt(0).Set(a) - } - - return big.NewInt(0).Set(b) -} diff --git a/pkg/solana/client/multinode_client.go b/pkg/solana/client/multinode_client.go index e6a70de9c..b97525044 100644 --- a/pkg/solana/client/multinode_client.go +++ b/pkg/solana/client/multinode_client.go @@ -12,8 +12,8 @@ import ( "github.com/gagliardetto/solana-go/rpc" "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/services" + mn "github.com/smartcontractkit/chainlink-framework/multinode" - mn "github.com/smartcontractkit/chainlink-solana/pkg/solana/client/multinode" "github.com/smartcontractkit/chainlink-solana/pkg/solana/config" ) @@ -102,8 +102,7 @@ func (m *MultiNodeClient) SubscribeToHeads(ctx context.Context) (<-chan *Head, m ctx, cancel, chStopInFlight, _ := m.acquireQueryCtx(ctx, m.cfg.TxTimeout()) defer cancel() - // TODO: BCFR-1070 - Add BlockPollInterval - pollInterval := m.cfg.MultiNode.FinalizedBlockPollInterval() // Use same interval as finalized polling + pollInterval := m.cfg.MultiNode.NewHeadsPollInterval() if pollInterval == 0 { return nil, nil, errors.New("PollInterval is 0") } diff --git a/pkg/solana/config/multinode.go b/pkg/solana/config/multinode.go deleted file mode 100644 index d002d489e..000000000 --- a/pkg/solana/config/multinode.go +++ /dev/null @@ -1,206 +0,0 @@ -package config - -import ( - "time" - - "github.com/smartcontractkit/chainlink-common/pkg/config" - - mn "github.com/smartcontractkit/chainlink-solana/pkg/solana/client/multinode" -) - -// MultiNodeConfig is a wrapper to provide required functions while keeping configs Public -type MultiNodeConfig struct { - MultiNode -} - -type MultiNode struct { - // Feature flag - Enabled *bool - - // Node Configs - PollFailureThreshold *uint32 - PollInterval *config.Duration - SelectionMode *string - SyncThreshold *uint32 - NodeIsSyncingEnabled *bool - LeaseDuration *config.Duration - FinalizedBlockPollInterval *config.Duration - EnforceRepeatableRead *bool - DeathDeclarationDelay *config.Duration - - // Chain Configs - NodeNoNewHeadsThreshold *config.Duration - NoNewFinalizedHeadsThreshold *config.Duration - FinalityDepth *uint32 - FinalityTagEnabled *bool - FinalizedBlockOffset *uint32 -} - -func (c *MultiNodeConfig) Enabled() bool { - return c.MultiNode.Enabled != nil && *c.MultiNode.Enabled -} - -func (c *MultiNodeConfig) PollFailureThreshold() uint32 { - return *c.MultiNode.PollFailureThreshold -} - -func (c *MultiNodeConfig) PollInterval() time.Duration { - return c.MultiNode.PollInterval.Duration() -} - -func (c *MultiNodeConfig) SelectionMode() string { - return *c.MultiNode.SelectionMode -} - -func (c *MultiNodeConfig) SyncThreshold() uint32 { - return *c.MultiNode.SyncThreshold -} - -func (c *MultiNodeConfig) NodeIsSyncingEnabled() bool { - return *c.MultiNode.NodeIsSyncingEnabled -} - -func (c *MultiNodeConfig) LeaseDuration() time.Duration { return c.MultiNode.LeaseDuration.Duration() } - -func (c *MultiNodeConfig) FinalizedBlockPollInterval() time.Duration { - return c.MultiNode.FinalizedBlockPollInterval.Duration() -} - -func (c *MultiNodeConfig) EnforceRepeatableRead() bool { return *c.MultiNode.EnforceRepeatableRead } - -func (c *MultiNodeConfig) DeathDeclarationDelay() time.Duration { - return c.MultiNode.DeathDeclarationDelay.Duration() -} - -func (c *MultiNodeConfig) NodeNoNewHeadsThreshold() time.Duration { - return c.MultiNode.NodeNoNewHeadsThreshold.Duration() -} - -func (c *MultiNodeConfig) NoNewFinalizedHeadsThreshold() time.Duration { - return c.MultiNode.NoNewFinalizedHeadsThreshold.Duration() -} - -func (c *MultiNodeConfig) FinalityDepth() uint32 { return *c.MultiNode.FinalityDepth } - -func (c *MultiNodeConfig) FinalityTagEnabled() bool { return *c.MultiNode.FinalityTagEnabled } - -func (c *MultiNodeConfig) FinalizedBlockOffset() uint32 { return *c.MultiNode.FinalizedBlockOffset } - -func (c *MultiNodeConfig) SetDefaults() { - // MultiNode is disabled as it's not fully implemented yet: BCFR-122 - if c.MultiNode.Enabled == nil { - c.MultiNode.Enabled = ptr(false) - } - - /* Node Configs */ - // Failure threshold for polling set to 5 to tolerate some polling failures before taking action. - if c.MultiNode.PollFailureThreshold == nil { - c.MultiNode.PollFailureThreshold = ptr(uint32(5)) - } - // Poll interval is set to 15 seconds to ensure timely updates while minimizing resource usage. - if c.MultiNode.PollInterval == nil { - c.MultiNode.PollInterval = config.MustNewDuration(15 * time.Second) - } - // Selection mode defaults to priority level to enable using node priorities - if c.MultiNode.SelectionMode == nil { - c.MultiNode.SelectionMode = ptr(mn.NodeSelectionModePriorityLevel) - } - // The sync threshold is set to 10 to allow for some flexibility in node synchronization before considering it out of sync. - if c.MultiNode.SyncThreshold == nil { - c.MultiNode.SyncThreshold = ptr(uint32(10)) - } - // Lease duration is set to 1 minute by default to allow node locks for a reasonable amount of time. - if c.MultiNode.LeaseDuration == nil { - c.MultiNode.LeaseDuration = config.MustNewDuration(time.Minute) - } - // Node syncing is not relevant for Solana and is disabled by default. - if c.MultiNode.NodeIsSyncingEnabled == nil { - c.MultiNode.NodeIsSyncingEnabled = ptr(false) - } - // The finalized block polling interval is set to 5 seconds to ensure timely updates while minimizing resource usage. - if c.MultiNode.FinalizedBlockPollInterval == nil { - c.MultiNode.FinalizedBlockPollInterval = config.MustNewDuration(5 * time.Second) - } - // Repeatable read guarantee should be enforced by default. - if c.MultiNode.EnforceRepeatableRead == nil { - c.MultiNode.EnforceRepeatableRead = ptr(true) - } - // The delay before declaring a node dead is set to 20 seconds to give nodes time to recover from temporary issues. - if c.MultiNode.DeathDeclarationDelay == nil { - c.MultiNode.DeathDeclarationDelay = config.MustNewDuration(20 * time.Second) - } - - /* Chain Configs */ - // Threshold for no new heads is set to 20 seconds, assuming that heads should update at a reasonable pace. - if c.MultiNode.NodeNoNewHeadsThreshold == nil { - c.MultiNode.NodeNoNewHeadsThreshold = config.MustNewDuration(20 * time.Second) - } - // Similar to heads, finalized heads should be updated within 20 seconds. - if c.MultiNode.NoNewFinalizedHeadsThreshold == nil { - c.MultiNode.NoNewFinalizedHeadsThreshold = config.MustNewDuration(20 * time.Second) - } - // Finality tags are used in Solana and enabled by default. - if c.MultiNode.FinalityTagEnabled == nil { - c.MultiNode.FinalityTagEnabled = ptr(true) - } - // Finality depth will not be used since finality tags are enabled. - if c.MultiNode.FinalityDepth == nil { - c.MultiNode.FinalityDepth = ptr(uint32(0)) - } - // Finalized block offset allows for RPCs to be slightly behind the finalized block. - if c.MultiNode.FinalizedBlockOffset == nil { - c.MultiNode.FinalizedBlockOffset = ptr(uint32(50)) - } -} - -func (c *MultiNodeConfig) SetFrom(f *MultiNodeConfig) { - if f.MultiNode.Enabled != nil { - c.MultiNode.Enabled = f.MultiNode.Enabled - } - - // Node Configs - if f.MultiNode.PollFailureThreshold != nil { - c.MultiNode.PollFailureThreshold = f.MultiNode.PollFailureThreshold - } - if f.MultiNode.PollInterval != nil { - c.MultiNode.PollInterval = f.MultiNode.PollInterval - } - if f.MultiNode.SelectionMode != nil { - c.MultiNode.SelectionMode = f.MultiNode.SelectionMode - } - if f.MultiNode.SyncThreshold != nil { - c.MultiNode.SyncThreshold = f.MultiNode.SyncThreshold - } - if f.MultiNode.NodeIsSyncingEnabled != nil { - c.MultiNode.NodeIsSyncingEnabled = f.MultiNode.NodeIsSyncingEnabled - } - if f.MultiNode.LeaseDuration != nil { - c.MultiNode.LeaseDuration = f.MultiNode.LeaseDuration - } - if f.MultiNode.FinalizedBlockPollInterval != nil { - c.MultiNode.FinalizedBlockPollInterval = f.MultiNode.FinalizedBlockPollInterval - } - if f.MultiNode.EnforceRepeatableRead != nil { - c.MultiNode.EnforceRepeatableRead = f.MultiNode.EnforceRepeatableRead - } - if f.MultiNode.DeathDeclarationDelay != nil { - c.MultiNode.DeathDeclarationDelay = f.MultiNode.DeathDeclarationDelay - } - - // Chain Configs - if f.MultiNode.NodeNoNewHeadsThreshold != nil { - c.MultiNode.NodeNoNewHeadsThreshold = f.MultiNode.NodeNoNewHeadsThreshold - } - if f.MultiNode.NoNewFinalizedHeadsThreshold != nil { - c.MultiNode.NoNewFinalizedHeadsThreshold = f.MultiNode.NoNewFinalizedHeadsThreshold - } - if f.MultiNode.FinalityDepth != nil { - c.MultiNode.FinalityDepth = f.MultiNode.FinalityDepth - } - if f.MultiNode.FinalityTagEnabled != nil { - c.MultiNode.FinalityTagEnabled = f.MultiNode.FinalityTagEnabled - } - if f.MultiNode.FinalizedBlockOffset != nil { - c.MultiNode.FinalizedBlockOffset = f.MultiNode.FinalizedBlockOffset - } -} diff --git a/pkg/solana/config/toml.go b/pkg/solana/config/toml.go index 9d46c0b65..5a1d64a10 100644 --- a/pkg/solana/config/toml.go +++ b/pkg/solana/config/toml.go @@ -12,6 +12,8 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/config" relaytypes "github.com/smartcontractkit/chainlink-common/pkg/types" + mn "github.com/smartcontractkit/chainlink-framework/multinode" + mnCfg "github.com/smartcontractkit/chainlink-framework/multinode/config" ) type TOMLConfigs []*TOMLConfig @@ -113,7 +115,7 @@ type TOMLConfig struct { // Do not access directly, use [IsEnabled] Enabled *bool Chain - MultiNode MultiNodeConfig + MultiNode mnCfg.MultiNodeConfig Nodes Nodes } @@ -312,12 +314,57 @@ func (c *TOMLConfig) ListNodes() Nodes { func (c *TOMLConfig) SetDefaults() { c.Chain.SetDefaults() - c.MultiNode.SetDefaults() + c.MultiNode.SetFrom(defaultMultiNodeConfig) } func NewDefault() *TOMLConfig { cfg := &TOMLConfig{} cfg.Chain.SetDefaults() - cfg.MultiNode.SetDefaults() + cfg.MultiNode.SetFrom(defaultMultiNodeConfig) + return cfg +} + +var defaultMultiNodeConfig = &mnCfg.MultiNodeConfig{ + MultiNode: mnCfg.MultiNode{ + // Have multinode disabled by default + Enabled: ptr(false), + /* Node Configs */ + // Failure threshold for polling set to 5 to tolerate some polling failures before taking action. + PollFailureThreshold: ptr(uint32(5)), + // Poll interval is set to 15 seconds to ensure timely updates while minimizing resource usage. + PollInterval: config.MustNewDuration(15 * time.Second), + // Selection mode defaults to priority level to enable using node priorities + SelectionMode: ptr(mn.NodeSelectionModePriorityLevel), + // The sync threshold is set to 10 to allow for some flexibility in node synchronization before considering it out of sync. + SyncThreshold: ptr(uint32(10)), + // Lease duration is set to 1 minute by default to allow node locks for a reasonable amount of time. + LeaseDuration: config.MustNewDuration(time.Minute), + // Node syncing is not relevant for Solana and is disabled by default. + NodeIsSyncingEnabled: ptr(false), + // The new heads polling interval is set to 5 seconds to ensure timely updates while minimizing resource usage. + NewHeadsPollInterval: config.MustNewDuration(5 * time.Second), + // The finalized block polling interval is set to 5 seconds to ensure timely updates while minimizing resource usage. + FinalizedBlockPollInterval: config.MustNewDuration(5 * time.Second), + // Repeatable read guarantee should be enforced by default. + EnforceRepeatableRead: ptr(true), + // The delay before declaring a node dead is set to 20 seconds to give nodes time to recover from temporary issues. + DeathDeclarationDelay: config.MustNewDuration(20 * time.Second), + /* Chain Configs */ + // Threshold for no new heads is set to 20 seconds, assuming that heads should update at a reasonable pace. + NodeNoNewHeadsThreshold: config.MustNewDuration(20 * time.Second), + // Similar to heads, finalized heads should be updated within 20 seconds. + NoNewFinalizedHeadsThreshold: config.MustNewDuration(20 * time.Second), + // Finality tags are used in Solana and enabled by default. + FinalityTagEnabled: ptr(true), + // Finality depth will not be used since finality tags are enabled. + FinalityDepth: ptr(uint32(0)), + // Finalized block offset allows for RPCs to be slightly behind the finalized block. + FinalizedBlockOffset: ptr(uint32(50)), + }, +} + +func NewDefaultMultiNodeConfig() mnCfg.MultiNodeConfig { + cfg := mnCfg.MultiNodeConfig{} + cfg.SetFrom(defaultMultiNodeConfig) return cfg }