From fd75761c982f5a6eb37e84bf21563c5914e45fe1 Mon Sep 17 00:00:00 2001 From: martin-cll <121895364+martin-cll@users.noreply.github.com> Date: Thu, 1 Aug 2024 15:17:03 +0200 Subject: [PATCH] MERC-6003 Add Mercury v4 schema (#73) * Add Mercury v4 types * MERC-6003 Move aggregator to v4 pkg * Fix lints --- go.mod | 40 +- go.sum | 80 +-- mercury/aggregate_functions_test.go | 31 + mercury/cmd/chainlink-mercury/plugin.go | 16 + mercury/v4/aggregate_functions.go | 39 ++ mercury/v4/aggregate_functions_test.go | 294 +++++++++ mercury/v4/mercury.go | 507 +++++++++++++++ mercury/v4/mercury_observation_v4.pb.go | 267 ++++++++ mercury/v4/mercury_observation_v4.proto | 24 + mercury/v4/mercury_test.go | 821 ++++++++++++++++++++++++ mercury/v4/observation.go | 110 ++++ 11 files changed, 2169 insertions(+), 60 deletions(-) create mode 100644 mercury/v4/aggregate_functions.go create mode 100644 mercury/v4/aggregate_functions_test.go create mode 100644 mercury/v4/mercury.go create mode 100644 mercury/v4/mercury_observation_v4.pb.go create mode 100644 mercury/v4/mercury_observation_v4.proto create mode 100644 mercury/v4/mercury_test.go create mode 100644 mercury/v4/observation.go diff --git a/go.mod b/go.mod index 503d434..56eb11b 100644 --- a/go.mod +++ b/go.mod @@ -4,12 +4,12 @@ go 1.21 require ( github.com/hashicorp/go-plugin v1.6.0 - github.com/shopspring/decimal v1.3.1 - github.com/smartcontractkit/chainlink-common v0.2.1-0.20240717132349-ee5af9b79834 + github.com/shopspring/decimal v1.4.0 + github.com/smartcontractkit/chainlink-common v0.2.2-0.20240731184516-249ef7ad0cdc github.com/smartcontractkit/libocr v0.0.0-20240419185742-fd3cab206b2c github.com/stretchr/testify v1.9.0 - golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa - google.golang.org/protobuf v1.34.1 + golang.org/x/exp v0.0.0-20240716175740-e3f259677ff7 + google.golang.org/protobuf v1.34.2 ) require ( @@ -17,12 +17,12 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/buger/jsonparser v1.1.1 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/fatih/color v1.14.1 // indirect github.com/fxamacker/cbor/v2 v2.5.0 // indirect github.com/go-json-experiment/json v0.0.0-20231102232822-2e55bd4e08b0 // indirect - github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/go-cmp v0.6.0 // indirect @@ -54,22 +54,22 @@ require ( github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect github.com/x448/float16 v0.8.4 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.52.0 // indirect - go.opentelemetry.io/otel v1.27.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0 // indirect - go.opentelemetry.io/otel/metric v1.27.0 // indirect - go.opentelemetry.io/otel/sdk v1.27.0 // indirect - go.opentelemetry.io/otel/trace v1.27.0 // indirect - go.opentelemetry.io/proto/otlp v1.2.0 // indirect + go.opentelemetry.io/otel v1.28.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0 // indirect + go.opentelemetry.io/otel/metric v1.28.0 // indirect + go.opentelemetry.io/otel/sdk v1.28.0 // indirect + go.opentelemetry.io/otel/trace v1.28.0 // indirect + go.opentelemetry.io/proto/otlp v1.3.1 // indirect go.uber.org/multierr v1.11.0 // indirect - go.uber.org/zap v1.26.0 // indirect - golang.org/x/crypto v0.24.0 // indirect - golang.org/x/net v0.25.0 // indirect - golang.org/x/sys v0.21.0 // indirect + go.uber.org/zap v1.27.0 // indirect + golang.org/x/crypto v0.25.0 // indirect + golang.org/x/net v0.27.0 // indirect + golang.org/x/sys v0.22.0 // indirect golang.org/x/text v0.16.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240520151616-dc85e6b867a5 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240520151616-dc85e6b867a5 // indirect - google.golang.org/grpc v1.64.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240711142825-46eb208f015d // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240711142825-46eb208f015d // indirect + google.golang.org/grpc v1.65.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 6db87a5..4b87023 100644 --- a/go.sum +++ b/go.sum @@ -11,8 +11,8 @@ github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx2 github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -31,8 +31,8 @@ github.com/fxamacker/cbor/v2 v2.5.0/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrt github.com/go-json-experiment/json v0.0.0-20231102232822-2e55bd4e08b0 h1:ymLjT4f35nQbASLnvxEde4XOBL+Sn7rFuV+FOJqkljg= github.com/go-json-experiment/json v0.0.0-20231102232822-2e55bd4e08b0/go.mod h1:6daplAwHHGbUGib4990V3Il26O0OC4aRyvewaaAihaA= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= -github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -127,10 +127,10 @@ github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/santhosh-tekuri/jsonschema/v5 v5.2.0 h1:WCcC4vZDS1tYNxjWlwRJZQy28r8CMoggKnxNzxsVDMQ= github.com/santhosh-tekuri/jsonschema/v5 v5.2.0/go.mod h1:FKdcjfQW6rpZSnxxUvEA5H/cDPdvJ/SZJQLWWXWGrZ0= -github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= -github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= -github.com/smartcontractkit/chainlink-common v0.2.1-0.20240717132349-ee5af9b79834 h1:pTf4xdcmiWBqWZ6rTy2RMTDBzhHk89VC1pM7jXKQztI= -github.com/smartcontractkit/chainlink-common v0.2.1-0.20240717132349-ee5af9b79834/go.mod h1:fh9eBbrReCmv31bfz52ENCAMa7nTKQbdhb2B3+S2VGo= +github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= +github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= +github.com/smartcontractkit/chainlink-common v0.2.2-0.20240731184516-249ef7ad0cdc h1:nNZqLasN8y5huDKX76JUZtni7WkUI36J61//czbJpDM= +github.com/smartcontractkit/chainlink-common v0.2.2-0.20240731184516-249ef7ad0cdc/go.mod h1:Jg1sCTsbxg76YByI8ifpFby3FvVqISStHT8ypy9ocmY= github.com/smartcontractkit/go-plugin v0.0.0-20240208201424-b3b91517de16 h1:TFe+FvzxClblt6qRfqEhUfa4kFQx5UobuoFGO2W4mMo= github.com/smartcontractkit/go-plugin v0.0.0-20240208201424-b3b91517de16/go.mod h1:lBS5MtSSBZk0SHc66KACcjjlU6WzEVP/8pwz68aMkCI= github.com/smartcontractkit/grpc-proxy v0.0.0-20230731113816-f1be6620749f h1:hgJif132UCdjo8u43i7iPN1/MFnu49hv7lFGFftCHKU= @@ -157,34 +157,34 @@ github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcY github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.52.0 h1:vS1Ao/R55RNV4O7TA2Qopok8yN+X0LIP6RVWLFkprck= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.52.0/go.mod h1:BMsdeOxN04K0L5FNUBfjFdvwWGNe/rkmSwH4Aelu/X0= -go.opentelemetry.io/otel v1.27.0 h1:9BZoF3yMK/O1AafMiQTVu0YDj5Ea4hPhxCs7sGva+cg= -go.opentelemetry.io/otel v1.27.0/go.mod h1:DMpAK8fzYRzs+bi3rS5REupisuqTheUlSZJ1WnZaPAQ= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 h1:R9DE4kQ4k+YtfLI2ULwX82VtNQ2J8yZmA7ZIF/D+7Mc= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0/go.mod h1:OQFyQVrDlbe+R7xrEyDr/2Wr67Ol0hRUgsfA+V5A95s= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0 h1:qFffATk0X+HD+f1Z8lswGiOQYKHRlzfmdJm0wEaVrFA= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0/go.mod h1:MOiCmryaYtc+V0Ei+Tx9o5S1ZjA7kzLucuVuyzBZloQ= -go.opentelemetry.io/otel/metric v1.27.0 h1:hvj3vdEKyeCi4YaYfNjv2NUje8FqKqUY8IlF0FxV/ik= -go.opentelemetry.io/otel/metric v1.27.0/go.mod h1:mVFgmRlhljgBiuk/MP/oKylr4hs85GZAylncepAX/ak= -go.opentelemetry.io/otel/sdk v1.27.0 h1:mlk+/Y1gLPLn84U4tI8d3GNJmGT/eXe3ZuOXN9kTWmI= -go.opentelemetry.io/otel/sdk v1.27.0/go.mod h1:Ha9vbLwJE6W86YstIywK2xFfPjbWlCuwPtMkKdz/Y4A= -go.opentelemetry.io/otel/trace v1.27.0 h1:IqYb813p7cmbHk0a5y6pD5JPakbVfftRXABGt5/Rscw= -go.opentelemetry.io/otel/trace v1.27.0/go.mod h1:6RiD1hkAprV4/q+yd2ln1HG9GoPx39SuvvstaLBl+l4= -go.opentelemetry.io/proto/otlp v1.2.0 h1:pVeZGk7nXDC9O2hncA6nHldxEjm6LByfA2aN8IOkz94= -go.opentelemetry.io/proto/otlp v1.2.0/go.mod h1:gGpR8txAl5M03pDhMC79G6SdqNV26naRm/KDsgaHD8A= +go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo= +go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 h1:3Q/xZUyC1BBkualc9ROb4G8qkH90LXEIICcs5zv1OYY= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0/go.mod h1:s75jGIWA9OfCMzF0xr+ZgfrB5FEbbV7UuYo32ahUiFI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0 h1:R3X6ZXmNPRR8ul6i3WgFURCHzaXjHdm0karRG/+dj3s= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0/go.mod h1:QWFXnDavXWwMx2EEcZsf3yxgEKAqsxQ+Syjp+seyInw= +go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q= +go.opentelemetry.io/otel/metric v1.28.0/go.mod h1:Fb1eVBFZmLVTMb6PPohq3TO9IIhUisDsbJoL/+uQW4s= +go.opentelemetry.io/otel/sdk v1.28.0 h1:b9d7hIry8yZsgtbmM0DKyPWMMUMlK9NEKuIG4aBqWyE= +go.opentelemetry.io/otel/sdk v1.28.0/go.mod h1:oYj7ClPUA7Iw3m+r7GeEjz0qckQRJK2B8zjcZEfu7Pg= +go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g= +go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI= +go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= +go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= -go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= -golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= +golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= +golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa h1:FRnLl4eNAQl8hwxVVC17teOw8kdjVDVAiFMtgUdTSRQ= -golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE= +golang.org/x/exp v0.0.0-20240716175740-e3f259677ff7 h1:wDLEX9a7YQoKdKNQt88rtydkqDxeGaBUTnIYc3iG/mA= +golang.org/x/exp v0.0.0-20240716175740-e3f259677ff7/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -200,8 +200,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210331212208-0fccb6fa2b5c/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= -golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= +golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -227,8 +227,8 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= -golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -254,17 +254,17 @@ google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoA google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20210401141331-865547bb08e2/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= -google.golang.org/genproto/googleapis/api v0.0.0-20240520151616-dc85e6b867a5 h1:P8OJ/WCl/Xo4E4zoe4/bifHpSmmKwARqyqE4nW6J2GQ= -google.golang.org/genproto/googleapis/api v0.0.0-20240520151616-dc85e6b867a5/go.mod h1:RGnPtTG7r4i8sPlNyDeikXF99hMM+hN6QMm4ooG9g2g= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240520151616-dc85e6b867a5 h1:Q2RxlXqh1cgzzUgV261vBO2jI5R/3DD1J2pM0nI4NhU= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240520151616-dc85e6b867a5/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= +google.golang.org/genproto/googleapis/api v0.0.0-20240711142825-46eb208f015d h1:kHjw/5UfflP/L5EbledDrcG4C2597RtymmGRZvHiCuY= +google.golang.org/genproto/googleapis/api v0.0.0-20240711142825-46eb208f015d/go.mod h1:mw8MG/Qz5wfgYr6VqVCiZcHe/GJEfI+oGGDCohaVgB0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240711142825-46eb208f015d h1:JU0iKnSg02Gmb5ZdV8nYsKEKsP6o/FGVWTrw4i1DA9A= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240711142825-46eb208f015d/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= -google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= +google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= +google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -276,8 +276,8 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= -google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/mercury/aggregate_functions_test.go b/mercury/aggregate_functions_test.go index 28df8ce..150a2ef 100644 --- a/mercury/aggregate_functions_test.go +++ b/mercury/aggregate_functions_test.go @@ -23,6 +23,8 @@ type testParsedAttributedObservation struct { LinkFeeValid bool NativeFee *big.Int NativeFeeValid bool + MarketStatus uint32 + MarketStatusValid bool } func (t testParsedAttributedObservation) GetObserver() commontypes.OracleID { return 0 } @@ -45,6 +47,10 @@ func (t testParsedAttributedObservation) GetLinkFee() (*big.Int, bool) { func (t testParsedAttributedObservation) GetNativeFee() (*big.Int, bool) { return t.NativeFee, t.NativeFeeValid } +func (t testParsedAttributedObservation) GetMarketStatus() (uint32, bool) { + return t.MarketStatus, t.MarketStatusValid +} + func newValidParsedAttributedObservations() []testParsedAttributedObservation { return []testParsedAttributedObservation{ testParsedAttributedObservation{ @@ -64,6 +70,9 @@ func newValidParsedAttributedObservations() []testParsedAttributedObservation { LinkFeeValid: true, NativeFee: big.NewInt(1), NativeFeeValid: true, + + MarketStatus: 1, + MarketStatusValid: true, }, testParsedAttributedObservation{ Timestamp: 1689648456, @@ -82,6 +91,9 @@ func newValidParsedAttributedObservations() []testParsedAttributedObservation { LinkFeeValid: true, NativeFee: big.NewInt(2), NativeFeeValid: true, + + MarketStatus: 1, + MarketStatusValid: true, }, testParsedAttributedObservation{ Timestamp: 1689648789, @@ -100,6 +112,9 @@ func newValidParsedAttributedObservations() []testParsedAttributedObservation { LinkFeeValid: true, NativeFee: big.NewInt(3), NativeFeeValid: true, + + MarketStatus: 2, + MarketStatusValid: true, }, testParsedAttributedObservation{ Timestamp: 1689648789, @@ -118,9 +133,13 @@ func newValidParsedAttributedObservations() []testParsedAttributedObservation { LinkFeeValid: true, NativeFee: big.NewInt(4), NativeFeeValid: true, + + MarketStatus: 3, + MarketStatusValid: true, }, } } + func NewValidParsedAttributedObservations(paos ...testParsedAttributedObservation) []testParsedAttributedObservation { if len(paos) == 0 { paos = newValidParsedAttributedObservations() @@ -152,6 +171,9 @@ func NewInvalidParsedAttributedObservations() []testParsedAttributedObservation LinkFeeValid: false, NativeFee: big.NewInt(1), NativeFeeValid: false, + + MarketStatus: 1, + MarketStatusValid: false, }, testParsedAttributedObservation{ Timestamp: 2, @@ -170,6 +192,9 @@ func NewInvalidParsedAttributedObservations() []testParsedAttributedObservation LinkFeeValid: false, NativeFee: big.NewInt(2), NativeFeeValid: false, + + MarketStatus: 1, + MarketStatusValid: false, }, testParsedAttributedObservation{ Timestamp: 2, @@ -188,6 +213,9 @@ func NewInvalidParsedAttributedObservations() []testParsedAttributedObservation LinkFeeValid: false, NativeFee: big.NewInt(3), NativeFeeValid: false, + + MarketStatus: 2, + MarketStatusValid: false, }, testParsedAttributedObservation{ Timestamp: 3, @@ -206,6 +234,9 @@ func NewInvalidParsedAttributedObservations() []testParsedAttributedObservation LinkFeeValid: true, NativeFee: big.NewInt(4), NativeFeeValid: true, + + MarketStatus: 3, + MarketStatusValid: false, }, } } diff --git a/mercury/cmd/chainlink-mercury/plugin.go b/mercury/cmd/chainlink-mercury/plugin.go index 432ce5f..d03d3df 100644 --- a/mercury/cmd/chainlink-mercury/plugin.go +++ b/mercury/cmd/chainlink-mercury/plugin.go @@ -12,10 +12,12 @@ import ( v1 "github.com/smartcontractkit/chainlink-common/pkg/types/mercury/v1" v2 "github.com/smartcontractkit/chainlink-common/pkg/types/mercury/v2" v3 "github.com/smartcontractkit/chainlink-common/pkg/types/mercury/v3" + v4 "github.com/smartcontractkit/chainlink-common/pkg/types/mercury/v4" ds_v1 "github.com/smartcontractkit/chainlink-data-streams/mercury/v1" ds_v2 "github.com/smartcontractkit/chainlink-data-streams/mercury/v2" ds_v3 "github.com/smartcontractkit/chainlink-data-streams/mercury/v3" + ds_v4 "github.com/smartcontractkit/chainlink-data-streams/mercury/v4" ) type Plugin struct { @@ -69,6 +71,20 @@ func (p *Plugin) NewMercuryV3Factory(ctx context.Context, provider types.Mercury return s, nil } +func (p *Plugin) NewMercuryV4Factory(ctx context.Context, provider types.MercuryProvider, dataSource v4.DataSource) (types.MercuryPluginFactory, error) { + var ctxVals loop.ContextValues + ctxVals.SetValues(ctx) + lggr := logger.With(p.Logger, ctxVals.Args()...) + + factory := ds_v4.NewFactory(dataSource, lggr, provider.OnchainConfigCodec(), provider.ReportCodecV4()) + + s := &mercuryPluginFactoryService{lggr: logger.Named(lggr, "MercuryV4PluginFactory"), MercuryPluginFactory: factory} + + p.SubService(s) + + return s, nil +} + type mercuryPluginFactoryService struct { services.StateMachine lggr logger.Logger diff --git a/mercury/v4/aggregate_functions.go b/mercury/v4/aggregate_functions.go new file mode 100644 index 0000000..617faa8 --- /dev/null +++ b/mercury/v4/aggregate_functions.go @@ -0,0 +1,39 @@ +package v4 + +import "fmt" + +type PAOMarketStatus interface { + GetMarketStatus() (uint32, bool) +} + +// GetConsensusMarketStatus gets the most common status, provided that it is at least F+1. +func GetConsensusMarketStatus(paos []PAOMarketStatus, f int) (uint32, error) { + marketStatusCounts := make(map[uint32]int) + for _, pao := range paos { + marketStatus, valid := pao.GetMarketStatus() + if valid { + marketStatusCounts[marketStatus]++ + } + } + + var mostCommonMarketStatus uint32 + var mostCommonCount int + for marketStatus, count := range marketStatusCounts { + if count > mostCommonCount { + mostCommonMarketStatus = marketStatus + mostCommonCount = count + } else if count == mostCommonCount { + // For stability, always prefer the smaller enum value in case of ties. + // In practice this will prefer CLOSED over OPEN. + if marketStatus < mostCommonMarketStatus { + mostCommonMarketStatus = marketStatus + } + } + } + + if mostCommonCount < f+1 { + return 0, fmt.Errorf("market status has fewer than f+1 observations (status %d got %d/%d)", mostCommonMarketStatus, mostCommonCount, len(paos)) + } + + return mostCommonMarketStatus, nil +} diff --git a/mercury/v4/aggregate_functions_test.go b/mercury/v4/aggregate_functions_test.go new file mode 100644 index 0000000..36170cb --- /dev/null +++ b/mercury/v4/aggregate_functions_test.go @@ -0,0 +1,294 @@ +package v4 + +import ( + "math/big" + "testing" + + "github.com/smartcontractkit/libocr/commontypes" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type testParsedAttributedObservation struct { + Timestamp uint32 + BenchmarkPrice *big.Int + BenchmarkPriceValid bool + Bid *big.Int + BidValid bool + Ask *big.Int + AskValid bool + MaxFinalizedTimestamp int64 + MaxFinalizedTimestampValid bool + LinkFee *big.Int + LinkFeeValid bool + NativeFee *big.Int + NativeFeeValid bool + MarketStatus uint32 + MarketStatusValid bool +} + +func (t testParsedAttributedObservation) GetObserver() commontypes.OracleID { return 0 } +func (t testParsedAttributedObservation) GetTimestamp() uint32 { return t.Timestamp } +func (t testParsedAttributedObservation) GetBenchmarkPrice() (*big.Int, bool) { + return t.BenchmarkPrice, t.BenchmarkPriceValid +} +func (t testParsedAttributedObservation) GetBid() (*big.Int, bool) { + return t.Bid, t.BidValid +} +func (t testParsedAttributedObservation) GetAsk() (*big.Int, bool) { + return t.Ask, t.AskValid +} +func (t testParsedAttributedObservation) GetMaxFinalizedTimestamp() (int64, bool) { + return t.MaxFinalizedTimestamp, t.MaxFinalizedTimestampValid +} +func (t testParsedAttributedObservation) GetLinkFee() (*big.Int, bool) { + return t.LinkFee, t.LinkFeeValid +} +func (t testParsedAttributedObservation) GetNativeFee() (*big.Int, bool) { + return t.NativeFee, t.NativeFeeValid +} +func (t testParsedAttributedObservation) GetMarketStatus() (uint32, bool) { + return t.MarketStatus, t.MarketStatusValid +} + +func convertTestPAOsToPAOs(testPAOs []testParsedAttributedObservation) []PAO { + var paos []PAO + for _, testPAO := range testPAOs { + paos = append(paos, testPAO) + } + return paos +} + +func newValidParsedAttributedObservations() []testParsedAttributedObservation { + return []testParsedAttributedObservation{ + testParsedAttributedObservation{ + Timestamp: 1689648456, + + BenchmarkPrice: big.NewInt(123), + BenchmarkPriceValid: true, + Bid: big.NewInt(120), + BidValid: true, + Ask: big.NewInt(130), + AskValid: true, + + MaxFinalizedTimestamp: 1679448456, + MaxFinalizedTimestampValid: true, + + LinkFee: big.NewInt(1), + LinkFeeValid: true, + NativeFee: big.NewInt(1), + NativeFeeValid: true, + + MarketStatus: 1, + MarketStatusValid: true, + }, + testParsedAttributedObservation{ + Timestamp: 1689648456, + + BenchmarkPrice: big.NewInt(456), + BenchmarkPriceValid: true, + Bid: big.NewInt(450), + BidValid: true, + Ask: big.NewInt(460), + AskValid: true, + + MaxFinalizedTimestamp: 1679448456, + MaxFinalizedTimestampValid: true, + + LinkFee: big.NewInt(2), + LinkFeeValid: true, + NativeFee: big.NewInt(2), + NativeFeeValid: true, + + MarketStatus: 1, + MarketStatusValid: true, + }, + testParsedAttributedObservation{ + Timestamp: 1689648789, + + BenchmarkPrice: big.NewInt(789), + BenchmarkPriceValid: true, + Bid: big.NewInt(780), + BidValid: true, + Ask: big.NewInt(800), + AskValid: true, + + MaxFinalizedTimestamp: 1679448456, + MaxFinalizedTimestampValid: true, + + LinkFee: big.NewInt(3), + LinkFeeValid: true, + NativeFee: big.NewInt(3), + NativeFeeValid: true, + + MarketStatus: 2, + MarketStatusValid: true, + }, + testParsedAttributedObservation{ + Timestamp: 1689648789, + + BenchmarkPrice: big.NewInt(456), + BenchmarkPriceValid: true, + Bid: big.NewInt(450), + BidValid: true, + Ask: big.NewInt(460), + AskValid: true, + + MaxFinalizedTimestamp: 1679513477, + MaxFinalizedTimestampValid: true, + + LinkFee: big.NewInt(4), + LinkFeeValid: true, + NativeFee: big.NewInt(4), + NativeFeeValid: true, + + MarketStatus: 3, + MarketStatusValid: true, + }, + } +} + +func NewValidParsedAttributedObservations(paos ...testParsedAttributedObservation) []testParsedAttributedObservation { + if len(paos) == 0 { + paos = newValidParsedAttributedObservations() + } + return []testParsedAttributedObservation{ + paos[0], + paos[1], + paos[2], + paos[3], + } +} + +func NewInvalidParsedAttributedObservations() []testParsedAttributedObservation { + return []testParsedAttributedObservation{ + testParsedAttributedObservation{ + Timestamp: 1, + + BenchmarkPrice: big.NewInt(123), + BenchmarkPriceValid: false, + Bid: big.NewInt(120), + BidValid: false, + Ask: big.NewInt(130), + AskValid: false, + + MaxFinalizedTimestamp: 1679648456, + MaxFinalizedTimestampValid: false, + + LinkFee: big.NewInt(1), + LinkFeeValid: false, + NativeFee: big.NewInt(1), + NativeFeeValid: false, + + MarketStatus: 1, + MarketStatusValid: false, + }, + testParsedAttributedObservation{ + Timestamp: 2, + + BenchmarkPrice: big.NewInt(456), + BenchmarkPriceValid: false, + Bid: big.NewInt(450), + BidValid: false, + Ask: big.NewInt(460), + AskValid: false, + + MaxFinalizedTimestamp: 1679648456, + MaxFinalizedTimestampValid: false, + + LinkFee: big.NewInt(2), + LinkFeeValid: false, + NativeFee: big.NewInt(2), + NativeFeeValid: false, + + MarketStatus: 1, + MarketStatusValid: false, + }, + testParsedAttributedObservation{ + Timestamp: 2, + + BenchmarkPrice: big.NewInt(789), + BenchmarkPriceValid: false, + Bid: big.NewInt(780), + BidValid: false, + Ask: big.NewInt(800), + AskValid: false, + + MaxFinalizedTimestamp: 1679648456, + MaxFinalizedTimestampValid: false, + + LinkFee: big.NewInt(3), + LinkFeeValid: false, + NativeFee: big.NewInt(3), + NativeFeeValid: false, + + MarketStatus: 2, + MarketStatusValid: false, + }, + testParsedAttributedObservation{ + Timestamp: 3, + + BenchmarkPrice: big.NewInt(456), + BenchmarkPriceValid: true, + Bid: big.NewInt(450), + BidValid: true, + Ask: big.NewInt(460), + AskValid: true, + + MaxFinalizedTimestamp: 1679513477, + MaxFinalizedTimestampValid: true, + + LinkFee: big.NewInt(4), + LinkFeeValid: true, + NativeFee: big.NewInt(4), + NativeFeeValid: true, + + MarketStatus: 3, + MarketStatusValid: false, + }, + } +} + +func Test_AggregateFunctions(t *testing.T) { + f := 1 + validPaos := NewValidParsedAttributedObservations() + invalidPaos := NewInvalidParsedAttributedObservations() + + t.Run("GetConsensusMarketStatus", func(t *testing.T) { + t.Run("gets consensus on market status when valid", func(t *testing.T) { + marketStatus, err := GetConsensusMarketStatus(convertMarketStatus(convertTestPAOsToPAOs(validPaos)), f) + require.NoError(t, err) + assert.Equal(t, uint32(1), marketStatus) + }) + t.Run("treats zero values as valid", func(t *testing.T) { + paos := NewValidParsedAttributedObservations() + for i := range paos { + paos[i].MarketStatus = 0 + } + marketStatus, err := GetConsensusMarketStatus(convertMarketStatus(convertTestPAOsToPAOs(paos)), f) + require.NoError(t, err) + assert.Equal(t, uint32(0), marketStatus) + }) + t.Run("is stable during ties", func(t *testing.T) { + paos := NewValidParsedAttributedObservations() + for i := range paos { + paos[i].MarketStatus = uint32(i % 2) + } + marketStatus, err := GetConsensusMarketStatus(convertMarketStatus(convertTestPAOsToPAOs(paos)), f) + require.NoError(t, err) + assert.Equal(t, uint32(0), marketStatus) + }) + t.Run("fails when the mode is less than f+1", func(t *testing.T) { + paos := NewValidParsedAttributedObservations() + for i := range paos { + paos[i].MarketStatus = uint32(i) + } + _, err := GetConsensusMarketStatus(convertMarketStatus(convertTestPAOsToPAOs(paos)), f) + assert.EqualError(t, err, "market status has fewer than f+1 observations (status 0 got 1/4)") + }) + t.Run("fails when all observations are invalid", func(t *testing.T) { + _, err := GetConsensusMarketStatus(convertMarketStatus(convertTestPAOsToPAOs(invalidPaos)), f) + assert.EqualError(t, err, "market status has fewer than f+1 observations (status 0 got 0/4)") + }) + }) +} diff --git a/mercury/v4/mercury.go b/mercury/v4/mercury.go new file mode 100644 index 0000000..1326f09 --- /dev/null +++ b/mercury/v4/mercury.go @@ -0,0 +1,507 @@ +package v4 + +import ( + "context" + "errors" + "fmt" + "math" + "math/big" + "time" + + "github.com/smartcontractkit/chainlink-common/pkg/logger" + mercurytypes "github.com/smartcontractkit/chainlink-common/pkg/types/mercury" + v4 "github.com/smartcontractkit/chainlink-common/pkg/types/mercury/v4" + "github.com/smartcontractkit/libocr/offchainreporting2plus/ocr3types" + "github.com/smartcontractkit/libocr/offchainreporting2plus/types" + "google.golang.org/protobuf/proto" + + "github.com/smartcontractkit/chainlink-data-streams/mercury" +) + +//go:generate protoc -I=. --go_out=. mercury_observation_v4.proto + +// DataSource implementations must be thread-safe. Observe may be called by many +// different threads concurrently. +type DataSource interface { + // Observe queries the data source. Returns a value or an error. Once the + // context is expires, Observe may still do cheap computations and return a + // result, but should return as quickly as possible. + // + // More details: In the current implementation, the context passed to + // Observe will time out after MaxDurationObservation. However, Observe + // should *not* make any assumptions about context timeout behavior. Once + // the context times out, Observe should prioritize returning as quickly as + // possible, but may still perform fast computations to return a result + // rather than error. For example, if Observe medianizes a number of data + // sources, some of which already returned a result to Observe prior to the + // context's expiry, Observe might still compute their median, and return it + // instead of an error. + // + // Important: Observe should not perform any potentially time-consuming + // actions like database access, once the context passed has expired. + Observe(ctx context.Context, repts types.ReportTimestamp, fetchMaxFinalizedTimestamp bool) (v4.Observation, error) +} + +var _ ocr3types.MercuryPluginFactory = Factory{} + +const maxObservationLength = 32 + // feedID + 4 + // timestamp + mercury.ByteWidthInt192 + // benchmarkPrice + mercury.ByteWidthInt192 + // bid + mercury.ByteWidthInt192 + // ask + 4 + // validFromTimestamp + mercury.ByteWidthInt192 + // linkFee + mercury.ByteWidthInt192 + // nativeFee + 4 + // marketStatus (enum is int32) + 18 /* overapprox. of protobuf overhead */ + +type Factory struct { + dataSource DataSource + logger logger.Logger + onchainConfigCodec mercurytypes.OnchainConfigCodec + reportCodec v4.ReportCodec +} + +func NewFactory(ds DataSource, lggr logger.Logger, occ mercurytypes.OnchainConfigCodec, rc v4.ReportCodec) Factory { + return Factory{ds, lggr, occ, rc} +} + +func (fac Factory) NewMercuryPlugin(configuration ocr3types.MercuryPluginConfig) (ocr3types.MercuryPlugin, ocr3types.MercuryPluginInfo, error) { + offchainConfig, err := mercury.DecodeOffchainConfig(configuration.OffchainConfig) + if err != nil { + return nil, ocr3types.MercuryPluginInfo{}, err + } + + onchainConfig, err := fac.onchainConfigCodec.Decode(configuration.OnchainConfig) + if err != nil { + return nil, ocr3types.MercuryPluginInfo{}, err + } + + maxReportLength, err := fac.reportCodec.MaxReportLength(configuration.N) + if err != nil { + return nil, ocr3types.MercuryPluginInfo{}, err + } + + r := &reportingPlugin{ + offchainConfig, + onchainConfig, + fac.dataSource, + fac.logger, + fac.reportCodec, + configuration.ConfigDigest, + configuration.F, + mercury.EpochRound{}, + new(big.Int), + maxReportLength, + } + + return r, ocr3types.MercuryPluginInfo{ + Name: "Mercury", + Limits: ocr3types.MercuryPluginLimits{ + MaxObservationLength: maxObservationLength, + MaxReportLength: maxReportLength, + }, + }, nil +} + +var _ ocr3types.MercuryPlugin = (*reportingPlugin)(nil) + +type reportingPlugin struct { + offchainConfig mercury.OffchainConfig + onchainConfig mercurytypes.OnchainConfig + dataSource DataSource + logger logger.Logger + reportCodec v4.ReportCodec + + configDigest types.ConfigDigest + f int + latestAcceptedEpochRound mercury.EpochRound + latestAcceptedMedian *big.Int + maxReportLength int +} + +var MissingPrice = big.NewInt(-1) + +func (rp *reportingPlugin) Observation(ctx context.Context, repts types.ReportTimestamp, previousReport types.Report) (types.Observation, error) { + obs, err := rp.dataSource.Observe(ctx, repts, previousReport == nil) + if err != nil { + return nil, fmt.Errorf("DataSource.Observe returned an error: %s", err) + } + + observationTimestamp := time.Now() + if observationTimestamp.Unix() > math.MaxUint32 { + return nil, fmt.Errorf("current unix epoch %d exceeds max uint32", observationTimestamp.Unix()) + } + p := MercuryObservationProto{Timestamp: uint32(observationTimestamp.Unix())} + var obsErrors []error + + var bpErr, bidErr, askErr error + if obs.BenchmarkPrice.Err != nil { + bpErr = fmt.Errorf("failed to observe BenchmarkPrice: %w", obs.BenchmarkPrice.Err) + obsErrors = append(obsErrors, bpErr) + } else if benchmarkPrice, err := mercury.EncodeValueInt192(obs.BenchmarkPrice.Val); err != nil { + bpErr = fmt.Errorf("failed to encode BenchmarkPrice; val=%s: %w", obs.BenchmarkPrice.Val, err) + obsErrors = append(obsErrors, bpErr) + } else { + p.BenchmarkPrice = benchmarkPrice + } + + if obs.Bid.Err != nil { + bidErr = fmt.Errorf("failed to observe Bid: %w", obs.Bid.Err) + obsErrors = append(obsErrors, bidErr) + } else if bid, err := mercury.EncodeValueInt192(obs.Bid.Val); err != nil { + bidErr = fmt.Errorf("failed to encode Bid; val=%s: %w", obs.Bid.Val, err) + obsErrors = append(obsErrors, bidErr) + } else { + p.Bid = bid + } + + if obs.Ask.Err != nil { + askErr = fmt.Errorf("failed to observe Ask: %w", obs.Ask.Err) + obsErrors = append(obsErrors, askErr) + } else if ask, err := mercury.EncodeValueInt192(obs.Ask.Val); err != nil { + askErr = fmt.Errorf("failed to encode Ask; val=%s: %w", obs.Ask.Val, err) + obsErrors = append(obsErrors, askErr) + } else { + p.Ask = ask + } + + if bpErr == nil && bidErr == nil && askErr == nil { + if err := validatePrices(obs.Bid.Val, obs.BenchmarkPrice.Val, obs.Ask.Val); err != nil { + rp.logger.Errorw("Cannot generate price observation: invalid bid/mid/ask", "err", err) + p.PricesValid = false + } else { + p.PricesValid = true + } + } + + var maxFinalizedTimestampErr error + if obs.MaxFinalizedTimestamp.Err != nil { + maxFinalizedTimestampErr = fmt.Errorf("failed to observe MaxFinalizedTimestamp: %w", obs.MaxFinalizedTimestamp.Err) + obsErrors = append(obsErrors, maxFinalizedTimestampErr) + } else { + p.MaxFinalizedTimestamp = obs.MaxFinalizedTimestamp.Val + p.MaxFinalizedTimestampValid = true + } + + var linkErr error + if obs.LinkPrice.Err != nil { + linkErr = fmt.Errorf("failed to observe LINK price: %w", obs.LinkPrice.Err) + obsErrors = append(obsErrors, linkErr) + } else if obs.LinkPrice.Val.Cmp(MissingPrice) <= 0 { + p.LinkFee = mercury.MaxInt192Enc + } else { + linkFee := mercury.CalculateFee(obs.LinkPrice.Val, rp.offchainConfig.BaseUSDFee) + if linkFeeEncoded, err := mercury.EncodeValueInt192(linkFee); err != nil { + linkErr = fmt.Errorf("failed to encode LINK fee; val=%s: %w", linkFee, err) + obsErrors = append(obsErrors, linkErr) + } else { + p.LinkFee = linkFeeEncoded + } + } + + if linkErr == nil { + p.LinkFeeValid = true + } + + var nativeErr error + if obs.NativePrice.Err != nil { + nativeErr = fmt.Errorf("failed to observe native price: %w", obs.NativePrice.Err) + obsErrors = append(obsErrors, nativeErr) + } else if obs.NativePrice.Val.Cmp(MissingPrice) <= 0 { + p.NativeFee = mercury.MaxInt192Enc + } else { + nativeFee := mercury.CalculateFee(obs.NativePrice.Val, rp.offchainConfig.BaseUSDFee) + if nativeFeeEncoded, err := mercury.EncodeValueInt192(nativeFee); err != nil { + nativeErr = fmt.Errorf("failed to encode native fee; val=%s: %w", nativeFee, err) + obsErrors = append(obsErrors, nativeErr) + } else { + p.NativeFee = nativeFeeEncoded + } + } + + if nativeErr == nil { + p.NativeFeeValid = true + } + + var marketStatusErr error + if obs.MarketStatus.Err != nil { + marketStatusErr = fmt.Errorf("failed to observe market status: %w", obs.MarketStatus.Err) + obsErrors = append(obsErrors, marketStatusErr) + } else { + p.MarketStatus = obs.MarketStatus.Val + p.MarketStatusValid = true + } + + if len(obsErrors) > 0 { + rp.logger.Warnw(fmt.Sprintf("Observe failed %d/7 observations", len(obsErrors)), "err", errors.Join(obsErrors...)) + } + + return proto.Marshal(&p) +} + +func validatePrices(bid, benchmarkPrice, ask *big.Int) error { + if bid.Cmp(benchmarkPrice) > 0 || benchmarkPrice.Cmp(ask) > 0 { + return fmt.Errorf("invariant violated: expected bid<=mid<=ask, got bid: %s, mid: %s, ask: %s", bid, benchmarkPrice, ask) + } + return nil +} + +func parseAttributedObservation(ao types.AttributedObservation) (PAO, error) { + var pao parsedAttributedObservation + var obs MercuryObservationProto + if err := proto.Unmarshal(ao.Observation, &obs); err != nil { + return parsedAttributedObservation{}, fmt.Errorf("attributed observation cannot be unmarshaled: %s", err) + } + + pao.Timestamp = obs.Timestamp + pao.Observer = ao.Observer + + if obs.PricesValid { + var err error + pao.BenchmarkPrice, err = mercury.DecodeValueInt192(obs.BenchmarkPrice) + if err != nil { + return parsedAttributedObservation{}, fmt.Errorf("benchmarkPrice cannot be converted to big.Int: %s", err) + } + pao.Bid, err = mercury.DecodeValueInt192(obs.Bid) + if err != nil { + return parsedAttributedObservation{}, fmt.Errorf("bid cannot be converted to big.Int: %s", err) + } + pao.Ask, err = mercury.DecodeValueInt192(obs.Ask) + if err != nil { + return parsedAttributedObservation{}, fmt.Errorf("ask cannot be converted to big.Int: %s", err) + } + if err := validatePrices(pao.Bid, pao.BenchmarkPrice, pao.Ask); err != nil { + // NOTE: since nodes themselves are not supposed to set + // PricesValid=true if this invariant is violated, this indicates a + // faulty/misbehaving node and the entire observation should be + // ignored + return parsedAttributedObservation{}, fmt.Errorf("observation claimed to be valid, but contains invalid prices: %w", err) + } + pao.PricesValid = true + } + + if obs.MaxFinalizedTimestampValid { + pao.MaxFinalizedTimestamp = obs.MaxFinalizedTimestamp + pao.MaxFinalizedTimestampValid = true + } + + if obs.LinkFeeValid { + var err error + pao.LinkFee, err = mercury.DecodeValueInt192(obs.LinkFee) + if err != nil { + return parsedAttributedObservation{}, fmt.Errorf("link price cannot be converted to big.Int: %s", err) + } + pao.LinkFeeValid = true + } + if obs.NativeFeeValid { + var err error + pao.NativeFee, err = mercury.DecodeValueInt192(obs.NativeFee) + if err != nil { + return parsedAttributedObservation{}, fmt.Errorf("native price cannot be converted to big.Int: %s", err) + } + pao.NativeFeeValid = true + } + + if obs.MarketStatusValid { + pao.MarketStatus = obs.MarketStatus + pao.MarketStatusValid = true + } + + return pao, nil +} + +func parseAttributedObservations(lggr logger.Logger, aos []types.AttributedObservation) []PAO { + paos := make([]PAO, 0, len(aos)) + for i, ao := range aos { + pao, err := parseAttributedObservation(ao) + if err != nil { + lggr.Warnw("parseAttributedObservations: dropping invalid observation", + "observer", ao.Observer, + "error", err, + "i", i, + ) + continue + } + paos = append(paos, pao) + } + return paos +} + +func (rp *reportingPlugin) Report(repts types.ReportTimestamp, previousReport types.Report, aos []types.AttributedObservation) (shouldReport bool, report types.Report, err error) { + paos := parseAttributedObservations(rp.logger, aos) + + if len(paos) == 0 { + return false, nil, errors.New("got zero valid attributed observations") + } + + // By assumption, we have at most f malicious oracles, so there should be at least f+1 valid paos + if !(rp.f+1 <= len(paos)) { + return false, nil, fmt.Errorf("only received %v valid attributed observations, but need at least f+1 (%v)", len(paos), rp.f+1) + } + + rf, err := rp.buildReportFields(previousReport, paos) + if err != nil { + rp.logger.Errorw("failed to build report fields", "paos", paos, "f", rp.f, "reportFields", rf, "repts", repts, "err", err) + return false, nil, err + } + + if rf.Timestamp < rf.ValidFromTimestamp { + rp.logger.Debugw("shouldReport: no (overlap)", "observationTimestamp", rf.Timestamp, "validFromTimestamp", rf.ValidFromTimestamp, "repts", repts) + return false, nil, nil + } + + if err = rp.validateReport(rf); err != nil { + rp.logger.Errorw("shouldReport: no (validation error)", "reportFields", rf, "err", err, "repts", repts, "paos", paos) + return false, nil, err + } + rp.logger.Debugw("shouldReport: yes", "repts", repts) + + report, err = rp.reportCodec.BuildReport(rf) + if err != nil { + rp.logger.Debugw("failed to BuildReport", "paos", paos, "f", rp.f, "reportFields", rf, "repts", repts) + return false, nil, err + } + + if !(len(report) <= rp.maxReportLength) { + return false, nil, fmt.Errorf("report with len %d violates MaxReportLength limit set by ReportCodec (%d)", len(report), rp.maxReportLength) + } else if len(report) == 0 { + return false, nil, errors.New("report may not have zero length (invariant violation)") + } + + return true, report, nil +} + +func (rp *reportingPlugin) buildReportFields(previousReport types.Report, paos []PAO) (rf v4.ReportFields, merr error) { + mPaos := convert(paos) + rf.Timestamp = mercury.GetConsensusTimestamp(mPaos) + + var err error + if previousReport != nil { + var maxFinalizedTimestamp uint32 + maxFinalizedTimestamp, err = rp.reportCodec.ObservationTimestampFromReport(previousReport) + merr = errors.Join(merr, err) + rf.ValidFromTimestamp = maxFinalizedTimestamp + 1 + } else { + var maxFinalizedTimestamp int64 + maxFinalizedTimestamp, err = mercury.GetConsensusMaxFinalizedTimestamp(convertMaxFinalizedTimestamp(paos), rp.f) + if err != nil { + merr = errors.Join(merr, err) + } else if maxFinalizedTimestamp < 0 { + // no previous observation timestamp available, e.g. in case of new + // feed; use current timestamp as start of range + rf.ValidFromTimestamp = rf.Timestamp + } else if maxFinalizedTimestamp+1 > math.MaxUint32 { + merr = errors.Join(err, fmt.Errorf("maxFinalizedTimestamp is too large, got: %d", maxFinalizedTimestamp)) + } else { + rf.ValidFromTimestamp = uint32(maxFinalizedTimestamp + 1) + } + } + + rf.BenchmarkPrice, err = mercury.GetConsensusBenchmarkPrice(mPaos, rp.f) + if err != nil { + merr = errors.Join(merr, fmt.Errorf("GetConsensusBenchmarkPrice failed: %w", err)) + } + + rf.Bid, err = mercury.GetConsensusBid(convertBid(paos), rp.f) + if err != nil { + merr = errors.Join(merr, fmt.Errorf("GetConsensusBid failed: %w", err)) + } + + rf.Ask, err = mercury.GetConsensusAsk(convertAsk(paos), rp.f) + if err != nil { + merr = errors.Join(merr, fmt.Errorf("GetConsensusAsk failed: %w", err)) + } + + rf.LinkFee, err = mercury.GetConsensusLinkFee(convertLinkFee(paos), rp.f) + if err != nil { + // It is better to generate a report that will validate for free, + // rather than no report at all, if we cannot come to consensus on a + // valid fee. + rp.logger.Errorw("Cannot come to consensus on LINK fee, falling back to 0", "err", err, "paos", paos) + rf.LinkFee = big.NewInt(0) + } + + rf.NativeFee, err = mercury.GetConsensusNativeFee(convertNativeFee(paos), rp.f) + if err != nil { + // It is better to generate a report that will validate for free, + // rather than no report at all, if we cannot come to consensus on a + // valid fee. + rp.logger.Errorw("Cannot come to consensus on Native fee, falling back to 0", "err", err, "paos", paos) + rf.NativeFee = big.NewInt(0) + } + + rf.MarketStatus, err = GetConsensusMarketStatus(convertMarketStatus(paos), rp.f) + if err != nil { + merr = errors.Join(merr, fmt.Errorf("GetConsensusMarketStatus failed: %w", err)) + } + + if int64(rf.Timestamp)+int64(rp.offchainConfig.ExpirationWindow) > math.MaxUint32 { + merr = errors.Join(merr, fmt.Errorf("timestamp %d + expiration window %d overflows uint32", rf.Timestamp, rp.offchainConfig.ExpirationWindow)) + } else { + rf.ExpiresAt = rf.Timestamp + rp.offchainConfig.ExpirationWindow + } + + return rf, merr +} + +func (rp *reportingPlugin) validateReport(rf v4.ReportFields) error { + return errors.Join( + mercury.ValidateBetween("median benchmark price", rf.BenchmarkPrice, rp.onchainConfig.Min, rp.onchainConfig.Max), + mercury.ValidateBetween("median bid", rf.Bid, rp.onchainConfig.Min, rp.onchainConfig.Max), + mercury.ValidateBetween("median ask", rf.Ask, rp.onchainConfig.Min, rp.onchainConfig.Max), + mercury.ValidateFee("median link fee", rf.LinkFee), + mercury.ValidateFee("median native fee", rf.NativeFee), + mercury.ValidateValidFromTimestamp(rf.Timestamp, rf.ValidFromTimestamp), + mercury.ValidateExpiresAt(rf.Timestamp, rf.ExpiresAt), + ) +} + +func (rp *reportingPlugin) Close() error { + return nil +} + +// convert funcs are necessary because go is not smart enough to cast +// []interface1 to []interface2 even if interface1 is a superset of interface2 +func convert(pao []PAO) (ret []mercury.PAO) { + for _, v := range pao { + ret = append(ret, v) + } + return ret +} +func convertMaxFinalizedTimestamp(pao []PAO) (ret []mercury.PAOMaxFinalizedTimestamp) { + for _, v := range pao { + ret = append(ret, v) + } + return ret +} +func convertBid(pao []PAO) (ret []mercury.PAOBid) { + for _, v := range pao { + ret = append(ret, v) + } + return ret +} +func convertAsk(pao []PAO) (ret []mercury.PAOAsk) { + for _, v := range pao { + ret = append(ret, v) + } + return ret +} +func convertLinkFee(pao []PAO) (ret []mercury.PAOLinkFee) { + for _, v := range pao { + ret = append(ret, v) + } + return ret +} +func convertNativeFee(pao []PAO) (ret []mercury.PAONativeFee) { + for _, v := range pao { + ret = append(ret, v) + } + return ret +} +func convertMarketStatus(pao []PAO) (ret []PAOMarketStatus) { + for _, v := range pao { + ret = append(ret, v) + } + return ret +} diff --git a/mercury/v4/mercury_observation_v4.pb.go b/mercury/v4/mercury_observation_v4.pb.go new file mode 100644 index 0000000..11d429a --- /dev/null +++ b/mercury/v4/mercury_observation_v4.pb.go @@ -0,0 +1,267 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.31.0 +// protoc v4.23.2 +// source: mercury_observation_v4.proto + +package v4 + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type MercuryObservationProto struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Timestamp uint32 `protobuf:"varint,1,opt,name=timestamp,proto3" json:"timestamp,omitempty"` + BenchmarkPrice []byte `protobuf:"bytes,2,opt,name=benchmarkPrice,proto3" json:"benchmarkPrice,omitempty"` + Bid []byte `protobuf:"bytes,3,opt,name=bid,proto3" json:"bid,omitempty"` + Ask []byte `protobuf:"bytes,4,opt,name=ask,proto3" json:"ask,omitempty"` + PricesValid bool `protobuf:"varint,5,opt,name=pricesValid,proto3" json:"pricesValid,omitempty"` + MaxFinalizedTimestamp int64 `protobuf:"varint,6,opt,name=maxFinalizedTimestamp,proto3" json:"maxFinalizedTimestamp,omitempty"` + MaxFinalizedTimestampValid bool `protobuf:"varint,7,opt,name=maxFinalizedTimestampValid,proto3" json:"maxFinalizedTimestampValid,omitempty"` + LinkFee []byte `protobuf:"bytes,8,opt,name=linkFee,proto3" json:"linkFee,omitempty"` + LinkFeeValid bool `protobuf:"varint,9,opt,name=linkFeeValid,proto3" json:"linkFeeValid,omitempty"` + NativeFee []byte `protobuf:"bytes,10,opt,name=nativeFee,proto3" json:"nativeFee,omitempty"` + NativeFeeValid bool `protobuf:"varint,11,opt,name=nativeFeeValid,proto3" json:"nativeFeeValid,omitempty"` + MarketStatus uint32 `protobuf:"varint,12,opt,name=marketStatus,proto3" json:"marketStatus,omitempty"` + MarketStatusValid bool `protobuf:"varint,13,opt,name=marketStatusValid,proto3" json:"marketStatusValid,omitempty"` +} + +func (x *MercuryObservationProto) Reset() { + *x = MercuryObservationProto{} + if protoimpl.UnsafeEnabled { + mi := &file_mercury_observation_v4_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *MercuryObservationProto) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*MercuryObservationProto) ProtoMessage() {} + +func (x *MercuryObservationProto) ProtoReflect() protoreflect.Message { + mi := &file_mercury_observation_v4_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use MercuryObservationProto.ProtoReflect.Descriptor instead. +func (*MercuryObservationProto) Descriptor() ([]byte, []int) { + return file_mercury_observation_v4_proto_rawDescGZIP(), []int{0} +} + +func (x *MercuryObservationProto) GetTimestamp() uint32 { + if x != nil { + return x.Timestamp + } + return 0 +} + +func (x *MercuryObservationProto) GetBenchmarkPrice() []byte { + if x != nil { + return x.BenchmarkPrice + } + return nil +} + +func (x *MercuryObservationProto) GetBid() []byte { + if x != nil { + return x.Bid + } + return nil +} + +func (x *MercuryObservationProto) GetAsk() []byte { + if x != nil { + return x.Ask + } + return nil +} + +func (x *MercuryObservationProto) GetPricesValid() bool { + if x != nil { + return x.PricesValid + } + return false +} + +func (x *MercuryObservationProto) GetMaxFinalizedTimestamp() int64 { + if x != nil { + return x.MaxFinalizedTimestamp + } + return 0 +} + +func (x *MercuryObservationProto) GetMaxFinalizedTimestampValid() bool { + if x != nil { + return x.MaxFinalizedTimestampValid + } + return false +} + +func (x *MercuryObservationProto) GetLinkFee() []byte { + if x != nil { + return x.LinkFee + } + return nil +} + +func (x *MercuryObservationProto) GetLinkFeeValid() bool { + if x != nil { + return x.LinkFeeValid + } + return false +} + +func (x *MercuryObservationProto) GetNativeFee() []byte { + if x != nil { + return x.NativeFee + } + return nil +} + +func (x *MercuryObservationProto) GetNativeFeeValid() bool { + if x != nil { + return x.NativeFeeValid + } + return false +} + +func (x *MercuryObservationProto) GetMarketStatus() uint32 { + if x != nil { + return x.MarketStatus + } + return 0 +} + +func (x *MercuryObservationProto) GetMarketStatusValid() bool { + if x != nil { + return x.MarketStatusValid + } + return false +} + +var File_mercury_observation_v4_proto protoreflect.FileDescriptor + +var file_mercury_observation_v4_proto_rawDesc = []byte{ + 0x0a, 0x1c, 0x6d, 0x65, 0x72, 0x63, 0x75, 0x72, 0x79, 0x5f, 0x6f, 0x62, 0x73, 0x65, 0x72, 0x76, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x76, 0x34, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x02, + 0x76, 0x34, 0x22, 0xf1, 0x03, 0x0a, 0x17, 0x4d, 0x65, 0x72, 0x63, 0x75, 0x72, 0x79, 0x4f, 0x62, + 0x73, 0x65, 0x72, 0x76, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x1c, + 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0d, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x26, 0x0a, 0x0e, + 0x62, 0x65, 0x6e, 0x63, 0x68, 0x6d, 0x61, 0x72, 0x6b, 0x50, 0x72, 0x69, 0x63, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0e, 0x62, 0x65, 0x6e, 0x63, 0x68, 0x6d, 0x61, 0x72, 0x6b, 0x50, + 0x72, 0x69, 0x63, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x62, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x0c, 0x52, 0x03, 0x62, 0x69, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x61, 0x73, 0x6b, 0x18, 0x04, 0x20, + 0x01, 0x28, 0x0c, 0x52, 0x03, 0x61, 0x73, 0x6b, 0x12, 0x20, 0x0a, 0x0b, 0x70, 0x72, 0x69, 0x63, + 0x65, 0x73, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0b, 0x70, + 0x72, 0x69, 0x63, 0x65, 0x73, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x12, 0x34, 0x0a, 0x15, 0x6d, 0x61, + 0x78, 0x46, 0x69, 0x6e, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, + 0x61, 0x6d, 0x70, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x15, 0x6d, 0x61, 0x78, 0x46, 0x69, + 0x6e, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, + 0x12, 0x3e, 0x0a, 0x1a, 0x6d, 0x61, 0x78, 0x46, 0x69, 0x6e, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x64, + 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x18, 0x07, + 0x20, 0x01, 0x28, 0x08, 0x52, 0x1a, 0x6d, 0x61, 0x78, 0x46, 0x69, 0x6e, 0x61, 0x6c, 0x69, 0x7a, + 0x65, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x56, 0x61, 0x6c, 0x69, 0x64, + 0x12, 0x18, 0x0a, 0x07, 0x6c, 0x69, 0x6e, 0x6b, 0x46, 0x65, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, + 0x0c, 0x52, 0x07, 0x6c, 0x69, 0x6e, 0x6b, 0x46, 0x65, 0x65, 0x12, 0x22, 0x0a, 0x0c, 0x6c, 0x69, + 0x6e, 0x6b, 0x46, 0x65, 0x65, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x18, 0x09, 0x20, 0x01, 0x28, 0x08, + 0x52, 0x0c, 0x6c, 0x69, 0x6e, 0x6b, 0x46, 0x65, 0x65, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x12, 0x1c, + 0x0a, 0x09, 0x6e, 0x61, 0x74, 0x69, 0x76, 0x65, 0x46, 0x65, 0x65, 0x18, 0x0a, 0x20, 0x01, 0x28, + 0x0c, 0x52, 0x09, 0x6e, 0x61, 0x74, 0x69, 0x76, 0x65, 0x46, 0x65, 0x65, 0x12, 0x26, 0x0a, 0x0e, + 0x6e, 0x61, 0x74, 0x69, 0x76, 0x65, 0x46, 0x65, 0x65, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x18, 0x0b, + 0x20, 0x01, 0x28, 0x08, 0x52, 0x0e, 0x6e, 0x61, 0x74, 0x69, 0x76, 0x65, 0x46, 0x65, 0x65, 0x56, + 0x61, 0x6c, 0x69, 0x64, 0x12, 0x22, 0x0a, 0x0c, 0x6d, 0x61, 0x72, 0x6b, 0x65, 0x74, 0x53, 0x74, + 0x61, 0x74, 0x75, 0x73, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0c, 0x6d, 0x61, 0x72, 0x6b, + 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x2c, 0x0a, 0x11, 0x6d, 0x61, 0x72, 0x6b, + 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x18, 0x0d, 0x20, + 0x01, 0x28, 0x08, 0x52, 0x11, 0x6d, 0x61, 0x72, 0x6b, 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x42, 0x06, 0x5a, 0x04, 0x2e, 0x3b, 0x76, 0x34, 0x62, 0x06, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_mercury_observation_v4_proto_rawDescOnce sync.Once + file_mercury_observation_v4_proto_rawDescData = file_mercury_observation_v4_proto_rawDesc +) + +func file_mercury_observation_v4_proto_rawDescGZIP() []byte { + file_mercury_observation_v4_proto_rawDescOnce.Do(func() { + file_mercury_observation_v4_proto_rawDescData = protoimpl.X.CompressGZIP(file_mercury_observation_v4_proto_rawDescData) + }) + return file_mercury_observation_v4_proto_rawDescData +} + +var file_mercury_observation_v4_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_mercury_observation_v4_proto_goTypes = []interface{}{ + (*MercuryObservationProto)(nil), // 0: v4.MercuryObservationProto +} +var file_mercury_observation_v4_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_mercury_observation_v4_proto_init() } +func file_mercury_observation_v4_proto_init() { + if File_mercury_observation_v4_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_mercury_observation_v4_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*MercuryObservationProto); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_mercury_observation_v4_proto_rawDesc, + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_mercury_observation_v4_proto_goTypes, + DependencyIndexes: file_mercury_observation_v4_proto_depIdxs, + MessageInfos: file_mercury_observation_v4_proto_msgTypes, + }.Build() + File_mercury_observation_v4_proto = out.File + file_mercury_observation_v4_proto_rawDesc = nil + file_mercury_observation_v4_proto_goTypes = nil + file_mercury_observation_v4_proto_depIdxs = nil +} diff --git a/mercury/v4/mercury_observation_v4.proto b/mercury/v4/mercury_observation_v4.proto new file mode 100644 index 0000000..74e79c9 --- /dev/null +++ b/mercury/v4/mercury_observation_v4.proto @@ -0,0 +1,24 @@ +syntax="proto3"; + +package v4; +option go_package = ".;v4"; + +message MercuryObservationProto { + uint32 timestamp = 1; + + bytes benchmarkPrice = 2; + bytes bid = 3; + bytes ask = 4; + bool pricesValid = 5; + + int64 maxFinalizedTimestamp = 6; + bool maxFinalizedTimestampValid = 7; + + bytes linkFee = 8; + bool linkFeeValid = 9; + bytes nativeFee = 10; + bool nativeFeeValid = 11; + + uint32 marketStatus = 12; + bool marketStatusValid = 13; +} diff --git a/mercury/v4/mercury_test.go b/mercury/v4/mercury_test.go new file mode 100644 index 0000000..d6b1f4b --- /dev/null +++ b/mercury/v4/mercury_test.go @@ -0,0 +1,821 @@ +package v4 + +import ( + "context" + "errors" + "math" + "math/big" + "math/rand" + "reflect" + "testing" + "time" + + "github.com/shopspring/decimal" + "github.com/smartcontractkit/chainlink-common/pkg/logger" + mercurytypes "github.com/smartcontractkit/chainlink-common/pkg/types/mercury" + v4 "github.com/smartcontractkit/chainlink-common/pkg/types/mercury/v4" + "github.com/smartcontractkit/libocr/commontypes" + "github.com/smartcontractkit/libocr/offchainreporting2plus/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" + + "github.com/smartcontractkit/chainlink-data-streams/mercury" +) + +type testDataSource struct { + Obs v4.Observation +} + +func (ds testDataSource) Observe(ctx context.Context, repts types.ReportTimestamp, fetchMaxFinalizedTimestamp bool) (v4.Observation, error) { + return ds.Obs, nil +} + +type testReportCodec struct { + observationTimestamp uint32 + builtReport types.Report + + builtReportFields *v4.ReportFields + err error +} + +func (rc *testReportCodec) BuildReport(rf v4.ReportFields) (types.Report, error) { + rc.builtReportFields = &rf + + return rc.builtReport, nil +} + +func (rc testReportCodec) MaxReportLength(n int) (int, error) { + return 123, nil +} + +func (rc testReportCodec) ObservationTimestampFromReport(types.Report) (uint32, error) { + return rc.observationTimestamp, rc.err +} + +func newTestReportPlugin(t *testing.T, codec *testReportCodec, ds *testDataSource) *reportingPlugin { + offchainConfig := mercury.OffchainConfig{ + ExpirationWindow: 1, + BaseUSDFee: decimal.NewFromInt32(1), + } + onchainConfig := mercurytypes.OnchainConfig{ + Min: big.NewInt(1), + Max: big.NewInt(1000), + } + maxReportLength, _ := codec.MaxReportLength(4) + return &reportingPlugin{ + offchainConfig: offchainConfig, + onchainConfig: onchainConfig, + dataSource: ds, + logger: logger.Test(t), + reportCodec: codec, + configDigest: types.ConfigDigest{}, + f: 1, + latestAcceptedEpochRound: mercury.EpochRound{}, + latestAcceptedMedian: big.NewInt(0), + maxReportLength: maxReportLength, + } +} + +func newValidProtos() []*MercuryObservationProto { + return []*MercuryObservationProto{ + &MercuryObservationProto{ + Timestamp: 42, + + BenchmarkPrice: mercury.MustEncodeValueInt192(big.NewInt(123)), + Bid: mercury.MustEncodeValueInt192(big.NewInt(120)), + Ask: mercury.MustEncodeValueInt192(big.NewInt(130)), + PricesValid: true, + + MaxFinalizedTimestamp: 40, + MaxFinalizedTimestampValid: true, + + LinkFee: mercury.MustEncodeValueInt192(big.NewInt(1.1e18)), + LinkFeeValid: true, + NativeFee: mercury.MustEncodeValueInt192(big.NewInt(2.1e18)), + NativeFeeValid: true, + + MarketStatus: 1, + MarketStatusValid: true, + }, + &MercuryObservationProto{ + Timestamp: 45, + + BenchmarkPrice: mercury.MustEncodeValueInt192(big.NewInt(234)), + Bid: mercury.MustEncodeValueInt192(big.NewInt(230)), + Ask: mercury.MustEncodeValueInt192(big.NewInt(240)), + PricesValid: true, + + MaxFinalizedTimestamp: 40, + MaxFinalizedTimestampValid: true, + + LinkFee: mercury.MustEncodeValueInt192(big.NewInt(1.2e18)), + LinkFeeValid: true, + NativeFee: mercury.MustEncodeValueInt192(big.NewInt(2.2e18)), + NativeFeeValid: true, + + MarketStatus: 1, + MarketStatusValid: true, + }, + &MercuryObservationProto{ + Timestamp: 47, + + BenchmarkPrice: mercury.MustEncodeValueInt192(big.NewInt(345)), + Bid: mercury.MustEncodeValueInt192(big.NewInt(340)), + Ask: mercury.MustEncodeValueInt192(big.NewInt(350)), + PricesValid: true, + + MaxFinalizedTimestamp: 39, + MaxFinalizedTimestampValid: true, + + LinkFee: mercury.MustEncodeValueInt192(big.NewInt(1.3e18)), + LinkFeeValid: true, + NativeFee: mercury.MustEncodeValueInt192(big.NewInt(2.3e18)), + NativeFeeValid: true, + + MarketStatus: 1, + MarketStatusValid: true, + }, + &MercuryObservationProto{ + Timestamp: 39, + + BenchmarkPrice: mercury.MustEncodeValueInt192(big.NewInt(456)), + Bid: mercury.MustEncodeValueInt192(big.NewInt(450)), + Ask: mercury.MustEncodeValueInt192(big.NewInt(460)), + PricesValid: true, + + MaxFinalizedTimestamp: 39, + MaxFinalizedTimestampValid: true, + + LinkFee: mercury.MustEncodeValueInt192(big.NewInt(1.4e18)), + LinkFeeValid: true, + NativeFee: mercury.MustEncodeValueInt192(big.NewInt(2.4e18)), + NativeFeeValid: true, + + MarketStatus: 1, + MarketStatusValid: true, + }, + } +} + +func newValidAos(t *testing.T, protos ...*MercuryObservationProto) (aos []types.AttributedObservation) { + if len(protos) == 0 { + protos = newValidProtos() + } + aos = make([]types.AttributedObservation, len(protos)) + for i := range aos { + marshalledObs, err := proto.Marshal(protos[i]) + require.NoError(t, err) + aos[i] = types.AttributedObservation{ + Observation: marshalledObs, + Observer: commontypes.OracleID(i), + } + } + return +} + +func Test_parseAttributedObservation(t *testing.T) { + t.Run("returns error if bid<=mid<=ask is violated, even if observation claims itself to be valid", func(t *testing.T) { + obs := &MercuryObservationProto{ + Timestamp: 42, + + BenchmarkPrice: mercury.MustEncodeValueInt192(big.NewInt(123)), + Bid: mercury.MustEncodeValueInt192(big.NewInt(130)), + Ask: mercury.MustEncodeValueInt192(big.NewInt(120)), + PricesValid: true, + + MaxFinalizedTimestamp: 40, + MaxFinalizedTimestampValid: true, + + LinkFee: mercury.MustEncodeValueInt192(big.NewInt(1.1e18)), + LinkFeeValid: true, + NativeFee: mercury.MustEncodeValueInt192(big.NewInt(2.1e18)), + NativeFeeValid: true, + } + + serialized, err := proto.Marshal(obs) + require.NoError(t, err) + + _, err = parseAttributedObservation(types.AttributedObservation{Observation: serialized, Observer: commontypes.OracleID(42)}) + require.Error(t, err) + assert.Equal(t, "observation claimed to be valid, but contains invalid prices: invariant violated: expected bid<=mid<=ask, got bid: 130, mid: 123, ask: 120", err.Error()) + }) +} + +func Test_Plugin_Report(t *testing.T) { + dataSource := &testDataSource{} + codec := &testReportCodec{ + builtReport: []byte{1, 2, 3, 4}, + } + rp := newTestReportPlugin(t, codec, dataSource) + repts := types.ReportTimestamp{} + + t.Run("when previous report is nil", func(t *testing.T) { + t.Run("errors if not enough attributed observations", func(t *testing.T) { + _, _, err := rp.Report(repts, nil, newValidAos(t)[0:1]) + assert.EqualError(t, err, "only received 1 valid attributed observations, but need at least f+1 (2)") + }) + + t.Run("errors if too many maxFinalizedTimestamp observations are invalid", func(t *testing.T) { + ps := newValidProtos() + ps[0].MaxFinalizedTimestampValid = false + ps[1].MaxFinalizedTimestampValid = false + ps[2].MaxFinalizedTimestampValid = false + aos := newValidAos(t, ps...) + + should, _, err := rp.Report(types.ReportTimestamp{}, nil, aos) + assert.False(t, should) + assert.EqualError(t, err, "fewer than f+1 observations have a valid maxFinalizedTimestamp (got: 1/4)") + }) + t.Run("errors if maxFinalizedTimestamp is too large", func(t *testing.T) { + ps := newValidProtos() + ps[0].MaxFinalizedTimestamp = math.MaxUint32 + ps[1].MaxFinalizedTimestamp = math.MaxUint32 + ps[2].MaxFinalizedTimestamp = math.MaxUint32 + ps[3].MaxFinalizedTimestamp = math.MaxUint32 + aos := newValidAos(t, ps...) + + should, _, err := rp.Report(types.ReportTimestamp{}, nil, aos) + assert.False(t, should) + assert.EqualError(t, err, "maxFinalizedTimestamp is too large, got: 4294967295") + }) + + t.Run("succeeds and generates validFromTimestamp from maxFinalizedTimestamp when maxFinalizedTimestamp is positive", func(t *testing.T) { + aos := newValidAos(t) + + should, report, err := rp.Report(types.ReportTimestamp{}, nil, aos) + assert.True(t, should) + assert.NoError(t, err) + assert.Equal(t, codec.builtReport, report) + require.NotNil(t, codec.builtReportFields) + assert.Equal(t, v4.ReportFields{ + ValidFromTimestamp: 41, // consensus maxFinalizedTimestamp is 40, so validFrom should be 40+1 + Timestamp: 45, + NativeFee: big.NewInt(2300000000000000000), // 2.3e18 + LinkFee: big.NewInt(1300000000000000000), // 1.3e18 + ExpiresAt: 46, + BenchmarkPrice: big.NewInt(345), + Bid: big.NewInt(340), + Ask: big.NewInt(350), + MarketStatus: 1, + }, *codec.builtReportFields) + }) + t.Run("succeeds and generates validFromTimestamp from maxFinalizedTimestamp when maxFinalizedTimestamp is zero", func(t *testing.T) { + protos := newValidProtos() + for i := range protos { + protos[i].MaxFinalizedTimestamp = 0 + } + aos := newValidAos(t, protos...) + + should, report, err := rp.Report(types.ReportTimestamp{}, nil, aos) + assert.True(t, should) + assert.NoError(t, err) + assert.Equal(t, codec.builtReport, report) + require.NotNil(t, codec.builtReportFields) + assert.Equal(t, v4.ReportFields{ + ValidFromTimestamp: 1, + Timestamp: 45, + NativeFee: big.NewInt(2300000000000000000), // 2.3e18 + LinkFee: big.NewInt(1300000000000000000), // 1.3e18 + ExpiresAt: 46, + BenchmarkPrice: big.NewInt(345), + Bid: big.NewInt(340), + Ask: big.NewInt(350), + MarketStatus: 1, + }, *codec.builtReportFields) + }) + t.Run("succeeds and generates validFromTimestamp from maxFinalizedTimestamp when maxFinalizedTimestamp is -1 (missing feed)", func(t *testing.T) { + protos := newValidProtos() + for i := range protos { + protos[i].MaxFinalizedTimestamp = -1 + } + aos := newValidAos(t, protos...) + + should, report, err := rp.Report(types.ReportTimestamp{}, nil, aos) + assert.True(t, should) + assert.NoError(t, err) + assert.Equal(t, codec.builtReport, report) + require.NotNil(t, codec.builtReportFields) + assert.Equal(t, v4.ReportFields{ + ValidFromTimestamp: 45, // in case of missing feed, ValidFromTimestamp=Timestamp for first report + Timestamp: 45, + NativeFee: big.NewInt(2300000000000000000), // 2.3e18 + LinkFee: big.NewInt(1300000000000000000), // 1.3e18 + ExpiresAt: 46, + BenchmarkPrice: big.NewInt(345), + Bid: big.NewInt(340), + Ask: big.NewInt(350), + MarketStatus: 1, + }, *codec.builtReportFields) + }) + + t.Run("succeeds, ignoring unparseable attributed observation", func(t *testing.T) { + aos := newValidAos(t) + aos[0] = newUnparseableAttributedObservation() + + should, report, err := rp.Report(repts, nil, aos) + require.NoError(t, err) + + assert.True(t, should) + assert.Equal(t, codec.builtReport, report) + require.NotNil(t, codec.builtReportFields) + assert.Equal(t, v4.ReportFields{ + ValidFromTimestamp: 40, // consensus maxFinalizedTimestamp is 39, so validFrom should be 39+1 + Timestamp: 45, + NativeFee: big.NewInt(2300000000000000000), // 2.3e18 + LinkFee: big.NewInt(1300000000000000000), // 1.3e18 + ExpiresAt: 46, + BenchmarkPrice: big.NewInt(345), + Bid: big.NewInt(340), + Ask: big.NewInt(350), + MarketStatus: 1, + }, *codec.builtReportFields) + }) + }) + + t.Run("when previous report is present", func(t *testing.T) { + *codec = testReportCodec{ + observationTimestamp: uint32(rand.Int31n(math.MaxInt16)), + builtReport: []byte{1, 2, 3, 4}, + } + previousReport := types.Report{} + + t.Run("succeeds and uses timestamp from previous report if valid", func(t *testing.T) { + protos := newValidProtos() + ts := codec.observationTimestamp + 1 + for i := range protos { + protos[i].Timestamp = ts + } + aos := newValidAos(t, protos...) + + should, report, err := rp.Report(repts, previousReport, aos) + require.NoError(t, err) + + assert.True(t, should) + assert.Equal(t, codec.builtReport, report) + require.NotNil(t, codec.builtReportFields) + assert.Equal(t, v4.ReportFields{ + ValidFromTimestamp: codec.observationTimestamp + 1, // previous observation timestamp +1 second + Timestamp: ts, + NativeFee: big.NewInt(2300000000000000000), // 2.3e18 + LinkFee: big.NewInt(1300000000000000000), // 1.3e18 + ExpiresAt: ts + 1, + BenchmarkPrice: big.NewInt(345), + Bid: big.NewInt(340), + Ask: big.NewInt(350), + MarketStatus: 1, + }, *codec.builtReportFields) + }) + t.Run("errors if cannot extract timestamp from previous report", func(t *testing.T) { + codec.err = errors.New("something exploded trying to extract timestamp") + aos := newValidAos(t) + + should, _, err := rp.Report(types.ReportTimestamp{}, previousReport, aos) + assert.False(t, should) + assert.EqualError(t, err, "something exploded trying to extract timestamp") + }) + t.Run("does not report if observationTimestamp < validFromTimestamp", func(t *testing.T) { + codec.observationTimestamp = 43 + codec.err = nil + + protos := newValidProtos() + for i := range protos { + protos[i].Timestamp = 42 + } + aos := newValidAos(t, protos...) + + should, _, err := rp.Report(types.ReportTimestamp{}, previousReport, aos) + assert.False(t, should) + assert.NoError(t, err) + }) + t.Run("uses 0 values for link/native if they are invalid", func(t *testing.T) { + codec.observationTimestamp = 42 + codec.err = nil + + protos := newValidProtos() + for i := range protos { + protos[i].LinkFeeValid = false + protos[i].NativeFeeValid = false + } + aos := newValidAos(t, protos...) + + should, report, err := rp.Report(types.ReportTimestamp{}, previousReport, aos) + assert.True(t, should) + assert.NoError(t, err) + + assert.True(t, should) + assert.Equal(t, codec.builtReport, report) + require.NotNil(t, codec.builtReportFields) + assert.Equal(t, "0", codec.builtReportFields.LinkFee.String()) + assert.Equal(t, "0", codec.builtReportFields.NativeFee.String()) + }) + }) + + t.Run("buildReport failures", func(t *testing.T) { + t.Run("Report errors when the report is too large", func(t *testing.T) { + aos := newValidAos(t) + codec.builtReport = make([]byte, 1<<16) + + _, _, err := rp.Report(types.ReportTimestamp{}, nil, aos) + + assert.EqualError(t, err, "report with len 65536 violates MaxReportLength limit set by ReportCodec (123)") + }) + + t.Run("Report errors when the report length is 0", func(t *testing.T) { + aos := newValidAos(t) + codec.builtReport = []byte{} + _, _, err := rp.Report(types.ReportTimestamp{}, nil, aos) + + assert.EqualError(t, err, "report may not have zero length (invariant violation)") + }) + }) +} + +func Test_Plugin_validateReport(t *testing.T) { + dataSource := &testDataSource{} + codec := &testReportCodec{} + rp := newTestReportPlugin(t, codec, dataSource) + + t.Run("valid reports", func(t *testing.T) { + rf := v4.ReportFields{ + ValidFromTimestamp: 42, + Timestamp: 43, + NativeFee: big.NewInt(100), + LinkFee: big.NewInt(50), + ExpiresAt: 44, + BenchmarkPrice: big.NewInt(150), + Bid: big.NewInt(140), + Ask: big.NewInt(160), + } + err := rp.validateReport(rf) + require.NoError(t, err) + + rf = v4.ReportFields{ + ValidFromTimestamp: 42, + Timestamp: 42, + NativeFee: big.NewInt(0), + LinkFee: big.NewInt(0), + ExpiresAt: 42, + BenchmarkPrice: big.NewInt(1), + Bid: big.NewInt(1), + Ask: big.NewInt(1), + } + err = rp.validateReport(rf) + require.NoError(t, err) + }) + t.Run("fails validation", func(t *testing.T) { + rf := v4.ReportFields{ + ValidFromTimestamp: 44, // later than timestamp not allowed + Timestamp: 43, + NativeFee: big.NewInt(-1), // negative value not allowed + LinkFee: big.NewInt(-1), // negative value not allowed + ExpiresAt: 42, // before timestamp + BenchmarkPrice: big.NewInt(150000), // exceeds max + Bid: big.NewInt(150000), // exceeds max + Ask: big.NewInt(150000), // exceeds max + } + err := rp.validateReport(rf) + require.Error(t, err) + + assert.Contains(t, err.Error(), "median benchmark price (Value: 150000) is outside of allowable range (Min: 1, Max: 1000)") + assert.Contains(t, err.Error(), "median bid (Value: 150000) is outside of allowable range (Min: 1, Max: 1000)") + assert.Contains(t, err.Error(), "median ask (Value: 150000) is outside of allowable range (Min: 1, Max: 1000)") + assert.Contains(t, err.Error(), "median link fee (Value: -1) is outside of allowable range (Min: 0, Max: 3138550867693340381917894711603833208051177722232017256447)") + assert.Contains(t, err.Error(), "median native fee (Value: -1) is outside of allowable range (Min: 0, Max: 3138550867693340381917894711603833208051177722232017256447)") + assert.Contains(t, err.Error(), "observationTimestamp (Value: 43) must be >= validFromTimestamp (Value: 44)") + assert.Contains(t, err.Error(), "expiresAt (Value: 42) must be ahead of observation timestamp (Value: 43)") + }) + + t.Run("zero values", func(t *testing.T) { + rf := v4.ReportFields{} + err := rp.validateReport(rf) + require.Error(t, err) + + assert.Contains(t, err.Error(), "median benchmark price: got nil value") + assert.Contains(t, err.Error(), "median bid: got nil value") + assert.Contains(t, err.Error(), "median ask: got nil value") + assert.Contains(t, err.Error(), "median native fee: got nil value") + assert.Contains(t, err.Error(), "median link fee: got nil value") + }) +} + +func mustDecodeBigInt(b []byte) *big.Int { + n, err := mercury.DecodeValueInt192(b) + if err != nil { + panic(err) + } + return n +} + +func Test_Plugin_Observation(t *testing.T) { + dataSource := &testDataSource{} + codec := &testReportCodec{} + rp := newTestReportPlugin(t, codec, dataSource) + t.Run("Observation protobuf doesn't exceed maxObservationLength", func(t *testing.T) { + obs := MercuryObservationProto{ + Timestamp: math.MaxUint32, + BenchmarkPrice: make([]byte, 24), + Bid: make([]byte, 24), + Ask: make([]byte, 24), + PricesValid: true, + MaxFinalizedTimestamp: math.MaxUint32, + MaxFinalizedTimestampValid: true, + LinkFee: make([]byte, 24), + LinkFeeValid: true, + NativeFee: make([]byte, 24), + NativeFeeValid: true, + } + // This assertion is here to force this test to fail if a new field is + // added to the protobuf. In this case, you must add the max value of + // the field to the MercuryObservationProto in the test and only after + // that increment the count below + numFields := reflect.TypeOf(obs).NumField() //nolint:all + // 3 fields internal to pbuf struct + require.Equal(t, 13, numFields-3) + + b, err := proto.Marshal(&obs) + require.NoError(t, err) + assert.LessOrEqual(t, len(b), maxObservationLength) + }) + + validBid := big.NewInt(rand.Int63() - 2) + validBenchmarkPrice := new(big.Int).Add(validBid, big.NewInt(1)) + validAsk := new(big.Int).Add(validBid, big.NewInt(2)) + + t.Run("all observations succeeded", func(t *testing.T) { + obs := v4.Observation{ + BenchmarkPrice: mercurytypes.ObsResult[*big.Int]{ + Val: validBenchmarkPrice, + }, + Bid: mercurytypes.ObsResult[*big.Int]{ + Val: validBid, + }, + Ask: mercurytypes.ObsResult[*big.Int]{ + Val: validAsk, + }, + MaxFinalizedTimestamp: mercurytypes.ObsResult[int64]{ + Val: rand.Int63(), + }, + LinkPrice: mercurytypes.ObsResult[*big.Int]{ + Val: big.NewInt(rand.Int63()), + }, + NativePrice: mercurytypes.ObsResult[*big.Int]{ + Val: big.NewInt(rand.Int63()), + }, + } + dataSource.Obs = obs + + parsedObs, err := rp.Observation(context.Background(), types.ReportTimestamp{}, nil) + require.NoError(t, err) + + var p MercuryObservationProto + require.NoError(t, proto.Unmarshal(parsedObs, &p)) + + assert.LessOrEqual(t, p.Timestamp, uint32(time.Now().Unix())) + assert.Equal(t, obs.BenchmarkPrice.Val, mustDecodeBigInt(p.BenchmarkPrice)) + assert.True(t, p.PricesValid) + assert.Equal(t, obs.MaxFinalizedTimestamp.Val, p.MaxFinalizedTimestamp) + assert.True(t, p.MaxFinalizedTimestampValid) + + fee := mercury.CalculateFee(obs.LinkPrice.Val, decimal.NewFromInt32(1)) + assert.Equal(t, fee, mustDecodeBigInt(p.LinkFee)) + assert.True(t, p.LinkFeeValid) + + fee = mercury.CalculateFee(obs.NativePrice.Val, decimal.NewFromInt32(1)) + assert.Equal(t, fee, mustDecodeBigInt(p.NativeFee)) + assert.True(t, p.NativeFeeValid) + }) + + t.Run("negative link/native prices set fee to max int192", func(t *testing.T) { + obs := v4.Observation{ + LinkPrice: mercurytypes.ObsResult[*big.Int]{ + Val: big.NewInt(-1), + }, + NativePrice: mercurytypes.ObsResult[*big.Int]{ + Val: big.NewInt(-1), + }, + } + dataSource.Obs = obs + + parsedObs, err := rp.Observation(context.Background(), types.ReportTimestamp{}, nil) + require.NoError(t, err) + + var p MercuryObservationProto + require.NoError(t, proto.Unmarshal(parsedObs, &p)) + + assert.Equal(t, mercury.MaxInt192, mustDecodeBigInt(p.LinkFee)) + assert.True(t, p.LinkFeeValid) + assert.Equal(t, mercury.MaxInt192, mustDecodeBigInt(p.NativeFee)) + assert.True(t, p.NativeFeeValid) + }) + + t.Run("some observations failed", func(t *testing.T) { + obs := v4.Observation{ + BenchmarkPrice: mercurytypes.ObsResult[*big.Int]{ + Val: big.NewInt(rand.Int63()), + Err: errors.New("bechmarkPrice error"), + }, + MaxFinalizedTimestamp: mercurytypes.ObsResult[int64]{ + Val: rand.Int63(), + Err: errors.New("maxFinalizedTimestamp error"), + }, + LinkPrice: mercurytypes.ObsResult[*big.Int]{ + Val: big.NewInt(rand.Int63()), + Err: errors.New("linkPrice error"), + }, + NativePrice: mercurytypes.ObsResult[*big.Int]{ + Val: big.NewInt(rand.Int63()), + }, + } + + dataSource.Obs = obs + + parsedObs, err := rp.Observation(context.Background(), types.ReportTimestamp{}, nil) + require.NoError(t, err) + + var p MercuryObservationProto + require.NoError(t, proto.Unmarshal(parsedObs, &p)) + + assert.LessOrEqual(t, p.Timestamp, uint32(time.Now().Unix())) + assert.Zero(t, p.BenchmarkPrice) + assert.False(t, p.PricesValid) + assert.Zero(t, p.MaxFinalizedTimestamp) + assert.False(t, p.MaxFinalizedTimestampValid) + assert.Zero(t, p.LinkFee) + assert.False(t, p.LinkFeeValid) + + fee := mercury.CalculateFee(obs.NativePrice.Val, decimal.NewFromInt32(1)) + assert.Equal(t, fee, mustDecodeBigInt(p.NativeFee)) + assert.True(t, p.NativeFeeValid) + }) + + t.Run("all observations failed", func(t *testing.T) { + obs := v4.Observation{ + BenchmarkPrice: mercurytypes.ObsResult[*big.Int]{ + Err: errors.New("bechmarkPrice error"), + }, + Bid: mercurytypes.ObsResult[*big.Int]{ + Err: errors.New("bid error"), + }, + Ask: mercurytypes.ObsResult[*big.Int]{ + Err: errors.New("ask error"), + }, + MaxFinalizedTimestamp: mercurytypes.ObsResult[int64]{ + Err: errors.New("maxFinalizedTimestamp error"), + }, + LinkPrice: mercurytypes.ObsResult[*big.Int]{ + Err: errors.New("linkPrice error"), + }, + NativePrice: mercurytypes.ObsResult[*big.Int]{ + Err: errors.New("nativePrice error"), + }, + } + + dataSource.Obs = obs + + parsedObs, err := rp.Observation(context.Background(), types.ReportTimestamp{}, nil) + require.NoError(t, err) + + var p MercuryObservationProto + require.NoError(t, proto.Unmarshal(parsedObs, &p)) + + assert.LessOrEqual(t, p.Timestamp, uint32(time.Now().Unix())) + assert.Zero(t, p.BenchmarkPrice) + assert.Zero(t, p.Bid) + assert.Zero(t, p.Ask) + assert.False(t, p.PricesValid) + assert.Zero(t, p.MaxFinalizedTimestamp) + assert.False(t, p.MaxFinalizedTimestampValid) + assert.Zero(t, p.LinkFee) + assert.False(t, p.LinkFeeValid) + assert.Zero(t, p.NativeFee) + assert.False(t, p.NativeFeeValid) + }) + + t.Run("encoding fails on some observations", func(t *testing.T) { + obs := v4.Observation{ + BenchmarkPrice: mercurytypes.ObsResult[*big.Int]{ + Val: new(big.Int).Exp(big.NewInt(2), big.NewInt(256), nil), + }, + MaxFinalizedTimestamp: mercurytypes.ObsResult[int64]{ + Val: rand.Int63(), + }, + LinkPrice: mercurytypes.ObsResult[*big.Int]{ + Val: new(big.Int).Exp(big.NewInt(2), big.NewInt(256), nil), + }, + NativePrice: mercurytypes.ObsResult[*big.Int]{ + Val: big.NewInt(rand.Int63()), + }, + } + + dataSource.Obs = obs + + parsedObs, err := rp.Observation(context.Background(), types.ReportTimestamp{}, nil) + require.NoError(t, err) + + var p MercuryObservationProto + require.NoError(t, proto.Unmarshal(parsedObs, &p)) + + assert.Zero(t, p.BenchmarkPrice) + assert.False(t, p.PricesValid) + }) + + t.Run("encoding fails on all observations", func(t *testing.T) { + obs := v4.Observation{ + BenchmarkPrice: mercurytypes.ObsResult[*big.Int]{ + Val: new(big.Int).Exp(big.NewInt(2), big.NewInt(256), nil), + }, + Bid: mercurytypes.ObsResult[*big.Int]{ + Val: new(big.Int).Exp(big.NewInt(2), big.NewInt(256), nil), + }, + Ask: mercurytypes.ObsResult[*big.Int]{ + Val: new(big.Int).Exp(big.NewInt(2), big.NewInt(256), nil), + }, + MaxFinalizedTimestamp: mercurytypes.ObsResult[int64]{ + Val: rand.Int63(), + }, + // encoding never fails on calculated fees + LinkPrice: mercurytypes.ObsResult[*big.Int]{ + Val: new(big.Int).Exp(big.NewInt(2), big.NewInt(256), nil), + }, + NativePrice: mercurytypes.ObsResult[*big.Int]{ + Val: new(big.Int).Exp(big.NewInt(2), big.NewInt(256), nil), + }, + } + + dataSource.Obs = obs + + parsedObs, err := rp.Observation(context.Background(), types.ReportTimestamp{}, nil) + require.NoError(t, err) + + var p MercuryObservationProto + require.NoError(t, proto.Unmarshal(parsedObs, &p)) + + assert.Zero(t, p.BenchmarkPrice) + assert.Zero(t, p.Bid) + assert.Zero(t, p.Ask) + assert.False(t, p.PricesValid) + }) + + t.Run("bid<=mid<=ask violation", func(t *testing.T) { + obs := v4.Observation{ + BenchmarkPrice: mercurytypes.ObsResult[*big.Int]{ + Val: big.NewInt(10), + }, + Bid: mercurytypes.ObsResult[*big.Int]{ + Val: big.NewInt(11), + }, + Ask: mercurytypes.ObsResult[*big.Int]{ + Val: big.NewInt(12), + }, + MaxFinalizedTimestamp: mercurytypes.ObsResult[int64]{ + Val: rand.Int63(), + }, + LinkPrice: mercurytypes.ObsResult[*big.Int]{ + Val: big.NewInt(rand.Int63()), + }, + NativePrice: mercurytypes.ObsResult[*big.Int]{ + Val: big.NewInt(rand.Int63()), + }, + } + dataSource.Obs = obs + + parsedObs, err := rp.Observation(context.Background(), types.ReportTimestamp{}, nil) + require.NoError(t, err) + + var p MercuryObservationProto + require.NoError(t, proto.Unmarshal(parsedObs, &p)) + + assert.LessOrEqual(t, p.Timestamp, uint32(time.Now().Unix())) + assert.Equal(t, obs.BenchmarkPrice.Val, mustDecodeBigInt(p.BenchmarkPrice)) + assert.False(t, p.PricesValid) // not valid! + + // other values passed through ok + assert.Equal(t, obs.MaxFinalizedTimestamp.Val, p.MaxFinalizedTimestamp) + assert.True(t, p.MaxFinalizedTimestampValid) + + fee := mercury.CalculateFee(obs.LinkPrice.Val, decimal.NewFromInt32(1)) + assert.Equal(t, fee, mustDecodeBigInt(p.LinkFee)) + assert.True(t, p.LinkFeeValid) + + fee = mercury.CalculateFee(obs.NativePrice.Val, decimal.NewFromInt32(1)) + assert.Equal(t, fee, mustDecodeBigInt(p.NativeFee)) + assert.True(t, p.NativeFeeValid) + + // test benchmark price higher than ask + obs.BenchmarkPrice.Val = big.NewInt(13) + dataSource.Obs = obs + + parsedObs, err = rp.Observation(context.Background(), types.ReportTimestamp{}, nil) + require.NoError(t, err) + require.NoError(t, proto.Unmarshal(parsedObs, &p)) + assert.False(t, p.PricesValid) // not valid! + }) +} + +func newUnparseableAttributedObservation() types.AttributedObservation { + return types.AttributedObservation{ + Observation: []byte{1, 2}, + Observer: commontypes.OracleID(42), + } +} diff --git a/mercury/v4/observation.go b/mercury/v4/observation.go new file mode 100644 index 0000000..edbc0da --- /dev/null +++ b/mercury/v4/observation.go @@ -0,0 +1,110 @@ +package v4 + +import ( + "math/big" + + "github.com/smartcontractkit/libocr/commontypes" + + "github.com/smartcontractkit/chainlink-data-streams/mercury" +) + +type PAO interface { + mercury.PAO + GetBid() (*big.Int, bool) + GetAsk() (*big.Int, bool) + GetMaxFinalizedTimestamp() (int64, bool) + GetLinkFee() (*big.Int, bool) + GetNativeFee() (*big.Int, bool) + GetMarketStatus() (uint32, bool) +} + +var _ PAO = parsedAttributedObservation{} + +type parsedAttributedObservation struct { + Timestamp uint32 + Observer commontypes.OracleID + + BenchmarkPrice *big.Int + Bid *big.Int + Ask *big.Int + PricesValid bool + + MaxFinalizedTimestamp int64 + MaxFinalizedTimestampValid bool + + LinkFee *big.Int + LinkFeeValid bool + + NativeFee *big.Int + NativeFeeValid bool + + MarketStatus uint32 + MarketStatusValid bool +} + +func NewParsedAttributedObservation(ts uint32, observer commontypes.OracleID, + bp *big.Int, bid *big.Int, ask *big.Int, pricesValid bool, mfts int64, + mftsValid bool, linkFee *big.Int, linkFeeValid bool, nativeFee *big.Int, nativeFeeValid bool, + marketStatus uint32, marketStatusValid bool) PAO { + return parsedAttributedObservation{ + Timestamp: ts, + Observer: observer, + + BenchmarkPrice: bp, + Bid: bid, + Ask: ask, + PricesValid: pricesValid, + + MaxFinalizedTimestamp: mfts, + MaxFinalizedTimestampValid: mftsValid, + + LinkFee: linkFee, + LinkFeeValid: linkFeeValid, + + NativeFee: nativeFee, + NativeFeeValid: nativeFeeValid, + + MarketStatus: marketStatus, + MarketStatusValid: marketStatusValid, + } +} + +func (pao parsedAttributedObservation) GetTimestamp() uint32 { + return pao.Timestamp +} + +func (pao parsedAttributedObservation) GetObserver() commontypes.OracleID { + return pao.Observer +} + +func (pao parsedAttributedObservation) GetBenchmarkPrice() (*big.Int, bool) { + return pao.BenchmarkPrice, pao.PricesValid +} + +func (pao parsedAttributedObservation) GetBid() (*big.Int, bool) { + return pao.Bid, pao.PricesValid +} + +func (pao parsedAttributedObservation) GetAsk() (*big.Int, bool) { + return pao.Ask, pao.PricesValid +} + +func (pao parsedAttributedObservation) GetMaxFinalizedTimestamp() (int64, bool) { + if pao.MaxFinalizedTimestamp < -1 { + // values below -1 are not valid + return 0, false + } + return pao.MaxFinalizedTimestamp, pao.MaxFinalizedTimestampValid +} + +func (pao parsedAttributedObservation) GetLinkFee() (*big.Int, bool) { + return pao.LinkFee, pao.LinkFeeValid +} + +func (pao parsedAttributedObservation) GetNativeFee() (*big.Int, bool) { + return pao.NativeFee, pao.NativeFeeValid +} + +func (pao parsedAttributedObservation) GetMarketStatus() (uint32, bool) { + return pao.MarketStatus, pao.MarketStatusValid +}