MERC-6190: Remove bid/ask fields from Mercury v4 schema (#14252)

* Remove bid/ask fields from Mercury v4 schema

* Add changeset

* make generate

* Add #internal changeset tag

* Update chainlink-data-streams dep

.changeset/neat-yaks-switch.md

@@ -0,0 +1,5 @@
+---
+"chainlink": patch
+---
+
+Remove bid/ask fields for Mercury v4 schema #internal

core/scripts/go.mod

@@ -22,7 +22,7 @@ require (
 	github.com/prometheus/client_golang v1.17.0
 	github.com/shopspring/decimal v1.4.0
 	github.com/smartcontractkit/chainlink-automation v1.0.4
-	github.com/smartcontractkit/chainlink-common v0.2.2-0.20240830180817-6a0f3d1e0f9e
+	github.com/smartcontractkit/chainlink-common v0.2.2-0.20240903184200-6488292a85e3
 	github.com/smartcontractkit/chainlink/v2 v2.0.0-00010101000000-000000000000
 	github.com/smartcontractkit/libocr v0.0.0-20240717100443-f6226e09bee7
 	github.com/spf13/cobra v1.8.0
@@ -272,7 +272,7 @@ require (
 	github.com/smartcontractkit/chain-selectors v1.0.21 // indirect
 	github.com/smartcontractkit/chainlink-ccip v0.0.0-20240828115624-442f1cff195b // indirect
 	github.com/smartcontractkit/chainlink-cosmos v0.4.1-0.20240710121324-3ed288aa9b45 // indirect
-	github.com/smartcontractkit/chainlink-data-streams v0.0.0-20240820130645-cf4b159fbba2 // indirect
+	github.com/smartcontractkit/chainlink-data-streams v0.0.0-20240904093355-e40169857652 // indirect
 	github.com/smartcontractkit/chainlink-feeds v0.0.0-20240710170203-5b41615da827 // indirect
 	github.com/smartcontractkit/chainlink-solana v1.1.1-0.20240821170223-a2f5c39f457f // indirect
 	github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240709043547-03612098f799 // indirect
@@ -363,5 +363,10 @@ require (
 	sigs.k8s.io/yaml v1.4.0 // indirect
 )
 
-// replicating the replace directive on cosmos SDK
-replace github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1
+replace (
+	// until merged upstream: https://github.com/omissis/go-jsonschema/pull/264
+	github.com/atombender/go-jsonschema => github.com/nolag/go-jsonschema v0.16.0-rtinianov
+
+	// replicating the replace directive on cosmos SDK
+	github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1
+)

core/scripts/go.sum

@@ -1190,12 +1190,12 @@ github.com/smartcontractkit/chainlink-automation v1.0.4 h1:iyW181JjKHLNMnDleI8um
 github.com/smartcontractkit/chainlink-automation v1.0.4/go.mod h1:u4NbPZKJ5XiayfKHD/v3z3iflQWqvtdhj13jVZXj/cM=
 github.com/smartcontractkit/chainlink-ccip v0.0.0-20240828115624-442f1cff195b h1:v1RnZVfUoHIm/lwIqRAH4eDRNTu+N+AtQE5Ik4U9hsU=
 github.com/smartcontractkit/chainlink-ccip v0.0.0-20240828115624-442f1cff195b/go.mod h1:Z9lQ5t20kRk28pzRLnqAJZUVOw8E6/siA3P3MLyKqoM=
-github.com/smartcontractkit/chainlink-common v0.2.2-0.20240830180817-6a0f3d1e0f9e h1:kxMSsv/RaTlH0Up9YTC6FE4K9n5QWcSKxbtiTS+JHzU=
-github.com/smartcontractkit/chainlink-common v0.2.2-0.20240830180817-6a0f3d1e0f9e/go.mod h1:D/qaCoq0SxXzg5NRN5FtBRv98VBf+D2NOC++RbvvuOc=
+github.com/smartcontractkit/chainlink-common v0.2.2-0.20240903184200-6488292a85e3 h1:fkfOoAPviqO2rN8ngvejsDa7WKcw4paGEFA4/Znu0L0=
+github.com/smartcontractkit/chainlink-common v0.2.2-0.20240903184200-6488292a85e3/go.mod h1:D/qaCoq0SxXzg5NRN5FtBRv98VBf+D2NOC++RbvvuOc=
 github.com/smartcontractkit/chainlink-cosmos v0.4.1-0.20240710121324-3ed288aa9b45 h1:NBQLtqk8zsyY4qTJs+NElI3aDFTcAo83JHvqD04EvB0=
 github.com/smartcontractkit/chainlink-cosmos v0.4.1-0.20240710121324-3ed288aa9b45/go.mod h1:LV0h7QBQUpoC2UUi6TcUvcIFm1xjP/DtEcqV8+qeLUs=
-github.com/smartcontractkit/chainlink-data-streams v0.0.0-20240820130645-cf4b159fbba2 h1:KH6tpCw5hu8u6UTtgll7a8mE4sIbHCbmtzHJdKuRwBw=
-github.com/smartcontractkit/chainlink-data-streams v0.0.0-20240820130645-cf4b159fbba2/go.mod h1:V/86loaFSH0dqqUEHqyXVbyNqDRSjvcf9BRomWFTljU=
+github.com/smartcontractkit/chainlink-data-streams v0.0.0-20240904093355-e40169857652 h1:0aZ3HiEz2bMM5ywHAyKlFMN95qTzpNDn7uvnHLrFX6s=
+github.com/smartcontractkit/chainlink-data-streams v0.0.0-20240904093355-e40169857652/go.mod h1:PwPcmQNAzVmU8r8JWKrDRgvXesDwxnqbMD6DvYt/Z7M=
 github.com/smartcontractkit/chainlink-feeds v0.0.0-20240710170203-5b41615da827 h1:BCHu4pNP6arrcHLEWx61XjLaonOd2coQNyL0NTUcaMc=
 github.com/smartcontractkit/chainlink-feeds v0.0.0-20240710170203-5b41615da827/go.mod h1:OPX+wC2TWQsyLNpR7daMt2vMpmsNcoBxbZyGTHr6tiA=
 github.com/smartcontractkit/chainlink-solana v1.1.1-0.20240821170223-a2f5c39f457f h1:b0Ifwk7eK3fwJ0R69Ovhv5XvZ1/TUAfHkU5Jp7wbNZ0=

core/services/ocr2/plugins/mercury/helpers_test.go

@@ -478,8 +478,6 @@ func addV4MercuryJob(
 	bootstrapPeerID string,
 	bootstrapNodePort int,
 	bmBridge,
-	bidBridge,
-	askBridge,
 	marketStatusBridge string,
 	servers map[string]string,
 	clientPubKey ed25519.PublicKey,
@@ -497,19 +495,19 @@ func addV4MercuryJob(
 	node.AddJob(t, fmt.Sprintf(`
 type = "offchainreporting2"
 schemaVersion = 1
-name = "mercury-%[1]d-%[11]s"
+name = "mercury-%[1]d-%[9]s"
 forwardingAllowed = false
 maxTaskDuration = "1s"
 contractID = "%[2]s"
-feedID = "0x%[10]x"
+feedID = "0x%[8]x"
 contractConfigTrackerPollInterval = "1s"
 ocrKeyBundleID = "%[3]s"
 p2pv2Bootstrappers = [
   "%[4]s"
 ]
 relay = "evm"
 pluginType = "mercury"
-transmitterID = "%[9]x"
+transmitterID = "%[7]x"
 observationSource = """
 	// Benchmark Price
 	price1          [type=bridge name="%[5]s" timeout="50ms" requestData="{\\"data\\":{\\"from\\":\\"ETH\\",\\"to\\":\\"USD\\"}}"];
@@ -518,31 +516,17 @@ observationSource = """
 
 	price1 -> price1_parse -> price1_multiply;
 
-	// Bid
-	bid          [type=bridge name="%[6]s" timeout="50ms" requestData="{\\"data\\":{\\"from\\":\\"ETH\\",\\"to\\":\\"USD\\"}}"];
-	bid_parse    [type=jsonparse path="result"];
-	bid_multiply [type=multiply times=100000000 index=1];
-
-	bid -> bid_parse -> bid_multiply;
-
-	// Ask
-	ask          [type=bridge name="%[7]s" timeout="50ms" requestData="{\\"data\\":{\\"from\\":\\"ETH\\",\\"to\\":\\"USD\\"}}"];
-	ask_parse    [type=jsonparse path="result"];
-	ask_multiply [type=multiply times=100000000 index=2];
-
-	ask -> ask_parse -> ask_multiply;
-
 	// Market Status
-	marketstatus       [type=bridge name="%[14]s" timeout="50ms" requestData="{\\"data\\":{\\"from\\":\\"ETH\\",\\"to\\":\\"USD\\"}}"];
-	marketstatus_parse [type=jsonparse path="result" index=3];
+	marketstatus       [type=bridge name="%[12]s" timeout="50ms" requestData="{\\"data\\":{\\"from\\":\\"ETH\\",\\"to\\":\\"USD\\"}}"];
+	marketstatus_parse [type=jsonparse path="result" index=1];
 
 	marketstatus -> marketstatus_parse;
 """
 
 [pluginConfig]
-servers = %[8]s
-linkFeedID = "0x%[12]x"
-nativeFeedID = "0x%[13]x"
+servers = %[6]s
+linkFeedID = "0x%[10]x"
+nativeFeedID = "0x%[11]x"
 
 [relayConfig]
 chainID = 1337
@@ -552,8 +536,6 @@ chainID = 1337
 		node.KeyBundle.ID(),
 		fmt.Sprintf("%[email protected]:%d", bootstrapPeerID, bootstrapNodePort),
 		bmBridge,
-		bidBridge,
-		askBridge,
 		serversStr,
 		clientPubKey,
 		feedID,

core/services/ocr2/plugins/mercury/integration_test.go

@@ -1105,7 +1105,7 @@ func integration_MercuryV4(t *testing.T) {
 		k := csakey.MustNewV2XXXTestingOnly(big.NewInt(int64(-(i + 1))))
 		reqs := make(chan request, 100)
 		srv := NewMercuryServer(t, ed25519.PrivateKey(k.Raw()), reqs, func() []byte {
-			report, err := (&reportcodecv4.ReportCodec{}).BuildReport(v4.ReportFields{BenchmarkPrice: big.NewInt(234567), Bid: big.NewInt(1), Ask: big.NewInt(1), LinkFee: big.NewInt(1), NativeFee: big.NewInt(1), MarketStatus: 1})
+			report, err := (&reportcodecv4.ReportCodec{}).BuildReport(v4.ReportFields{BenchmarkPrice: big.NewInt(234567), LinkFee: big.NewInt(1), NativeFee: big.NewInt(1), MarketStatus: 1})
 			if err != nil {
 				panic(err)
 			}
@@ -1205,8 +1205,6 @@ func integration_MercuryV4(t *testing.T) {
 	for i, node := range nodes {
 		for j, feed := range feeds {
 			bmBridge := createBridge(fmt.Sprintf("benchmarkprice-%d", j), i, feed.baseBenchmarkPrice, 0, node.App.BridgeORM())
-			bidBridge := createBridge(fmt.Sprintf("bid-%d", j), i, feed.baseBid, 0, node.App.BridgeORM())
-			askBridge := createBridge(fmt.Sprintf("ask-%d", j), i, feed.baseAsk, 0, node.App.BridgeORM())
 			marketStatusBridge := createBridge(fmt.Sprintf("marketstatus-%d", j), i, nil, feed.baseMarketStatus, node.App.BridgeORM())
 
 			addV4MercuryJob(
@@ -1217,8 +1215,6 @@ func integration_MercuryV4(t *testing.T) {
 				bootstrapPeerID,
 				bootstrapNodePort,
 				bmBridge,
-				bidBridge,
-				askBridge,
 				marketStatusBridge,
 				servers,
 				clientPubKeys[i],
@@ -1312,8 +1308,6 @@ func integration_MercuryV4(t *testing.T) {
 
 			assert.GreaterOrEqual(t, int(reportElems["observationsTimestamp"].(uint32)), int(testStartTimeStamp))
 			assert.InDelta(t, feed.baseBenchmarkPrice.Int64(), reportElems["benchmarkPrice"].(*big.Int).Int64(), 5000000)
-			assert.InDelta(t, feed.baseBid.Int64(), reportElems["bid"].(*big.Int).Int64(), 5000000)
-			assert.InDelta(t, feed.baseAsk.Int64(), reportElems["ask"].(*big.Int).Int64(), 5000000)
 			assert.NotZero(t, reportElems["validFromTimestamp"].(uint32))
 			assert.GreaterOrEqual(t, reportElems["observationsTimestamp"].(uint32), reportElems["validFromTimestamp"].(uint32))
 			assert.Equal(t, expectedExpiresAt, reportElems["expiresAt"].(uint32))

core/services/ocrcommon/telemetry.go

@@ -294,7 +294,7 @@ func (e *EnhancedTelemetryService[T]) collectMercuryEnhancedTelemetry(d Enhanced
 	var bn int64
 	var bh string
 	var bt uint64
-	// v1+v2+v3 fields
+	// v1+v2+v3+v4 fields
 	bp := big.NewInt(0)
 	// v1+v3 fields
 	bid := big.NewInt(0)
@@ -373,12 +373,6 @@ func (e *EnhancedTelemetryService[T]) collectMercuryEnhancedTelemetry(d Enhanced
 		if obs.BenchmarkPrice.Err == nil && obs.BenchmarkPrice.Val != nil {
 			bp = obs.BenchmarkPrice.Val
 		}
-		if obs.Bid.Err == nil && obs.Bid.Val != nil {
-			bid = obs.Bid.Val
-		}
-		if obs.Ask.Err == nil && obs.Ask.Val != nil {
-			ask = obs.Ask.Val
-		}
 		if obs.MarketStatus.Err == nil {
 			marketStatus = telem.MarketStatus(obs.MarketStatus.Val)
 		}

core/services/relay/evm/mercury/v4/data_source.go

@@ -104,8 +104,6 @@ func (ds *datasource) Observe(ctx context.Context, repts ocrtypes.ReportTimestam
 			return
 		}
 		obs.BenchmarkPrice = parsed.benchmarkPrice
-		obs.Bid = parsed.bid
-		obs.Ask = parsed.ask
 		obs.MarketStatus = parsed.marketStatus
 	}()
 
@@ -188,8 +186,6 @@ func toBigInt(val interface{}) (*big.Int, error) {
 
 type parseOutput struct {
 	benchmarkPrice mercury.ObsResult[*big.Int]
-	bid            mercury.ObsResult[*big.Int]
-	ask            mercury.ObsResult[*big.Int]
 	marketStatus   mercury.ObsResult[uint32]
 }
 
@@ -204,15 +200,13 @@ func (ds *datasource) parse(trrs pipeline.TaskRunResults) (o parseOutput, merr e
 
 	// pipeline.TaskRunResults comes ordered asc by index, this is guaranteed
 	// by the pipeline executor
-	if len(finaltrrs) != 4 {
-		return o, fmt.Errorf("invalid number of results, expected: 4, got: %d", len(finaltrrs))
+	if len(finaltrrs) != 2 {
+		return o, fmt.Errorf("invalid number of results, expected: 2, got: %d", len(finaltrrs))
 	}
 
 	merr = errors.Join(
 		setBenchmarkPrice(&o, finaltrrs[0].Result),
-		setBid(&o, finaltrrs[1].Result),
-		setAsk(&o, finaltrrs[2].Result),
-		setMarketStatus(&o, finaltrrs[3].Result),
+		setMarketStatus(&o, finaltrrs[1].Result),
 	)
 
 	return o, merr
@@ -231,32 +225,6 @@ func setBenchmarkPrice(o *parseOutput, res pipeline.Result) error {
 	return nil
 }
 
-func setBid(o *parseOutput, res pipeline.Result) error {
-	if res.Error != nil {
-		o.bid.Err = res.Error
-		return res.Error
-	}
-	val, err := toBigInt(res.Value)
-	if err != nil {
-		return fmt.Errorf("failed to parse Bid: %w", err)
-	}
-	o.bid.Val = val
-	return nil
-}
-
-func setAsk(o *parseOutput, res pipeline.Result) error {
-	if res.Error != nil {
-		o.ask.Err = res.Error
-		return res.Error
-	}
-	val, err := toBigInt(res.Value)
-	if err != nil {
-		return fmt.Errorf("failed to parse Ask: %w", err)
-	}
-	o.ask.Val = val
-	return nil
-}
-
 func setMarketStatus(o *parseOutput, res pipeline.Result) error {
 	if res.Error != nil {
 		o.marketStatus.Err = res.Error

core/services/relay/evm/mercury/v4/data_source_test.go

@@ -86,16 +86,6 @@ func Test_Datasource(t *testing.T) {
 			Result: pipeline.Result{Value: "122.345"},
 			Task:   &mercurymocks.MockTask{},
 		},
-		{
-			// bid
-			Result: pipeline.Result{Value: "121.993"},
-			Task:   &mercurymocks.MockTask{},
-		},
-		{
-			// ask
-			Result: pipeline.Result{Value: "123.111"},
-			Task:   &mercurymocks.MockTask{},
-		},
 		{
 			// marketStatus
 			Result: pipeline.Result{Value: "1"},
@@ -193,10 +183,6 @@ func Test_Datasource(t *testing.T) {
 
 				assert.Equal(t, big.NewInt(122), obs.BenchmarkPrice.Val)
 				assert.NoError(t, obs.BenchmarkPrice.Err)
-				assert.Equal(t, big.NewInt(121), obs.Bid.Val)
-				assert.NoError(t, obs.Bid.Err)
-				assert.Equal(t, big.NewInt(123), obs.Ask.Val)
-				assert.NoError(t, obs.Ask.Err)
 				assert.Equal(t, int64(123123), obs.MaxFinalizedTimestamp.Val)
 				assert.NoError(t, obs.MaxFinalizedTimestamp.Err)
 				assert.Equal(t, big.NewInt(122), obs.LinkPrice.Val)
@@ -239,17 +225,7 @@ func Test_Datasource(t *testing.T) {
 			badTrrs := []pipeline.TaskRunResult{
 				{
 					// benchmark price
-					Result: pipeline.Result{Value: "122.345"},
-					Task:   &mercurymocks.MockTask{},
-				},
-				{
-					// bid
-					Result: pipeline.Result{Value: "121.993"},
-					Task:   &mercurymocks.MockTask{},
-				},
-				{
-					// ask
-					Result: pipeline.Result{Error: errors.New("some error with ask")},
+					Result: pipeline.Result{Error: errors.New("some error with bp")},
 					Task:   &mercurymocks.MockTask{},
 				},
 				{
@@ -265,7 +241,7 @@ func Test_Datasource(t *testing.T) {
 			}
 
 			_, err := ds.Observe(ctx, repts, false)
-			assert.EqualError(t, err, "Observe failed while parsing run results: some error with ask")
+			assert.EqualError(t, err, "Observe failed while parsing run results: some error with bp")
 		})
 
 		t.Run("when run execution succeeded", func(t *testing.T) {
@@ -282,10 +258,6 @@ func Test_Datasource(t *testing.T) {
 
 				assert.Equal(t, big.NewInt(122), obs.BenchmarkPrice.Val)
 				assert.NoError(t, obs.BenchmarkPrice.Err)
-				assert.Equal(t, big.NewInt(121), obs.Bid.Val)
-				assert.NoError(t, obs.Bid.Err)
-				assert.Equal(t, big.NewInt(123), obs.Ask.Val)
-				assert.NoError(t, obs.Ask.Err)
 				assert.Equal(t, int64(0), obs.MaxFinalizedTimestamp.Val)
 				assert.NoError(t, obs.MaxFinalizedTimestamp.Err)
 				assert.Equal(t, big.NewInt(122), obs.LinkPrice.Val)
@@ -333,15 +305,13 @@ func buildSamplev4Report() []byte {
 	feedID := sampleFeedID
 	timestamp := uint32(124)
 	bp := big.NewInt(242)
-	bid := big.NewInt(243)
-	ask := big.NewInt(244)
 	validFromTimestamp := uint32(123)
 	expiresAt := uint32(456)
 	linkFee := big.NewInt(3334455)
 	nativeFee := big.NewInt(556677)
 	marketStatus := uint32(1)
 
-	b, err := reportcodecv4.ReportTypes.Pack(feedID, validFromTimestamp, timestamp, nativeFee, linkFee, expiresAt, bp, bid, ask, marketStatus)
+	b, err := reportcodecv4.ReportTypes.Pack(feedID, validFromTimestamp, timestamp, nativeFee, linkFee, expiresAt, bp, marketStatus)
 	if err != nil {
 		panic(err)
 	}

core/services/relay/evm/mercury/v4/reportcodec/report_codec.go

@@ -35,12 +35,6 @@ func (r *ReportCodec) BuildReport(rf v4.ReportFields) (ocrtypes.Report, error) {
 	if rf.BenchmarkPrice == nil {
 		merr = errors.Join(merr, errors.New("benchmarkPrice may not be nil"))
 	}
-	if rf.Bid == nil {
-		merr = errors.Join(merr, errors.New("bid may not be nil"))
-	}
-	if rf.Ask == nil {
-		merr = errors.Join(merr, errors.New("ask may not be nil"))
-	}
 	if rf.LinkFee == nil {
 		merr = errors.Join(merr, errors.New("linkFee may not be nil"))
 	} else if rf.LinkFee.Cmp(zero) < 0 {
@@ -54,7 +48,7 @@ func (r *ReportCodec) BuildReport(rf v4.ReportFields) (ocrtypes.Report, error) {
 	if merr != nil {
 		return nil, merr
 	}
-	reportBytes, err := ReportTypes.Pack(r.feedID, rf.ValidFromTimestamp, rf.Timestamp, rf.NativeFee, rf.LinkFee, rf.ExpiresAt, rf.BenchmarkPrice, rf.Bid, rf.Ask, rf.MarketStatus)
+	reportBytes, err := ReportTypes.Pack(r.feedID, rf.ValidFromTimestamp, rf.Timestamp, rf.NativeFee, rf.LinkFee, rf.ExpiresAt, rf.BenchmarkPrice, rf.MarketStatus)
 	return ocrtypes.Report(reportBytes), pkgerrors.Wrap(err, "failed to pack report blob")
 }