integration-tests/smoke: add manual exec test (#15242)

* integration-tests/smoke: add manual exec test

Manually execute the message that failed to execute due to low callback
gas. This is the first manual exec scenario, others will be covered in
future PRs.

* fix lint

* updates after PR feedback

* use ctx, fill out filteropts

deployment/ccip/add_lane_test.go

@@ -96,14 +96,14 @@ func TestAddLane(t *testing.T) {
 	startBlock := latesthdr.Number.Uint64()
 	// Send traffic on the first lane and it should not be processed by the plugin as onRamp is disabled
 	// we will check this by confirming that the message is not executed by the end of the test
-	seqNum1 := TestSendRequest(t, e.Env, state, chain1, chain2, false, router.ClientEVM2AnyMessage{
+	msgSentEvent1 := TestSendRequest(t, e.Env, state, chain1, chain2, false, router.ClientEVM2AnyMessage{
 		Receiver:     common.LeftPadBytes(state.Chains[chain2].Receiver.Address().Bytes(), 32),
 		Data:         []byte("hello world"),
 		TokenAmounts: nil,
 		FeeToken:     common.HexToAddress("0x0"),
 		ExtraArgs:    nil,
 	})
-	require.Equal(t, uint64(1), seqNum1)
+	require.Equal(t, uint64(1), msgSentEvent1.SequenceNumber)
 
 	// Add another lane
 	require.NoError(t, AddLane(e.Env, state, chain2, chain1))
@@ -112,18 +112,18 @@ func TestAddLane(t *testing.T) {
 	latesthdr, err = e.Env.Chains[chain1].Client.HeaderByNumber(testcontext.Get(t), nil)
 	require.NoError(t, err)
 	startBlock2 := latesthdr.Number.Uint64()
-	seqNum2 := TestSendRequest(t, e.Env, state, chain2, chain1, false, router.ClientEVM2AnyMessage{
+	msgSentEvent2 := TestSendRequest(t, e.Env, state, chain2, chain1, false, router.ClientEVM2AnyMessage{
 		Receiver:     common.LeftPadBytes(state.Chains[chain2].Receiver.Address().Bytes(), 32),
 		Data:         []byte("hello world"),
 		TokenAmounts: nil,
 		FeeToken:     common.HexToAddress("0x0"),
 		ExtraArgs:    nil,
 	})
-	require.Equal(t, uint64(1), seqNum2)
-	require.NoError(t, commonutils.JustError(ConfirmExecWithSeqNr(t, e.Env.Chains[chain2], e.Env.Chains[chain1], state.Chains[chain1].OffRamp, &startBlock2, seqNum2)))
+	require.Equal(t, uint64(1), msgSentEvent2.SequenceNumber)
+	require.NoError(t, commonutils.JustError(ConfirmExecWithSeqNr(t, e.Env.Chains[chain2], e.Env.Chains[chain1], state.Chains[chain1].OffRamp, &startBlock2, msgSentEvent2.SequenceNumber)))
 
 	// now check for the previous message from chain 1 to chain 2 that it has not been executed till now as the onRamp was disabled
-	ConfirmNoExecConsistentlyWithSeqNr(t, e.Env.Chains[chain1], e.Env.Chains[chain2], state.Chains[chain2].OffRamp, seqNum1, 30*time.Second)
+	ConfirmNoExecConsistentlyWithSeqNr(t, e.Env.Chains[chain1], e.Env.Chains[chain2], state.Chains[chain2].OffRamp, msgSentEvent1.SequenceNumber, 30*time.Second)
 
 	// enable the onRamp on OffRamp
 	enableRampTx, err := state.Chains[chain2].OffRamp.ApplySourceChainConfigUpdates(e.Env.Chains[chain2].DeployerKey, []offramp.OffRampSourceChainConfigArgs{
@@ -146,5 +146,5 @@ func TestAddLane(t *testing.T) {
 	ReplayLogs(t, e.Env.Offchain, replayBlocks)
 	time.Sleep(30 * time.Second)
 	// Now that the onRamp is enabled, the request should be processed
-	require.NoError(t, commonutils.JustError(ConfirmExecWithSeqNr(t, e.Env.Chains[chain1], e.Env.Chains[chain2], state.Chains[chain2].OffRamp, &startBlock, seqNum1)))
+	require.NoError(t, commonutils.JustError(ConfirmExecWithSeqNr(t, e.Env.Chains[chain1], e.Env.Chains[chain2], state.Chains[chain2].OffRamp, &startBlock, msgSentEvent1.SequenceNumber)))
 }

deployment/ccip/changeset/active_candidate_test.go

@@ -84,14 +84,14 @@ func TestActiveCandidate(t *testing.T) {
 			require.NoError(t, err)
 			block := latesthdr.Number.Uint64()
 			startBlocks[dest] = &block
-			seqNum := ccdeploy.TestSendRequest(t, e, state, src, dest, false, router.ClientEVM2AnyMessage{
+			msgSentEvent := ccdeploy.TestSendRequest(t, e, state, src, dest, false, router.ClientEVM2AnyMessage{
 				Receiver:     common.LeftPadBytes(state.Chains[dest].Receiver.Address().Bytes(), 32),
 				Data:         []byte("hello world"),
 				TokenAmounts: nil,
 				FeeToken:     common.HexToAddress("0x0"),
 				ExtraArgs:    nil,
 			})
-			expectedSeqNum[dest] = seqNum
+			expectedSeqNum[dest] = msgSentEvent.SequenceNumber
 		}
 	}
 

deployment/ccip/changeset/add_chain_test.go

@@ -212,7 +212,7 @@ func TestAddChainInbound(t *testing.T) {
 	latesthdr, err := e.Env.Chains[newChain].Client.HeaderByNumber(testcontext.Get(t), nil)
 	require.NoError(t, err)
 	startBlock := latesthdr.Number.Uint64()
-	seqNr := ccipdeployment.TestSendRequest(t, e.Env, state, initialDeploy[0], newChain, true, router.ClientEVM2AnyMessage{
+	msgSentEvent := ccipdeployment.TestSendRequest(t, e.Env, state, initialDeploy[0], newChain, true, router.ClientEVM2AnyMessage{
 		Receiver:     common.LeftPadBytes(state.Chains[newChain].Receiver.Address().Bytes(), 32),
 		Data:         []byte("hello world"),
 		TokenAmounts: nil,
@@ -222,10 +222,10 @@ func TestAddChainInbound(t *testing.T) {
 	require.NoError(t,
 		ccipdeployment.ConfirmCommitWithExpectedSeqNumRange(t, e.Env.Chains[initialDeploy[0]], e.Env.Chains[newChain], state.Chains[newChain].OffRamp, &startBlock, cciptypes.SeqNumRange{
 			cciptypes.SeqNum(1),
-			cciptypes.SeqNum(seqNr),
+			cciptypes.SeqNum(msgSentEvent.SequenceNumber),
 		}))
 	require.NoError(t,
-		commonutils.JustError(ccipdeployment.ConfirmExecWithSeqNr(t, e.Env.Chains[initialDeploy[0]], e.Env.Chains[newChain], state.Chains[newChain].OffRamp, &startBlock, seqNr)))
+		commonutils.JustError(ccipdeployment.ConfirmExecWithSeqNr(t, e.Env.Chains[initialDeploy[0]], e.Env.Chains[newChain], state.Chains[newChain].OffRamp, &startBlock, msgSentEvent.SequenceNumber)))
 
 	linkAddress := state.Chains[newChain].LinkToken.Address()
 	feeQuoter := state.Chains[newChain].FeeQuoter

deployment/ccip/changeset/initial_deploy_test.go

@@ -74,14 +74,14 @@ func TestInitialDeploy(t *testing.T) {
 			require.NoError(t, err)
 			block := latesthdr.Number.Uint64()
 			startBlocks[dest] = &block
-			seqNum := ccdeploy.TestSendRequest(t, e, state, src, dest, false, router.ClientEVM2AnyMessage{
+			msgSentEvent := ccdeploy.TestSendRequest(t, e, state, src, dest, false, router.ClientEVM2AnyMessage{
 				Receiver:     common.LeftPadBytes(state.Chains[dest].Receiver.Address().Bytes(), 32),
 				Data:         []byte("hello"),
 				TokenAmounts: nil,
 				FeeToken:     common.HexToAddress("0x0"),
 				ExtraArgs:    nil,
 			})
-			expectedSeqNum[dest] = seqNum
+			expectedSeqNum[dest] = msgSentEvent.SequenceNumber
 		}
 	}
 

deployment/ccip/test_helpers.go

@@ -42,6 +42,7 @@ import (
 
 	"github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/burn_mint_token_pool"
 	"github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/mock_v3_aggregator_contract"
+	"github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/onramp"
 	"github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/router"
 	"github.com/smartcontractkit/chainlink/v2/core/gethwrappers/generated/aggregator_v3_interface"
 )
@@ -262,7 +263,7 @@ func TestSendRequest(
 	src, dest uint64,
 	testRouter bool,
 	evm2AnyMessage router.ClientEVM2AnyMessage,
-) (seqNum uint64) {
+) (msgSentEvent *onramp.OnRampCCIPMessageSent) {
 	t.Logf("Sending CCIP request from chain selector %d to chain selector %d",
 		src, dest)
 	tx, blockNum, err := CCIPSendRequest(
@@ -280,12 +281,16 @@ func TestSendRequest(
 	}, []uint64{dest}, []uint64{})
 	require.NoError(t, err)
 	require.True(t, it.Next())
-	seqNum = it.Event.Message.Header.SequenceNumber
-	nonce := it.Event.Message.Header.Nonce
-	sender := it.Event.Message.Sender
-	t.Logf("CCIP message sent from chain selector %d to chain selector %d tx %s seqNum %d nonce %d sender %s",
-		src, dest, tx.Hash().String(), seqNum, nonce, sender.String())
-	return seqNum
+	t.Logf("CCIP message (id %x) sent from chain selector %d to chain selector %d tx %s seqNum %d nonce %d sender %s",
+		it.Event.Message.Header.MessageId[:],
+		src,
+		dest,
+		tx.Hash().String(),
+		it.Event.SequenceNumber,
+		it.Event.Message.Header.Nonce,
+		it.Event.Message.Sender.String(),
+	)
+	return it.Event
 }
 
 // MakeEVMExtraArgsV2 creates the extra args for the EVM2Any message that is destined
@@ -440,23 +445,23 @@ func ConfirmRequestOnSourceAndDest(t *testing.T, env deployment.Environment, sta
 	require.NoError(t, err)
 	startBlock := latesthdr.Number.Uint64()
 	fmt.Printf("startblock %d", startBlock)
-	seqNum := TestSendRequest(t, env, state, sourceCS, destCS, false, router.ClientEVM2AnyMessage{
+	msgSentEvent := TestSendRequest(t, env, state, sourceCS, destCS, false, router.ClientEVM2AnyMessage{
 		Receiver:     common.LeftPadBytes(state.Chains[destCS].Receiver.Address().Bytes(), 32),
 		Data:         []byte("hello world"),
 		TokenAmounts: nil,
 		FeeToken:     common.HexToAddress("0x0"),
 		ExtraArgs:    nil,
 	})
-	require.Equal(t, expectedSeqNr, seqNum)
+	require.Equal(t, expectedSeqNr, msgSentEvent.SequenceNumber)
 
-	fmt.Printf("Request sent for seqnr %d", seqNum)
+	fmt.Printf("Request sent for seqnr %d", msgSentEvent.SequenceNumber)
 	require.NoError(t,
 		ConfirmCommitWithExpectedSeqNumRange(t, env.Chains[sourceCS], env.Chains[destCS], state.Chains[destCS].OffRamp, &startBlock, cciptypes.SeqNumRange{
-			cciptypes.SeqNum(seqNum),
-			cciptypes.SeqNum(seqNum),
+			cciptypes.SeqNum(msgSentEvent.SequenceNumber),
+			cciptypes.SeqNum(msgSentEvent.SequenceNumber),
 		}))
 
-	fmt.Printf("Commit confirmed for seqnr %d", seqNum)
+	fmt.Printf("Commit confirmed for seqnr %d", msgSentEvent.SequenceNumber)
 	require.NoError(
 		t,
 		commonutils.JustError(
@@ -466,7 +471,7 @@ func ConfirmRequestOnSourceAndDest(t *testing.T, env deployment.Environment, sta
 				env.Chains[destCS],
 				state.Chains[destCS].OffRamp,
 				&startBlock,
-				seqNum,
+				msgSentEvent.SequenceNumber,
 			),
 		),
 	)