add RPC chaos test example for CRIB (#14274)

.github/workflows/crib-integration-test.yml

@@ -104,7 +104,7 @@ jobs:
           # RESTY_DEBUG: true
           TEST_PERSISTENCE: true
         run: |-
-          go test -v -run TestCRIB
+          go test -v -run TestCRIBChaos
       - name: Destroy CRIB Environment
         id: destroy
         if: always() && steps.deploy-crib.outputs.devspace-namespace != ''

integration-tests/crib/README.md

@@ -19,7 +19,7 @@ export CRIB_NODES=5 # min 5 nodes
 #export RESTY_DEBUG=true
 #export TEST_PERSISTENCE=true # to run the chaos test
 export GAP_URL=https://localhost:8080/primary # only applicable in CI, unset the var to connect locally
-go test -v -run TestCRIB
+go test -v -run TestCRIBChaos
 ```
 
 ## Configuring CI workflow

integration-tests/crib/connect.go

@@ -37,16 +37,17 @@ func ConnectRemote() (
 	*msClient.MockserverClient,
 	*client.ChainlinkK8sClient,
 	[]*client.ChainlinkK8sClient,
+	*crib.CoreDONConnectionConfig,
 	error,
 ) {
 	vars, err := crib.CoreDONSimulatedConnection()
 	if err != nil {
-		return nil, nil, nil, nil, err
+		return nil, nil, nil, nil, nil, err
 	}
 	// TODO: move all the parts of ConnectRemote() to CTF when Seth config refactor is finalized
 	config, err := tc.GetConfig([]string{"CRIB"}, tc.OCR)
 	if err != nil {
-		return nil, nil, nil, nil, err
+		return nil, nil, nil, nil, nil, err
 	}
 	var sethClient *seth.Client
 	switch vars.Network {
@@ -69,10 +70,10 @@ func ConnectRemote() (
 		}
 		sethClient, err = seth_utils.GetChainClient(config, net)
 		if err != nil {
-			return nil, nil, nil, nil, err
+			return nil, nil, nil, nil, nil, err
 		}
 	default:
-		return nil, nil, nil, nil, errors.New("CRIB network is not supported")
+		return nil, nil, nil, nil, nil, errors.New("CRIB network is not supported")
 	}
 	// bootstrap node
 	clClients := make([]*client.ChainlinkK8sClient, 0)
@@ -84,7 +85,7 @@ func ConnectRemote() (
 		Headers:    vars.NodeHeaders[0],
 	}, vars.NodeInternalDNS[0], vars.Namespace)
 	if err != nil {
-		return nil, nil, nil, nil, err
+		return nil, nil, nil, nil, nil, err
 	}
 	clClients = append(clClients, c)
 	// all the other nodes, indices of nodes in CRIB starts with 1
@@ -97,7 +98,7 @@ func ConnectRemote() (
 			Headers:    vars.NodeHeaders[i],
 		}, vars.NodeInternalDNS[i], vars.Namespace)
 		if err != nil {
-			return nil, nil, nil, nil, err
+			return nil, nil, nil, nil, nil, err
 		}
 		clClients = append(clClients, cl)
 	}
@@ -108,5 +109,5 @@ func ConnectRemote() (
 	})
 
 	//nolint:gosec // G602 - false positive https://github.com/securego/gosec/issues/1005
-	return sethClient, mockServerClient, clClients[0], clClients[1:], nil
+	return sethClient, mockServerClient, clClients[0], clClients[1:], vars, nil
 }

integration-tests/crib/ocr_test.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/smartcontractkit/chainlink-testing-framework/client"
 	"github.com/smartcontractkit/chainlink-testing-framework/havoc"
 
 	"github.com/stretchr/testify/require"
@@ -16,10 +17,11 @@ import (
 	"github.com/smartcontractkit/chainlink-testing-framework/logging"
 )
 
-func TestCRIB(t *testing.T) {
+// TestCRIBChaos an example of how we can run chaos tests with havoc and core CRIB
+func TestCRIBChaos(t *testing.T) {
 	l := logging.GetTestLogger(t)
 
-	sethClient, msClient, bootstrapNode, workerNodes, err := ConnectRemote()
+	sethClient, msClient, bootstrapNode, workerNodes, _, err := ConnectRemote()
 	require.NoError(t, err)
 
 	lta, err := actions.SetupOCRv1Cluster(l, sethClient, workerNodes)
@@ -55,3 +57,30 @@ func TestCRIB(t *testing.T) {
 		}, 20*time.Minute, 5*time.Second)
 	}
 }
+
+// TestCRIBRPCChaos and example of how we can run RPC chaos with Geth or Anvil
+func TestCRIBRPCChaos(t *testing.T) {
+	l := logging.GetTestLogger(t)
+
+	sethClient, msClient, bootstrapNode, workerNodes, vars, err := ConnectRemote()
+	require.NoError(t, err)
+
+	lta, err := actions.SetupOCRv1Cluster(l, sethClient, workerNodes)
+	require.NoError(t, err)
+	ocrInstances, err := actions.SetupOCRv1Feed(l, sethClient, lta, msClient, bootstrapNode, workerNodes)
+	require.NoError(t, err)
+
+	err = actions.SetAllAdapterResponsesToTheSameValue(10, ocrInstances, workerNodes, msClient)
+	require.NoError(t, err)
+	actions.SimulateOCRv1EAActivity(l, 3*time.Second, ocrInstances, workerNodes, msClient)
+
+	err = actions.WatchNewOCRRound(l, sethClient, 1, contracts.V1OffChainAgrregatorToOffChainAggregatorWithRounds(ocrInstances), 5*time.Minute)
+	require.NoError(t, err, "Error watching for new OCR round")
+
+	ac := client.NewRPCClient(sethClient.URL, vars.BlockchainNodeHeaders)
+	err = ac.GethSetHead(10)
+	require.NoError(t, err)
+
+	err = actions.WatchNewOCRRound(l, sethClient, 3, contracts.V1OffChainAgrregatorToOffChainAggregatorWithRounds(ocrInstances), 5*time.Minute)
+	require.NoError(t, err, "Error watching for new OCR round")
+}

integration-tests/load/ocr/ocr_test.go

@@ -28,7 +28,7 @@ func TestOCRLoad(t *testing.T) {
 	config, err := tc.GetConfig([]string{"Load"}, tc.OCR)
 	require.NoError(t, err)
 
-	sethClient, msClient, bootstrapNode, workerNodes, err := crib.ConnectRemote()
+	sethClient, msClient, bootstrapNode, workerNodes, _, err := crib.ConnectRemote()
 	require.NoError(t, err)
 
 	lta, err := actions.SetupOCRv1Cluster(l, sethClient, workerNodes)
@@ -61,7 +61,7 @@ func TestOCRVolume(t *testing.T) {
 	config, err := tc.GetConfig([]string{"Volume"}, tc.OCR)
 	require.NoError(t, err)
 
-	sethClient, msClient, bootstrapNode, workerNodes, err := crib.ConnectRemote()
+	sethClient, msClient, bootstrapNode, workerNodes, _, err := crib.ConnectRemote()
 	require.NoError(t, err)
 
 	lta, err := actions.SetupOCRv1Cluster(l, sethClient, workerNodes)