Adding on demand soak

.github/workflows/integration-tests-soak.yml

@@ -0,0 +1,83 @@
+name: Integration Tests - Soak
+on:
+  workflow_dispatch:
+    inputs:
+      base64_config:
+        description: Your .toml file as base64
+        required: true
+      cl_image_tag:
+        description: Core image tag
+        required: true
+        default: develop
+        type: string
+      test_runner_tag:
+        description: Remote runner tag that will run the tests
+        default: develop
+        required: true
+        type: string
+
+env:
+  TEST_LOG_LEVEL: debug
+  CL_ECR: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
+  ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-starknet-tests:${{ inputs.test_runner_tag }}
+
+jobs:
+  run_tests:
+    name: Run soak Tests
+    runs-on: ubuntu20.04-16cores-64GB
+    environment: integration
+    env:
+      TEST_SUITE: soak
+      DETACH_RUNNER: true
+      INTERNAL_DOCKER_REPO: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com
+    permissions:
+      checks: write
+      pull-requests: write
+      id-token: write
+      contents: read
+    steps:
+      - name: Collect Metrics
+        id: collect-gha-metrics
+        uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1
+        with:
+          id: starknet-e2e-soak
+          org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
+          basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
+          hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
+          this-job-name: Run soak Tests
+          test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}'
+        continue-on-error: true
+      - name: Checkout the repo
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      - name: Install Nix
+        uses: cachix/install-nix-action@ba0dd844c9180cbf77aa72a116d6fbc515d0e87b # v27
+        with:
+          nix_path: nixpkgs=channel:nixos-unstable
+      - name: Install Cairo
+        uses: ./.github/actions/install-cairo
+      - name: Build contracts
+        run: |
+          cd contracts && scarb --profile release build
+      - name: Build gauntlet
+        run: |
+          yarn install && yarn build
+      - name: Mask base64 config
+        # shellcheck disable=SC2086
+        run: |
+          BASE64_CONFIG_OVERRIDE=$(jq -r '.inputs.base64_config' "$GITHUB_EVENT_PATH")
+          echo "::add-mask::$BASE64_CONFIG_OVERRIDE"
+          echo "BASE64_CONFIG_OVERRIDE=$BASE64_CONFIG_OVERRIDE" >> "$GITHUB_ENV"
+      - name: Run Tests
+        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@fc3e0df622521019f50d772726d6bf8dc919dd38 # v2.3.19
+        with:
+          aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
+          test_command_to_run: cd ./integration-tests && go test -timeout 24h -count=1 -run TestOCRBasicSoak/embedded ./soak
+          test_download_vendor_packages_command: cd integration-tests && nix develop -c go mod download
+          cl_repo: ${{ env.CL_ECR }}
+          token: ${{ secrets.GITHUB_TOKEN }}
+          go_mod_path: ./integration-tests/go.mod
+          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
+          QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
+          artifacts_location: /home/runner/work/chainlink-starknet/chainlink-starknet/integration-tests/soak/logs
+

integration-tests/README.md

@@ -23,6 +23,30 @@ In order to run the tests on Testnet, additional variables need to be specified
 
 Set `inside_k8 = true` under `[Common]`.
 
-#### Run tests
+#### Run smoke tests
+
+`cd integration-tests && go test --timeout=2h -v -count=1 -json ./smoke`
+
+
+### On demand soak test
+
+Navigate to the [workflow](https://github.com/smartcontractkit/chainlink-starknet/actions/workflows/integration-tests-soak.yml). The workflow takes in 3 parameters:
+
+- Base64 string of the .toml configuration
+- Core image tag which defaults to develop
+- Test runner tag, only tag needs to be supplied
+
+Create an `overrides.toml` file in `integration-tests/testconfig` and run `cat overrides.toml | base64`. `inside_k8` needs to be set to true in the .toml in order to run the tests in kubernetes.
+
+#### Local
+
+If you want to kick off the test from local:
+
+- `export TEST_SUITE: soak`
+- `export DETACH_RUNNER: true`
+- `export ENV_JOB_IMAGE: <internal_repo>/chainlink-solana-tests:<tag>`
+- Base64 the .toml config
+- Run `export BASE64_CONFIG_OVERRIDE="<config>"`
+- `cd integration-tests/soak && go test -timeout 24h -count=1 -run TestOCRBasicSoak/embedded -test.timeout 30m;`
+
 
-`cd integration-tests && go test --timeout=2h -v -count=1 -json ./smoke`

integration-tests/common/common.go

@@ -133,7 +133,7 @@ func (c *Common) Default(t *testing.T, namespacePrefix string) (*Common, error)
 			},
 			"db": map[string]any{
 				"image": map[string]any{
-					"version": "15.5",
+					"version": *c.TestConfig.Common.PostgresVersion,
 				},
 				"stateful": c.TestConfig.Common.Stateful,
 			},

integration-tests/common/test_common.go

@@ -133,11 +133,20 @@ func (m *OCRv2TestState) DeployCluster() {
 	// When running soak we need to use K8S
 	if *m.Common.TestConfig.Common.InsideK8s {
 		m.DeployEnv()
+
 		if m.Common.Env.WillUseRemoteRunner() {
 			return
 		}
-		// Setting RPC details
+
 		m.Common.RPCDetails.RPCL2External = m.Common.Env.URLs["starknet-dev"][0]
+
+		// Checking whether we are running in a remote runner since the forwarding is not working there and we need the public IP
+		// In that case it is http://127.0.0.1:0 so we do a check and get the public IP
+		if m.Common.RPCDetails.RPCL2External == "http://127.0.0.1:0" {
+			m.Common.RPCDetails.RPCL2External = m.Common.Env.URLs["starknet-dev"][1]
+		}
+
+		// Setting RPC details
 		if *m.Common.TestConfig.Common.Network == "testnet" {
 			m.Common.RPCDetails.RPCL2External = *m.Common.TestConfig.Common.L2RPCUrl
 			m.Common.RPCDetails.RPCL2Internal = *m.Common.TestConfig.Common.L2RPCUrl
@@ -189,12 +198,15 @@ func (m *OCRv2TestState) DeployCluster() {
 	}
 
 	m.TestConfig.Resty = resty.New().SetBaseURL(m.Common.RPCDetails.RPCL2External)
-	m.SetupClients()
+
 	if *m.Common.TestConfig.Common.InsideK8s {
+		m.ChainlinkNodesK8s, m.TestConfig.err = client.ConnectChainlinkNodes(m.Common.Env)
+		require.NoError(m.TestConfig.T, m.TestConfig.err)
 		m.Clients.ChainlinkClient.ChainlinkNodes = m.GetChainlinkNodes()
 		m.Clients.ChainlinkClient.NKeys, m.TestConfig.err = m.Common.CreateNodeKeysBundle(m.Clients.ChainlinkClient.ChainlinkNodes)
 		require.NoError(m.TestConfig.T, m.TestConfig.err)
 	} else {
+		m.Clients.ChainlinkClient.ChainlinkNodes = m.Clients.DockerEnv.ClCluster.NodeAPIs()
 		m.Clients.ChainlinkClient.NKeys, m.TestConfig.err = m.Common.CreateNodeKeysBundle(m.Clients.DockerEnv.ClCluster.NodeAPIs())
 		require.NoError(m.TestConfig.T, m.TestConfig.err)
 	}
@@ -225,16 +237,6 @@ func (m *OCRv2TestState) DeployEnv() {
 	require.NoError(m.TestConfig.T, err)
 }
 
-// SetupClients Sets up the starknet client
-func (m *OCRv2TestState) SetupClients() {
-	if *m.Common.TestConfig.Common.InsideK8s {
-		m.ChainlinkNodesK8s, m.TestConfig.err = client.ConnectChainlinkNodes(m.Common.Env)
-		require.NoError(m.TestConfig.T, m.TestConfig.err)
-	} else {
-		m.Clients.ChainlinkClient.ChainlinkNodes = m.Clients.DockerEnv.ClCluster.NodeAPIs()
-	}
-}
-
 // LoadOCR2Config Loads and returns the default starknet gauntlet config
 func (m *OCRv2TestState) LoadOCR2Config() (*ops.OCR2Config, error) {
 	var offChaiNKeys []string

integration-tests/smoke/ocr2_test.go

@@ -82,7 +82,7 @@ func TestOCRBasic(t *testing.T) {
 
 			state.SetUpNodes()
 
-			err = state.ValidateRounds(*config.OCR2.Smoke.NumberOfRounds, false)
+			err = state.ValidateRounds(*config.OCR2.NumberOfRounds, false)
 			require.NoError(t, err, "Validating round should not fail")
 		})
 	}

integration-tests/soak/ocr2_test.go

@@ -1,54 +1,94 @@
-package soak_test
-
-//import (
-//"flag"
-//"fmt"
-//"testing"
-
-//"github.com/stretchr/testify/require"
-//"go.uber.org/zap/zapcore"
-
-//"github.com/smartcontractkit/chainlink-starknet/integration-tests/common"
-//"github.com/smartcontractkit/chainlink-starknet/ops/gauntlet"
-//"github.com/smartcontractkit/chainlink-starknet/ops/utils"
-
-//"github.com/smartcontractkit/chainlink/integration-tests/actions"
-//)
-
-//var (
-//keepAlive bool
-//err       error
-//testState *common.Test
-//decimals  = 9
-//)
-
-//func init() {
-//flag.BoolVar(&keepAlive, "keep-alive", false, "enable to keep the cluster alive")
-//}
-//func TestOCRSoak(t *testing.T) {
-//testState = &common.Test{
-//T: t,
-//}
-//testState.Common = common.New(t)
-//// Setting this to the root of the repo for cmd exec func for Gauntlet
-//testState.Sg, err = gauntlet.NewStarknetGauntlet(fmt.Sprintf("%s/", utils.ProjectRoot))
-//require.NoError(t, err, "Could not get a new gauntlet struct")
-//testState.DeployCluster()
-//require.NoError(t, err, "Deploying cluster should not fail")
-//if testState.Common.Env.WillUseRemoteRunner() {
-//return // short circuit here if using a remote runner
-//}
-//err = testState.Sg.SetupNetwork(testState.Common.L2RPCUrl)
-//require.NoError(t, err, "Setting up network should not fail")
-//err = testState.DeployGauntlet(0, 100000000000, decimals, "auto", 1, 1)
-//require.NoError(t, err, "Deploying contracts should not fail")
-//if !testState.Common.Testnet {
-//testState.Devnet.AutoLoadState(testState.OCR2Client, testState.OCRAddr)
-//}
-//err = testState.ValidateRounds(99999999, true)
-//require.NoError(t, err, "Validating round should not fail")
-//t.Cleanup(func() {
-//err = actions.TeardownSuite(t, testState.Common.Env, testState.Cc.ChainlinkNodes, nil, zapcore.ErrorLevel, nil, nil)
-//require.NoError(t, err, "Error tearing down environment")
-//})
-//}
+package smoke_test
+
+import (
+	"flag"
+	"fmt"
+	"maps"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"go.uber.org/zap/zapcore"
+
+	"github.com/smartcontractkit/chainlink-testing-framework/logging"
+	"github.com/smartcontractkit/chainlink/integration-tests/actions"
+	"github.com/smartcontractkit/chainlink/integration-tests/docker/test_env"
+
+	"github.com/smartcontractkit/chainlink-starknet/integration-tests/common"
+	tc "github.com/smartcontractkit/chainlink-starknet/integration-tests/testconfig"
+	"github.com/smartcontractkit/chainlink-starknet/ops/gauntlet"
+	"github.com/smartcontractkit/chainlink-starknet/ops/utils"
+)
+
+var (
+	keepAlive bool
+	decimals  = 9
+)
+
+func init() {
+	flag.BoolVar(&keepAlive, "keep-alive", false, "enable to keep the cluster alive")
+}
+
+func TestOCRBasicSoak(t *testing.T) {
+	for _, test := range []struct {
+		name string
+		env  map[string]string
+	}{
+		{name: "embedded"},
+		{name: "plugins", env: map[string]string{
+			"CL_MEDIAN_CMD": "chainlink-feeds",
+			"CL_SOLANA_CMD": "chainlink-solana",
+		}},
+	} {
+		config, err := tc.GetConfig("Soak", tc.OCR2)
+		if err != nil {
+			t.Fatal(err)
+		}
+		err = os.Setenv("CHAINLINK_ENV_USER", *config.Common.User)
+		require.NoError(t, err, "Could not set CHAINLINK_ENV_USER")
+		err = os.Setenv("INTERNAL_DOCKER_REPO", *config.Common.InternalDockerRepo)
+		require.NoError(t, err, "Could not set INTERNAL_DOCKER_REPO")
+		test := test
+		t.Run(test.name, func(t *testing.T) {
+			t.Parallel()
+			logging.Init()
+			//
+			state, err := common.NewOCRv2State(t, "soak-ocr2", &config)
+			require.NoError(t, err, "Could not setup the ocrv2 state")
+
+			// K8s specific config and cleanup
+			if *config.Common.InsideK8s {
+				t.Cleanup(func() {
+					if err = actions.TeardownSuite(t, state.Common.Env, state.ChainlinkNodesK8s, nil, zapcore.PanicLevel, nil); err != nil {
+						state.TestConfig.L.Error().Err(err).Msg("Error tearing down environment")
+					}
+				})
+			}
+			if len(test.env) > 0 {
+				state.Common.TestEnvDetails.NodeOpts = append(state.Common.TestEnvDetails.NodeOpts, func(n *test_env.ClNode) {
+					if n.ContainerEnvs == nil {
+						n.ContainerEnvs = map[string]string{}
+					}
+					maps.Copy(n.ContainerEnvs, test.env)
+				})
+			}
+			state.DeployCluster()
+
+			if state.Common.Env.WillUseRemoteRunner() {
+				return
+			}
+
+			state.Clients.GauntletClient, err = gauntlet.NewStarknetGauntlet(fmt.Sprintf("%s/", utils.ProjectRoot))
+			require.NoError(t, err, "Setting up gauntlet should not fail")
+			err = state.Clients.GauntletClient.SetupNetwork(state.Common.RPCDetails.RPCL2External, state.Account.Account, state.Account.PrivateKey)
+			require.NoError(t, err, "Setting up gauntlet network should not fail")
+			err = state.DeployGauntlet(0, 100000000000, decimals, "auto", 1, 1)
+			require.NoError(t, err, "Deploying contracts should not fail")
+
+			state.SetUpNodes()
+
+			err = state.ValidateRounds(*config.OCR2.NumberOfRounds, true)
+			require.NoError(t, err, "Validating round should not fail")
+		})
+	}
+}

integration-tests/testconfig/default.toml

@@ -27,10 +27,12 @@ network = "localnet"
 user = "satoshi"
 stateful_db = false
 devnet_image = "shardlabs/starknet-devnet-rs:a147b4cd72f9ce9d1fa665d871231370db0f51c7"
+postgres_version = "15.7"
 
 [OCR2]
 node_count = 6
 test_duration = "30m"
+number_of_rounds = 10
 
 [OCR2.Smoke]
-number_of_rounds = 10
+enabled = true

integration-tests/testconfig/ocr2/ocr2.go

@@ -5,9 +5,9 @@ import (
 )
 
 type Config struct {
-	Smoke        *SmokeConfig `toml:"Smoke"`
-	NodeCount    *int         `toml:"node_count"`
-	TestDuration *string      `toml:"test_duration"`
+	NumberOfRounds *int    `toml:"number_of_rounds"`
+	NodeCount      *int    `toml:"node_count"`
+	TestDuration   *string `toml:"test_duration"`
 }
 
 func (o *Config) Validate() error {
@@ -19,24 +19,9 @@ func (o *Config) Validate() error {
 		return errors.New("test_duration must be set")
 	}
 
-	if o.Smoke == nil {
-		return errors.New("smoke must be defined")
-	}
-	err := o.Smoke.Validate()
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-type SmokeConfig struct {
-	NumberOfRounds *int `toml:"number_of_rounds"`
-}
-
-func (o *SmokeConfig) Validate() error {
 	if o.NumberOfRounds == nil {
 		return errors.New("number_of_rounds must be set")
 	}
+
 	return nil
 }