e2e: test program upgrade

.github/workflows/e2e_custom_cl.yml

@@ -395,22 +395,20 @@ jobs:
           # shellcheck disable=SC2086
           echo "BASE64_CONFIG_OVERRIDE=$BASE64_CONFIG_OVERRIDE" >> $GITHUB_ENV
       - name: Run Upgrade Test
-        run: |
-          echo "TODO: implement OCR2 upgrade test!"
-      #   uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@fc3e0df622521019f50d772726d6bf8dc919dd38 # v2.3.19
-      #   with:
-      #     test_command_to_run: cd ./integration-tests && go test -timeout 24h -count=1 -run TestSolanaOCRV2Smoke -json $(args) ./smoke 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci= -singlepackage=true -hidepassingtests=false -hidepassinglogs=false
-      #     test_download_vendor_packages_command: cd ./integration-tests && go mod download
-      #     download_contract_artifacts_path: ${{ env.CONTRACT_ARTIFACTS_PATH }} 
-      #     go_mod_path: ./integration-tests/go.mod
-      #     cl_repo: ${{ env.CL_ECR }}
-      #     cl_image_tag: solana.${{ env.CUSTOM_CORE_REF || github.event.inputs.cl_branch_ref || github.sha }}
-      #     token: ${{ secrets.GITHUB_TOKEN }}
-      #     aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
-      #     artifacts_name: smoke-test-logs
-      #     artifacts_location: ./integration-tests/smoke/logs/
-      #     QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
-      #     QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
-      #     QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
-      #     cache_key_id: solana-e2e-${{ env.MOD_CACHE_VERSION }}
-      #     cache_restore_only: "false"
+        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@fc3e0df622521019f50d772726d6bf8dc919dd38 # v2.3.19
+        with:
+          test_command_to_run: cd ./integration-tests && go test -timeout 24h -count=1 -run TestSolanaOCRV2UpgradeSmoke -json $(args) ./smoke 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci=true -singlepackage=true -hidepassingtests=false -hidepassinglogs=false
+          test_download_vendor_packages_command: cd ./integration-tests && go mod download
+          download_contract_artifacts_path: ${{ env.CONTRACT_ARTIFACTS_PATH }} 
+          go_mod_path: ./integration-tests/go.mod
+          cl_repo: ${{ env.CL_ECR }}
+          cl_image_tag: solana.${{ env.CUSTOM_CORE_REF || github.event.inputs.cl_branch_ref || github.sha }}
+          token: ${{ secrets.GITHUB_TOKEN }}
+          aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
+          artifacts_name: smoke-test-logs
+          artifacts_location: ./integration-tests/smoke/logs/
+          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
+          QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
+          cache_key_id: solana-e2e-${{ env.MOD_CACHE_VERSION }}
+          cache_restore_only: "false"

integration-tests/common/test_common.go

@@ -4,6 +4,8 @@ import (
 	"fmt"
 	"math/big"
 	"net/http"
+	"path/filepath"
+	"strings"
 	"testing"
 	"time"
 
@@ -221,16 +223,44 @@ func (m *OCRv2TestState) SetupClients() {
 }
 
 // DeployContracts deploys contracts
-func (m *OCRv2TestState) DeployContracts(contractsDir string) {
+// baseDir is the root folder where contracts are stored
+// subDir allows for pointing to a subdirectory within baseDir (can be left empty)
+func (m *OCRv2TestState) DeployContracts(baseDir, subDir string) {
 	var err error
 	m.Clients.ChainlinkClient.NKeys, err = m.Common.CreateNodeKeysBundle(m.Clients.ChainlinkClient.ChainlinkNodes)
 	require.NoError(m.Config.T, err)
 	cd, err := solclient.NewContractDeployer(m.Clients.SolanaClient, nil)
 	require.NoError(m.Config.T, err)
 	if *m.Config.TestConfig.Common.InsideK8s {
-		err = cd.DeployAnchorProgramsRemote(contractsDir, m.Common.Env)
+		err = cd.DeployAnchorProgramsRemote(baseDir, m.Common.Env)
 	} else {
-		err = cd.DeployAnchorProgramsRemoteDocker(contractsDir, m.Common.DockerEnv.Sol)
+		err = cd.DeployAnchorProgramsRemoteDocker(baseDir, subDir, m.Common.DockerEnv.Sol, solclient.BuildProgramIDKeypairPath)
+	}
+	require.NoError(m.Config.T, err)
+}
+
+func (m *OCRv2TestState) UpgradeContracts(baseDir, subDir string) {
+	cd, err := solclient.NewContractDeployer(m.Clients.SolanaClient, nil)
+	require.NoError(m.Config.T, err)
+
+	// fetch corresponding program address for program
+	programIDBuilder := func(programName string) string {
+		// remove extra directories + .so suffix from lookup
+		programName, _ = strings.CutSuffix(filepath.Base(programName), ".so")
+		ids := map[string]string{
+			"ocr_2":             m.Common.ChainDetails.ProgramAddresses.OCR2,
+			"access_controller": m.Common.ChainDetails.ProgramAddresses.AccessController,
+			"store":             m.Common.ChainDetails.ProgramAddresses.Store,
+		}
+		val, ok := ids[programName]
+		require.True(m.Config.T, ok, fmt.Sprintf("unable to find corresponding key (%s) within %+v", programName, ids))
+		return val
+	}
+
+	if *m.Config.TestConfig.Common.InsideK8s {
+		err = fmt.Errorf("not implemented")
+	} else {
+		err = cd.DeployAnchorProgramsRemoteDocker(baseDir, subDir, m.Common.DockerEnv.Sol, programIDBuilder)
 	}
 	require.NoError(m.Config.T, err)
 }

integration-tests/smoke/ocr2_test.go

@@ -38,13 +38,13 @@ func TestSolanaOCRV2Smoke(t *testing.T) {
 		test := test
 		t.Run(test.name, func(t *testing.T) {
 			t.Parallel()
-			_, sg := startOCR2DataFeedsSmokeTest(t, test.name, test.env, config)
+			_, sg := startOCR2DataFeedsSmokeTest(t, test.name, test.env, config, "")
 			validateRounds(t, test.name, sg, *config.OCR2.NumberOfRounds)
 		})
 	}
 }
 
-func startOCR2DataFeedsSmokeTest(t *testing.T, testname string, testenv map[string]string, config tc.TestConfig) (*common.OCRv2TestState, *gauntlet.SolanaGauntlet) {
+func startOCR2DataFeedsSmokeTest(t *testing.T, testname string, testenv map[string]string, config tc.TestConfig, subDir string) (*common.OCRv2TestState, *gauntlet.SolanaGauntlet) {
 	name := "gauntlet-" + testname
 	state, err := common.NewOCRv2State(t, 1, name, &config)
 	require.NoError(t, err, "Could not setup the ocrv2 state")
@@ -58,7 +58,7 @@ func startOCR2DataFeedsSmokeTest(t *testing.T, testname string, testenv map[stri
 	}
 
 	state.DeployCluster(utils.ContractsDir)
-	state.DeployContracts(utils.ContractsDir)
+	state.DeployContracts(utils.ContractsDir, subDir)
 	if state.Common.Env.WillUseRemoteRunner() {
 		return state, nil
 	}

integration-tests/smoke/ocr2upgrade_test.go

@@ -0,0 +1,32 @@
+package smoke
+
+import (
+	"testing"
+
+	"github.com/rs/zerolog/log"
+
+	tc "github.com/smartcontractkit/chainlink-solana/integration-tests/testconfig"
+	"github.com/smartcontractkit/chainlink-solana/integration-tests/utils"
+)
+
+func TestSolanaOCRV2UpgradeSmoke(t *testing.T) {
+	name := "plugins-program-upgrade"
+	env := map[string]string{
+		"CL_MEDIAN_CMD": "chainlink-feeds",
+		"CL_SOLANA_CMD": "chainlink-solana",
+	}
+	config, err := tc.GetConfig("Smoke", tc.OCR2)
+	if err != nil {
+		t.Fatal(err)
+	}
+	s, sg := startOCR2DataFeedsSmokeTest(t, name, env, config, "previous")
+	// validate cluster is functioning
+	validateRounds(t, name, sg, *config.OCR2.NumberOfRounds)
+
+	log.Info().Msg("------ REDEPLOYING CONTRACTS ------")
+	s.UpgradeContracts(utils.ContractsDir, "")
+	log.Info().Msg("-----------------------------------")
+
+	// validate cluster is still functioning after program upgrade
+	validateRounds(t, name, sg, *config.OCR2.NumberOfRounds)
+}

integration-tests/solclient/deployer.go

@@ -361,12 +361,19 @@ func (c *ContractDeployer) DeployProgramRemote(programName string, env *environm
 	return nil
 }
 
-func (c *ContractDeployer) DeployProgramRemoteLocal(programName string, sol *test_env_sol.Solana) error {
+func BuildProgramIDKeypairPath(programName string) string {
+	programKeyFileName := strings.Replace(programName, ".so", keypairSuffix, -1)
+	return filepath.Join("programs", programKeyFileName)
+}
+
+// DeployProgramRemoteLocal takes in a programIDBuilder which allows for building the program keypair path from the name or passing the deployed program address
+func (c *ContractDeployer) DeployProgramRemoteLocal(programName string, sol *test_env_sol.Solana, programIDBuilder func(string) string) error {
+
 	log.Info().Str("Program", programName).Msg("Deploying program")
 	programPath := filepath.Join("programs", programName)
-	programKeyFileName := strings.Replace(programName, ".so", keypairSuffix, -1)
-	programKeyFilePath := filepath.Join("programs", programKeyFileName)
-	cmd := fmt.Sprintf("solana program deploy --program-id %s %s", programKeyFilePath, programPath)
+
+	cmd := fmt.Sprintf("solana program deploy --program-id %s %s", programIDBuilder(programName), programPath)
+	log.Info().Str("Cmd", cmd).Msg("Deploying " + programName)
 	_, res, err := sol.Container.Exec(context.Background(), strings.Split(cmd, " "))
 	if err != nil {
 		return err
@@ -505,17 +512,17 @@ func (c *ContractDeployer) DeployAnchorProgramsRemote(contractsDir string, env *
 	return g.Wait()
 }
 
-func (c *ContractDeployer) DeployAnchorProgramsRemoteDocker(contractsDir string, sol *test_env_sol.Solana) error {
-	contractBinaries, err := c.Client.ListDirFilenamesByExt(contractsDir, ".so")
+func (c *ContractDeployer) DeployAnchorProgramsRemoteDocker(baseDir, subDir string, sol *test_env_sol.Solana, programIDBuilder func(string) string) error {
+	contractBinaries, err := c.Client.ListDirFilenamesByExt(filepath.Join(baseDir, subDir), ".so")
 	if err != nil {
 		return err
 	}
-	log.Info().Interface("Binaries", contractBinaries).Msg("Program binaries")
+	log.Info().Interface("Binaries", contractBinaries).Msg(fmt.Sprintf("Program binaries [%s]", filepath.Join("programs", subDir)))
 	g := errgroup.Group{}
 	for _, bin := range contractBinaries {
 		bin := bin
 		g.Go(func() error {
-			return c.DeployProgramRemoteLocal(bin, sol)
+			return c.DeployProgramRemoteLocal(filepath.Join(subDir, bin), sol, programIDBuilder)
 		})
 	}
 	return g.Wait()

integration-tests/solclient/solclient.go

@@ -292,8 +292,8 @@ func (c *Client) AirdropAddresses(addr []string, solAmount uint64) error {
 func (c *Client) ListDirFilenamesByExt(dir string, ext string) ([]string, error) {
 	keyFiles := make([]string, 0)
 	err := filepath.Walk(dir, func(path string, info fs.FileInfo, err error) error {
-		if info.IsDir() {
-			return nil
+		if info.IsDir() && info.Name() != filepath.Base(dir) {
+			return filepath.SkipDir // only check first depth of folders
 		}
 		if filepath.Ext(path) == ext {
 			keyFiles = append(keyFiles, info.Name())