Merge branch 'develop' into feat/gha-commit-signing

.github/workflows/e2e_custom_cl.yml

@@ -129,6 +129,8 @@ jobs:
         with:
           name: artifacts
           path: ${{ env.CONTRACT_ARTIFACTS_PATH }}
+      - name: Install Solana CLI # required for ensuring the local test validator is configured correctly
+        run: ./scripts/install-solana-ci.sh
       - name: Generate config overrides
         run: | # https://github.com/smartcontractkit/chainlink-testing-framework/blob/main/config/README.md
           cat << EOF > config.toml

integration-tests/common/test_common.go

@@ -312,6 +312,7 @@ func (m *OCRv2TestState) DeployContracts(contractsDir string) {
 	}
 	require.NoError(m.T, err)
 	cd.RegisterAnchorPrograms()
+	require.NoError(m.T, cd.ValidateProgramsDeployed())
 	m.Client.LinkToken, err = cd.DeployLinkTokenContract()
 	require.NoError(m.T, err)
 	err = FundOracles(m.Client, m.NodeKeysBundle, big.NewFloat(1e4))

integration-tests/docker/test_env/sol.go

@@ -2,8 +2,11 @@ package test_env
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"os"
+	"os/exec"
+	"strings"
 	"testing"
 	"time"
 
@@ -12,6 +15,7 @@ import (
 	"github.com/rs/zerolog/log"
 	tc "github.com/testcontainers/testcontainers-go"
 	tcwait "github.com/testcontainers/testcontainers-go/wait"
+	"golang.org/x/exp/slices"
 
 	"github.com/smartcontractkit/chainlink-solana/integration-tests/utils"
 	"github.com/smartcontractkit/chainlink-testing-framework/docker/test_env"
@@ -75,7 +79,14 @@ func (s *Solana) StartContainer() error {
 			L: s.l,
 		}
 	}
-	cReq, err := s.getContainerRequest()
+
+	// get disabled/unreleased features on mainnet
+	inactiveMainnetFeatures, err := GetInactiveFeatureHashes("mainnet-beta")
+	if err != nil {
+		return err
+	}
+
+	cReq, err := s.getContainerRequest(inactiveMainnetFeatures)
 	if err != nil {
 		return err
 	}
@@ -114,10 +125,18 @@ func (s *Solana) StartContainer() error {
 		Str("containerName", s.ContainerName).
 		Msgf("Started Solana container")
 
+	// validate features are properly set
+	inactiveLocalFeatures, err := GetInactiveFeatureHashes(s.ExternalHttpUrl)
+	if err != nil {
+		return err
+	}
+	if !slices.Equal(inactiveMainnetFeatures, inactiveLocalFeatures) {
+		return fmt.Errorf("Localnet features does not match mainnet features")
+	}
 	return nil
 }
 
-func (ms *Solana) getContainerRequest() (*tc.ContainerRequest, error) {
+func (ms *Solana) getContainerRequest(inactiveFeatures InactiveFeatures) (*tc.ContainerRequest, error) {
 	configYml, err := os.CreateTemp("", "config.yml")
 	if err != nil {
 		return nil, err
@@ -138,7 +157,7 @@ func (ms *Solana) getContainerRequest() (*tc.ContainerRequest, error) {
 
 	return &tc.ContainerRequest{
 		Name:         ms.ContainerName,
-		Image:        "solanalabs/solana:v1.13.3",
+		Image:        "solanalabs/solana:v1.17.22",
 		ExposedPorts: []string{test_env.NatPortFormat(SOL_HTTP_PORT), test_env.NatPortFormat(SOL_WS_PORT)},
 		Env: map[string]string{
 			"SERVER_PORT": "1080",
@@ -169,6 +188,49 @@ func (ms *Solana) getContainerRequest() (*tc.ContainerRequest, error) {
 				},
 			},
 		},
-		Entrypoint: []string{"sh", "-c", "mkdir -p /root/.config/solana/cli && solana-test-validator -r --mint AAxAoGfkbWnbgsiQeAanwUvjv6bQrM5JS8Vxv1ckzVxg"},
+		Entrypoint: []string{"sh", "-c", "mkdir -p /root/.config/solana/cli && solana-test-validator -r --mint=AAxAoGfkbWnbgsiQeAanwUvjv6bQrM5JS8Vxv1ckzVxg " + inactiveFeatures.CLIString()},
 	}, nil
 }
+
+type FeatureStatuses struct {
+	Features []FeatureStatus
+	// note: there are other unused params in the json response
+}
+
+type FeatureStatus struct {
+	ID          string
+	Description string
+	Status      string
+	SinceSlot   int
+}
+
+type InactiveFeatures []string
+
+func (f InactiveFeatures) CLIString() string {
+	return "--deactivate-feature=" + strings.Join(f, " --deactivate-feature=")
+}
+
+// GetInactiveFeatureHashes uses the solana CLI to fetch inactive solana features
+// This is used in conjuction with the solana-test-validator command to produce a solana network that has the same features as mainnet
+// the solana-test-validator has all features on by default (released + unreleased)
+func GetInactiveFeatureHashes(url string) (output InactiveFeatures, err error) {
+	cmd := exec.Command("solana", "feature", "status", "-u="+url, "--output=json") // -um is for mainnet url
+	stdout, err := cmd.Output()
+	if err != nil {
+		return nil, fmt.Errorf("Failed to get feature status: %w", err)
+	}
+
+	statuses := FeatureStatuses{}
+	if err = json.Unmarshal(stdout, &statuses); err != nil {
+		return nil, fmt.Errorf("Failed to unmarshal feature status: %w", err)
+	}
+
+	for _, f := range statuses.Features {
+		if f.Status == "inactive" {
+			output = append(output, f.ID)
+		}
+	}
+
+	slices.Sort(output)
+	return output, err
+}

integration-tests/go.mod

@@ -21,7 +21,8 @@ require (
 	github.com/stretchr/testify v1.8.4
 	github.com/testcontainers/testcontainers-go v0.23.0
 	go.uber.org/zap v1.26.0
-	golang.org/x/crypto v0.18.0
+	golang.org/x/crypto v0.19.0
+	golang.org/x/exp v0.0.0-20240213143201-ec583247a57a
 	golang.org/x/sync v0.6.0
 	gopkg.in/guregu/null.v4 v4.0.0
 )
@@ -420,15 +421,14 @@ require (
 	go.uber.org/ratelimit v0.2.0 // indirect
 	go4.org/netipx v0.0.0-20230125063823-8449b0a6169f // indirect
 	golang.org/x/arch v0.6.0 // indirect
-	golang.org/x/exp v0.0.0-20231127185646-65229373498e // indirect
-	golang.org/x/mod v0.14.0 // indirect
-	golang.org/x/net v0.20.0 // indirect
+	golang.org/x/mod v0.15.0 // indirect
+	golang.org/x/net v0.21.0 // indirect
 	golang.org/x/oauth2 v0.15.0 // indirect
-	golang.org/x/sys v0.16.0 // indirect
-	golang.org/x/term v0.16.0 // indirect
+	golang.org/x/sys v0.17.0 // indirect
+	golang.org/x/term v0.17.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
-	golang.org/x/tools v0.16.0 // indirect
+	golang.org/x/tools v0.18.0 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect
 	gonum.org/v1/gonum v0.14.0 // indirect
 	google.golang.org/appengine v1.6.8 // indirect

integration-tests/go.sum

@@ -1705,8 +1705,8 @@ golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0
 golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
 golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
 golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
-golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc=
-golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg=
+golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo=
+golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -1717,8 +1717,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
-golang.org/x/exp v0.0.0-20231127185646-65229373498e h1:Gvh4YaCaXNs6dKTlfgismwWZKyjVZXwOPfIyUaqU3No=
-golang.org/x/exp v0.0.0-20231127185646-65229373498e/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI=
+golang.org/x/exp v0.0.0-20240213143201-ec583247a57a h1:HinSgX1tJRX3KsL//Gxynpw5CTOAIPhgL4W8PNiIpVE=
+golang.org/x/exp v0.0.0-20240213143201-ec583247a57a/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -1745,8 +1745,8 @@ golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0=
-golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/mod v0.15.0 h1:SernR4v+D55NyBH2QiEQrlBAnj1ECL6AGrA5+dPaMY8=
+golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -1808,8 +1808,8 @@ golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
 golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
-golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo=
-golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY=
+golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
+golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -1935,8 +1935,8 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
-golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
+golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
@@ -1946,8 +1946,8 @@ golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
 golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
-golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE=
-golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY=
+golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U=
+golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -2036,8 +2036,8 @@ golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
 golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
-golang.org/x/tools v0.16.0 h1:GO788SKMRunPIBCXiQyo2AaexLstOrVhuAL5YwsckQM=
-golang.org/x/tools v0.16.0/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0=
+golang.org/x/tools v0.18.0 h1:k8NLag8AGHnn+PHbl7g43CtqZAwG60vZkLqgyZgIHgQ=
+golang.org/x/tools v0.18.0/go.mod h1:GL7B4CwcLLeo59yx/9UWWuNOW1n3VZ4f5axWfML7Lcg=
 golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

integration-tests/solclient/deployer.go

@@ -3,6 +3,7 @@ package solclient
 import (
 	"context"
 	"fmt"
+	"io"
 	"path/filepath"
 	"strings"
 
@@ -39,6 +40,7 @@ const (
 	OCROffChainConfigSize            = uint64(8 + 4096 + 8)
 	OCRConfigSize                    = 32 + 32 + 32 + 32 + 32 + 32 + 16 + 16 + (1 + 1 + 2 + 4 + 4 + 32) + (4 + 32 + 8) + (4 + 4)
 	OCRAccountSize                   = Discriminator + 1 + 1 + 2 + 4 + solana.PublicKeyLength + OCRConfigSize + OCROffChainConfigSize + OCROraclesSize
+	keypairSuffix                    = "-keypair.json"
 )
 
 type Authority struct {
@@ -104,7 +106,7 @@ func (c *ContractDeployer) SetupAssociatedAccount() (*solana.PublicKey, *solana.
 	c.Accounts.OCRVaultAssociatedPubKey = aaccount
 	err := c.Client.TXSync(
 		"Setup associated account",
-		rpc.CommitmentFinalized,
+		rpc.CommitmentConfirmed,
 		instr,
 		func(key solana.PublicKey) *solana.PrivateKey {
 			if key.Equals(payer.PublicKey()) {
@@ -200,7 +202,7 @@ func (c *ContractDeployer) CreateFeed(desc string, decimals uint8, granularity i
 	}
 	err = c.Client.TXSync(
 		"Create feed",
-		rpc.CommitmentFinalized,
+		rpc.CommitmentConfirmed,
 		[]solana.Instruction{
 			feedAccInstruction,
 			store2.NewCreateFeedInstruction(
@@ -252,12 +254,9 @@ func (c *ContractDeployer) DeployLinkTokenContract() (*LinkToken, error) {
 		return nil, err
 	}
 	err = c.Client.TXAsync(
-		"Createing LINK Token and associated accounts",
+		"Creating LINK Token and associated accounts",
 		instr,
 		func(key solana.PublicKey) *solana.PrivateKey {
-			if key.Equals(c.Accounts.OCRVault.PublicKey()) {
-				return &c.Accounts.OCRVault.PrivateKey
-			}
 			if key.Equals(c.Accounts.Mint.PublicKey()) {
 				return &c.Accounts.Mint.PrivateKey
 			}
@@ -323,7 +322,7 @@ func (c *ContractDeployer) InitOCR2(billingControllerAddr string, requesterContr
 			Build())
 	err = c.Client.TXSync(
 		"Initializing OCRv2",
-		rpc.CommitmentFinalized,
+		rpc.CommitmentConfirmed,
 		instr,
 		func(key solana.PublicKey) *solana.PrivateKey {
 			if key.Equals(payer.PublicKey()) {
@@ -361,9 +360,9 @@ func (c *ContractDeployer) InitOCR2(billingControllerAddr string, requesterContr
 func (c *ContractDeployer) DeployProgramRemote(programName string, env *environment.Environment) error {
 	log.Debug().Str("Program", programName).Msg("Deploying program")
 	programPath := filepath.Join("programs", programName)
-	programKeyFileName := strings.Replace(programName, ".so", "-keypair.json", -1)
+	programKeyFileName := strings.Replace(programName, ".so", keypairSuffix, -1)
 	programKeyFilePath := filepath.Join("programs", programKeyFileName)
-	cmd := fmt.Sprintf("solana deploy %s %s", programPath, programKeyFilePath)
+	cmd := fmt.Sprintf("solana program deploy --program-id %s %s", programKeyFilePath, programPath)
 	pl, err := env.Client.ListPods(env.Cfg.Namespace, "app=sol")
 	if err != nil {
 		return err
@@ -374,13 +373,21 @@ func (c *ContractDeployer) DeployProgramRemote(programName string, env *environm
 }
 
 func (c *ContractDeployer) DeployProgramRemoteLocal(programName string, sol *test_env_sol.Solana) error {
-	log.Debug().Str("Program", programName).Msg("Deploying program")
+	log.Info().Str("Program", programName).Msg("Deploying program")
 	programPath := filepath.Join("programs", programName)
-	programKeyFileName := strings.Replace(programName, ".so", "-keypair.json", -1)
+	programKeyFileName := strings.Replace(programName, ".so", keypairSuffix, -1)
 	programKeyFilePath := filepath.Join("programs", programKeyFileName)
-	cmd := fmt.Sprintf("solana deploy %s %s", programPath, programKeyFilePath)
-	_, _, err := sol.Container.Exec(context.Background(), strings.Split(cmd, " "))
-	return err
+	cmd := fmt.Sprintf("solana program deploy --program-id %s %s", programKeyFilePath, programPath)
+	_, res, err := sol.Container.Exec(context.Background(), strings.Split(cmd, " "))
+	if err != nil {
+		return err
+	}
+	out, err := io.ReadAll(res)
+	if err != nil {
+		return err
+	}
+	log.Info().Str("Output", string(out)).Msg("Deploying " + programName)
+	return nil
 }
 
 func (c *ContractDeployer) DeployOCRv2AccessController() (*AccessController, error) {
@@ -431,6 +438,47 @@ func (c *ContractDeployer) RegisterAnchorPrograms() {
 	ocr_2.SetProgramID(c.Client.ProgramWallets["ocr2-keypair.json"].PublicKey())
 }
 
+func (c *ContractDeployer) ValidateProgramsDeployed() error {
+	keys := []solana.PublicKey{}
+	names := []string{}
+	for i := range c.Client.ProgramWallets {
+		keys = append(keys, c.Client.ProgramWallets[i].PublicKey())
+		names = append(names, strings.TrimSuffix(i, keypairSuffix))
+	}
+
+	res, err := c.Client.RPC.GetMultipleAccountsWithOpts(
+		context.Background(),
+		keys,
+		&rpc.GetMultipleAccountsOpts{
+			Commitment: rpc.CommitmentConfirmed,
+		},
+	)
+	if err != nil {
+		return fmt.Errorf("failed to get accounts: %w", err)
+	}
+
+	output := []string{}
+	invalid := false
+	for i := range res.Value {
+		if res.Value[i] == nil {
+			invalid = true
+			output = append(output, fmt.Sprintf("%s=nil", names[i]))
+			continue
+		}
+		if !res.Value[i].Executable {
+			invalid = true
+			output = append(output, fmt.Sprintf("%s=notProgram(%s)", names[i], keys[i].String()))
+			continue
+		}
+		output = append(output, fmt.Sprintf("%s=valid(%s)", names[i], keys[i].String()))
+	}
+
+	if invalid {
+		return fmt.Errorf("Programs not deployed: %s", strings.Join(output, " "))
+	}
+	return nil
+}
+
 func (c *ContractDeployer) LoadPrograms(contractsDir string) error {
 	keyFiles, err := c.Client.ListDirFilenamesByExt(contractsDir, ".json")
 	if err != nil {