From 59ad9c3e660282d5eedfb530e7103d180d1d30d7 Mon Sep 17 00:00:00 2001
From: Jade Park <jonghyeon.park@smartcontract.com>
Date: Thu, 5 Sep 2024 11:54:52 +0100
Subject: [PATCH] [BCI-4073] Add Solana CLI / Test validator to Nix environment
 (#839)

* feat: solana-test-validator nix flake

* fix: missing semicolon

* feat: Dockerfile for the latest test-validator image

* fix: scripts src path

* fix: copy necessary script to nix bin

* fix: added version prefix

* chore: remove logs from test validator

* feat: added solana cli devshell

* feat: add solana cli env package for ci

* chore: refactor flake

* chore: update structure

* test: solana packages test on ci

* fix: remove command

* test: solana packages test on ci - fixed typo

* test: solana packages test on ci - check connection

* test: solana packages test on ci - check connection

* test: solana packages test on ci - should fail

* fix: static version for localnet

* chore: clean up comments

* fix: remove unnecessary changes

* chore: test default option

* chore: testing ci checks with default devShells

* fix: currying in callPackage

* chore: rename workflow file

* feat: add version update for docker image / nix packages

* chore: update name for context

* chore: manual solana version bump

* chore: update job name

* feat: update nix hashes on version update
---
 .github/workflows/dependency-updates.yml |  2 +-
 .github/workflows/nix-packages-test.yml  | 29 +++++++
 Makefile                                 |  4 +-
 flake.nix                                | 13 +++-
 scripts/setup-localnet/localnet.down.sh  | 15 ++++
 scripts/setup-localnet/localnet.sh       | 62 +++++++++++++++
 scripts/update-solana-nix-hashes.sh      | 51 +++++++++++++
 scripts/update-solana.sh                 | 18 ++++-
 solana.nix                               | 97 ++++++++++++++++++++++++
 9 files changed, 285 insertions(+), 6 deletions(-)
 create mode 100644 .github/workflows/nix-packages-test.yml
 create mode 100644 scripts/setup-localnet/localnet.down.sh
 create mode 100755 scripts/setup-localnet/localnet.sh
 create mode 100755 scripts/update-solana-nix-hashes.sh
 create mode 100644 solana.nix

diff --git a/.github/workflows/dependency-updates.yml b/.github/workflows/dependency-updates.yml
index e7219013d..5d38d5a0b 100644
--- a/.github/workflows/dependency-updates.yml
+++ b/.github/workflows/dependency-updates.yml
@@ -10,7 +10,7 @@ jobs:
     with:
       reviewers: aalu1418
       run: |
-        make upgrade-e2e-solana-image
+        make upgrade-solana-image
         image=$(curl https://api.github.com/repos/solana-labs/solana/releases/latest | jq -r '.tag_name')
 
         # outputs
diff --git a/.github/workflows/nix-packages-test.yml b/.github/workflows/nix-packages-test.yml
new file mode 100644
index 000000000..a1441ec09
--- /dev/null
+++ b/.github/workflows/nix-packages-test.yml
@@ -0,0 +1,29 @@
+on:
+  pull_request:
+  push:
+    branches:
+      - develop
+
+jobs:
+  nix-packages-test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5
+
+      - name: Install Nix
+        uses: cachix/install-nix-action@8887e596b4ee1134dae06b98d573bd674693f47c # v26
+        with:
+          nix_path: nixpkgs=channel:nixos-unstable
+          extra_nix_config: "sandbox = false"
+
+      - name: Build and test solana-test-validator
+        run: nix run .#solana-test-validator
+
+      - name: Build solana-cli-env
+        run: nix build .#solana-cli-env --print-out-paths
+
+      - name: Test solana-cli-shell
+        run: nix develop .#solana-cli
+
+
+        
\ No newline at end of file
diff --git a/Makefile b/Makefile
index 06e3b0077..22f59f596 100644
--- a/Makefile
+++ b/Makefile
@@ -120,8 +120,8 @@ lint-go-integration-tests:
 lint-go-relay:
 	cd ./pkg && golangci-lint --max-issues-per-linter 0 --max-same-issues 0 --color=always --exclude=dot-imports --timeout 10m --out-format checkstyle:golangci-lint-relay-report.xml run
 
-.PHONY: upgrade-e2e-solana-image
-upgrade-e2e-solana-image:
+.PHONY: upgrade-solana-image
+upgrade-solana-image:
 	./scripts/update-solana.sh
 
 .PHONY: update-e2e-core-deps
diff --git a/flake.nix b/flake.nix
index 339966756..0761c5330 100644
--- a/flake.nix
+++ b/flake.nix
@@ -11,7 +11,16 @@
     flake-utils.lib.eachDefaultSystem (system:
       let
         pkgs = import nixpkgs { inherit system; overlays = [ rust-overlay.overlays.default ]; };
+        solanaPkgs = pkgs.callPackage ./solana.nix {};
       in rec {
-        devShell = pkgs.callPackage ./shell.nix {};
-      });
+        devShells = {
+          default = pkgs.callPackage ./shell.nix {};
+          solana-cli = solanaPkgs.solana-cli-shell;
+        };
+
+        packages = {
+          solana-test-validator = solanaPkgs.solana-test-validator;
+          solana-cli-env = solanaPkgs.solana-cli-env;
+        };
+    });
 }
diff --git a/scripts/setup-localnet/localnet.down.sh b/scripts/setup-localnet/localnet.down.sh
new file mode 100644
index 000000000..0987e1855
--- /dev/null
+++ b/scripts/setup-localnet/localnet.down.sh
@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+echo "Cleaning up test validator container.."
+
+echo "Checking for existing 'chainlink-solana.test-validator' docker container..."
+dpid=`docker ps -a | grep chainlink-solana.test-validator | awk '{print $1}'`;
+if [ -z "$dpid" ]
+then
+    echo "No docker test validator container running.";
+else
+    docker kill $dpid;
+    docker rm $dpid;
+fi
+
+echo "Cleanup finished."
\ No newline at end of file
diff --git a/scripts/setup-localnet/localnet.sh b/scripts/setup-localnet/localnet.sh
new file mode 100755
index 000000000..a00857d23
--- /dev/null
+++ b/scripts/setup-localnet/localnet.sh
@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+cpu_struct="linux";
+
+# Clean up first
+bash "$(dirname -- "$0";)/localnet.down.sh"
+
+container_version=v1.18.23
+container_name="chainlink-solana.test-validator"
+
+echo "Starting $container_name@$container_version"
+
+# NOTE: solanalabs/solana docker image only supports linux/amd64
+# https://hub.docker.com/r/solanalabs/solana/tags
+# If you are running on an ARM machine, Following error will be thrown:
+# "Incompatible CPU detected: missing AVX support. Please build from source on the target "
+docker run -d \
+  --platform linux/amd64 \
+  -p 127.0.0.1:8899:8899 \
+  -p 127.0.0.1:8900:8900 \
+  -p 127.0.0.1:9900:9900 \
+  --name "${container_name}" \
+  --entrypoint /bin/sh \
+  "solanalabs/solana:${container_version}" \
+  -c "solana-test-validator && echo 'Validator started successfully'"
+  # 	--network-alias "${container_name}" \
+  # 	--network chainlink \
+
+echo "Waiting for test validator to become ready.."
+start_time=$(date +%s)
+prev_output=""
+while true
+do
+  output=$(docker logs chainlink-solana.test-validator 2>&1)
+  if [[ "${output}" != "${prev_output}" ]]; then
+    echo -n "${output#$prev_output}"
+    prev_output="${output}"
+  fi
+
+  if [[ $output == *"Finalized Slot"* ]]; then
+    echo ""
+    echo "solana-test-validator is ready."
+    exit 0
+  fi
+
+  if [[ $output == *"Incompatible CPU detected"* || $output == *"Aborted"* ]]; then
+    echo ""
+    echo "solanalabs/solana docker image only supports linux/amd64"
+    exit 1
+  fi
+
+  current_time=$(date +%s)
+  elapsed_time=$((current_time - start_time))
+
+  if (( elapsed_time > 10 )); then
+    echo "Error: Command did not become ready within 10 seconds"
+    exit 1
+  fi
+
+  sleep 3
+done
\ No newline at end of file
diff --git a/scripts/update-solana-nix-hashes.sh b/scripts/update-solana-nix-hashes.sh
new file mode 100755
index 000000000..058274c5a
--- /dev/null
+++ b/scripts/update-solana-nix-hashes.sh
@@ -0,0 +1,51 @@
+#!/bin/bash
+
+# Solana Nix Package SHA256 Hash Updater
+#
+# This script updates the SHA256 hashes for Solana binaries in the solana.nix file.
+#
+# Usage: ./update_nix_hashes.sh <version>
+
+set -e
+
+if [ $# -eq 0 ]; then
+  echo "Error: No version provided"
+  echo "Usage: $0 <version>"
+  exit 1
+fi
+
+VERSION=$1
+
+echo "Updating SHA256 hashes for Nix packages (Solana version: $VERSION)"
+
+get_nix_hash() {
+  local url=$1
+  nix hash convert --hash-algo sha256 $(nix-prefetch-url --unpack $url)
+}
+
+linux_hash=$(get_nix_hash https://github.com/anza-xyz/agave/releases/download/${VERSION}/solana-release-x86_64-unknown-linux-gnu.tar.bz2)
+darwin_hash=$(get_nix_hash https://github.com/anza-xyz/agave/releases/download/${VERSION}/solana-release-aarch64-apple-darwin.tar.bz2)
+
+echo "Linux Hash: $linux_hash"
+echo "Darwin Hash: $darwin_hash"
+
+LINUX_START_MARKER="### BEGIN_LINUX_SHA256 ###"
+LINUX_END_MARKER="### END_LINUX_SHA256 ###"
+DARWIN_START_MARKER="### BEGIN_DARWIN_SHA256 ###"
+DARWIN_END_MARKER="### END_DARWIN_SHA256 ###"
+
+if [ "$(uname -s)" = "Darwin" ]; then
+  sed_in_place=(-i '')
+else
+  sed_in_place=(-i)
+fi
+
+sed "${sed_in_place[@]}" \
+  -e "/$LINUX_START_MARKER/,/$LINUX_END_MARKER/ s|sha256 = \"[^\"]*\"|sha256 = \"$linux_hash\"|" \
+  solana.nix
+
+sed "${sed_in_place[@]}" \
+  -e "/$DARWIN_START_MARKER/,/$DARWIN_END_MARKER/ s|sha256 = \"[^\"]*\"|sha256 = \"$darwin_hash\"|" \
+  solana.nix
+
+echo "Updated hashes in solana.nix"
\ No newline at end of file
diff --git a/scripts/update-solana.sh b/scripts/update-solana.sh
index a115fbf53..1d433c7ae 100755
--- a/scripts/update-solana.sh
+++ b/scripts/update-solana.sh
@@ -9,9 +9,17 @@ testVersion=$(grep -oh "solanalabs/solana:v[0-9]*.[0-9]*.[0-9]*" testconfig/defa
 echo "Current E2E Test Version: $testVersion"
 cd ..
 
+localnetVersion=$(grep -oh "container_version=v[0-9]*.[0-9]*.[0-9]*" scripts/setup-localnet/localnet.sh)
+echo "Current Version in Localnet Container: $localnetVersion"
+
+nixVersion=$(grep -oh "version = \"v[0-9]*.[0-9]*.[0-9]*\"" solana.nix)
+echo "Current Version in Nix packages: $nixVersion"
+
 latestTag=$(curl https://api.github.com/repos/solana-labs/solana/releases/latest | jq -r '.tag_name')
 latestVersion="solanalabs/solana:$latestTag"
 latestCLI="release.solana.com/$latestTag"
+latestLocalnet="container_version=$latestTag"
+latestNix="version = \"$latestTag\""
 echo "Latest Solana Mainnet Version: $latestTag"
 
 if [ "$testVersion" = "$latestVersion" ] && [ "$cliVersion" = "$latestCLI" ] ; then
@@ -25,11 +33,19 @@ if [ "$(uname -s)" = "Darwin" ]; then
   sed -i '' -e "s~$cliVersion~$latestCLI~" scripts/install-solana-ci.sh
   cd integration-tests
   sed -i '' -e "s~$testVersion~$latestVersion~" testconfig/default.toml
+  cd ..
+  sed -i '' -e "s~$localnetVersion~$latestLocalnet~" scripts/setup-localnet/localnet.sh
+  sed -i '' -e "s~$nixVersion~$latestNix~" solana.nix
 else
   sed -i -e "s~$cliVersion~$latestCLI~" scripts/install-solana-ci.sh
   cd integration-tests
   sed -i -e "s~$testVersion~$latestVersion~" testconfig/default.toml
+  cd ..
+  sed -i -e "s~$containerVersion~$latestContainer~" scripts/setup-localnet/localnet.sh
+  sed -i -e "s~$nixVersion~$latestNix~" solana.nix
 fi
-cd ..
+
+# Update the Nix hashes
+"$(dirname -- "$0")/update-solana-nix-hashes.sh" "$latestTag"
 
 echo "Done"
diff --git a/solana.nix b/solana.nix
new file mode 100644
index 000000000..f60e2289c
--- /dev/null
+++ b/solana.nix
@@ -0,0 +1,97 @@
+{
+  system ? builtins.currentSystem,
+  pkgs ? import <nixpkgs> { inherit system; },
+}:
+
+# Solana integration
+let
+  version = "v1.18.23";
+  getBinDerivation =
+    {
+      name,
+      filename,
+      sha256,
+    }:
+    pkgs.stdenv.mkDerivation rec {
+      inherit name;
+      url = "https://github.com/anza-xyz/agave/releases/download/${version}/${filename}";
+      src = pkgs.fetchzip {
+        inherit url sha256;
+      };
+
+      installPhase = ''
+        mkdir -p $out/bin
+        ls -lah $src
+        cp -r $src/bin/* $out/bin
+      '';
+    };
+
+  # It provides two derivations, one for x86_64-linux and another for aarch64-apple-darwin.
+  # Each derivation downloads the corresponding Solana release.
+
+  # The SHA256 hashes below are automatically updated by action.(dependency-updates.yml)
+  # The update script(./scripts/update-solana-nix-hashes.sh) looks for the BEGIN and END markers to locate the lines to modify.
+  # Do not modify these markers or the lines between them manually.
+  solanaBinaries = {
+    x86_64-linux = getBinDerivation {
+      name = "solana-cli-x86_64-linux";
+      filename = "solana-release-x86_64-unknown-linux-gnu.tar.bz2";
+      ### BEGIN_LINUX_SHA256 ###
+      sha256 = "sha256-L7N8z1MjDWkSoOKLAe4y/iuKTRgLpqg2mDpb9h1RXH0=";
+      ### END_LINUX_SHA256 ###
+    };
+    aarch64-apple-darwin = getBinDerivation {
+      name = "solana-cli-aarch64-apple-darwin";
+      filename = "solana-release-aarch64-apple-darwin.tar.bz2";
+      ### BEGIN_DARWIN_SHA256 ###
+      sha256 = "sha256-D6hJL3yQncHltuWtF4QMAzvp/s7LV/S3NHwHiJG8wQ0=";
+      ### END_DARWIN_SHA256 ###
+    };
+  };
+in
+{
+  # Provides environment package for Solana CLI
+  solana-cli-env = pkgs.buildEnv {
+    name = "solana-cli-env";
+    paths = pkgs.lib.optionals pkgs.stdenv.isLinux [
+      solanaBinaries.x86_64-linux
+    ]
+    ++ pkgs.lib.optionals (pkgs.stdenv.isDarwin && pkgs.stdenv.hostPlatform.isAarch64) [
+      solanaBinaries.aarch64-apple-darwin
+    ];
+  };
+
+  # Provides interactive dev shell with Solana CLI tool accessibility.
+  solana-cli-shell = pkgs.mkShell {
+    buildInputs = pkgs.lib.optionals pkgs.stdenv.isLinux [
+      solanaBinaries.x86_64-linux
+    ]
+    ++ pkgs.lib.optionals (pkgs.stdenv.isDarwin && pkgs.stdenv.hostPlatform.isAarch64) [
+      solanaBinaries.aarch64-apple-darwin
+    ];
+    shellHook = ''
+      echo "===================================================="
+      echo "Welcome to the Solana CLI dev shell."
+      echo "Current environment: $(uname -a)"
+      echo "You are using the package for ${pkgs.stdenv.hostPlatform.system}."
+      echo "----------------------------------------------------"
+      echo "Solana CLI information:"
+      solana --version
+      solana config get
+      echo "===================================================="
+    '';
+  };
+
+  # Provides dockerized Solana test validator accessibility. https://hub.docker.com/r/solanalabs/solana
+  # Currently the official docker image only supports x86_64-linux.(https://github.com/anza-xyz/agave/tree/master/sdk/docker-solana)
+  solana-test-validator = pkgs.stdenv.mkDerivation rec {
+    name = "solana-test-validator";
+    src = ./scripts/setup-localnet;
+    installPhase = ''
+      mkdir -p $out/bin
+      cp $src/localnet.sh $out/bin/${name}
+      cp $src/localnet.down.sh $out/bin/
+      chmod +x $out/bin/${name}
+    '';
+  };
+}
\ No newline at end of file