From 75fa690e9d44269b98aedfdfbf2e465341a0ddd1 Mon Sep 17 00:00:00 2001 From: Juraj Piar Date: Mon, 6 May 2024 19:32:22 +0100 Subject: [PATCH] feat(ci): adjusts mining-tests --- .github/mining-test/run.sh | 76 +++++ .github/workflows/build_and_test.yml | 437 ++++++++++++++++----------- 2 files changed, 344 insertions(+), 169 deletions(-) create mode 100755 .github/mining-test/run.sh diff --git a/.github/mining-test/run.sh b/.github/mining-test/run.sh new file mode 100755 index 00000000000..21272fee06a --- /dev/null +++ b/.github/mining-test/run.sh @@ -0,0 +1,76 @@ +#!/bin/bash + +#echo "create docker network for bitcoin nodes exposing it to host" +#sudo docker network create bitcoin-network +#docker network ls +#docker network inspect bitcoin-network +#docker network connect bitcoin-network bitcoind01 +#docker network connect bitcoin-network bitcoind02 + +#sudo docker run -d --network bitcoin-network --name bitcoind01 -p 31591:31591 -p 32591:32591 --entrypoint "/usr/local/bin/bitcoind" kylemanna/bitcoind:latest -printtoconsole -regtest -debug -server -listen -port=31591 -connect=localhost:31592 -rpcbind=0.0.0.0:32591 -rpcallowip=0.0.0.0/0 -rpcuser=admin -rpcpassword=admin +#sudo docker run -d --network bitcoin-network --name bitcoind02 -p 31592:31592 -p 32592:32592 --entrypoint "/usr/local/bin/bitcoind" kylemanna/bitcoind:latest -printtoconsole -regtest -debug -server -listen -port=31592 -connect=localhost:31591 -rpcbind=0.0.0.0:32592 -rpcallowip=0.0.0.0/0 -rpcuser=admin -rpcpassword=admin + +sudo docker run -d --name bitcoind01 -p 31591:31591 -p 32591:32591 --entrypoint "/usr/local/bin/bitcoind" kylemanna/bitcoind:latest -printtoconsole -regtest -debug -server -listen -port=31591 -connect=localhost:31592 -rpcbind=0.0.0.0:32591 -rpcallowip=0.0.0.0/0 -rpcuser=admin -rpcpassword=admin +sudo docker run -d --name bitcoind02 -p 31592:31592 -p 32592:32592 --entrypoint "/usr/local/bin/bitcoind" kylemanna/bitcoind:latest -printtoconsole -regtest -debug -server -listen -port=31592 -connect=localhost:31591 -rpcbind=0.0.0.0:32592 -rpcallowip=0.0.0.0/0 -rpcuser=admin -rpcpassword=admin + +while ! docker ps --format "{{.Names}}" | grep -wq bitcoind01; do + echo "Waiting for container bitcoind01 to start..." + sleep 5 # Wait for 5 seconds before checking again +done +echo "bitcoind01 container is running." + +while ! docker ps --format "{{.Names}}" | grep -wq bitcoind02; do + echo "Waiting for container bitcoind02 to start..." + sleep 5 # Wait for 5 seconds before checking again +done +echo "bitcoind02 container is running." + +#echo "Verify nodes are attached to the network" +#docker network ls +#docker network inspect bitcoin-network +#if ! docker network inspect bitcoin-network | grep -q "\"Name\": \"bitcoind01\""; then +# echo "bitcoind01 is not on the network" +# exit 1 +#fi +#if ! docker network inspect bitcoin-network | grep -q "\"Name\": \"bitcoind02\""; then +# echo "bitcoind02 is not on the network" +# exit 1 +#fi + +ports=(31591 32591 31592 32592) +for port in "${ports[@]}"; do + if ! sudo lsof -i:$port -sTCP:LISTEN > /dev/null; then + echo "Error: No process is listening on port $port" + exit 1 + fi +done +echo "Success: All specified ports are being listened to." + + +echo Test bitcoin node 1 +curl -s -u admin:admin --data-binary '{"jsonrpc":"1.0","id":"curltext","method":"getblockchaininfo","params":[]}' -H 'content-type:text/plain;' http://localhost:32591 --fail +if [ $? -ne 0 ]; then + echo "Error: Failed to connect to the Bitcoin bitcoind01 node" + exit 1 +fi +echo Test bitcoin node 2 +curl -s -u admin:admin --data-binary '{"jsonrpc":"1.0","id":"curltext","method":"getblockchaininfo","params":[]}' -H 'content-type:text/plain;' http://localhost:32592 --fail +if [ $? -ne 0 ]; then + echo "Error: Failed to connect to the Bitcoin bitcoind02 node" + exit 1 +fi +# +#echo "Linking btc nodes to localhost" +#echo "127.0.0.1 bitcoind01" | sudo tee -a /etc/host +#echo "127.0.0.1 bitcoind02" | sudo tee -a /etc/host +#echo "/etc/hosts" +#cat /etc/hosts + +echo "Modify test suite config" +cat mining-integration-tests/config.json +jq --arg new_url "localhost" '.bitcoind.url = $new_url' "mining-integration-tests/config.json" > temp.json && mv temp.json "mining-integration-tests/config.json" +cat mining-integration-tests/config.json + +echo "Generate BTC blocks" +(cd mining-integration-tests && node generateBtcBlocks.js || exit 1) + diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 1c20558c51b..2c8553d1453 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -10,208 +10,307 @@ on: - '**' jobs: - build: +# build: +# runs-on: ubuntu-latest +# container: +# image: openjdk:8-jdk +# env: +# _JAVA_OPTIONS: "-Xmx3G -Xms2G" +# defaults: +# run: +# working-directory: . +# steps: +# - uses: actions/checkout@v2 +# - name: Setup System Tools +# run: | +# apt update -y +# apt install -y gnupg2 curl +# - name: Install Node.js +# run: | +# curl -fsSL https://deb.nodesource.com/setup_16.x | bash - +# apt-get install -y nodejs +# - name: Check Node.js version +# run: node --version +# - name: Verify files +# run: | +# curl -sSL https://secchannel.rsk.co/SUPPORT.asc | gpg2 --import - +# gpg2 --verify SHA256SUMS.asc && sha256sum --check SHA256SUMS.asc +# +# - uses: actions/cache@v4 +# name: Cache Gradle +# with: +# path: | +# .gradle/caches +# gradle/wrapper +# key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} +# restore-keys: | +# ${{ runner.os }}-gradle- +# +# - uses: actions/cache@v4 +# name: Cache build +# with: +# path: | +# build +# key: ${{ runner.os }}-build-${{ hashFiles('build') }} +# restore-keys: | +# ${{ runner.os }}-build- +# +# - name: Build +# run: | +# ./configure.sh +# ./gradlew --no-daemon dependencies +# ./gradlew --no-daemon --stacktrace build -x test +# ls -la ./gradle/wrapper/gradle-wrapper.jar +# +# sonarqube: +# needs: build +# runs-on: ubuntu-latest +# container: +# image: eclipse-temurin:17-jdk +# defaults: +# run: +# working-directory: . +# steps: +# - uses: actions/checkout@v2 +# - name: Run SonarQube analysis +# run: | +# echo "Installing git" +# apt-get update && apt-get install -yqq git +# extra_flags="" +# echo "Get GH event type" +# event_name="${{ github.event_name }}" +# if [ "$event_name" = "pull_request" ]; then +# echo "Setting up PR analysis" +# echo "switch to master" +# git branch -f master origin/master +# pr_number=${{ github.event.pull_request.number }} +# echo "PR number: $pr_number" +# extra_flags="-Dsonar.pullrequest.base=master -Dsonar.pullrequest.branch=${{ github.head_ref }} -Dsonar.pullrequest.key=$pr_number" +# echo "PR analysis flags: $extra_flags" +# else +# echo "Setting up branch analysis" +# extra_flags="-Dsonar.branch.name=master" +# echo "Branch analysis flags: $extra_flags" +# fi +# +# sonar_url=${{ secrets.SONAR_URL }} +# sonar_token=${{ secrets.SONAR_TOKEN }} +# echo "Event name: $event_name" +# echo "Sonar URL: $sonar_url" +# +# echo "Running SonarQube analysis" +# ./configure.sh +# ./gradlew sonarqube --no-daemon -x build -x test $extra_flags \ +# -Dsonar.organization=rsksmart -Dsonar.host.url="$sonar_url" -Dsonar.login="$sonar_token" +# +# tests: +# needs: build +# runs-on: ubuntu-latest +# container: +# image: openjdk:8-jdk +# defaults: +# run: +# working-directory: . +# steps: +# - uses: actions/checkout@v2 +# +# - name: Setup System Tools +# run: | +# apt update -y +# apt install -y curl +# - name: Install Node.js +# run: | +# curl -fsSL https://deb.nodesource.com/setup_16.x | bash - +# apt-get install -y nodejs +# - name: Check Node.js version +# run: node --version +# +# - uses: actions/cache@v4 +# name: Restore gradle cache +# with: +# path: | +# .gradle/caches +# gradle/wrapper +# key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} +# restore-keys: | +# ${{ runner.os }}-gradle- +# +# - name: Get Gradle jar +# run: | +# ./configure.sh +# +# - name: Unit tests +# run: | +# chmod +x ./gradlew +# ./gradlew --no-daemon --stacktrace test +# - name: Integration tests +# run: | +# chmod +x ./gradlew +# ./gradlew --no-daemon --stacktrace test +# +# - name: Save test results +# run: | +# mkdir -p ~/junit/ +# find rskj-core/build/test-results -type f -name "*.xml" -exec cp {} ~/junit/ \; +# if: always() + + mining-tests: +# needs: build runs-on: ubuntu-latest - container: - image: openjdk:8-jdk - env: - _JAVA_OPTIONS: "-Xmx3G -Xms2G" defaults: run: working-directory: . + +# services: +# bitcoind01: +# image: kylemanna/bitcoind:latest +# env: +# DISABLEWALLET: "0" +# ports: +# - 31591:31591 +# # options: >- +# # --entrypoint "/usr/local/bin/bitcoind +# +# bitcoind02: +# image: kylemanna/bitcoind:latest +# env: +# DISABLEWALLET: "0" +# ports: +# - 31592:31592 +# # options: >- +# # --entrypoint "/usr/local/bin/bitcoind + steps: - uses: actions/checkout@v2 - name: Setup System Tools run: | - apt update -y - apt install -y gnupg2 curl - - name: Install Node.js - run: | - curl -fsSL https://deb.nodesource.com/setup_16.x | bash - - apt-get install -y nodejs + sudo apt-get update -y + sudo apt-get install -y jq + # - name: Install Node 12 + # run: | + # sudo apt-get install -y fnm + + - name: Setup Node.js + uses: actions/setup-node@v2 + with: + node-version: '12' - name: Check Node.js version run: node --version - - name: Verify files - run: | - curl -sSL https://secchannel.rsk.co/SUPPORT.asc | gpg2 --import - - gpg2 --verify SHA256SUMS.asc && sha256sum --check SHA256SUMS.asc - - uses: actions/cache@v4 - name: Cache Gradle + - name: Checkout Mining Integration Tests Repository + uses: actions/checkout@v2 with: - path: | - .gradle/caches - gradle/wrapper - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} - restore-keys: | - ${{ runner.os }}-gradle- + repository: rsksmart/mining-integration-tests +# ref: ${{ secrets.MINING_INTEGRATION_TESTS_REF }} +# token: ${{ secrets.GITHUB_TOKEN }} + path: mining-integration-tests - - uses: actions/cache@v4 - name: Cache build + # - uses: actions/cache@v4 + # name: Restore gradle cache + # with: + # path: | + # .gradle/caches + # gradle/wrapper + # key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} + # restore-keys: | + # ${{ runner.os }}-gradle- + # + # - name: Get Gradle jar + # run: | + # ./configure.sh + + - uses: actions/cache@v2 with: path: | - build - key: ${{ runner.os }}-build-${{ hashFiles('build') }} + node_modules + key: ${{ runner.os }}-node-${{ hashFiles('package-lock.json') }} restore-keys: | - ${{ runner.os }}-build- + ${{ runner.os }}-node- - - name: Build + - name: Install Mining Integration Tests dependencies + working-directory: mining-integration-tests run: | - ./configure.sh - ./gradlew --no-daemon dependencies - ./gradlew --no-daemon --stacktrace build -x test - ls -la ./gradle/wrapper/gradle-wrapper.jar + npm install + # + # - name: Start Bitcoin nodes + # run: | + # docker run -itd bitcoind01 /usr/local/bin/bitcoind -printtoconsole -regtest -debug -server -listen -port=31591 -connect=localhost:31592 -rpcbind=0.0.0.0:32591 -rpcallowip=0.0.0.0/0 -rpcuser=admin -rpcpassword=admin + # docker run -itd bitcoind02 /usr/local/bin/bitcoind -printtoconsole -regtest -debug -server -listen -port=31592 -connect=localhost:31591 -rpcbind=0.0.0.0:32591 -rpcallowip=0.0.0.0/0 -rpcuser=admin -rpcpassword=admin +# - name: Start Bitcoin nodes +# run: | +# docker run -d --name bitcoind01 -p 31591:31591 --entrypoint "/usr/local/bin/bitcoind" kylemanna/bitcoind:latest -printtoconsole -regtest -debug -server -listen -port=31591 -connect=localhost:31592 -rpcbind=0.0.0.0:32591 -rpcallowip=0.0.0.0/0 -rpcuser=admin -rpcpassword=admin +# docker run -d --name bitcoind02 -p 31592:31592 --entrypoint "/usr/local/bin/bitcoind" kylemanna/bitcoind:latest -printtoconsole -regtest -debug -server -listen -port=31592 -connect=localhost:31591 -rpcbind=0.0.0.0:32592 -rpcallowip=0.0.0.0/0 -rpcuser=admin -rpcpassword=admin +# +# +# - name: Test bitcoin nodes +# run: | +# echo Test bitcoin node 1 +# curl -s -u admin:admin --data-binary '{"jsonrpc":"1.0","id":"curltext","method":"getblockchaininfo","params":[]}' -H 'content-type:text/plain;' http://localhost:32591 +# echo Test bitcoin node 2 +# curl -s -u admin:admin --data-binary '{"jsonrpc":"1.0","id":"curltext","method":"getblockchaininfo","params":[]}' -H 'content-type:text/plain;' http://localhost:32592 +# +# - name: Generate BTC blocks +# working-directory: mining-integration-tests +# run: | +# node generateBtcBlocks.js - sonarqube: - needs: build - runs-on: ubuntu-latest - container: - image: eclipse-temurin:17-jdk - defaults: - run: - working-directory: . - steps: - - uses: actions/checkout@v2 - - name: Run SonarQube analysis + - name: Run Mining Integration Tests run: | - echo "Installing git" - apt-get update && apt-get install -yqq git - extra_flags="" - echo "Get GH event type" - event_name="${{ github.event_name }}" - if [ "$event_name" = "pull_request" ]; then - echo "Setting up PR analysis" - echo "switch to master" - git branch -f master origin/master - pr_number=${{ github.event.pull_request.number }} - echo "PR number: $pr_number" - extra_flags="-Dsonar.pullrequest.base=master -Dsonar.pullrequest.branch=${{ github.head_ref }} -Dsonar.pullrequest.key=$pr_number" - echo "PR analysis flags: $extra_flags" - else - echo "Setting up branch analysis" - extra_flags="-Dsonar.branch.name=master" - echo "Branch analysis flags: $extra_flags" - fi - - sonar_url=${{ secrets.SONAR_URL }} - sonar_token=${{ secrets.SONAR_TOKEN }} - echo "Event name: $event_name" - echo "Sonar URL: $sonar_url" - - echo "Running SonarQube analysis" - ./configure.sh - ./gradlew sonarqube --no-daemon -x build -x test $extra_flags \ - -Dsonar.organization=rsksmart -Dsonar.host.url="$sonar_url" -Dsonar.login="$sonar_token" - - tests: - needs: build - runs-on: ubuntu-latest - container: - image: openjdk:8-jdk - defaults: - run: - working-directory: . - steps: - - uses: actions/checkout@v2 + pwd + ls -al + ls -al ../ + chmod +x .github/mining-test/run.sh + ./.github/mining-test/run.sh - - name: Setup System Tools - run: | - apt update -y - apt install -y curl - - name: Install Node.js + - name: Checkout RSKj + uses: actions/checkout@v2 + + - name: Setup Java JDK + uses: actions/setup-java@v2 + with: + java-version: '17' + distribution: 'temurin' + + - name: Verify Java installation + run: java -version + + - name: Get RSKj version run: | - curl -fsSL https://deb.nodesource.com/setup_16.x | bash - - apt-get install -y nodejs - - name: Check Node.js version - run: node --version + version=$(tr -d "'\"" < rskj-core/src/main/resources/version.properties \ + | cut -d = -f 2- | paste -sd - -) - uses: actions/cache@v4 - name: Restore gradle cache + name: Restore build with: path: | - .gradle/caches - gradle/wrapper - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} + build + key: ${{ runner.os }}-build-${{ hashFiles('build') }} restore-keys: | - ${{ runner.os }}-gradle- + ${{ runner.os }}-build- - - name: Get Gradle jar + - name: Start RSKj run: | - ./configure.sh + java -Drsk.conf.file=./rsk-integration-test.conf -cp rskj-core/build/libs/rskj-core-"$version"-all.jar co.rsk.Start --regtest + rskpid=$! + echo "RSKj PID: $rskpid" + until nc -z 127.0.0.1 4444 + do + echo "Waiting for RskJ..." + sleep 1 + done - - name: Unit tests - run: | - chmod +x ./gradlew - ./gradlew --no-daemon --stacktrace test - - name: Integration tests + - name: Run Mining Integration Tests + working-directory: mining-integration-tests run: | - chmod +x ./gradlew - ./gradlew --no-daemon --stacktrace test + npm test - - name: Save test results - run: | - mkdir -p ~/junit/ - find rskj-core/build/test-results -type f -name "*.xml" -exec cp {} ~/junit/ \; + - name: Stop RSKj if: always() - - mining-tests: - runs-on: ubuntu-latest - - steps: - - name: Checkout code - uses: actions/checkout@v2 - - - name: Setup run: | - sudo apt-get update - sudo apt-get install -y curl jq + kill $rskpid - - name: Mining integration tests + - name: Stop and Remove Docker Containers + if: always() run: | - function wait_for_completion() { - build_number=$1 - poll_interval=60 - i=0 - max_count=20 - while [ $i -lt $max_count ]; do - res=$(curl -Ssfu "${{ secrets.GITHUB_TOKEN }}" \ - "https://circleci.com/api/v1.1/project/github/$MIT_ORGANIZATION/$MIT_PROJECT/$build_numbe" \ - | jq -r '[.status, .conclusion] | @tsv') - IFS=" " set -- $res - status=${1:-} - conclusion=${2:-} - if [ "$status" = "queued" ]; then - echo "Build is enqueued. Waiting..." - else - echo "[$i/$max_count] Waiting for build $build_url ..." - i=$((i + 1)) - fi - if [ "$status" = "completed" ]; then - echo "Build $build_number finished. Outcome: \"$conclusion\"." - test "$conclusion" = "success" - return $? - fi - sleep $poll_interval - done - return 1 - } - json_payload='{ - "ref": "'${{ github.ref }}'", - "inputs": { - "RSKJ_CIRCLE_BRANCH": "${{ github.ref }}", - "RSKJ_CIRCLE_USERNAME": "${{ github.actor }}", - "RSKJ_CIRCLE_REPONAME": "${{ github.repository }}", - "RSKJ_PR_NUMBER": "${{ github.event.pull_request.number }}", - "RSKJ_CIRCLE_SHA1": "${{ github.sha }}" - } - }' - res=$(curl -Ssf -u "${{ secrets.GITHUB_TOKEN }}:" \ - -H "Content-type: application/json" -d "$json_payload" \ - "https://circleci.com/api/v1.1/project/github/$MIT_ORGANIZATION/$MIT_PROJECT/tree/$MIT_BRANCH" \ - | jq -r '[.url, .id] | @tsv') - IFS=" " set -- $res - build_url=$1 - build_num=$2 - echo "Running mining integration tests. Follow it on:\n\n $build_url\n\n" - sleep 10 - wait_for_completion "$build_num" \ No newline at end of file + /usr/bin/docker stop bitcoind01 bitcoind02 + /usr/bin/docker rm bitcoind01 bitcoind02 \ No newline at end of file