Skip to content

Commit

Permalink
[TT-524] [TT-729] More Products and Reporting for Live Tests (#11327)
Browse files Browse the repository at this point in the history 
* More Specific Tests

* Disable sepolia

* Specify automation test

* Don't give up

* Reduce default funding

* Fix test name

* Messing with jq

* Change all OCR config

* Fiddling with jq more

* Little better reports

* I think I got the formatting now

* Ensure prerequisites

* Debugging

* Remove Sepolia

* Formatting

* Remove if (we'll make it pretty later)

* Strip out quotes

* Fix GHA label

* Remove debug

* More accurate colors

* Ease the debugging process

* More debugging

* Fix needs check

* Check fail

* Check fail better

* Learned some tricks

* Successes

* Extra paren

* Fix emojis

* Purty

* Remove debug and mocks

* Cleanup
  • Loading branch information
@kalverra
kalverra authored Nov 21, 2023
1 parent 98ac92b commit 530225a
Show file tree
Hide file tree
Showing 2 changed files with 128 additions and 88 deletions.
205 changes: 124 additions & 81 deletions .github/workflows/live-testnet-tests.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,16 @@ env:
CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
INTERNAL_DOCKER_REPO: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com
MOD_CACHE_VERSION: 2
CHAINLINK_NODE_FUNDING: .1

CHAINLINK_COMMIT_SHA: ${{ github.sha }}
CHAINLINK_ENV_USER: ${{ github.actor }}
TEST_LOG_LEVEL: debug
EVM_KEYS: ${{ secrets.QA_EVM_KEYS }}

SEPOLIA_URLS: ${{ secrets.QA_SEPOLIA_URLS }}
SEPOLIA_HTTP_URLS: ${{ secrets.QA_SEPOLIA_HTTP_URLS }}

OPTIMISM_GOERLI_URLS: ${{ secrets.QA_OPTIMISM_GOERLI_URLS }}
OPTIMISM_GOERLI_HTTP_URLS: ${{ secrets.QA_OPTIMISM_GOERLI_HTTP_URLS }}

Expand Down Expand Up @@ -55,61 +59,66 @@ jobs:
AWS_REGION: ${{ secrets.QA_AWS_REGION }}
AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}


sepolia-smoke-tests:
environment: integration
permissions:
checks: write
pull-requests: write
id-token: write
contents: read
needs: [build-chainlink]
env:
SELECTED_NETWORKS: SEPOLIA
strategy:
fail-fast: false
matrix:
product: [ocr, automation]
name: Sepolia ${{ matrix.product }} Tests
runs-on: ubuntu-latest
steps:
- name: Checkout the repo
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
with:
ref: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }}
- name: Run Tests
uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@e865e376b8c2d594028c8d645dd6c47169b72974 # v2.2.16
env:
PYROSCOPE_SERVER: ${{ secrets.QA_PYROSCOPE_INSTANCE }}
PYROSCOPE_ENVIRONMENT: ci-smoke-${{ matrix.product }}-sepolia
PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}
with:
test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=1 ./smoke/${{ matrix.product }}_test.go 2>&1 | tee /tmp/gotest.log | gotestfmt
test_download_vendor_packages_command: cd ./integration-tests && go mod download
cl_repo: ${{ env.CHAINLINK_IMAGE }}
cl_image_tag: ${{ github.sha }}
aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
dockerhub_username: ${{ secrets.DOCKERHUB_READONLY_USERNAME }}
dockerhub_password: ${{ secrets.DOCKERHUB_READONLY_PASSWORD }}
artifacts_location: ./integration-tests/smoke/logs
publish_check_name: Seplia ${{ matrix.product }} Smoke Test Results
token: ${{ secrets.GITHUB_TOKEN }}
go_mod_path: ./integration-tests/go.mod
cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }}
cache_restore_only: "true"
QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
- name: Collect Metrics
if: always()
id: collect-gha-metrics
uses: smartcontractkit/push-gha-metrics-action@d1618b772a97fd87e6505de97b872ee0b1f1729a # v2.0.2
with:
basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }}
hostname: ${{ secrets.GRAFANA_CLOUD_HOST }}
this-job-name: Sepolia ${{ matrix.product }} Tests
test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}'
continue-on-error: true
# TODO: Re-enable when we have secrets properly configured
# sepolia-smoke-tests:
# environment: integration
# permissions:
# checks: write
# pull-requests: write
# id-token: write
# contents: read
# needs: [build-chainlink]
# env:
# SELECTED_NETWORKS: SEPOLIA
# strategy:
# max-parallel: 1
# fail-fast: false
# matrix:
# include: # https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs#example-adding-configurations
# - product: OCR
# test: TestOCRBasic
# - product: Automation
# test: TestAutomationBasic/registry_2_0
# name: Sepolia ${{ matrix.product }} Tests
# runs-on: ubuntu-latest
# steps:
# - name: Checkout the repo
# uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
# with:
# ref: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }}
# - name: Run Tests
# uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@e865e376b8c2d594028c8d645dd6c47169b72974 # v2.2.16
# env:
# PYROSCOPE_SERVER: ${{ secrets.QA_PYROSCOPE_INSTANCE }}
# PYROSCOPE_ENVIRONMENT: ci-smoke-${{ matrix.product }}-sepolia
# PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}
# with:
# test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=1 -run ${{ matrix.test }} ./smoke 2>&1 | tee /tmp/gotest.log | gotestfmt
# test_download_vendor_packages_command: cd ./integration-tests && go mod download
# cl_repo: ${{ env.CHAINLINK_IMAGE }}
# cl_image_tag: ${{ github.sha }}
# aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
# dockerhub_username: ${{ secrets.DOCKERHUB_READONLY_USERNAME }}
# dockerhub_password: ${{ secrets.DOCKERHUB_READONLY_PASSWORD }}
# artifacts_location: ./integration-tests/smoke/logs
# publish_check_name: Seplia ${{ matrix.product }} Smoke Test Results
# token: ${{ secrets.GITHUB_TOKEN }}
# go_mod_path: ./integration-tests/go.mod
# cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }}
# cache_restore_only: "true"
# QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
# QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
# QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
# - name: Collect Metrics
# if: always()
# id: collect-gha-metrics
# uses: smartcontractkit/push-gha-metrics-action@d1618b772a97fd87e6505de97b872ee0b1f1729a # v2.0.2
# with:
# basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }}
# hostname: ${{ secrets.GRAFANA_CLOUD_HOST }}
# this-job-name: Sepolia ${{ matrix.product }} Tests
# test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}'
# continue-on-error: true

optimism-goerli-smoke-tests:
environment: integration
Expand All @@ -123,8 +132,13 @@ jobs:
SELECTED_NETWORKS: OPTIMISM_GOERLI
strategy:
fail-fast: false
max-parallel: 1
matrix:
product: [ocr, automation]
include: # https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs#example-adding-configurations
- product: OCR
test: TestOCRBasic
- product: Automation
test: TestAutomationBasic/registry_2_0
name: Optimism Goerli ${{ matrix.product }} Tests
runs-on: ubuntu-latest
steps:
Expand All @@ -139,7 +153,7 @@ jobs:
PYROSCOPE_ENVIRONMENT: ci-smoke-${{ matrix.product }}-optimism-goerli
PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}
with:
test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=1 ./smoke/${{ matrix.product }}_test.go 2>&1 | tee /tmp/gotest.log | gotestfmt
test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=1 -run ${{ matrix.test }} ./smoke 2>&1 | tee /tmp/gotest.log | gotestfmt
test_download_vendor_packages_command: cd ./integration-tests && go mod download
cl_repo: ${{ env.CHAINLINK_IMAGE }}
cl_image_tag: ${{ github.sha }}
Expand Down Expand Up @@ -177,9 +191,14 @@ jobs:
env:
SELECTED_NETWORKS: ARBITRUM_GOERLI
strategy:
max-parallel: 1
fail-fast: false
matrix:
product: [ocr, automation]
include: # https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs#example-adding-configurations
- product: OCR
test: TestOCRBasic
- product: Automation
test: TestAutomationBasic/registry_2_0
name: Arbitrum Goerli ${{ matrix.product }} Tests
runs-on: ubuntu-latest
steps:
Expand All @@ -194,7 +213,7 @@ jobs:
PYROSCOPE_ENVIRONMENT: ci-smoke-${{ matrix.product }}-arbitrum-goerli
PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}
with:
test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=1 ./smoke/${{ matrix.product }}_test.go 2>&1 | tee /tmp/gotest.log | gotestfmt
test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=1 -run ${{ matrix.test }} ./smoke 2>&1 | tee /tmp/gotest.log | gotestfmt
test_download_vendor_packages_command: cd ./integration-tests && go mod download
cl_repo: ${{ env.CHAINLINK_IMAGE }}
cl_image_tag: ${{ github.sha }}
Expand Down Expand Up @@ -233,10 +252,10 @@ jobs:
id-token: write
contents: read
runs-on: ubuntu-latest
needs: [sepolia-smoke-tests, optimism-goerli-smoke-tests, arbitrum-goerli-smoke-tests]
needs: [optimism-goerli-smoke-tests, arbitrum-goerli-smoke-tests]
steps:
- name: Debug Result
run: echo ${{needs.*.result}}
run: echo ${{ join(needs.*.result, ',') }}
- name: Main Slack Notification
uses: slackapi/slack-github-action@e28cf165c92ffef168d23c5c9000cffc8a25e117 # v1.24.0
id: slack
Expand All @@ -246,13 +265,13 @@ jobs:
{
"attachments": [
{
"color": "${{ needs.*.result == 'success' && '#2E7D32' || '#C62828' }}",
"color": "${{ contains(join(needs.*.result, ','), 'failure') && '#C62828' || '#2E7D32' }}",
"blocks": [
{
"type": "header",
"text": {
"type": "plain_text",
"text": "Live Smoke Test Results ${{ needs.*.result == 'success' && ':white_check_mark:' || ':x:'}}",
"text": "Live Smoke Test Results ${{ contains(join(needs.*.result, ','), 'failure') && ':x:' || ':white_check_mark:'}}",
"emoji": true
}
},
Expand All @@ -274,7 +293,7 @@ jobs:
SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}

testnet-smoke-tests-results:
name: Post Test Results
name: Post Test Results for ${{ matrix.network }}
if: ${{ always() && needs.*.result != 'skipped' && needs.*.result != 'cancelled' }}
environment: integration
permissions:
Expand All @@ -287,23 +306,53 @@ jobs:
strategy:
fail-fast: false
matrix:
testnet: [sepolia, optimism-goerli, arbitrum-goerli]
network: [Optimism Goerli, Arbitrum Goerli]
steps:
- name: Get Results
id: test-results
run: |
# I feel like there's some clever, fully jq way to do this, but I ain't got the motivation to figure it out
echo "Querying test results"
echo "status=$(curl \
PARSED_RESULTS=$(curl \
-H "Authorization: Bearer ${{ github.token }}" \
'https://api.github.com/repos/${{github.repository}}/actions/runs/${{ github.run_id }}/jobs' \
| jq -r '.jobs[] | select(.name == "Live Testnet Smoke Tests ${{ matrix.testnet }}-smoke-tests").steps[] | select(.name == "Run Tests").conclusion')" >> $GITHUB_OUTPUT
| jq -r --arg pattern "${{ matrix.network }} (?<product>\\w+) Tests" '.jobs[]
| select(.name | test($pattern)) as $job
| $job.steps[]
| select(.name == "Run Tests")
| { conclusion: (if .conclusion == "success" then ":white_check_mark:" else ":x:" end), product: ("*" + ($job.name | capture($pattern).product) + "*") }')
echo "status=$(curl \
-H "Authorization: Bearer ${{ github.token }}" \
'https://api.github.com/repos/${{github.repository}}/actions/runs/${{ github.run_id }}/jobs' \
| jq -r '.jobs[] | select(.name == "Live Testnet Smoke Tests ${{ matrix.testnet }}-smoke-tests"").steps[] | select(.name == "Run Tests").conclusion')"
echo "thread_ts=${{ needs.testnet-smoke-tests-notify.outputs.thread_ts }}"
echo "Parsed Results:"
echo $PARSED_RESULTS
ALL_SUCCESS=true
for row in $(echo "$PARSED_RESULTS" | jq -s | jq -r '.[] | select(.conclusion != ":white_check_mark:")'); do
success=false
break
done
echo all_success=$ALL_SUCCESS >> $GITHUB_OUTPUT
FORMATTED_RESULTS=$(echo $PARSED_RESULTS | jq -s '[.[]
| {
conclusion: .conclusion,
product: .product
}
]
| map("{\"type\": \"section\", \"text\": {\"type\": \"mrkdwn\", \"text\": \"\(.product): \(.conclusion)\"}}")
| join(",")')
echo "Formatted Results:"
echo $FORMATTED_RESULTS
# Cleans out backslashes and quotes from jq
CLEAN_RESULTS=$(echo "$FORMATTED_RESULTS" | sed 's/\\\"/"/g' | sed 's/^"//;s/"$//')
echo "Clean Results"
echo $CLEAN_RESULTS
echo results=$CLEAN_RESULTS >> $GITHUB_OUTPUT
- name: Test Details
uses: slackapi/slack-github-action@e28cf165c92ffef168d23c5c9000cffc8a25e117 # v1.24.0
Expand All @@ -314,26 +363,20 @@ jobs:
"thread_ts": "${{ needs.testnet-smoke-tests-notify.outputs.thread_ts }}",
"attachments": [
{
"color": "${{ steps.test-results.outputs.status == 'success' && '#2E7D32' || '#C62828' }}",
"color": "${{ steps.test-results.outputs.all_success && '#2E7D32' || '#C62828' }}",
"blocks": [
{
"type": "header",
"text": {
"type": "plain_text",
"text": "${{ matrix.testnet }} Smoke Test Results ${{ steps.test-results.outputs.status == 'success' && ':white_check_mark:' || ':x:'}}",
"text": "${{ matrix.network }} ${{ steps.test-results.outputs.all_success && ':white_check_mark:' || ':x: Notifying <@U01Q4N37KFG>'}}",
"emoji": true
}
},
{
"type": "divider"
},
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "OCR ${{ steps.test-results.outputs.status == 'success' && ':white_check_mark:' || ':x:'}}"
}
}
${{ steps.test-results.outputs.results }}
]
}
]
Expand Down
11 changes: 4 additions & 7 deletions integration-tests/soak/ocr_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,16 +1,13 @@
package soak

import (
"fmt"
"testing"

"github.com/stretchr/testify/require"

"github.com/smartcontractkit/chainlink-testing-framework/logging"
"github.com/smartcontractkit/chainlink-testing-framework/networks"

"github.com/smartcontractkit/chainlink/integration-tests/actions"
"github.com/smartcontractkit/chainlink/integration-tests/config"
"github.com/smartcontractkit/chainlink/integration-tests/testsetups"
)

Expand All @@ -19,10 +16,10 @@ func TestOCRSoak(t *testing.T) {
// Use this variable to pass in any custom EVM specific TOML values to your Chainlink nodes
customNetworkTOML := ``
// Uncomment below for debugging TOML issues on the node
network := networks.MustGetSelectedNetworksFromEnv()[0]
fmt.Println("Using Chainlink TOML\n---------------------")
fmt.Println(networks.AddNetworkDetailedConfig(config.BaseOCR1Config, customNetworkTOML, network))
fmt.Println("---------------------")
// network := networks.MustGetSelectedNetworksFromEnv()[0]
// fmt.Println("Using Chainlink TOML\n---------------------")
// fmt.Println(networks.AddNetworkDetailedConfig(config.BaseOCR1Config, customNetworkTOML, network))
// fmt.Println("---------------------")

ocrSoakTest, err := testsetups.NewOCRSoakTest(t, false)
require.NoError(t, err, "Error creating soak test")
Expand Down

0 comments on commit 530225a

Please sign in to comment.