Skip to content

Commit

Permalink
Added metrics to smoke (#520)
Browse files Browse the repository at this point in the history
Co-authored-by: Tate <[email protected]>
  • Loading branch information
smickovskid and tateexon authored Jul 25, 2023
1 parent e36a9ca commit 5bc4c50
Show file tree
Hide file tree
Showing 14 changed files with 1,588 additions and 260 deletions.
2 changes: 1 addition & 1 deletion .github/actions/build-test-image/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ runs:
file: ./integration-tests/test.Dockerfile
build-args: |
BASE_IMAGE=${{ inputs.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ inputs.QA_AWS_REGION }}.amazonaws.com/test-base-image
IMAGE_VERSION=v0.33.0
IMAGE_VERSION=v0.35.0
SUITES="soak smoke"
AWS_REGION: ${{ inputs.QA_AWS_REGION }}
AWS_ROLE_TO_ASSUME: ${{ inputs.QA_AWS_ROLE_TO_ASSUME }}
Expand Down
20 changes: 15 additions & 5 deletions .github/workflows/e2e_custom_cl.yml
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ jobs:
steps:
- name: Check if image exists
id: check-image
uses: smartcontractkit/chainlink-github-actions/docker/image-exists@336617ae6d70fec60c15cc3382e17a4d2615a801 # v2.2.0
uses: smartcontractkit/chainlink-github-actions/docker/image-exists@ad22fbd6f4d108b82aaf49b527bcf40f32babea8 # v2.2.1
with:
repository: chainlink-solana-tests
tag: ${{ github.sha }}
Expand Down Expand Up @@ -84,15 +84,15 @@ jobs:
steps:
- name: Check if image exists
id: check-image
uses: smartcontractkit/chainlink-github-actions/docker/image-exists@336617ae6d70fec60c15cc3382e17a4d2615a801 # v2.2.0
uses: smartcontractkit/chainlink-github-actions/docker/image-exists@ad22fbd6f4d108b82aaf49b527bcf40f32babea8 # v2.2.1
with:
repository: chainlink
tag: solana.${{ github.sha }}
AWS_REGION: ${{ secrets.QA_AWS_REGION }}
AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
- name: Build Image
if: steps.check-image.outputs.exists == 'false'
uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/build-image@cosmos_one_off # we will want to switch to this once cosmos settles back down 336617ae6d70fec60c15cc3382e17a4d2615a801 # v2.2.0
uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/build-image@d5ab9fe53da374dd789b5d9ef10f3282197e088d # we will want to switch to this once cosmos settles back down 336617ae6d70fec60c15cc3382e17a4d2615a801 # v2.2.0
with:
cl_repo: smartcontractkit/chainlink
cl_ref: ${{ github.event.inputs.cl_branch_ref }}
Expand Down Expand Up @@ -139,6 +139,7 @@ jobs:
id-token: write
contents: read
runs-on: ubuntu-latest

needs: [e2e_custom_build_artifacts, e2e_custom_build_custom_chainlink_image, build_test_image]
env:
TEST_SUITE: smoke
Expand All @@ -148,12 +149,21 @@ jobs:
TEST_LOG_LEVEL: debug
SELECTED_NETWORKS: SIMULATED
steps:
- name: Collect Metrics
if: always()
id: collect-gha-metrics
uses: smartcontractkit/push-gha-metrics-action@8163dcea2f01a0a8fec84b284406ff7af1d2e1c0
with:
basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }}
hostname: ${{ secrets.GRAFANA_CLOUD_HOST }}
this-job-name: E2E Custom Run Smoke Tests
continue-on-error: true
- name: Checkout the repo
uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
- name: Run Tests
uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@336617ae6d70fec60c15cc3382e17a4d2615a801 # v2.2.0
uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@ad22fbd6f4d108b82aaf49b527bcf40f32babea8 # v2.2.1
with:
test_command_to_run: cd ./integration-tests && go test -timeout 24h -count=1 -json $(args) ./smoke 2>&1 | tee /tmp/gotest.log | gotestfmt
test_command_to_run: cd ./integration-tests && go test -timeout 24h -count=1 -run TestSolanaOCRV2Smoke -json $(args) ./smoke 2>&1 | tee /tmp/gotest.log | gotestfmt
go_mod_path: ./integration-tests/go.mod
cl_repo: ${{ env.CL_ECR }}
cl_image_tag: solana.${{ github.sha }}
Expand Down
198 changes: 198 additions & 0 deletions .github/workflows/e2e_testnet_daily.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,198 @@
name: e2e_testnet_daily
on:
workflow_dispatch:
inputs:
cl_branch_ref:
description: Chainlink repo branch to integrate with
required: true
default: develop
type: string
RPC_URL:
description: RPC URL for the tests
required: true
default: https://api.devnet.solana.com
type: string
WS_URL:
description: WS URL for the tests
required: true
default: wss://api.devnet.solana.com/
type: string
PRIVATE_KEY:
description: Private key in byte format [12, 12 ...]
required: true
type: string
schedule:
- cron: '0 * * * *'
env:
CL_ECR: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-solana-tests:${{ github.sha }}
CONTRACT_ARTIFACTS_PATH: contracts/target/deploy

# Only run 1 of this workflow at a time per PR
concurrency:
group: integration-tests-solana-${{ github.ref }}
cancel-in-progress: true

jobs:
get_projectserum_version:
name: Get ProjectSerum Version
environment: integration
runs-on: ubuntu-latest
outputs:
projectserum_version: ${{ steps.psversion.outputs.projectserum_version }}
steps:
- name: Checkout the repo
uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
- name: Get ProjectSerum Version
id: psversion
uses: ./.github/actions/projectserum_version

test-image-exists:
environment: integration
permissions:
checks: write
pull-requests: write
id-token: write
contents: read
name: Check If Solana Test Image Exists
runs-on: ubuntu-latest
outputs:
exists: ${{ steps.check-image.outputs.exists }}
steps:
- name: Check if image exists
id: check-image
uses: smartcontractkit/chainlink-github-actions/docker/image-exists@ad22fbd6f4d108b82aaf49b527bcf40f32babea8 # v2.2.1
with:
repository: chainlink-solana-tests
tag: ${{ github.sha }}
AWS_REGION: ${{ secrets.QA_AWS_REGION }}
AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}

e2e_custom_build_artifacts:
name: E2E Custom Build Artifacts
environment: integration
permissions:
id-token: write
contents: read
runs-on: ubuntu-latest
needs: [get_projectserum_version, test-image-exists]
container:
image: projectserum/build:${{ needs.get_projectserum_version.outputs.projectserum_version }}
env:
RUSTUP_HOME: "/root/.rustup"
FORCE_COLOR: 1
steps:
- name: Checkout the repo
uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
- name: Build contracts
if: needs.test-image-exists.outputs.exists == 'false'
uses: ./.github/actions/build_contract_artifacts

e2e_custom_build_custom_chainlink_image:
name: E2E Custom Build Custom CL Image
runs-on: ubuntu-latest
environment: integration
permissions:
id-token: write
contents: read
steps:
- name: Check if image exists
id: check-image
uses: smartcontractkit/chainlink-github-actions/docker/image-exists@ad22fbd6f4d108b82aaf49b527bcf40f32babea8 # v2.2.1
with:
repository: chainlink
tag: solana.${{ github.sha }}
AWS_REGION: ${{ secrets.QA_AWS_REGION }}
AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
- name: Build Image
if: steps.check-image.outputs.exists == 'false'
uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/build-image@d5ab9fe53da374dd789b5d9ef10f3282197e088d # we will want to switch to this once cosmos settles back down 336617ae6d70fec60c15cc3382e17a4d2615a801 # v2.2.0
with:
cl_repo: smartcontractkit/chainlink
cl_ref: ${{ github.event.inputs.cl_branch_ref }}
dep_solana_sha: ${{ github.event.pull_request.head.sha }}
push_tag: ${{ env.CL_ECR }}:solana.${{ github.sha }}
QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}

build_test_image:
environment: integration
permissions:
id-token: write
contents: read
name: Build Test Image
runs-on: ubuntu-latest
needs: [e2e_custom_build_artifacts]
steps:
- name: Collect Metrics
id: collect-gha-metrics
uses: smartcontractkit/push-gha-metrics-action@v1
with:
basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }}
hostname: ${{ secrets.GRAFANA_CLOUD_HOST }}
this-job-name: Build Test Image
continue-on-error: true
- name: Checkout the repo
uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
with:
ref: ${{ github.event.pull_request.head.sha }}
- name: Build Image
uses: ./.github/actions/build-test-image
with:
artifacts_path: ${{ env.CONTRACT_ARTIFACTS_PATH }}
QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}

e2e_custom_run_daily_testnet_smoke_tests:
name: E2E Run Daily Smoke Tests
environment: integration
permissions:
checks: write
pull-requests: write
id-token: write
contents: read
runs-on: ubuntu-latest

needs: [e2e_custom_build_artifacts, e2e_custom_build_custom_chainlink_image, build_test_image]
env:
TEST_SUITE: smoke
TEST_ARGS: -test.timeout 30m
CHAINLINK_COMMIT_SHA: ${{ github.sha }}
CHAINLINK_ENV_USER: ${{ github.actor }}
TEST_LOG_LEVEL: debug
SELECTED_NETWORKS: SIMULATED
RPC_URL: ${{ github.event_name == 'schedule' && 'https://api.devnet.solana.com' || github.event.inputs.RPC_URL }}
WS_URL: ${{ github.event_name == 'schedule' && 'wss://api.devnet.solana.com/' || github.event.inputs.WS_URL }}
PROGRAM_ID_OCR2: 9v5sGbNhyTM23yVvczzPbk5NtCqrARQpZjuzebBur43h
PROGRAM_ID_ACCESS_CONTROLLER: 5Gm8JySC3RzGonihF1hj1kZkjHyyW7qxJNvyzq1ieAY5
PROGRAM_ID_STORE: ANooJg7j9UxgJ2DNuLqUJs5h956qzc5LdPLFFtQdDb8M
LINK_TOKEN: Dmw5mDvteezKfop9zd3RQbJmZfBATF3QuSqDU66axyts
VAULT_ADDRESS: G27m7KxTh4KVLapxB9MXfEA8HLUfYuGYQ1ELEs2zQdiQ
PRIVATE_KEY: ${{ github.event_name == 'schedule' && secrets.PRIVATE_KEY || github.event.inputs.PRIVATE_KEY }}

steps:
- name: Collect Metrics
if: always()
id: collect-gha-metrics
uses: smartcontractkit/push-gha-metrics-action@8163dcea2f01a0a8fec84b284406ff7af1d2e1c0
with:
basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }}
hostname: ${{ secrets.GRAFANA_CLOUD_HOST }}
this-job-name: E2E Run Daily Smoke Tests
continue-on-error: true
- name: Checkout the repo
uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
- name: Run Tests
uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@ad22fbd6f4d108b82aaf49b527bcf40f32babea8 # v2.2.1
with:
test_command_to_run: cd ./integration-tests && go test -timeout 24h -count=1 -run TestSolanaGauntletOCRV2Smoke -json $(args) ./smoke 2>&1 | tee /tmp/gotest.log | gotestfmt
go_mod_path: ./integration-tests/go.mod
cl_repo: ${{ env.CL_ECR }}
cl_image_tag: solana.${{ github.sha }}
artifacts_location: /home/runner/work/chainlink-solana/chainlink-solana/integration-tests/logs
token: ${{ secrets.GITHUB_TOKEN }}
QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
should_cleanup: false
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -41,3 +41,4 @@ tests/e2e/smoke/logs
tests/e2e/smoke/contracts-chaos-state.json
tmp-manifest-*
tests-smoke-report.xml
.env.test*
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ test_relay_unit:

test_smoke:
cd ./integration-tests &&\
SELECTED_NETWORKS=SIMULATED go test -timeout 24h -count=1 -json $(args) ./smoke 2>&1 | tee /tmp/gotest.log | gotestfmt
SELECTED_NETWORKS=SIMULATED go test -timeout 24h -count=1 -json $(args) -run TestSolanaOCRV2Smoke ./smoke 2>&1 | tee /tmp/gotest.log | gotestfmt

test_ocr_soak:
cd ./integration-tests &&\
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,13 @@ type Oracle = {
apis: string[]
}

type Transmission = {
latestTransmissionNo: number
roundId: number
answer: any
transmitter: string
}

// Returns a formatted oracle log given a list of addresses and a start tab
// if a list of oracles is provided, add context to logs
const makeOracleLog = (addresses: string[], startTab = 2, oracles?: Oracle[]): string[] => {
Expand Down Expand Up @@ -116,7 +123,7 @@ export default class OCR2InspectResponses extends SolanaCommand {

// Store observers from each transmission
const observerRounds: PublicKey[][] = []

const transmissionDetails: Transmission[] = []
events.forEach((event, i) => {
// Map observer indices into addresses
const observers = (event.observers as []).slice(0, event.observerCount).map((observer) => transmitters[observer])
Expand Down Expand Up @@ -144,7 +151,12 @@ export default class OCR2InspectResponses extends SolanaCommand {
${event.observerCount}/${transmitters.length} oracles are responding
`,
)

transmissionDetails.push({
latestTransmissionNo: i + 1,
roundId: event.roundId,
answer: parseInt(event.answer.toString(), 2),
transmitter: transmitters[event.transmitter].toString(),
})
// Log oracles that are not responsive
var notResponding: number = 0
transmitters.forEach((transmitter) => {
Expand Down Expand Up @@ -185,6 +197,9 @@ export default class OCR2InspectResponses extends SolanaCommand {
const successfulInspection = inspection.inspect(inspections)

return {
data: {
latestTransmissions: transmissionDetails,
},
responses: [
{
tx: this.wrapInspectResponse(successfulInspection, state.toString()),
Expand Down
Loading

0 comments on commit 5bc4c50

Please sign in to comment.