From e6ce4a055ba8227197d923138e82bc5ce1a8fb2c Mon Sep 17 00:00:00 2001 From: Alexandr Yepishev Date: Mon, 30 Sep 2024 20:31:51 +0100 Subject: [PATCH] Add OCR test run book --- .github/E2E_TESTS_ON_GITHUB_CI.md | 60 ++++---- integration-tests/Makefile | 16 ++- integration-tests/README.md | 125 ++++++++++++---- integration-tests/run-books/OCR.md | 214 ++++++++++++++++++++++++++++ integration-tests/smoke/ocr_test.go | 8 +- 5 files changed, 362 insertions(+), 61 deletions(-) create mode 100644 integration-tests/run-books/OCR.md diff --git a/.github/E2E_TESTS_ON_GITHUB_CI.md b/.github/E2E_TESTS_ON_GITHUB_CI.md index 02144eff64f..4cbcffacbdb 100644 --- a/.github/E2E_TESTS_ON_GITHUB_CI.md +++ b/.github/E2E_TESTS_ON_GITHUB_CI.md @@ -1,26 +1,40 @@ # E2E Tests on GitHub CI -E2E tests are executed on GitHub CI using the [E2E Tests Reusable Workflow](#about-the-reusable-workflow) or dedicated workflows. +- [E2E Tests on GitHub CI](#e2e-tests-on-github-ci) + - [Scheduled test workflows](#scheduled-test-workflows) + - [PR E2E Tests](#pr-e2e-tests) + - [Nightly E2E Tests](#nightly-e2e-tests) + - [Release E2E Tests](#release-e2e-tests) + - [Smoke Tests](#smoke-tests) + - [Client Compatibility Tests](#client-compatibility-tests) + - [On-Demand Workflows](#on-demand-workflows) + - [Test workflows setup in CI](#test-workflows-setup-in-ci) -## Automatic workflows +E2E tests are executed on GitHub CI using the [E2E Tests Reusable Workflow](https://github.com/smartcontractkit/.github/blob/main/.github/workflows/README.md) or dedicated workflows. -These workflows are designed to run automatically at crucial stages of the software development process, such as on every commit in a PR, nightly or before release. +## Scheduled test workflows + +These workflows are designed to run on every commit in a PR, nightly or before release (see `triggers` in [e2e-tests.yaml](./e2e-tests.yml)). ### PR E2E Tests -Run on every commit in a PR to ensure changes do not introduce regressions. +Smoke tests triggered on every commit in a PR to ensure changes do not introduce regressions. [Link](https://github.com/smartcontractkit/chainlink/blob/develop/.github/workflows/integration-tests.yml) ### Nightly E2E Tests -Conducted nightly to catch issues that may develop over time or with accumulated changes. +Nightly E2E test runs (). [Link](https://github.com/smartcontractkit/chainlink/blob/develop/.github/workflows/run-nightly-e2e-tests.yml) ### Release E2E Tests -This section contains automatic workflows triggering E2E tests at release. +E2E tests triggered on a release tag. + +#### Smoke Tests + +[Link](https://github.com/smartcontractkit/chainlink/blob/develop/.github/workflows/integration-tests.yml) #### Client Compatibility Tests @@ -28,31 +42,29 @@ This section contains automatic workflows triggering E2E tests at release. ## On-Demand Workflows -Triggered manually by QA for specific testing needs. +These are dispatched parametrized workflows, that may be triggered manually for specific testing needs. For more details refer [integration-tests README](../integration-tests/README.md) and [per-product test run books](../integration-tests/run-books/). **Examples:** +- [Selected E2E Tests Workflow](https://github.com/smartcontractkit/chainlink/actions/workflows/run-selected-e2e-tests.yml) +- [Client Compatibility Tests](https://github.com/smartcontractkit/chainlink/actions/workflows/client-compatibility-tests.yml) +- [OCR Soak Tests](https://github.com/smartcontractkit/chainlink/actions/workflows/on-demand-ocr-soak-test.yml) - [On-Demand Automation Tests](https://github.com/smartcontractkit/chainlink/actions/workflows/automation-ondemand-tests.yml) - [CCIP Chaos Tests](https://github.com/smartcontractkit/chainlink/actions/workflows/ccip-chaos-tests.yml) -- [OCR Soak Tests](https://github.com/smartcontractkit/chainlink/actions/workflows/on-demand-ocr-soak-test.yml) - [VRFv2Plus Smoke Tests](https://github.com/smartcontractkit/chainlink/actions/workflows/on-demand-vrfv2plus-smoke-tests.yml) -## Test Configs - -E2E tests utilize TOML files to define their parameters. Each test is equipped with a default TOML config, which can be overridden by specifying an alternative TOML config. This allows for running tests with varied parameters, such as on a non-default blockchain network. For tests executed on GitHub CI, both the default configs and any override configs must reside within the git repository. The `test_config_override_path` workflow input is used to provide a path to an override config. - -Config overrides should be stored in `testconfig/*/overrides/*.toml`. Placing files here will not trigger a rebuild of the test runner image. - -**Important Note:** The use of `base64Config` input is deprecated in favor of `test_config_override_path`. For more details, refer to [the decision log](https://smartcontract-it.atlassian.net/wiki/spaces/TT/pages/927596563/Storing+All+Test+Configs+In+Git). - -To learn more about test configs see [CTF Test Config](https://github.com/smartcontractkit/chainlink-testing-framework/blob/main/lib/config/README.md). - -## Test Secrets - -For security reasons, test secrets and sensitive information are not stored directly within the test config TOML files. Instead, these secrets are securely injected into tests using environment variables. For a detailed explanation on managing test secrets, refer to our [Test Secrets documentation](https://github.com/smartcontractkit/chainlink-testing-framework/blob/main/lib/config/README.md#test-secrets). +### Test workflows setup in CI -If you need to run a GitHub workflow using custom secrets, please refer to the [guide on running GitHub workflows with your test secrets](https://github.com/smartcontractkit/chainlink-testing-framework/blob/main/lib/config/README.md#run-github-workflow-with-your-test-secrets). +Even though tests have default configs, for parametrization purposes, those can be overridden (read more about [integration-tests test configs](../integration-tests/testconfig/README.md)) and [CTF Test Config](https://github.com/smartcontractkit/chainlink-testing-framework/blob/main/lib/config/README.md). -## About the E2E Test Reusable Workflow +> [!NOTE] +> **Config Overrides** +> +> 1. For tests executed on GitHub CI, both (defaults and overrides) should be stored in repository under `testconfig//overrides/.toml` (see example [here](../integration-tests/testconfig/ocr2/overrides/base_sepolia.toml)). Placing files here will not trigger a rebuild of the test runner image. +> 2. Use `test_config_override_path` to point to an override config. -For information on the E2E Test Reusable Workflow, visit the documentation in the [smartcontractkit/.github repository](https://github.com/smartcontractkit/.github/blob/main/.github/workflows/README.md). +> [!IMPORTANT] +> **Test Secrets** +> +> 1. Never store your test secrets in repository. +> 2. It is recommended to use and upload your secrets to GitHub to avoid test fails caused by default values. Read more in [CTF Test Secrets documentation](https://github.com/smartcontractkit/chainlink-testing-framework/blob/main/lib/config/README.md#test-secrets) and [guide on running GitHub workflows with your test secrets](https://github.com/smartcontractkit/chainlink-testing-framework/blob/main/lib/config/README.md#run-github-workflow-with-your-test-secrets). diff --git a/integration-tests/Makefile b/integration-tests/Makefile index 1e6828fd35c..e28b133e1f5 100644 --- a/integration-tests/Makefile +++ b/integration-tests/Makefile @@ -102,20 +102,20 @@ test_smoke_product: ## Run smoke tests for specific product ex: make test_smoke_ # Chaos .PHONY: test_chaos_pods_raw -test_chaos_pods_raw: +test_chaos_pods_raw: ## Run all chaos pod tests go test -timeout 2h -v -count=1 $(args) -p 2 -run 'Test/.*pod-chaos' ./chaos .PHONY: test_chaos_network_raw -test_chaos_network_raw: +test_chaos_network_raw: ## Run all chaos network tests go test -timeout 2h -v -count=1 $(args) -p 2 -run 'Test/.*network-chaos' ./chaos .PHONY: test_chaos_pods -test_chaos_pods: install_gotestloghelper ## Run all chaos pod tests +test_chaos_pods: install_gotestloghelper ## Run all chaos pod tests with decorated output TEST_LOG_LEVEL="disabled" \ go test -timeout 2h -count=1 -json $(args) -run 'Test/.*pod-chaos' ./chaos 2>&1 | tee ./gotest.log | gotestloghelper -json -hidepassingtests -tlogprefix -color -singlepackage .PHONY: test_chaos_network -test_chaos_network: install_gotestloghelper ## Run all chaos network tests +test_chaos_network: install_gotestloghelper ## Run all chaos network tests with decorated output TEST_LOG_LEVEL="disabled" \ go test -timeout 2h -count=1 -json $(args) -run 'Test/.*network-chaos' ./chaos 2>&1 | tee ./gotest.log | gotestloghelper -json -hidepassingtests -tlogprefix -color -singlepackage @@ -123,6 +123,14 @@ test_chaos_network: install_gotestloghelper ## Run all chaos network tests test_chaos_verbose: ## Run all smoke tests with verbose logging go test -timeout 24h -count=1 -v $(args) ./chaos +.PHONY: test_chaos_ocr +test_chaos_ocr: ## Run only OCR chaos tests + go test -timeout 2h -v -count=1 $(args) -p 2 -run 'TestOCRChaos' ./chaos + +.PHONY: test_chaos_automation +test_chaos_automation: ## Run only Automation chaos tests + go test -timeout 2h -v -count=1 $(args) -p 2 -run 'TestAutomationChaos' ./chaos + # Migrations .PHONY: test_node_migrations test_node_migrations: install_gotestloghelper ## Run all node migration tests. diff --git a/integration-tests/README.md b/integration-tests/README.md index 75fa57985ae..49aaec12dd6 100644 --- a/integration-tests/README.md +++ b/integration-tests/README.md @@ -6,11 +6,12 @@ - [Pre-requisites](#pre-requisites) - [Test and node configuration](#test-and-node-configuration) - [Run Tests](#run-tests) - - [Locally](#locally) + - [Locally (in Docker)](#locally-in-docker) - [All tests in a suite](#all-tests-in-a-suite) - [A single test](#a-single-test) - - [In CI](#in-ci) - [In Kubernetes](#in-kubernetes) + - [From local machine](#from-local-machine) + - [CI/GitHub Actions](#cigithub-actions) ## Summary @@ -25,15 +26,19 @@ This directory represent a place for different types of integration and system l ### Pre-requisites 1. [Installed Go](https://go.dev/) -2. For local testing, [Installed Docker](https://www.docker.com/). Consider [increasing resources limits needed by Docker](https://stackoverflow.com/questions/44533319/how-to-assign-more-memory-to-docker-container) as most tests require building several containers for a Decentralized Oracle Network (e.g. OCR requires 6 nodes and DBs, a simulated blockchain, and a mock server). +2. For local testing, [Installed Docker](https://www.docker.com/). Consider [increasing resources limits needed by Docker](https://stackoverflow.com/questions/44533319/how-to-assign-more-memory-to-docker-container) as most tests require building several containers for a Decentralized Oracle Network (e.g. OCR requires 6 nodes, 6 DBs, and a mock server). 3. For remote testing, access to Kubernetes cluster/AWS Docker registry (if you are pulling images from private links). 4. Docker image. If there is no image to pull from a registry, you may run tests against a custom build. Run the following command to build the image: -```bash -make build_docker_image image= tag= -``` + ```bash + make build_docker_image image= tag= + ``` -Example: `make build_docker_image image=chainlink tag=test-tag` + Example: `make build_docker_image image=chainlink tag=test-tag` + +5. RPC node/s (for testnets/mainnets). +6. EOA's (wallet) Private Key (see [How to export an account's private key](https://support.metamask.io/ru/managing-my-wallet/secret-recovery-phrase-and-private-keys/how-to-export-an-accounts-private-key/)) +7. Sufficient amount of native token and LINK on EOA per a target chain. #### Test and node configuration @@ -43,55 +48,117 @@ Example: `make build_docker_image image=chainlink tag=test-tag` 1. `cd integration-tests` 2. `source .env` -2. Setup test secrets. See "how-to" details in the [CTF config README](https://github.com/smartcontractkit/chainlink-testing-framework/blob/main/lib/config/README.md#test-secrets). +2. Setup test secrets. See "how-to" details in the [CTF config README](https://github.com/smartcontractkit/chainlink-testing-framework/blob/main/lib/config/README.md#test-secrets). If you want to run tests in CI, you will have to push test secrets to GitHub (see [Run GitHub Workflow with your test secrets](https://github.com/smartcontractkit/chainlink-testing-framework/blob/main/lib/config/README.md#run-github-workflow-with-your-test-secrets)). 3. Provide test and node configuration (for more details refer to [testconfig README](./testconfig/README.md) and `example.toml` files): 1. **Defaults** for all products are defined in `./testconfig//.toml` files. 2. To **override default values**, create a `./testconfig/overrides.toml` file (yes, in the root of `testconfig`, not a product directory) specifying the values to override by your test (see some examples in [./testconfig/ocr2/overrides](./testconfig/ocr2/overrides)). > [!IMPORTANT] > **Image version and node configs** - > 1. Do not forget to set `[ChainlinkImage].version` to test against the necessary remotely accessible version or [custom build](#optional-build-docker-image). + > 1. Pay attention to the `[ChainlinkImage].version` to test against the necessary remotely accessible version or [custom build](#pre-requisites). > 2. When running OCR-related tests, pay attention to which version of OCR you enable/override in your `overrides.toml`. - > 3. Pay attention to not committing any sensitive data. + > 3. Do not commit any sensitive data. 4. [Optional] Configure Seth (or use defaults), an evm client used by tests. Detailed instructions on how to configure it can be found in the [Seth README](./README_SETH.md) and [Seth repository](https://github.com/smartcontractkit/chainlink-testing-framework/tree/main/seth). + > [!IMPORTANT] + > **Simulated mode (no test secrets needed)** + > Tests may run in a simulated mode, on a simulated chain (1337). In the `overrides.toml` file, set the following: + > 1. `[Network].selected_networks=["simulated"]` + > 2. `[[Seth.networks]].name = "Default"` + ### Run Tests -#### Locally +#### Locally (in Docker) > [!NOTE] -> **Resources utilization** +> **Resources utilization by Docker** > It's recommended to run only one test at a time (run tests sequentially) on a local machine as it needs a lot of docker containers and can peg your resources otherwise. You will see docker containers spin up on your machine for each component of the test where you can inspect logs. ##### All tests in a suite -1. Run CLI command: `go test ./smoke/_test.go` - Example: `go test ./smoke/ocr_test.go` -2. Logs of each Chainlink container will dump into the `smoke/logs/`. -3. To enable debugging of HTTP and RPC clients set the following env vars: +1. Run CLI command(with `override.toml`): -```bash -export SETH_LOG_LEVEL=debug -export RESTY_DEBUG=true -``` + ```bash + BASE64_CONFIG_OVERRIDE=$(cat ./testconfig/overrides.toml | base64) go test -v -p 1 ./smoke/_test.go + ``` + + Example: + + ```bash + BASE64_CONFIG_OVERRIDE=$(cat ./testconfig/overrides.toml | base64) go test -v -p 1 ./smoke/ocr_test.go + ``` + + > [!WARNING] + > **Parallelized tests and nonce issues** + > Most tests are paralelized by default. To avoid nonce-related issues, it is recommended to run tests with disabled parallelization, e.g. with `-p 1`. + +2. Alternatively, you may use `make` commands (see more in [Makefile .PHONY lines](./Makefile)) for running suites of tests. + Example: + + ```bash + make test_smoke_product product="ocr" ./scripts/run_product_tests + ``` + +3. Logs of each Chainlink container will dump into the `smoke/logs/`. +4. To enable debugging of HTTP and RPC clients set the following env vars: + + ```bash + export SETH_LOG_LEVEL=debug + export RESTY_DEBUG=true + ``` ##### A single test -1. Run CLI command: `go test ./smoke/_test.go -run ` - Example: `go test ./smoke/ocr_test.go -run TestOCRBasic` +Run CLI command (with `override.toml`): + +```bash +BASE64_CONFIG_OVERRIDE=$(cat ./testconfig/overrides.toml | base64) go test -v -timeout 15m -run <"TestNameToRun"> ./ +``` -#### In CI +Example: -1. Refer [Tests Run Books](./run-books/) for more details. -2. Logs in CI uploaded as GitHub artifacts. +```bash +BASE64_CONFIG_OVERRIDE=$(cat ./testconfig/overrides.toml | base64) go test -v -timeout 15m -run "TestOCRv2Basic" ./smoke +``` #### In Kubernetes Such tests as Soak, Performance, Benchmark, and Chaos Tests remain bound to a Kubernetes run environment. -1. Ensure all necessary configuration is provided (see [Test Configuration](#test-configuration)) -2. You are logged in to your Kubernetes cluster (with `aws sso login`) -3. Run CLI command: `make test_` (see commands in [Makefile .PHONY lines](./Makefile) for more details) +1. Refer [Tests Run Books](./run-books/) to get more details on how to run specific per-product tests. +2. Logs in CI are uploaded as GitHub artifacts. + +##### From local machine + +1. Ensure all necessary configurations are provided (see [Test and node configuration](#test-and-node-configuration)) +2. [Optional] Encode `override.toml`, if any, with the following command: + + ```bash + cat testconfig/overrides.toml | base64 + ``` + +3. Copy and paste the output of the encoded `override.toml` into `BASE64_CONFIG_OVERRIDE` var in `.env` file to be later used by Kubernetes. + Example: + + ```bash + export BASE64_CONFIG_OVERRIDE='Base_64_encoded_content_of_override.toml' + ``` + + Ensure to **update you environment** with the following command: + + ```bash + source .env + ``` + +4. Log in to your Kubernetes cluster (with `aws sso login`) +5. Run test with the following CLI command: `make test_` (see make-commands in [Makefile .PHONY lines](./Makefile)). Example: `make test_soak_ocr`, `make test_soak_ocr2`, `test_node_migrations`, etc. -4. Navigate to Grafana dashboards to see test and node logs, and results. +6. Use Kubernetes namespace printed out in logs to monitor and analyze test runs. +7. Navigate to Grafana dashboards to for test results and logs. + +#### CI/GitHub Actions + +1. Ensure all necessary configurations are provided (see [Test and node configuration](#test-and-node-configuration)). +2. Follow instructions provided in [E2E Tests on GitHub CI](../.github/E2E_TESTS_ON_GITHUB_CI.md). +3. Refer [Tests Run Books](./run-books/) to get more details on how to run specific per-product tests. diff --git a/integration-tests/run-books/OCR.md b/integration-tests/run-books/OCR.md new file mode 100644 index 00000000000..e654001194d --- /dev/null +++ b/integration-tests/run-books/OCR.md @@ -0,0 +1,214 @@ + +# OCR Tests Run-Book + +- [OCR Tests Run-Book](#ocr-tests-run-book) + - [Summary](#summary) + - [Instructions](#instructions) + - [Pre-requisites](#pre-requisites) + - [Run Tests](#run-tests) + - [COMMON COMMANDS](#common-commands) + - [Docker (with overrides)](#docker-with-overrides) + - [CI (with overrides)](#ci-with-overrides) + - [SMOKE](#smoke) + - [Docker](#docker) + - [CI](#ci) + - [SOAK](#soak) + - [Kubernetes](#kubernetes) + - [CI](#ci-1) + - [LOAD](#load) + - [CHAOS](#chaos) + - [Kubernetes](#kubernetes-1) + - [CI](#ci-2) + - [MIGRATION](#migration) + +## Summary + +This run-book is a guideline for running on demand OCR tests against any blockchain. + +## Instructions + +### Pre-requisites + +> [!IMPORTANT] +> +> 1. Ensure [main Pre-requisites](../README.md#pre-requisites) are met. +> 2. Pay attention to the OCR version enabled in `overreride.toml`. +> 3. Use `-p 1` to disable tests parallelization and avoid nonce-related issues (or comment `t.Parallel()`). +> 4. For running tests in Kubernetes and CI, ensure test secrets are provided/uploaded to GitHub (ref. [CTF README#test-secrets](https://github.com/smartcontractkit/chainlink-testing-framework/blob/main/lib/config/README.md#test-secrets)). + +### Run Tests + +Below you may find instructions for running tests in different environments. + +#### COMMON COMMANDS + +Reuse the commands below to run different tests by their types/suites. + +##### Docker (with overrides) + +Any test suite/test can be run in Docker using the following `go` command: + +```bash +BASE64_CONFIG_OVERRIDE=$(cat ./testconfig/overrides.toml | base64) go test -v -timeout -p 1 <./path/to/_test.go file> +``` + +Example: + +```bash +BASE64_CONFIG_OVERRIDE=$(cat ./testconfig/overrides.toml | base64) go test -v -timeout 60m -p 1 ./smoke/ocr2_test.go +``` + +##### CI (with overrides) + +For the most tests run [Selected E2E Tests Workflow](https://github.com/smartcontractkit/chainlink/actions/workflows/run-selected-e2e-tests.yml) in GitHub, either manually or using `gh` command unless the otherwise stated: + +```bash +gh workflow run "run-selected-e2e-tests.yml" \ +--ref \ +-f chainlink_version="v" \ # Optional, default is image created from develop branch. Not needed if you run tests against existing environment +-f workflow_run_name="Any name" \ # Optional +-f test_ids="" \ # see /chainlink/.github/e2e-tests.yml for IDS +-f test_secrets_override_key= \ # Optional, can be obtained when secrets are uploaded to GitHub +-f test_config_override_path= \ # Optional +-f with_existing_remote_runner_version= \ # Optional +``` + +Example: + +```bash +gh workflow run "run-selected-e2e-tests.yml" \ +--ref develop \ +-f chainlink_version="v2.17.0-beta0" \ +-f test_ids="smoke/ocr2_test.go:*" \ +-f workflow_run_name="Smoke:OCR2:2.17.0-beta0" \ +-f test_secrets_override_key=YOUR_TEST_SECRETS_ID \ +-f test_config_override_path=./testconfig/ocr2/overrides/base_sepolia.toml +``` + +#### SMOKE + +##### Docker + +Refer [COMMON COMMANDS#Docker (with overrides)](#docker-with-overrides). Override path as follows: + +With forwarders: +`./smoke/forwarder_ocr_test.go` +`./smoke/forwarder_ocr2_test.go` + +No forwarders: +`./smoke/ocr_test.go` +`./smoke/ocr2_test.go` + +##### CI + +Refer [COMMON COMMANDS#CI](#ci). Override `test_ids` as follows: + +`smoke/ocr_test.go:*` - all OCR1 tests +`smoke/ocr2_test.go:*` - all OCR2 tests + +#### SOAK + +> [!IMPORTANT] +> These tests require logging in to Kubernetes cluster (`aws sso login`). + +Refer [Test config README](../testconfig/README.md) for more details about Soak tests configuration. + +##### Kubernetes + +Refer [COMMON COMMANDS#Docker (with overrides)](#docker-with-overrides). Override path as follows: + +With forwarders: +`-run 'TestForwarderOCRv1Soak' ./soak` or `make test_soak_forwarder_ocr1` +`-run 'TestForwarderOCRv2Soak' ./soak` or `make test_soak_forwarder_ocr2` + +No forwarders: +`-run 'TestOCRv1Soak' ./soak` or `make test_soak_ocr` +`-run 'TestOCRv2Soak' ./soak` or `make test_soak_ocr2` + +With reorg below finality and `FinalityTagEnabled=false`: +`-run 'TestOCRSoak_GethReorgBelowFinality_FinalityTagDisabled' ./soak` or `make test_soak_ocr_reorg_1` + +With reorg below finality and `FinalityTagEnabled=true`: +`-run 'TestOCRSoak_GethReorgBelowFinality_FinalityTagEnabled' ./soak` or `make test_soak_ocr_reorg_2` + +With gas spike: +`-run 'TestOCRSoak_GasSpike' ./soak` or `make test_soak_ocr_gas_spike` + +With change of a block gas limit (creating block congestion): +`-run 'TestOCRSoak_ChangeBlockGasLimit' ./soak` or `make test_soak_ocr_gas_limit_change` + +All RPCs get down for all nodes: +`-run 'TestOCRSoak_RPCDownForAllCLNodes' ./soak` or `make test_soak_ocr_rpc_down_all_cl_nodes` + +50% of nodes get RPCs down: +`-run 'TestOCRSoak_RPCDownForHalfCLNodes' ./soak` or `make test_soak_ocr_rpc_down_half_cl_nodes` + +##### CI + +Use [On Demand OCR Soak Test](https://github.com/smartcontractkit/chainlink/actions/workflows/on-demand-ocr-soak-test.yml) workflow in GitHub. + +OR + +Run [On Demand OCR Soak Test](https://github.com/smartcontractkit/chainlink/actions/workflows/run-on-demand-ocr-soak-test.yml) workflow with `gh` as follows: + +```bash +gh workflow run "on-demand-ocr-soak-test.yml" \ +--ref \ +-f chainlink_version="v" \ # Optional, default is image created from develop branch. Not needed if you run tests against existing environment +-f workflow_run_name="Any name" \ # Optional +-f test_ids="" \ # see /chainlink/.github/e2e-tests.yml for IDS +-f test_secrets_override_key= \ # Optional, can be obtained when secrets are uploaded to GitHub +-f test_config_override_path= \ # Optional +``` + +Example: + +```bash +gh workflow run "on-demand-ocr-soak-test.yml" \ +--ref develop \ +-f chainlink_version="v2.17.0-beta0" \ +-f testToRun="soak/ocr_test.go:TestOCRv2Soak" \ +-f test_secrets_override_key=YOUR_TEST_SECRETS_ID \ +-f slackMemberID="YOUR_SLACK_MEMBER_ID" +``` + +The following values may be used in the `testToRun` field (ref. [Soak#Kubernetes](#kubernetes) for more details): +`soak/ocr_test.go:TestOCRv1Soak` +`soak/ocr_test.go:TestOCRv2Soak` +`soak/ocr_test.go:TestForwarderOCRv1Soak` +`soak/ocr_test.go:TestForwarderOCRv2Soak` +`soak/ocr_test.go:TestOCRSoak_GethReorgBelowFinality_FinalityTagDisabled` +`soak/ocr_test.go:TestOCRSoak_GethReorgBelowFinality_FinalityTagEnabled` +`soak/ocr_test.go:TestOCRSoak_GasSpike` +`soak/ocr_test.go:TestOCRSoak_ChangeBlockGasLimit` +`soak/ocr_test.go:TestOCRSoak_RPCDownForAllCLNodes` +`soak/ocr_test.go:TestOCRSoak_RPCDownForHalfCLNodes` + +#### LOAD + +Ref: [Load Tests README](../load/ocr/README.md) + +#### CHAOS + +> [!IMPORTANT] +> These tests require logging in to Kubernetes cluster (`aws sso login`). + +##### Kubernetes + +Refer [COMMON COMMANDS#Docker (with overrides)](#docker-with-overrides). Override path as follows: + +`-run 'TestOCRChaos' ./chaos` + +OR + +`make test_ocr_chaos` + +##### CI + +Refer [COMMON COMMANDS#CI](#ci). Override `test_ids` as follows: + +`chaos/ocr_chaos_test.go` + +#### MIGRATION + +TODO: Add README under [migration directory](../migration/) diff --git a/integration-tests/smoke/ocr_test.go b/integration-tests/smoke/ocr_test.go index 6556591b6d5..e3172e23b8c 100644 --- a/integration-tests/smoke/ocr_test.go +++ b/integration-tests/smoke/ocr_test.go @@ -111,18 +111,18 @@ func prepareORCv1SmokeTestEnv(t *testing.T, l zerolog.Logger, firstRoundResult i }) linkContract, err := actions.LinkTokenContract(l, sethClient, config.OCR) - require.NoError(t, err, "Error loading/deploying link token contract") + require.NoError(t, err, "Error loading/deploying LinkToken contract") ocrInstances, err := actions.SetupOCRv1Contracts(l, sethClient, config.OCR, common.HexToAddress(linkContract.Address()), contracts.ChainlinkClientToChainlinkNodeWithKeysAndAddress(workerNodes)) require.NoError(t, err, "Error deploying OCR contracts") - err = actions.CreateOCRJobsLocal(ocrInstances, bootstrapNode, workerNodes, 5, env.MockAdapter, big.NewInt(sethClient.ChainID)) - require.NoError(t, err, "Error creating OCR jobs") - // there is no need to fund the nodes unless the OCR contract and job are configured err = actions.FundChainlinkNodesFromRootAddress(l, sethClient, contracts.ChainlinkClientToChainlinkNodeWithKeysAndAddress(workerNodes), big.NewFloat(*config.Common.ChainlinkNodeFunding)) require.NoError(t, err, "Error funding Chainlink nodes") + err = actions.CreateOCRJobsLocal(ocrInstances, bootstrapNode, workerNodes, 5, env.MockAdapter, big.NewInt(sethClient.ChainID)) + require.NoError(t, err, "Error creating OCR jobs") + err = actions.WatchNewOCRRound(l, sethClient, 1, contracts.V1OffChainAgrregatorToOffChainAggregatorWithRounds(ocrInstances), time.Duration(3*time.Minute)) require.NoError(t, err, "Error watching for new OCR round")