add automation test configs (#14455)

* add automation test configs

* update grafana config

* update testid in workflow

* update testid in e2e-tests

* restructure configs

* update config

* add testType input in benchmark workflow

* add testType input in e2e reusable workflow

* Revert "add testType input in e2e reusable workflow"

This reverts commit 937607c.

* update benchmark workflow

* update test log level for load and benchmark

---------

Co-authored-by: lukaszcl <[email protected]>

.github/e2e-tests.yml

@@ -490,6 +490,7 @@ runner-test-matrix:
     remote_runner_memory: 4Gi
     test_secrets_required: true
     test_env_vars:
+      TEST_LOG_LEVEL: info
       TEST_SUITE: automationv2_1
     pyroscope_env: automation-load-test
 
@@ -591,7 +592,7 @@ runner-test-matrix:
     test_env_vars:
       TEST_SUITE: chaos
 
-  - id: benchmark/automation_test.go:^TestAutomationBenchmark$
+  - id: benchmark/automation_test.go:TestAutomationBenchmark
     path: integration-tests/benchmark/automation_test.go
     test_env_type: k8s-remote-runner
     remote_runner_memory: 4Gi
@@ -601,9 +602,24 @@ runner-test-matrix:
     test_cmd: cd integration-tests/benchmark && go test -v -test.run ^TestAutomationBenchmark$ -test.parallel=1 -timeout 30m -count=1 -json
     pyroscope_env: ci-benchmark-automation-nightly
     test_env_vars:
+      TEST_LOG_LEVEL: info
       TEST_SUITE: benchmark
       TEST_TYPE: benchmark
 
+  - id: soak/automation_test.go:TestAutomationBenchmark
+    path: integration-tests/benchmark/automation_test.go
+    test_env_type: k8s-remote-runner
+    remote_runner_memory: 4Gi
+    runs_on: ubuntu-latest
+    # workflows:
+    # - Nightly E2E Tests
+    test_cmd: cd integration-tests/benchmark && go test -v -test.run ^TestAutomationBenchmark$ -test.parallel=1 -timeout 30m -count=1 -json
+    pyroscope_env: ci-benchmark-automation-nightly
+    test_env_vars:
+      TEST_LOG_LEVEL: info
+      TEST_SUITE: benchmark
+      TEST_TYPE: soak
+
   # END: Automation tests
 
   # START: VRF tests

.github/workflows/automation-benchmark-tests.yml

@@ -14,14 +14,19 @@ on:
         description: Notifies test results (Not your @)
         required: true
         default: U02Q14G80TY
-        type: string     
+        type: string
+      testType:
+        description: Type of test to run (benchmark, soak)
+        required: true
+        default: benchmark
+        type: string
 
 jobs:
   run-e2e-tests-workflow:
     name: Run E2E Tests
     uses: ./.github/workflows/run-e2e-tests-reusable-workflow.yml
     with:
-      test_ids: 'benchmark/keeper_test.go:TestAutomationBenchmark'
+      test_ids: '${{ inputs.testType }}/automation_test.go:TestAutomationBenchmark'
       test_config_override_path: ${{ inputs.test_config_override_path }}
       SLACK_USER: ${{ inputs.slackMemberID }}
       SLACK_CHANNEL: C03KJ5S7KEK

integration-tests/testconfig/automation/automation.toml

@@ -2,6 +2,9 @@
 [Common]
 chainlink_node_funding = 2.0
 
+[Pyroscope]
+enabled=false
+
 [NodeConfig]
 BaseConfigTOML = """
 [Feature]
@@ -207,6 +210,10 @@ max_perform_data_size=5_000
 max_revert_data_size=5_000
 
 # load test specific overrides
+[Load.Logging.Grafana]
+base_url="https://grafana.ops.prod.cldev.sh"
+dashboard_url="/d/a4899f53-f709-430a-aec2-24f32198dcc9/chainlink-automation-v2-load-test"
+
 [Load.Seth]
 ephemeral_addresses_number = 100
 root_key_funds_buffer = 1_000_000
@@ -301,6 +308,9 @@ max_revert_data_size=5_000
 enabled=false
 
 # automation benchmark test specific overrides
+[Benchmark.Logging.Grafana]
+base_url="https://grafana.ops.prod.cldev.sh"
+dashboard_url="/d/Q8n6m1unz/chainlink-automation-benchmark-test"
 
 # will retry roughly for 1h before giving up (900 * 4s)
 [Benchmark.Automation.Resiliency]
@@ -420,6 +430,9 @@ max_perform_data_size=5_000
 max_revert_data_size=5_000
 
 # automation soak test specific overrides
+[Soak.Logging.Grafana]
+base_url="https://grafana.ops.prod.cldev.sh"
+dashboard_url="/d/Q8n6m1unz/chainlink-automation-benchmark-test"
 
 # will retry roughly for 1h before giving up (900 * 4s)
 [Soak.Automation.Resiliency]

integration-tests/testconfig/automation/overrides/benchmark/1000Upkeeps-1h-2_1.toml

@@ -0,0 +1,15 @@
+[ChainlinkImage]
+version="latest"
+
+[Benchmark.Automation.Benchmark]
+registry_to_test = "2_1"
+number_of_registries = 1
+number_of_nodes = 6
+number_of_upkeeps = 1000
+upkeep_gas_limit = 1500000
+check_gas_to_burn = 10000
+perform_gas_to_burn = 1000
+block_range = 3600
+block_interval = 60
+forces_single_tx_key = false
+delete_jobs_on_end = true

integration-tests/testconfig/automation/overrides/benchmark/1000Upkeeps-1h-2_3.toml

@@ -0,0 +1,15 @@
+[ChainlinkImage]
+version="latest"
+
+[Benchmark.Automation.Benchmark]
+registry_to_test = "2_3"
+number_of_registries = 1
+number_of_nodes = 6
+number_of_upkeeps = 1000
+upkeep_gas_limit = 1500000
+check_gas_to_burn = 10000
+perform_gas_to_burn = 1000
+block_range = 3600
+block_interval = 60
+forces_single_tx_key = false
+delete_jobs_on_end = true

integration-tests/testconfig/automation/overrides/load/500Upkeeps-1x-1h.toml

@@ -0,0 +1,47 @@
+[ChainlinkImage]
+version="latest"
+
+[Load.Seth]
+root_key_funds_buffer = 1_000_000
+ephemeral_addresses_number = 300
+
+[Load.Seth.nonce_manager]
+key_sync_timeout = "100s"
+
+[Load.Common]
+chainlink_node_funding = 1000
+
+[Load.Automation.AutomationConfig]
+use_log_buffer_v1=false
+
+[Load.Automation.AutomationConfig.PluginConfig.LogProviderConfig]
+block_rate=1
+log_limit=2
+
+[Load.Automation]
+[Load.Automation.General]
+number_of_nodes=6
+duration=3600
+block_time=1
+spec_type="recommended"
+chainlink_node_log_level="debug"
+use_prometheus=true
+remove_namespace = true
+
+[Load.Automation.DataStreams]
+enabled=false
+
+[[Load.Automation.Load]]
+number_of_upkeeps=500
+number_of_events = 1
+number_of_spam_matching_events = 0
+number_of_spam_non_matching_events = 0
+check_burn_amount = 0
+perform_burn_amount = 0
+upkeep_gas_limit = 1000000
+shared_trigger = false
+is_streams_lookup = false
+feeds = []
+
+[Pyroscope]
+enabled=false

integration-tests/testconfig/automation/overrides/load/50Upkeeps-1x-12h.toml

@@ -0,0 +1,43 @@
+[ChainlinkImage]
+version="latest"
+
+[Load.Seth]
+root_key_funds_buffer = 1_000_000
+
+[Load.Common]
+chainlink_node_funding = 10000
+
+[Load.Automation.AutomationConfig]
+use_log_buffer_v1=false
+
+[Load.Automation.AutomationConfig.PluginConfig.LogProviderConfig]
+block_rate=1
+log_limit=2
+
+[Load.Automation]
+[Load.Automation.General]
+number_of_nodes=6
+duration=43200
+block_time=1
+spec_type="recommended"
+chainlink_node_log_level="debug"
+use_prometheus=true
+remove_namespace = true
+
+[Load.Automation.DataStreams]
+enabled=false
+
+[[Load.Automation.Load]]
+number_of_upkeeps=50
+number_of_events = 1
+number_of_spam_matching_events = 0
+number_of_spam_non_matching_events = 0
+check_burn_amount = 0
+perform_burn_amount = 0
+upkeep_gas_limit = 1000000
+shared_trigger = false
+is_streams_lookup = false
+feeds = []
+
+[Pyroscope]
+enabled=false

integration-tests/testconfig/automation/overrides/load/50Upkeeps-1x-1h.toml

@@ -0,0 +1,43 @@
+[ChainlinkImage]
+version="latest"
+
+[Load.Seth]
+root_key_funds_buffer = 1_000_000
+
+[Load.Common]
+chainlink_node_funding = 1000
+
+[Load.Automation.AutomationConfig]
+use_log_buffer_v1=false
+
+[Load.Automation.AutomationConfig.PluginConfig.LogProviderConfig]
+block_rate=1
+log_limit=2
+
+[Load.Automation]
+[Load.Automation.General]
+number_of_nodes=6
+duration=3600
+block_time=1
+spec_type="recommended"
+chainlink_node_log_level="debug"
+use_prometheus=true
+remove_namespace = true
+
+[Load.Automation.DataStreams]
+enabled=false
+
+[[Load.Automation.Load]]
+number_of_upkeeps=50
+number_of_events = 1
+number_of_spam_matching_events = 0
+number_of_spam_non_matching_events = 0
+check_burn_amount = 0
+perform_burn_amount = 0
+upkeep_gas_limit = 1000000
+shared_trigger = false
+is_streams_lookup = false
+feeds = []
+
+[Pyroscope]
+enabled=false

integration-tests/testconfig/automation/overrides/soak/50Upkeeps-8h-2_1.toml

@@ -0,0 +1,15 @@
+[ChainlinkImage]
+version="latest"
+
+[Soak.Automation.Benchmark]
+registry_to_test = "2_1"
+number_of_registries = 1
+number_of_nodes = 6
+number_of_upkeeps = 50
+upkeep_gas_limit = 1500000
+check_gas_to_burn = 10000
+perform_gas_to_burn = 1000
+block_range = 28800
+block_interval = 300
+forces_single_tx_key = false
+delete_jobs_on_end = true

integration-tests/testconfig/automation/overrides/soak/50Upkeeps-8h-2_3.toml

@@ -0,0 +1,15 @@
+[ChainlinkImage]
+version="latest"
+
+[Soak.Automation.Benchmark]
+registry_to_test = "2_3"
+number_of_registries = 1
+number_of_nodes = 6
+number_of_upkeeps = 50
+upkeep_gas_limit = 1500000
+check_gas_to_burn = 10000
+perform_gas_to_burn = 1000
+block_range = 28800
+block_interval = 300
+forces_single_tx_key = false
+delete_jobs_on_end = true

integration-tests/testconfig/keeper/keeper.toml

@@ -32,4 +32,4 @@ HTTPSPort = 0
 
 [Keeper]
 TurnLookBack = 0
-"""
+"""