use correct repositories for compatibility tests

smartcontractkit · Oct 31, 2024 · 8b02474 · 8b02474
1 parent d888177
commit 8b02474
Show file tree

Hide file tree

Showing 2 changed files with 73 additions and 50 deletions.
diff --git a/.github/workflows/ccip-client-compatibility-tests.yml b/.github/workflows/ccip-client-compatibility-tests.yml
@@ -238,14 +238,14 @@ jobs:
       matrix:
         mirror:
           - name: ethereum/client-go
-            expression: '^(alltools-v|v)[0-9]\.[0-9]+\.[0-9]+$'
+            expression: '^(alltools-v|v)?[0-9]\.[0-9]+\.[0-9]+$'
           - name: hyperledger/besu
-            expression: '^[0-9]+\.[0-9]+(\.[0-9]+)?$'
+            expression: 'v?^[0-9]+\.[0-9]+(\.[0-9]+)?$'
             page_size: 300
           - name: thorax/erigon
-            expression: '^v[0-9]+\.[0-9]+\.[0-9]+$'
+            expression: '^v?[0-9]+\.[0-9]+\.[0-9]+$'
           - name: nethermind/nethermind
-            expression: '^[0-9]+\.[0-9]+\.[0-9]+$'
+            expression: '^v?[0-9]+\.[0-9]+\.[0-9]+$'
           - name: tofelb/ethereum-genesis-generator
             expression: '^[0-9]+\.[0-9]+\.[0-9]+(\-slots\-per\-epoch)?'
     steps:
@@ -330,7 +330,7 @@ jobs:
       - name: Set Up ecrimagefetcher
         shell: bash
         run: |
-          go install github.com/smartcontractkit/chainlink-testing-framework/tools/ecrimagefetcher@v1.0.1
+          go install github.com/smartcontractkit/chainlink-testing-framework/tools/ecrimagefetcher@v1.50.2
       - name: Get latest docker images from ECR
         if: ${{ github.event.inputs.base64TestList == '' }}
         env:
@@ -360,7 +360,7 @@ jobs:
           
           if [[ "$ETH_IMPLEMENTATIONS" == *"erigon"* ]]; then
             # 2.60.0 and 2.60.1 are ignored as they stopped working with CL node
-            erigon_images=$(ecrimagefetcher 'thorax/erigon' '^v[0-9]+\.[0-9]+\.[0-9]+$' ${{ env.LATEST_IMAGE_COUNT }} "<v2.60.0")
+            erigon_images=$(ecrimagefetcher 'thorax/erigon' '^v?[0-9]+\.[0-9]+\.[0-9]+$' ${{ env.LATEST_IMAGE_COUNT }} "<v2.60.0 || >v2.60.2")
             echo "ERIGON_IMAGES=$erigon_images" >> $GITHUB_ENV
             echo "Erigon latest images: $erigon_images" 
           fi          
@@ -463,47 +463,44 @@ jobs:
 
   run-client-compatibility-matrix:
     name: CCIP Compatibility with ${{ matrix.evm_node.visible_name }}
-    if: always() && needs.should-run.outputs.should_run == 'true'
+    if: always() && needs.should-run.outputs.should_run == 'true' && (needs.build-chainlink.result == 'success' || needs.build-chainlink.result == 'skipped') && needs.prepare-compatibility-matrix.outputs.matrix != ''
     environment: integration
     permissions:
       checks: write
       pull-requests: write
       id-token: write
       contents: read
-    needs: [build-chainlink, prepare-compatibility-matrix, should-run, select-versions]
+    needs:
+      - build-chainlink
+      - prepare-compatibility-matrix
+      - should-run
+      - select-versions
     env:
+      SELECTED_NETWORKS: SIMULATED,SIMULATED_1,SIMULATED_2
       CHAINLINK_COMMIT_SHA: ${{ needs.select-versions.outputs.chainlink_version }}
       CHAINLINK_ENV_USER: ${{ github.actor }}
       TEST_LOG_LEVEL: debug
     strategy:
       fail-fast: false
+      max-parallel: 10
       matrix:
         evm_node: ${{fromJson(needs.prepare-compatibility-matrix.outputs.matrix)}}
     runs-on: ubuntu-latest
     steps:
       - name: Checkout the repo
         uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
         with:
-          repository: smartcontractkit/ccip
+          repository: smartcontractkit/chainlink
           ref: ${{ needs.select-versions.outputs.chainlink_version }}
-      - name: Prepare Base64 TOML override
-        uses: ./.github/actions/setup-create-base64-config
-        with:
-          runId: ${{ github.run_id }}
-          testLogCollect: ${{ vars.TEST_LOG_COLLECT }}
-          selectedNetworks: ${{ matrix.evm_node.networks }}
-          chainlinkVersion: ${{ needs.select-versions.outputs.chainlink_version }}
-          logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }}
-      - name: Prepare Base64 TOML override for CCIP secrets
-        uses: ./.github/actions/setup-create-base64-config-ccip
-        id: setup_create_base64_config_ccip
+      - name: Setup GAP for Grafana
+        uses: smartcontractkit/.github/actions/setup-gap@d316f66b2990ea4daa479daa3de6fc92b00f863e # [email protected]
         with:
-          runId: ${{ github.run_id }}
-          selectedNetworks: ${{ matrix.evm_node.networks }}
-          testLogCollect: ${{ vars.TEST_LOG_COLLECT }}
-          chainlinkVersion: ${{ needs.select-versions.outputs.chainlink_version }}
-          logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }}
-          customEvmNodes: ${{ matrix.evm_node.docker_image }}
+          # aws inputs
+          aws-region: ${{ secrets.AWS_REGION }}
+          aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
+          api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}
+          # other inputs
+          duplicate-authorization-header: "true"
       - name: Prepare test log name
         run: |
           replace_special_chars() {
@@ -525,27 +522,46 @@ jobs:
 
             echo "$modified_string"
           }
-          echo "TEST_LOG_NAME=$(replace_special_chars "ccip-${{ matrix.evm_node.name }}-test-logs")" >> $GITHUB_ENV
-      - name: Print Test details - ${{ matrix.evm_node.docker_image }}
-        run: |
-          echo "EVM Implementation Docker Image: ${{ matrix.evm_node.docker_image }}"
-          echo "EVM Implementation Networks: ${{ matrix.evm_node.networks }}"
-          echo "Test identifier: ${{ matrix.evm_node.name }}"
+          echo "TEST_LOG_NAME=$(replace_special_chars "${{ matrix.evm_node.product }}-${{ matrix.evm_node.docker_image }}-test-logs")" >> $GITHUB_ENV
+      #      - name: Collect Workflow Telemetry
+      #        uses: catchpoint/workflow-telemetry-action@v2
+      #        with:
+      #          comment_on_pr: false
+      #          theme: 'dark'
+      - name: Prepare Base64 CCIP TOML secrets
+        uses: ./.github/actions/setup-create-base64-config-ccip
+        id: setup_create_base64_config_ccip
+        with:
+          runId: ${{ github.run_id }}
+          testLogCollect: ${{ vars.TEST_LOG_COLLECT }}
+          selectedNetworks: SIMULATED_1,SIMULATED_2
+          chainlinkVersion: ${{ needs.select-versions.outputs.chainlink_version }}
+          logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }}
       - name: Run Tests
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@d38226be720c5ccc1ff4d3cee40608ebf264cd59 # v2.3.26
+        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@aa8eea635029ab8d95abd3c206f56dae1e22e623 # v2.3.28
         env:
           BASE64_CONFIG_OVERRIDE: ${{ steps.setup_create_base64_config_ccip.outputs.base64_config }}
           TEST_BASE64_CONFIG_OVERRIDE: ${{ steps.setup_create_base64_config_ccip.outputs.base64_config }}
+          DEBUG_RESTY: false
         with:
-          test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=2 ${{ matrix.evm_node.run }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci
+          test_command_to_run: cd ./integration-tests && touch .root_dir && go test -timeout 30m -count=1 -json ${{ matrix.evm_node.run }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs
           test_download_vendor_packages_command: cd ./integration-tests && go mod download
+          test_config_chainlink_version: ${{ needs.select-versions.outputs.chainlink_version }}
+          test_config_selected_networks: ${{ env.SELECTED_NETWORKS}}
+          test_config_logging_run_id: ${{ github.run_id }}
+          test_config_test_log_collect: ${{ vars.TEST_LOG_COLLECT }}
+          test_config_logstream_log_targets: ${{ vars.LOGSTREAM_LOG_TARGETS }}
+          test_config_private_ethereum_network_execution_layer: ${{ matrix.evm_node.eth_implementation || 'geth' }}
+          test_config_private_ethereum_network_custom_docker_image: ${{ matrix.evm_node.docker_image }}
+          cl_repo: ${{ env.CHAINLINK_IMAGE }}
+          cl_image_tag: ${{ needs.select-versions.outputs.chainlink_version }}
           aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
-          artifacts_name:  ${{ env.TEST_LOG_NAME }}
+          artifacts_name: ${{ env.TEST_LOG_NAME }}
           artifacts_location: |
             ./integration-tests/smoke/logs/
             ./integration-tests/ccip-tests/smoke/logs/*
             /tmp/gotest.log
-          publish_check_name:  ${{ matrix.evm_node.name }}
+          publish_check_name: ${{ matrix.evm_node.product }}-${{ matrix.evm_node.eth_implementation }}
           token: ${{ secrets.GITHUB_TOKEN }}
           go_mod_path: ./integration-tests/go.mod
           cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }}
@@ -554,16 +570,23 @@ jobs:
           QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
           QA_KUBECONFIG: ""
           should_tidy: "false"
-          DEFAULT_LOKI_TENANT_ID: ${{ vars.LOKI_TENANT_ID }}
-          DEFAULT_LOKI_ENDPOINT: ${{ secrets.LOKI_URL_CI }}
-          DEFAULT_LOKI_BASIC_AUTH: ${{ secrets.LOKI_BASIC_AUTH }}
+          go_coverage_src_dir: /var/tmp/go-coverage
+          go_coverage_dest_dir: ${{ github.workspace }}/.covdata
           DEFAULT_CHAINLINK_IMAGE: ${{ env.CHAINLINK_IMAGE }}
-          DEFAULT_GRAFANA_BASE_URL: ${{ vars.GRAFANA_URL }}
+          DEFAULT_LOKI_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
+          DEFAULT_LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push
+          DEFAULT_LOKI_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
+          DEFAULT_GRAFANA_BASE_URL: "http://localhost:8080/primary"
           DEFAULT_GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
-          DEFAULT_PYROSCOPE_SERVER_URL: ${{ !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725
+          DEFAULT_GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
+          DEFAULT_PYROSCOPE_SERVER_URL: ${{ secrets.QA_PYROSCOPE_INSTANCE }}
           DEFAULT_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}
-          DEFAULT_PYROSCOPE_ENVIRONMENT: ci-ccip-bidirectional-lane-${{ matrix.evm_node.name }}
-          DEFAULT_PYROSCOPE_ENABLED: 'true'
+          DEFAULT_PYROSCOPE_ENVIRONMENT: ci-client-compatability-${{ matrix.eth_client }}-testnet
+          DEFAULT_PYROSCOPE_ENABLED: "true"
+
+      - name: Print failed test summary
+        if: always()
+        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/show-test-summary@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25
 
       - name: Print failed test summary
         if: always()

diff --git a/.github/workflows/client-compatibility-tests.yml b/.github/workflows/client-compatibility-tests.yml
@@ -29,7 +29,7 @@ on:
         type: string
 
 env:
-  CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-ccip
+  CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
   INTERNAL_DOCKER_REPO: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com
   MOD_CACHE_VERSION: 2
 
@@ -283,14 +283,14 @@ jobs:
       matrix:
         mirror:
           - name: ethereum/client-go
-            expression: '^(alltools-v|v)[0-9]\.[0-9]+\.[0-9]+$'
+            expression: '^(alltools-v|v)?[0-9]\.[0-9]+\.[0-9]+$'
           - name: hyperledger/besu
-            expression: '^[0-9]+\.[0-9]+(\.[0-9]+)?$'
+            expression: 'v?^[0-9]+\.[0-9]+(\.[0-9]+)?$'
             page_size: 300
           - name: thorax/erigon
-            expression: '^v[0-9]+\.[0-9]+\.[0-9]+$'
+            expression: '^v?[0-9]+\.[0-9]+\.[0-9]+$'
           - name: nethermind/nethermind
-            expression: '^[0-9]+\.[0-9]+\.[0-9]+$'
+            expression: '^v?[0-9]+\.[0-9]+\.[0-9]+$'
           - name: tofelb/ethereum-genesis-generator
             expression: '^[0-9]+\.[0-9]+\.[0-9]+(\-slots\-per\-epoch)?'
           - name: ghcr.io/paradigmxyz/reth
@@ -386,7 +386,7 @@ jobs:
       - name: Set Up ecrimagefetcher
         shell: bash
         run: |
-          go install github.com/smartcontractkit/chainlink-testing-framework/tools/ecrimagefetcher@v1.0.1
+          go install github.com/smartcontractkit/chainlink-testing-framework/tools/ecrimagefetcher@v1.50.2
       - name: Get latest docker images from ECR
         if: ${{ github.event.inputs.base64TestList == '' }}
         env:
@@ -416,7 +416,7 @@ jobs:
 
           if [[ "$ETH_IMPLEMENTATIONS" == *"erigon"* ]]; then
             # 2.60.0 and 2.60.1 are ignored as they stopped working with CL node
-            erigon_images=$(ecrimagefetcher 'thorax/erigon' '^v[0-9]+\.[0-9]+\.[0-9]+$' ${{ env.LATEST_IMAGE_COUNT }} "<v2.60.0")
+            erigon_images=$(ecrimagefetcher 'thorax/erigon' '^v?[0-9]+\.[0-9]+\.[0-9]+$' ${{ env.LATEST_IMAGE_COUNT }} "<v2.60.0 || >v2.60.2")
             echo "ERIGON_IMAGES=$erigon_images" >> $GITHUB_ENV
             echo "Erigon latest images: $erigon_images"
           fi