DEVSVCS-138: adding billing test and GHA workflow running smoke tests (…

…#14265)

* DEVSVCS-138: adding billing test and GHA workflow running smoke tests

* DEVSVCS-138: minor fixes

* DEVSVCS-138: adding sub_billing_tolerance_wei

.github/workflows/on-demand-vrfv2plus-smoke-tests.yml

@@ -0,0 +1,115 @@
+name: On Demand VRFV2 Plus Smoke Tests
+on:
+  workflow_dispatch:
+    inputs:
+      base64Config:
+        description: base64-ed config
+        required: true
+        type: string
+      test_suite:
+        description: "Test Suite to run"
+        required: true
+        type: choice
+        options:
+          - "All Tests"
+          - "Selected Tests"
+      test_list_regex:
+        description: "Regex for 'Selected Tests' to run"
+        required: false
+        default: "(TestVRFv2Plus$/(Link_Billing|Native_Billing|Direct_Funding)|TestVRFV2PlusWithBHS|TestVRFV2PlusWithBHF)"
+      test_secrets_override_key:
+        description: 'Key to run tests with custom test secrets'
+        required: false
+        type: string
+
+jobs:
+  vrfv2plus_smoke_test:
+    name: VRFV2 Plus Smoke Test
+    environment: integration
+    runs-on: ubuntu22.04-8cores-32GB
+    permissions:
+      checks: write
+      pull-requests: write
+      id-token: write
+      contents: read
+    env:
+      LOKI_URL: ${{ secrets.LOKI_URL }}
+      LOKI_TENANT_ID: ${{ secrets.LOKI_TENANT_ID }}
+      LOKI_BASIC_AUTH: ${{ secrets.LOKI_BASIC_AUTH }}
+      TEST_LOG_LEVEL: debug
+      REF_NAME: ${{ github.head_ref || github.ref_name }}
+      SLACK_API_KEY: ${{ secrets.QA_SLACK_API_KEY }}
+      SLACK_CHANNEL: ${{ secrets.QA_VRF_SLACK_CHANNEL }}
+      GRAFANA_URL: "http://localhost:8080/primary"
+      GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
+      GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
+      WASP_LOG_LEVEL: info
+    steps:
+      - name: Collect Metrics
+        id: collect-gha-metrics
+        uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1
+        with:
+          id: on-demand-vrfv2plus-smoke-tests
+          org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
+          basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
+          hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
+          this-job-name: ${{ inputs.network }} VRFV2 Plus Smoke Test
+        continue-on-error: true
+      - name: Checkout code
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+        with:
+          fetch-depth: 0
+      - name: Mask base64 config
+        run: |
+          BASE64_CONFIG_OVERRIDE=$(jq -r '.inputs.base64Config' $GITHUB_EVENT_PATH)
+          echo ::add-mask::$BASE64_CONFIG_OVERRIDE
+          echo "BASE64_CONFIG_OVERRIDE=$BASE64_CONFIG_OVERRIDE" >> $GITHUB_ENV
+      - name: Merge and export base64 config
+        uses: ./.github/actions/setup-merge-base64-config
+        with:
+          base64Config: ${{ env.BASE64_CONFIG_OVERRIDE }}
+      - name: Send details to Step Summary
+        shell: bash
+        run: |
+          echo "### chainlink image used for this test run :link:" >>$GITHUB_STEP_SUMMARY
+          echo "\`${{ env.CHAINLINK_IMAGE }}\`" >>$GITHUB_STEP_SUMMARY
+          echo "### chainlink-tests image tag for this test run :ship:" >>$GITHUB_STEP_SUMMARY
+          echo "\`${GITHUB_SHA}\`" >>$GITHUB_STEP_SUMMARY
+          echo "### Networks on which test was run" >>$GITHUB_STEP_SUMMARY
+          echo "\`${{ env.NETWORKS }}\`" >>$GITHUB_STEP_SUMMARY
+      - name: Run Tests
+        if: ${{ github.event.inputs.test_suite == 'Selected Tests' }}
+        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@2967f2287bd3f3ddbac7b476e9568993df01796e # v2.3.27
+        with:
+          test_command_to_run: cd ./integration-tests/smoke && go test -v -count=1 -parallel=1 -timeout 2h -run "${{ inputs.test_list_regex }}"
+          test_download_vendor_packages_command: cd ./integration-tests && go mod download
+          test_secrets_override_base64: ${{ secrets[inputs.test_secrets_override_key] }}
+          cl_repo: ${{ env.CHAINLINK_IMAGE }}
+          cl_image_tag: ${{ env.CHAINLINK_VERSION }}
+          aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
+          artifacts_name: vrf-test-logs
+          artifacts_location: ./integration-tests/smoke/vrfv2plus/logs/
+          token: ${{ secrets.GITHUB_TOKEN }}
+          go_mod_path: ./integration-tests/go.mod
+          should_cleanup: false
+          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
+          QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
+      - name: Run Tests
+        if: ${{ github.event.inputs.test_suite == 'All Tests' }}
+        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@2967f2287bd3f3ddbac7b476e9568993df01796e # v2.3.27
+        with:
+          test_command_to_run: cd ./integration-tests/smoke && go test -v -count=1 -parallel=1 -timeout 3h vrfv2plus_test.go
+          test_download_vendor_packages_command: cd ./integration-tests && go mod download
+          test_secrets_override_base64: ${{ secrets[inputs.test_secrets_override_key] }}
+          cl_repo: ${{ env.CHAINLINK_IMAGE }}
+          cl_image_tag: ${{ env.CHAINLINK_VERSION }}
+          aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
+          artifacts_name: vrf-test-logs
+          artifacts_location: ./integration-tests/smoke/vrfv2plus/logs/
+          token: ${{ secrets.GITHUB_TOKEN }}
+          go_mod_path: ./integration-tests/go.mod
+          should_cleanup: false
+          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
+          QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}

integration-tests/actions/actions.go

@@ -1285,3 +1285,12 @@ func ContinuouslyGenerateTXsOnChain(sethClient *seth.Client, stopChannel chan bo
 		}
 	}
 }
+
+func WithinTolerance(a, b, tolerance float64) (bool, float64) {
+	if a == b {
+		return true, 0
+	}
+	diff := math.Abs(a - b)
+	isWithinTolerance := diff < tolerance
+	return isWithinTolerance, diff
+}

integration-tests/actions/vrf/vrfv2plus/contract_steps.go

@@ -41,7 +41,7 @@ func DeployVRFV2_5Contracts(
 		if err != nil {
 			return nil, fmt.Errorf(vrfcommon.ErrGenericFormat, ErrDeployCoordinatorV2Plus, err)
 		}
-		err = opStackCoordinator.SetL1FeeCalculation(configGeneral.L1FeeCalculationMode, configGeneral.L1FeeCoefficient)
+		err = opStackCoordinator.SetL1FeeCalculation(*configGeneral.L1FeeCalculationMode, *configGeneral.L1FeeCoefficient)
 		if err != nil {
 			return nil, fmt.Errorf(vrfcommon.ErrGenericFormat, ErrSetL1FeeCalculation, err)
 		}
@@ -437,7 +437,7 @@ func DeployVRFV2PlusDirectFundingContracts(
 		if err != nil {
 			return nil, fmt.Errorf(vrfcommon.ErrGenericFormat, ErrDeployWrapper, err)
 		}
-		err = opStackWrapper.SetL1FeeCalculation(configGeneral.L1FeeCalculationMode, configGeneral.L1FeeCoefficient)
+		err = opStackWrapper.SetL1FeeCalculation(*configGeneral.L1FeeCalculationMode, *configGeneral.L1FeeCoefficient)
 		if err != nil {
 			return nil, fmt.Errorf(vrfcommon.ErrGenericFormat, ErrSetL1FeeCalculation, err)
 		}

integration-tests/actions/vrf/vrfv2plus/logging_helpers.go

@@ -4,9 +4,11 @@ import (
 	"fmt"
 	"math/big"
 
+	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/rs/zerolog"
 
 	"github.com/smartcontractkit/chainlink/integration-tests/contracts"
+	tc "github.com/smartcontractkit/chainlink/integration-tests/testconfig"
 	vrfv2plus_config "github.com/smartcontractkit/chainlink/integration-tests/testconfig/vrfv2plus"
 	"github.com/smartcontractkit/chainlink/v2/core/gethwrappers/generated/vrf_coordinator_v2_5"
 	"github.com/smartcontractkit/chainlink/v2/core/gethwrappers/generated/vrf_v2plus_upgraded_version"
@@ -55,3 +57,18 @@ func LogSubDetailsAfterMigration(l zerolog.Logger, newCoordinator contracts.Coor
 		Interface("Subscription Consumers", migratedSubscription.Consumers).
 		Msg("Subscription Data After Migration to New Coordinator")
 }
+
+func LogPaymentDetails(l zerolog.Logger, fulfillmentTxFeeWei *big.Int, fulfillmentTxReceipt *types.Receipt, actualSubPaymentWei *big.Int, expectedSubPaymentWei *big.Float, configCopy tc.TestConfig) {
+	l.Info().
+		Str("Tx Fee in Wei", fulfillmentTxFeeWei.String()).
+		Str("Effective Gas Price", fulfillmentTxReceipt.EffectiveGasPrice.String()).
+		Uint64("Gas Used", fulfillmentTxReceipt.GasUsed).
+		Str("Actual Subscription Payment in Wei", actualSubPaymentWei.String()).
+		Str("Expected Subscription Payment in Wei", expectedSubPaymentWei.String()).
+		Uint8("Native Premium Percentage", *configCopy.VRFv2Plus.General.NativePremiumPercentage).
+		Uint8("Link Premium Percentage", *configCopy.VRFv2Plus.General.LinkPremiumPercentage).
+		Uint32("FulfillmentFlatFeeNativePPM", *configCopy.VRFv2Plus.General.FulfillmentFlatFeeNativePPM).
+		Uint32("FulfillmentFlatFeeLinkDiscountPPM", *configCopy.VRFv2Plus.General.FulfillmentFlatFeeLinkDiscountPPM).
+		Uint32("GasAfterPaymentCalculation", *configCopy.VRFv2Plus.General.GasAfterPaymentCalculation).
+		Msg("Randomness Fulfillment Payment Details")
+}

integration-tests/contracts/contract_vrf_models.go

@@ -125,6 +125,7 @@ type VRFCoordinatorV2_5 interface {
 	GetBlockHashStoreAddress(ctx context.Context) (common.Address, error)
 	GetLinkAddress(ctx context.Context) (common.Address, error)
 	GetLinkNativeFeed(ctx context.Context) (common.Address, error)
+	GetConfig(ctx context.Context) (vrf_coordinator_v2_5.SConfig, error)
 }
 
 type VRFCoordinatorV2PlusUpgradedVersion interface {