[single node perf] Recalibrate and improve regression perf test

Recalibrate for RG change Update limits to be based on min_ratio / max_ratio of many runs update module working set to 100
aptos-labs · Oct 8, 2024 · 8164ee9 · 8164ee9
1 parent 67f7ee6
commit 8164ee9
Show file tree

Hide file tree

Showing 3 changed files with 157 additions and 101 deletions.
diff --git a/.github/workflows/execution-performance.yaml b/.github/workflows/execution-performance.yaml
@@ -23,6 +23,6 @@ jobs:
       GIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
       RUNNER_NAME: executor-benchmark-runner
        # Run all tests only on the scheduled cadence, or explicitly requested
-      IS_FULL_RUN: ${{ github.event_name == 'schedule' || contains(github.event.pull_request.labels.*.name, 'CICD:run-execution-performance-full-test') }}
+      FLOW: ${{ (github.event_name == 'schedule' || contains(github.event.pull_request.labels.*.name, 'CICD:run-execution-performance-full-test')) && 'CONTINUOUS' || 'LAND_BLOCKING' }}
       # Ignore target determination if on the scheduled cadence, or explicitly requested
       IGNORE_TARGET_DETERMINATION: ${{ github.event_name == 'schedule' || contains(github.event.pull_request.labels.*.name, 'CICD:run-execution-performance-test') || contains(github.event.pull_request.labels.*.name, 'CICD:run-execution-performance-full-test') }}
diff --git a/.github/workflows/workflow-run-execution-performance.yaml b/.github/workflows/workflow-run-execution-performance.yaml
@@ -12,11 +12,11 @@ on:
         required: false
         default: executor-benchmark-runner 
         type: string
-      IS_FULL_RUN:
+      FLOW:
         required: false
-        default: false
-        type: boolean
-        description: Run complete version of the tests
+        default: LAND_BLOCKING
+        type: string
+        description: Which set of tests to run.
       IGNORE_TARGET_DETERMINATION:
         required: false
         default: false
@@ -36,15 +36,25 @@ on:
         type: choice
         options:
         - executor-benchmark-runner
+        - benchmark-t2d-32
+        - benchmark-t2d-60
+        - benchmark-c3d-30
+        - benchmark-n4-32
+        - benchmark-c4-32
         description: The name of the runner to use for the test.
-      IS_FULL_RUN:
+      FLOW:
         required: false
-        default: false
-        type: boolean
-        description: Run complete version of the tests
+        default: LAND_BLOCKING
+        options:
+        - LAND_BLOCKING
+        - CONTINUOUS
+        - MAINNET
+        - MAINNET_LARGE_DB
+        type: choice
+        description: Which set of tests to run. MAINNET/MAINNET_LARGE_DB are for performance validation of mainnet nodes.
       IGNORE_TARGET_DETERMINATION:
         required: false
-        default: false
+        default: true
         type: boolean
         description: Ignore target determination and run the tests
 
@@ -63,7 +73,7 @@ jobs:
   # Run single node execution performance tests
   single-node-performance:
     needs: test-target-determinator
-    timeout-minutes: 60
+    timeout-minutes: 120
     runs-on: ${{ inputs.RUNNER_NAME }}
     steps:
       - uses: actions/checkout@v4
@@ -78,13 +88,8 @@ jobs:
 
       - name: Run single node execution benchmark in performance build mode
         shell: bash
-        run: TABULATE_INSTALL=lib-only pip install tabulate && testsuite/single_node_performance.py
-        if: ${{ !inputs.IS_FULL_RUN && (inputs.IGNORE_TARGET_DETERMINATION || needs.test-target-determinator.outputs.run_execution_performance_test == 'true') }}
-
-      - name: Run full version of the single node execution benchmark in performance build mode
-        shell: bash
-        run: TABULATE_INSTALL=lib-only pip install tabulate && FLOW=CONTINUOUS testsuite/single_node_performance.py
-        if: ${{ inputs.IS_FULL_RUN && (inputs.IGNORE_TARGET_DETERMINATION || needs.test-target-determinator.outputs.run_execution_performance_test == 'true') }}
+        run: TABULATE_INSTALL=lib-only pip install tabulate && FLOW=${{ inputs.FLOW }} testsuite/single_node_performance.py
+        if: ${{ (inputs.IGNORE_TARGET_DETERMINATION || needs.test-target-determinator.outputs.run_execution_performance_test == 'true') }}
 
       - run: echo "Skipping single node execution performance! Unrelated changes detected."
         if: ${{ !inputs.IGNORE_TARGET_DETERMINATION && needs.test-target-determinator.outputs.run_execution_performance_test != 'true' }}