[CI/CD] Add target determination for execution performance.

.github/actions/framework-upgrade-determinator/action.yaml → .github/actions/test-target-determinator/action.yaml

@@ -1,10 +1,13 @@
-name: Framework Upgrade Test Determinator
-description: Runs the framework upgrade test determinator
+name: Test Target Determinator
+description: Runs the test target determinator
 inputs:
   GIT_CREDENTIALS:
     description: "Optional credentials to pass to git. Useful if you need to pull private repos for dependencies"
     required: false
 outputs:
+  run_execution_performance_test:
+    description: "Returns true if the execution performance test should be run"
+    value: ${{ steps.execution_performance_determinator.outputs.run_execution_performance_test }}
   run_framework_upgrade_test:
     description: "Returns true if the framework upgrade test should be run"
     value: ${{ steps.framework_upgrade_determinator.outputs.run_framework_upgrade_test }}
@@ -31,6 +34,15 @@ runs:
       run: cargo x affected-packages -vv
       shell: bash
 
+    # Run the execution performance test determinator
+    - name: Run the execution performance test determinator
+      id: execution_performance_determinator
+      run: |
+          export RESULT=$(cargo x targeted-execution-performance-tests | awk -F'Execution performance test required: ' '{print $2}')
+          echo "Execution performance test required: $RESULT"
+          echo "run_execution_performance_test=$RESULT" >> $GITHUB_OUTPUT
+      shell: bash
+
     # Run the framework upgrade test determinator
     - name: Run the framework upgrade test determinator
       id: framework_upgrade_determinator

.github/workflows/docker-build-test.yaml

@@ -125,17 +125,17 @@ jobs:
         id: determine_file_changes
         uses: ./.github/actions/file-change-determinator
 
-  # This job determines if the framework upgrade test should be run
-  framework-upgrade-determinator:
+  # This job determines which tests to run
+  test-target-determinator:
     needs: [permission-check]
     runs-on: ubuntu-latest
     outputs:
-      run_framework_upgrade_test: ${{ steps.determine_framework_upgrade_test.outputs.run_framework_upgrade_test }}
+      run_framework_upgrade_test: ${{ steps.determine_test_targets.outputs.run_framework_upgrade_test }}
     steps:
       - uses: actions/checkout@v3
-      - name: Run the framework upgrade test determinator
-        id: determine_framework_upgrade_test
-        uses: ./.github/actions/framework-upgrade-determinator
+      - name: Run the test target determinator
+        id: determine_test_targets
+        uses: ./.github/actions/test-target-determinator
 
   # This is a PR required job.
   rust-images:
@@ -327,7 +327,7 @@ jobs:
       - rust-images-failpoints
       - rust-images-performance
       - rust-images-consensus-only-perf-test
-      - framework-upgrade-determinator
+      - test-target-determinator
     if: |
       !failure() && !cancelled() && needs.permission-check.result == 'success' && (
         (github.event_name == 'push' && github.ref_name != 'main') ||
@@ -344,7 +344,7 @@ jobs:
       FORGE_RUNNER_DURATION_SECS: 3600
       COMMENT_HEADER: forge-framework-upgrade
       FORGE_NAMESPACE: forge-framework-upgrade-${{ needs.determine-docker-build-metadata.outputs.targetCacheId }}
-      SKIP_JOB: ${{ !contains(github.event.pull_request.labels.*.name, 'CICD:run-framework-upgrade-test') && (needs.framework-upgrade-determinator.outputs.run_framework_upgrade_test == 'false') }}
+      SKIP_JOB: ${{ !contains(github.event.pull_request.labels.*.name, 'CICD:run-framework-upgrade-test') && (needs.test-target-determinator.outputs.run_framework_upgrade_test == 'false') }}
 
   forge-consensus-only-perf-test:
     needs:

.github/workflows/execution-performance.yaml

@@ -15,8 +15,7 @@ jobs:
         contains(github.event.pull_request.labels.*.name, 'CICD:run-e2e-tests') ||
         contains(github.event.pull_request.labels.*.name, 'CICD:run-execution-performance-test') ||
         contains(github.event.pull_request.labels.*.name, 'CICD:run-execution-performance-full-test') ||
-        github.event.pull_request.auto_merge != null) ||
-        contains(github.event.pull_request.body, '#e2e'
+        github.event.pull_request.auto_merge != null
       )
     uses: ./.github/workflows/workflow-run-execution-performance.yaml
     secrets: inherit
@@ -25,3 +24,5 @@ jobs:
       RUNNER_NAME: executor-benchmark-runner
        # Run all tests only on the scheduled cadence, or explicitly requested
       IS_FULL_RUN: ${{ github.event_name == 'schedule' || contains(github.event.pull_request.labels.*.name, 'CICD:run-execution-performance-full-test') }}
+      # Ignore target determination if on the scheduled cadence, or explicitly requested
+      IGNORE_TARGET_DETERMINATION: ${{ github.event_name == 'schedule' || contains(github.event.pull_request.labels.*.name, 'CICD:run-execution-performance-test') || contains(github.event.pull_request.labels.*.name, 'CICD:run-execution-performance-full-test') }}

.github/workflows/workflow-run-execution-performance.yaml

@@ -17,6 +17,11 @@ on:
         default: false
         type: boolean
         description: Run complete version of the tests
+      IGNORE_TARGET_DETERMINATION:
+        required: false
+        default: false
+        type: boolean
+        description: Ignore target determination and run the tests
   # This allows the workflow to be triggered manually from the Github UI or CLI
   # NOTE: because the "number" type is not supported, we default to 720 minute timeout
   workflow_dispatch:
@@ -37,44 +42,49 @@ on:
         default: false
         type: boolean
         description: Run complete version of the tests
+      IGNORE_TARGET_DETERMINATION:
+        required: false
+        default: false
+        type: boolean
+        description: Ignore target determination and run the tests
 
 jobs:
-  # This job determines which files were changed
-  file_change_determinator:
+  # This job determines which tests to run
+  test-target-determinator:
     runs-on: ubuntu-latest
     outputs:
-      only_docs_changed: ${{ steps.determine_file_changes.outputs.only_docs_changed }}
+      run_execution_performance_test: ${{ steps.determine_test_targets.outputs.run_execution_performance_test }}
     steps:
       - uses: actions/checkout@v3
-      - name: Run the file change determinator
-        id: determine_file_changes
-        uses: ./.github/actions/file-change-determinator
+      - name: Run the test target determinator
+        id: determine_test_targets
+        uses: ./.github/actions/test-target-determinator
 
   # Run single node execution performance tests
   single-node-performance:
-    needs: file_change_determinator
+    needs: test-target-determinator
     timeout-minutes: 60
     runs-on: ${{ inputs.RUNNER_NAME }}
     steps:
       - uses: actions/checkout@v3
         with:
           ref: ${{ inputs.GIT_SHA }}
-        if: needs.file_change_determinator.outputs.only_docs_changed != 'true'
+        if: ${{ inputs.IGNORE_TARGET_DETERMINATION || needs.test-target-determinator.outputs.run_execution_performance_test == 'true' }}
 
       - uses: aptos-labs/aptos-core/.github/actions/rust-setup@main
         with:
           GIT_CREDENTIALS: ${{ secrets.GIT_CREDENTIALS }}
-        if: needs.file_change_determinator.outputs.only_docs_changed != 'true'
+        if: ${{ inputs.IGNORE_TARGET_DETERMINATION || needs.test-target-determinator.outputs.run_execution_performance_test == 'true' }}
 
       - name: Run single node execution benchmark in performance build mode
         shell: bash
         run: TABULATE_INSTALL=lib-only pip install tabulate && testsuite/single_node_performance.py
-        if: ${{ needs.file_change_determinator.outputs.only_docs_changed != 'true' && !inputs.IS_FULL_RUN }}
+        if: ${{ !inputs.IS_FULL_RUN && (inputs.IGNORE_TARGET_DETERMINATION || needs.test-target-determinator.outputs.run_execution_performance_test == 'true') }}
 
       - name: Run full version of the single node execution benchmark in performance build mode
         shell: bash
         run: TABULATE_INSTALL=lib-only pip install tabulate && FLOW=CONTINUOUS testsuite/single_node_performance.py
-        if: ${{ needs.file_change_determinator.outputs.only_docs_changed != 'true' && inputs.IS_FULL_RUN }}
+        if: ${{ inputs.IS_FULL_RUN && (inputs.IGNORE_TARGET_DETERMINATION || needs.test-target-determinator.outputs.run_execution_performance_test == 'true') }}
 
       - run: echo "Skipping single node execution performance! Unrelated changes detected."
-        if: needs.file_change_determinator.outputs.only_docs_changed == 'true'
+        if: ${{ !inputs.IGNORE_TARGET_DETERMINATION && needs.test-target-determinator.outputs.run_execution_performance_test != 'true' }}

devtools/aptos-cargo-cli/src/lib.rs

@@ -19,19 +19,28 @@ const APTOS_CLI_PACKAGE_NAME: &str = "aptos";
 // Note: these paths should be relative to the root of the `aptos-core` repository,
 // and will be transformed into UTF-8 paths for cross-platform compatibility.
 const RELEVANT_FILE_PATHS_FOR_COMPILER_V2: [&str; 7] = [
+    ".github/actions/move-tests-compiler-v2",
+    ".github/workflows/move-test-compiler-v2.yaml",
     "aptos-move/aptos-transactional-test-harness",
     "aptos-move/e2e-move-tests",
     "aptos-move/framework",
     "aptos-move/move-examples",
     "third_party/move",
-    ".github/workflows/move-test-compiler-v2.yaml",
-    ".github/actions/move-tests-compiler-v2",
+];
+const RELEVANT_FILE_PATHS_FOR_EXECUTION_PERFORMANCE_TESTS: [&str; 5] = [
+    ".github/workflows/execution-performance.yaml",
+    ".github/workflows/workflow-run-execution-performance.yaml",
+    "aptos-move/e2e-benchmark",
+    "execution/aptos-executor-benchmark",
+    "testsuite/single_node_performance.py",
 ];
 const RELEVANT_FILE_PATHS_FOR_FRAMEWORK_UPGRADE_TESTS: [&str; 2] =
     ["aptos-move/aptos-release-builder", "aptos-move/framework"];
 
 // Relevant packages to monitor when deciding to run the targeted tests
 const RELEVANT_PACKAGES_FOR_COMPILER_V2: [&str; 2] = ["aptos-framework", "e2e-move-tests"];
+const RELEVANT_PACKAGES_FOR_EXECUTION_PERFORMANCE_TESTS: [&str; 2] =
+    ["aptos-executor-benchmark", "aptos-move-e2e-benchmark"];
 const RELEVANT_PACKAGES_FOR_FRAMEWORK_UPGRADE_TESTS: [&str; 2] =
     ["aptos-framework", "aptos-release-builder"];
 
@@ -66,6 +75,7 @@ pub enum AptosCargoCommand {
     Nextest(CommonArgs),
     TargetedCLITests(CommonArgs),
     TargetedCompilerV2Tests(CommonArgs),
+    TargetedExecutionPerformanceTests(CommonArgs),
     TargetedFrameworkUpgradeTests(CommonArgs),
     TargetedUnitTests(CommonArgs),
     Test(CommonArgs),
@@ -93,6 +103,7 @@ impl AptosCargoCommand {
             AptosCargoCommand::Nextest(args) => args,
             AptosCargoCommand::TargetedCLITests(args) => args,
             AptosCargoCommand::TargetedCompilerV2Tests(args) => args,
+            AptosCargoCommand::TargetedExecutionPerformanceTests(args) => args,
             AptosCargoCommand::TargetedFrameworkUpgradeTests(args) => args,
             AptosCargoCommand::TargetedUnitTests(args) => args,
             AptosCargoCommand::Test(args) => args,
@@ -204,6 +215,30 @@ impl AptosCargoCommand {
                 println!("Skipping targeted compiler v2 tests because no relevant files or packages were affected!");
                 Ok(())
             },
+            AptosCargoCommand::TargetedExecutionPerformanceTests(_) => {
+                // Determine if the execution performance tests should be run.
+                // Start by calculating the changed files and affected packages.
+                let (_, _, changed_files) = package_args.identify_changed_files()?;
+                let (_, _, affected_package_paths) =
+                    self.get_args_and_affected_packages(package_args)?;
+
+                // Determine if any relevant files or packages were changed
+                let relevant_changes_detected = detect_relevant_changes(
+                    RELEVANT_FILE_PATHS_FOR_EXECUTION_PERFORMANCE_TESTS.to_vec(),
+                    RELEVANT_PACKAGES_FOR_EXECUTION_PERFORMANCE_TESTS.to_vec(),
+                    changed_files,
+                    affected_package_paths,
+                );
+
+                // Output if relevant changes were detected that require the execution performance
+                // test. This will be consumed by Github Actions and used to run the test.
+                println!(
+                    "Execution performance test required: {}",
+                    relevant_changes_detected
+                );
+
+                Ok(())
+            },
             AptosCargoCommand::TargetedFrameworkUpgradeTests(_) => {
                 // Determine if the framework upgrade tests should be run.
                 // Start by calculating the changed files and affected packages.
@@ -221,7 +256,6 @@ impl AptosCargoCommand {
 
                 // Output if relevant changes were detected that require the framework upgrade
                 // test. This will be consumed by Github Actions and used to run the test.
-                // TODO: is there a cleaner way to output this for Github Actions?
                 println!(
                     "Framework upgrade test required: {}",
                     relevant_changes_detected