Merge branch 'main' into fix_restore_for_sharding

aptos-labs · Dec 3, 2024 · a87fad6 · a87fad6
2 parents a3583ca + b011781
commit a87fad6
Show file tree

Hide file tree

Showing 361 changed files with 10,107 additions and 7,674 deletions.
diff --git a/.github/workflows/provision-replay-verify-archive-disks.yaml b/.github/workflows/provision-replay-verify-archive-disks.yaml
@@ -0,0 +1,65 @@
+name: "provision-replay-verify-archive-disks"
+on:
+  # Allow triggering manually
+  workflow_dispatch:
+    inputs:
+      NETWORK:
+        required: true
+        type: choice
+        description: The network to provision storage for.If not specified, it will provision snapshot for both testnet and mainnet.
+        options: [testnet, mainnet, all]
+        default: all
+  pull_request:
+    paths:
+      - ".github/workflows/provision-replay-verify-archive-disks.yaml"
+      - ".github/workflows/workflow-run-replay-verify-archive-storage-provision.yaml"
+  schedule:
+    - cron: "0 22 * * 1,3,5" # This runs every Mon,Wed,Fri
+
+permissions:
+  contents: read
+  id-token: write #required for GCP Workload Identity federation which we use to login into Google Artifact Registry
+  issues: read
+  pull-requests: read
+
+# cancel redundant builds
+concurrency:
+  # cancel redundant builds on PRs (only on PR, not on branches)
+  group: ${{ github.workflow }}-${{ (github.event_name == 'pull_request' && github.ref) || github.sha }}
+  cancel-in-progress: true
+
+jobs:
+  determine-test-metadata:
+    runs-on: ubuntu-latest
+    steps:
+      # checkout the repo first, so check-aptos-core can use it and cancel the workflow if necessary
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/check-aptos-core
+        with:
+          cancel-workflow: ${{ github.event_name == 'schedule' }} # Cancel the workflow if it is scheduled on a fork
+
+      - name: Debug
+        run: |
+          echo "Event name: ${{ github.event_name }}"
+          echo "Network: ${{ inputs.NETWORK }}"
+
+  provision-testnet:
+    if: |
+      github.event_name == 'schedule' ||
+      github.event_name == 'workflow_dispatch' && (inputs.NETWORK == 'testnet' || inputs.NETWORK == 'all')
+    needs: determine-test-metadata
+    uses: ./.github/workflows/workflow-run-replay-verify-archive-storage-provision.yaml
+    secrets: inherit
+    with:
+      NETWORK: testnet
+
+  provision-mainnet:
+    if: |
+      github.event_name == 'schedule' ||
+      github.event_name == 'push' ||
+      github.event_name == 'workflow_dispatch' && (inputs.NETWORK == 'testnet' || inputs.NETWORK == 'all')
+    needs: determine-test-metadata
+    uses: ./.github/workflows/workflow-run-replay-verify-archive-storage-provision.yaml
+    secrets: inherit
+    with:
+      NETWORK: mainnet
diff --git a/.github/workflows/replay-verify-on-archive.yaml b/.github/workflows/replay-verify-on-archive.yaml
@@ -0,0 +1,82 @@
+# This defines a workflow to replay transactions on the given chain with the latest aptos node software.
+# In order to trigger it go to the Actions Tab of the Repo, click "replay-verify" and then "Run Workflow".
+#
+# On PR, a single test case will run. On workflow_dispatch, you may specify the CHAIN_NAME to verify.
+
+name: "replay-verify-on-archive"
+on:
+  # Allow triggering manually
+  workflow_dispatch:
+    inputs:
+      NETWORK:
+        required: true
+        type: choice
+        options: [testnet, mainnet, all]
+        default: all
+        description: The chain name to test. If not specified, it will test both testnet and mainnet.
+      IMAGE_TAG:
+        required: false
+        type: string
+        description: The image tag of the feature branch to test, if not specified, it will use the latest commit on current branch.
+      START_VERSION:
+        required: false
+        type: string
+        description: Optional version to start replaying. If not specified, replay-verify will determines start version itself.
+      END_VERSION:
+        required: false
+        type: string
+        description: Optional version to end replaying. If not specified, replay-verify will determines end version itself.
+  pull_request:
+    paths:
+      - ".github/workflows/replay-verify-on-archive.yaml"
+      - ".github/workflows/workflow-run-replay-verify-on-archive.yaml"
+  schedule:
+    - cron: "0 22 * * 0,2,4" # The main branch cadence. This runs every Sun,Tues,Thurs
+
+permissions:
+  contents: read
+  id-token: write #required for GCP Workload Identity federation which we use to login into Google Artifact Registry
+  issues: read
+  pull-requests: read
+
+# cancel redundant builds
+concurrency:
+  # cancel redundant builds on PRs (only on PR, not on branches)
+  group: ${{ github.workflow }}-${{ (github.event_name == 'pull_request' && github.ref) || github.sha }}
+  cancel-in-progress: true
+
+jobs:
+  determine-test-metadata:
+    runs-on: ubuntu-latest-32-core
+    steps:
+      # checkout the repo first, so check-aptos-core can use it and cancel the workflow if necessary
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/check-aptos-core
+        with:
+          cancel-workflow: ${{ github.event_name == 'schedule' }} # Cancel the workflow if it is scheduled on a fork
+
+  replay-testnet:
+    if: |
+      github.event_name == 'schedule' ||
+      github.event_name == 'workflow_dispatch' && (inputs.NETWORK == 'testnet' || inputs.NETWORK == 'all')
+    needs: determine-test-metadata
+    uses: ./.github/workflows/workflow-run-replay-verify-on-archive.yaml
+    secrets: inherit
+    with:
+      NETWORK: "testnet"
+      IMAGE_TAG: ${{ inputs.IMAGE_TAG }}
+      START_VERSION: ${{ inputs.START_VERSION }}
+      END_VERSION: ${{ inputs.END_VERSION }}
+
+  replay-mainnet:
+    if: |
+      github.event_name == 'schedule' ||
+      github.event_name == 'workflow_dispatch' && (inputs.NETWORK == 'mainnet' || inputs.NETWORK == 'all' )
+    needs: determine-test-metadata
+    uses: ./.github/workflows/workflow-run-replay-verify-on-archive.yaml
+    secrets: inherit
+    with:
+      NETWORK: "mainnet"
+      IMAGE_TAG: ${{ inputs.IMAGE_TAG }}
+      START_VERSION: ${{ inputs.START_VERSION }}
+      END_VERSION: ${{ inputs.END_VERSION }}
diff --git a/.github/workflows/workflow-run-replay-verify-archive-storage-provision.yaml b/.github/workflows/workflow-run-replay-verify-archive-storage-provision.yaml
@@ -0,0 +1,61 @@
+name: "*run archive storage provision workflow"
+
+on:
+  # This allows the workflow to be triggered from another workflow
+  workflow_call:
+    inputs:
+      NETWORK:
+        required: true
+        type: string
+        description: The network to provision storage for.
+  workflow_dispatch:
+    inputs:
+      NETWORK:
+        description: The network to provision storage for.
+        type: string
+        required: true
+jobs:
+  provision:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.ref || github.ref }}
+
+      # Authenticate to Google Cloud the project is aptos-ci
+      - name: Authenticate to Google Cloud
+        id: auth
+        uses: "google-github-actions/auth@v2"
+        with:
+          workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}
+          service_account: ${{ secrets.GCP_SERVICE_ACCOUNT_EMAIL }}
+          export_environment_variables: false
+          create_credentials_file: true
+
+      # This is required since we need to switch from aptos-ci to aptos-devinfra-0
+      - name: Setup Credentials  
+        run: |
+          echo "GOOGLE_APPLICATION_CREDENTIALS=${{ steps.auth.outputs.credentials_file_path }}" >> $GITHUB_ENV
+          echo "CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=${{ steps.auth.outputs.credentials_file_path }}" >> $GITHUB_ENV 
+          echo "GOOGLE_GHA_CREDS_PATH=${{ steps.auth.outputs.credentials_file_path }}" >> $GITHUB_ENV
+          echo "CLOUDSDK_AUTH_ACCESS_TOKEN=${{ steps.auth.outputs.access_token }}" >> $GITHUB_ENV
+      
+      - name: Set up Cloud SDK
+        uses: "google-github-actions/setup-gcloud@v2"
+        with:
+          install_components: "kubectl, gke-gcloud-auth-plugin"
+
+      - name: "Setup GCloud Project"
+        shell: bash
+        run: gcloud config set project aptos-devinfra-0
+
+      - uses: ./.github/actions/python-setup
+        with:
+          pyproject_directory: testsuite/replay-verify
+
+      - name: "Provision Storage"
+        env:
+          GOOGLE_CLOUD_PROJECT: aptos-devinfra-0
+        run: cd testsuite/replay-verify && poetry run python archive_disk_utils.py --network ${{ inputs.NETWORK }}
+
diff --git a/.github/workflows/workflow-run-replay-verify-on-archive.yaml b/.github/workflows/workflow-run-replay-verify-on-archive.yaml
@@ -0,0 +1,120 @@
+name: "*run replay-verify on archive reusable workflow"
+
+on:
+  # This allows the workflow to be triggered from another workflow
+  workflow_call:
+    inputs:
+      NETWORK:
+        required: true
+        type: string
+        description: The network to run replay verify on.
+      IMAGE_TAG:
+        required: false
+        type: string
+        description: The image tag of the feature branch to test, if not specified, it will use the latest commit on current branch.
+      START_VERSION:
+        required: false
+        type: string
+        description: Optional version to start replaying. If not specified, replay-verify will determines start version itself.
+      END_VERSION:
+        required: false
+        type: string
+        description: Optional version to end replaying. If not specified, replay-verify will determines end version itself.
+
+  workflow_dispatch:
+    inputs:
+      NETWORK:
+        required: true
+        type: string
+        description: The network to run replay verify on.
+      IMAGE_TAG:
+        required: false
+        type: string
+        description: The image tag of the feature branch to test, if not specified, it will use the latest commit on current branch.
+      START_VERSION:
+        required: false
+        type: string
+        description: The history start to use for the backup. If not specified, it will use the default history start.
+      END_VERSION:
+        required: false
+        type: string
+        description: The end version to use for the backup. If not specified, it will use the latest version.
+jobs:
+  run-replay-verify:
+    runs-on: ubuntu-latest-32-core
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.ref || github.ref }}
+
+      - uses: aptos-labs/aptos-core/.github/actions/docker-setup@main
+        id: docker-setup
+        with:
+          GCP_WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}
+          GCP_SERVICE_ACCOUNT_EMAIL: ${{ secrets.GCP_SERVICE_ACCOUNT_EMAIL }}
+          EXPORT_GCP_PROJECT_VARIABLES: "false"
+          GIT_CREDENTIALS: ${{ secrets.GIT_CREDENTIALS }}
+
+      # Authenticate to Google Cloud the project is aptos-ci with credentails files generated
+      - name: Authenticate to Google Cloud
+        id: auth
+        uses: "google-github-actions/auth@v2"
+        with:
+          workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}
+          service_account: ${{ secrets.GCP_SERVICE_ACCOUNT_EMAIL }}
+          export_environment_variables: false
+          create_credentials_file: true
+
+      # This is required since we need to switch from aptos-ci to aptos-devinfra-0
+      - name: Setup credentials  
+        run: |
+          echo "GOOGLE_APPLICATION_CREDENTIALS=${{ steps.auth.outputs.credentials_file_path }}" >> $GITHUB_ENV
+          echo "CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=${{ steps.auth.outputs.credentials_file_path }}" >> $GITHUB_ENV 
+          echo "GOOGLE_GHA_CREDS_PATH=${{ steps.auth.outputs.credentials_file_path }}" >> $GITHUB_ENV
+          echo "CLOUDSDK_AUTH_ACCESS_TOKEN=${{ steps.auth.outputs.access_token }}" >> $GITHUB_ENV
+
+      - name: Set up Cloud SDK
+        uses: "google-github-actions/setup-gcloud@v2"
+        with:
+          install_components: "kubectl, gke-gcloud-auth-plugin"
+
+      - name: "Setup GCloud project"
+        shell: bash
+        run: gcloud config set project aptos-devinfra-0
+
+      - uses: ./.github/actions/python-setup
+        with:
+          pyproject_directory: testsuite/replay-verify
+
+      - name: Schedule replay verify
+        env:
+          GOOGLE_CLOUD_PROJECT: aptos-devinfra-0
+        run: | 
+          cd testsuite/replay-verify  
+          CMD="poetry run python main.py --network ${{ inputs.NETWORK }}"  
+          if [ -n "${{ inputs.START_VERSION }}" ]; then  
+            CMD="$CMD --start ${{ inputs.START_VERSION }}"  
+          fi  
+          if [ -n "${{ inputs.END_VERSION }}" ]; then  
+            CMD="$CMD --end ${{ inputs.END_VERSION }}"  
+          fi
+
+           if [ -n "${{ inputs.IMAGE_TAG }}" ]; then  
+            CMD="$CMD --end ${{ inputs.IMAGE_TAG }}"  
+          fi
+
+          eval $CMD
+      # This is in case user manually cancel the step above, we still want to cleanup the resources
+      - name: Post-run cleanup
+        env:
+          GOOGLE_CLOUD_PROJECT: aptos-devinfra-0
+        if: ${{ always() }}  
+        run: |
+          cd testsuite/replay-verify
+          poetry run python main.py  --network ${{ inputs.NETWORK }} --cleanup
+          echo "Cleanup completed"
+
+          
+          
+