Benchmark Master Branch #8
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Workflow to trigger/schedule benchmarking, cleaning, aggregating on master branch only and storing results in S3 | |
| name: Benchmark Master Branch | |
| on: | |
| # push: | |
| # branches: ["master"] # Uncomment post v1.0 | |
| schedule: | |
| - cron: '00 09 * * SUN' # UTC 9:00(2:00 PST Time) every Sunday | |
| env: | |
| AG_BRANCH_NAME: master | |
| permissions: | |
| id-token: write | |
| contents: read | |
| jobs: | |
| generate_amlb_user_dir: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| AG_MODULE: [tabular, timeseries, multimodal] | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v2 | |
| - name: Setup Env Vars | |
| uses: ./.github/actions/setup-env-vars | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v1 | |
| with: | |
| role-to-assume: arn:aws:iam::369469875935:role/AutoGluonCIBenchmarkConfig | |
| role-duration-seconds: 3600 | |
| aws-region: us-east-1 | |
| - name: Generate AMLB User Dir - ${{ matrix.AG_MODULE }} | |
| run: | | |
| /bin/bash CI/bench/generate_amlb_user_dir.sh ${{ matrix.AG_MODULE }} ${{ github.repository }} ${{ env.AG_BRANCH_NAME }} ${{ github.sha }} | |
| benchmark-tabular: | |
| needs: generate_amlb_user_dir | |
| runs-on: ubuntu-latest | |
| defaults: | |
| run: | |
| shell: bash | |
| steps: | |
| - name: Free Disk Space (Ubuntu) | |
| # uses: jlumbroso/[email protected] | |
| uses: hirnidrin/free-disk-space@main # revert back once fix in https://github.com/jlumbroso/free-disk-space/pull/11 | |
| with: | |
| tool-cache: false | |
| android: true | |
| dotnet: true | |
| haskell: true | |
| large-packages: true | |
| docker-images: true | |
| swap-storage: true | |
| - name: Checkout repository for PR | |
| uses: actions/checkout@v2 | |
| - name: Setup Python | |
| uses: actions/setup-python@v1 | |
| with: | |
| python-version: '3.9' | |
| - name: Setup npm | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version: 'latest' | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v1 | |
| with: | |
| role-to-assume: arn:aws:iam::369469875935:role/AutoGluonCIBenchmark | |
| role-duration-seconds: 36000 | |
| aws-region: us-east-1 | |
| - name: Run benchmark - Tabular | |
| shell: bash -l {0} | |
| run: | | |
| nvm install 20 | |
| npm install -g aws-cdk | |
| /bin/bash ./.github/workflow_scripts/run_benchmark.sh tabular best tabular_full 1h8c ${{ env.AG_BRANCH_NAME }} ${{ github.sha }} | |
| benchmark-timeseries: | |
| needs: benchmark-tabular | |
| runs-on: ubuntu-latest | |
| defaults: | |
| run: | |
| shell: bash | |
| steps: | |
| - name: Free Disk Space (Ubuntu) | |
| # uses: jlumbroso/[email protected] | |
| uses: hirnidrin/free-disk-space@main # revert back once fix in https://github.com/jlumbroso/free-disk-space/pull/11 | |
| with: | |
| tool-cache: false | |
| android: true | |
| dotnet: true | |
| haskell: true | |
| large-packages: true | |
| docker-images: true | |
| swap-storage: true | |
| - name: Checkout repository for PR | |
| uses: actions/checkout@v2 | |
| - name: Setup Python | |
| uses: actions/setup-python@v1 | |
| with: | |
| python-version: '3.9' | |
| - name: Setup npm | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version: 'latest' | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v1 | |
| with: | |
| role-to-assume: arn:aws:iam::369469875935:role/AutoGluonCIBenchmark | |
| role-duration-seconds: 36000 | |
| aws-region: us-east-1 | |
| - name: Run benchmark - Timeseries | |
| shell: bash -l {0} | |
| run: | | |
| nvm install 20 | |
| npm install -g aws-cdk | |
| /bin/bash ./.github/workflow_scripts/run_benchmark.sh timeseries dev timeseries_small 10m4c ${{ env.AG_BRANCH_NAME }} ${{ github.sha }} | |
| benchmark-multimodal-text: | |
| needs: benchmark-timeseries | |
| runs-on: ubuntu-latest | |
| defaults: | |
| run: | |
| shell: bash | |
| steps: | |
| - name: Free Disk Space (Ubuntu) | |
| # uses: jlumbroso/[email protected] | |
| uses: hirnidrin/free-disk-space@main # revert back once fix in https://github.com/jlumbroso/free-disk-space/pull/11 | |
| with: | |
| tool-cache: false | |
| android: true | |
| dotnet: true | |
| haskell: true | |
| large-packages: true | |
| docker-images: true | |
| swap-storage: true | |
| - name: Checkout repository for PR | |
| uses: actions/checkout@v2 | |
| - name: Setup Python | |
| uses: actions/setup-python@v1 | |
| with: | |
| python-version: '3.9' | |
| - name: Setup npm | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version: 'latest' | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v1 | |
| with: | |
| role-to-assume: arn:aws:iam::369469875935:role/AutoGluonCIBenchmark | |
| role-duration-seconds: 36000 | |
| aws-region: us-east-1 | |
| - name: Run benchmark - Multimodal | |
| shell: bash -l {0} | |
| run: | | |
| nvm install 20 | |
| npm install -g aws-cdk | |
| /bin/bash ./.github/workflow_scripts/run_benchmark.sh multimodal master_medium automm-text g4_12x ${{ env.AG_BRANCH_NAME }} ${{ github.sha }} | |
| benchmark-multimodal-text-tabular: | |
| needs: benchmark-multimodal-text | |
| runs-on: ubuntu-latest | |
| defaults: | |
| run: | |
| shell: bash | |
| steps: | |
| - name: Free Disk Space (Ubuntu) | |
| # uses: jlumbroso/[email protected] | |
| uses: hirnidrin/free-disk-space@main # revert back once fix in https://github.com/jlumbroso/free-disk-space/pull/11 | |
| with: | |
| tool-cache: false | |
| android: true | |
| dotnet: true | |
| haskell: true | |
| large-packages: true | |
| docker-images: true | |
| swap-storage: true | |
| - name: Checkout repository for PR | |
| uses: actions/checkout@v2 | |
| - name: Setup Python | |
| uses: actions/setup-python@v1 | |
| with: | |
| python-version: '3.9' | |
| - name: Setup npm | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version: 'latest' | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v1 | |
| with: | |
| role-to-assume: arn:aws:iam::369469875935:role/AutoGluonCIBenchmark | |
| role-duration-seconds: 36000 | |
| aws-region: us-east-1 | |
| - name: Run benchmark - Multimodal | |
| shell: bash -l {0} | |
| run: | | |
| nvm install 20 | |
| npm install -g aws-cdk | |
| /bin/bash ./.github/workflow_scripts/run_benchmark.sh multimodal master_medium automm-text-tabular g4_12x ${{ env.AG_BRANCH_NAME }} ${{ github.sha }} | |
| benchmark-multimodal-image: | |
| needs: benchmark-multimodal-text-tabular | |
| runs-on: ubuntu-latest | |
| defaults: | |
| run: | |
| shell: bash | |
| steps: | |
| - name: Free Disk Space (Ubuntu) | |
| # uses: jlumbroso/[email protected] | |
| uses: hirnidrin/free-disk-space@main # revert back once fix in https://github.com/jlumbroso/free-disk-space/pull/11 | |
| with: | |
| tool-cache: false | |
| android: true | |
| dotnet: true | |
| haskell: true | |
| large-packages: true | |
| docker-images: true | |
| swap-storage: true | |
| - name: Checkout repository for PR | |
| uses: actions/checkout@v2 | |
| - name: Setup Python | |
| uses: actions/setup-python@v1 | |
| with: | |
| python-version: '3.9' | |
| - name: Setup npm | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version: 'latest' | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v1 | |
| with: | |
| role-to-assume: arn:aws:iam::369469875935:role/AutoGluonCIBenchmark | |
| role-duration-seconds: 36000 | |
| aws-region: us-east-1 | |
| - name: Run benchmark - Multimodal | |
| shell: bash -l {0} | |
| run: | | |
| nvm install 20 | |
| npm install -g aws-cdk | |
| /bin/bash ./.github/workflow_scripts/run_benchmark.sh multimodal master_medium automm-image g4_12x ${{ env.AG_BRANCH_NAME }} ${{ github.sha }} |