Continuous benchmarking - Baseline experiments #223
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # MIT License | |
| # | |
| # Copyright (c) 2022 Dilina Dehigama and EASE Lab | |
| # | |
| # Permission is hereby granted, free of charge, to any person obtaining a copy | |
| # of this software and associated documentation files (the "Software"), to deal | |
| # in the Software without restriction, including without limitation the rights | |
| # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell | |
| # copies of the Software, and to permit persons to whom the Software is | |
| # furnished to do so, subject to the following conditions: | |
| # | |
| # The above copyright notice and this permission notice shall be included in all | |
| # copies or substantial portions of the Software. | |
| # | |
| # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
| # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
| # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE | |
| # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | |
| # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, | |
| # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
| # SOFTWARE. | |
| name: Continuous benchmarking - Baseline experiments | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: "0 0 * * 1" | |
| jobs: | |
| build_client: | |
| name: Build framework | |
| runs-on: ubuntu-latest | |
| env: | |
| working-directory: ./src | |
| steps: | |
| - name: Check out code into the Go module directory | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: continuous-benchmarking | |
| - name: Set up Go 1.21 | |
| uses: actions/setup-go@v2 | |
| with: | |
| go-version: 1.21 | |
| - name: Build client binary | |
| working-directory: ${{ env.working-directory }} | |
| run: env GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o stellar main.go | |
| - name: Package client artifact | |
| working-directory: ${{ env.working-directory }} | |
| run: tar -czvf build.tar ./stellar | |
| - name: Upload client artifact | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: STeLLAR-build | |
| path: ${{ env.working-directory }}/build.tar | |
| retention-days: 1 | |
| warm-baseline-aws: | |
| name: AWS warm baseline experiment | |
| needs: build_client | |
| runs-on: [ self-hosted, aws ] | |
| timeout-minutes: 1200 | |
| env: | |
| working-directory: src | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_KEY }} | |
| steps: | |
| - name: "Cleanup build folder" | |
| run: | | |
| sudo rm -rf ./* || true | |
| sudo rm -rf ./.??* || true | |
| - name: Check out code into the Go module directory | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: continuous-benchmarking | |
| - name: Configure AWS credentials using EASE lab account | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_KEY }} | |
| aws-region: us-west-1 | |
| - name: Set up Node.js 18 | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version: 18 | |
| - name: Cleanup serverless folder | |
| run: | | |
| rm -rf /home/ubuntu/actions-runner/_work/_tool/node/18.20.4/x64/lib/node_modules/serverless | |
| - name: Install Serverless framework and related plugins | |
| run: npm install -g [email protected] | |
| - name: Download client artifact | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: STeLLAR-build | |
| - name: Untar client build | |
| working-directory: ${{ env.working-directory }} | |
| run: tar --strip-components=1 -xvf ../build.tar -C . | |
| - name: AWS Warm Function Invocation - Baseline | |
| uses: nick-fields/retry@v2 | |
| with: | |
| timeout_minutes: 1440 | |
| max_attempts: 5 | |
| retry_wait_seconds: 60 | |
| command: cd src && ./stellar -a 356764711652 -o latency-samples -c ../continuous-benchmarking/experiments/warm-function-invocations/warm-baseline-aws.json -db -w | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: warm-baseline-aws | |
| path: ${{ env.working-directory }}/latency-samples | |
| - name: Add zero'ed data (Failed experiment) | |
| if: ${{ failure() }} | |
| env: | |
| DATA_INSERT_URL: ${{ secrets.DATA_INSERT_URL}} | |
| run: | | |
| curl -XPOST -H "Content-type: application/json" -d '{"experiment_type":"warm-baseline-aws","date":"'$(date +%F)'","min":"0","max":"0","median":"0","tail_latency":"0","first_quartile":"0","third_quartile":"0","standard_deviation":"0","payload_size":"0","burst_size":"0","IATType":"0","count":"0","provider":"aws"}' $DATA_INSERT_URL | |
| - name: Send Slack message using Incoming Webhooks | |
| if: ${{ failure() }} | |
| uses: slackapi/[email protected] | |
| with: | |
| payload: | | |
| { | |
| "blocks": [ | |
| { | |
| "type": "header", | |
| "text": { | |
| "type": "plain_text", | |
| "text": "Scheduled experiment (Job status)" | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "fields": [ | |
| { | |
| "type": "mrkdwn", | |
| "text": "*Workflow Name*: ${{ github.workflow }}" | |
| }, | |
| { | |
| "type": "mrkdwn", | |
| "text": "*Job Name*: ${{ github.job }}" | |
| }, | |
| { | |
| "type": "mrkdwn", | |
| "text": "*Job Status*: ${{ job.status }} ❌" | |
| } | |
| ] | |
| }, | |
| { | |
| "type": "section", | |
| "fields": [ | |
| { | |
| "type": "mrkdwn", | |
| "text": "<!channel>" | |
| } | |
| ] | |
| } | |
| ] | |
| } | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK | |
| warm-baseline-gcr: | |
| name: GCR warm baseline experiment | |
| needs: build_client | |
| runs-on: [ self-hosted, gcr ] | |
| timeout-minutes: 1200 | |
| env: | |
| working-directory: src | |
| DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} | |
| DOCKER_HUB_ACCESS_TOKEN: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} | |
| steps: | |
| - name: "Cleanup build folder" | |
| run: | | |
| sudo rm -rf ./* || true | |
| sudo rm -rf ./.??* || true | |
| - name: Check out code into the Go module directory | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: continuous-benchmarking | |
| - id: auth | |
| name: Configure GCR credentials | |
| uses: google-github-actions/auth@v1 | |
| with: | |
| credentials_json: ${{ secrets.GCR_CREDENTIALS }} | |
| - name: Set up gcloud | |
| uses: google-github-actions/setup-gcloud@v1 | |
| with: | |
| version: ">= 363.0.0" | |
| - name: Download client artifact | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: STeLLAR-build | |
| - name: Untar client build | |
| working-directory: ${{ env.working-directory }} | |
| run: tar --strip-components=1 -xvf ../build.tar -C . | |
| - name: GCR Warm Function Invocation - Baseline | |
| uses: nick-fields/retry@v2 | |
| with: | |
| timeout_minutes: 1440 | |
| max_attempts: 5 | |
| retry_wait_seconds: 60 | |
| command: cd src && ./stellar -a 356764711652 -o latency-samples -c ../continuous-benchmarking/experiments/warm-function-invocations/warm-baseline-gcr.json -db -w | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: warm-baseline-gcr | |
| path: ${{ env.working-directory }}/latency-samples | |
| - name: Add zero'ed data (Failed experiment) | |
| if: ${{ failure() }} | |
| env: | |
| DATA_INSERT_URL: ${{ secrets.DATA_INSERT_URL}} | |
| run: | | |
| curl -XPOST -H "Content-type: application/json" -d '{"experiment_type":"warm-baseline-gcr","date":"'$(date +%F)'","min":"0","max":"0","median":"0","tail_latency":"0","first_quartile":"0","third_quartile":"0","standard_deviation":"0","payload_size":"0","burst_size":"0","IATType":"0","count":"0","provider":"gcr"}' $DATA_INSERT_URL | |
| - name: Send Slack message using Incoming Webhooks | |
| if: ${{ failure() }} | |
| uses: slackapi/[email protected] | |
| with: | |
| payload: | | |
| { | |
| "blocks": [ | |
| { | |
| "type": "header", | |
| "text": { | |
| "type": "plain_text", | |
| "text": "Scheduled experiment (Job status)" | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "fields": [ | |
| { | |
| "type": "mrkdwn", | |
| "text": "*Workflow Name*: ${{ github.workflow }}" | |
| }, | |
| { | |
| "type": "mrkdwn", | |
| "text": "*Job Name*: ${{ github.job }}" | |
| }, | |
| { | |
| "type": "mrkdwn", | |
| "text": "*Job Status*: ${{ job.status }} ❌" | |
| } | |
| ] | |
| }, | |
| { | |
| "type": "section", | |
| "fields": [ | |
| { | |
| "type": "mrkdwn", | |
| "text": "<!channel>" | |
| } | |
| ] | |
| } | |
| ] | |
| } | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK | |
| warm-baseline-cloudflare: | |
| name: Cloudflare warm baseline experiment | |
| needs: build_client | |
| runs-on: [ self-hosted, cloudflare ] | |
| timeout-minutes: 1200 | |
| env: | |
| working-directory: src | |
| CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} | |
| steps: | |
| - name: "Cleanup build folder" | |
| run: | | |
| sudo rm -rf ./* || true | |
| sudo rm -rf ./.??* || true | |
| - name: Check out code into the Go module directory | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: continuous-benchmarking | |
| - name: Set up Node.js 18 | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version: 18 | |
| - name: Install Cloudflare Wrangler | |
| run: npm install -g wrangler | |
| - name: Download client artifact | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: STeLLAR-build | |
| - name: Untar client build | |
| working-directory: ${{ env.working-directory }} | |
| run: tar --strip-components=1 -xvf ../build.tar -C . | |
| - name: Cloudflare Warm Function Invocation - Baseline | |
| uses: nick-fields/retry@v2 | |
| with: | |
| timeout_minutes: 1440 | |
| max_attempts: 5 | |
| retry_wait_seconds: 60 | |
| command: cd src && ./stellar -a 356764711652 -o latency-samples -c ../continuous-benchmarking/experiments/warm-function-invocations/warm-baseline-cloudflare.json -db -w | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: warm-baseline-cloudflare | |
| path: ${{ env.working-directory }}/latency-samples | |
| - name: Add zero'ed data (Failed experiment) | |
| if: ${{ failure() }} | |
| env: | |
| DATA_INSERT_URL: ${{ secrets.DATA_INSERT_URL}} | |
| run: | | |
| curl -XPOST -H "Content-type: application/json" -d '{"experiment_type":"warm-baseline-cloudflare","date":"'$(date +%F)'","min":"0","max":"0","median":"0","tail_latency":"0","first_quartile":"0","third_quartile":"0","standard_deviation":"0","payload_size":"0","burst_size":"0","IATType":"0","count":"0","provider":"cloudflare"}' $DATA_INSERT_URL | |
| - name: Send Slack message using Incoming Webhooks | |
| if: ${{ failure() }} | |
| uses: slackapi/[email protected] | |
| with: | |
| payload: | | |
| { | |
| "blocks": [ | |
| { | |
| "type": "header", | |
| "text": { | |
| "type": "plain_text", | |
| "text": "Scheduled experiment (Job status)" | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "fields": [ | |
| { | |
| "type": "mrkdwn", | |
| "text": "*Workflow Name*: ${{ github.workflow }}" | |
| }, | |
| { | |
| "type": "mrkdwn", | |
| "text": "*Job Name*: ${{ github.job }}" | |
| }, | |
| { | |
| "type": "mrkdwn", | |
| "text": "*Job Status*: ${{ job.status }} ❌" | |
| } | |
| ] | |
| }, | |
| { | |
| "type": "section", | |
| "fields": [ | |
| { | |
| "type": "mrkdwn", | |
| "text": "<!channel>" | |
| } | |
| ] | |
| } | |
| ] | |
| } | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK | |
| warm-baseline-azure: | |
| name: Azure warm baseline experiment | |
| needs: build_client | |
| runs-on: [ self-hosted, azure ] | |
| timeout-minutes: 1200 | |
| env: | |
| working-directory: src | |
| AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} | |
| AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} | |
| AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} | |
| AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} | |
| steps: | |
| - name: "Cleanup build folder" | |
| run: | | |
| sudo rm -rf ./* || true | |
| sudo rm -rf ./.??* || true | |
| - name: Check out code into the Go module directory | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: continuous-benchmarking | |
| - name: Set up Node.js 18 | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version: 18 | |
| - name: Install Serverless framework and related plugins | |
| run: npm install -g [email protected] serverless-azure-functions functions-have-names serverless-aliyun-function-compute | |
| - name: Download client artifact | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: STeLLAR-build | |
| - name: Untar client build | |
| working-directory: ${{ env.working-directory }} | |
| run: tar --strip-components=1 -xvf ../build.tar -C . | |
| - name: Azure Warm Function Invocation - Baseline | |
| uses: nick-fields/retry@v2 | |
| with: | |
| timeout_minutes: 1440 | |
| max_attempts: 5 | |
| retry_wait_seconds: 60 | |
| command: cd src && ./stellar -c ../continuous-benchmarking/experiments/warm-function-invocations/warm-baseline-azure.json -db -w | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: warm-baseline-azure | |
| path: ${{ env.working-directory }}/latency-samples | |
| - name: Add zero'ed data (Failed experiment) | |
| if: ${{ failure() }} | |
| env: | |
| DATA_INSERT_URL: ${{ secrets.DATA_INSERT_URL}} | |
| run: | | |
| curl -XPOST -H "Content-type: application/json" -d '{"experiment_type":"warm-baseline-azure","date":"'$(date +%F)'","min":"0","max":"0","median":"0","tail_latency":"0","first_quartile":"0","third_quartile":"0","standard_deviation":"0","payload_size":"0","burst_size":"0","IATType":"0","count":"0","provider":"azure"}' $DATA_INSERT_URL | |
| - name: Send Slack message using Incoming Webhooks | |
| if: ${{ failure() }} | |
| uses: slackapi/[email protected] | |
| with: | |
| payload: | | |
| { | |
| "blocks": [ | |
| { | |
| "type": "header", | |
| "text": { | |
| "type": "plain_text", | |
| "text": "Scheduled experiment (Job status)" | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "fields": [ | |
| { | |
| "type": "mrkdwn", | |
| "text": "*Workflow Name*: ${{ github.workflow }}" | |
| }, | |
| { | |
| "type": "mrkdwn", | |
| "text": "*Job Name*: ${{ github.job }}" | |
| }, | |
| { | |
| "type": "mrkdwn", | |
| "text": "*Job Status*: ${{ job.status }} ❌" | |
| } | |
| ] | |
| }, | |
| { | |
| "type": "section", | |
| "fields": [ | |
| { | |
| "type": "mrkdwn", | |
| "text": "<!channel>" | |
| } | |
| ] | |
| } | |
| ] | |
| } | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK | |
| cold-baseline-aws: | |
| name: AWS cold baseline experiment | |
| needs: build_client | |
| runs-on: [ self-hosted, aws ] | |
| timeout-minutes: 1200 | |
| env: | |
| working-directory: src | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_KEY }} | |
| steps: | |
| - name: "Cleanup build folder" | |
| run: | | |
| sudo rm -rf ./* || true | |
| sudo rm -rf ./.??* || true | |
| - name: Check out code into the Go module directory | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: continuous-benchmarking | |
| - name: Configure AWS credentials using EASE lab account | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_KEY }} | |
| aws-region: us-west-1 | |
| - name: Set up Node.js 18 | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version: 18 | |
| - name: Cleanup serverless folder | |
| run: | | |
| rm -rf /home/ubuntu/actions-runner/_work/_tool/node/18.20.4/x64/lib/node_modules/serverless | |
| - name: Install Serverless framework and related plugins | |
| run: npm install -g [email protected] | |
| - name: Download client artifact | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: STeLLAR-build | |
| - name: Untar client build | |
| working-directory: ${{ env.working-directory }} | |
| run: tar --strip-components=1 -xvf ../build.tar -C . | |
| - name: AWS Cold Function Invocation - Baseline | |
| uses: nick-fields/retry@v2 | |
| with: | |
| timeout_minutes: 1440 | |
| max_attempts: 5 | |
| retry_wait_seconds: 60 | |
| command: cd src && ./stellar -a 356764711652 -o latency-samples -c ../continuous-benchmarking/experiments/cold-function-invocations/cold-baseline-aws.json -db | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: cold-baseline-aws | |
| path: ${{ env.working-directory }}/latency-samples | |
| - name: Add zero'ed data (Failed experiment) | |
| if: ${{ failure() }} | |
| env: | |
| DATA_INSERT_URL: ${{ secrets.DATA_INSERT_URL}} | |
| run: | | |
| curl -XPOST -H "Content-type: application/json" -d '{"experiment_type":"cold-baseline-aws","date":"'$(date +%F)'","min":"0","max":"0","median":"0","tail_latency":"0","first_quartile":"0","third_quartile":"0","standard_deviation":"0","payload_size":"0","burst_size":"0","IATType":"0","count":"0","provider":"aws"}' $DATA_INSERT_URL | |
| - name: Send Slack message using Incoming Webhooks | |
| if: ${{ failure() }} | |
| uses: slackapi/[email protected] | |
| with: | |
| payload: | | |
| { | |
| "blocks": [ | |
| { | |
| "type": "header", | |
| "text": { | |
| "type": "plain_text", | |
| "text": "Scheduled experiment (Job status)" | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "fields": [ | |
| { | |
| "type": "mrkdwn", | |
| "text": "*Workflow Name*: ${{ github.workflow }}" | |
| }, | |
| { | |
| "type": "mrkdwn", | |
| "text": "*Job Name*: ${{ github.job }}" | |
| }, | |
| { | |
| "type": "mrkdwn", | |
| "text": "*Job Status*: ${{ job.status }} ❌" | |
| } | |
| ] | |
| }, | |
| { | |
| "type": "section", | |
| "fields": [ | |
| { | |
| "type": "mrkdwn", | |
| "text": "<!channel>" | |
| } | |
| ] | |
| } | |
| ] | |
| } | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK | |
| cold-baseline-gcr: | |
| name: GCR cold baseline experiment | |
| needs: build_client | |
| runs-on: [ self-hosted, gcr ] | |
| timeout-minutes: 1200 | |
| env: | |
| working-directory: src | |
| DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} | |
| DOCKER_HUB_ACCESS_TOKEN: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} | |
| steps: | |
| - name: "Cleanup build folder" | |
| run: | | |
| sudo rm -rf ./* || true | |
| sudo rm -rf ./.??* || true | |
| - name: Check out code into the Go module directory | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: continuous-benchmarking | |
| - id: auth | |
| name: Configure GCR credentials | |
| uses: google-github-actions/auth@v1 | |
| with: | |
| credentials_json: ${{ secrets.GCR_CREDENTIALS }} | |
| - name: Set up gcloud | |
| uses: google-github-actions/setup-gcloud@v1 | |
| with: | |
| version: ">= 363.0.0" | |
| - name: Download client artifact | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: STeLLAR-build | |
| - name: Untar client build | |
| working-directory: ${{ env.working-directory }} | |
| run: tar --strip-components=1 -xvf ../build.tar -C . | |
| - name: GCR Cold Function Invocation - Baseline | |
| uses: nick-fields/retry@v2 | |
| with: | |
| timeout_minutes: 1440 | |
| max_attempts: 5 | |
| retry_wait_seconds: 60 | |
| command: cd src && ./stellar -a 356764711652 -o latency-samples -c ../continuous-benchmarking/experiments/cold-function-invocations/cold-baseline-gcr.json -db | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: cold-baseline-gcr | |
| path: ${{ env.working-directory }}/latency-samples | |
| - name: Add zero'ed data (Failed experiment) | |
| if: ${{ failure() }} | |
| env: | |
| DATA_INSERT_URL: ${{ secrets.DATA_INSERT_URL}} | |
| run: | | |
| curl -XPOST -H "Content-type: application/json" -d '{"experiment_type":"cold-baseline-gcr","date":"'$(date +%F)'","min":"0","max":"0","median":"0","tail_latency":"0","first_quartile":"0","third_quartile":"0","standard_deviation":"0","payload_size":"0","burst_size":"0","IATType":"0","count":"0","provider":"gcr"}' $DATA_INSERT_URL | |
| - name: Send Slack message using Incoming Webhooks | |
| if: ${{ failure() }} | |
| uses: slackapi/[email protected] | |
| with: | |
| payload: | | |
| { | |
| "blocks": [ | |
| { | |
| "type": "header", | |
| "text": { | |
| "type": "plain_text", | |
| "text": "Scheduled experiment (Job status)" | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "fields": [ | |
| { | |
| "type": "mrkdwn", | |
| "text": "*Workflow Name*: ${{ github.workflow }}" | |
| }, | |
| { | |
| "type": "mrkdwn", | |
| "text": "*Job Name*: ${{ github.job }}" | |
| }, | |
| { | |
| "type": "mrkdwn", | |
| "text": "*Job Status*: ${{ job.status }} ❌" | |
| } | |
| ] | |
| }, | |
| { | |
| "type": "section", | |
| "fields": [ | |
| { | |
| "type": "mrkdwn", | |
| "text": "<!channel>" | |
| } | |
| ] | |
| } | |
| ] | |
| } | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK | |
| cold-baseline-cloudflare: | |
| name: Cloudflare cold baseline experiment | |
| needs: build_client | |
| runs-on: [ self-hosted, cloudflare ] | |
| timeout-minutes: 1200 | |
| env: | |
| working-directory: src | |
| CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} | |
| steps: | |
| - name: "Cleanup build folder" | |
| run: | | |
| sudo rm -rf ./* || true | |
| sudo rm -rf ./.??* || true | |
| - name: Check out code into the Go module directory | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: continuous-benchmarking | |
| - name: Set up Node.js 18 | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version: 18 | |
| - name: Install Cloudflare Wrangler | |
| run: npm install -g wrangler | |
| - name: Download client artifact | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: STeLLAR-build | |
| - name: Untar client build | |
| working-directory: ${{ env.working-directory }} | |
| run: tar --strip-components=1 -xvf ../build.tar -C . | |
| - name: Cloudflare Cold Function Invocation - Baseline | |
| uses: nick-fields/retry@v2 | |
| with: | |
| timeout_minutes: 1440 | |
| max_attempts: 5 | |
| retry_wait_seconds: 60 | |
| command: cd src && ./stellar -a 356764711652 -o latency-samples -c ../continuous-benchmarking/experiments/cold-function-invocations/cold-baseline-cloudflare.json -db | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: cold-baseline-cloudflare | |
| path: ${{ env.working-directory }}/latency-samples | |
| - name: Add zero'ed data (Failed experiment) | |
| if: ${{ failure() }} | |
| env: | |
| DATA_INSERT_URL: ${{ secrets.DATA_INSERT_URL}} | |
| run: | | |
| curl -XPOST -H "Content-type: application/json" -d '{"experiment_type":"cold-baseline-cloudflare","date":"'$(date +%F)'","min":"0","max":"0","median":"0","tail_latency":"0","first_quartile":"0","third_quartile":"0","standard_deviation":"0","payload_size":"0","burst_size":"0","IATType":"0","count":"0","provider":"cloudflare"}' $DATA_INSERT_URL | |
| - name: Send Slack message using Incoming Webhooks | |
| if: ${{ failure() }} | |
| uses: slackapi/[email protected] | |
| with: | |
| payload: | | |
| { | |
| "blocks": [ | |
| { | |
| "type": "header", | |
| "text": { | |
| "type": "plain_text", | |
| "text": "Scheduled experiment (Job status)" | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "fields": [ | |
| { | |
| "type": "mrkdwn", | |
| "text": "*Workflow Name*: ${{ github.workflow }}" | |
| }, | |
| { | |
| "type": "mrkdwn", | |
| "text": "*Job Name*: ${{ github.job }}" | |
| }, | |
| { | |
| "type": "mrkdwn", | |
| "text": "*Job Status*: ${{ job.status }} ❌" | |
| } | |
| ] | |
| }, | |
| { | |
| "type": "section", | |
| "fields": [ | |
| { | |
| "type": "mrkdwn", | |
| "text": "<!channel>" | |
| } | |
| ] | |
| } | |
| ] | |
| } | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK | |
| cold-baseline-azure: | |
| name: Azure cold baseline experiment | |
| needs: build_client | |
| runs-on: [ self-hosted, azure ] | |
| timeout-minutes: 1200 | |
| env: | |
| working-directory: src | |
| AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} | |
| AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} | |
| AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} | |
| AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} | |
| steps: | |
| - name: "Cleanup build folder" | |
| run: | | |
| sudo rm -rf ./* || true | |
| sudo rm -rf ./.??* || true | |
| - name: Check out code into the Go module directory | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: continuous-benchmarking | |
| - name: Set up Node.js 18 | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version: 18 | |
| - name: Install Serverless framework and related plugins | |
| run: npm install -g [email protected] serverless-azure-functions functions-have-names serverless-aliyun-function-compute | |
| - name: Download client artifact | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: STeLLAR-build | |
| - name: Untar client build | |
| working-directory: ${{ env.working-directory }} | |
| run: tar --strip-components=1 -xvf ../build.tar -C . | |
| - name: Azure Cold Function Invocation - Baseline | |
| uses: nick-fields/retry@v2 | |
| with: | |
| timeout_minutes: 1440 | |
| max_attempts: 5 | |
| retry_wait_seconds: 60 | |
| command: cd src && ./stellar -c ../continuous-benchmarking/experiments/cold-function-invocations/cold-baseline-azure.json -db -l debug | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: cold-baseline-azure | |
| path: ${{ env.working-directory }}/latency-samples | |
| - name: Add zero'ed data (Failed experiment) | |
| if: ${{ failure() }} | |
| env: | |
| DATA_INSERT_URL: ${{ secrets.DATA_INSERT_URL}} | |
| run: | | |
| curl -XPOST -H "Content-type: application/json" -d '{"experiment_type":"cold-baseline-azure","date":"'$(date +%F)'","min":"0","max":"0","median":"0","tail_latency":"0","first_quartile":"0","third_quartile":"0","standard_deviation":"0","payload_size":"0","burst_size":"0","IATType":"0","count":"0","provider":"azure"}' $DATA_INSERT_URL | |
| - name: Send Slack message using Incoming Webhooks | |
| if: ${{ failure() }} | |
| uses: slackapi/[email protected] | |
| with: | |
| payload: | | |
| { | |
| "blocks": [ | |
| { | |
| "type": "header", | |
| "text": { | |
| "type": "plain_text", | |
| "text": "Scheduled experiment (Job status)" | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "fields": [ | |
| { | |
| "type": "mrkdwn", | |
| "text": "*Workflow Name*: ${{ github.workflow }}" | |
| }, | |
| { | |
| "type": "mrkdwn", | |
| "text": "*Job Name*: ${{ github.job }}" | |
| }, | |
| { | |
| "type": "mrkdwn", | |
| "text": "*Job Status*: ${{ job.status }} ❌" | |
| } | |
| ] | |
| }, | |
| { | |
| "type": "section", | |
| "fields": [ | |
| { | |
| "type": "mrkdwn", | |
| "text": "<!channel>" | |
| } | |
| ] | |
| } | |
| ] | |
| } | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK |