Skip to content

Continuous benchmarking - Image size experiments #193

Continuous benchmarking - Image size experiments

Continuous benchmarking - Image size experiments #193

name: Continuous benchmarking - Image size experiments
on:
workflow_dispatch:
schedule:
- cron: "0 12 * * 1"
jobs:
build_client:
name: Build framework
runs-on: ubuntu-latest
env:
working-directory: ./src
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v4
with:
ref: continuous-benchmarking
- name: Set up Go 1.21
uses: actions/setup-go@v3
with:
go-version: 1.21
- name: Build client binary
working-directory: ${{ env.working-directory }}
run: env GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o stellar main.go
- name: Package client artifact
working-directory: ${{ env.working-directory }}
run: tar -czvf build.tar ./stellar
- name: Upload client artifact
uses: actions/upload-artifact@v3
with:
name: STeLLAR-build
path: ${{ env.working-directory }}/build.tar
retention-days: 1
cold-image-size-50-aws:
name: AWS 50MB image size experiment
needs: build_client
runs-on: [ self-hosted, aws ]
timeout-minutes: 1200
env:
working-directory: src
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_KEY }}
steps:
- name: "Cleanup build folder"
run: |
sudo rm -rf ./* || true
sudo rm -rf ./.??* || true
- name: Check out code into the Go module directory
uses: actions/checkout@v4
with:
ref: continuous-benchmarking
- name: Configure AWS credentials using EASE lab account
uses: aws-actions/configure-aws-credentials@v4
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_KEY }}
aws-region: us-west-1
- name: Set up Node.js 18
uses: actions/setup-node@v3
with:
node-version: 18
- name: Cleanup serverless folder
run: |
rm -rf /home/ubuntu/actions-runner/_work/_tool/node/18.20.4/x64/lib/node_modules/serverless
- name: Install Serverless framework and related plugins
run: npm install -g [email protected]
- name: Download client artifact
uses: actions/download-artifact@v3
with:
name: STeLLAR-build
- name: Untar client build
working-directory: ${{ env.working-directory }}
run: tar --strip-components=1 -xvf ../build.tar -C .
- name: AWS Image Size 50MB
uses: nick-fields/retry@v2
with:
timeout_minutes: 1440
max_attempts: 5
retry_wait_seconds: 60
command: cd src && ./stellar -a 356764711652 -o latency-samples -c ../continuous-benchmarking/experiments/cold-function-invocations/image-size/cold-image-size-50-aws.json -db
- uses: actions/upload-artifact@v3
with:
name: cold-image-size-50-aws
path: ${{ env.working-directory }}/latency-samples
- name: Add zero'ed data (Failed experiment)
if: ${{ failure() }}
env:
DATA_INSERT_URL: ${{ secrets.DATA_INSERT_URL}}
run: |
curl -XPOST -H "Content-type: application/json" -d '{"experiment_type":"cold-image-size-50-aws","date":"'$(date +%F)'","min":"0","max":"0","median":"0","tail_latency":"0","first_quartile":"0","third_quartile":"0","standard_deviation":"0","payload_size":"0","burst_size":"0","IATType":"0","count":"0","provider":"aws"}' $DATA_INSERT_URL
- name: Send Slack message using Incoming Webhooks
if: ${{ failure() }}
uses: slackapi/[email protected]
with:
payload: |
{
"blocks": [
{
"type": "header",
"text": {
"type": "plain_text",
"text": "Scheduled experiment (Job status)"
}
},
{
"type": "section",
"fields": [
{
"type": "mrkdwn",
"text": "*Workflow Name*: ${{ github.workflow }}"
},
{
"type": "mrkdwn",
"text": "*Job Name*: ${{ github.job }}"
},
{
"type": "mrkdwn",
"text": "*Job Status*: ${{ job.status }} ❌"
}
]
},
{
"type": "section",
"fields": [
{
"type": "mrkdwn",
"text": "<!channel>"
}
]
}
]
}
env:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
cold-image-size-100-aws:
name: AWS 100MB image size experiment
needs: build_client
runs-on: [ self-hosted, aws ]
timeout-minutes: 1200
env:
working-directory: src
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_KEY }}
steps:
- name: "Cleanup build folder"
run: |
sudo rm -rf ./* || true
sudo rm -rf ./.??* || true
- name: Check out code into the Go module directory
uses: actions/checkout@v4
with:
ref: continuous-benchmarking
- name: Configure AWS credentials using EASE lab account
uses: aws-actions/configure-aws-credentials@v4
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_KEY }}
aws-region: us-west-1
- name: Set up Node.js 18
uses: actions/setup-node@v3
with:
node-version: 18
- name: Cleanup serverless folder
run: |
rm -rf /home/ubuntu/actions-runner/_work/_tool/node/18.20.4/x64/lib/node_modules/serverless
- name: Install Serverless framework and related plugins
run: npm install -g [email protected]
- name: Download client artifact
uses: actions/download-artifact@v3
with:
name: STeLLAR-build
- name: Untar client build
working-directory: ${{ env.working-directory }}
run: tar --strip-components=1 -xvf ../build.tar -C .
- name: AWS Image Size 100MB
uses: nick-fields/retry@v2
with:
timeout_minutes: 1440
max_attempts: 5
retry_wait_seconds: 60
command: cd src && ./stellar -a 356764711652 -o latency-samples -c ../continuous-benchmarking/experiments/cold-function-invocations/image-size/cold-image-size-100-aws.json -db
- uses: actions/upload-artifact@v3
with:
name: cold-image-size-100-aws
path: ${{ env.working-directory }}/latency-samples
- name: Add zero'ed data (Failed experiment)
if: ${{ failure() }}
env:
DATA_INSERT_URL: ${{ secrets.DATA_INSERT_URL}}
run: |
curl -XPOST -H "Content-type: application/json" -d '{"experiment_type":"cold-image-size-100-aws","date":"'$(date +%F)'","min":"0","max":"0","median":"0","tail_latency":"0","first_quartile":"0","third_quartile":"0","standard_deviation":"0","payload_size":"0","burst_size":"0","IATType":"0","count":"0","provider":"aws"}' $DATA_INSERT_URL
- name: Send Slack message using Incoming Webhooks
if: ${{ failure() }}
uses: slackapi/[email protected]
with:
payload: |
{
"blocks": [
{
"type": "header",
"text": {
"type": "plain_text",
"text": "Scheduled experiment (Job status)"
}
},
{
"type": "section",
"fields": [
{
"type": "mrkdwn",
"text": "*Workflow Name*: ${{ github.workflow }}"
},
{
"type": "mrkdwn",
"text": "*Job Name*: ${{ github.job }}"
},
{
"type": "mrkdwn",
"text": "*Job Status*: ${{ job.status }} ❌"
}
]
},
{
"type": "section",
"fields": [
{
"type": "mrkdwn",
"text": "<!channel>"
}
]
}
]
}
env:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
cold-image-size-50-azure:
name: Azure 50MB image size experiment
needs: build_client
runs-on: [ self-hosted, azure ]
timeout-minutes: 1200
env:
working-directory: src
AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
steps:
- name: "Cleanup build folder"
run: |
sudo rm -rf ./* || true
sudo rm -rf ./.??* || true
- name: Check out code into the Go module directory
uses: actions/checkout@v4
with:
ref: continuous-benchmarking
- name: Set up Node.js 18
uses: actions/setup-node@v3
with:
node-version: 18
- name: Install Serverless framework and related plugins
run: npm install -g [email protected] serverless-azure-functions functions-have-names serverless-aliyun-function-compute
- name: Download client artifact
uses: actions/download-artifact@v3
with:
name: STeLLAR-build
- name: Untar client build
working-directory: ${{ env.working-directory }}
run: tar --strip-components=1 -xvf ../build.tar -C .
- name: Azure Image Size 50MB
uses: nick-fields/retry@v2
with:
timeout_minutes: 1440
max_attempts: 5
retry_wait_seconds: 60
command: cd src && ./stellar -c ../continuous-benchmarking/experiments/cold-function-invocations/image-size/cold-image-size-50-azure.json -db
- uses: actions/upload-artifact@v3
with:
name: cold-image-size-50-azure
path: ${{ env.working-directory }}/latency-samples
- name: Add zero'ed data (Failed experiment)
if: ${{ failure() }}
env:
DATA_INSERT_URL: ${{ secrets.DATA_INSERT_URL}}
run: |
curl -XPOST -H "Content-type: application/json" -d '{"experiment_type":"cold-image-size-50-azure","date":"'$(date +%F)'","min":"0","max":"0","median":"0","tail_latency":"0","first_quartile":"0","third_quartile":"0","standard_deviation":"0","payload_size":"0","burst_size":"0","IATType":"0","count":"0","provider":"azure"}' $DATA_INSERT_URL
- name: Send Slack message using Incoming Webhooks
if: ${{ failure() }}
uses: slackapi/[email protected]
with:
payload: |
{
"blocks": [
{
"type": "header",
"text": {
"type": "plain_text",
"text": "Scheduled experiment (Job status)"
}
},
{
"type": "section",
"fields": [
{
"type": "mrkdwn",
"text": "*Workflow Name*: ${{ github.workflow }}"
},
{
"type": "mrkdwn",
"text": "*Job Name*: ${{ github.job }}"
},
{
"type": "mrkdwn",
"text": "*Job Status*: ${{ job.status }} ❌"
}
]
},
{
"type": "section",
"fields": [
{
"type": "mrkdwn",
"text": "<!channel>"
}
]
}
]
}
env:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
cold-image-size-100-azure:
name: Azure 100MB image size experiment
needs: build_client
runs-on: [ self-hosted, azure ]
timeout-minutes: 1200
env:
working-directory: src
AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
steps:
- name: "Cleanup build folder"
run: |
sudo rm -rf ./* || true
sudo rm -rf ./.??* || true
- name: Check out code into the Go module directory
uses: actions/checkout@v4
with:
ref: continuous-benchmarking
- name: Set up Node.js 18
uses: actions/setup-node@v3
with:
node-version: 18
- name: Install Serverless framework and related plugins
run: npm install -g [email protected] serverless-azure-functions functions-have-names serverless-aliyun-function-compute
- name: Download client artifact
uses: actions/download-artifact@v3
with:
name: STeLLAR-build
- name: Untar client build
working-directory: ${{ env.working-directory }}
run: tar --strip-components=1 -xvf ../build.tar -C .
- name: Azure Image Size 100MB
uses: nick-fields/retry@v2
with:
timeout_minutes: 1440
max_attempts: 5
retry_wait_seconds: 60
command: cd src && ./stellar -c ../continuous-benchmarking/experiments/cold-function-invocations/image-size/cold-image-size-100-azure.json -db
- uses: actions/upload-artifact@v3
with:
name: cold-image-size-100-azure
path: ${{ env.working-directory }}/latency-samples
- name: Add zero'ed data (Failed experiment)
if: ${{ failure() }}
env:
DATA_INSERT_URL: ${{ secrets.DATA_INSERT_URL}}
run: |
curl -XPOST -H "Content-type: application/json" -d '{"experiment_type":"cold-image-size-100-azure","date":"'$(date +%F)'","min":"0","max":"0","median":"0","tail_latency":"0","first_quartile":"0","third_quartile":"0","standard_deviation":"0","payload_size":"0","burst_size":"0","IATType":"0","count":"0","provider":"azure"}' $DATA_INSERT_URL
- name: Send Slack message using Incoming Webhooks
if: ${{ failure() }}
uses: slackapi/[email protected]
with:
payload: |
{
"blocks": [
{
"type": "header",
"text": {
"type": "plain_text",
"text": "Scheduled experiment (Job status)"
}
},
{
"type": "section",
"fields": [
{
"type": "mrkdwn",
"text": "*Workflow Name*: ${{ github.workflow }}"
},
{
"type": "mrkdwn",
"text": "*Job Name*: ${{ github.job }}"
},
{
"type": "mrkdwn",
"text": "*Job Status*: ${{ job.status }} ❌"
}
]
},
{
"type": "section",
"fields": [
{
"type": "mrkdwn",
"text": "<!channel>"
}
]
}
]
}
env:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
cold-image-size-50-gcr:
name: GCR 50MB image size experiment
needs: build_client
runs-on: [ self-hosted, gcr ]
timeout-minutes: 1200
env:
working-directory: src
DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }}
DOCKER_HUB_ACCESS_TOKEN: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
steps:
- name: "Cleanup build folder"
run: |
sudo rm -rf ./* || true
sudo rm -rf ./.??* || true
- name: Check out code into the Go module directory
uses: actions/checkout@v4
with:
ref: continuous-benchmarking
- id: auth
name: Configure GCR credentials
uses: google-github-actions/auth@v1
with:
credentials_json: ${{ secrets.GCR_CREDENTIALS }}
- name: Set up gcloud
uses: google-github-actions/setup-gcloud@v1
with:
version: ">= 363.0.0"
- name: Download client artifact
uses: actions/download-artifact@v3
with:
name: STeLLAR-build
- name: Untar client build
working-directory: ${{ env.working-directory }}
run: tar --strip-components=1 -xvf ../build.tar -C .
- name: GCR Image Size 50MB
uses: nick-fields/retry@v2
with:
timeout_minutes: 1440
max_attempts: 5
retry_wait_seconds: 60
command: cd src && ./stellar -c ../continuous-benchmarking/experiments/cold-function-invocations/image-size/cold-image-size-50-gcr.json -db
- uses: actions/upload-artifact@v3
with:
name: cold-image-size-50-gcr
path: ${{ env.working-directory }}/latency-samples
- name: Add zero'ed data (Failed experiment)
if: ${{ failure() }}
env:
DATA_INSERT_URL: ${{ secrets.DATA_INSERT_URL}}
run: |
curl -XPOST -H "Content-type: application/json" -d '{"experiment_type":"cold-image-size-50-gcr","date":"'$(date +%F)'","min":"0","max":"0","median":"0","tail_latency":"0","first_quartile":"0","third_quartile":"0","standard_deviation":"0","payload_size":"0","burst_size":"0","IATType":"0","count":"0","provider":"gcr"}' $DATA_INSERT_URL
- name: Send Slack message using Incoming Webhooks
if: ${{ failure() }}
uses: slackapi/[email protected]
with:
payload: |
{
"blocks": [
{
"type": "header",
"text": {
"type": "plain_text",
"text": "Scheduled experiment (Job status)"
}
},
{
"type": "section",
"fields": [
{
"type": "mrkdwn",
"text": "*Workflow Name*: ${{ github.workflow }}"
},
{
"type": "mrkdwn",
"text": "*Job Name*: ${{ github.job }}"
},
{
"type": "mrkdwn",
"text": "*Job Status*: ${{ job.status }} ❌"
}
]
},
{
"type": "section",
"fields": [
{
"type": "mrkdwn",
"text": "<!channel>"
}
]
}
]
}
env:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
cold-image-size-100-gcr:
name: GCR 100MB image size experiment
needs: build_client
runs-on: [ self-hosted, gcr ]
timeout-minutes: 1200
env:
working-directory: src
DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }}
DOCKER_HUB_ACCESS_TOKEN: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
steps:
- name: "Cleanup build folder"
run: |
sudo rm -rf ./* || true
sudo rm -rf ./.??* || true
- name: Check out code into the Go module directory
uses: actions/checkout@v4
with:
ref: continuous-benchmarking
- id: auth
name: Configure GCR credentials
uses: google-github-actions/auth@v1
with:
credentials_json: ${{ secrets.GCR_CREDENTIALS }}
- name: Set up gcloud
uses: google-github-actions/setup-gcloud@v1
with:
version: ">= 363.0.0"
- name: Download client artifact
uses: actions/download-artifact@v3
with:
name: STeLLAR-build
- name: Untar client build
working-directory: ${{ env.working-directory }}
run: tar --strip-components=1 -xvf ../build.tar -C .
- name: GCR Image Size 100MB
uses: nick-fields/retry@v2
with:
timeout_minutes: 1440
max_attempts: 5
retry_wait_seconds: 60
command: cd src && ./stellar -c ../continuous-benchmarking/experiments/cold-function-invocations/image-size/cold-image-size-100-gcr.json -db
- uses: actions/upload-artifact@v3
with:
name: cold-image-size-100-gcr
path: ${{ env.working-directory }}/latency-samples
- name: Add zero'ed data (Failed experiment)
if: ${{ failure() }}
env:
DATA_INSERT_URL: ${{ secrets.DATA_INSERT_URL}}
run: |
curl -XPOST -H "Content-type: application/json" -d '{"experiment_type":"cold-image-size-100-gcr","date":"'$(date +%F)'","min":"0","max":"0","median":"0","tail_latency":"0","first_quartile":"0","third_quartile":"0","standard_deviation":"0","payload_size":"0","burst_size":"0","IATType":"0","count":"0","provider":"gcr"}' $DATA_INSERT_URL
- name: Send Slack message using Incoming Webhooks
if: ${{ failure() }}
uses: slackapi/[email protected]
with:
payload: |
{
"blocks": [
{
"type": "header",
"text": {
"type": "plain_text",
"text": "Scheduled experiment (Job status)"
}
},
{
"type": "section",
"fields": [
{
"type": "mrkdwn",
"text": "*Workflow Name*: ${{ github.workflow }}"
},
{
"type": "mrkdwn",
"text": "*Job Name*: ${{ github.job }}"
},
{
"type": "mrkdwn",
"text": "*Job Status*: ${{ job.status }} ❌"
}
]
},
{
"type": "section",
"fields": [
{
"type": "mrkdwn",
"text": "<!channel>"
}
]
}
]
}
env:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK