Add sleeping and spinning benchmark (#846)

This PR add 2 new benchmarks sleeping (to simulate I/O bounded task) and spinning (to simulate CPU bounded task) such that i can investigate how dynamically configure cpu frequency affects latency of such tasks --------- Signed-off-by: HermioneKT <[email protected]>
vhive-serverless · Apr 6, 2024 · 82eb01d · 82eb01d
1 parent a33cb18
commit 82eb01d
Show file tree

Hide file tree

Showing 21 changed files with 6,094 additions and 0 deletions.
diff --git a/.github/workflows/e2e-spin-sleep.yml b/.github/workflows/e2e-spin-sleep.yml
@@ -0,0 +1,188 @@
+name: Spin-Sleep End-to-End Tests
+
+on:
+  workflow_dispatch:
+    schedule:
+      - cron: "0 9 * * 1"
+  push:
+    branches: [main]
+    paths:
+      - "benchmarks/sleeping/**"
+      - "benchmarks/spinning/**"
+      - "utils/**"
+      - "tools/**"
+      - "runner/**"
+
+  pull_request:
+    branches: [main]
+    paths:
+      - "benchmarks/sleeping/**"
+      - "benchmarks/spinning/**"
+      - "utils/**"
+      - "tools/**"
+      - "runner/**"
+
+env:
+  GOOS: linux
+  GO111MODULE: on
+  PORT: 50051
+  PLATFORMS: linux/amd64,linux/arm64
+
+jobs:
+  build-and-push:
+    name: Build and push all images
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - service: sleeping-go
+            DOCKER_DIR: benchmarks/sleeping/docker/Dockerfile
+          - service: spinning-go
+            DOCKER_DIR: benchmarks/spinning/docker/Dockerfile
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v4
+        with:
+          lfs: "true"
+
+      - uses: actions/setup-go@v5
+        with:
+          go-version: '1.21'
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build and push
+        env:
+            GOPRIVATE_KEY: ${{ secrets.XDT_REPO_ACCESS_KEY }}
+        uses: docker/build-push-action@v5
+        with:
+          push: true
+          file: ${{ matrix.DOCKER_DIR }}
+          platforms: ${{ env.PLATFORMS }}
+          tags: vhiveease/${{ matrix.service }}:latest
+          build-args: SERVICE=${{ matrix.service }}
+          context: .
+
+
+
+  test-compose:
+    name: Test Docker Compose
+    needs: build-and-push
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - service: sleeping-go
+            YAML_DIR: benchmarks/sleeping/yamls/docker-compose/
+          - service: spinning-go
+            YAML_DIR: benchmarks/spinning/yamls/docker-compose/
+
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v4
+        with:
+          lfs: "true"
+
+      - name: start docker-compose benchmark
+        run: |
+          docker-compose -f ${{ matrix.YAML_DIR }}/dc-${{ matrix.service }}.yaml pull
+          docker-compose -f ${{ matrix.YAML_DIR }}/dc-${{ matrix.service }}.yaml up &> log_file &
+          sleep 60s
+          cat log_file
+      - name: invoke the function directly
+        run: |
+          ./tools/bin/grpcurl -plaintext localhost:50000 helloworld.Greeter.SayHello
+      - name: invoke function via relay
+        working-directory: tools/test-client
+        run: |
+          go build ./test-client.go
+          ./test-client --addr localhost:50000 --name "Example text for CI"
+      - name: show docker-compose log
+        run: cat log_file
+
+  test-knative:
+    name: Test Knative Deployment
+    needs: build-and-push
+    env:
+      KIND_VERSION: v0.14.0
+      K8S_VERSION: v1.23
+
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - service: sleeping-go
+            file: kn-sleeping-go.yaml
+            YAML_DIR: benchmarks/sleeping/yamls/knative/
+          - service: spinning-go
+            file: kn-spinning-go.yaml
+            YAML_DIR: benchmarks/spinning/yamls/knative/
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          lfs: "true"
+      - name: Checkout LFS objects
+        run: git lfs checkout
+
+      - uses: actions/setup-go@v5
+        with:
+          go-version: '1.21'
+
+      ## Setup a Knative cluster to test the service
+      - name: Create k8s Kind Cluster
+        run: bash ./runner/scripts/01-kind.sh
+
+      - name: Install Serving
+        run: bash ./runner/scripts/02-serving.sh
+
+      - name: Install Kourier
+        run: bash ./runner/scripts/02-kourier.sh
+
+      - name: Setup domain
+        run: |
+          INGRESS_HOST="127.0.0.1"
+          KNATIVE_DOMAIN=$INGRESS_HOST.sslip.io
+          kubectl patch configmap -n knative-serving config-domain -p "{\"data\": {\"$KNATIVE_DOMAIN\": \"\"}}"
+      - name: Deploy knative
+        run: |
+          kubectl apply -f ${{ matrix.YAML_DIR }}/${{ matrix.file }}
+      - name: Check if service is ready
+        run: |
+          kubectl wait --for=condition=Ready -f ${{ matrix.YAML_DIR }}/${{ matrix.file }} --timeout 900s
+          kubectl get service
+          kubectl get -f ${{ matrix.YAML_DIR }}/${{ matrix.file }}
+      - name: Test invoking once
+        working-directory: tools/test-client
+        run: |
+          set -x
+          go build ./test-client.go
+          NODEPORT=80
+          url=$(kubectl get kservice ${{ matrix.service }} | awk '$2 ~ /http/ {sub(/http\:\/\//,""); print $2}')
+          ./test-client --addr $url:$NODEPORT --name "Example text for CI"
+      - name: Print logs
+        if: ${{ always() }}
+        run: |
+          set -x
+          pod_list=$(kubectl get pods -n default -o jsonpath="{.items[*].name}")
+          for pod in $pod_list
+          do
+            kubectl logs $pod
+          done
+      - name: Down
+        if: ${{ always() }}
+        run: |
+          kubectl delete -f ${{ matrix.YAML_DIR }}/${{ matrix.file }} --namespace default --wait
diff --git a/benchmarks/sleeping/Makefile b/benchmarks/sleeping/Makefile
@@ -0,0 +1,53 @@
+# MIT License
+
+# Copyright (c) 2022 EASE lab
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+
+clean: clean-proto
+
+
+ROOT = ../../
+
+FUNCTIONS = sleeping-go
+ALL_IMAGES = $(addsuffix -image, $(FUNCTIONS))
+
+all: $(ALL_IMAGES)
+
+sleeping-go-image: docker/Dockerfile go/server.go
+	DOCKER_BUILDKIT=1 docker buildx build \
+	--tag vhiveease/sleeping-go:latest \
+	--target sleepingGo \
+	-f docker/Dockerfile \
+	$(ROOT) --load
+
+## Push images
+
+push-%: %-image
+	docker push docker.io/vhiveease/$(subst push-,,$@):latest
+
+push: $(addprefix push-, $(FUNCTIONS))
+
+## Pull images from docker hub
+
+pull-%:
+	docker pull docker.io/vhiveease/$(subst pull-,,$@):latest
+
+pull: $(addprefix pull-, $(FUNCTIONS))
diff --git a/benchmarks/sleeping/README.md b/benchmarks/sleeping/README.md
@@ -0,0 +1,36 @@
+# Sleeping benchmark
+
+The sleeping benchmark simulates a simple I/O bounded task by sleeping for 50 ms.
+
+This benchmark can be used to verify that increasing in CPU frequency would not significantly affect the latency of I/O bounded workload.
+
+The functionality is implemented in golang.
+
+## Running this benchmark (using knative)
+
+1. Build or pull the function images using `make all` or `make pull`.
+2. Start the function with knative
+   ```bash
+   kn service apply -f ./yamls/knative/kn-sleeping-go.yaml
+   ```
+3. **Note the URL provided in the output. The part without the `http://` we'll call `$URL`. Replace any instance of `$URL` in the code below with it.**
+### Invoke once
+4. In a new terminal, invoke the interface function with test-client.
+   ```bash
+   ./test-client --addr sleeping-go.default.192.168.1.240.sslip.io:80 --name "Example text for Sleeping"
+   ```
+### Invoke multiple times
+4. Run the invoker
+   ```bash
+   # build the invoker binary
+   cd ../../tools/invoker
+   make invoker
+
+   # Specify the hostname through "endpoints.json"
+   echo '[ { "hostname": "$URL" } ]' > endpoints.json
+
+   # Start the invoker with a chosen RPS rate and time
+   ./invoker -port 80 -dbg -time 10 -rps 1
+   ```
+## Tracing
+This Benchmark supports distributed tracing for all runtimes. For the general use see vSwarm docs for tracing [locally](../../docs/running_locally.md#tracing) and with [knative](../../docs/running_benchmarks.md#tracing).
diff --git a/benchmarks/sleeping/docker/Dockerfile b/benchmarks/sleeping/docker/Dockerfile
@@ -0,0 +1,45 @@
+# MIT License
+
+# Copyright (c) 2022 EASE lab
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+#---------- GoLang -----------#
+# First stage (Builder):
+FROM vhiveease/golang-builder:latest AS sleepingGoBuilder
+WORKDIR /app/app/
+RUN apt-get install git ca-certificates
+
+COPY ./utils/tracing/go ../../utils/tracing/go
+COPY ./benchmarks/sleeping/go/go.mod ./
+COPY ./benchmarks/sleeping/go/go.sum ./
+COPY ./benchmarks/sleeping/go/server.go ./
+
+RUN go mod tidy
+RUN CGO_ENABLED=0 GOOS=linux go build -v -o ./server server.go
+
+# Second stage (Runner):
+FROM scratch as sleepingGo
+WORKDIR /app/
+COPY --from=sleepingGoBuilder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
+COPY --from=sleepingGoBuilder /app/app/server .
+
+ENTRYPOINT [ "/app/server" ]
+
+EXPOSE 50051
diff --git a/benchmarks/sleeping/go/go.mod b/benchmarks/sleeping/go/go.mod
@@ -0,0 +1,52 @@
+// MIT License
+
+// Copyright (c) 2022 EASE lab
+
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+module aes
+
+go 1.21
+
+replace github.com/vhive-serverless/vSwarm/utils/tracing/go => ../../../utils/tracing/go
+
+require (
+	github.com/sirupsen/logrus v1.9.3
+	github.com/vhive-serverless/vSwarm-proto v0.4.2
+	github.com/vhive-serverless/vSwarm/utils/tracing/go v0.0.0-20231030155054-d3b2e7d0ff16
+	google.golang.org/grpc v1.61.0
+)
+
+require (
+	github.com/go-logr/logr v1.4.1 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/openzipkin/zipkin-go v0.4.2 // indirect
+	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 // indirect
+	go.opentelemetry.io/otel v1.22.0 // indirect
+	go.opentelemetry.io/otel/exporters/zipkin v1.21.0 // indirect
+	go.opentelemetry.io/otel/metric v1.22.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.21.0 // indirect
+	go.opentelemetry.io/otel/trace v1.22.0 // indirect
+	golang.org/x/net v0.20.0 // indirect
+	golang.org/x/sys v0.16.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 // indirect
+	google.golang.org/protobuf v1.32.0 // indirect
+)