Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Added load generator and profiler #1028

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open

Added load generator and profiler #1028

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
45d97c1
Added all files for profiling and loadgen.
Aug 2, 2024
3dc0583
Added compression to profiling.
Aug 12, 2024
a5af69f
Bug fixes for loadgen.
Sep 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
15 changes: 15 additions & 0 deletions .github/configs/wordlist.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ authorizer
autodetect
autoscaling
autostart
AverageAllocatedMb
avi
AVI
aws
Expand Down Expand Up @@ -158,6 +159,8 @@ dockerizing
doi
Dragojevic
Drumond
dur
durations
DVFS
eba
ECR
Expand Down Expand Up @@ -234,6 +237,9 @@ gVisor
Gptj
gpt
Harald
HashApp
HashFunction
HashOwner
HDDs
HelloRequest
helloworld
Expand Down Expand Up @@ -262,6 +268,7 @@ init
InitBasicTracer
InitCustomTracer
Initializr
injective
inproceedings
insteadOf
integ
Expand All @@ -275,6 +282,7 @@ ISCA
ISPASS
Istio
istioctl
iter
iteratively
jekyll
Jesalpura
Expand Down Expand Up @@ -340,13 +348,15 @@ microarchitecture
Marios
matchers
maxVMNum
metset
md
MECS
memcached
metadata
Metatrainer
metatrainer
memcached
MiB
Michal
Micheli
microarchitecting
Expand All @@ -359,6 +369,7 @@ microservices
Microservice
Microservices
microVMs
millicpu
minio
MinIO
Mirzadeh
Expand Down Expand Up @@ -424,13 +435,15 @@ Petrov
Ph
Picorel
Pilani
Pipelined
plaintext
Plamen
plamenpp
pmu
pmutool
pn
Pnevmatikatos
png
postfix
PPoPP
PPOPP
Expand Down Expand Up @@ -462,6 +475,7 @@ PTEMagnet
pv
PV
pvc
py
qO
QoS
QOS
Expand Down Expand Up @@ -499,6 +513,7 @@ runtimes
Rustem
s
SABRes
SampleCount
SayHello
Sazeides
Scalability
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/e2e-aes.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -146,8 +146,8 @@ jobs:

- name: start docker-compose benchmark
run: |
docker-compose -f ${{ env.YAML_DIR }}/dc-${{ matrix.service }}.yaml pull
docker-compose -f ${{ env.YAML_DIR }}/dc-${{ matrix.service }}.yaml up &> log_file &
docker compose -f ${{ env.YAML_DIR }}/dc-${{ matrix.service }}.yaml pull
docker compose -f ${{ env.YAML_DIR }}/dc-${{ matrix.service }}.yaml up &> log_file &
sleep 60s
cat log_file

Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/e2e-auth.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -146,8 +146,8 @@ jobs:

- name: start docker-compose benchmark
run: |
docker-compose -f ${{ env.YAML_DIR }}/dc-${{ matrix.service }}.yaml pull
docker-compose -f ${{ env.YAML_DIR }}/dc-${{ matrix.service }}.yaml up &> log_file &
docker compose -f ${{ env.YAML_DIR }}/dc-${{ matrix.service }}.yaml pull
docker compose -f ${{ env.YAML_DIR }}/dc-${{ matrix.service }}.yaml up &> log_file &
sleep 60s
cat log_file

Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/e2e-fibonacci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,8 +125,8 @@ jobs:

- name: start docker-compose benchmark
run: |
docker-compose -f ${{ env.YAML_DIR }}/dc-${{ matrix.service }}.yaml pull
docker-compose -f ${{ env.YAML_DIR }}/dc-${{ matrix.service }}.yaml up &> log_file &
docker compose -f ${{ env.YAML_DIR }}/dc-${{ matrix.service }}.yaml pull
docker compose -f ${{ env.YAML_DIR }}/dc-${{ matrix.service }}.yaml up &> log_file &
sleep 60s
cat log_file

Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/e2e-gptj.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,8 +97,8 @@ jobs:

- name: start docker-compose benchmark
run: |
docker-compose -f ${{ env.YAML_DIR }}/dc-${{ matrix.service }}.yaml pull
docker-compose -f ${{ env.YAML_DIR }}/dc-${{ matrix.service }}.yaml up &> log_file &
docker compose -f ${{ env.YAML_DIR }}/dc-${{ matrix.service }}.yaml pull
docker compose -f ${{ env.YAML_DIR }}/dc-${{ matrix.service }}.yaml up &> log_file &
sleep 60s
cat log_file

Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/e2e-hotel.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,8 +105,8 @@ jobs:

- name: start docker-compose benchmark
run: |
docker-compose -f ${{ env.YAML_DIR }}/dc-${{ matrix.service }}.yaml pull
docker-compose -f ${{ env.YAML_DIR }}/dc-${{ matrix.service }}.yaml up &> log_file &
docker compose -f ${{ env.YAML_DIR }}/dc-${{ matrix.service }}.yaml pull
docker compose -f ${{ env.YAML_DIR }}/dc-${{ matrix.service }}.yaml up &> log_file &
sleep 60s
cat log_file

Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/e2e-image-rotate.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,8 +99,8 @@ jobs:

- name: start docker-compose benchmark
run: |
docker-compose -f ${{ env.YAML_DIR }}/dc-${{ matrix.service }}.yaml pull
docker-compose -f ${{ env.YAML_DIR }}/dc-${{ matrix.service }}.yaml up &> log_file &
docker compose -f ${{ env.YAML_DIR }}/dc-${{ matrix.service }}.yaml pull
docker compose -f ${{ env.YAML_DIR }}/dc-${{ matrix.service }}.yaml up &> log_file &
sleep 60s
cat log_file

Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/e2e-map-reduce.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ jobs:
- name: start docker-compose benchmark
working-directory: benchmarks/map-reduce
run: |
docker-compose -f docker-compose-s3.yml up &> log_file &
docker compose -f docker-compose-s3.yml up &> log_file &
sleep 120s
cat log_file

Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/e2e-online-shop.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,8 +113,8 @@ jobs:

- name: start docker-compose benchmark
run: |
docker-compose -f ${{ env.YAML_DIR }}/dc-${{ matrix.service }}.yaml pull
docker-compose -f ${{ env.YAML_DIR }}/dc-${{ matrix.service }}.yaml up &> log_file &
docker compose -f ${{ env.YAML_DIR }}/dc-${{ matrix.service }}.yaml pull
docker compose -f ${{ env.YAML_DIR }}/dc-${{ matrix.service }}.yaml up &> log_file &
sleep 60s
cat log_file

Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/e2e-rnn-serving.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,8 +94,8 @@ jobs:

- name: start docker-compose benchmark
run: |
docker-compose -f ${{ env.YAML_DIR }}/dc-${{ matrix.service }}.yaml pull
docker-compose -f ${{ env.YAML_DIR }}/dc-${{ matrix.service }}.yaml up &> log_file &
docker compose -f ${{ env.YAML_DIR }}/dc-${{ matrix.service }}.yaml pull
docker compose -f ${{ env.YAML_DIR }}/dc-${{ matrix.service }}.yaml up &> log_file &
sleep 60s
cat log_file

Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/e2e-spin-sleep.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,8 +98,8 @@ jobs:

- name: start docker-compose benchmark
run: |
docker-compose -f ${{ matrix.YAML_DIR }}/dc-${{ matrix.service }}.yaml pull
docker-compose -f ${{ matrix.YAML_DIR }}/dc-${{ matrix.service }}.yaml up &> log_file &
docker compose -f ${{ matrix.YAML_DIR }}/dc-${{ matrix.service }}.yaml pull
docker compose -f ${{ matrix.YAML_DIR }}/dc-${{ matrix.service }}.yaml up &> log_file &
sleep 60s
cat log_file
- name: invoke the function directly
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/e2e-stacking-training.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ jobs:
env:
BUCKET_NAME: vhive-stacking
run: |
docker-compose -f docker-compose-s3.yml up &> log_file &
docker compose -f docker-compose-s3.yml up &> log_file &
sleep 60s
- name: invoke the chain
run: |
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/e2e-tuning-halving.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ jobs:
env:
BUCKET_NAME: vhive-tuning
run: |
docker-compose -f docker-compose-s3.yml up &> log_file &
docker compose -f docker-compose-s3.yml up &> log_file &
sleep 60s

- name: invoke the chain
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/e2e-video-analytics-standalone.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,8 +96,8 @@ jobs:

- name: start docker-compose benchmark
run: |
docker-compose -f ${{ env.YAML_DIR }}/dc-${{ matrix.service }}.yaml pull
docker-compose -f ${{ env.YAML_DIR }}/dc-${{ matrix.service }}.yaml up &> log_file &
docker compose -f ${{ env.YAML_DIR }}/dc-${{ matrix.service }}.yaml pull
docker compose -f ${{ env.YAML_DIR }}/dc-${{ matrix.service }}.yaml up &> log_file &
sleep 60s
cat log_file

Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/e2e-video-analytics.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ jobs:
BUCKET_NAME: "vhive-video-bench"
working-directory: benchmarks/video-analytics/yamls/docker-compose
run: |
docker-compose -f docker-compose-s3.yml up &> log_file &
docker compose -f docker-compose-s3.yml up &> log_file &
sleep 60s

- name: invoke the chain
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/e2e-video-processing.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,8 +96,8 @@ jobs:

- name: start docker-compose benchmark
run: |
docker-compose -f ${{ env.YAML_DIR }}/dc-${{ matrix.service }}.yaml pull
docker-compose -f ${{ env.YAML_DIR }}/dc-${{ matrix.service }}.yaml up &> log_file &
docker compose -f ${{ env.YAML_DIR }}/dc-${{ matrix.service }}.yaml pull
docker compose -f ${{ env.YAML_DIR }}/dc-${{ matrix.service }}.yaml up &> log_file &
sleep 60s
cat log_file

Expand Down
4 changes: 2 additions & 2 deletions benchmarks/rnn-serving/python/requirements.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
-f https://download.pytorch.org/whl/torch_stable.html
numpy
torch
grpcio==1.53.2
torch==2.0.0+cpu
grpcio-tools==1.45.0
opentelemetry-api==1.3.0
opentelemetry-exporter-zipkin==1.3.0
Expand Down
30 changes: 28 additions & 2 deletions tools/invoker/client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,30 @@ func runExperiment(endpoints []*endpoint.Endpoint, runDuration int, targetRPS fl

Start(TimeseriesDBAddr, endpoints, workflowIDs)

// Calculation to find the right RPS
for _, ep := range endpoints {
if ep.Eventing {
invokeEventingFunction(ep)
} else {
invokeServingFunction(ep)
}
}
latSlice.Lock()
var maxLatency int64
maxLatency = 0
for _, value := range latSlice.slice {
if value > maxLatency {
maxLatency = value
}
}
latSlice.slice = []int64{}
latSlice.Unlock()
profSlice.Lock()
profSlice.slice = []int64{}
profSlice.Unlock()

// ACTUAL EXPERIMENT

timeout := time.After(time.Duration(runDuration) * time.Second)
d := time.Duration(1000000/targetRPS) * time.Microsecond
if d <= 0 {
Expand Down Expand Up @@ -196,11 +220,14 @@ func SayHello(address, workflowID string) {
log.Warnf("Failed to invoke %v, err=%v", address, err)
} else {
if *funcDurEnableFlag {
log.Debugf("Inside if\n")
words := strings.Fields(response.Message)
lastWord := words[len(words)-1]
duration, err := strconv.ParseInt(lastWord, 10, 64)
if err != nil {
log.Warnf("Failed to parse the duration from the response: %v", err)
}
if err == nil {
log.Debugf("Invoked %v. Response: %v\n", address, response.Message)
profSlice.Lock()
profSlice.slice = append(profSlice.slice, duration)
profSlice.Unlock()
Expand Down Expand Up @@ -290,7 +317,6 @@ func writeFunctionDurations(funcDurationOutputFile string) {
}

datawriter := bufio.NewWriter(file)

for _, dur := range profSlice.slice {
_, err := datawriter.WriteString(strconv.FormatInt(dur, 10) + "\n")
if err != nil {
Expand Down
Loading
Loading