considering settings & benchmarks docker-compose

.dockerignore

@@ -1,5 +1,6 @@
 .git
 LICENSE
+docker-compose.yaml
 *.md
 assets
 Dockerfile

README.md

@@ -1,9 +1,10 @@
-# framer
-framer is the most performant grpc load generator
+# ozon-framer
+ozon-framer is the most performant grpc load generator
 
 ## Performance
 ![benchmark chart](./assets/benchmark_chart.png)
 
+Benchmarks are done with `11th Gen Intel(R) Core(TM) i7-1165G7 @ 2.80GHz`.
 Load generators was limited in 2 CPU.
 Load generators configurations are available in [benchmarks directory](./benchmarks)
 
@@ -17,7 +18,7 @@ Load generators configurations are available in [benchmarks directory](./benchma
 This is alpha version. Public api and request file format may be changed.
 
 ## Install
-Download binary from [github release page](./releases/latest) and place it in your PATH.
+Download binary from [github release page](https://github.com/ozontech/framer/releases/latest) and place it in your PATH.
 
 ### Compile
 **Build using go**
@@ -77,6 +78,13 @@ framer load --addr=localhost:9090 --requests-file=test_files/requests --clients
 It makes 10 rps from 10 clients in 10 second.
 
 ## Converter
+`framer convert` command may be used to convert requests file between different formats.
+Now is supported next formats:
+* ozon.binary - [see format description above](#ozon.binary-file-format);
+* pandora.json - grpc json format of pandora load generator. [See documentation](https://yandex.cloud/ru/docs/load-testing/concepts/payloads/grpc-json);
+* ozon.json - same as pandora.json, but has ability to store repeatable meta value.
+
+### Supported formats
 ### Ozon.binary file format
 Rules are using [ABNF syntax](https://tools.ietf.org/html/rfc5234).
 
@@ -94,6 +102,9 @@ Body = 1*({any byte})
 
 [Example requests file](https://github.com/ozontech/framer/-/blob/master/test_files/requests)
 
+#### Programatic ozon.binary generation example
+[Full example](./examples/requestsgen)
+
 ### Usage
 ```
 Usage: framer convert --help
@@ -123,12 +134,10 @@ framer convert --from=ozon.json --to=ozon.binary --reflection-proto=formats/grpc
 ```
 It converts requests file from ozon.json format to ozon.binary format using protofile.
 
-### Programatic requests generation example
-[Full example](./examples/requestsgen)
-
 ## TODO
-- [ ] Installation from homebrew for macOS;
-- [ ] Publish to dockerhub;
+- [ ] Installation
+    - [ ] Homebrew suport for macOS;
+    - [ ] Publish to dockerhub;
 - [ ] Configuration file support;
 - [ ] Requests scheduling strategys combination;
 - [ ] More reporting variants;

benchmarks/dumb-server/Dockerfile

@@ -0,0 +1,13 @@
+FROM golang:1.22-alpine AS build-stage
+WORKDIR /app
+RUN apk add make
+COPY go.mod go.sum ./
+RUN go mod download
+COPY . .
+WORKDIR /app/benchmarks/dumb-server
+RUN make build
+
+FROM alpine
+WORKDIR /
+COPY --from=build-stage /tmp/bin/dumb-server /dumb-server
+ENTRYPOINT ["/dumb-server"]

benchmarks/dumb-server/docker-compose.yaml

@@ -0,0 +1,9 @@
+version: "3.8"
+services:
+  server:
+    container_name: server-${COMPOSE_PROJECT_NAME}
+    image: server
+    build:
+      context: ../..
+      dockerfile: benchmarks/dumb-server/Dockerfile
+    network_mode: host

benchmarks/dumb-server/main.go

@@ -122,8 +122,6 @@ func newResps() (resp1 []byte, resp1Mod respModifier, resp2 []byte, resp2Mod res
 		resp = append(resp, headerBuf.Bytes()...)
 
 		msgPrefix := make([]byte, 5) // не кодированный ответ 0 длины = любое сообщение со всеми пустыми полями
-		// TODO: remove debug
-		// msgPrefix := []byte{255, 254, 253, 252, 251}
 		dataLen := len(msgPrefix)
 
 		respMod.dataStreamIDindx = len(resp) + 5
@@ -305,8 +303,16 @@ func handleConn(conn net.Conn, i int) (err error) {
 	processor := reciever.NewProcessor([]reciever.FrameTypeProcessor{
 		http2.FrameData:         endStreamProcessor,
 		http2.FrameHeaders:      endStreamProcessor,
-		http2.FrameContinuation: endStreamProcessor,
 		http2.FramePing:         &pingProcessor{cmds},
+		http2.FrameContinuation: endStreamProcessor,
+
+		http2.FrameRSTStream:    noopFrameProcessor{},
+		http2.FrameGoAway:       noopFrameProcessor{},
+		http2.FrameWindowUpdate: noopFrameProcessor{},
+
+		// http.FramePriority not supported
+		http2.FrameSettings: settingsProcessor{},
+		// http.FramePushPromise not supported
 	})
 	g.Go(func() error {
 		defer println("processor done")
@@ -369,7 +375,6 @@ func handleConn(conn net.Conn, i int) (err error) {
 }
 
 func writeLoop(ctx context.Context, conn io.Writer, commands <-chan *command, _ types.FlowControl, _ int) (err error) {
-	f := http2.NewFramer(conn, nil)
 	r1Original, r1mod, r2Original, r2mod := newResps()
 	select {
 	case cmd := <-commands:
@@ -381,7 +386,7 @@ func writeLoop(ctx context.Context, conn io.Writer, commands <-chan *command, _
 				return err
 			}
 		case commandTypePing:
-			err := f.WritePing(true, cmd.pingPayload)
+			err := http2.NewFramer(conn, nil).WritePing(true, cmd.pingPayload)
 			if err != nil {
 				return err
 			}
@@ -393,8 +398,9 @@ func writeLoop(ctx context.Context, conn io.Writer, commands <-chan *command, _
 	const limit = 1024
 
 	type t struct {
-		bufSrc   [limit][]byte
-		requests [limit][]byte
+		bufSrc    [limit][]byte
+		requests  [limit][]byte
+		pingFrame []byte
 	}
 
 	var t1, t2 t
@@ -403,16 +409,23 @@ func writeLoop(ctx context.Context, conn io.Writer, commands <-chan *command, _
 		t2.requests[i] = bytes.Clone(r2Original)
 	}
 
-	writeChan := make(chan net.Buffers, 1)
-	go func() {
-		ticker := time.NewTicker(time.Millisecond)
-		defer close(writeChan)
-		defer ticker.Stop()
+	{
 		pingFrame := make([]byte, 9+8)
 		pingHeader := frameheader.FrameHeader(pingFrame)
 		pingHeader.SetType(http2.FramePing)
+		pingHeader.SetLength(8)
 		pingHeader.SetFlags(http2.FlagPingAck)
 
+		t1.pingFrame = pingFrame
+		t2.pingFrame = bytes.Clone(pingFrame)
+	}
+
+	writeChan := make(chan net.Buffers) // важно чтобы канал был небуферизованный!
+	go func() {
+		ticker := time.NewTicker(time.Millisecond)
+		defer close(writeChan)
+		defer ticker.Stop()
+
 		nextT, t := &t1, &t2
 		for {
 			t, nextT = nextT, t
@@ -430,16 +443,16 @@ func writeLoop(ctx context.Context, conn io.Writer, commands <-chan *command, _
 						scheduledBytesOUT.Add(uint64(len(r2)))
 
 						if len(buf) == limit {
-							writeBuf = buf[1:] // не пишем резервную позицию
+							writeBuf = buf[1:] // НЕ пишем резервную позицию
 						}
 					case commandTypePing:
-						copy(pingFrame[9:], cmd.pingPayload[:])
-						buf[0] = pingFrame
-						writeBuf = buf
+						copy(t.pingFrame[9:], cmd.pingPayload[:])
+						buf[0] = t.pingFrame
+						writeBuf = buf // пишем резервную позицию
 					}
 					commandsPool.Release(cmd)
 				case <-ticker.C:
-					writeBuf = buf[1:] // не пишем резервную позицию
+					writeBuf = buf[1:] // НЕ пишем резервную позицию
 				case <-ctx.Done():
 					return
 				}
@@ -463,7 +476,7 @@ func writeLoop(ctx context.Context, conn io.Writer, commands <-chan *command, _
 		cmdsAfter := len(commands)
 
 		since := time.Since(writeStart)
-		if since > time.Millisecond*10 {
+		if since > time.Millisecond*10 && false {
 			println(since.String(), cmdsBefore, cmdsAfter)
 		}
 		bytesOUT.Add(l * uint64(len(r2Original)))
@@ -503,18 +516,16 @@ func configureConn(conn io.ReadWriter, buf []byte) (settings, error) {
 		return s, fmt.Errorf("first frame from other end is not settings, got %T", frame)
 	}
 
-	if !sf.IsAck() {
-		if val, ok := sf.Value(http2.SettingInitialWindowSize); ok {
-			s.InitialWindowSize = val
-		}
-		if val, ok := sf.Value(http2.SettingMaxConcurrentStreams); ok {
-			s.MaxConcurrentStreams = val
-		}
+	if val, ok := sf.Value(http2.SettingInitialWindowSize); ok {
+		s.InitialWindowSize = val
+	}
+	if val, ok := sf.Value(http2.SettingMaxConcurrentStreams); ok {
+		s.MaxConcurrentStreams = val
+	}
 
-		err = framer.WriteSettingsAck()
-		if err != nil {
-			return s, fmt.Errorf("writing settings ack: %w", err)
-		}
+	err = framer.WriteSettingsAck()
+	if err != nil {
+		return s, fmt.Errorf("writing settings ack: %w", err)
 	}
 
 	// у h2load на валидное значение происходит переполнение буффера с отвалом соедиенинения, поэтому: - 65_535
@@ -527,6 +538,16 @@ func configureConn(conn io.ReadWriter, buf []byte) (settings, error) {
 	return s, nil
 }
 
+type noopFrameProcessor struct{}
+
+func (p noopFrameProcessor) Process(
+	_ frameheader.FrameHeader,
+	_ []byte,
+	incomplete bool,
+) error {
+	return nil
+}
+
 type endStreamProcessor struct {
 	respChan chan<- *command
 }
@@ -557,11 +578,11 @@ type pingProcessor struct {
 }
 
 func (p *pingProcessor) Process(
-	_ frameheader.FrameHeader,
+	header frameheader.FrameHeader,
 	payload []byte,
 	incomplete bool,
 ) error {
-	if incomplete {
+	if incomplete || !header.Flags().Has(http2.FlagPingAck) {
 		return nil
 	}
 
@@ -575,3 +596,20 @@ func (p *pingProcessor) Process(
 
 	return nil
 }
+
+type settingsProcessor struct{}
+
+func (p settingsProcessor) Process(
+	header frameheader.FrameHeader,
+	payload []byte,
+	incomplete bool,
+) error {
+	if incomplete {
+		return nil
+	}
+
+	if !header.Flags().Has(http2.FlagSettingsAck) {
+		return errors.New("update settings in runtime not supported")
+	}
+	return nil
+}

benchmarks/framer/README.md

@@ -0,0 +1,6 @@
+How to run
+- Run in docker (needs docker-compose v2): `docker compose up --abort-on-container-exit`
+- Run local:
+    * Install [framer](../../README.md#install)
+    * Run dumb server: `(cd ../dumb-server && make run)`
+    * Execute `./run.sh`

benchmarks/framer/docker-compose.yaml

@@ -0,0 +1,22 @@
+name: framer-benchmark
+include:
+ - ../dumb-server/docker-compose.yaml
+services:
+  framer:
+    container_name: framer
+    image: framer
+    build:
+      context: ../..
+      dockerfile: Dockerfile
+    network_mode: host
+    volumes:
+       - ./requests.bin:/tmp/requests.bin
+    command: >
+      load --addr=localhost:9090 --inmem-requests
+      --requests-file=/tmp/requests.bin --clients 10
+      unlimited --duration 1m
+    deploy:
+      resources:
+        limits:
+          cpus: '2'
+          memory: 2G

benchmarks/gatling/.dockerignore

@@ -0,0 +1 @@
+build

benchmarks/gatling/Dockerfile

@@ -0,0 +1,5 @@
+FROM openjdk:17-oracle
+WORKDIR /app
+COPY . /app
+RUN ./gradlew
+ENTRYPOINT ["./gradlew", "gatlingRun-bench.BenchKt"]

benchmarks/gatling/README.md

@@ -1,4 +1,6 @@
 How to run
-- Install java/gradle
-- Run dumb server: `(cd ../dumb-server && make run)`
-- Execute `./run.sh`
+- Run in docker (needs docker-compose v2): `docker compose up --abort-on-container-exit`
+- Run local:
+    * Install java and gradle
+    * Run dumb server: `(cd ../dumb-server && make run)`
+    * Execute `./run.sh`

benchmarks/gatling/docker-compose.yaml

@@ -0,0 +1,19 @@
+name: gatling-benchmark
+include:
+ - ../dumb-server/docker-compose.yaml
+services:
+  gatling:
+    container_name: gatling
+    image: gatling
+    build:
+      context: ./
+      dockerfile: Dockerfile
+    network_mode: host
+    working_dir: /app
+    volumes:
+       - ./:/app
+    deploy:
+      resources:
+        limits:
+          cpus: '2'
+          memory: 2G

benchmarks/ghz/Dockerfile

@@ -0,0 +1,5 @@
+FROM ubuntu:24.04
+WORKDIR /app
+ADD https://github.com/bojand/ghz/releases/download/v0.120.0/ghz-linux-x86_64.tar.gz ./
+RUN tar xvf ghz-linux-x86_64.tar.gz
+ENTRYPOINT ["/app/ghz"]

benchmarks/ghz/README.md

@@ -0,0 +1,6 @@
+How to run
+- Run in docker (needs docker-compose v2): `docker compose up --abort-on-container-exit`
+- Run local:
+    - Install [ghz](https://ghz.sh/docs/install)
+    - Run dumb server: `(cd ../dumb-server && make run)`
+    - Execute `./run.sh`