From 4466d9fb6b10ecc90aec6e310ee1d1d723c72463 Mon Sep 17 00:00:00 2001
From: Anna Pendleton <pendleton@google.com>
Date: Wed, 27 Mar 2024 23:36:13 -0700
Subject: [PATCH] Custom exporter (#336)

* add new docker container for locust custom metrics

* add custom metrics collection into UI

* add custom exporter into related deployments.
---
 .../tools/locust-load-inference/build.tf      |   8 +
 .../locust-custom-exporter/Dockerfile         |  20 ++
 .../locust-custom-exporter/Makefile           |  17 +
 .../locust-custom-exporter/Makefile.common    | 302 ++++++++++++++++++
 .../locust-custom-exporter/README.md          |  58 ++++
 .../locust-custom-exporter/go.mod             |   9 +
 .../locust-custom-exporter/go.sum             | 150 +++++++++
 .../locust-custom-exporter/main.go            | 186 +++++++++++
 .../locust-tasks/custom_metric_aggregator.py  |  52 ++-
 .../locust-docker/locust-tasks/tasks.py       |  65 +++-
 .../locust-runner/metrics.yaml                |  20 ++
 .../locust-master-controller.yaml.tpl         |  13 +-
 .../pod-monitoring.yaml.tpl                   |   3 +-
 13 files changed, 884 insertions(+), 19 deletions(-)
 create mode 100644 benchmarks/benchmark/tools/locust-load-inference/locust-custom-exporter/Dockerfile
 create mode 100644 benchmarks/benchmark/tools/locust-load-inference/locust-custom-exporter/Makefile
 create mode 100644 benchmarks/benchmark/tools/locust-load-inference/locust-custom-exporter/Makefile.common
 create mode 100644 benchmarks/benchmark/tools/locust-load-inference/locust-custom-exporter/README.md
 create mode 100644 benchmarks/benchmark/tools/locust-load-inference/locust-custom-exporter/go.mod
 create mode 100644 benchmarks/benchmark/tools/locust-load-inference/locust-custom-exporter/go.sum
 create mode 100644 benchmarks/benchmark/tools/locust-load-inference/locust-custom-exporter/main.go

diff --git a/benchmarks/benchmark/tools/locust-load-inference/build.tf b/benchmarks/benchmark/tools/locust-load-inference/build.tf
index 3cda25ae7..2d60f5735 100644
--- a/benchmarks/benchmark/tools/locust-load-inference/build.tf
+++ b/benchmarks/benchmark/tools/locust-load-inference/build.tf
@@ -14,3 +14,11 @@ resource "null_resource" "build_and_push_runner_image" {
     command     = "gcloud builds submit --tag ${var.artifact_registry}/locust-runner:latest locust-runner"
   }
 }
+
+resource "null_resource" "build_and_push_exporter_image" {
+
+  provisioner "local-exec" {
+    working_dir = path.module
+    command     = "gcloud builds submit --tag ${var.artifact_registry}/locust-custom-exporter:latest locust-custom-exporter"
+  }
+}
\ No newline at end of file
diff --git a/benchmarks/benchmark/tools/locust-load-inference/locust-custom-exporter/Dockerfile b/benchmarks/benchmark/tools/locust-load-inference/locust-custom-exporter/Dockerfile
new file mode 100644
index 000000000..abd0aff9e
--- /dev/null
+++ b/benchmarks/benchmark/tools/locust-load-inference/locust-custom-exporter/Dockerfile
@@ -0,0 +1,20 @@
+FROM golang:1.20
+
+# Set destination for COPY
+WORKDIR /app
+
+# Download Go modules
+COPY go.mod go.sum ./
+RUN go mod download
+
+# Copy the source code. Note the slash at the end, as explained in
+# https://docs.docker.com/reference/dockerfile/#copy
+COPY *.go ./
+
+# Build
+RUN CGO_ENABLED=0 GOOS=linux go build -o /locust_exporter
+
+EXPOSE 8080
+
+# Run
+CMD ["/locust_exporter"]
diff --git a/benchmarks/benchmark/tools/locust-load-inference/locust-custom-exporter/Makefile b/benchmarks/benchmark/tools/locust-load-inference/locust-custom-exporter/Makefile
new file mode 100644
index 000000000..a7b48bc1e
--- /dev/null
+++ b/benchmarks/benchmark/tools/locust-load-inference/locust-custom-exporter/Makefile
@@ -0,0 +1,17 @@
+# Copyright 2018 The Prometheus Authors
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+include Makefile.common
+
+## This is a copy!
+## https://github.com/prometheus/prometheus/blob/main/Makefile.common
diff --git a/benchmarks/benchmark/tools/locust-load-inference/locust-custom-exporter/Makefile.common b/benchmarks/benchmark/tools/locust-load-inference/locust-custom-exporter/Makefile.common
new file mode 100644
index 000000000..2b73d86a6
--- /dev/null
+++ b/benchmarks/benchmark/tools/locust-load-inference/locust-custom-exporter/Makefile.common
@@ -0,0 +1,302 @@
+# Copyright 2018 The Prometheus Authors
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# A common Makefile that includes rules to be reused in different prometheus projects.
+# !!! Open PRs only against the prometheus/prometheus/Makefile.common repository!
+
+# Example usage :
+# Create the main Makefile in the root project directory.
+# include Makefile.common
+# customTarget:
+# 	@echo ">> Running customTarget"
+#
+
+# Ensure GOBIN is not set during build so that promu is installed to the correct path
+unexport GOBIN
+
+GO           ?= go
+GOFMT        ?= $(GO)fmt
+FIRST_GOPATH := $(firstword $(subst :, ,$(shell $(GO) env GOPATH)))
+GOOPTS       ?=
+GOHOSTOS     ?= $(shell $(GO) env GOHOSTOS)
+GOHOSTARCH   ?= $(shell $(GO) env GOHOSTARCH)
+
+GO_VERSION        ?= $(shell $(GO) version)
+GO_VERSION_NUMBER ?= $(word 3, $(GO_VERSION))
+PRE_GO_111        ?= $(shell echo $(GO_VERSION_NUMBER) | grep -E 'go1\.(10|[0-9])\.')
+
+GOVENDOR :=
+GO111MODULE :=
+ifeq (, $(PRE_GO_111))
+	ifneq (,$(wildcard go.mod))
+		# Enforce Go modules support just in case the directory is inside GOPATH (and for Travis CI).
+		GO111MODULE := on
+
+		ifneq (,$(wildcard vendor))
+			# Always use the local vendor/ directory to satisfy the dependencies.
+			GOOPTS := $(GOOPTS) -mod=vendor
+		endif
+	endif
+else
+	ifneq (,$(wildcard go.mod))
+		ifneq (,$(wildcard vendor))
+$(warning This repository requires Go >= 1.11 because of Go modules)
+$(warning Some recipes may not work as expected as the current Go runtime is '$(GO_VERSION_NUMBER)')
+		endif
+	else
+		# This repository isn't using Go modules (yet).
+		GOVENDOR := $(FIRST_GOPATH)/bin/govendor
+	endif
+endif
+PROMU        := $(FIRST_GOPATH)/bin/promu
+pkgs          = ./...
+
+ifeq (arm, $(GOHOSTARCH))
+	GOHOSTARM ?= $(shell GOARM= $(GO) env GOARM)
+	GO_BUILD_PLATFORM ?= $(GOHOSTOS)-$(GOHOSTARCH)v$(GOHOSTARM)
+else
+	GO_BUILD_PLATFORM ?= $(GOHOSTOS)-$(GOHOSTARCH)
+endif
+
+GOTEST := $(GO) test
+GOTEST_DIR :=
+ifneq ($(CIRCLE_JOB),)
+ifneq ($(shell which gotestsum),)
+	GOTEST_DIR := test-results
+	GOTEST := gotestsum --junitfile $(GOTEST_DIR)/unit-tests.xml --
+endif
+endif
+
+PROMU_VERSION ?= 0.7.0
+PROMU_URL     := https://github.com/prometheus/promu/releases/download/v$(PROMU_VERSION)/promu-$(PROMU_VERSION).$(GO_BUILD_PLATFORM).tar.gz
+
+GOLANGCI_LINT :=
+GOLANGCI_LINT_OPTS ?=
+GOLANGCI_LINT_VERSION ?= v1.36.0
+# golangci-lint only supports linux, darwin and windows platforms on i386/amd64.
+# windows isn't included here because of the path separator being different.
+ifeq ($(GOHOSTOS),$(filter $(GOHOSTOS),linux darwin))
+	ifeq ($(GOHOSTARCH),$(filter $(GOHOSTARCH),amd64 i386))
+		GOLANGCI_LINT := $(FIRST_GOPATH)/bin/golangci-lint
+	endif
+endif
+
+PREFIX                  ?= $(shell pwd)
+BIN_DIR                 ?= $(shell pwd)
+DOCKER_IMAGE_TAG        ?= $(subst /,-,$(shell git rev-parse --abbrev-ref HEAD))
+DOCKERFILE_PATH         ?= ./Dockerfile
+DOCKERBUILD_CONTEXT     ?= ./
+DOCKER_REPO             ?= prom
+
+DOCKER_ARCHS            ?= amd64
+
+BUILD_DOCKER_ARCHS = $(addprefix common-docker-,$(DOCKER_ARCHS))
+PUBLISH_DOCKER_ARCHS = $(addprefix common-docker-publish-,$(DOCKER_ARCHS))
+TAG_DOCKER_ARCHS = $(addprefix common-docker-tag-latest-,$(DOCKER_ARCHS))
+
+ifeq ($(GOHOSTARCH),amd64)
+        ifeq ($(GOHOSTOS),$(filter $(GOHOSTOS),linux freebsd darwin windows))
+                # Only supported on amd64
+                test-flags := -race
+        endif
+endif
+
+# This rule is used to forward a target like "build" to "common-build".  This
+# allows a new "build" target to be defined in a Makefile which includes this
+# one and override "common-build" without override warnings.
+%: common-% ;
+
+.PHONY: common-all
+common-all: precheck style check_license lint unused build test
+
+.PHONY: common-style
+common-style:
+	@echo ">> checking code style"
+	@fmtRes=$$($(GOFMT) -d $$(find . -path ./vendor -prune -o -name '*.go' -print)); \
+	if [ -n "$${fmtRes}" ]; then \
+		echo "gofmt checking failed!"; echo "$${fmtRes}"; echo; \
+		echo "Please ensure you are using $$($(GO) version) for formatting code."; \
+		exit 1; \
+	fi
+
+.PHONY: common-check_license
+common-check_license:
+	@echo ">> checking license header"
+	@licRes=$$(for file in $$(find . -type f -iname '*.go' ! -path './vendor/*') ; do \
+               awk 'NR<=3' $$file | grep -Eq "(Copyright|generated|GENERATED)" || echo $$file; \
+       done); \
+       if [ -n "$${licRes}" ]; then \
+               echo "license header checking failed:"; echo "$${licRes}"; \
+               exit 1; \
+       fi
+
+.PHONY: common-deps
+common-deps:
+	@echo ">> getting dependencies"
+ifdef GO111MODULE
+	GO111MODULE=$(GO111MODULE) $(GO) mod download
+else
+	$(GO) get $(GOOPTS) -t ./...
+endif
+
+.PHONY: update-go-deps
+update-go-deps:
+	@echo ">> updating Go dependencies"
+	@for m in $$($(GO) list -mod=readonly -m -f '{{ if and (not .Indirect) (not .Main)}}{{.Path}}{{end}}' all); do \
+		$(GO) get $$m; \
+	done
+	GO111MODULE=$(GO111MODULE) $(GO) mod tidy
+ifneq (,$(wildcard vendor))
+	GO111MODULE=$(GO111MODULE) $(GO) mod vendor
+endif
+
+.PHONY: common-test-short
+common-test-short: $(GOTEST_DIR)
+	@echo ">> running short tests"
+	GO111MODULE=$(GO111MODULE) $(GOTEST) -short $(GOOPTS) $(pkgs)
+
+.PHONY: common-test
+common-test: $(GOTEST_DIR)
+	@echo ">> running all tests"
+	GO111MODULE=$(GO111MODULE) $(GOTEST) $(test-flags) $(GOOPTS) $(pkgs)
+
+$(GOTEST_DIR):
+	@mkdir -p $@
+
+.PHONY: common-format
+common-format:
+	@echo ">> formatting code"
+	GO111MODULE=$(GO111MODULE) $(GO) fmt $(pkgs)
+
+.PHONY: common-vet
+common-vet:
+	@echo ">> vetting code"
+	GO111MODULE=$(GO111MODULE) $(GO) vet $(GOOPTS) $(pkgs)
+
+.PHONY: common-lint
+common-lint: $(GOLANGCI_LINT)
+ifdef GOLANGCI_LINT
+	@echo ">> running golangci-lint"
+ifdef GO111MODULE
+# 'go list' needs to be executed before staticcheck to prepopulate the modules cache.
+# Otherwise staticcheck might fail randomly for some reason not yet explained.
+	GO111MODULE=$(GO111MODULE) $(GO) list -e -compiled -test=true -export=false -deps=true -find=false -tags= -- ./... > /dev/null
+	GO111MODULE=$(GO111MODULE) $(GOLANGCI_LINT) run $(GOLANGCI_LINT_OPTS) $(pkgs)
+else
+	$(GOLANGCI_LINT) run $(pkgs)
+endif
+endif
+
+# For backward-compatibility.
+.PHONY: common-staticcheck
+common-staticcheck: lint
+
+.PHONY: common-unused
+common-unused: $(GOVENDOR)
+ifdef GOVENDOR
+	@echo ">> running check for unused packages"
+	@$(GOVENDOR) list +unused | grep . && exit 1 || echo 'No unused packages'
+else
+ifdef GO111MODULE
+	@echo ">> running check for unused/missing packages in go.mod"
+	GO111MODULE=$(GO111MODULE) $(GO) mod tidy
+ifeq (,$(wildcard vendor))
+	@git diff --exit-code -- go.sum go.mod
+else
+	@echo ">> running check for unused packages in vendor/"
+	GO111MODULE=$(GO111MODULE) $(GO) mod vendor
+	@git diff --exit-code -- go.sum go.mod vendor/
+endif
+endif
+endif
+
+.PHONY: common-build
+common-build: promu
+	@echo ">> building binaries"
+	GO111MODULE=$(GO111MODULE) $(PROMU) build --prefix $(PREFIX) $(PROMU_BINARIES)
+
+.PHONY: common-tarball
+common-tarball: promu
+	@echo ">> building release tarball"
+	$(PROMU) tarball --prefix $(PREFIX) $(BIN_DIR)
+
+.PHONY: common-docker $(BUILD_DOCKER_ARCHS)
+common-docker: $(BUILD_DOCKER_ARCHS)
+$(BUILD_DOCKER_ARCHS): common-docker-%:
+	docker build -t "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(DOCKER_IMAGE_TAG)" \
+		-f $(DOCKERFILE_PATH) \
+		--build-arg ARCH="$*" \
+		--build-arg OS="linux" \
+		$(DOCKERBUILD_CONTEXT)
+
+.PHONY: common-docker-publish $(PUBLISH_DOCKER_ARCHS)
+common-docker-publish: $(PUBLISH_DOCKER_ARCHS)
+$(PUBLISH_DOCKER_ARCHS): common-docker-publish-%:
+	docker push "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(DOCKER_IMAGE_TAG)"
+
+DOCKER_MAJOR_VERSION_TAG = $(firstword $(subst ., ,$(shell cat VERSION)))
+.PHONY: common-docker-tag-latest $(TAG_DOCKER_ARCHS)
+common-docker-tag-latest: $(TAG_DOCKER_ARCHS)
+$(TAG_DOCKER_ARCHS): common-docker-tag-latest-%:
+	docker tag "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(DOCKER_IMAGE_TAG)" "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:latest"
+	docker tag "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(DOCKER_IMAGE_TAG)" "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:v$(DOCKER_MAJOR_VERSION_TAG)"
+
+.PHONY: common-docker-manifest
+common-docker-manifest:
+	DOCKER_CLI_EXPERIMENTAL=enabled docker manifest create -a "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_TAG)" $(foreach ARCH,$(DOCKER_ARCHS),$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$(ARCH):$(DOCKER_IMAGE_TAG))
+	DOCKER_CLI_EXPERIMENTAL=enabled docker manifest push "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_TAG)"
+
+.PHONY: promu
+promu: $(PROMU)
+
+$(PROMU):
+	$(eval PROMU_TMP := $(shell mktemp -d))
+	curl -s -L $(PROMU_URL) | tar -xvzf - -C $(PROMU_TMP)
+	mkdir -p $(FIRST_GOPATH)/bin
+	cp $(PROMU_TMP)/promu-$(PROMU_VERSION).$(GO_BUILD_PLATFORM)/promu $(FIRST_GOPATH)/bin/promu
+	rm -r $(PROMU_TMP)
+
+.PHONY: proto
+proto:
+	@echo ">> generating code from proto files"
+	@./scripts/genproto.sh
+
+ifdef GOLANGCI_LINT
+$(GOLANGCI_LINT):
+	mkdir -p $(FIRST_GOPATH)/bin
+	curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/$(GOLANGCI_LINT_VERSION)/install.sh \
+		| sed -e '/install -d/d' \
+		| sh -s -- -b $(FIRST_GOPATH)/bin $(GOLANGCI_LINT_VERSION)
+endif
+
+ifdef GOVENDOR
+.PHONY: $(GOVENDOR)
+$(GOVENDOR):
+	GOOS= GOARCH= $(GO) get -u github.com/kardianos/govendor
+endif
+
+.PHONY: precheck
+precheck::
+
+define PRECHECK_COMMAND_template =
+precheck:: $(1)_precheck
+
+PRECHECK_COMMAND_$(1) ?= $(1) $$(strip $$(PRECHECK_OPTIONS_$(1)))
+.PHONY: $(1)_precheck
+$(1)_precheck:
+	@if ! $$(PRECHECK_COMMAND_$(1)) 1>/dev/null 2>&1; then \
+		echo "Execution of '$$(PRECHECK_COMMAND_$(1))' command failed. Is $(1) installed?"; \
+		exit 1; \
+	fi
+endef
diff --git a/benchmarks/benchmark/tools/locust-load-inference/locust-custom-exporter/README.md b/benchmarks/benchmark/tools/locust-load-inference/locust-custom-exporter/README.md
new file mode 100644
index 000000000..9147b1e9a
--- /dev/null
+++ b/benchmarks/benchmark/tools/locust-load-inference/locust-custom-exporter/README.md
@@ -0,0 +1,58 @@
+# Locust Exporter
+
+Prometheus exporter for [Locust](https://github.com/locustio/locust). This exporter was inspired by [containersol/locust_exporter](https://github.com/ContainerSolutions/locust_exporter/tree/main) and added on to for collecting Inferencing Benchmarking custom metrics.
+
+## Build Locust Exporter
+
+With Docker:
+
+    ```bash
+    docker build --tag locust-custom-exporter .
+    ```
+
+### Flags
+
+- `--locust.uri`
+  Address of Locust. Default is `http://localhost:8089`.
+
+- `--locust.timeout`
+  Timeout request to Locust. Default is `5s`.
+
+- `--web.listen-address`
+  Address to listen on for web interface and telemetry. Default is `:8080`.
+
+- `--web.telemetry-path`
+  Path under which to expose metrics. Default is `/metrics`.
+
+- `--locust.namespace`
+  Namespace for prometheus metrics. Default `locust`.
+
+- `--log.level`
+  Set logging level: one of `debug`, `info`, `warn`, `error`, `fatal`
+
+- `--log.format`
+  Set the log output target and format. e.g. `logger:syslog?appname=bob&local=7` or `logger:stdout?json=true`
+  Defaults to `logger:stderr`.
+
+### Environment Variables
+
+The following environment variables configure the exporter:
+
+- `LOCUST_EXPORTER_URI`
+  Address of Locust. Default is `http://localhost:8089`.
+
+- `LOCUST_EXPORTER_TIMEOUT`
+  Timeout reqeust to Locust. Default is `5s`.
+
+- `LOCUST_EXPORTER_WEB_LISTEN_ADDRESS`
+  Address to listen on for web interface and telemetry. Default is `:8080`.
+
+- `LOCUST_EXPORTER_WEB_TELEMETRY_PATH`
+  Path under which to expose metrics. Default is `/metrics`.
+
+- `LOCUST_METRIC_NAMESPACE`
+  Namespace for prometheus metrics. Default `locust`.
+
+## License
+
+Apache License. Please see [License File](LICENSE.md) for more information.
diff --git a/benchmarks/benchmark/tools/locust-load-inference/locust-custom-exporter/go.mod b/benchmarks/benchmark/tools/locust-load-inference/locust-custom-exporter/go.mod
new file mode 100644
index 000000000..b4f51ddb7
--- /dev/null
+++ b/benchmarks/benchmark/tools/locust-load-inference/locust-custom-exporter/go.mod
@@ -0,0 +1,9 @@
+module github.com/ContainerSolutions/locust_exporter
+
+go 1.12
+
+require (
+	github.com/prometheus/client_golang v1.11.1
+	github.com/prometheus/common v0.26.0
+	gopkg.in/alecthomas/kingpin.v2 v2.2.6
+)
diff --git a/benchmarks/benchmark/tools/locust-load-inference/locust-custom-exporter/go.sum b/benchmarks/benchmark/tools/locust-load-inference/locust-custom-exporter/go.sum
new file mode 100644
index 000000000..cdfa61ebe
--- /dev/null
+++ b/benchmarks/benchmark/tools/locust-load-inference/locust-custom-exporter/go.sum
@@ -0,0 +1,150 @@
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
+github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E=
+github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
+github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
+github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
+github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
+github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
+github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
+github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
+github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8=
+github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
+github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
+github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
+github.com/prometheus/client_golang v1.11.1 h1:+4eQaD7vAZ6DsfsxB15hbE0odUjGI5ARs9yskGu1v4s=
+github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
+github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=
+github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
+github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ=
+github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
+github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
+github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4=
+github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
+github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I=
+github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40 h1:JWgyZ1qgdTaF3N3oxC+MdTV7qvEEgHo3otj+HB5CM7Q=
+golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.26.0-rc.1 h1:7QnIQpGRHE5RnLKnESfDoxm2dTapTZua5a0kS0A+VXQ=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
+gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
diff --git a/benchmarks/benchmark/tools/locust-load-inference/locust-custom-exporter/main.go b/benchmarks/benchmark/tools/locust-load-inference/locust-custom-exporter/main.go
new file mode 100644
index 000000000..51e7802fe
--- /dev/null
+++ b/benchmarks/benchmark/tools/locust-load-inference/locust-custom-exporter/main.go
@@ -0,0 +1,186 @@
+package main
+
+import (
+	"crypto/tls"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"os"
+	"sync"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/prometheus/common/log"
+	"github.com/prometheus/common/version"
+	"gopkg.in/alecthomas/kingpin.v2"
+)
+
+var (
+	namespace string
+	NameSpace *string
+)
+
+// Exporter structure
+type Exporter struct {
+	uri   string
+	mutex sync.RWMutex
+	fetch func(endpoint string) (io.ReadCloser, error)
+
+	locustAvgTokensSent,
+	locustAvgTokensReceived,
+	locustAvgTestTime,
+	locustAvgOutputTokenLatency prometheus.Gauge
+}
+
+// NewExporter function
+func NewExporter(uri string, timeout time.Duration) (*Exporter, error) {
+	u, err := url.Parse(uri)
+	if err != nil {
+		return nil, err
+	}
+
+	var fetch func(endpoint string) (io.ReadCloser, error)
+	switch u.Scheme {
+	case "http", "https", "file":
+		fetch = fetchHTTP(uri, timeout)
+	default:
+		return nil, fmt.Errorf("unsupported scheme: %q", u.Scheme)
+	}
+
+	return &Exporter{
+		uri:   uri,
+		fetch: fetch,
+		locustAvgTokensSent: prometheus.NewGauge(
+			prometheus.GaugeOpts{
+				Namespace: namespace,
+				Subsystem: "custom_metrics",
+				Name:      "avg_tokens_sent",
+			},
+		),
+		locustAvgTokensReceived: prometheus.NewGauge(
+			prometheus.GaugeOpts{
+				Namespace: namespace,
+				Subsystem: "custom_metrics",
+				Name:      "avg_tokens_received",
+			},
+		),
+		locustAvgTestTime: prometheus.NewGauge(
+			prometheus.GaugeOpts{
+				Namespace: namespace,
+				Subsystem: "custom_metrics",
+				Name:      "avg_test_time",
+			},
+		),
+		locustAvgOutputTokenLatency: prometheus.NewGauge(
+			prometheus.GaugeOpts{
+				Namespace: namespace,
+				Subsystem: "custom_metrics",
+				Name:      "avg_output_token_latency",
+			},
+		),
+	}, nil
+}
+
+// Describe function of Exporter
+func (e *Exporter) Describe(ch chan<- *prometheus.Desc) {
+	e.mutex.Lock()
+	defer e.mutex.Unlock()
+
+	ch <- e.locustAvgTokensSent.Desc()
+	ch <- e.locustAvgTokensReceived.Desc()
+	ch <- e.locustAvgTestTime.Desc()
+	ch <- e.locustAvgOutputTokenLatency.Desc()
+}
+
+// Collect function of Exporter
+func (e *Exporter) Collect(ch chan<- prometheus.Metric) {
+	e.mutex.Lock()
+	defer e.mutex.Unlock()
+
+	var locustStats locustStats
+
+	body, err := e.fetch("/stats/custom_metrics")
+	if err != nil {
+		log.Errorf("Can't scrape Pack: %v", err)
+		return
+	}
+	defer body.Close()
+
+	bodyAll, err := ioutil.ReadAll(body)
+	if err != nil {
+		return
+	}
+
+	_ = json.Unmarshal([]byte(bodyAll), &locustStats)
+
+	ch <- prometheus.MustNewConstMetric(e.locustAvgTokensSent.Desc(), prometheus.GaugeValue, float64(locustStats.AvgTokensSent))
+	ch <- prometheus.MustNewConstMetric(e.locustAvgTokensReceived.Desc(), prometheus.GaugeValue, float64(locustStats.AvgTokensReceived))
+	ch <- prometheus.MustNewConstMetric(e.locustAvgTestTime.Desc(), prometheus.GaugeValue, float64(locustStats.AvgTestTime))
+	ch <- prometheus.MustNewConstMetric(e.locustAvgOutputTokenLatency.Desc(), prometheus.GaugeValue, float64(locustStats.AvgOutputTokenLatency))
+}
+
+type locustStats struct {
+	AvgTokensSent         float64 `json:"average-tokens-sent"`
+	AvgTokensReceived     float64 `json:"average-tokens-received"`
+	AvgTestTime           float64 `json:"average-test-time"`
+	AvgOutputTokenLatency float64 `json:"average-output-token-latency"`
+}
+
+func fetchHTTP(uri string, timeout time.Duration) func(endpoint string) (io.ReadCloser, error) {
+	tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
+	client := http.Client{
+		Timeout:   timeout,
+		Transport: tr,
+	}
+
+	return func(endpoint string) (io.ReadCloser, error) {
+		resp, err := client.Get(uri + endpoint)
+		if err != nil {
+			return nil, err
+		}
+		if !(resp.StatusCode >= 200 && resp.StatusCode < 300) {
+			resp.Body.Close()
+			return nil, fmt.Errorf("HTTP status %d", resp.StatusCode)
+		}
+		return resp.Body, nil
+	}
+}
+
+func main() {
+	var (
+		listenAddress = kingpin.Flag("web.listen-address", "Address to listen on for web interface and telemetry.").Default(":8080").Envar("locust_custom_exporter_WEB_LISTEN_ADDRESS").String()
+		metricsPath   = kingpin.Flag("web.telemetry-path", "Path under which to expose metrics.").Default("/metrics").Envar("locust_custom_exporter_WEB_TELEMETRY_PATH").String()
+		uri           = kingpin.Flag("locust.uri", "URI of Locust.").Default("http://localhost:8089").Envar("locust_custom_exporter_URI").String()
+		NameSpace     = kingpin.Flag("locust.namespace", "Namespace for prometheus metrics.").Default("locust").Envar("LOCUST_METRIC_NAMESPACE").String()
+		timeout       = kingpin.Flag("locust.timeout", "Scrape timeout").Default("5s").Envar("locust_custom_exporter_TIMEOUT").Duration()
+	)
+
+	log.AddFlags(kingpin.CommandLine)
+	kingpin.Version(version.Print("locust_exporter"))
+	kingpin.HelpFlag.Short('h')
+	kingpin.Parse()
+
+	namespace = *NameSpace
+	log.Infoln("Starting locust_custom_exporter", version.Info())
+	log.Infoln("Build context", version.BuildContext())
+
+	exporter, err := NewExporter(*uri, *timeout)
+	if err != nil {
+		log.Fatal(err)
+	}
+	prometheus.MustRegister(exporter)
+	prometheus.MustRegister(version.NewCollector("locustexporter"))
+
+	http.Handle(*metricsPath, promhttp.Handler())
+	http.HandleFunc("/quitquitquit", func(http.ResponseWriter, *http.Request) { os.Exit(0) })
+	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		_, _ = w.Write([]byte(`<html><head><title>Locust Exporter</title></head><body><h1>Locust Exporter</h1><p><a href='` + *metricsPath + `'>Metrics</a></p></body></html>`))
+	})
+
+	log.Infoln("Listening on", *listenAddress)
+	log.Fatal(http.ListenAndServe(*listenAddress, nil))
+}
diff --git a/benchmarks/benchmark/tools/locust-load-inference/locust-docker/locust-tasks/custom_metric_aggregator.py b/benchmarks/benchmark/tools/locust-load-inference/locust-docker/locust-tasks/custom_metric_aggregator.py
index 773be423a..a7621c7d9 100644
--- a/benchmarks/benchmark/tools/locust-load-inference/locust-docker/locust-tasks/custom_metric_aggregator.py
+++ b/benchmarks/benchmark/tools/locust-load-inference/locust-docker/locust-tasks/custom_metric_aggregator.py
@@ -1,5 +1,6 @@
 import datetime
 import logging
+import json
 
 class TokenMetricCollector:
     def __init__(self):
@@ -8,28 +9,56 @@ def __init__(self):
         self.test_time = []
         self.success_count = 0
         self.failure_count = 0
-        
 
-    def add_metric(self, sent, received, test_time, request_successful_bool):
-        if request_successful_bool==1:
+    def add_metric(self, sent, received, test_time, request_succesful_bool):
+        if request_succesful_bool == 1:
             self.tokens_sent.append(sent)
             self.tokens_received.append(received)
             self.test_time.append(test_time)
             self.success_count += 1
         else:
             self.failure_count += 1
-            
+
+    def add_metrics(self, tokens_sent, tokens_received, test_time, success_count, failure_count):
+        self.tokens_sent = self.tokens_sent + tokens_sent
+        self.tokens_received = self.tokens_received + tokens_received
+        self.test_time = self.test_time + test_time
+        self.success_count += success_count
+        self.failure_count += failure_count
+
+    def share_stats(self):
+        return self.tokens_sent, self.tokens_received, self.test_time, self.success_count, self.failure_count
 
     def calculate_average_tokens(self):
-        avg_sent = sum(self.tokens_sent) / len(self.tokens_sent) if self.tokens_sent else 0
-        avg_received = sum(self.tokens_received) / len(self.tokens_received) if self.tokens_received else 0
-        avg_test_time = sum(self.test_time) / len(self.test_time) if self.tokens_sent else 0
-        
-        return avg_sent, avg_received, avg_test_time
+        if self.tokens_sent and len(self.tokens_sent) > 0:
+            avg_sent = sum(self.tokens_sent) / \
+                len(self.tokens_sent) if self.tokens_sent else 0
+            avg_received = sum(self.tokens_received) / \
+                len(self.tokens_received) if self.tokens_received else 0
+            avg_test_time = sum(self.test_time) / \
+                len(self.test_time) if self.tokens_sent else 0
+            avg_output_token_latency = 0
+            for i in range(0, self.success_count):
+                avg_output_token_latency += (
+                    self.tokens_received[i] / self.test_time[i])
+            avg_output_token_latency = avg_output_token_latency / \
+                self.success_count
+            return avg_sent, avg_received, avg_test_time, avg_output_token_latency
+        return 0, 0, 0, 0
+
+    def json_dump_report(self):
+        avg_sent, avg_received, avg_test_time, avg_output_token_latency = self.calculate_average_tokens()
+        stats = {
+            "average-tokens-sent": avg_sent,
+            "average-tokens-received": avg_received,
+            "average-output-token-latency": avg_output_token_latency,
+            "average-test-time": avg_test_time,
+        }
+        return json.dumps(stats)
 
     def write_to_csv(self, file_path='custom_metrics.csv'):
         import csv
-        avg_sent, avg_received, avg_test_time = self.calculate_average_tokens()
+        avg_sent, avg_received, avg_test_time, avg_output_token_latency = self.calculate_average_tokens()
         with open(file_path, mode='w', newline='') as file:
             writer = csv.writer(file)
             writer.writerow(['Metric', 'Average Value'])
@@ -38,5 +67,6 @@ def write_to_csv(self, file_path='custom_metrics.csv'):
             writer.writerow(['Avg Tokens Sent Per Req', avg_sent])
             writer.writerow(['Avg Tokens Received Per Req', avg_received])
             writer.writerow(['Avg Test Time', avg_test_time])
+            writer.writerow(['Avg Output Tokens Latency',
+                            avg_output_token_latency])
             writer.writerow(['Timestamp', datetime.datetime.now()])
-            
diff --git a/benchmarks/benchmark/tools/locust-load-inference/locust-docker/locust-tasks/tasks.py b/benchmarks/benchmark/tools/locust-load-inference/locust-docker/locust-tasks/tasks.py
index ebb55dfc8..4f9370d2c 100644
--- a/benchmarks/benchmark/tools/locust-load-inference/locust-docker/locust-tasks/tasks.py
+++ b/benchmarks/benchmark/tools/locust-load-inference/locust-docker/locust-tasks/tasks.py
@@ -30,6 +30,7 @@
 
 from custom_metric_aggregator import TokenMetricCollector
 metric_collector = TokenMetricCollector()
+local_metric_collector = TokenMetricCollector()
 
 logging.basicConfig(level=logging.INFO)
 
@@ -112,7 +113,9 @@ def get_token_count(prompt, resp):
     if backend == "vllm":
         number_of_output_tokens = 0  # to be added
     elif backend == "tgi":
-        number_of_output_tokens = 0  # to be added
+        resp_dict = json.loads(resp.content.decode('utf-8'))
+        number_of_output_tokens = len(
+            tokenizer.encode(resp_dict['generated_text']))
     elif backend == "tensorrt_llm_triton":
         resp_dict = json.loads(resp.content.decode('utf-8'))
         number_of_output_tokens = len(
@@ -165,6 +168,8 @@ def handle_successful_response(self, prompt, reponse, start_time):
             request_successful_bool = 1
             tokens_sent, tokens_received = get_token_count(prompt, reponse)
 
+            local_metric_collector.add_metric(
+                tokens_sent, tokens_received, test_time, request_successful_bool)
             logging.info(
                 f'sending to master: metric_update: {[tokens_sent, tokens_received, test_time, request_successful_bool]}')
             self.environment.runner.send_message("metric_update", [
@@ -180,6 +185,8 @@ def handle_failed_response(self, request, response):
             test_time = -1
             request_successful_bool = 0
 
+            local_metric_collector.add_metric(
+                tokens_sent, tokens_received, test_time, request_successful_bool)
             logging.info(
                 f'sending to master: metric_update: {[tokens_sent, tokens_received, test_time, request_successful_bool]}')
             self.environment.runner.send_message("metric_update", [
@@ -197,7 +204,7 @@ def collect_metrics(msg, **_kwargs):
     received = msg.data[1]
     test_time = msg.data[2]
     request_successful_bool = msg.data[3]
-    logging.info(f'recevied from worker {msg.data}')
+    logging.info(f'received from worker {msg.data}')
     metric_collector.add_metric(
         sent, received, test_time, request_successful_bool)
 
@@ -232,6 +239,40 @@ def on_test_stop(environment, **kwargs):
         metric_collector.write_to_csv('custom_metrics_final.csv')
         metric_collector.__init__()
         metric_collector.write_to_csv()
+        local_metric_collector.__init__()
+
+
+"""
+Methods for collecting request latencies to share to master webui
+"""
+
+
+@events.report_to_master.add_listener
+def on_report_to_master(client_id, data):
+    """
+    This event is triggered on the worker instances every time a stats report is
+    to be sent to the locust master. It will allow us to add our local workers metrics
+    to the dict that is being sent, and then we clear the local stats in the worker, so
+    as to avoid sending duplicate data to the master on the next run.
+    """
+    tokens_sent, tokens_recieved, test_time, success_count, failure_count = local_metric_collector.share_stats()
+    data["tokens-sent"] = tokens_sent
+    data["tokens-received"] = tokens_recieved
+    data["test-time"] = test_time
+    data["success-count"] = success_count
+    data["failure-count"] = failure_count
+    local_metric_collector.__init__
+
+
+@events.worker_report.add_listener
+def on_worker_report(client_id, data):
+    """
+    This event is triggered on the master instance when a new stats report arrives
+    from a worker. Here we just add the local stats to the master's aggregated
+    stats dict.
+    """
+    local_metric_collector.add_metrics(
+        data["tokens-sent"], data["tokens-received"], data["test-time"], data["success-count"], data["failure-count"])
 
 
 @events.init_command_line_parser.add_listener
@@ -260,6 +301,7 @@ def _(environment, **kwargs):
         global model_params
         global test_data
         global metric_collector
+        global local_metric_collector
         global tokenizer
 
         tokenizer = AutoTokenizer.from_pretrained(
@@ -292,3 +334,22 @@ def _(environment, **kwargs):
         setup_periodic_metrics_writer(environment)
         setup_custom_route(environment)
         environment.runner.register_message("metric_update", collect_metrics)
+
+
+@events.init.add_listener
+def locust_init(environment, **kwargs):
+    """
+    We need somewhere to store the stats 
+
+    On the master node the metric_collector will contain the aggregated sum of all content-lengths,
+    while on the worker nodes this will be the sum of the content-lengths since the
+    last stats report was sent to the master
+    """
+    if environment.web_ui:
+        # this code is only run on the master node (the web_ui instance doesn't exist on workers)
+        @environment.web_ui.app.route("/stats/custom_metrics")
+        def total_content_length():
+            """
+            Add a route to the Locust web app, where we can see the total content-length
+            """
+            return local_metric_collector.json_dump_report()
diff --git a/benchmarks/benchmark/tools/locust-load-inference/locust-runner/metrics.yaml b/benchmarks/benchmark/tools/locust-load-inference/locust-runner/metrics.yaml
index 81475deb6..8b5dffa64 100644
--- a/benchmarks/benchmark/tools/locust-load-inference/locust-runner/metrics.yaml
+++ b/benchmarks/benchmark/tools/locust-load-inference/locust-runner/metrics.yaml
@@ -9,6 +9,26 @@
             'filter': 'metric.type = "prometheus.googleapis.com/DCGM_FI_DEV_FB_USED/gauge"',
             'type': 'GAUGE',
             'aggregation': ''
+        },
+        'AVG_TEST_TIME': {
+            'filter': 'metric.type = "prometheus/locust_custom_metrics_avg_test_time/gauge"',
+            'type': 'GAUGE',
+            'aggregation': ''
+        },
+        'AVG_OUTPUT_TOKEN_LATENCY': {
+            'filter': 'metric.type = "prometheus/locust_custom_metrics_avg_output_token_latency/gauge"',
+            'type': 'GAUGE',
+            'aggregation': ''
+        },
+        'AVG_TOKENS_RECEIVED': {
+            'filter': 'metric.type = "prometheus/locust_custom_metrics_avg_tokens_received/gauge"',
+            'type': 'GAUGE',
+            'aggregation': ''
+        },
+        'AVG_TOKENS_SENT': {
+            'filter': 'metric.type = "prometheus/locust_custom_metrics_avg_tokens_sent/gauge"',
+            'type': 'GAUGE',
+            'aggregation': ''
         }
     }
 }
\ No newline at end of file
diff --git a/benchmarks/benchmark/tools/locust-load-inference/manifest-templates/locust-master-controller.yaml.tpl b/benchmarks/benchmark/tools/locust-load-inference/manifest-templates/locust-master-controller.yaml.tpl
index bcef25201..f30c968d2 100644
--- a/benchmarks/benchmark/tools/locust-load-inference/manifest-templates/locust-master-controller.yaml.tpl
+++ b/benchmarks/benchmark/tools/locust-load-inference/manifest-templates/locust-master-controller.yaml.tpl
@@ -38,12 +38,15 @@ spec:
             - name: loc-master-p2
               containerPort: 5558
               protocol: TCP
+        - name: locust-custom-metrics-exporter
+          image: ${artifact_registry}/locust-custom-exporter:latest
+          ports:
+            - name: cmetrics-port
+              containerPort: 8080
+              protocol: TCP
         - name: locust-metrics-exporter
           image: containersol/locust_exporter
-          env:
-            - name: LOCUST_EXPORTER_WEB_LISTEN_ADDRESS
-              value: ":8080"
           ports:
             - name: metrics-port
-              containerPort: 8080
-              protocol: TCP
+              containerPort: 9646
+              protocol: TCP
\ No newline at end of file
diff --git a/benchmarks/benchmark/tools/locust-load-inference/manifest-templates/pod-monitoring.yaml.tpl b/benchmarks/benchmark/tools/locust-load-inference/manifest-templates/pod-monitoring.yaml.tpl
index ef9111797..63f81b5eb 100644
--- a/benchmarks/benchmark/tools/locust-load-inference/manifest-templates/pod-monitoring.yaml.tpl
+++ b/benchmarks/benchmark/tools/locust-load-inference/manifest-templates/pod-monitoring.yaml.tpl
@@ -7,7 +7,8 @@ spec:
   selector:
     matchLabels:
       app: locust-master
-
   endpoints:
   - port: 8080
+    interval: 5s
+  - port: 9646
     interval: 5s
\ No newline at end of file