From 977d96f42eaafd568ee777ed4a475c3dd8fad7ff Mon Sep 17 00:00:00 2001 From: Michael Gasch Date: Fri, 29 May 2020 22:38:51 +0200 Subject: [PATCH 01/86] Update Linter and Unit Test Action Update Golang linter to latest version. Unit tests will now also verify whether container image can be build. Signed-off-by: Michael Gasch --- .github/workflows/router-unit-tests.yml | 12 ++++++++---- vmware-event-router/Dockerfile | 2 +- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/router-unit-tests.yml b/.github/workflows/router-unit-tests.yml index 9d8ba85f..eaa65beb 100644 --- a/.github/workflows/router-unit-tests.yml +++ b/.github/workflows/router-unit-tests.yml @@ -1,8 +1,13 @@ name: VMware Event Router Unit Tests -# triggered on PRs but only when changes inside vmware-event-router (sub)dir(s) +# triggered on every push and PRs but only when changes inside +# vmware-event-router (sub)dir(s) on: pull_request: + types: [opened, synchronize, reopened] + paths: + - 'vmware-event-router/**' + push: paths: - 'vmware-event-router/**' @@ -18,6 +23,5 @@ jobs: steps: - name: checkout source uses: actions/checkout@master - - name: run unit tests - run: make test - + - name: run unit tests and verify image build (w/out push) + run: make build diff --git a/vmware-event-router/Dockerfile b/vmware-event-router/Dockerfile index f5be9a32..b5f176ae 100644 --- a/vmware-event-router/Dockerfile +++ b/vmware-event-router/Dockerfile @@ -6,7 +6,7 @@ ARG COMMIT WORKDIR /build # install linter into ./bin/ -RUN curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s v1.23.7 +RUN curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s v1.27.0 COPY go.mod . COPY go.sum . From 312d726f806e392412434392bc9c4e5579df6881 Mon Sep 17 00:00:00 2001 From: Michael Gasch Date: Fri, 29 May 2020 23:22:43 +0200 Subject: [PATCH 02/86] Decouple from types.BaseEvent This marks the first commit in a series to decouple the VMware Event Router processing logic from vCenter. Specifically, this commit changes the processor interface to using fully leveraging CloudEvents v1 Spec (SDKv2 package). Before invoking a processor stream provider events, e.g. vCenter, are converted to CloudEvents v1 spec and validated. The (vCenter) stream provider now invokes each processor via a single CloudEvent (instead of []types.BaseEvent). Batching, e.g. as was in EventBridge, could be re-implemented at a later stage (see note on state management below). Both, stream providers and stream processors have better error handling (depending on the implementation though). Note that there is still coupling between (vCenter) stream provider logic to the processor which need to be addressed in separate PR(s) via state management. Signed-off-by: Michael Gasch --- .gitignore | 5 +- vmware-event-router/Makefile | 23 ++- vmware-event-router/cmd/main.go | 6 +- vmware-event-router/go.mod | 16 +- vmware-event-router/go.sum | 151 ++++++++++++++-- .../internal/connection/connection.go | 7 + vmware-event-router/internal/events/events.go | 79 ++++---- .../internal/events/events_test.go | 43 +---- .../internal/metrics/metrics.go | 5 +- .../internal/metrics/server.go | 8 + .../internal/processor/aws_event_bridge.go | 157 +++++++--------- .../processor/aws_event_bridge_test.go | 143 --------------- .../internal/processor/aws_option.go | 38 ++++ .../internal/processor/openfaas.go | 167 +++++++++-------- .../internal/processor/openfaas_option.go | 57 ++++++ .../internal/processor/processor.go | 10 +- .../internal/stream/fake/vcenter_fake.go | 68 +++++++ .../internal/stream/fake/vcenter_fake_test.go | 168 ++++++++++++++++++ vmware-event-router/internal/stream/stream.go | 3 +- .../internal/stream/vcenter.go | 55 +++++- .../internal/stream/vcenter_option.go | 20 +++ 21 files changed, 788 insertions(+), 441 deletions(-) delete mode 100644 vmware-event-router/internal/processor/aws_event_bridge_test.go create mode 100644 vmware-event-router/internal/processor/aws_option.go create mode 100644 vmware-event-router/internal/processor/openfaas_option.go create mode 100644 vmware-event-router/internal/stream/fake/vcenter_fake.go create mode 100644 vmware-event-router/internal/stream/fake/vcenter_fake_test.go create mode 100644 vmware-event-router/internal/stream/vcenter_option.go diff --git a/.gitignore b/.gitignore index 5792af44..050e1112 100644 --- a/.gitignore +++ b/.gitignore @@ -10,4 +10,7 @@ secret*.json .jekyll-cache _site Gemfile.lock -.tweet-cache \ No newline at end of file +.tweet-cache +*.coverprofile +faas-cli* +template diff --git a/vmware-event-router/Makefile b/vmware-event-router/Makefile index 7a909460..72805ec9 100644 --- a/vmware-event-router/Makefile +++ b/vmware-event-router/Makefile @@ -1,5 +1,10 @@ COMMIT := $(shell git rev-parse --short HEAD) VERSION := $(shell /bin/cat ../VERSION) + +# fail fast +$(if $(value COMMIT),,$(error COMMIT must be set (.git project?))) +$(if $(value VERSION),,$(error VERSION must be set)) + IMAGE_REPO=vmware IMAGE_NAME=$(IMAGE_REPO)/veba-event-router DIST_FOLDER=dist @@ -12,11 +17,7 @@ GIT_BRANCH := $(shell git branch --show-current) GIT_NOT_CLEAN_CHECK := $(shell git status --porcelain) export GO111MODULE=on -PKGS = $(or $(PKG),$(shell env GO111MODULE=on go list ./...)) -TESTPKGS = $(shell env GO111MODULE=on go list -f \ - '{{ if or .TestGoFiles .XTestGoFiles }}{{ .ImportPath }}{{ end }}' \ - $(PKGS)) -TIMEOUT = 20 +TEST_TIMEOUT = 20 .PHONY: release tidy build tag push output vendor test @@ -34,11 +35,11 @@ tidy: go mod tidy -v # intended for local dev use (won't check for unclean git) -binary: test tidy +binary: unit-test tidy $(info Make: Building binary "$(DIST_FOLDER)/$(BINARY)".) CGO_ENABLED=0 go build -a -installsuffix nocgo -ldflags="-X main.version=${VERSION} -X main.commit=${COMMIT}" -o $(DIST_FOLDER)/$(BINARY) cmd/main.go -build: test tidy +build: unit-test tidy $(info Make: Building image "$(IMAGE_NAME)".) $(if $(GIT_NOT_CLEAN_CHECK), $(error "Dirty Git repository!")) docker build -t $(BUILD_TAG) --build-arg COMMIT=$(COMMIT) --build-arg VERSION=$(VERSION) . @@ -47,8 +48,12 @@ gofmt: $(info Make: Checking code is gofmted.) @test -z "$(shell gofmt -s -l -d -e ./cmd | tee /dev/stderr)" -test: gofmt - GORACE=history_size=5 go test -race -timeout $(TIMEOUT)s -cover $(TESTPKGS) +unit-test: gofmt + $(eval TAG=unit) + $(eval TEST_PKGS=$(shell go list -tags=${TAG} -f '{{ if or .TestGoFiles .XTestGoFiles }}{{ .ImportPath }}{{ end }}' ./...)) + @GORACE=history_size=5 go test -count 1 -tags=${TAG} -race -timeout $(TEST_TIMEOUT)s -cover ${TEST_PKGS} + +test: unit-test tag: $(info Make: Tagging image "$(IMAGE_NAME)" with "$(BUILD_TAG)", "$(LATEST_TAG) and "$(VERSION_TAG)".) diff --git a/vmware-event-router/cmd/main.go b/vmware-event-router/cmd/main.go index 29ab575a..bda398ec 100644 --- a/vmware-event-router/cmd/main.go +++ b/vmware-event-router/cmd/main.go @@ -90,7 +90,7 @@ func main() { switch cfg.Provider { case stream.ProviderVSphere: logger.Printf("connecting to vCenter %s", cfg.Address) - streamer, err = stream.NewVCenterStream(ctx, cfg, metricsServer) + streamer, err = stream.NewVCenterStream(ctx, cfg, metricsServer, stream.WithVCenterVerbose(verbose)) if err != nil { logger.Fatalf("could not connect to vCenter: %v", err) } @@ -107,13 +107,13 @@ func main() { async = true } logger.Printf("connecting to OpenFaaS gateway %s (async mode: %v)", cfg.Address, async) - proc, err = processor.NewOpenFaaSProcessor(ctx, cfg, streamer.Source(), verbose, metricsServer) + proc, err = processor.NewOpenFaaSProcessor(ctx, cfg, metricsServer, processor.WithOpenFaaSVerbose(verbose)) if err != nil { logger.Fatalf("could not connect to OpenFaaS: %v", err) } case processor.ProviderAWS: logger.Printf("connecting to AWS EventBridge (arn: %s)", cfg.Options["aws_eventbridge_rule_arn"]) - proc, err = processor.NewAWSEventBridgeProcessor(ctx, cfg, streamer.Source(), verbose, metricsServer) + proc, err = processor.NewAWSEventBridgeProcessor(ctx, cfg, metricsServer, processor.WithAWSVerbose(verbose)) if err != nil { logger.Fatalf("could not connect to AWS EventBridge: %v", err) } diff --git a/vmware-event-router/go.mod b/vmware-event-router/go.mod index 33bf7a87..3c2c6706 100644 --- a/vmware-event-router/go.mod +++ b/vmware-event-router/go.mod @@ -1,16 +1,16 @@ module github.com/vmware-samples/vcenter-event-broker-appliance/vmware-event-router -go 1.12 +go 1.14 require ( github.com/aws/aws-sdk-go v1.27.3 - github.com/cloudevents/sdk-go v0.10.2 + github.com/cloudevents/sdk-go/v2 v2.0.0-RC2 github.com/google/uuid v1.1.1 + github.com/onsi/ginkgo v1.12.2 // indirect github.com/openfaas-incubator/connector-sdk v0.0.0-20191214130609-df5d76475412 - github.com/openfaas/faas-provider v0.0.0-20200101101649-8f7c35975e1b - github.com/pkg/errors v0.8.1 - github.com/stretchr/testify v1.4.0 // indirect - github.com/vmware/govmomi v0.21.0 - golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 // indirect - golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e + github.com/openfaas/faas-provider v0.15.1 + github.com/pkg/errors v0.9.1 + github.com/vmware/govmomi v0.22.2 + golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a + golang.org/x/sys v0.0.0-20200523222454-059865788121 // indirect ) diff --git a/vmware-event-router/go.sum b/vmware-event-router/go.sum index 7edc0846..07c93ff4 100644 --- a/vmware-event-router/go.sum +++ b/vmware-event-router/go.sum @@ -5,6 +5,7 @@ cloud.google.com/go v0.40.0/go.mod h1:Tk58MuI9rbLMKlAjeO/bDnteAx7tX2gJIXw4T5Jwlr contrib.go.opencensus.io/exporter/ocagent v0.4.12/go.mod h1:450APlNTSR6FrvC3CTRqYosuDstRB9un7SOx2k/9ckA= contrib.go.opencensus.io/exporter/prometheus v0.1.0/go.mod h1:cGFniUXGZlKRjzOyuZJ6mgB+PgBcCIa79kEKR8YCW+A= github.com/Azure/azure-sdk-for-go v30.1.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/go-amqp v0.12.7/go.mod h1:qApuH6OFTSKZFmCOxccvAv5rLizBQf4v8pRmG138DPo= github.com/Azure/go-autorest/autorest v0.2.0/go.mod h1:AKyIcETwSUFxIcs/Wnq/C+kwCtlEYGUVd7FPNb2slmg= github.com/Azure/go-autorest/autorest/adal v0.1.0/go.mod h1:MeS4XhScH55IST095THyTxElntu7WqB7pNbZo8Q5G3E= github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= @@ -15,18 +16,32 @@ github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6L github.com/Azure/go-autorest/tracing v0.1.0/go.mod h1:ROEEAFwXycQw7Sn3DXNtEedEvdeRAgDr0izn4z5Ij88= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= +github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg= +github.com/aslakhellesoy/gox v1.0.100/go.mod h1:AJl542QsKKG96COVsv0N74HHzVQgDIQPceVUh1aeU2M= github.com/aws/aws-sdk-go v1.27.3 h1:CBWC7Yot0U6OU/uosUmq7tKJVBTq6HrhgW1Vjpt9SMw= github.com/aws/aws-sdk-go v1.27.3/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= github.com/census-instrumentation/opencensus-proto v0.2.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= +github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cloudevents/sdk-go v0.10.2 h1:CAqHqDHmBkCG4OUeUBt7q2Ql8KV25U+bgPUtlcJelZ4= -github.com/cloudevents/sdk-go v0.10.2/go.mod h1:EHG6NmU3XkIeuueER6+vbnhYfWlgVlfUQVzPC+UK7ao= +github.com/cloudevents/sdk-go v1.1.2 h1:mg/7d+BzubBPrPpH1bdeF85BQZYV85j7Ljqat3+m+qE= +github.com/cloudevents/sdk-go v1.1.2/go.mod h1:ss+jWJ88wypiewnPEzChSBzTYXGpdcILoN9YHk8uhTQ= +github.com/cloudevents/sdk-go/v2 v2.0.0-RC2 h1:XXqj/WXjOWhxUR8/+Ovn5YtSuIE83uOD6Gy3vUnBdUQ= +github.com/cloudevents/sdk-go/v2 v2.0.0-RC2/go.mod h1:f6d2RzSysHwhr4EsysDapUIWyJOFKqIhDisATXEa6Wk= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/cucumber/gherkin-go/v11 v11.0.0/go.mod h1:CX33k2XU2qog4e+TFjOValoq6mIUq0DmVccZs238R9w= +github.com/cucumber/godog v0.9.0/go.mod h1:roWCHkpeK6UTOyIRRl7IR+fgfBeZ4vZR7OSq2J/NbM4= +github.com/cucumber/messages-go/v10 v10.0.1/go.mod h1:kA5T38CBlBbYLU12TIrJ4fk4wSkVVOgyh7Enyy8WnSg= +github.com/cucumber/messages-go/v10 v10.0.3/go.mod h1:9jMZ2Y8ZxjLY6TG2+x344nt5rXstVVDYSdS5ySfI1WY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -35,26 +50,46 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= +github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0 h1:xU6/SpYbvkNYiptHJYEDRseDLvYE7wSqhYYNy0QSUzI= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/uuid v0.0.0-20170306145142-6a5e28554805 h1:skl44gU1qEIcRpwKjb9bhlRwjvr96wLdvpTogCBBJe8= @@ -62,47 +97,86 @@ github.com/google/uuid v0.0.0-20170306145142-6a5e28554805/go.mod h1:TIyPZe4Mgqvf github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/gorilla/context v0.0.0-20160226214623-1ea25387ff6f/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= +github.com/gorilla/mux v1.6.2 h1:Pgr17XVTNXAk3q/r4CpKzC5xBM/qW1uVLV+IhRZpIIk= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/grpc-ecosystem/grpc-gateway v1.8.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-hclog v0.9.1/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-version v1.0.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/raft v1.1.1/go.mod h1:vPAJM8Asw6u8LxC3eJCUZmRP/E4QmUGE1R7g7k8sG/8= +github.com/hashicorp/raft-boltdb v0.0.0-20171010151810-6e5ba93211ea/go.mod h1:pNv7Wc3ycL6F5oOWn+tPGo2gWD4a5X+yp/ntwdKLjRk= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lightstep/tracecontext.go v0.0.0-20181129014701-1757c391b1ac h1:+2b6iGRJe3hvV/yVXrd41yVEjxuFHxasJqDhkIjS4gk= +github.com/lightstep/tracecontext.go v0.0.0-20181129014701-1757c391b1ac/go.mod h1:Frd2bnT3w5FB5q49ENTfVlztJES+1k/7lyWX2+9gq/M= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k= +github.com/nats-io/nats-server/v2 v2.1.4/go.mod h1:Jw1Z28soD/QasIA2uWjXyM9El1jly3YwyFOuR8tH1rg= +github.com/nats-io/nats-streaming-server v0.17.0/go.mod h1:ewPBEsmp62Znl3dcRsYtlcfwudxHEdYMtYqUQSt4fE0= github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/nats-io/stan.go v0.6.0/go.mod h1:eIcD5bi3pqbHT/xIIvXMwvzXYElgouBvaVRftaE+eac= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.12.2 h1:Ke9m3h2Hu0wsZ45yewCqhYr3Z+emcNTuLY2nMWCkrSI= +github.com/onsi/ginkgo v1.12.2/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/openfaas-incubator/connector-sdk v0.0.0-20191214130609-df5d76475412 h1:9xbx42wa4TtNE7QuWhhQnbpCBclrI4fC6O9Q0oty/9I= github.com/openfaas-incubator/connector-sdk v0.0.0-20191214130609-df5d76475412/go.mod h1:jHCtd1HCZwhuwdPy4OB8CQU7ZqzxBdWNhIhH/khCJqQ= -github.com/openfaas/faas-provider v0.0.0-20200101101649-8f7c35975e1b h1:3zAdXYHiFYX0rP6dEQyj0Ua3X51X06cRSu1GENTEQ/k= -github.com/openfaas/faas-provider v0.0.0-20200101101649-8f7c35975e1b/go.mod h1:W4OIp33RUOpR7wW+omJB/7GhIydRmYXvKf/VqUKI4yM= +github.com/openfaas/faas-provider v0.15.1 h1:5M+DrGbuKlZxhN3/otFgLgBUnPQcYm6XosQDSbKXh30= +github.com/openfaas/faas-provider v0.15.1/go.mod h1:ChVioeB3snmfYxBTu7SrWGYLoYN9fZkQlnuEQ86a6l0= github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= +github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -110,11 +184,13 @@ github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4 github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -123,11 +199,16 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/vmware/govmomi v0.21.0 h1:jc8uMuxpcV2xMAA/cnEDlnsIjvqcMra5Y8onh/U3VuY= -github.com/vmware/govmomi v0.21.0/go.mod h1:zbnFoBQ9GIjs2RVETy8CNEpb+L+Lwkjs3XZUL0B3/m0= +github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/vmware/govmomi v0.22.2 h1:hmLv4f+RMTTseqtJRijjOWzwELiaLMIoHv2D6H3bF4I= +github.com/vmware/govmomi v0.22.2/go.mod h1:Y+Wq4lst78L85Ge/F8+ORXIWiKYqaro1vhAulACy9Lc= github.com/vmware/vmw-guestinfo v0.0.0-20170707015358-25eff159a728/go.mod h1:x9oS4Wk2s2u4tS29nEaDLdzvuHdB19CvSGJjPgkZJNk= +go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= @@ -135,13 +216,17 @@ go.opencensus.io v0.22.0 h1:C9hSCOW830chIVkdja34wa6Ky+IzWllkUinR+BtRZd4= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/goleak v0.10.0/go.mod h1:VCZuO8V8mFPlL0F5J5GK1rtHV3DrFcQ1R8ryq7FK0aI= go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 h1:HuIa8hRrWRSrqYzx1qI49NNxhdi2PrY7gxVSq1JjLDc= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200206161412-a0c6ece9d31a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -160,21 +245,22 @@ golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c h1:uOCk1iQW6Vc18bnC13MfzScl+wdKBmM9Y9kU7Z83/lw= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 h1:efeOvDhwQ29Dj3SdAV/MJf8oukgn+8D8WgaCaRMchF8= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7 h1:AeiKBIuRw3UomYXSbLy0Mc2dDLfdtbT/IVn4keq83P0= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a h1:WXEvlFVvvGxCJLG6REjsT03iWnKLEWinaScsxF2Vm2o= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -185,22 +271,38 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190523142557-0e01d883c5c5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e h1:D5TXcfTk7xF7hvieo4QErS3qqCB4teTffacDWr7CI+0= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121 h1:rITEj+UZHYC927n8GT97eC3zrpzXdb/voyeOuVKS46o= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.6.0/go.mod h1:btoxGiFvQNVUZQ8W08zLtrVS08CNpINPEfxXxgJL1Q4= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -210,24 +312,45 @@ google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRn google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.19.1/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= pack.ag/amqp v0.11.0/go.mod h1:4/cbmt4EJXSKlG6LCfWHoqmN0uFdy5i/+YFz+fTfhV4= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= diff --git a/vmware-event-router/internal/connection/connection.go b/vmware-event-router/internal/connection/connection.go index 9c24dbd1..f06fb32d 100644 --- a/vmware-event-router/internal/connection/connection.go +++ b/vmware-event-router/internal/connection/connection.go @@ -5,8 +5,12 @@ import ( "io" ) +// Configs is a list of configurations for stream processors, providers and +// internal services of the VMware Event Router (e.g. metrics) type Configs []Config +// Config is used to configure stream processors, providers and internal +// services of the VMware Event Router (e.g. metrics) type Config struct { Type string `json:"type,omitempty"` // "stream", "processor" Provider string `json:"provider,omitempty"` // "vmware_vcenter", "openfaas", "aws_event_bridge" @@ -15,11 +19,14 @@ type Config struct { Options map[string]string `json:"options,omitempty"` } +// Authentication can hold generic authentication data for different stream +// providers and processors type Authentication struct { Method string `json:"method"` Secret map[string]string `json:"secret"` } +// Parse parses a list of configurations func Parse(cfg io.Reader) (Configs, error) { var cfgs Configs err := json.NewDecoder(cfg).Decode(&cfgs) diff --git a/vmware-event-router/internal/events/events.go b/vmware-event-router/internal/events/events.go index d30af1e0..2514e826 100644 --- a/vmware-event-router/internal/events/events.go +++ b/vmware-event-router/internal/events/events.go @@ -4,38 +4,18 @@ import ( "reflect" "time" + cloudevents "github.com/cloudevents/sdk-go/v2" "github.com/google/uuid" + "github.com/pkg/errors" "github.com/vmware/govmomi/vim25/types" ) const ( eventCanonicalType = "com.vmware.event.router" - eventSpecVersion = "1.0" // CloudEvents spec version used - eventContentType = "application/json" + eventSpecVersion = cloudevents.VersionV1 + eventContentType = cloudevents.ApplicationJSON ) -// CloudEvent is the JSON object sent to subscribed functions. We follow -// CloudEvents v1.0 spec as defined in -// https://github.com/cloudevents/sdk-go/blob/6c55828dbb6915e1594e5ace8bd8a19980731867/pkg/cloudevents/eventcontext_v1.go#L22 -type CloudEvent struct { - // ID of the event; must be non-empty and unique within the scope of the producer. - ID string `json:"id"` - // Source - URI of the event producer, e.g. http(s)://vcenter.domain.ext/sdk. - Source string `json:"source"` - // SpecVersion - The version of the CloudEvents specification the event router. - SpecVersion string `json:"specversion"` - // Type - canonicalType + vcenter event category (event, eventex, extendedevent). - Type string `json:"type"` - // Subject - vcenter event name used for topic subscriptions - Subject string `json:"subject"` - // Time - Timestamp set by this event router when this message was created. - Time time.Time `json:"time"` - // Data - Event payload as received from vcenter (includes event creation timestamp set by vcenter). - Data types.BaseEvent `json:"data"` - // DataContentType - A MIME (RFC2046) string describing the media type of `data`. - DataContentType string `json:"datacontenttype"` -} - // VCenterEventInfo contains the name and category of an event received from vCenter // supported event categories: event, eventex, extendedevent // category to name convention: @@ -50,7 +30,7 @@ type VCenterEventInfo struct { // GetDetails retrieves the underlying vSphere event category and name for // the given BaseEvent, e.g. VmPoweredOnEvent (event) or // com.vmware.applmgmt.backup.job.failed.event (extendedevent) -func GetDetails(event types.BaseEvent) VCenterEventInfo { +func getDetails(event types.BaseEvent) (VCenterEventInfo, error) { eventInfo := VCenterEventInfo{} switch e := event.(type) { @@ -60,24 +40,51 @@ func GetDetails(event types.BaseEvent) VCenterEventInfo { case *types.ExtendedEvent: eventInfo.Category = "extendedevent" eventInfo.Name = e.EventTypeId + + // TODO: make agnostic to vCenter events default: eType := reflect.TypeOf(event).Elem().Name() eventInfo.Category = "event" eventInfo.Name = eType } - return eventInfo + return eventInfo, nil } // NewCloudEvent returns a compliant CloudEvent -func NewCloudEvent(event types.BaseEvent, eventInfo VCenterEventInfo, source string) CloudEvent { - return CloudEvent{ - ID: uuid.New().String(), - Source: source, - SpecVersion: eventSpecVersion, - Type: eventCanonicalType + "/" + eventInfo.Category, - Subject: eventInfo.Name, - Time: time.Now().UTC(), - Data: event, - DataContentType: eventContentType, +// TODO: make agnostic to just vCenter event types +func NewCloudEvent(event types.BaseEvent, source string) (*cloudevents.Event, error) { + eventInfo, err := getDetails(event) + if err != nil { + return nil, errors.Wrap(err, "could not retrieve event information") + } + + ce := cloudevents.NewEvent(eventSpecVersion) + + // set ID of the event; must be non-empty and unique within the scope of the producer. + ce.SetID(uuid.New().String()) + + // set source - URI of the event producer, e.g. http(s)://vcenter.domain.ext/sdk. + ce.SetSource(source) + + // set type - canonicalType + vcenter event category (event, eventex, extendedevent). + ce.SetType(eventCanonicalType + "/" + eventInfo.Category) + + // set subject - vcenter event name used for topic subscriptions + ce.SetSubject(eventInfo.Name) + + // set time - Timestamp set by this event router when this message was created. + ce.SetTime(time.Now().UTC()) + + // set data - Event payload as received from processor (includes event + // creation timestamp, e.g. as set by vcenter). + err = ce.SetData(eventContentType, event) + if err != nil { + return nil, errors.Wrap(err, "could not create CloudEvent") } + + if err = ce.Validate(); err != nil { + return nil, errors.Wrap(err, "validation for CloudEvent failed") + } + + return &ce, nil } diff --git a/vmware-event-router/internal/events/events_test.go b/vmware-event-router/internal/events/events_test.go index 92a04af1..ba8e16f3 100644 --- a/vmware-event-router/internal/events/events_test.go +++ b/vmware-event-router/internal/events/events_test.go @@ -1,10 +1,10 @@ +// +build unit + package events import ( - "encoding/json" "testing" - cloudevents "github.com/cloudevents/sdk-go" "github.com/vmware/govmomi/vim25/types" ) @@ -35,45 +35,14 @@ func Test_GetEventDetails(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := GetDetails(tt.args.event); got != tt.want { + // TODO: handle error + if got, _ := getDetails(tt.args.event); got != tt.want { t.Errorf("getEventDetails() = %v, want %v", got, tt.want) } }) } } -func Test_ConvertToCloudEventV1(t *testing.T) { - vmEvent := newVMPoweredOnEvent() - eInfo := GetDetails(vmEvent) - e := NewCloudEvent(vmEvent, eInfo, getSource()) - b, err := json.Marshal(e) - if err != nil { - t.Fatalf("could not marshal cloud event: %v", err) - } - - ce := cloudevents.NewEvent(cloudevents.VersionV1) - err = ce.UnmarshalJSON(b) - if err != nil { - t.Fatalf("could not unmarshal outbound cloud event into cloud events v1 spec: %v", err) - } - - if e.ID != ce.ID() { - t.Fatalf("ID of outbound cloud event and cloud event v1 does not match: %q vs %q", e.ID, ce.ID()) - } - - if e.Source != ce.Source() { - t.Fatalf("Source of outbound cloud event and cloud event v1 does not match: %q vs %q", e.Source, ce.Source()) - } - - if e.SpecVersion != ce.SpecVersion() { - t.Fatalf("SpecVersions of outbound cloud event and cloud event v1 does not match: %q vs %q", e.SpecVersion, ce.SpecVersion()) - } - - if e.Subject != ce.Subject() { - t.Fatalf("Subject of outbound cloud event and cloud event v1 don't match: %q vs %q", e.Subject, ce.Subject()) - } -} - func newVMPoweredOnEvent() types.BaseEvent { return &types.VmPoweredOnEvent{ VmEvent: types.VmEvent{ @@ -103,7 +72,3 @@ func newEventExEvent() types.BaseEvent { EventTypeId: "com.vmware.cl.PublishLibraryEvent", } } - -func getSource() string { - return "https://vcenter.corp.local/sdk" -} diff --git a/vmware-event-router/internal/metrics/metrics.go b/vmware-event-router/internal/metrics/metrics.go index 07d23358..8990168e 100644 --- a/vmware-event-router/internal/metrics/metrics.go +++ b/vmware-event-router/internal/metrics/metrics.go @@ -13,7 +13,7 @@ const ( mapName = "vmware.event.router.stats" // PushInterval defines the default interval event streams and processors // push their metrics to the server - PushInterval = time.Second * 5 + PushInterval = time.Second * 1 ) // EventStats are provided and continously updated by event streams and @@ -23,7 +23,8 @@ type EventStats struct { ProviderType string `json:"provider_type"` // stream or processor Name string `json:"name"` Started time.Time `json:"started"` - EventsTotal *int `json:"events_total,omitempty"` // only used by event streams + EventsTotal *int `json:"events_total,omitempty"` // only used by event streams, total events received + EventsErr *int `json:"events_err,omitempty"` // only used by event streams, events received which lead to error EventsSec *float64 `json:"events_per_sec,omitempty"` // only used by event streams Invocations map[string]int `json:"invocations,omitempty"` // event.Category to invocations - only used by event processors } diff --git a/vmware-event-router/internal/metrics/server.go b/vmware-event-router/internal/metrics/server.go index fa619efd..57735346 100644 --- a/vmware-event-router/internal/metrics/server.go +++ b/vmware-event-router/internal/metrics/server.go @@ -26,6 +26,14 @@ var ( eventRouterStats = expvar.NewMap(mapName) ) +// Receiver receives metrics from metric providers +type Receiver interface { + Receive(stats EventStats) +} + +// verify that metrics server implements Receiver +var _ Receiver = (*Server)(nil) + // Server is the implementation of the metrics server type Server struct { http *http.Server diff --git a/vmware-event-router/internal/processor/aws_event_bridge.go b/vmware-event-router/internal/processor/aws_event_bridge.go index 6c1a1936..3598c59a 100644 --- a/vmware-event-router/internal/processor/aws_event_bridge.go +++ b/vmware-event-router/internal/processor/aws_event_bridge.go @@ -14,30 +14,32 @@ import ( "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/eventbridge" "github.com/aws/aws-sdk-go/service/eventbridge/eventbridgeiface" + cloudevents "github.com/cloudevents/sdk-go/v2" "github.com/pkg/errors" "github.com/vmware-samples/vcenter-event-broker-appliance/vmware-event-router/internal/color" "github.com/vmware-samples/vcenter-event-broker-appliance/vmware-event-router/internal/connection" - "github.com/vmware-samples/vcenter-event-broker-appliance/vmware-event-router/internal/events" "github.com/vmware-samples/vcenter-event-broker-appliance/vmware-event-router/internal/metrics" - "github.com/vmware/govmomi/vim25/types" ) const ( - // ProviderAWS is the name used to identify this provider in the + // ProviderAWS variable is the name used to identify this provider in the // VMware Event Router configuration file - ProviderAWS = "aws_event_bridge" - authMethodAWS = "access_key" // only this method is supported by the processor - resyncInterval = time.Minute * 5 // resync rule patterns after interval - pageLimit = 50 // max 50 results per page for list operations - batchSize = 10 // max 10 input events per batch sent to AWS + ProviderAWS = "aws_event_bridge" + authMethodAWS = "access_key" // only this method is supported by the processor + defaultResyncInterval = time.Minute * 5 // resync rule patterns after interval + defaultPageLimit = 50 // max 50 results per page for list operations + defaultBatchSize = 10 // max 10 input events per batch sent to AWS ) // awsEventBridgeProcessor implements the Processor interface type awsEventBridgeProcessor struct { session session.Session eventbridgeiface.EventBridgeAPI - source string - verbose bool + + // options + verbose bool + resyncInterval time.Duration + batchSize int *log.Logger mu sync.RWMutex @@ -53,13 +55,18 @@ type eventPattern struct { // NewAWSEventBridgeProcessor returns an AWS EventBridge processor for the given // stream source. -func NewAWSEventBridgeProcessor(ctx context.Context, cfg connection.Config, source string, verbose bool, ms *metrics.Server) (Processor, error) { +func NewAWSEventBridgeProcessor(ctx context.Context, cfg connection.Config, ms metrics.Receiver, opts ...AWSOption) (Processor, error) { logger := log.New(os.Stdout, color.Yellow("[AWS EventBridge] "), log.LstdFlags) eventBridge := awsEventBridgeProcessor{ - source: source, - verbose: verbose, - Logger: logger, - patternMap: make(map[string]string), + resyncInterval: defaultResyncInterval, + batchSize: defaultBatchSize, + Logger: logger, + patternMap: make(map[string]string), + } + + // apply options + for _, opt := range opts { + opt(&eventBridge) } var accessKey, secretKey, region, eventbus, ruleARN string @@ -109,8 +116,8 @@ func NewAWSEventBridgeProcessor(ctx context.Context, cfg connection.Config, sour var nextToken *string for !found { rules, err := eventBridge.ListRulesWithContext(ctx, &eventbridge.ListRulesInput{ - EventBusName: aws.String(eventbus), // explicitely passing eventbus name because list assumes "default" otherwise - Limit: aws.Int64(pageLimit), // up to n results per page for requests. + EventBusName: aws.String(eventbus), // explicitely passing eventbus name because list assumes "default" otherwise + Limit: aws.Int64(defaultPageLimit), // up to n results per page for requests. NextToken: nextToken, }) if err != nil { @@ -171,89 +178,57 @@ func NewAWSEventBridgeProcessor(ctx context.Context, cfg connection.Config, sour return &eventBridge, nil } -// Process implements the stream processor interface TODO: handle -// throttling/batching -// https://docs.aws.amazon.com/eventbridge/latest/userguide/cloudwatch-limits-eventbridge.html#putevents-limits -func (awsEventBridge *awsEventBridgeProcessor) Process(moref types.ManagedObjectReference, baseEvent []types.BaseEvent) error { - batchInput, err := awsEventBridge.createPutEventsInput(baseEvent) - if err != nil { - errMsg := fmt.Errorf("could not create PutEventsInput for event(s): %v", err) - awsEventBridge.Println(errMsg) - return processorError(ProviderAWS, errMsg) - } - - // nothing to send - if len(batchInput) == 0 { - return nil +// Process implements the stream processor interface +func (awsEventBridge *awsEventBridgeProcessor) Process(ce cloudevents.Event) error { + if awsEventBridge.verbose { + awsEventBridge.Printf("processing event (ID %s): %v", ce.ID(), ce) } - for idx, input := range batchInput { - resp, err := awsEventBridge.PutEvents(&input) - if err != nil { - awsEventBridge.Printf("could not send event(s) for batch %d: %v", idx, err) - continue - } + awsEventBridge.mu.RLock() + defer awsEventBridge.mu.RUnlock() + if _, ok := awsEventBridge.patternMap[ce.Subject()]; !ok { + // no event bridge rule pattern (subscription) for event, skip if awsEventBridge.verbose { - awsEventBridge.Printf("successfully sent event(s) from source %s: %+v batch: %d", - awsEventBridge.source, - resp, - idx) + awsEventBridge.Printf("pattern rule does not match, skipping event (ID %s): %v", ce.ID(), ce) } + return nil } - return nil -} - -func (awsEventBridge *awsEventBridgeProcessor) createPutEventsInput(baseEvent []types.BaseEvent) ([]eventbridge.PutEventsInput, error) { - awsEventBridge.mu.Lock() - defer awsEventBridge.mu.Unlock() - batch := []eventbridge.PutEventsInput{} - - tmpInput := eventbridge.PutEventsInput{ - Entries: []*eventbridge.PutEventsRequestEntry{}, + jsonBytes, err := json.Marshal(ce) + if err != nil { + msg := fmt.Errorf("could not marshal event %v: %v", ce, err) + awsEventBridge.Println(msg) + return processorError(ProviderAWS, msg) } - for idx := range baseEvent { - // process slice in reverse order to maintain Event.Key ordering - event := baseEvent[len(baseEvent)-1-idx] - - if awsEventBridge.verbose { - awsEventBridge.Printf("processing event [%d] of type %T from source %s: %+v", idx, event, awsEventBridge.source, event) - } - eventInfo := events.GetDetails(event) - if _, ok := awsEventBridge.patternMap[eventInfo.Name]; !ok { - // no event bridge rule pattern (subscription) for event, skip - continue - } - cloudEvent := events.NewCloudEvent(event, eventInfo, awsEventBridge.source) - jsonBytes, err := json.Marshal(cloudEvent) - if err != nil { - return nil, errors.Wrapf(err, "could not marshal cloud event for vSphere event %d from source %s", event.GetEvent().Key, awsEventBridge.source) - } - jsonString := string(jsonBytes) - entry := eventbridge.PutEventsRequestEntry{ - Detail: aws.String(jsonString), - EventBusName: aws.String(awsEventBridge.patternMap[eventInfo.Name]), - Source: aws.String(cloudEvent.Source), - DetailType: aws.String(cloudEvent.Subject), - } - tmpInput.Entries = append(tmpInput.Entries, &entry) + jsonString := string(jsonBytes) + entry := eventbridge.PutEventsRequestEntry{ + Detail: aws.String(jsonString), + EventBusName: aws.String(awsEventBridge.patternMap[ce.Subject()]), + Source: aws.String(ce.Source()), + DetailType: aws.String(ce.Subject()), + } - // update metrics - awsEventBridge.stats.Invocations[eventInfo.Name]++ + // update metrics + awsEventBridge.stats.Invocations[ce.Subject()]++ - if idx%batchSize == 0 && idx != 0 { - batch = append(batch, tmpInput) - tmpInput = eventbridge.PutEventsInput{ - Entries: []*eventbridge.PutEventsRequestEntry{}, - } - } + input := eventbridge.PutEventsInput{ + Entries: []*eventbridge.PutEventsRequestEntry{&entry}, } - if len(tmpInput.Entries) > 0 { - batch = append(batch, tmpInput) + awsEventBridge.Printf("sending event %s", ce.ID()) + resp, err := awsEventBridge.PutEvents(&input) + if err != nil { + msg := fmt.Errorf("could not send event %v: %v", ce, err) + awsEventBridge.Println(msg) + return processorError(ProviderAWS, msg) } - return batch, nil + if awsEventBridge.verbose { + awsEventBridge.Printf("successfully sent event %v: %v", ce, resp) + } else { + awsEventBridge.Printf("successfully sent event %s", ce.ID()) + } + return nil } func (awsEventBridge *awsEventBridgeProcessor) syncPatternMap(ctx context.Context, eventbus string, ruleARN string) { @@ -261,12 +236,12 @@ func (awsEventBridge *awsEventBridgeProcessor) syncPatternMap(ctx context.Contex select { case <-ctx.Done(): return - case <-time.After(resyncInterval): + case <-time.After(awsEventBridge.resyncInterval): awsEventBridge.Printf("syncing pattern map for rule ARN %s", ruleARN) err := awsEventBridge.syncRules(ctx, eventbus, ruleARN) if err != nil { awsEventBridge.Printf("could not sync pattern map for rule ARN %s: %v", ruleARN, err) - awsEventBridge.Printf("retrying after %v", resyncInterval) + awsEventBridge.Printf("retrying after %v", awsEventBridge.resyncInterval) } awsEventBridge.Printf("successfully synced pattern map for rule ARN %s", ruleARN) } @@ -284,7 +259,7 @@ func (awsEventBridge *awsEventBridgeProcessor) syncRules(ctx context.Context, ev for !found { rules, err := awsEventBridge.ListRulesWithContext(ctx, &eventbridge.ListRulesInput{ EventBusName: aws.String(eventbus), // explicitely passing eventbus name because list assumes "default" otherwise - Limit: aws.Int64(pageLimit), + Limit: aws.Int64(defaultPageLimit), NextToken: nextToken, }) if err != nil { @@ -337,7 +312,7 @@ func (awsEventBridge *awsEventBridgeProcessor) syncRules(ctx context.Context, ev return nil } -func (awsEventBridge *awsEventBridgeProcessor) PushMetrics(ctx context.Context, ms *metrics.Server) { +func (awsEventBridge *awsEventBridgeProcessor) PushMetrics(ctx context.Context, ms metrics.Receiver) { ticker := time.NewTicker(metrics.PushInterval) defer ticker.Stop() diff --git a/vmware-event-router/internal/processor/aws_event_bridge_test.go b/vmware-event-router/internal/processor/aws_event_bridge_test.go deleted file mode 100644 index 5e2b1d22..00000000 --- a/vmware-event-router/internal/processor/aws_event_bridge_test.go +++ /dev/null @@ -1,143 +0,0 @@ -package processor - -import ( - "testing" - - "github.com/vmware-samples/vcenter-event-broker-appliance/vmware-event-router/internal/metrics" - "github.com/vmware/govmomi/vim25/types" -) - -func Test_batching_createPutEventsInput(t *testing.T) { - tests := []struct { - title string - baseEvents []types.BaseEvent - desiredBatches int - desiredEntries int - desiredType string - }{ - { - title: "13 VmPoweredOnEvent events 2 batches", - baseEvents: baseEventsMockVMPoweredOn(13), - desiredBatches: 2, - desiredEntries: 13, - desiredType: "VmPoweredOnEvent", - }, - { - title: "10 VmPoweredOnEvent events 1 batch", - baseEvents: baseEventsMockVMPoweredOn(10), - desiredBatches: 1, - desiredEntries: 10, - desiredType: "VmPoweredOnEvent", - }, - { - title: "3 VmPoweredOnEvent events 1 batch", - baseEvents: baseEventsMockVMPoweredOn(3), - desiredBatches: 1, - desiredEntries: 3, - desiredType: "VmPoweredOnEvent", - }, - { - title: "23 VmPoweredOnEvent events 3 batches", - baseEvents: baseEventsMockVMPoweredOn(23), - desiredBatches: 3, - desiredEntries: 23, - desiredType: "VmPoweredOnEvent", - }, - { - title: "0 events 0 batches unsubscribed event", - baseEvents: baseEventsMockCustomizedDVPortEvent(23), - desiredBatches: 0, - desiredEntries: 0, - desiredType: "", - }, - { - title: "5 VmPoweredOnEvent events 1 batches", - baseEvents: baseEventsMockHalfVMPoweredOn(10), - desiredBatches: 1, - desiredEntries: 5, - desiredType: "VmPoweredOnEvent", - }, - { - title: "0 events 0 batches", - baseEvents: []types.BaseEvent{}, - desiredBatches: 0, - desiredEntries: 0, - desiredType: "", - }, - } - for _, test := range tests { - t.Run(test.title, func(t *testing.T) { - awsEventBridgeStub := createAWSObjectStubVMPoweredOn() - batchEvents, err := awsEventBridgeStub.createPutEventsInput(test.baseEvents) - if err != nil { - t.Errorf("unexpected error: %s", err.Error()) - } - actualBatches := 0 - actualEntries := 0 - for _, v := range batchEvents { - for _, entry := range v.Entries { - actualEntries++ - if test.desiredType != *entry.DetailType { - t.Errorf("wanted entry type: %s got: %s", - test.desiredType, - *entry.DetailType) - } - } - actualBatches++ - } - if test.desiredBatches != actualBatches { - t.Errorf("wanted: %v batches got: %v", - test.desiredBatches, - actualBatches) - } - if test.desiredEntries != actualEntries { - t.Errorf("wanted: %v entries got: %v", - test.desiredEntries, - actualEntries) - } - }) - } -} - -func baseEventsMockVMPoweredOn(numberOfEvents int) []types.BaseEvent { - baseEvents := []types.BaseEvent{} - for numberOfEvents > 0 { - numberOfEvents = numberOfEvents - 1 - baseEvents = append(baseEvents, &types.VmPoweredOnEvent{}) - } - return baseEvents -} - -func baseEventsMockCustomizedDVPortEvent(numberOfEvents int) []types.BaseEvent { - baseEvents := []types.BaseEvent{} - for numberOfEvents > 0 { - numberOfEvents = numberOfEvents - 1 - baseEvents = append(baseEvents, &types.VmPoweringOnWithCustomizedDVPortEvent{}) - } - return baseEvents -} - -func baseEventsMockHalfVMPoweredOn(numberOfEvents int) []types.BaseEvent { - baseEvents := []types.BaseEvent{} - switchFlag := true - for numberOfEvents > 0 { - if switchFlag { - baseEvents = append(baseEvents, &types.VmPoweringOnWithCustomizedDVPortEvent{}) - switchFlag = false - } else { - baseEvents = append(baseEvents, &types.VmPoweredOnEvent{}) - switchFlag = true - } - numberOfEvents = numberOfEvents - 1 - } - return baseEvents -} - -func createAWSObjectStubVMPoweredOn() awsEventBridgeProcessor { - return awsEventBridgeProcessor{ - patternMap: map[string]string{"VmPoweredOnEvent": ""}, - stats: metrics.EventStats{ - Invocations: make(map[string]int), - }, - } -} diff --git a/vmware-event-router/internal/processor/aws_option.go b/vmware-event-router/internal/processor/aws_option.go new file mode 100644 index 00000000..46ea0081 --- /dev/null +++ b/vmware-event-router/internal/processor/aws_option.go @@ -0,0 +1,38 @@ +package processor + +import ( + "log" + "time" +) + +// AWSOption configures the AWS processor +type AWSOption func(*awsEventBridgeProcessor) + +// WithAWSVerbose enables verbose logging for the AWS processor +func WithAWSVerbose(verbose bool) AWSOption { + return func(aws *awsEventBridgeProcessor) { + aws.verbose = verbose + } +} + +// WithAWSLogger sets an alternative logger for the AWS processor +func WithAWSLogger(logger *log.Logger) AWSOption { + return func(aws *awsEventBridgeProcessor) { + aws.Logger = logger + } +} + +// WithAWSResyncInterval configures the interval to sync AWS EventBridge event +// pattern rules +func WithAWSResyncInterval(interval time.Duration) AWSOption { + return func(aws *awsEventBridgeProcessor) { + aws.resyncInterval = interval + } +} + +// WithAWSBatchSize sets the batch size for PutEvents requests +func WithAWSBatchSize(size int) AWSOption { + return func(aws *awsEventBridgeProcessor) { + aws.batchSize = size + } +} diff --git a/vmware-event-router/internal/processor/openfaas.go b/vmware-event-router/internal/processor/openfaas.go index 9fc078a2..2c05fe47 100644 --- a/vmware-event-router/internal/processor/openfaas.go +++ b/vmware-event-router/internal/processor/openfaas.go @@ -3,37 +3,51 @@ package processor import ( "context" "encoding/json" + "fmt" "log" + "net/http" "os" "sync" "time" - sdk "github.com/openfaas-incubator/connector-sdk/types" + cloudevents "github.com/cloudevents/sdk-go/v2" + ofsdk "github.com/openfaas-incubator/connector-sdk/types" "github.com/openfaas/faas-provider/auth" "github.com/pkg/errors" "github.com/vmware-samples/vcenter-event-broker-appliance/vmware-event-router/internal/color" "github.com/vmware-samples/vcenter-event-broker-appliance/vmware-event-router/internal/connection" - "github.com/vmware-samples/vcenter-event-broker-appliance/vmware-event-router/internal/events" "github.com/vmware-samples/vcenter-event-broker-appliance/vmware-event-router/internal/metrics" - - "github.com/vmware/govmomi/vim25/types" ) const ( - // ProviderOpenFaaS is the name used to identify this provider in the - // VMware Event Router configuration file - ProviderOpenFaaS = "openfaas" - topicDelimiter = "," - rebuildInterval = time.Second * 10 - timeout = time.Second * 15 - authMethodOpenFaaS = "basic_auth" // only this method is supported by the processor + // ProviderOpenFaaS variable is the name used to identify this provider in + // the VMware Event Router configuration file + ProviderOpenFaaS = "openfaas" + authMethodOpenFaaS = "basic_auth" // only this method is supported by the processor + defaultTopicDelimiter = "," + defaultRebuildInterval = time.Second * 10 + defaultTimeout = time.Second * 15 ) +// responseFunc implements ResponseSubscriber and is used to configure the +// default response handler for the OpenFaaS processor +type responseFunc func(ofsdk.InvokerResponse) + +func (r responseFunc) Response(res ofsdk.InvokerResponse) { + r(res) +} + // openfaasProcessor implements the Processor interface type openfaasProcessor struct { - controller sdk.Controller - source string - verbose bool + controller ofsdk.Controller + ofsdk.ResponseSubscriber + + // options + verbose bool + topicDelimiter string + rebuildInterval time.Duration + gatewayTimeout time.Duration + // TODO (@embano1): make log interface for all processors/streams *log.Logger lock sync.RWMutex @@ -43,12 +57,20 @@ type openfaasProcessor struct { // NewOpenFaaSProcessor returns an OpenFaaS processor for the given stream // source. Asynchronous function invokation can be configured for // high-throughput (non-blocking) requirements. -func NewOpenFaaSProcessor(ctx context.Context, cfg connection.Config, source string, verbose bool, ms *metrics.Server) (Processor, error) { +func NewOpenFaaSProcessor(ctx context.Context, cfg connection.Config, ms metrics.Receiver, opts ...OpenFaaSOption) (Processor, error) { + // defaults logger := log.New(os.Stdout, color.Purple("[OpenFaaS] "), log.LstdFlags) - openfaas := openfaasProcessor{ - source: source, - verbose: verbose, - Logger: logger, + ofProcessor := openfaasProcessor{ + topicDelimiter: defaultTopicDelimiter, + rebuildInterval: defaultRebuildInterval, + gatewayTimeout: defaultTimeout, + Logger: logger, + } + ofProcessor.ResponseSubscriber = defaultResponseHandler(&ofProcessor) + + // apply options + for _, opt := range opts { + opt(&ofProcessor) } var creds auth.BasicAuthCredentials @@ -64,94 +86,83 @@ func NewOpenFaaSProcessor(ctx context.Context, cfg connection.Config, source str if cfg.Options["async"] == "true" { async = true } - ofconfig := sdk.ControllerConfig{ + ofconfig := ofsdk.ControllerConfig{ GatewayURL: cfg.Address, - TopicAnnotationDelimiter: topicDelimiter, - RebuildInterval: rebuildInterval, - UpstreamTimeout: timeout, + TopicAnnotationDelimiter: ofProcessor.topicDelimiter, + RebuildInterval: ofProcessor.rebuildInterval, + UpstreamTimeout: ofProcessor.gatewayTimeout, AsyncFunctionInvocation: async, - PrintSync: verbose, + PrintSync: ofProcessor.verbose, } - ofcontroller := sdk.NewController(&creds, &ofconfig) - - openfaas.controller = ofcontroller - openfaas.controller.Subscribe(&openfaas) - openfaas.controller.BeginMapBuilder() + ofcontroller := ofsdk.NewController(&creds, &ofconfig) + ofProcessor.controller = ofcontroller + ofProcessor.controller.Subscribe(&ofProcessor) + ofProcessor.controller.BeginMapBuilder() // prepopulate the metrics stats - openfaas.stats = metrics.EventStats{ + ofProcessor.stats = metrics.EventStats{ Provider: ProviderOpenFaaS, ProviderType: cfg.Type, Name: cfg.Address, Started: time.Now().UTC(), Invocations: make(map[string]int), } - go openfaas.PushMetrics(ctx, ms) + go ofProcessor.PushMetrics(ctx, ms) - return &openfaas, nil + return &ofProcessor, nil } -// Response prints status information for each function invokation -func (openfaas *openfaasProcessor) Response(res sdk.InvokerResponse) { - // update stats - // TODO: currently we only support metrics when in sync invokation mode - // because we don't have a callback for async invocations - openfaas.lock.Lock() - openfaas.stats.Invocations[res.Topic]++ - openfaas.lock.Unlock() - - if res.Error != nil { - openfaas.Printf("function %s for topic %s returned status %d with error: %v", res.Function, res.Topic, res.Status, res.Error) - return +// defaultResponseHandler prints status information for each function invokation +func defaultResponseHandler(openfaas *openfaasProcessor) responseFunc { + return func(res ofsdk.InvokerResponse) { + // update stats + // TODO: currently we only support metrics when in sync invokation mode + // because we don't have a callback for async invocations + openfaas.lock.Lock() + openfaas.stats.Invocations[res.Topic]++ + openfaas.lock.Unlock() + + if res.Error != nil || res.Status != http.StatusOK { + openfaas.Printf("function %s for topic %s returned status %d with error: %v", res.Function, res.Topic, res.Status, res.Error) + return + } + openfaas.Printf("successfully invoked function %s for topic %s", res.Function, res.Topic) } - openfaas.Printf("successfully invoked function %s for topic %s", res.Function, res.Topic) } // Process implements the stream processor interface -func (openfaas *openfaasProcessor) Process(moref types.ManagedObjectReference, baseEvent []types.BaseEvent) error { - for idx := range baseEvent { - // process slice in reverse order to maintain Event.Key ordering - event := baseEvent[len(baseEvent)-1-idx] - - if openfaas.verbose { - openfaas.Printf("processing event [%d] of type %T from source %s: %+v", idx, event, openfaas.source, event) - } - - topic, message, err := handleEvent(event, openfaas.source) - if err != nil { - openfaas.Printf("error handling event: %v", err) - continue - } +func (openfaas *openfaasProcessor) Process(ce cloudevents.Event) error { + if openfaas.verbose { + openfaas.Printf("processing event (ID %s): %v", ce.ID(), ce) + } - if openfaas.verbose { - openfaas.Printf("created new outbound cloud event for subscribers: %s", string(message)) - } + topic, message, err := handleEvent(ce) + if err != nil { + msg := fmt.Errorf("error handling event %v: %v", ce, err) + openfaas.Println(msg) + return processorError(ProviderOpenFaaS, msg) + } - openfaas.Printf("invoking function(s) on topic: %s", topic) - openfaas.controller.Invoke(topic, &message) + if openfaas.verbose { + openfaas.Printf("created new outbound event for subscribers: %s", string(message)) } + + openfaas.Printf("invoking function(s) for event %s on topic: %s", ce.ID(), topic) + openfaas.controller.Invoke(topic, &message) return nil } // handleEvent returns the OpenFaaS subscription topic, e.g. VmPoweredOnEvent, -// and outbound event message for the given BaseEvent and source -func handleEvent(event types.BaseEvent, source string) (string, []byte, error) { - // Sanity check to avoid nil pointer exception - if event == nil { - return "", nil, errors.New("source event must not be nil") - } - - // Get the category and name of the event used for subscribed topic matching - eventInfo := events.GetDetails(event) - cloudEvent := events.NewCloudEvent(event, eventInfo, source) - message, err := json.Marshal(cloudEvent) +// and outbound event message ([]byte(CloudEvent) for the given CloudEvent +func handleEvent(event cloudevents.Event) (string, []byte, error) { + message, err := json.Marshal(event) if err != nil { - return "", nil, errors.Wrapf(err, "could not marshal cloud event for vSphere event %d from source %s", event.GetEvent().Key, source) + return "", nil, errors.Wrapf(err, "could not JSON-encode CloudEvent %v", event) } - return eventInfo.Name, message, nil + return event.Subject(), message, nil } -func (openfaas *openfaasProcessor) PushMetrics(ctx context.Context, ms *metrics.Server) { +func (openfaas *openfaasProcessor) PushMetrics(ctx context.Context, ms metrics.Receiver) { ticker := time.NewTicker(metrics.PushInterval) defer ticker.Stop() for { diff --git a/vmware-event-router/internal/processor/openfaas_option.go b/vmware-event-router/internal/processor/openfaas_option.go new file mode 100644 index 00000000..15578c9b --- /dev/null +++ b/vmware-event-router/internal/processor/openfaas_option.go @@ -0,0 +1,57 @@ +package processor + +import ( + "log" + "time" + + ofsdk "github.com/openfaas-incubator/connector-sdk/types" +) + +// OpenFaaSOption configures the OpenFaaS processor +type OpenFaaSOption func(*openfaasProcessor) + +// WithOpenFaaSVerbose enables verbose logging for the OpenFaaS processor +func WithOpenFaaSVerbose(verbose bool) OpenFaaSOption { + return func(o *openfaasProcessor) { + o.verbose = verbose + } +} + +// WithOpenFaaSLogger sets an alternative logger for the OpenFaaS processor +func WithOpenFaaSLogger(logger *log.Logger) OpenFaaSOption { + return func(o *openfaasProcessor) { + o.Logger = logger + } +} + +// WithOpenFaaSDelimiter changes the default topic delimiter (comma-separated +// strings) for the OpenFaaS processor +func WithOpenFaaSDelimiter(delim string) OpenFaaSOption { + return func(o *openfaasProcessor) { + o.topicDelimiter = delim + } +} + +// WithOpenFaaSTimeout changes the default gateway timeout for the OpenFaaS +// processor +func WithOpenFaaSTimeout(timeout time.Duration) OpenFaaSOption { + return func(o *openfaasProcessor) { + o.gatewayTimeout = timeout + } +} + +// WithOpenFaaSRebuildInterval changes the default gateway topic synchronization +// interval for the OpenFaaS processor +func WithOpenFaaSRebuildInterval(interval time.Duration) OpenFaaSOption { + return func(o *openfaasProcessor) { + o.rebuildInterval = interval + } +} + +// WithOpenFaaSResponseHandler sets an alternative response handler for the +// OpenFaaS processor +func WithOpenFaaSResponseHandler(handler ofsdk.ResponseSubscriber) OpenFaaSOption { + return func(o *openfaasProcessor) { + o.ResponseSubscriber = handler + } +} diff --git a/vmware-event-router/internal/processor/processor.go b/vmware-event-router/internal/processor/processor.go index 12d0c23f..f5202369 100644 --- a/vmware-event-router/internal/processor/processor.go +++ b/vmware-event-router/internal/processor/processor.go @@ -3,15 +3,13 @@ package processor import ( "fmt" - "github.com/vmware/govmomi/vim25/types" + cloudevents "github.com/cloudevents/sdk-go/v2" ) -// Processor handles incoming vCenter events. This enables different FaaS -// implementations for vCenter event processing. Note: in the case of processing -// failure the current behavior is to log but return nil until at-least-once -// semantics are implemented. +// Processor handles incoming stream events to decouple event stream providers, +// e.g. vCenter, from processors, e.g. OpenFaaS, knative, AWS EventBridge, etc. type Processor interface { - Process(types.ManagedObjectReference, []types.BaseEvent) error + Process(cloudevents.Event) error } // Error struct contains the generic error content used by the processors diff --git a/vmware-event-router/internal/stream/fake/vcenter_fake.go b/vmware-event-router/internal/stream/fake/vcenter_fake.go new file mode 100644 index 00000000..a0dfb47e --- /dev/null +++ b/vmware-event-router/internal/stream/fake/vcenter_fake.go @@ -0,0 +1,68 @@ +package fake + +import ( + "context" + "log" + "os" + + "github.com/vmware-samples/vcenter-event-broker-appliance/vmware-event-router/internal/color" + "github.com/vmware-samples/vcenter-event-broker-appliance/vmware-event-router/internal/events" + "github.com/vmware-samples/vcenter-event-broker-appliance/vmware-event-router/internal/metrics" + "github.com/vmware-samples/vcenter-event-broker-appliance/vmware-event-router/internal/processor" + "github.com/vmware-samples/vcenter-event-broker-appliance/vmware-event-router/internal/stream" + "github.com/vmware/govmomi/vim25/types" +) + +const source = "https://fake.vcenter01.testing.io/sdk" + +// verify that FakeVCenter implements the streamer interface +var _ stream.Streamer = (*FakeVCenter)(nil) + +// FakeVCenter implements the streamer interface +type FakeVCenter struct { + eventCh <-chan []types.BaseEvent // channel which simulates events + *log.Logger +} + +// NewFakeVCenter returns a fake vcenter event stream provider streaming events +// received from the specified generator channel +func NewFakeVCenter(generator <-chan []types.BaseEvent) *FakeVCenter { + return &FakeVCenter{ + eventCh: generator, + Logger: log.New(os.Stdout, color.Magenta("[Fake vCenter] "), log.LstdFlags), + } +} + +// PushMetrics is a no-op +func (f *FakeVCenter) PushMetrics(context.Context, metrics.Receiver) {} + +// Stream streams events generated by the Generator specified in the FakeVCenter +// server +func (f *FakeVCenter) Stream(ctx context.Context, p processor.Processor) error { + for { + select { + case <-ctx.Done(): + return nil + case baseEvent := <-f.eventCh: + for idx := range baseEvent { + // process slice in reverse order to maintain Event.Key ordering + event := baseEvent[len(baseEvent)-1-idx] + + ce, err := events.NewCloudEvent(event, source) + if err != nil { + log.Printf("skipping event %v because it could not be converted to CloudEvent format: %v", event, err) + continue + } + err = p.Process(*ce) + if err != nil { + f.Logger.Printf("could not process event %v: %v", ce, err) + } + } + } + } +} + +// Shutdown is a no-op +func (f *FakeVCenter) Shutdown(context.Context) error { + return nil +} diff --git a/vmware-event-router/internal/stream/fake/vcenter_fake_test.go b/vmware-event-router/internal/stream/fake/vcenter_fake_test.go new file mode 100644 index 00000000..1cec47ab --- /dev/null +++ b/vmware-event-router/internal/stream/fake/vcenter_fake_test.go @@ -0,0 +1,168 @@ +// +build unit + +package fake + +import ( + "context" + "testing" + "time" + + cloudevents "github.com/cloudevents/sdk-go/v2" + "github.com/vmware/govmomi/vim25/types" +) + +type noOpProcessor struct { + invokations int +} + +func (n *noOpProcessor) Process(event cloudevents.Event) error { + n.invokations++ + return nil +} + +func TestFakeVCenter_Stream(t *testing.T) { + type fields struct { + ctxTimeout time.Duration // used to shutdown the stream + genDelay *time.Duration // delay for generating events + events [][]types.BaseEvent // events to send and expected to receive + } + type args struct { + p *noOpProcessor + } + tests := []struct { + name string + fields fields + args args + wantEvents int + wantErr bool + }{ + { + name: "receive 8 events with context timeout of 200ms", + fields: fields{ + ctxTimeout: 200 * time.Millisecond, + genDelay: makeDelay(50 * time.Millisecond), + events: [][]types.BaseEvent{createVMPoweredOnEvents(3), createVMPoweredOnEvents(5)}, + }, + wantEvents: 8, + args: args{ + &noOpProcessor{}, + }, + }, + { + name: "expect no events with context timeout of 200ms", + fields: fields{ + ctxTimeout: 200 * time.Millisecond, + genDelay: makeDelay(50 * time.Millisecond), + events: nil, + }, + wantEvents: 0, + args: args{ + &noOpProcessor{}, + }, + }, + { + name: "expect no events since no delay is specified", + fields: fields{ + ctxTimeout: 200 * time.Millisecond, + genDelay: nil, + events: [][]types.BaseEvent{}, + }, + wantEvents: 0, + args: args{ + &noOpProcessor{}, + }, + }, + { + name: "expect no events since no events are specified", + fields: fields{ + ctxTimeout: 200 * time.Millisecond, + genDelay: makeDelay(50 * time.Millisecond), + events: nil, + }, + args: args{ + &noOpProcessor{}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), tt.fields.ctxTimeout) + defer cancel() + + eventCh := createGenerator(ctx, tt.fields.genDelay, tt.fields.events) + f := NewFakeVCenter(eventCh) + _ = f.Stream(ctx, tt.args.p) // ignore errors + + got := tt.args.p.invokations + want := tt.wantEvents + if got != want { + t.Errorf("FakeVCenter.Stream() invokations = %d, wanted %d", got, want) + } + }) + } +} + +// the returned generator will send each []types.BaseEvent in the +// [][]types.BaseEvent slice via the returned channel. If neither a delay for +// the time between sending these events is specified or no events are specified +// at all a nil channel is returned which blocks forever +func createGenerator(ctx context.Context, delay *time.Duration, events [][]types.BaseEvent) <-chan []types.BaseEvent { + if events == nil || delay == nil { + return nil + } + + genCh := make(chan []types.BaseEvent, len(events)) + go func() { + for { + select { + case <-ctx.Done(): + return + case <-time.After(*delay): + // if we consumed all events set generator channel to block + // forever and return + if len(events) == 0 { + genCh = nil + return + } + + baseEvent := events[0] + genCh <- baseEvent + events = events[1:] + } + } + }() + + return genCh +} + +func createVMPoweredOnEvents(count int) []types.BaseEvent { + if count == 0 { + return nil + } + + var events []types.BaseEvent + for i := 0; i < count; i++ { + e := createVMPowerOnEvent() + events = append(events, &e) + } + return events +} + +func createVMPowerOnEvent() types.VmPoweredOnEvent { + return types.VmPoweredOnEvent{ + VmEvent: types.VmEvent{ + Event: types.Event{ + Vm: &types.VmEventArgument{ + Vm: types.ManagedObjectReference{ + Type: "VirtualMachine", + Value: "vm-1234", + }, + }, + }, + }, + } +} + +func makeDelay(t time.Duration) *time.Duration { + return &t +} diff --git a/vmware-event-router/internal/stream/stream.go b/vmware-event-router/internal/stream/stream.go index 6db2307a..7af6d660 100644 --- a/vmware-event-router/internal/stream/stream.go +++ b/vmware-event-router/internal/stream/stream.go @@ -10,8 +10,7 @@ import ( // Streamer establishes a connection to a stream provider and invokes a stream // processor. type Streamer interface { - PushMetrics(context.Context, *metrics.Server) + PushMetrics(context.Context, metrics.Receiver) Stream(context.Context, processor.Processor) error Shutdown(context.Context) error - Source() string } diff --git a/vmware-event-router/internal/stream/vcenter.go b/vmware-event-router/internal/stream/vcenter.go index 6312d535..0c19cfa7 100644 --- a/vmware-event-router/internal/stream/vcenter.go +++ b/vmware-event-router/internal/stream/vcenter.go @@ -2,13 +2,17 @@ package stream import ( "context" + "log" "math" "net/url" + "os" "sync" "time" "github.com/pkg/errors" + "github.com/vmware-samples/vcenter-event-broker-appliance/vmware-event-router/internal/color" "github.com/vmware-samples/vcenter-event-broker-appliance/vmware-event-router/internal/connection" + "github.com/vmware-samples/vcenter-event-broker-appliance/vmware-event-router/internal/events" "github.com/vmware-samples/vcenter-event-broker-appliance/vmware-event-router/internal/metrics" "github.com/vmware-samples/vcenter-event-broker-appliance/vmware-event-router/internal/processor" "github.com/vmware/govmomi" @@ -28,19 +32,28 @@ const ( type vCenterStream struct { client govmomi.Client stream event.Manager + *log.Logger + verbose bool lock sync.RWMutex stats metrics.EventStats } // NewVCenterStream returns a vCenter event manager for a given configuration and metrics server -func NewVCenterStream(ctx context.Context, cfg connection.Config, ms *metrics.Server) (Streamer, error) { +func NewVCenterStream(ctx context.Context, cfg connection.Config, ms metrics.Receiver, opts ...VCenterOption) (Streamer, error) { var vCenter vCenterStream + logger := log.New(os.Stdout, color.Magenta("[vCenter] "), log.LstdFlags) + vCenter.Logger = logger parsedURL, err := soap.ParseURL(cfg.Address) if err != nil { return nil, errors.Wrap(err, "error parsing URL") } + // apply options + for _, opt := range opts { + opt(&vCenter) + } + var username, password string switch cfg.Auth.Method { case authMethodvSphere: @@ -71,6 +84,7 @@ func NewVCenterStream(ctx context.Context, cfg connection.Config, ms *metrics.Se Name: client.URL().String(), Started: time.Now().UTC(), EventsTotal: new(int), + EventsErr: new(int), EventsSec: new(float64), } go vCenter.PushMetrics(ctx, ms) @@ -103,7 +117,7 @@ func (vcenter *vCenterStream) Source() string { return vcenter.client.URL().String() } -func (vcenter *vCenterStream) PushMetrics(ctx context.Context, ms *metrics.Server) { +func (vcenter *vCenterStream) PushMetrics(ctx context.Context, ms metrics.Receiver) { ticker := time.NewTicker(metrics.PushInterval) defer ticker.Stop() for { @@ -124,12 +138,35 @@ func (vcenter *vCenterStream) PushMetrics(ctx context.Context, ms *metrics.Serve // processor func (vcenter *vCenterStream) streamCallbackFn(p processor.Processor) func(types.ManagedObjectReference, []types.BaseEvent) error { return func(moref types.ManagedObjectReference, baseEvent []types.BaseEvent) error { - // update stats before invoking the processor - vcenter.lock.Lock() - total := *vcenter.stats.EventsTotal + len(baseEvent) - vcenter.stats.EventsTotal = &total - vcenter.lock.Unlock() - - return p.Process(moref, baseEvent) + var errCount int + + // update stats + defer func() { + vcenter.lock.Lock() + total := *vcenter.stats.EventsTotal + len(baseEvent) + vcenter.stats.EventsTotal = &total + errTotal := *vcenter.stats.EventsErr + errCount + vcenter.stats.EventsErr = &errTotal + vcenter.lock.Unlock() + }() + + for idx := range baseEvent { + // process slice in reverse order to maintain Event.Key ordering + event := baseEvent[len(baseEvent)-1-idx] + + ce, err := events.NewCloudEvent(event, vcenter.Source()) + if err != nil { + vcenter.Logger.Printf("skipping event %v because it could not be converted to CloudEvent format: %v", event, err) + errCount++ + continue + } + + err = p.Process(*ce) + if err != nil { + vcenter.Logger.Printf("could not process event %v: %v", ce, err) + errCount++ + } + } + return nil } } diff --git a/vmware-event-router/internal/stream/vcenter_option.go b/vmware-event-router/internal/stream/vcenter_option.go new file mode 100644 index 00000000..47ddf79e --- /dev/null +++ b/vmware-event-router/internal/stream/vcenter_option.go @@ -0,0 +1,20 @@ +package stream + +import "log" + +// VCenterOption configures the vCenter stream provider +type VCenterOption func(*vCenterStream) + +// WithVCenterVerbose enables verbose logging for the AWS processor +func WithVCenterVerbose(verbose bool) VCenterOption { + return func(vc *vCenterStream) { + vc.verbose = verbose + } +} + +// WithVCenterLogger sets an alternative logger for the vCenter stream provider +func WithVCenterLogger(logger *log.Logger) VCenterOption { + return func(vc *vCenterStream) { + vc.Logger = logger + } +} \ No newline at end of file From 7fbf2cb9525eadf461ddefbcfd4ef89191b69b5d Mon Sep 17 00:00:00 2001 From: Michael Gasch Date: Tue, 2 Jun 2020 09:55:51 +0200 Subject: [PATCH 03/86] Update Docker images used in VMware Event Router Dockerfile build and deploy stage will use latest available Docker images for Golang/Debian as of filing this commit. Signed-off-by: Michael Gasch --- vmware-event-router/Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/vmware-event-router/Dockerfile b/vmware-event-router/Dockerfile index b5f176ae..1511c03d 100644 --- a/vmware-event-router/Dockerfile +++ b/vmware-event-router/Dockerfile @@ -1,5 +1,5 @@ -# golang:1.13.7-stretch -FROM golang@sha256:d16d1e0b4021e15dfc17dc58e794ad1e794155abf74c579a4c7b8a2c83ff8682 AS builder +# golang:1.14.4-stretch amd64 +FROM golang@sha256:8b6b86e45c7847fea9cf401da52e41747cfb18bbfb4aebe924b4ef13a3366521 AS builder ARG VERSION ARG COMMIT @@ -18,8 +18,8 @@ COPY internal internal RUN ./bin/golangci-lint run ./... RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix nocgo -ldflags="-X main.version=${VERSION} -X main.commit=${COMMIT}" -o vmware-event-router cmd/main.go -# debian:stable-slim -FROM debian@sha256:55f6837fd25a8e2ab94790bc73e762ffeb0d5d2e510177aef76101d2822d5937 +# debian:stable-slim amd64 +FROM debian@sha256:aa1db351593d2849034c0395bc604dd65aa80ae4471da4a37a0c38e30aed3ab8 ARG VERSION ARG COMMIT LABEL maintainer="mgasch@vmware.com" \ From 5fc30e20676c9578bc89fe08fc76971d5933fc87 Mon Sep 17 00:00:00 2001 From: Michael Gasch Date: Wed, 3 Jun 2020 00:31:13 +0200 Subject: [PATCH 04/86] Add faas-cli version to BOM Signed-off-by: Michael Gasch --- veba-bom.json | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/veba-bom.json b/veba-bom.json index f40b7433..3cd1bb21 100644 --- a/veba-bom.json +++ b/veba-bom.json @@ -92,7 +92,10 @@ "name" : "nats-streaming", "version" : "0.11.2" } - ] + ], + "faas-cli": { + "version": "0.12.4" + } }, "tinywww" :{ "version" : "latest", From 8895230e9746538d62d7adaa5c47a8cb8d61ceb6 Mon Sep 17 00:00:00 2001 From: William Lam Date: Sun, 24 May 2020 05:40:13 -0700 Subject: [PATCH 05/86] Refactor VEBA components to reference BOM file Signed-off-by: William Lam --- build.sh | 18 ++++++++++++-- files/kubeconfig.yml | 6 ----- files/setup-04-kubernetes.sh | 19 ++++++++++----- photon-version.json | 1 - photon.json | 10 ++++---- scripts/photon-containers.sh | 46 ++++++++++++++---------------------- scripts/photon-settings.sh | 3 ++- 7 files changed, 54 insertions(+), 49 deletions(-) delete mode 100644 files/kubeconfig.yml diff --git a/build.sh b/build.sh index f7fb46e7..c0e9846e 100755 --- a/build.sh +++ b/build.sh @@ -4,6 +4,18 @@ set -euo pipefail +VEBA_BOM_FILE=veba-bom.json + +if [ ! -e ${VEBA_BOM_FILE} ]; then + echo "Unable to locate veba-bom.json in current directory which is required" + exit 1 +fi + +#command -v jqw > /dev/null 2>&1 +if [ ! -e /usr/local/bin/jq ]; then + echo "jq utility is not installed on this system" + exit 1 +fi if [ $# -ne 1 ]; then echo -e "\n\tUsage: $0 [master|release]\n" @@ -17,12 +29,14 @@ fi rm -f output-vmware-iso/*.ova +VEBA_VERSION_FROM_BOM=$(jq -r < ${VEBA_BOM_FILE} '.veba.version') + if [ "$1" == "release" ]; then echo "Building VEBA OVA release ..." - packer build -var "VEBA_VERSION=$(cat VERSION)-release" -var "VEBA_COMMIT=$(git rev-parse --short HEAD)" -var-file=photon-builder.json -var-file=photon-version.json photon.json + packer build -var "VEBA_VERSION=${VEBA_VERSION_FROM_BOM}-release" -var "VEBA_COMMIT=$(git rev-parse --short HEAD)" -var-file=photon-builder.json -var-file=photon-version.json photon.json elif [ "$1" == "master" ]; then echo "Building VEBA OVA master ..." - packer build -var "VEBA_VERSION=$(cat VERSION)" -var "VEBA_COMMIT=$(git rev-parse --short HEAD)" -var-file=photon-builder.json -var-file=photon-version.json photon.json + packer build -var "VEBA_VERSION=${VEBA_VERSION_FROM_BOM}" -var "VEBA_COMMIT=$(git rev-parse --short HEAD)" -var-file=photon-builder.json -var-file=photon-version.json photon.json else echo -e "\nPlease specify release or master to build ...\n" fi \ No newline at end of file diff --git a/files/kubeconfig.yml b/files/kubeconfig.yml deleted file mode 100644 index 4e5c527c..00000000 --- a/files/kubeconfig.yml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: kubeadm.k8s.io/v1beta1 -kind: InitConfiguration ---- -apiVersion: kubeadm.k8s.io/v1beta1 -kind: ClusterConfiguration -kubernetesVersion: v1.14.9 \ No newline at end of file diff --git a/files/setup-04-kubernetes.sh b/files/setup-04-kubernetes.sh index 24053855..824cc91b 100755 --- a/files/setup-04-kubernetes.sh +++ b/files/setup-04-kubernetes.sh @@ -6,6 +6,8 @@ set -euo pipefail +VEBA_BOM_FILE=/root/config/veba-bom.json + echo -e "\e[92mStarting Docker ..." > /dev/console systemctl daemon-reload systemctl start docker.service @@ -20,14 +22,19 @@ if [ -z "${POD_NETWORK_CIDR}" ]; then POD_NETWORK_CIDR="10.16.0.0/16" fi -cat >> /root/config/kubeconfig.yml << EOF - -networking: - podSubnet: ${POD_NETWORK_CIDR} -EOF - # Setup k8s echo -e "\e[92mSetting up k8s ..." > /dev/console +K8S_VERSION=$(jq -r < ${VEBA_BOM_FILE} '.["kubernetes"].version') +cat > /root/config/kubeconfig.yml << __EOF__ +apiVersion: kubeadm.k8s.io/v1beta1 +kind: InitConfiguration +--- +apiVersion: kubeadm.k8s.io/v1beta1 +kind: ClusterConfiguration +kubernetesVersion: ${K8S_VERSION} +networking: + podSubnet: ${POD_NETWORK_CIDR} +__EOF__ echo -e "\e[92mDeloying kubeadm ..." > /dev/console HOME=/root diff --git a/photon-version.json b/photon-version.json index 27068b60..3c1515e1 100644 --- a/photon-version.json +++ b/photon-version.json @@ -1,5 +1,4 @@ { - "version": "0.4.0", "description": "Photon Build for vCenter Event Broker Appliance", "vm_name": "vCenter_Event_Broker_Appliance", "iso_checksum": "f6619bcff94cef63d0d6d7ead7dd3878816ebfa6a1ef5717175bb0d08d4ccc719e4ec7daa7db3c5dc07ea3547fc24412b4dc6827a4ac332ada9d5bfc842c4229", diff --git a/photon.json b/photon.json index f3e8e1f0..c9503f16 100644 --- a/photon.json +++ b/photon.json @@ -61,6 +61,11 @@ "scripts/photon-docker.sh" ] }, + { + "type": "file", + "source": "veba-bom.json", + "destination": "/root/config/veba-bom.json" + }, { "type": "shell", "environment_vars": [ @@ -137,11 +142,6 @@ "source": "files/tinywww-debug.yml", "destination": "/root/config/tinywww-debug.yml" }, - { - "type": "file", - "source": "files/kubeconfig.yml", - "destination": "/root/config/kubeconfig.yml" - }, { "type": "file", "source": "files/veba-dcui", diff --git a/scripts/photon-containers.sh b/scripts/photon-containers.sh index 9ee21ee7..7c9cb6d3 100644 --- a/scripts/photon-containers.sh +++ b/scripts/photon-containers.sh @@ -4,52 +4,42 @@ echo '> Pre-Downloading Kubeadm Docker Containers' -CONTAINERS=( -k8s.gcr.io/kube-apiserver:v1.14.9 -k8s.gcr.io/kube-controller-manager:v1.14.9 -k8s.gcr.io/kube-scheduler:v1.14.9 -k8s.gcr.io/kube-proxy:v1.14.9 -k8s.gcr.io/pause:3.1 -k8s.gcr.io/etcd:3.3.10 -k8s.gcr.io/coredns:1.3.1 -antrea/antrea-ubuntu:v0.6.0 -embano1/tinywww:latest -projectcontour/contour:v1.0.0-beta.1 -openfaas/faas-netes:0.9.0 -openfaas/gateway:0.17.4 -openfaas/basic-auth-plugin:0.17.0 -openfaas/queue-worker:0.8.0 -openfaas/faas-idler:0.2.1 -envoyproxy/envoy:v1.11.1 -prom/prometheus:v2.11.0 -prom/alertmanager:v0.18.0 -nats-streaming:0.11.2 -vmware/veba-event-router:${VEBA_VERSION} -) +VEBA_BOM_FILE=/root/config/veba-bom.json -for i in ${CONTAINERS[@]}; +for component_name in $(jq '. | keys | .[]' ${VEBA_BOM_FILE}); do - docker pull $i + if [ $component_name != "\"veba\"" ]; then + for i in $(jq ".$component_name.containers | keys | .[]" ${VEBA_BOM_FILE}); do + value=$(jq -r ".$component_name.containers[$i]" ${VEBA_BOM_FILE}); + container_name=$(jq -r '.name' <<< "$value"); + container_version=$(jq -r '.version' <<< "$value"); + docker pull "$container_name:$container_version" + done + fi done mkdir -p /root/download && cd /root/download echo '> Downloading FaaS-Netes...' +OPENFAAS_VERSION=$(jq -r < ${VEBA_BOM_FILE} '.["openfaas"].version') git clone https://github.com/openfaas/faas-netes cd faas-netes -git checkout 0.9.2 +git checkout ${OPENFAAS_VERSION} sed -i 's/imagePullPolicy: Always/imagePullPolicy: IfNotPresent/g' yaml/*.yml cd .. echo '> Downloading Contour...' +CONTOUR_VERSION=$(jq -r < ${VEBA_BOM_FILE} '.["contour"].version') git clone https://github.com/projectcontour/contour.git cd contour -git checkout v1.0.0-beta.1 +git checkout ${CONTOUR_VERSION} sed -i '/^---/i \ dnsPolicy: ClusterFirstWithHostNet\n hostNetwork: true' examples/contour/03-envoy.yaml sed -i 's/imagePullPolicy: Always/imagePullPolicy: IfNotPresent/g' examples/contour/*.yaml cd .. echo '> Downloading Antrea...' -wget https://github.com/vmware-tanzu/antrea/releases/download/v0.6.0/antrea.yml -O /root/download/antrea.yml -sed -i 's/image: antrea\/antrea-ubuntu:.*/image: antrea\/antrea-ubuntu:v0.6.0/g' /root/download/antrea.yml +ANTREA_VERSION=$(jq -r < ${VEBA_BOM_FILE} '.["antrea"].version') +ANTREA_CONTAINER_VERSION=$(jq -r < ${VEBA_BOM_FILE} '.["antrea"].containers' | jq -r '.[] | select(.name | contains("antrea/antrea-ubuntu")).version') +wget https://github.com/vmware-tanzu/antrea/releases/download/${ANTREA_VERSION}/antrea.yml -O /root/download/antrea.yml +sed -i "s/image: antrea\/antrea-ubuntu:.*/image: antrea\/antrea-ubuntu:${ANTREA_CONTAINER_VERSION}/g" /root/download/antrea.yml sed -i '/image:.*/i \ imagePullPolicy: IfNotPresent' /root/download/antrea.yml \ No newline at end of file diff --git a/scripts/photon-settings.sh b/scripts/photon-settings.sh index 5cc441d3..ec2a0293 100644 --- a/scripts/photon-settings.sh +++ b/scripts/photon-settings.sh @@ -25,7 +25,8 @@ tdnf install -y \ unzip \ awk \ tar \ - kubernetes-kubeadm + kubernetes-kubeadm \ + jq echo '> Creating directory for setup scripts and configuration files' mkdir -p /root/setup From 88bd3f29ae4a396814dbcfd829f41af01865cd9e Mon Sep 17 00:00:00 2001 From: William Lam Date: Sun, 24 May 2020 06:46:00 -0700 Subject: [PATCH 06/86] Update K8s, Contour, OpenFaaS to latest stable release Signed-off-by: William Lam --- files/setup-04-kubernetes.sh | 4 +- photon.json | 14 ++-- scripts/photon-containers.sh | 5 +- scripts/photon-settings.sh | 21 +++++- veba-bom.json | 123 ++++++++++++++++------------------- 5 files changed, 89 insertions(+), 78 deletions(-) diff --git a/files/setup-04-kubernetes.sh b/files/setup-04-kubernetes.sh index 824cc91b..45eb0fea 100755 --- a/files/setup-04-kubernetes.sh +++ b/files/setup-04-kubernetes.sh @@ -26,10 +26,10 @@ fi echo -e "\e[92mSetting up k8s ..." > /dev/console K8S_VERSION=$(jq -r < ${VEBA_BOM_FILE} '.["kubernetes"].version') cat > /root/config/kubeconfig.yml << __EOF__ -apiVersion: kubeadm.k8s.io/v1beta1 +apiVersion: kubeadm.k8s.io/v1beta2 kind: InitConfiguration --- -apiVersion: kubeadm.k8s.io/v1beta1 +apiVersion: kubeadm.k8s.io/v1beta2 kind: ClusterConfiguration kubernetesVersion: ${K8S_VERSION} networking: diff --git a/photon.json b/photon.json index c9503f16..e5793667 100644 --- a/photon.json +++ b/photon.json @@ -49,6 +49,15 @@ } ], "provisioners": [ + { + "type": "shell", + "inline": ["mkdir -p /root/config"] + }, + { + "type": "file", + "source": "veba-bom.json", + "destination": "/root/config/veba-bom.json" + }, { "type": "shell", "environment_vars": [ @@ -61,11 +70,6 @@ "scripts/photon-docker.sh" ] }, - { - "type": "file", - "source": "veba-bom.json", - "destination": "/root/config/veba-bom.json" - }, { "type": "shell", "environment_vars": [ diff --git a/scripts/photon-containers.sh b/scripts/photon-containers.sh index 7c9cb6d3..81a2ea60 100644 --- a/scripts/photon-containers.sh +++ b/scripts/photon-containers.sh @@ -8,7 +8,8 @@ VEBA_BOM_FILE=/root/config/veba-bom.json for component_name in $(jq '. | keys | .[]' ${VEBA_BOM_FILE}); do - if [ $component_name != "\"veba\"" ]; then + HAS_CONTAINERS=$(jq ".$component_name | select(.containers != null)" ${VEBA_BOM_FILE}) + if [ "${HAS_CONTAINERS}" != "" ]; then for i in $(jq ".$component_name.containers | keys | .[]" ${VEBA_BOM_FILE}); do value=$(jq -r ".$component_name.containers[$i]" ${VEBA_BOM_FILE}); container_name=$(jq -r '.name' <<< "$value"); @@ -39,7 +40,7 @@ cd .. echo '> Downloading Antrea...' ANTREA_VERSION=$(jq -r < ${VEBA_BOM_FILE} '.["antrea"].version') -ANTREA_CONTAINER_VERSION=$(jq -r < ${VEBA_BOM_FILE} '.["antrea"].containers' | jq -r '.[] | select(.name | contains("antrea/antrea-ubuntu")).version') +ANTREA_CONTAINER_VERSION=$(jq -r < ${VEBA_BOM_FILE} '.["antrea"].containers | .[] | select(.name | contains("antrea/antrea-ubuntu")).version') wget https://github.com/vmware-tanzu/antrea/releases/download/${ANTREA_VERSION}/antrea.yml -O /root/download/antrea.yml sed -i "s/image: antrea\/antrea-ubuntu:.*/image: antrea\/antrea-ubuntu:${ANTREA_CONTAINER_VERSION}/g" /root/download/antrea.yml sed -i '/image:.*/i \ imagePullPolicy: IfNotPresent' /root/download/antrea.yml \ No newline at end of file diff --git a/scripts/photon-settings.sh b/scripts/photon-settings.sh index ec2a0293..a849327c 100644 --- a/scripts/photon-settings.sh +++ b/scripts/photon-settings.sh @@ -8,12 +8,13 @@ echo '> vCenter Event Broker Appliance Settings...' +VEBA_BOM_FILE=/root/config/veba-bom.json + echo '> Disable IPv6' echo "net.ipv6.conf.all.disable_ipv6 = 1" >> /etc/sysctl.conf echo '> Applying latest Updates...' tdnf -y update -#tdnf upgrade linux-esx echo '> Installing Additional Packages...' tdnf install -y \ @@ -25,12 +26,26 @@ tdnf install -y \ unzip \ awk \ tar \ - kubernetes-kubeadm \ jq +echo '> Adding K8s Repo' +curl -L https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg -o /etc/pki/rpm-gpg/GOOGLE-RPM-GPG-KEY +rpm --import /etc/pki/rpm-gpg/GOOGLE-RPM-GPG-KEY +cat > /etc/yum.repos.d/kubernetes.repo << EOF +[kubernetes] +name=Kubernetes +baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64 +enabled=1 +gpgcheck=1 +repo_gpgcheck=1 +gpgkey=file:///etc/pki/rpm-gpg/GOOGLE-RPM-GPG-KEY +EOF +K8S_VERSION=$(jq -r < ${VEBA_BOM_FILE} '.["kubernetes"].version' | sed 's/v//g') +# Ensure kubelet is updated to the latest desired K8s version +tdnf install -y kubelet-${K8S_VERSION} kubectl-${K8S_VERSION} kubeadm-${K8S_VERSION} + echo '> Creating directory for setup scripts and configuration files' mkdir -p /root/setup -mkdir -p /root/config echo '> Creating tools.conf to prioritize eth0 interface...' cat > /etc/vmware-tools/tools.conf << EOF diff --git a/veba-bom.json b/veba-bom.json index 3cd1bb21..f9a346c7 100644 --- a/veba-bom.json +++ b/veba-bom.json @@ -1,118 +1,109 @@ { - "veba" : { - "version" : "v0.4.0" + "veba": { + "version": "v0.4.0" }, - "antrea" : { - "version" : "v0.6.0", - "containers" : [ - { - "name": "antrea/antrea-ubuntu", - "version" : "v0.6.0" - } - ] + "antrea": { + "version": "v0.6.0", + "containers": [{ + "name": "antrea/antrea-ubuntu", + "version": "v0.6.0" + }] }, - "contour" : { - "version" : "v1.0.0-beta.1", - "containers" : [ - { + "contour": { + "version": "v1.4.0", + "containers": [{ "name": "projectcontour/contour", - "version": "v1.0.0-beta.1" + "version": "v1.4.0" }, { "name": "envoyproxy/envoy", - "version": "v1.11.1" + "version": "v1.14.1" } ] }, - "kubernetes" : { - "version" : "v1.14.9", - "containers": [ - { + "kubernetes": { + "version": "v1.18.3", + "containers": [{ "name": "k8s.gcr.io/kube-apiserver", - "version": "v1.14.9" + "version": "v1.18.3" }, { "name": "k8s.gcr.io/kube-controller-manager", - "version": "v1.14.9" + "version": "v1.18.3" }, { "name": "k8s.gcr.io/kube-scheduler", - "version": "v1.14.9" + "version": "v1.18.3" }, { "name": "k8s.gcr.io/kube-proxy", - "version": "v1.14.9" + "version": "v1.18.3" }, { - "name" : "k8s.gcr.io/pause", - "version" : "3.1" + "name": "k8s.gcr.io/pause", + "version": "3.2" }, { - "name" : "k8s.gcr.io/etcd", - "version" : "3.3.10" + "name": "k8s.gcr.io/etcd", + "version": "3.4.3-0" }, { - "name" : "k8s.gcr.io/coredns", - "version" : "1.3.1" + "name": "k8s.gcr.io/coredns", + "version": "1.6.7" } ] }, - "openfaas" : { - "version" : "0.9.2", - "containers" : [ - { - "name" : "openfaas/faas-netes", - "version" : "0.9.0" + "openfaas": { + "version": "0.10.5", + "containers": [{ + "name": "openfaas/faas-netes", + "version": "0.10.3" }, { - "name" : "openfaas/gateway", - "version" : "0.17.4" + "name": "openfaas/gateway", + "version": "0.18.17" }, { - "name" : "openfaas/basic-auth-plugin", - "version" : "0.17.0" + "name": "openfaas/basic-auth-plugin", + "version": "0.18.17" }, { - "name" : "openfaas/queue-worker", - "version" : "0.8.0" + "name": "openfaas/queue-worker", + "version": "0.11.0" }, { - "name" : "openfaas/faas-idler", - "version" : "0.2.1" + "name": "openfaas/faas-idler", + "version": "0.3.0" }, { - "name" : "prom/prometheus", - "version" : "v2.11.0" + "name": "prom/prometheus", + "version": "v2.11.0" }, { - "name" : "prom/alertmanager", - "version" : "v0.18.0" + "name": "prom/alertmanager", + "version": "v0.18.0" }, { - "name" : "nats-streaming", - "version" : "0.11.2" + "name": "nats-streaming", + "version": "0.17.0" } ], "faas-cli": { "version": "0.12.4" } }, - "tinywww" :{ - "version" : "latest", - "containers" : [ - { - "name" : "embano1/tinywww", - "version" : "latest" - } - ] + "tinywww": { + "version": "latest", + "containers": [{ + "name": "embano1/tinywww", + "version": "latest" + }] }, - "vmware-event-router" : { - "version" : "v0.4.0", - "containers" : [ - { - "name": "vmware/veba-event-router", - "version": "v0.4.0" - } - ] + "vmware-event-router": { + "version": "v0.4.0", + "containers": [{ + "name": "vmware/veba-event-router", + "version": "v0.4.0" + }] } } \ No newline at end of file From 7b5be22993085538aa94e51060c6c2c4116aaa57 Mon Sep 17 00:00:00 2001 From: William Lam Date: Wed, 3 Jun 2020 15:22:27 -0700 Subject: [PATCH 07/86] VEBA Issue and Feature Enhancement Templates Signed-off-by: William Lam --- .github/ISSUE_TEMPLATE/bug-report.md | 31 +++++++++++++++++++ .github/ISSUE_TEMPLATE/feature-enhancement.md | 20 ++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug-report.md create mode 100644 .github/ISSUE_TEMPLATE/feature-enhancement.md diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/bug-report.md new file mode 100644 index 00000000..76f5a4bd --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug-report.md @@ -0,0 +1,31 @@ +--- +name: Bug Report +about: Create a bug report to help us improve VEBA +title: '' +labels: bug +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Steps to reproduce the behavior: +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Screenshots** +If applicable, add screenshots to help explain your problem. + +**Version (please complete the following information):** + - VEBA Form Factor: [e.g. Appliance or K8s] + - VEBA Version: [e.g. v0.4.0] + +**Additional context** +Add any other context about the problem here. \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/feature-enhancement.md b/.github/ISSUE_TEMPLATE/feature-enhancement.md new file mode 100644 index 00000000..6b4e2806 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature-enhancement.md @@ -0,0 +1,20 @@ +--- +name: Feature Enhancement +about: Suggest a new idea or enhancement for VEBA +title: '' +labels: enhancement +assignees: lamw + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. \ No newline at end of file From f7fcc4d8c5ae36c116436ff7a3b365e40c817615 Mon Sep 17 00:00:00 2001 From: pksrc Date: Fri, 5 Jun 2020 20:56:19 -0700 Subject: [PATCH 08/86] Search functionality added to Documentation Updating Jquery version to 3.5.1 and requiring the full version Adding the capability to index external content that are shown in docs side nav Signed-off-by: pksrc --- docs/_config.yml | 7 + docs/_includes/footer.html | 12 +- docs/_includes/head-docs.html | 1 + docs/_includes/nav.html | 7 + docs/_layouts/docs.html | 3 + docs/assets/_scss/_styles.scss | 1 + .../_scss/site/objects/_tipuesearch.scss | 261 ++++++++++++++++++ docs/assets/js/scripts.js | 2 + docs/assets/tipuesearch/search.png | Bin 0 -> 1366 bytes docs/assets/tipuesearch/tipuesearch.min.js | 178 ++++++++++++ .../assets/tipuesearch/tipuesearch_content.js | 95 +++++++ docs/assets/tipuesearch/tipuesearch_set.js | 80 ++++++ docs/site/search.html | 14 + licenses/github.com/tipuesearch/LICENSE | 21 ++ 14 files changed, 678 insertions(+), 4 deletions(-) create mode 100755 docs/assets/_scss/site/objects/_tipuesearch.scss create mode 100644 docs/assets/tipuesearch/search.png create mode 100644 docs/assets/tipuesearch/tipuesearch.min.js create mode 100644 docs/assets/tipuesearch/tipuesearch_content.js create mode 100644 docs/assets/tipuesearch/tipuesearch_set.js create mode 100644 docs/site/search.html create mode 100644 licenses/github.com/tipuesearch/LICENSE diff --git a/docs/_config.yml b/docs/_config.yml index 858ad934..6b0ee83d 100644 --- a/docs/_config.yml +++ b/docs/_config.yml @@ -65,6 +65,13 @@ versioning: false #- v0.3.0 #- v0.2.0 +# Tipue search settings +tipue_search: + include: + pages: true + exclude: + files: [README.md, search.html, site/resources.md, assets/css/fonts/Metropolis/Open Font License.md, assets/css/fonts/Metropolis/README.md, zcleanup/DESIGN.md, zcleanup/getting-started-build.md, zcleanup/getting-started.md, assets/css/fonts/Metropolis/README.md, zcleanup/advanced-deploy-k8s.md] + # Build settings permalink: /blog/:title/ sass: diff --git a/docs/_includes/footer.html b/docs/_includes/footer.html index 73035d59..10a4b466 100644 --- a/docs/_includes/footer.html +++ b/docs/_includes/footer.html @@ -42,12 +42,16 @@
{{ site.footer.title }}
- + - - +{% if page.tipue_search_active or layout.tipue_search_active %} + + + +{% endif %} + + \ No newline at end of file diff --git a/docs/_includes/head-docs.html b/docs/_includes/head-docs.html index 9546b360..ec856cf6 100644 --- a/docs/_includes/head-docs.html +++ b/docs/_includes/head-docs.html @@ -5,6 +5,7 @@ + {% seo %} {% include google-analytics.html %} diff --git a/docs/_includes/nav.html b/docs/_includes/nav.html index 483bf183..99fd32fb 100644 --- a/docs/_includes/nav.html +++ b/docs/_includes/nav.html @@ -1,6 +1,13 @@