From a1645e1d0943d619ef09a7f50240313df661df90 Mon Sep 17 00:00:00 2001 From: Sanskarzz Date: Sun, 7 Jul 2024 05:22:03 +0530 Subject: [PATCH 1/4] chore: Add demo of kyverno-envoy-plugin with Gloo-Edge Signed-off-by: Sanskarzz --- demo/gloo-edge/manifest/gloo.yaml | 6 +++ .../manifest/kyverno-deployment.yaml | 51 +++++++++++++++++++ demo/gloo-edge/manifest/lint.yaml | 34 +++++++++++++ demo/gloo-edge/manifest/policy.yaml | 14 +++++ demo/gloo-edge/manifest/virtual-service.yaml | 28 ++++++++++ demo/gloo-edge/manifest/vs-patch.yaml | 5 ++ 6 files changed, 138 insertions(+) create mode 100644 demo/gloo-edge/manifest/gloo.yaml create mode 100644 demo/gloo-edge/manifest/kyverno-deployment.yaml create mode 100644 demo/gloo-edge/manifest/lint.yaml create mode 100644 demo/gloo-edge/manifest/policy.yaml create mode 100644 demo/gloo-edge/manifest/virtual-service.yaml create mode 100644 demo/gloo-edge/manifest/vs-patch.yaml diff --git a/demo/gloo-edge/manifest/gloo.yaml b/demo/gloo-edge/manifest/gloo.yaml new file mode 100644 index 0000000..3a98b13 --- /dev/null +++ b/demo/gloo-edge/manifest/gloo.yaml @@ -0,0 +1,6 @@ +global: + extensions: + extAuth: + extauthzServerRef: + name: gloo-system-kyverno-9191 + namespace: gloo-system \ No newline at end of file diff --git a/demo/gloo-edge/manifest/kyverno-deployment.yaml b/demo/gloo-edge/manifest/kyverno-deployment.yaml new file mode 100644 index 0000000..6cb64d4 --- /dev/null +++ b/demo/gloo-edge/manifest/kyverno-deployment.yaml @@ -0,0 +1,51 @@ +apiVersion: v1 +kind: Service +metadata: + name: kyverno + labels: + app: kyverno +spec: + ports: + - name: grpc + port: 9191 + targetPort: 9191 + protocol: TCP + selector: + app: kyverno +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kyverno + labels: + app: kyverno +spec: + replicas: 1 + selector: + matchLabels: + app: kyverno + template: + metadata: + labels: + app: kyverno + spec: + containers: + - name: kyverno + image: sanskardevops/plugin:0.0.34 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 8181 + - containerPort: 9191 + volumeMounts: + - readOnly: true + mountPath: /policy + name: kyverno-policy + args: + - "serve" + - "--policy=/policy/policy.yaml" + - "--address=:9191" + - "--healthaddress=:8181" + volumes: + - name: kyverno-policy + secret: + secretName: kyverno-policy \ No newline at end of file diff --git a/demo/gloo-edge/manifest/lint.yaml b/demo/gloo-edge/manifest/lint.yaml new file mode 100644 index 0000000..3907242 --- /dev/null +++ b/demo/gloo-edge/manifest/lint.yaml @@ -0,0 +1,34 @@ +# yaml-language-server: $schema=https://json.schemastore.org/github-workflow.json + +name: Lint + +permissions: {} + +on: + pull_request: + branches: + - main + push: + branches: + - main + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + required: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + - name: Setup Go + uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 + with: + go-version-file: go.mod + cache-dependency-path: go.sum + - name: golangci-lint + uses: golangci/golangci-lint-action@9d1e0624a798bb64f6c3cea93db47765312263dc # v3.7.1 + with: + skip-cache: true + args: --timeout 5m diff --git a/demo/gloo-edge/manifest/policy.yaml b/demo/gloo-edge/manifest/policy.yaml new file mode 100644 index 0000000..4147aad --- /dev/null +++ b/demo/gloo-edge/manifest/policy.yaml @@ -0,0 +1,14 @@ +apiVersion: json.kyverno.io/v1alpha1 +kind: ValidatingPolicy +metadata: + name: checkrequest +spec: + rules: + - name: deny-post-request + assert: + any: + - message: "POST method calls are not allowed only GET call are allowed" + check: + request: + http: + method: GET \ No newline at end of file diff --git a/demo/gloo-edge/manifest/virtual-service.yaml b/demo/gloo-edge/manifest/virtual-service.yaml new file mode 100644 index 0000000..179c07e --- /dev/null +++ b/demo/gloo-edge/manifest/virtual-service.yaml @@ -0,0 +1,28 @@ +apiVersion: gloo.solo.io/v1 +kind: Upstream +metadata: + name: httpbin +spec: + static: + hosts: + - addr: httpbin.org + port: 80 +--- +apiVersion: gateway.solo.io/v1 +kind: VirtualService +metadata: + name: httpbin +spec: + virtualHost: + domains: + - '*' + routes: + - matchers: + - prefix: / + routeAction: + single: + upstream: + name: httpbin + namespace: gloo-system + options: + autoHostRewrite: true \ No newline at end of file diff --git a/demo/gloo-edge/manifest/vs-patch.yaml b/demo/gloo-edge/manifest/vs-patch.yaml new file mode 100644 index 0000000..c8bf57e --- /dev/null +++ b/demo/gloo-edge/manifest/vs-patch.yaml @@ -0,0 +1,5 @@ +spec: + virtualHost: + options: + extauth: + customAuth: {} \ No newline at end of file From 2f2098ac451d995b8ba4ba78f9f16787b4eb4356 Mon Sep 17 00:00:00 2001 From: Sanskarzz Date: Sun, 7 Jul 2024 05:22:45 +0530 Subject: [PATCH 2/4] add readme of the demo gloo edge Signed-off-by: Sanskarzz --- demo/gloo-edge/README.md | 283 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 283 insertions(+) create mode 100644 demo/gloo-edge/README.md diff --git a/demo/gloo-edge/README.md b/demo/gloo-edge/README.md new file mode 100644 index 0000000..9d75a7b --- /dev/null +++ b/demo/gloo-edge/README.md @@ -0,0 +1,283 @@ +# Gloo Edge demo + +This GLoo Edge demo is the prototype of Kyverno-Envoy-plugin. + +## Overview + +[Gloo Edge](https://docs.solo.io/gloo-edge/latest/) is a Kubernetes-native API gateway built on top of Envoy Proxy. Gloo Edge is designed to be extensible and pluggable, and can be used to secure, control, and observe traffic to and from any application. `Gloo Edge` an API Gateway built on Envoy, offers a `Kubernetes` Custom Resource Definition (CRD) for managing Envoy configurations to handle traffic management and routing. + +`Gloo Edge` supports the creation of a [Custom External Authorization Service](https://docs.solo.io/gloo-edge/latest/guides/security/auth/custom_auth/) that adheres to the Envoy specification for an External Authorization Server. + +This tutorial demonstrates how to use the Kyverno-Envoy-Plugin with Gloo Edge to enforce security policies for upstream services. + +## Demo instructions + +### Required tools + +1. [`minikube`](https://minikube.sigs.k8s.io/docs/start/) +2. [`kubectl`](https://kubernetes.io/docs/tasks/tools/install-kubectl/) +3. [`helm`](https://helm.sh/docs/intro/install/) + +### Create a local cluster and install Gloo Edge + +Start a local cluster with `minikube`: + +```bash +$ minikube start +``` + +Setup and configure `Gloo Edge` using the below commands: + +```bash +$ helm repo add gloo https://storage.googleapis.com/solo-public-helm +$ helm upgrade --install --namespace gloo-system --create-namespace gloo gloo/gloo +$ kubectl config set-context $(kubectl config current-context) --namespace=gloo-system +``` +### Create a VirtualService and Upstream + +[VirtualService](https://docs.solo.io/gloo-edge/latest/introduction/architecture/concepts/#virtual-services) define a set of route rules that live under a domain or set of domains. Route rules consist of matchers, which specify the kind of function calls to match (requests and events, are currently supported), and the name of the destination (or destinations) where to route them. + +[Upstreams](https://docs.solo.io/gloo-edge/latest/introduction/architecture/concepts/#upstreams) define destinations for routes. Upstreams tell Gloo Edge what to route to and how to route to them. Gloo Edge determines how to handle routing for the Upstream based on its spec field. Upstreams have a type-specific spec field that provides routing information to Gloo Edge. + +In this tutorial, we will create a VirtualService and Upstream that will route requests to the `httpbin.org` service. + +```bash +$ kubectl apply -f - < Note: Since the name of the service port is `grpc`, `Gloo` will understand that traffic should be routed using HTTP2 protocol. + +```bash +$ kubectl apply -f - < POST method calls are not allowed only GET call are allowed + -> any[0].check.request.http.method: Invalid value: "POST": Expected value: "GET" +``` \ No newline at end of file From b076e801a46333bc9d1030e717a633620cee7590 Mon Sep 17 00:00:00 2001 From: Sanskarzz Date: Sun, 7 Jul 2024 05:25:26 +0530 Subject: [PATCH 3/4] Added gloo edge demo to the website docs Signed-off-by: Sanskarzz --- website/docs/tutorials/gloo.md | 293 +++++++++++++++++++++++++++++++++ website/mkdocs.yaml | 1 + 2 files changed, 294 insertions(+) create mode 100644 website/docs/tutorials/gloo.md diff --git a/website/docs/tutorials/gloo.md b/website/docs/tutorials/gloo.md new file mode 100644 index 0000000..2d0ce58 --- /dev/null +++ b/website/docs/tutorials/gloo.md @@ -0,0 +1,293 @@ +# Gloo Edge + +This GLoo Edge demo is the prototype of Kyverno-Envoy-plugin. + +## Overview + +[Gloo Edge](https://docs.solo.io/gloo-edge/latest/) is a Kubernetes-native API gateway built on top of Envoy Proxy. Gloo Edge is designed to be extensible and pluggable, and can be used to secure, control, and observe traffic to and from any application. `Gloo Edge` an API Gateway built on Envoy, offers a `Kubernetes` Custom Resource Definition (CRD) for managing Envoy configurations to handle traffic management and routing. + +`Gloo Edge` supports the creation of a [Custom External Authorization Service](https://docs.solo.io/gloo-edge/latest/guides/security/auth/custom_auth/) that adheres to the Envoy specification for an External Authorization Server. + +This tutorial demonstrates how to use the Kyverno-Envoy-Plugin with Gloo Edge to enforce security policies for upstream services. + +## Demo instructions + +### Required tools + +1. [`minikube`](https://minikube.sigs.k8s.io/docs/start/) +2. [`kubectl`](https://kubernetes.io/docs/tasks/tools/install-kubectl/) +3. [`helm`](https://helm.sh/docs/intro/install/) + +### Create a local cluster and install Gloo Edge + +Start a local cluster with `minikube`: + +```bash +$ minikube start +``` + +Setup and configure `Gloo Edge` using the below commands: + +```bash +$ helm repo add gloo https://storage.googleapis.com/solo-public-helm +$ helm upgrade --install --namespace gloo-system --create-namespace gloo gloo/gloo +$ kubectl config set-context $(kubectl config current-context) --namespace=gloo-system +``` +### Create a VirtualService and Upstream + +[VirtualService](https://docs.solo.io/gloo-edge/latest/introduction/architecture/concepts/#virtual-services) define a set of route rules that live under a domain or set of domains. Route rules consist of matchers, which specify the kind of function calls to match (requests and events, are currently supported), and the name of the destination (or destinations) where to route them. + +[Upstreams](https://docs.solo.io/gloo-edge/latest/introduction/architecture/concepts/#upstreams) define destinations for routes. Upstreams tell Gloo Edge what to route to and how to route to them. Gloo Edge determines how to handle routing for the Upstream based on its spec field. Upstreams have a type-specific spec field that provides routing information to Gloo Edge. + +In this tutorial, we will create a VirtualService and Upstream that will route requests to the `httpbin.org` service. + +```bash +$ kubectl apply -f - < Note: Since the name of the service port is `grpc`, `Gloo` will understand that traffic should be routed using HTTP2 protocol. + +```bash +$ kubectl apply -f - < POST method calls are not allowed only GET call are allowed + -> any[0].check.request.http.method: Invalid value: "POST": Expected value: "GET" +``` + +### Cleanup and Wrap up + +```bash +$ kubectl delete ns gloo-system +``` + +Congratulations on completing the tutorial! + +In this tutorial, you learned how to utilize the `Kyverno-Envoy-Plugin` with `Gloo Edge` to enforce security policies on upstream services. Additionally, you created and tested a policy that permits `GET` or `POST` requests based on user roles. \ No newline at end of file diff --git a/website/mkdocs.yaml b/website/mkdocs.yaml index b6b5fa4..b31d5d2 100644 --- a/website/mkdocs.yaml +++ b/website/mkdocs.yaml @@ -16,6 +16,7 @@ nav: - tutorials/standalone-envoy.md - tutorials/istio.md - tutorials/mtls-istio.md + - tutorials/gloo.md - Performance: performance.md From 1e5e5106fa673bfa37c375d95de6711abfa764f3 Mon Sep 17 00:00:00 2001 From: Sanskarzz Date: Sun, 7 Jul 2024 05:26:30 +0530 Subject: [PATCH 4/4] fixed some error in performace.md websitedocs file Signed-off-by: Sanskarzz --- website/docs/performance.md | 44 +++++++++++++++++++++++-------------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/website/docs/performance.md b/website/docs/performance.md index ff5e348..890933e 100644 --- a/website/docs/performance.md +++ b/website/docs/performance.md @@ -217,7 +217,7 @@ EOF ```console k6 run --out json=report.json k6-script.js ``` -5. ***Repeat for different scenarios**: +5. **Repeat for different scenarios**: - # App only In this case , request are sent directly to the sample application ie no Envoy and Kyverno-plugin in the request path . @@ -226,8 +226,10 @@ k6 run --out json=report.json k6-script.js ```shell $ kubectl apply -f https://raw.githubusercontent.com/kyverno/kyverno-envoy-plugin/main/tests/performance-test/manifest/app.yaml ``` - Results of the k6 when only application is applied - ```bash + + Results of the k6 when only application is applied: + +```bash /\ |‾‾| /‾‾/ /‾‾/ /\ / \ | |/ / / / @@ -269,9 +271,10 @@ k6 run --out json=report.json k6-script.js running (2m00.6s), 000/100 VUs, 9048 complete and 0 interrupted iterations default ✓ [======================================] 000/100 VUs 2m0s - ``` +``` - # App and Envoy + In this case, Kyverno-envoy-plugin is not included in the path but Envoy is but Envoy External Authorization API disabled For this run this command to apply the sample application with envoy. @@ -280,7 +283,8 @@ default ✓ [======================================] 000/100 VUs 2m0s ``` Results of k6 after applying sample-application with envoy. - ```bash + +```bash /\ |‾‾| /‾‾/ /‾‾/ /\ / \ | |/ / / / @@ -322,9 +326,10 @@ default ✓ [======================================] 000/100 VUs 2m0s running (2m00.7s), 000/100 VUs, 9031 complete and 0 interrupted iterations default ✓ [======================================] 000/100 VUs 2m0s - ``` +``` + +- # App, Envoy and Kyverno-envoy-plugin -- # App, Envoy and Kyverno-envoy-plugin In this case, performance measurements are observed with Envoy External Authorization API enabled and a sample real-world RBAC policy loaded into kyverno-envoy-plugin . For this apply this command to apply sample-application, envoy and kyverno-envoy-plugin @@ -333,7 +338,8 @@ default ✓ [======================================] 000/100 VUs 2m0s ``` Results of k6 after applying sample-application, Envoy and kyverno-envoy-plugin . - ```console + +```bash /\ |‾‾| /‾‾/ /‾‾/ /\ / \ | |/ / / / @@ -375,7 +381,8 @@ default ✓ [======================================] 000/100 VUs 2m0s running (2m00.2s), 000/100 VUs, 8655 complete and 0 interrupted iterations default ✓ [======================================] 000/100 VUs 2m0s - ``` +``` + ## Measuring Performance The following metrics should be measured to evaluate the performance impact of the `kyverno-envoy-plugin`: @@ -445,14 +452,19 @@ The following metrics should be measured to evaluate the performance impact of t testapp-5955cd6f8b-dbvgd test-application 1m 11Mi ``` - Observations: - - After the load test completed and the request volume returned to normal levels, the CPU and memory utilization of the kyverno-envoy-plugin container returned to their initial values. This indicates that the kyverno-envoy-plugin can efficiently handle the increased load during the test and release the additional resources when the load subsides. +Observations: + +- After the load test completed and the request volume returned to normal levels, the CPU and memory utilization of the kyverno-envoy-plugin container returned to their initial values. This indicates that the kyverno-envoy-plugin can efficiently handle the increased load during the test and release the additional resources when the load subsides. + +Correlation with k6 results: + +- The k6 script simulated a load test scenario with 100 virtual users, ramping up over 30 seconds, staying at 100 users for 1 minute, and then ramping down over 30 seconds. + +- During the load test, when the request volume was at its peak (100 virtual users), the kyverno-envoy-plugin container experienced a significant increase in CPU utilization, reaching 895m. + +- This CPU utilization spike aligns with the increased processing demand on the kyverno-envoy-plugin to evaluate the incoming requests against the configured Kyverno policies. - Correlation with k6 results: - - The k6 script simulated a load test scenario with 100 virtual users, ramping up over 30 seconds, staying at 100 users for 1 minute, and then ramping down over 30 seconds. - - During the load test, when the request volume was at its peak (100 virtual users), the kyverno-envoy-plugin container experienced a significant increase in CPU utilization, reaching 895m. - - This CPU utilization spike aligns with the increased processing demand on the kyverno-envoy-plugin to evaluate the incoming requests against the configured Kyverno policies. - - The memory utilization increase during the load test was relatively modest, suggesting that the policy evaluation did not significantly impact the memory requirements of the kyverno-envoy-plugin. +- The memory utilization increase during the load test was relatively modest, suggesting that the policy evaluation did not significantly impact the memory requirements of the kyverno-envoy-plugin.