Merge pull request #1695 from rksharma95/feat-secure-grpc

feat(gRPC): add tls configurations to the operator and helm

.github/workflows/ci-test-controllers.yml

@@ -56,11 +56,18 @@ jobs:
       - name: Build KubeArmorController
         run: make -C pkg/KubeArmorController/ docker-build TAG=latest
 
+      - name: Build Kubearmor-Operator
+        working-directory: pkg/KubeArmorOperator
+        run: |
+          make docker-build
+
       - name: Install KubeArmor Latest and KubeArmorController using Helm
         run: |
           # save images
           docker save kubearmor/kubearmor-controller:latest | sudo k3s ctr images import -
-
+          docker save kubearmor/kubearmor-operator:latest | sudo k3s ctr images import -
+          docker save kubearmor/kubearmor-snitch:latest | sudo k3s ctr images import -
+          
           helm upgrade --install kubearmor-operator ./deployments/helm/KubeArmorOperator -n kubearmor --create-namespace --set kubearmorOperator.image.tag=latest
           kubectl wait --for=condition=ready --timeout=5m -n kubearmor pod -l kubearmor-app=kubearmor-operator
           kubectl get pods -A

.github/workflows/ci-test-ubi-image.yml

@@ -71,11 +71,12 @@ jobs:
           sudo podman pull docker-daemon:kubearmor/kubearmor-operator:latest
           sudo podman pull docker-daemon:kubearmor/kubearmor-snitch:latest
           helm upgrade --install kubearmor-operator ./deployments/helm/KubeArmorOperator -n kubearmor --create-namespace --set kubearmorOperator.image.tag=latest
-          kubectl wait --for=condition=ready --timeout=5m -n kubearmor pod -l kubearmor-app=kubearmor-operator
           kubectl get pods -A
+          kubectl wait --for=condition=ready --timeout=5m -n kubearmor pod -l kubearmor-app=kubearmor-operator
           kubectl apply -f pkg/KubeArmorOperator/config/samples/kubearmor-ubi-test.yaml
           kubectl wait -n kubearmor --timeout=5m --for=jsonpath='{.status.phase}'=Running kubearmorconfigs/kubearmorconfig-test
-          kubectl wait --timeout=5m --for=condition=ready pod -l kubearmor-app,kubearmor-app!=kubearmor-snitch -n kubearmor
+          kubectl wait --timeout=7m --for=condition=ready pod -l kubearmor-app,kubearmor-app!=kubearmor-snitch,kubearmor-app!=kubearmor-controller -n kubearmor
+          kubectl wait --timeout=1m --for=condition=ready pod -l kubearmor-app=kubearmor-controller -n kubearmor
           kubectl get pods -A
 
       - name: Test KubeArmor using Ginkgo

.github/workflows/install-k3s.sh

@@ -5,7 +5,13 @@
 echo "RUNTIME="$RUNTIME
 
 if [ "$RUNTIME" == "crio" ]; then
-    ./contribution/self-managed-k8s/crio/install_crio.sh
+    status=$(systemctl is-active crio)
+    if [ "$status" == "active" ]; then
+        echo "CRI-O is already installed."
+    else
+        echo "CRI-O is not installed"
+        ./contribution/self-managed-k8s/crio/install_crio.sh
+    fi
 fi
 
 ./contribution/k3s/install_k3s.sh

KubeArmor/cert/cert.go

@@ -2,10 +2,10 @@
 // Copyright 2022 Authors of KubeArmor
 
 // Package cert is responsible for generating certs dynamically and loading the certs from external sources.
-
 package cert
 
 import (
+	"bytes"
 	"context"
 	"crypto/rand"
 	"crypto/rsa"
@@ -21,6 +21,7 @@ import (
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
+	"k8s.io/klog/v2"
 )
 
 const (
@@ -80,6 +81,19 @@ type CertPath struct {
 	CertOnly bool   // if true read certificate only
 }
 
+func GetPemCertFromx509Cert(cert x509.Certificate) []byte {
+	certPem := &pem.Block{
+		Type:  "CERTIFICATE",
+		Bytes: cert.Raw,
+	}
+	var pemBuffer bytes.Buffer
+	err := pem.Encode(&pemBuffer, certPem)
+	if err != nil {
+		klog.Error("error while encoding certificate to pem format", err)
+	}
+	return pemBuffer.Bytes()
+}
+
 // GetCACertPath func returns CA certificate (full) path
 func GetCACertPath(base string) CertPath {
 	return CertPath{
@@ -114,13 +128,13 @@ func GetCertKeyPairFromCertBytes(certBytes *CertBytes) (*CertKeyPair, error) {
 
 	crt, err := x509.ParseCertificate(certPem.Bytes)
 	if err != nil {
-		fmt.Println("Error parsing CA certificate:", err)
+		klog.Error("Error parsing CA certificate:", err)
 		return nil, err
 	}
 
 	key, err := x509.ParsePKCS1PrivateKey(keyPem.Bytes)
 	if err != nil {
-		fmt.Println("Error parsing CA private key:", err)
+		klog.Error("Error parsing CA private key:", err)
 		return nil, err
 	}
 
@@ -138,26 +152,26 @@ func ReadCertFromFile(certPath *CertPath) (*CertBytes, error) {
 
 	// Check if certificate file exists
 	if _, err := os.Stat(certFile); os.IsNotExist(err) {
-		fmt.Println("certificate file does not exist:", err)
+		klog.Error("certificate file does not exist:", err)
 		return nil, err
 	}
 
 	certBytes, err := os.ReadFile(certFile)
 	if err != nil {
-		fmt.Println("Error reading CA certificate file:", err)
+		klog.Error("Error reading CA certificate file:", err)
 		return nil, err
 	}
 
 	if !certPath.CertOnly {
 		// Check if key file exists
 		if _, err := os.Stat(keyFile); os.IsNotExist(err) {
-			fmt.Println("CA key file does not exist:", err)
+			klog.Error("CA key file does not exist:", err)
 			return nil, err
 		}
 
 		keyBytes, err = os.ReadFile(keyFile)
 		if err != nil {
-			fmt.Println("Error reading CA key file:", err)
+			klog.Error("Error reading CA key file:", err)
 			return nil, err
 		}
 	}
@@ -186,11 +200,27 @@ func ReadCertFromK8sSecret(client *kubernetes.Clientset, namespace, secret strin
 	}, nil
 }
 
+func GenerateCA(cfg *CertConfig) (*CertBytes, error) {
+	crtTemp, err := GenerateCert(cfg)
+	if err != nil {
+		klog.Errorf("error generating ca cert: %s\n", err)
+		return &CertBytes{}, err
+	}
+	crtBytes, err := GenerateSelfSignedCert(crtTemp, cfg)
+	if err != nil {
+		return &CertBytes{}, nil
+	}
+	return &CertBytes{
+		Crt: crtBytes.Crt,
+		Key: crtBytes.Key,
+	}, nil
+}
+
 func GenerateCert(cfg *CertConfig) (*CertKeyPair, error) {
 	// Generate a new RSA private key for the server
 	key, err := rsa.GenerateKey(rand.Reader, 2048)
 	if err != nil {
-		fmt.Println("error generating cert private key:", err)
+		klog.Error("error generating cert private key:", err)
 		return nil, err
 	}
 
@@ -224,7 +254,7 @@ func GenerateSelfSignedCert(ca *CertKeyPair, cfg *CertConfig) (*CertBytes, error
 	// Create the certificate signed by the CA
 	certBytes, err := x509.CreateCertificate(rand.Reader, certKeyPair.Crt, ca.Crt, &certKeyPair.Key.PublicKey, ca.Key)
 	if err != nil {
-		fmt.Println("Error creating certificate:", err)
+		klog.Error("Error creating certificate:", err)
 		return nil, err
 	}
 

contribution/self-managed-k8s/crio/install_crio.sh

@@ -12,21 +12,18 @@ fi
 OS="x${NAME}_${VERSION_ID}"
 
 # install cri-o corresponding to the latest k3s version
-VERSION=$(curl -w '%{url_effective}' -L -s -S https://update.k3s.io/v1-release/channels/stable -o /dev/null | sed -e 's|.*/v||' | cut -d '.' -f1,2)
+VERSION=$(curl -w '%{url_effective}' -L -s -S https://update.k3s.io/v1-release/channels/stable -o /dev/null | sed -e 's|.*/||' | cut -d '.' -f1,2)
 echo "Installing CRI-O version $VERSION"
 
-# get signing keys
-echo "deb [signed-by=/usr/share/keyrings/libcontainers-archive-keyring.gpg] https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/$OS/ /" | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list
-echo "deb [signed-by=/usr/share/keyrings/libcontainers-crio-archive-keyring.gpg] https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/$VERSION/$OS/ /" | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable:cri-o:$VERSION.list
+curl -fsSL https://pkgs.k8s.io/addons:/cri-o://stable:/$VERSION/deb/Release.key |
+    sudo gpg --dearmor -o /etc/apt/keyrings/cri-o-apt-keyring.gpg
 
-# add repositories
-sudo mkdir -p /usr/share/keyrings
-curl -L https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/$OS/Release.key | sudo gpg --yes --dearmor -o /usr/share/keyrings/libcontainers-archive-keyring.gpg
-curl -L https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/$VERSION/$OS/Release.key | sudo gpg --yes --dearmor -o /usr/share/keyrings/libcontainers-crio-archive-keyring.gpg
+echo "deb [signed-by=/etc/apt/keyrings/cri-o-apt-keyring.gpg] https://pkgs.k8s.io/addons:/cri-o:/stable:/$VERSION/deb/ /" |
+    sudo tee /etc/apt/sources.list.d/cri-o.list
 
 # install
 sudo apt-get update
-sudo apt-get install -y cri-o cri-o-runc
+sudo apt-get install -y cri-o
 
 # this option is not supported in ubuntu 18.04
 if [ "$VERSION_ID" == "18.04" ]; then

deployments/helm/KubeArmor/templates/daemonset.yaml

@@ -19,6 +19,9 @@ spec:
       containers:
       - args:
         - -gRPC=32767
+        {{printf "- -tlsEnabled=%t" .Values.tls.enabled}}
+        {{printf "- -tlsCertPath=%s" .Values.kubearmor.tls.tlsCertPath}}
+        {{printf "- -tlsCertProvider=%s" .Values.kubearmor.tls.tlsCertProvider}}
         image: {{printf "%s:%s" .Values.kubearmor.image.repository .Values.kubearmor.image.tag}}
         imagePullPolicy: {{ .Values.kubearmor.imagePullPolicy }}
         env:
@@ -49,6 +52,9 @@ spec:
         terminationMessagePolicy: File
         volumeMounts:
         {{- toYaml .Values.kubearmor.commonMounts | trim | nindent 10 }}
+        {{- if .Values.tls.enabled -}}
+          {{- toYaml .Values.kubearmor.tls.kubearmorCACertVolumeMount | trim | nindent 10 }}
+        {{- end -}}
         {{- if eq .Values.environment.name "docker" }}
           {{- toYaml .Values.kubearmor.volumeMountsDocker | trim | nindent 10 }}
         {{- else if eq .Values.environment.name "crio" }}
@@ -105,6 +111,9 @@ spec:
       - operator: Exists
       volumes:
       {{- toYaml .Values.kubearmor.commonVolumes | trim | nindent 8}}
+      {{- if .Values.tls.enabled -}}
+        {{- toYaml .Values.kubearmor.tls.kubearmorCACertVolume | trim | nindent 8 }}
+      {{- end -}}
       {{- if eq .Values.environment.name "docker" }}
         {{- toYaml .Values.kubearmor.volumesDocker | trim | nindent 8 }}
       {{- else if eq .Values.environment.name "crio" }}

deployments/helm/KubeArmor/templates/deployment.yaml

@@ -19,7 +19,11 @@ spec:
         kubearmor-app: kubearmor-relay
     spec:
       containers:
-      - image: {{printf "%s:%s" .Values.kubearmorRelay.image.repository .Values.kubearmorRelay.image.tag}}
+      - args:
+        {{printf "- -tlsEnabled=%t" .Values.tls.enabled}}
+        {{printf "- -tlsCertPath=%s" .Values.kubearmorRelay.tls.tlsCertPath}}
+        {{printf "- -tlsCertProvider=%s" .Values.kubearmorRelay.tls.tlsCertProvider}}
+        image: {{printf "%s:%s" .Values.kubearmorRelay.image.repository .Values.kubearmorRelay.image.tag}}
         imagePullPolicy: {{ .Values.kubearmorRelay.imagePullPolicy }}
         name: kubearmor-relay-server
         env:
@@ -31,9 +35,17 @@ spec:
           value: {{ quote .Values.kubearmorRelay.enableStdoutMsg }}
         ports:
         - containerPort: 32767
+        {{- if .Values.tls.enabled }}
+        volumeMounts:
+          {{- toYaml .Values.kubearmorRelay.tls.certVolumeMount | trim | nindent 10 }}
+        {{- end}}
       nodeSelector:
         kubernetes.io/os: linux
       serviceAccountName: kubearmor-relay
+      {{- if .Values.tls.enabled }}
+      volumes:
+        {{- toYaml .Values.kubearmorRelay.tls.certVolume | trim | nindent 8 }}
+      {{- end }}
 {{- end }}
 ---
 apiVersion: apps/v1

deployments/helm/KubeArmor/templates/kubearmor-tls-secrets.yaml

@@ -0,0 +1,52 @@
+# kubearmor-ca secret
+{{- $ca := genCA "kubearmor"  1095 -}}
+{{- if .Values.tls.enabled }}
+apiVersion: v1
+kind: Secret
+type: kubernetes.io/tls
+metadata:
+  name: {{ .Values.kubearmor.tls.caSecretName }}
+  labels:
+    kubearmor-app: {{ .Values.kubearmor.tls.caSecretName }}
+  namespace: {{.Release.Namespace}}
+data:
+  tls.key: {{ $ca.Key | b64enc }}
+  tls.crt: {{ $ca.Cert | b64enc }}
+{{- end}}
+---
+{{- if .Values.tls.enabled }}
+{{- $altNames := list ( printf "kubearmor.%s" .Release.Namespace ) ( printf "kubearmor.%s.svc" .Release.Namespace ) ( printf "kubearmor.%s.svc.cluster.local" .Release.Namespace ) -}}
+{{- $dns := concat .Values.kubearmorRelay.tls.extraDnsNames $altNames -}}
+{{- $ip := .Values.kubearmorRelay.tls.extraIpAddresses -}}
+{{- $cert := genSignedCert "kubearmor-relay" $ip $dns 1095 $ca -}}
+# kubearmor-relay-certs secret
+apiVersion: v1
+kind: Secret
+type: kubernetes.io/tls
+metadata:
+  name: {{ .Values.kubearmorRelay.tls.certSecretName }}
+  labels:
+    kubearmor-app: {{ .Values.kubearmorRelay.tls.certSecretName }}
+  namespace: {{.Release.Namespace}}
+data:
+  tls.key: {{ $cert.Key | b64enc }}
+  tls.crt: {{ $cert.Cert | b64enc }}
+  ca.crt: {{ $ca.Cert | b64enc }}
+{{- end}}
+---
+{{- if .Values.tls.enabled }}
+{{- $cert := genSignedCert "kubearmor-relay" nil nil 1095 $ca -}}
+# kubearmor-client-certs secret
+apiVersion: v1
+kind: Secret
+type: kubernetes.io/tls
+metadata:
+  name: {{ .Values.kubearmor.tls.clientCertSecretName }}
+  labels:
+    kubearmor-app: {{ .Values.kubearmor.tls.clientCertSecretName }}
+  namespace: {{.Release.Namespace}}
+data:
+  tls.key: {{ $cert.Key | b64enc }}
+  tls.crt: {{ $cert.Cert | b64enc }}
+  ca.crt: {{ $ca.Cert | b64enc }}
+{{- end}}

deployments/helm/KubeArmor/values.yaml

@@ -5,6 +5,9 @@ environment:
   # The target environment to install KubeArmor in. Possible values: generic, GKE, EKS, BottleRocket, k0s, k3s, minikube, microk8s
   name: generic
 
+tls:
+  enabled: false  
+
 kubearmorRelay:
   # to enable/disable kubearmor-relay
   enabled: true
@@ -20,6 +23,39 @@ kubearmorRelay:
   enableStdoutAlerts: "false"
   enableStdoutMsg: "false"
 
+  tls:
+    extraDnsNames: ["localhost"]
+    extraIpAddresses: ["127.0.0.1"]
+    tlsCertPath: /var/lib/kubearmor/tls
+    tlsCertProvider: external
+    certSecretName: kubearmor-relay-server-certs
+    certVolumeMount:
+    - mountPath: /var/lib/kubearmor/tls
+      name: kubearmor-relay-certs-secrets
+      readOnly: true
+    certVolume:
+    - name: kubearmor-relay-certs-secrets
+      projected:
+        defaultMode: 0444
+        sources:
+        - secret:
+            name: kubearmor-relay-server-certs
+            items:
+            - key: tls.crt
+              path: server.crt
+            - key: tls.key
+              path: server.key
+            - key: ca.crt
+              path: ca.crt
+        - secret:
+            name: kubearmor-client-certs
+            items:
+            - key: tls.crt
+              path: client.crt
+            - key: tls.key
+              path: client.key  
+
+
 kubearmorInit:
   image:
     # kubearmor-init image repo
@@ -73,6 +109,28 @@ kubearmor:
   # kubearmor daemonset arguments. See `kubearmor --help`
   args: []
 
+  tls:
+    tlsCertPath: /var/lib/kubearmor/tls
+    tlsCertProvider: self
+    caSecretName: kubearmor-ca
+    clientCertSecretName: kubearmor-client-certs
+    kubearmorCACertVolumeMount:
+    - mountPath: /var/lib/kubearmor/tls
+      name: kubearmor-ca-secret
+      readOnly: true
+    kubearmorCACertVolume:
+    - name: kubearmor-ca-secret
+      projected:
+        defaultMode: 0444
+        sources:
+        - secret:
+            name: kubearmor-ca
+            items:
+            - key: tls.crt
+              path: ca.crt
+            - key: tls.key
+              path: ca.key
+
   capabilities:
     add:
     - SETUID