Migrate kube-bench and image analyzer into kvisor cli (#204)

castai · Dec 6, 2023 · 651a20e · 651a20e
1 parent f88fb58
commit 651a20e
Show file tree

Hide file tree

Showing 99 changed files with 11,156 additions and 239 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -1,4 +1,3 @@
 testdata
-cmd
 node_modules
-dist
+dist
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -41,29 +41,15 @@ jobs:
           key: ${{ runner.os }}-build-${{ hashFiles('**/go.sum') }}
           restore-keys: ${{ runner.os }}-build-
 
-      - name: Build agent go binary amd64
-        run: go build -ldflags "-s -w -X main.GitCommit=$GITHUB_SHA -X main.GitRef=$GITHUB_REF -X main.Version=${RELEASE_TAG:-commit-$GITHUB_SHA}" -o bin/castai-kvisor-amd64 ./cmd/agent
-        env:
-          GOOS: linux
-          GOARCH: amd64
-          CGO_ENABLED: 0
-
-      - name: Build imgcollector go binary amd64
-        run: go build -ldflags "-s -w -X main.GitCommit=$GITHUB_SHA -X main.GitRef=$GITHUB_REF -X main.Version=${RELEASE_TAG:-commit-$GITHUB_SHA}" -o bin/castai-imgcollector-amd64 ./cmd/imgcollector
+      - name: Build kvisor go binary amd64
+        run: go build -ldflags "-s -w -X main.GitCommit=$GITHUB_SHA -X main.GitRef=$GITHUB_REF -X main.Version=${RELEASE_TAG:-commit-$GITHUB_SHA}" -o bin/castai-kvisor-amd64 ./cmd/kvisor
         env:
           GOOS: linux
           GOARCH: amd64
           CGO_ENABLED: 0
 
       - name: Build agent go binary arm64
-        run: go build -ldflags "-s -w -X main.GitCommit=$GITHUB_SHA -X main.GitRef=$GITHUB_REF -X main.Version=${RELEASE_TAG:-commit-$GITHUB_SHA}" -o bin/castai-kvisor-arm64 ./cmd/agent
-        env:
-          GOOS: linux
-          GOARCH: arm64
-          CGO_ENABLED: 0
-
-      - name: Build imgcollector go binary arm64
-        run: go build -ldflags "-s -w -X main.GitCommit=$GITHUB_SHA -X main.GitRef=$GITHUB_REF -X main.Version=${RELEASE_TAG:-commit-$GITHUB_SHA}" -o bin/castai-imgcollector-arm64 ./cmd/imgcollector
+        run: go build -ldflags "-s -w -X main.GitCommit=$GITHUB_SHA -X main.GitRef=$GITHUB_REF -X main.Version=${RELEASE_TAG:-commit-$GITHUB_SHA}" -o bin/castai-kvisor-arm64 ./cmd/kvisor
         env:
           GOOS: linux
           GOARCH: arm64
@@ -106,45 +92,26 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Build and push pr (agent)
+      - name: Build and push pr (kvisor)
         if: ${{ github.event_name == 'pull_request' }}
         uses: docker/build-push-action@v2
         with:
           context: .
-          file: ./Dockerfile.agent
+          file: ./Dockerfile
           platforms: linux/arm64,linux/amd64
           push: ${{ github.event_name == 'pull_request' }}
           tags: ghcr.io/castai/kvisor/kvisor:${{ github.sha }}
 
-      - name: Build and push pr (imgcollector)
-        if: ${{ github.event_name == 'pull_request' }}
-        uses: docker/build-push-action@v2
-        with:
-          context: .
-          file: ./Dockerfile.imgcollector
-          platforms: linux/arm64,linux/amd64
-          push: ${{ github.event_name == 'pull_request' }}
-          tags: ghcr.io/castai/kvisor/kvisor-imgcollector:${{ github.sha }}
-
-      - name: Build and push main (agent)
+      - name: Build and push main (kvisor)
         if: ${{ github.event_name != 'pull_request' && github.event_name != 'release' }}
         uses: docker/build-push-action@v2
         with:
           context: .
-          file: ./Dockerfile.agent
+          file: ./Dockerfile
           platforms: linux/arm64,linux/amd64
           push: ${{ github.event_name != 'pull_request' }}
           tags: us-docker.pkg.dev/castai-hub/library/kvisor:${{ github.sha }}
 
-      - name: Build and push main (imgcollector)
-        if: ${{ github.event_name != 'pull_request' && github.event_name != 'release' }}
-        uses: docker/build-push-action@v2
-        with:
-          context: .
-          file: ./Dockerfile.imgcollector
-          platforms: linux/arm64,linux/amd64
-          push: ${{ github.event_name != 'pull_request' }}
-          tags: us-docker.pkg.dev/castai-hub/library/kvisor-imgcollector:${{ github.sha }}
   e2e:
     name: E2E
     runs-on: ubuntu-20.04

diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -36,29 +36,15 @@ jobs:
       - name: Get release tag
         run: echo "RELEASE_TAG=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
 
-      - name: Build agent go binary amd64
-        run: go build -ldflags "-s -w -X main.GitCommit=$GITHUB_SHA -X main.GitRef=$GITHUB_REF -X main.Version=${RELEASE_TAG:-commit-$GITHUB_SHA}" -o bin/castai-kvisor-amd64 ./cmd/agent
+      - name: Build kvisor go binary amd64
+        run: go build -ldflags "-s -w -X main.GitCommit=$GITHUB_SHA -X main.GitRef=$GITHUB_REF -X main.Version=${RELEASE_TAG:-commit-$GITHUB_SHA}" -o bin/castai-kvisor-amd64 ./cmd/kvisor
         env:
           GOOS: linux
           GOARCH: amd64
           CGO_ENABLED: 0
 
-      - name: Build imgcollector go binary amd64
-        run: go build -ldflags "-s -w -X main.GitCommit=$GITHUB_SHA -X main.GitRef=$GITHUB_REF -X main.Version=${RELEASE_TAG:-commit-$GITHUB_SHA}" -o bin/castai-imgcollector-amd64 ./cmd/imgcollector
-        env:
-          GOOS: linux
-          GOARCH: amd64
-          CGO_ENABLED: 0
-
-      - name: Build agent go binary arm64
-        run: go build -ldflags "-s -w -X main.GitCommit=$GITHUB_SHA -X main.GitRef=$GITHUB_REF -X main.Version=${RELEASE_TAG:-commit-$GITHUB_SHA}" -o bin/castai-kvisor-arm64 ./cmd/agent
-        env:
-          GOOS: linux
-          GOARCH: arm64
-          CGO_ENABLED: 0
-
-      - name: Build imgcollector go binary arm64
-        run: go build -ldflags "-s -w -X main.GitCommit=$GITHUB_SHA -X main.GitRef=$GITHUB_REF -X main.Version=${RELEASE_TAG:-commit-$GITHUB_SHA}" -o bin/castai-imgcollector-arm64 ./cmd/imgcollector
+      - name: Build kvisor go binary arm64
+        run: go build -ldflags "-s -w -X main.GitCommit=$GITHUB_SHA -X main.GitRef=$GITHUB_REF -X main.Version=${RELEASE_TAG:-commit-$GITHUB_SHA}" -o bin/castai-kvisor-arm64 ./cmd/kvisor
         env:
           GOOS: linux
           GOARCH: arm64
@@ -77,28 +63,17 @@ jobs:
           username: _json_key
           password: ${{ secrets.ARTIFACT_BUILDER_JSON_KEY }}
 
-      - name: Build and push release (agent)
+      - name: Build and push release (kvisor)
         uses: docker/build-push-action@v2
         with:
           context: .
           push: true
-          file: ./Dockerfile.agent
+          file: ./Dockerfile
           platforms: linux/arm64,linux/amd64
           tags: |
             us-docker.pkg.dev/castai-hub/library/kvisor:${{ env.RELEASE_TAG }}
             us-docker.pkg.dev/castai-hub/library/kvisor:latest
 
-      - name: Build and push release (imgcollector)
-        uses: docker/build-push-action@v2
-        with:
-          context: .
-          push: true
-          file: ./Dockerfile.imgcollector
-          platforms: linux/arm64,linux/amd64
-          tags: |
-            us-docker.pkg.dev/castai-hub/library/kvisor-imgcollector:${{ env.RELEASE_TAG }}
-            us-docker.pkg.dev/castai-hub/library/kvisor-imgcollector:latest
-
   release_chart:
     name: Release Helm Chart
     runs-on: ubuntu-20.04

diff --git a/.golangci.yaml b/.golangci.yaml
@@ -20,3 +20,4 @@ run:
     - .github
     - charts
     - examples
+    - cmd/kvisor/kubebench # TODO: Fix kubebench issues
diff --git a/Dockerfile b/Dockerfile
@@ -0,0 +1,9 @@
+FROM alpine:3.18.5
+
+# Needed for kube-bench.
+RUN apk --no-cache add procps
+
+ARG TARGETARCH
+COPY ./bin/castai-kvisor-$TARGETARCH /usr/local/bin/castai-kvisor
+COPY ./cmd/kvisor/kubebench/kubebench-rules /etc/kubebench-rules
+ENTRYPOINT ["/usr/local/bin/castai-kvisor"]
diff --git a/Dockerfile.agent b/Dockerfile.agent
diff --git a/Dockerfile.imgcollector b/Dockerfile.imgcollector
diff --git a/Dockerfile.imgcollector.tilt b/Dockerfile.imgcollector.tilt
diff --git a/Dockerfile.tilt b/Dockerfile.tilt
@@ -11,5 +11,5 @@ RUN /busybox --install
 FROM base-with-shell
 
 COPY ./bin/castai-kvisor /usr/local/bin/castai-kvisor
-CMD ["/usr/local/bin/castai-kvisor"]
+ENTRYPOINT ["/usr/local/bin/castai-kvisor"]
 
diff --git a/charts/castai-kvisor/templates/deployment.yaml b/charts/castai-kvisor/templates/deployment.yaml
@@ -68,9 +68,11 @@ spec:
       securityContext:
       {{- toYaml .Values.securityContext | nindent 8 }}
       containers:
-        - name: kvisor
+        - name: kvisor # It's important to keep this name as is since we  search for image name in kube controller.
           image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
           imagePullPolicy: {{ .Values.image.pullPolicy }}
+          args:
+            - "agent"
           env:
             - name: POD_IP
               valueFrom:

diff --git a/charts/castai-kvisor/values.yaml b/charts/castai-kvisor/values.yaml
@@ -69,6 +69,7 @@ config: |
     rateLimit:
       burst: 150
       qps: 25
+  provider: "" # Kubernetes provider (aks, eks, gke)
   log:
     level: "debug"
   deltaSyncInterval: "15s"
@@ -79,7 +80,6 @@ config: |
     enabled: true
     scanInterval: "30s"
     image:
-      name: "ghcr.io/castai/kvisor/kube-bench:v0.8.0"
       pullPolicy: IfNotPresent
   imageScan:
     enabled: true
@@ -89,7 +89,6 @@ config: |
     serviceAccountName: "{{ (.Values.imageScanServiceAccount | default dict).name }}"
     apiUrl: "http://kvisor.{{ .Release.Namespace }}.svc.cluster.local.:6060"
     image:
-      name: "{{ .Values.image.repository }}-imgcollector:{{ .Values.image.tag | default .Chart.AppVersion }}"
       pullPolicy: IfNotPresent
     {{ if .Values.imageScanSecret }}
     pullSecret: "{{ .Values.imageScanSecret }}"

diff --git a/cmd/agent/main.go → cmd/kvisor/agent/agent.go b/cmd/agent/main.go → cmd/kvisor/agent/agent.go
@@ -1,4 +1,4 @@
-package main
+package agent
 
 import (
 	"context"
@@ -21,6 +21,7 @@ import (
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/samber/lo"
 	"github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
 	"k8s.io/apimachinery/pkg/api/meta"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
@@ -56,64 +57,63 @@ import (
 	"github.com/castai/kvisor/version"
 )
 
-// These should be set via `go build` during a release.
-var (
-	GitCommit = "undefined"
-	GitRef    = "no-ref"
-	Version   = "local"
-)
-
-var (
-	configPath = flag.String("config", "/etc/castai/config/config.yaml", "Config file path")
-)
-
-func main() {
-	flag.Parse()
-
-	logger := logrus.New()
-	cfg, err := config.Load(*configPath)
-	if err != nil {
-		logger.Fatal(err)
-	}
-	lvl, _ := logrus.ParseLevel(cfg.Log.Level)
-	logger.SetLevel(lvl)
+func NewCommand(version, gitCommit, gitRef string) *cobra.Command {
+	var configPath string
+	cmd := &cobra.Command{
+		Use:   "agent",
+		Short: "Run kvisor agent server",
+		Run: func(cmd *cobra.Command, args []string) {
+			flag.Parse()
 
-	binVersion := config.SecurityAgentVersion{
-		GitCommit: GitCommit,
-		GitRef:    GitRef,
-		Version:   Version,
-	}
+			logger := logrus.New()
+			cfg, err := config.Load(configPath)
+			if err != nil {
+				logger.Fatal(err)
+			}
+			lvl, _ := logrus.ParseLevel(cfg.Log.Level)
+			logger.SetLevel(lvl)
 
-	client := castai.NewClient(
-		cfg.API.URL, cfg.API.Key,
-		logger,
-		cfg.API.ClusterID,
-		cfg.PolicyEnforcement.Enabled,
-		"castai-kvisor",
-		binVersion,
-	)
+			binVersion := config.SecurityAgentVersion{
+				GitCommit: gitCommit,
+				GitRef:    gitRef,
+				Version:   version,
+			}
 
-	log := logrus.WithFields(logrus.Fields{})
-	e := agentlog.NewExporter(logger, client, []logrus.Level{
-		logrus.ErrorLevel,
-		logrus.FatalLevel,
-		logrus.PanicLevel,
-		logrus.InfoLevel,
-		logrus.WarnLevel,
-	})
+			client := castai.NewClient(
+				cfg.API.URL, cfg.API.Key,
+				logger,
+				cfg.API.ClusterID,
+				cfg.PolicyEnforcement.Enabled,
+				"castai-kvisor",
+				binVersion,
+			)
+
+			log := logrus.WithFields(logrus.Fields{})
+			e := agentlog.NewExporter(logger, client, []logrus.Level{
+				logrus.ErrorLevel,
+				logrus.FatalLevel,
+				logrus.PanicLevel,
+				logrus.InfoLevel,
+				logrus.WarnLevel,
+			})
 
-	logger.AddHook(e)
-	logrus.RegisterExitHandler(e.Wait)
+			logger.AddHook(e)
+			logrus.RegisterExitHandler(e.Wait)
 
-	ctx := signals.SetupSignalHandler()
-	if err := run(ctx, logger, client, cfg, binVersion); err != nil && !errors.Is(err, context.Canceled) {
-		logErr := &logContextErr{}
-		if errors.As(err, &logErr) {
-			log = logger.WithFields(logErr.fields)
-		}
-		log.Fatalf("castai-kvisor failed: %v", err)
+			ctx := signals.SetupSignalHandler()
+			if err := run(ctx, logger, client, cfg, binVersion); err != nil && !errors.Is(err, context.Canceled) {
+				logErr := &logContextErr{}
+				if errors.As(err, &logErr) {
+					log = logger.WithFields(logErr.fields)
+				}
+				log.Fatalf("castai-kvisor failed: %v", err)
+			}
+			log.Info("castai-kvisor stopped")
+		},
 	}
-	log.Info("castai-kvisor stopped")
+	cmd.PersistentFlags().StringVar(&configPath, "config", "/etc/castai/config/config.yaml", "Config file path")
+
+	return cmd
 }
 
 func run(ctx context.Context, logger logrus.FieldLogger, castaiClient castai.Client, cfg config.Config, binVersion config.SecurityAgentVersion) (reterr error) {
@@ -160,7 +160,7 @@ func run(ctx context.Context, logger logrus.FieldLogger, castaiClient castai.Cli
 	snapshotProvider := delta.NewSnapshotProvider()
 
 	informersFactory := informers.NewSharedInformerFactory(clientSet, 0)
-	kubeCtrl := kube.NewController(log, informersFactory, k8sVersion)
+	kubeCtrl := kube.NewController(log, informersFactory, k8sVersion, cfg.PodNamespace)
 
 	deltaCtrl := delta.NewController(
 		log,
@@ -209,6 +209,7 @@ func run(ctx context.Context, logger logrus.FieldLogger, castaiClient castai.Cli
 			cfg.KubeBench.ScanInterval,
 			castaiClient,
 			podLogReader,
+			kubeCtrl,
 			scannedNodes,
 		)
 		kubeCtrl.AddSubscribers(kubeBenchCtrl)

diff --git a/cmd/agent/main_test.go → cmd/kvisor/agent/agent_test.go b/cmd/agent/main_test.go → cmd/kvisor/agent/agent_test.go
@@ -1,4 +1,4 @@
-package main
+package agent
 
 import (
 	"errors"

diff --git a/cmd/imgcollector/collector/collector.go → ...visor/imgcollector/collector/collector.go b/cmd/imgcollector/collector/collector.go → ...visor/imgcollector/collector/collector.go
@@ -13,6 +13,7 @@ import (
 	"time"
 
 	fanalyzer "github.com/aquasecurity/trivy/pkg/fanal/analyzer"
+	"github.com/castai/kvisor/cmd/kvisor/imgcollector/config"
 	"github.com/cenkalti/backoff/v4"
 	"github.com/google/go-containerregistry/pkg/name"
 	"github.com/samber/lo"
@@ -23,7 +24,6 @@ import (
 	"github.com/castai/image-analyzer/image"
 	"github.com/castai/image-analyzer/image/hostfs"
 	"github.com/castai/kvisor/castai"
-	"github.com/castai/kvisor/cmd/imgcollector/config"
 )
 
 func New(log logrus.FieldLogger, cfg config.Config, cache analyzer.CacheClient, hostfsConfig *hostfs.ContainerdHostFSConfig) *Collector {
-Original file line number
+Diff line change
@@ Expand Up / @@ -20,3 +20,4 @@ run: @@
         - .github
         - charts
         - examples
+        - cmd/kvisor/kubebench # TODO: Fix kubebench issues