Skip to content

Commit

Permalink
Find kvisor image from spec
Browse files Browse the repository at this point in the history
  • Loading branch information
anjmao committed Dec 6, 2023
1 parent 8f93cd2 commit 1e50e7d
Show file tree
Hide file tree
Showing 12 changed files with 75 additions and 36 deletions.
2 changes: 1 addition & 1 deletion charts/castai-kvisor/templates/deployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ spec:
securityContext:
{{- toYaml .Values.securityContext | nindent 8 }}
containers:
- name: kvisor
- name: kvisor # It's important to keep this name as is since we search for image name in kube controller.
image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
imagePullPolicy: {{ .Values.image.pullPolicy }}
args:
Expand Down
2 changes: 0 additions & 2 deletions charts/castai-kvisor/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,6 @@ config: |
enabled: true
scanInterval: "30s"
image:
name: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
pullPolicy: IfNotPresent
imageScan:
enabled: true
Expand All @@ -90,7 +89,6 @@ config: |
serviceAccountName: "{{ (.Values.imageScanServiceAccount | default dict).name }}"
apiUrl: "http://kvisor.{{ .Release.Namespace }}.svc.cluster.local.:6060"
image:
name: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
pullPolicy: IfNotPresent
{{ if .Values.imageScanSecret }}
pullSecret: "{{ .Values.imageScanSecret }}"
Expand Down
8 changes: 1 addition & 7 deletions config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,6 @@ type ImageScan struct {
}

type ImageScanImage struct {
Name string `envconfig:"IMAGE_SCAN_IMAGE_NAME" yaml:"name"`
PullPolicy string `envconfig:"IMAGE_SCAN_IMAGE_PULL_POLICY" yaml:"pullPolicy"`
}

Expand All @@ -103,7 +102,6 @@ type KubeBench struct {
}

type KubeBenchImage struct {
Name string `envconfig:"KUBE_BENCH_IMAGE_NAME" yaml:"name"`
PullPolicy string `envconfig:"KUBE_BENCH_IMAGE_PULL_POLICY" yaml:"pullPolicy"`
}

Expand Down Expand Up @@ -174,9 +172,6 @@ func Load(configPath string) (Config, error) {
}
}
if cfg.ImageScan.Enabled {
if cfg.ImageScan.Image.Name == "" {
return cfg, required("IMAGE_SCAN_IMAGE_NAME")
}
if cfg.ImageScan.Image.PullPolicy == "" {
cfg.ImageScan.Image.PullPolicy = "IfNotPresent"
}
Expand Down Expand Up @@ -221,8 +216,7 @@ func Load(configPath string) (Config, error) {
if cfg.KubeBench.ScanInterval == 0 {
cfg.KubeBench.ScanInterval = 30 * time.Second
}
if cfg.KubeBench.Image.Name == "" {
cfg.KubeBench.Image.Name = "ghcr.io/castai/kvisor/kube-bench:v0.8.0"
if cfg.KubeBench.Image.PullPolicy == "" {
cfg.KubeBench.Image.PullPolicy = "IfNotPresent"
}
}
Expand Down
2 changes: 0 additions & 2 deletions config/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,6 @@ func newTestConfig() Config {
MaxConcurrentScans: 3,
InitDelay: 60 * time.Second,
Image: ImageScanImage{
Name: "collector-img",
PullPolicy: "IfNotPresent",
},
Mode: "mode",
Expand All @@ -94,7 +93,6 @@ func newTestConfig() Config {
Enabled: true,
ScanInterval: 15 * time.Second,
Image: KubeBenchImage{
Name: "ghcr.io/castai/kvisor/kube-bench:v0.8.",
PullPolicy: "IfNotPresent",
},
},
Expand Down
7 changes: 6 additions & 1 deletion imagescan/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -312,6 +312,11 @@ func (s *Controller) scanImage(ctx context.Context, img *image) (rerr error) {
metrics.ObserveScanDuration(metrics.ScanTypeImage, start)
}()

collectorImageDetails, found := s.kubeController.GetKvisorImageDetails()
if !found {
return errors.New("kvisor image details not found")
}

return s.imageScanner.ScanImage(ctx, ScanImageParams{
ImageName: img.name,
ImageID: img.id,
Expand All @@ -324,7 +329,7 @@ func (s *Controller) scanImage(ctx context.Context, img *image) (rerr error) {
WaitDurationAfterCompletion: 30 * time.Second,
Architecture: img.architecture,
Os: img.os,
ImageCollectorPullSecrets: s.kubeController.GetKvisorImagePullSecret(),
CollectorImageDetails: collectorImageDetails,
})
}

Expand Down
12 changes: 10 additions & 2 deletions imagescan/controller_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"time"

imgcollectorconfig "github.com/castai/kvisor/cmd/kvisor/imgcollector/config"
"github.com/castai/kvisor/kube"
"github.com/google/uuid"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/mock"
Expand Down Expand Up @@ -236,6 +237,10 @@ func TestSubscriber(t *testing.T) {
WaitDurationAfterCompletion: 30 * time.Second,
Architecture: defaultImageArch,
Os: defaultImageOs,
CollectorImageDetails: kube.KvisorImageDetails{
ImageName: "kvisor",
ImagePullSecrets: nil,
},
}, ngnxImage)
r.Len(client.getImagesResourcesChanges(), 1)
r.Len(client.getImagesResourcesChanges()[0].Images, 3)
Expand Down Expand Up @@ -882,8 +887,11 @@ func (m *mockImageScanner) getScanImageParams() []ScanImageParams {
type mockKubeController struct {
}

func (m *mockKubeController) GetKvisorImagePullSecret() []corev1.LocalObjectReference {
return nil
func (m *mockKubeController) GetKvisorImageDetails() (kube.KvisorImageDetails, bool) {
return kube.KvisorImageDetails{
ImageName: "kvisor",
ImagePullSecrets: nil,
}, true
}

func (m *mockKubeController) GetPodOwnerID(pod *corev1.Pod) string {
Expand Down
2 changes: 1 addition & 1 deletion imagescan/delta.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ const defaultImageArch = "amd64"

type kubeController interface {
GetPodOwnerID(pod *corev1.Pod) string
GetKvisorImagePullSecret() []corev1.LocalObjectReference
GetKvisorImageDetails() (kube.KvisorImageDetails, bool)
}

func newImage() *image {
Expand Down
11 changes: 6 additions & 5 deletions imagescan/scanner.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"time"

imgcollectorconfig "github.com/castai/kvisor/cmd/kvisor/imgcollector/config"
"github.com/castai/kvisor/kube"
"github.com/samber/lo"
batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
Expand Down Expand Up @@ -66,7 +67,7 @@ type ScanImageParams struct {
WaitDurationAfterCompletion time.Duration
Architecture string
Os string
ImageCollectorPullSecrets []corev1.LocalObjectReference
CollectorImageDetails kube.KvisorImageDetails
}

func (s *Scanner) ScanImage(ctx context.Context, params ScanImageParams) (rerr error) {
Expand Down Expand Up @@ -249,7 +250,7 @@ func (s *Scanner) ScanImage(ctx context.Context, params ScanImageParams) (rerr e
vols,
tolerations,
s.cfg.ImageScan,
params.ImageCollectorPullSecrets,
params.CollectorImageDetails,
)
jobs := s.client.BatchV1().Jobs(s.cfg.PodNamespace)

Expand Down Expand Up @@ -387,7 +388,7 @@ func scanJobSpec(
vol volumesAndMounts,
tolerations []corev1.Toleration,
cfg config.ImageScan,
collectorImagePullSecrets []corev1.LocalObjectReference,
collectorImageDetails kube.KvisorImageDetails,
) *batchv1.Job {
job := &batchv1.Job{
TypeMeta: metav1.TypeMeta{
Expand Down Expand Up @@ -437,7 +438,7 @@ func scanJobSpec(
Tolerations: tolerations,
AutomountServiceAccountToken: lo.ToPtr(false),
ServiceAccountName: cfg.ServiceAccountName,
ImagePullSecrets: collectorImagePullSecrets,
ImagePullSecrets: collectorImageDetails.ImagePullSecrets,
Containers: []corev1.Container{
{
SecurityContext: &corev1.SecurityContext{
Expand All @@ -446,7 +447,7 @@ func scanJobSpec(
AllowPrivilegeEscalation: lo.ToPtr(false),
},
Name: "collector",
Image: cfg.Image.Name,
Image: collectorImageDetails.ImageName,
Args: []string{
"analyze-image",
},
Expand Down
18 changes: 12 additions & 6 deletions imagescan/scanner_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"testing"
"time"

"github.com/castai/kvisor/kube"
"github.com/samber/lo"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/require"
Expand Down Expand Up @@ -35,9 +36,7 @@ func TestScanner(t *testing.T) {
PodIP: "10.10.5.77",
PodNamespace: ns,
ImageScan: config.ImageScan{
Image: config.ImageScanImage{
Name: "imgcollector:1.0.0",
},
Image: config.ImageScanImage{},
APIUrl: "http://kvisor:6060",
DockerOptionsPath: "/etc/docker/config.json",
CPURequest: "500m",
Expand All @@ -61,6 +60,9 @@ func TestScanner(t *testing.T) {
ResourceIDs: []string{"p1", "p2"},
Architecture: "amd64",
Os: "linux",
CollectorImageDetails: kube.KvisorImageDetails{
ImageName: "imgcollector:1.0.0",
},
})
r.NoError(err)

Expand Down Expand Up @@ -265,6 +267,9 @@ func TestScanner(t *testing.T) {
NodeName: "n1",
ResourceIDs: []string{"p1", "p2"},
DeleteFinishedJob: true,
CollectorImageDetails: kube.KvisorImageDetails{
ImageName: "imgcollector:1.0.0",
},
})
r.NoError(err)

Expand Down Expand Up @@ -306,9 +311,7 @@ func TestScanner(t *testing.T) {
scanner := NewImageScanner(client, config.Config{
PodNamespace: ns,
ImageScan: config.ImageScan{
Image: config.ImageScanImage{
Name: "imgcollector:1.0.0",
},
Image: config.ImageScanImage{},
},
})
scanner.jobCheckInterval = 1 * time.Microsecond
Expand All @@ -321,6 +324,9 @@ func TestScanner(t *testing.T) {
NodeName: "n1",
ResourceIDs: []string{"p1", "p2"},
WaitForCompletion: true,
CollectorImageDetails: kube.KvisorImageDetails{
ImageName: "imgcollector:1.0.0",
},
})
r.ErrorContains(err, "[type=Ready, status=False, reason=no cpu], [type=PodScheduled, status=False, reason=no cpu]")
})
Expand Down
27 changes: 24 additions & 3 deletions kube/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -172,13 +172,34 @@ func (c *Controller) GetPodOwnerID(pod *corev1.Pod) string {
return string(pod.UID)
}

func (c *Controller) GetKvisorImagePullSecret() []corev1.LocalObjectReference {
type KvisorImageDetails struct {
ImageName string
ImagePullSecrets []corev1.LocalObjectReference
}

// GetKvisorImageDetails returns kvisor image details.
// This is used for image analyzer and kube-bench dynamic jobs to schedule using the same image.
func (c *Controller) GetKvisorImageDetails() (KvisorImageDetails, bool) {
spec, found := c.getKvisorDeploymentSpec()
if !found {
c.log.Warn("kvisor deployment not found")
return nil
return KvisorImageDetails{}, false
}
var imageName string
for _, container := range spec.Template.Spec.Containers {
if container.Name == "kvisor" {
imageName = container.Image
break
}
}
if imageName == "" {
c.log.Warn("kvisor container image not found")
return KvisorImageDetails{}, false
}
return spec.Template.Spec.ImagePullSecrets
return KvisorImageDetails{
ImageName: imageName,
ImagePullSecrets: spec.Template.Spec.ImagePullSecrets,
}, true
}

func (c *Controller) getKvisorDeploymentSpec() (appsv1.DeploymentSpec, bool) {
Expand Down
12 changes: 8 additions & 4 deletions linters/kubebench/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ const (
)

type kubeController interface {
GetKvisorImagePullSecret() []corev1.LocalObjectReference
GetKvisorImageDetails() (kube.KvisorImageDetails, bool)
}

func NewController(
Expand Down Expand Up @@ -287,12 +287,16 @@ func (s *Controller) createKubebenchJob(ctx context.Context, node *corev1.Node,
specFn := resolveSpec(s.provider, node)
jobSpec := specFn(node.GetName(), jobName)

// Set image from config.
// Set image.
imageDetails, found := s.kubeController.GetKvisorImageDetails()
if !found {
return nil, errors.New("kvisor image details not found")
}
cont := jobSpec.Spec.Template.Spec.Containers[0]
cont.Image = s.cfg.Image.Name
cont.Image = imageDetails.ImageName
cont.ImagePullPolicy = corev1.PullPolicy(s.cfg.Image.PullPolicy)
jobSpec.Spec.Template.Spec.Containers[0] = cont
jobSpec.Spec.Template.Spec.ImagePullSecrets = s.kubeController.GetKvisorImagePullSecret()
jobSpec.Spec.Template.Spec.ImagePullSecrets = imageDetails.ImagePullSecrets

job, err := s.client.BatchV1().
Jobs(s.castaiNamespace).
Expand Down
8 changes: 6 additions & 2 deletions linters/kubebench/controller_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (

"github.com/castai/kvisor/castai"
"github.com/castai/kvisor/config"
"github.com/castai/kvisor/kube"
"github.com/castai/kvisor/log"
"github.com/golang/mock/gomock"
"github.com/google/uuid"
Expand Down Expand Up @@ -285,8 +286,11 @@ func readReport() []byte {
type mockKubeController struct {
}

func (m *mockKubeController) GetKvisorImagePullSecret() []corev1.LocalObjectReference {
return []corev1.LocalObjectReference{}
func (m *mockKubeController) GetKvisorImageDetails() (kube.KvisorImageDetails, bool) {
return kube.KvisorImageDetails{
ImageName: "kvisor",
ImagePullSecrets: nil,
}, true
}

func (m *mockKubeController) GetPodOwnerID(pod *corev1.Pod) string {
Expand Down

0 comments on commit 1e50e7d

Please sign in to comment.