From 2ee899ee481a1ab996b7afadb4678bb5a6a27415 Mon Sep 17 00:00:00 2001 From: anjmao Date: Thu, 16 Jan 2025 12:03:35 +0200 Subject: [PATCH] Export node stats --- api/v1/runtime/common.pb.go | 240 +++++++----------- api/v1/runtime/common.proto | 11 +- charts/kvisor/values-local.yaml | 4 +- cmd/agent/daemon/state/controller.go | 18 +- cmd/agent/daemon/state/stats_pipeline.go | 182 +++++++++---- .../state/castai_controller_test.go | 2 +- cmd/mock-server/main.go | 2 +- pkg/castai/client_test.go | 2 +- pkg/cgroup/cgroup_memory.go | 78 +----- pkg/cgroup/psi.go | 16 +- pkg/proc/proc.go | 4 +- 11 files changed, 244 insertions(+), 315 deletions(-) diff --git a/api/v1/runtime/common.pb.go b/api/v1/runtime/common.pb.go index 9cfd0066..ad7f38db 100644 --- a/api/v1/runtime/common.pb.go +++ b/api/v1/runtime/common.pb.go @@ -1508,10 +1508,7 @@ type PSIData struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Avg10 float64 `protobuf:"fixed64,1,opt,name=avg10,proto3" json:"avg10,omitempty"` - Avg60 float64 `protobuf:"fixed64,2,opt,name=avg60,proto3" json:"avg60,omitempty"` - Avg300 float64 `protobuf:"fixed64,3,opt,name=avg300,proto3" json:"avg300,omitempty"` - Total uint64 `protobuf:"varint,4,opt,name=total,proto3" json:"total,omitempty"` + Total uint64 `protobuf:"varint,1,opt,name=total,proto3" json:"total,omitempty"` } func (x *PSIData) Reset() { @@ -1546,27 +1543,6 @@ func (*PSIData) Descriptor() ([]byte, []int) { return file_api_v1_runtime_common_proto_rawDescGZIP(), []int{16} } -func (x *PSIData) GetAvg10() float64 { - if x != nil { - return x.Avg10 - } - return 0 -} - -func (x *PSIData) GetAvg60() float64 { - if x != nil { - return x.Avg60 - } - return 0 -} - -func (x *PSIData) GetAvg300() float64 { - if x != nil { - return x.Avg300 - } - return 0 -} - func (x *PSIData) GetTotal() uint64 { if x != nil { return x.Total @@ -1784,16 +1760,10 @@ type MemoryStats struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // memory used for cache - Cache uint64 `protobuf:"varint,1,opt,name=cache,proto3" json:"cache,omitempty"` - // usage of memory - Usage *MemoryData `protobuf:"bytes,2,opt,name=usage,proto3" json:"usage,omitempty"` - // usage of memory + swap - SwapUsage *MemoryData `protobuf:"bytes,3,opt,name=swap_usage,json=swapUsage,proto3" json:"swap_usage,omitempty"` - // usage of swap only + Cache uint64 `protobuf:"varint,1,opt,name=cache,proto3" json:"cache,omitempty"` + Usage *MemoryData `protobuf:"bytes,2,opt,name=usage,proto3" json:"usage,omitempty"` SwapOnlyUsage *MemoryData `protobuf:"bytes,4,opt,name=swap_only_usage,json=swapOnlyUsage,proto3" json:"swap_only_usage,omitempty"` - // usage of kernel memory - Psi *PSIStats `protobuf:"bytes,8,opt,name=psi,proto3" json:"psi,omitempty"` + Psi *PSIStats `protobuf:"bytes,8,opt,name=psi,proto3" json:"psi,omitempty"` } func (x *MemoryStats) Reset() { @@ -1842,13 +1812,6 @@ func (x *MemoryStats) GetUsage() *MemoryData { return nil } -func (x *MemoryStats) GetSwapUsage() *MemoryData { - if x != nil { - return x.SwapUsage - } - return nil -} - func (x *MemoryStats) GetSwapOnlyUsage() *MemoryData { if x != nil { return x.SwapOnlyUsage @@ -2139,99 +2102,91 @@ var file_api_v1_runtime_common_proto_rawDesc = []byte{ 0x0a, 0x0a, 0x74, 0x78, 0x5f, 0x70, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x04, 0x52, 0x09, 0x74, 0x78, 0x50, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x78, 0x5f, 0x70, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x18, 0x0d, 0x20, 0x01, 0x28, - 0x04, 0x52, 0x09, 0x72, 0x78, 0x50, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x22, 0x63, 0x0a, 0x07, - 0x50, 0x53, 0x49, 0x44, 0x61, 0x74, 0x61, 0x12, 0x14, 0x0a, 0x05, 0x61, 0x76, 0x67, 0x31, 0x30, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x01, 0x52, 0x05, 0x61, 0x76, 0x67, 0x31, 0x30, 0x12, 0x14, 0x0a, - 0x05, 0x61, 0x76, 0x67, 0x36, 0x30, 0x18, 0x02, 0x20, 0x01, 0x28, 0x01, 0x52, 0x05, 0x61, 0x76, - 0x67, 0x36, 0x30, 0x12, 0x16, 0x0a, 0x06, 0x61, 0x76, 0x67, 0x33, 0x30, 0x30, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x01, 0x52, 0x06, 0x61, 0x76, 0x67, 0x33, 0x30, 0x30, 0x12, 0x14, 0x0a, 0x05, 0x74, - 0x6f, 0x74, 0x61, 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x74, 0x6f, 0x74, 0x61, - 0x6c, 0x22, 0x5c, 0x0a, 0x08, 0x50, 0x53, 0x49, 0x53, 0x74, 0x61, 0x74, 0x73, 0x12, 0x27, 0x0a, - 0x04, 0x73, 0x6f, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x72, 0x75, - 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x53, 0x49, 0x44, 0x61, 0x74, 0x61, - 0x52, 0x04, 0x73, 0x6f, 0x6d, 0x65, 0x12, 0x27, 0x0a, 0x04, 0x66, 0x75, 0x6c, 0x6c, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, - 0x31, 0x2e, 0x50, 0x53, 0x49, 0x44, 0x61, 0x74, 0x61, 0x52, 0x04, 0x66, 0x75, 0x6c, 0x6c, 0x22, - 0x83, 0x02, 0x0a, 0x08, 0x43, 0x70, 0x75, 0x53, 0x74, 0x61, 0x74, 0x73, 0x12, 0x1f, 0x0a, 0x0b, - 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x75, 0x73, 0x61, 0x67, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x04, 0x52, 0x0a, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x55, 0x73, 0x61, 0x67, 0x65, 0x12, 0x2e, 0x0a, - 0x13, 0x75, 0x73, 0x61, 0x67, 0x65, 0x5f, 0x69, 0x6e, 0x5f, 0x6b, 0x65, 0x72, 0x6e, 0x65, 0x6c, - 0x6d, 0x6f, 0x64, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x11, 0x75, 0x73, 0x61, 0x67, - 0x65, 0x49, 0x6e, 0x4b, 0x65, 0x72, 0x6e, 0x65, 0x6c, 0x6d, 0x6f, 0x64, 0x65, 0x12, 0x2a, 0x0a, - 0x11, 0x75, 0x73, 0x61, 0x67, 0x65, 0x5f, 0x69, 0x6e, 0x5f, 0x75, 0x73, 0x65, 0x72, 0x6d, 0x6f, - 0x64, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0f, 0x75, 0x73, 0x61, 0x67, 0x65, 0x49, - 0x6e, 0x55, 0x73, 0x65, 0x72, 0x6d, 0x6f, 0x64, 0x65, 0x12, 0x2b, 0x0a, 0x11, 0x74, 0x68, 0x72, - 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x64, 0x5f, 0x70, 0x65, 0x72, 0x69, 0x6f, 0x64, 0x73, 0x18, 0x04, - 0x20, 0x01, 0x28, 0x04, 0x52, 0x10, 0x74, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x64, 0x50, - 0x65, 0x72, 0x69, 0x6f, 0x64, 0x73, 0x12, 0x25, 0x0a, 0x0e, 0x74, 0x68, 0x72, 0x6f, 0x74, 0x74, - 0x6c, 0x65, 0x64, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0d, - 0x74, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x26, 0x0a, - 0x03, 0x70, 0x73, 0x69, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x72, 0x75, 0x6e, - 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x53, 0x49, 0x53, 0x74, 0x61, 0x74, 0x73, - 0x52, 0x03, 0x70, 0x73, 0x69, 0x22, 0x38, 0x0a, 0x0a, 0x4d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x44, - 0x61, 0x74, 0x61, 0x12, 0x14, 0x0a, 0x05, 0x75, 0x73, 0x61, 0x67, 0x65, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x04, 0x52, 0x05, 0x75, 0x73, 0x61, 0x67, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x6c, 0x69, 0x6d, - 0x69, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x22, - 0xf0, 0x01, 0x0a, 0x0b, 0x4d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x53, 0x74, 0x61, 0x74, 0x73, 0x12, - 0x14, 0x0a, 0x05, 0x63, 0x61, 0x63, 0x68, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, - 0x63, 0x61, 0x63, 0x68, 0x65, 0x12, 0x2c, 0x0a, 0x05, 0x75, 0x73, 0x61, 0x67, 0x65, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, - 0x31, 0x2e, 0x4d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x44, 0x61, 0x74, 0x61, 0x52, 0x05, 0x75, 0x73, - 0x61, 0x67, 0x65, 0x12, 0x35, 0x0a, 0x0a, 0x73, 0x77, 0x61, 0x70, 0x5f, 0x75, 0x73, 0x61, 0x67, - 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, - 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x44, 0x61, 0x74, 0x61, 0x52, - 0x09, 0x73, 0x77, 0x61, 0x70, 0x55, 0x73, 0x61, 0x67, 0x65, 0x12, 0x3e, 0x0a, 0x0f, 0x73, 0x77, - 0x61, 0x70, 0x5f, 0x6f, 0x6e, 0x6c, 0x79, 0x5f, 0x75, 0x73, 0x61, 0x67, 0x65, 0x18, 0x04, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, - 0x2e, 0x4d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x44, 0x61, 0x74, 0x61, 0x52, 0x0d, 0x73, 0x77, 0x61, - 0x70, 0x4f, 0x6e, 0x6c, 0x79, 0x55, 0x73, 0x61, 0x67, 0x65, 0x12, 0x26, 0x0a, 0x03, 0x70, 0x73, - 0x69, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, - 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x53, 0x49, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x03, 0x70, - 0x73, 0x69, 0x22, 0x3b, 0x0a, 0x09, 0x50, 0x69, 0x64, 0x73, 0x53, 0x74, 0x61, 0x74, 0x73, 0x12, - 0x18, 0x0a, 0x07, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, - 0x52, 0x07, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x6c, 0x69, 0x6d, - 0x69, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x22, - 0x31, 0x0a, 0x07, 0x49, 0x4f, 0x53, 0x74, 0x61, 0x74, 0x73, 0x12, 0x26, 0x0a, 0x03, 0x70, 0x73, - 0x69, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, - 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x53, 0x49, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x03, 0x70, - 0x73, 0x69, 0x2a, 0x44, 0x0a, 0x0d, 0x46, 0x6c, 0x6f, 0x77, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x12, 0x10, 0x0a, 0x0c, 0x46, 0x4c, 0x4f, 0x57, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, - 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x10, 0x0a, 0x0c, 0x46, 0x4c, 0x4f, 0x57, 0x5f, 0x49, 0x4e, - 0x47, 0x52, 0x45, 0x53, 0x53, 0x10, 0x01, 0x12, 0x0f, 0x0a, 0x0b, 0x46, 0x4c, 0x4f, 0x57, 0x5f, - 0x45, 0x47, 0x52, 0x45, 0x53, 0x53, 0x10, 0x02, 0x2a, 0x84, 0x01, 0x0a, 0x10, 0x53, 0x69, 0x67, - 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x49, 0x44, 0x12, 0x15, 0x0a, - 0x11, 0x53, 0x49, 0x47, 0x4e, 0x41, 0x54, 0x55, 0x52, 0x45, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, - 0x57, 0x4e, 0x10, 0x00, 0x12, 0x1e, 0x0a, 0x1a, 0x53, 0x49, 0x47, 0x4e, 0x41, 0x54, 0x55, 0x52, - 0x45, 0x5f, 0x53, 0x54, 0x44, 0x49, 0x4f, 0x5f, 0x56, 0x49, 0x41, 0x5f, 0x53, 0x4f, 0x43, 0x4b, - 0x45, 0x54, 0x10, 0x01, 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x49, 0x47, 0x4e, 0x41, 0x54, 0x55, 0x52, - 0x45, 0x5f, 0x54, 0x54, 0x59, 0x5f, 0x44, 0x45, 0x54, 0x45, 0x43, 0x54, 0x45, 0x44, 0x10, 0x02, - 0x12, 0x1d, 0x0a, 0x19, 0x53, 0x49, 0x47, 0x4e, 0x41, 0x54, 0x55, 0x52, 0x45, 0x5f, 0x53, 0x4f, - 0x43, 0x4b, 0x53, 0x35, 0x5f, 0x44, 0x45, 0x54, 0x45, 0x43, 0x54, 0x45, 0x44, 0x10, 0x03, 0x2a, - 0x55, 0x0a, 0x0a, 0x53, 0x4f, 0x43, 0x4b, 0x53, 0x35, 0x52, 0x6f, 0x6c, 0x65, 0x12, 0x17, 0x0a, - 0x13, 0x53, 0x4f, 0x43, 0x4b, 0x53, 0x35, 0x5f, 0x52, 0x4f, 0x4c, 0x45, 0x5f, 0x55, 0x4e, 0x4b, - 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f, 0x43, 0x4b, 0x53, 0x35, - 0x5f, 0x52, 0x4f, 0x4c, 0x45, 0x5f, 0x43, 0x4c, 0x49, 0x45, 0x4e, 0x54, 0x10, 0x01, 0x12, 0x16, - 0x0a, 0x12, 0x53, 0x4f, 0x43, 0x4b, 0x53, 0x35, 0x5f, 0x52, 0x4f, 0x4c, 0x45, 0x5f, 0x53, 0x45, - 0x52, 0x56, 0x45, 0x52, 0x10, 0x02, 0x2a, 0x95, 0x01, 0x0a, 0x11, 0x53, 0x4f, 0x43, 0x4b, 0x53, - 0x35, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1f, 0x0a, 0x1b, - 0x53, 0x4f, 0x43, 0x4b, 0x53, 0x35, 0x5f, 0x41, 0x44, 0x44, 0x52, 0x45, 0x53, 0x53, 0x5f, 0x54, - 0x59, 0x50, 0x45, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x1c, 0x0a, - 0x18, 0x53, 0x4f, 0x43, 0x4b, 0x53, 0x35, 0x5f, 0x41, 0x44, 0x44, 0x52, 0x45, 0x53, 0x53, 0x5f, - 0x54, 0x59, 0x50, 0x45, 0x5f, 0x49, 0x50, 0x76, 0x34, 0x10, 0x01, 0x12, 0x23, 0x0a, 0x1f, 0x53, - 0x4f, 0x43, 0x4b, 0x53, 0x35, 0x5f, 0x41, 0x44, 0x44, 0x52, 0x45, 0x53, 0x53, 0x5f, 0x54, 0x59, - 0x50, 0x45, 0x5f, 0x44, 0x4f, 0x4d, 0x41, 0x49, 0x4e, 0x5f, 0x4e, 0x41, 0x4d, 0x45, 0x10, 0x03, - 0x12, 0x1c, 0x0a, 0x18, 0x53, 0x4f, 0x43, 0x4b, 0x53, 0x35, 0x5f, 0x41, 0x44, 0x44, 0x52, 0x45, - 0x53, 0x53, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x49, 0x50, 0x76, 0x36, 0x10, 0x04, 0x2a, 0x63, - 0x0a, 0x0f, 0x4e, 0x65, 0x74, 0x66, 0x6c, 0x6f, 0x77, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, - 0x6c, 0x12, 0x1c, 0x0a, 0x18, 0x4e, 0x45, 0x54, 0x46, 0x4c, 0x4f, 0x57, 0x5f, 0x50, 0x52, 0x4f, - 0x54, 0x4f, 0x43, 0x4f, 0x4c, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, - 0x18, 0x0a, 0x14, 0x4e, 0x45, 0x54, 0x46, 0x4c, 0x4f, 0x57, 0x5f, 0x50, 0x52, 0x4f, 0x54, 0x4f, - 0x43, 0x4f, 0x4c, 0x5f, 0x54, 0x43, 0x50, 0x10, 0x01, 0x12, 0x18, 0x0a, 0x14, 0x4e, 0x45, 0x54, - 0x46, 0x4c, 0x4f, 0x57, 0x5f, 0x50, 0x52, 0x4f, 0x54, 0x4f, 0x43, 0x4f, 0x4c, 0x5f, 0x55, 0x44, - 0x50, 0x10, 0x02, 0x42, 0x2a, 0x5a, 0x28, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, - 0x6d, 0x2f, 0x63, 0x61, 0x73, 0x74, 0x61, 0x69, 0x2f, 0x6b, 0x76, 0x69, 0x73, 0x6f, 0x72, 0x64, - 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2f, 0x76, 0x31, 0x62, - 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x04, 0x52, 0x09, 0x72, 0x78, 0x50, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x22, 0x1f, 0x0a, 0x07, + 0x50, 0x53, 0x49, 0x44, 0x61, 0x74, 0x61, 0x12, 0x14, 0x0a, 0x05, 0x74, 0x6f, 0x74, 0x61, 0x6c, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x22, 0x5c, 0x0a, + 0x08, 0x50, 0x53, 0x49, 0x53, 0x74, 0x61, 0x74, 0x73, 0x12, 0x27, 0x0a, 0x04, 0x73, 0x6f, 0x6d, + 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, + 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x53, 0x49, 0x44, 0x61, 0x74, 0x61, 0x52, 0x04, 0x73, 0x6f, + 0x6d, 0x65, 0x12, 0x27, 0x0a, 0x04, 0x66, 0x75, 0x6c, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x13, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x53, + 0x49, 0x44, 0x61, 0x74, 0x61, 0x52, 0x04, 0x66, 0x75, 0x6c, 0x6c, 0x22, 0x83, 0x02, 0x0a, 0x08, + 0x43, 0x70, 0x75, 0x53, 0x74, 0x61, 0x74, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x74, 0x6f, 0x74, 0x61, + 0x6c, 0x5f, 0x75, 0x73, 0x61, 0x67, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0a, 0x74, + 0x6f, 0x74, 0x61, 0x6c, 0x55, 0x73, 0x61, 0x67, 0x65, 0x12, 0x2e, 0x0a, 0x13, 0x75, 0x73, 0x61, + 0x67, 0x65, 0x5f, 0x69, 0x6e, 0x5f, 0x6b, 0x65, 0x72, 0x6e, 0x65, 0x6c, 0x6d, 0x6f, 0x64, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x11, 0x75, 0x73, 0x61, 0x67, 0x65, 0x49, 0x6e, 0x4b, + 0x65, 0x72, 0x6e, 0x65, 0x6c, 0x6d, 0x6f, 0x64, 0x65, 0x12, 0x2a, 0x0a, 0x11, 0x75, 0x73, 0x61, + 0x67, 0x65, 0x5f, 0x69, 0x6e, 0x5f, 0x75, 0x73, 0x65, 0x72, 0x6d, 0x6f, 0x64, 0x65, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x04, 0x52, 0x0f, 0x75, 0x73, 0x61, 0x67, 0x65, 0x49, 0x6e, 0x55, 0x73, 0x65, + 0x72, 0x6d, 0x6f, 0x64, 0x65, 0x12, 0x2b, 0x0a, 0x11, 0x74, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, + 0x65, 0x64, 0x5f, 0x70, 0x65, 0x72, 0x69, 0x6f, 0x64, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, + 0x52, 0x10, 0x74, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x64, 0x50, 0x65, 0x72, 0x69, 0x6f, + 0x64, 0x73, 0x12, 0x25, 0x0a, 0x0e, 0x74, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x64, 0x5f, + 0x74, 0x69, 0x6d, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0d, 0x74, 0x68, 0x72, 0x6f, + 0x74, 0x74, 0x6c, 0x65, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x26, 0x0a, 0x03, 0x70, 0x73, 0x69, + 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, + 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x53, 0x49, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x03, 0x70, 0x73, + 0x69, 0x22, 0x38, 0x0a, 0x0a, 0x4d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x44, 0x61, 0x74, 0x61, 0x12, + 0x14, 0x0a, 0x05, 0x75, 0x73, 0x61, 0x67, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, + 0x75, 0x73, 0x61, 0x67, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x22, 0xb9, 0x01, 0x0a, 0x0b, + 0x4d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x53, 0x74, 0x61, 0x74, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x63, + 0x61, 0x63, 0x68, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x63, 0x61, 0x63, 0x68, + 0x65, 0x12, 0x2c, 0x0a, 0x05, 0x75, 0x73, 0x61, 0x67, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x16, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x65, + 0x6d, 0x6f, 0x72, 0x79, 0x44, 0x61, 0x74, 0x61, 0x52, 0x05, 0x75, 0x73, 0x61, 0x67, 0x65, 0x12, + 0x3e, 0x0a, 0x0f, 0x73, 0x77, 0x61, 0x70, 0x5f, 0x6f, 0x6e, 0x6c, 0x79, 0x5f, 0x75, 0x73, 0x61, + 0x67, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, + 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x44, 0x61, 0x74, 0x61, + 0x52, 0x0d, 0x73, 0x77, 0x61, 0x70, 0x4f, 0x6e, 0x6c, 0x79, 0x55, 0x73, 0x61, 0x67, 0x65, 0x12, + 0x26, 0x0a, 0x03, 0x70, 0x73, 0x69, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x72, + 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x53, 0x49, 0x53, 0x74, 0x61, + 0x74, 0x73, 0x52, 0x03, 0x70, 0x73, 0x69, 0x22, 0x3b, 0x0a, 0x09, 0x50, 0x69, 0x64, 0x73, 0x53, + 0x74, 0x61, 0x74, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x07, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x12, 0x14, + 0x0a, 0x05, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x6c, + 0x69, 0x6d, 0x69, 0x74, 0x22, 0x31, 0x0a, 0x07, 0x49, 0x4f, 0x53, 0x74, 0x61, 0x74, 0x73, 0x12, + 0x26, 0x0a, 0x03, 0x70, 0x73, 0x69, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x72, + 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x53, 0x49, 0x53, 0x74, 0x61, + 0x74, 0x73, 0x52, 0x03, 0x70, 0x73, 0x69, 0x2a, 0x44, 0x0a, 0x0d, 0x46, 0x6c, 0x6f, 0x77, 0x44, + 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x10, 0x0a, 0x0c, 0x46, 0x4c, 0x4f, 0x57, + 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x10, 0x0a, 0x0c, 0x46, 0x4c, + 0x4f, 0x57, 0x5f, 0x49, 0x4e, 0x47, 0x52, 0x45, 0x53, 0x53, 0x10, 0x01, 0x12, 0x0f, 0x0a, 0x0b, + 0x46, 0x4c, 0x4f, 0x57, 0x5f, 0x45, 0x47, 0x52, 0x45, 0x53, 0x53, 0x10, 0x02, 0x2a, 0x84, 0x01, + 0x0a, 0x10, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x45, 0x76, 0x65, 0x6e, 0x74, + 0x49, 0x44, 0x12, 0x15, 0x0a, 0x11, 0x53, 0x49, 0x47, 0x4e, 0x41, 0x54, 0x55, 0x52, 0x45, 0x5f, + 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x1e, 0x0a, 0x1a, 0x53, 0x49, 0x47, + 0x4e, 0x41, 0x54, 0x55, 0x52, 0x45, 0x5f, 0x53, 0x54, 0x44, 0x49, 0x4f, 0x5f, 0x56, 0x49, 0x41, + 0x5f, 0x53, 0x4f, 0x43, 0x4b, 0x45, 0x54, 0x10, 0x01, 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x49, 0x47, + 0x4e, 0x41, 0x54, 0x55, 0x52, 0x45, 0x5f, 0x54, 0x54, 0x59, 0x5f, 0x44, 0x45, 0x54, 0x45, 0x43, + 0x54, 0x45, 0x44, 0x10, 0x02, 0x12, 0x1d, 0x0a, 0x19, 0x53, 0x49, 0x47, 0x4e, 0x41, 0x54, 0x55, + 0x52, 0x45, 0x5f, 0x53, 0x4f, 0x43, 0x4b, 0x53, 0x35, 0x5f, 0x44, 0x45, 0x54, 0x45, 0x43, 0x54, + 0x45, 0x44, 0x10, 0x03, 0x2a, 0x55, 0x0a, 0x0a, 0x53, 0x4f, 0x43, 0x4b, 0x53, 0x35, 0x52, 0x6f, + 0x6c, 0x65, 0x12, 0x17, 0x0a, 0x13, 0x53, 0x4f, 0x43, 0x4b, 0x53, 0x35, 0x5f, 0x52, 0x4f, 0x4c, + 0x45, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x16, 0x0a, 0x12, 0x53, + 0x4f, 0x43, 0x4b, 0x53, 0x35, 0x5f, 0x52, 0x4f, 0x4c, 0x45, 0x5f, 0x43, 0x4c, 0x49, 0x45, 0x4e, + 0x54, 0x10, 0x01, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f, 0x43, 0x4b, 0x53, 0x35, 0x5f, 0x52, 0x4f, + 0x4c, 0x45, 0x5f, 0x53, 0x45, 0x52, 0x56, 0x45, 0x52, 0x10, 0x02, 0x2a, 0x95, 0x01, 0x0a, 0x11, + 0x53, 0x4f, 0x43, 0x4b, 0x53, 0x35, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x54, 0x79, 0x70, + 0x65, 0x12, 0x1f, 0x0a, 0x1b, 0x53, 0x4f, 0x43, 0x4b, 0x53, 0x35, 0x5f, 0x41, 0x44, 0x44, 0x52, + 0x45, 0x53, 0x53, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, + 0x10, 0x00, 0x12, 0x1c, 0x0a, 0x18, 0x53, 0x4f, 0x43, 0x4b, 0x53, 0x35, 0x5f, 0x41, 0x44, 0x44, + 0x52, 0x45, 0x53, 0x53, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x49, 0x50, 0x76, 0x34, 0x10, 0x01, + 0x12, 0x23, 0x0a, 0x1f, 0x53, 0x4f, 0x43, 0x4b, 0x53, 0x35, 0x5f, 0x41, 0x44, 0x44, 0x52, 0x45, + 0x53, 0x53, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x44, 0x4f, 0x4d, 0x41, 0x49, 0x4e, 0x5f, 0x4e, + 0x41, 0x4d, 0x45, 0x10, 0x03, 0x12, 0x1c, 0x0a, 0x18, 0x53, 0x4f, 0x43, 0x4b, 0x53, 0x35, 0x5f, + 0x41, 0x44, 0x44, 0x52, 0x45, 0x53, 0x53, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x49, 0x50, 0x76, + 0x36, 0x10, 0x04, 0x2a, 0x63, 0x0a, 0x0f, 0x4e, 0x65, 0x74, 0x66, 0x6c, 0x6f, 0x77, 0x50, 0x72, + 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x1c, 0x0a, 0x18, 0x4e, 0x45, 0x54, 0x46, 0x4c, 0x4f, + 0x57, 0x5f, 0x50, 0x52, 0x4f, 0x54, 0x4f, 0x43, 0x4f, 0x4c, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, + 0x57, 0x4e, 0x10, 0x00, 0x12, 0x18, 0x0a, 0x14, 0x4e, 0x45, 0x54, 0x46, 0x4c, 0x4f, 0x57, 0x5f, + 0x50, 0x52, 0x4f, 0x54, 0x4f, 0x43, 0x4f, 0x4c, 0x5f, 0x54, 0x43, 0x50, 0x10, 0x01, 0x12, 0x18, + 0x0a, 0x14, 0x4e, 0x45, 0x54, 0x46, 0x4c, 0x4f, 0x57, 0x5f, 0x50, 0x52, 0x4f, 0x54, 0x4f, 0x43, + 0x4f, 0x4c, 0x5f, 0x55, 0x44, 0x50, 0x10, 0x02, 0x42, 0x2a, 0x5a, 0x28, 0x67, 0x69, 0x74, 0x68, + 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x61, 0x73, 0x74, 0x61, 0x69, 0x2f, 0x6b, 0x76, + 0x69, 0x73, 0x6f, 0x72, 0x64, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, + 0x65, 0x2f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -2299,15 +2254,14 @@ var file_api_v1_runtime_common_proto_depIdxs = []int32{ 21, // 17: runtime.v1.PSIStats.full:type_name -> runtime.v1.PSIData 22, // 18: runtime.v1.CpuStats.psi:type_name -> runtime.v1.PSIStats 24, // 19: runtime.v1.MemoryStats.usage:type_name -> runtime.v1.MemoryData - 24, // 20: runtime.v1.MemoryStats.swap_usage:type_name -> runtime.v1.MemoryData - 24, // 21: runtime.v1.MemoryStats.swap_only_usage:type_name -> runtime.v1.MemoryData - 22, // 22: runtime.v1.MemoryStats.psi:type_name -> runtime.v1.PSIStats - 22, // 23: runtime.v1.IOStats.psi:type_name -> runtime.v1.PSIStats - 24, // [24:24] is the sub-list for method output_type - 24, // [24:24] is the sub-list for method input_type - 24, // [24:24] is the sub-list for extension type_name - 24, // [24:24] is the sub-list for extension extendee - 0, // [0:24] is the sub-list for field type_name + 24, // 20: runtime.v1.MemoryStats.swap_only_usage:type_name -> runtime.v1.MemoryData + 22, // 21: runtime.v1.MemoryStats.psi:type_name -> runtime.v1.PSIStats + 22, // 22: runtime.v1.IOStats.psi:type_name -> runtime.v1.PSIStats + 23, // [23:23] is the sub-list for method output_type + 23, // [23:23] is the sub-list for method input_type + 23, // [23:23] is the sub-list for extension type_name + 23, // [23:23] is the sub-list for extension extendee + 0, // [0:23] is the sub-list for field type_name } func init() { file_api_v1_runtime_common_proto_init() } diff --git a/api/v1/runtime/common.proto b/api/v1/runtime/common.proto index 44dcdc00..8e02949f 100644 --- a/api/v1/runtime/common.proto +++ b/api/v1/runtime/common.proto @@ -159,10 +159,7 @@ message NetflowDestination { } message PSIData { - double avg10 = 1; - double avg60 = 2; - double avg300 = 3; - uint64 total = 4; + uint64 total = 1; } message PSIStats { @@ -194,15 +191,9 @@ message MemoryData { } message MemoryStats { - // memory used for cache uint64 cache = 1; - // usage of memory MemoryData usage = 2; - // usage of memory + swap - MemoryData swap_usage = 3; - // usage of swap only MemoryData swap_only_usage = 4; - // usage of kernel memory PSIStats psi = 8; } diff --git a/charts/kvisor/values-local.yaml b/charts/kvisor/values-local.yaml index 2bb32605..2c583cd1 100644 --- a/charts/kvisor/values-local.yaml +++ b/charts/kvisor/values-local.yaml @@ -18,8 +18,8 @@ agent: pyroscope-addr: http://kvisord-pyroscope:4040 file-hash-enricher-enabled: true signature-socks5-detection-enabled: true - container-stats-enabled: true - container-stats-scrape-interval: 5s + stats-enabled: true + stats-scrape-interval: 5s ebpf-events-enabled: true netflow-enabled: true netflow-sample-submit-interval-seconds: 1 diff --git a/cmd/agent/daemon/state/controller.go b/cmd/agent/daemon/state/controller.go index e662a6ac..f1aeecb2 100644 --- a/cmd/agent/daemon/state/controller.go +++ b/cmd/agent/daemon/state/controller.go @@ -131,7 +131,7 @@ func NewController( enrichmentService: enrichmentService, kubeClient: kubeClient, nodeName: os.Getenv("NODE_NAME"), - resourcesStatsScrapePoints: map[uint64]*containerStatsScrapePoint{}, + containerStatsScrapePoints: map[uint64]*containerStatsScrapePoint{}, mutedNamespaces: map[string]struct{}{}, dnsCache: dnsCache, podCache: podCache, @@ -157,8 +157,9 @@ type Controller struct { nodeName string // Scrape points are used to calculate deltas between scrapes. - resourcesStatsScrapePointsMu sync.RWMutex - resourcesStatsScrapePoints map[uint64]*containerStatsScrapePoint + containerStatsScrapePointsMu sync.RWMutex + containerStatsScrapePoints map[uint64]*containerStatsScrapePoint + nodeScrapePoint *nodeScrapePoint mutedNamespacesMu sync.RWMutex mutedNamespaces map[string]struct{} @@ -216,20 +217,15 @@ func (c *Controller) onNewContainer(container *containers.Container) { } func (c *Controller) onDeleteContainer(container *containers.Container) { - c.resourcesStatsScrapePointsMu.Lock() - delete(c.resourcesStatsScrapePoints, container.CgroupID) - c.resourcesStatsScrapePointsMu.Unlock() + c.containerStatsScrapePointsMu.Lock() + delete(c.containerStatsScrapePoints, container.CgroupID) + c.containerStatsScrapePointsMu.Unlock() c.dnsCache.Remove(container.CgroupID) c.log.Debugf("removed cgroup %d", container.CgroupID) } -type containerStatsScrapePoint struct { - ts time.Time - cpuStat *castaipb.CpuStats -} - func (c *Controller) MuteNamespace(namespace string) error { c.mutedNamespacesMu.Lock() c.mutedNamespaces[namespace] = struct{}{} diff --git a/cmd/agent/daemon/state/stats_pipeline.go b/cmd/agent/daemon/state/stats_pipeline.go index c1c20ec2..e6d3d586 100644 --- a/cmd/agent/daemon/state/stats_pipeline.go +++ b/cmd/agent/daemon/state/stats_pipeline.go @@ -10,6 +10,18 @@ import ( "github.com/castai/kvisor/pkg/containers" ) +type containerStatsScrapePoint struct { + cpuStat *castaipb.CpuStats + memStat *castaipb.MemoryStats + ioStat *castaipb.IOStats +} + +type nodeScrapePoint struct { + cpuStat *castaipb.CpuStats + memStat *castaipb.MemoryStats + ioStat *castaipb.IOStats +} + func (c *Controller) runStatsPipeline(ctx context.Context) error { c.log.Info("running stats pipeline") defer c.log.Info("stats pipeline done") @@ -17,7 +29,8 @@ func (c *Controller) runStatsPipeline(ctx context.Context) error { ticker := time.NewTicker(c.cfg.StatsScrapeInterval) defer ticker.Stop() - // Initial scrape to populate container metrics for cpu diff. + // Initial scrape to populate initial points for diffs. + c.scrapeNodeStats(nil) c.scrapeContainersResourceStats(nil) for { @@ -25,14 +38,18 @@ func (c *Controller) runStatsPipeline(ctx context.Context) error { case <-ctx.Done(): return ctx.Err() case <-ticker.C: - batch := &castaipb.StatsBatch{} - c.scrapeNodeStats(batch) - c.scrapeContainersResourceStats(batch) - if len(batch.Items) > 0 { - for _, exp := range c.exporters.Stats { - exp.Enqueue(batch) + func() { + start := time.Now() + batch := &castaipb.StatsBatch{} + c.scrapeNodeStats(batch) + c.scrapeContainersResourceStats(batch) + if len(batch.Items) > 0 { + for _, exp := range c.exporters.Stats { + exp.Enqueue(batch) + } } - } + c.log.Debugf("stats exported, duration=%v", time.Since(start)) + }() } } } @@ -44,7 +61,6 @@ func (c *Controller) scrapeContainersResourceStats(batch *castaipb.StatsBatch) { } func (c *Controller) scrapeContainerResourcesStats(cont *containers.Container, batch *castaipb.StatsBatch) { - now := time.Now().UTC() cgStats, err := c.containersClient.GetCgroupStats(cont) if err != nil { if c.log.IsEnabled(slog.LevelDebug) { @@ -55,18 +71,21 @@ func (c *Controller) scrapeContainerResourcesStats(cont *containers.Container, b } currScrape := &containerStatsScrapePoint{ - ts: now, cpuStat: cgStats.CpuStats, + memStat: cgStats.MemoryStats, + ioStat: cgStats.IOStats, } - // We need at least 2 scrapes to calculate cpu diff count. - c.resourcesStatsScrapePointsMu.RLock() - prevScrape, found := c.resourcesStatsScrapePoints[cont.CgroupID] - c.resourcesStatsScrapePointsMu.RUnlock() + // We need at least 2 scrapes to calculate diffs. + // Diffs are needed for always increasing counters only because we store them as deltas. + // This includes cpu usage and psi total value. + c.containerStatsScrapePointsMu.RLock() + prevScrape, found := c.containerStatsScrapePoints[cont.CgroupID] + c.containerStatsScrapePointsMu.RUnlock() if !found { - c.resourcesStatsScrapePointsMu.Lock() - c.resourcesStatsScrapePoints[cont.CgroupID] = currScrape - c.resourcesStatsScrapePointsMu.Unlock() + c.containerStatsScrapePointsMu.Lock() + c.containerStatsScrapePoints[cont.CgroupID] = currScrape + c.containerStatsScrapePointsMu.Unlock() return } @@ -80,10 +99,10 @@ func (c *Controller) scrapeContainerResourcesStats(cont *containers.Container, b ContainerName: cont.Name, PodUid: cont.PodUID, ContainerId: cont.ID, - CpuStats: getCPUStatsDiff(prevScrape, currScrape), - MemoryStats: cgStats.MemoryStats, + CpuStats: getCPUStatsDiff(prevScrape.cpuStat, currScrape.cpuStat), + MemoryStats: getMemoryStatsDiff(prevScrape.memStat, currScrape.memStat), PidsStats: cgStats.PidsStats, - IoStats: cgStats.IOStats, + IoStats: getIOStatsDiff(prevScrape.ioStat, currScrape.ioStat), } if podInfo, ok := c.getPodInfo(cont.PodUID); ok { item.NodeName = podInfo.NodeName @@ -92,38 +111,69 @@ func (c *Controller) scrapeContainerResourcesStats(cont *containers.Container, b } batch.Items = append(batch.Items, &castaipb.StatsItem{Data: &castaipb.StatsItem_Container{Container: item}}) - prevScrape.ts = currScrape.ts prevScrape.cpuStat = currScrape.cpuStat + prevScrape.memStat = currScrape.memStat + prevScrape.ioStat = currScrape.ioStat } func (c *Controller) scrapeNodeStats(batch *castaipb.StatsBatch) { - item := &castaipb.NodeStats{} if err := func() error { - if c.procHandler.PSIEnabled() { - cpuPSI, err := c.procHandler.GetPSIStats("cpu") - if err != nil { - return err - } - item.CpuStats = &castaipb.CpuStats{Psi: cpuPSI} - - memStats, err := c.procHandler.GetMeminfoStats() - if err != nil { - return err - } - item.MemoryStats = memStats - memoryPSI, err := c.procHandler.GetPSIStats("memory") - if err != nil { - return err - } - item.MemoryStats.Psi = memoryPSI - - ioPSI, err := c.procHandler.GetPSIStats("io") - if err != nil { - return err - } - item.IoStats = &castaipb.IOStats{Psi: ioPSI} - batch.Items = append(batch.Items, &castaipb.StatsItem{Data: &castaipb.StatsItem_Node{Node: item}}) + // For now, we only care about PSI related metrics on node. + if !c.procHandler.PSIEnabled() { + return nil + } + + cpuPSI, err := c.procHandler.GetPSIStats("cpu") + if err != nil { + return err + } + memStats, err := c.procHandler.GetMeminfoStats() + if err != nil { + return err + } + memoryPSI, err := c.procHandler.GetPSIStats("memory") + if err != nil { + return err + } + ioPSI, err := c.procHandler.GetPSIStats("io") + if err != nil { + return err + } + + currScrape := &nodeScrapePoint{ + cpuStat: &castaipb.CpuStats{Psi: cpuPSI}, + memStat: &castaipb.MemoryStats{ + Usage: memStats.Usage, + SwapOnlyUsage: memStats.SwapOnlyUsage, + Psi: memoryPSI, + }, + ioStat: &castaipb.IOStats{Psi: ioPSI}, + } + + // We need at least 2 scrapes to calculate diffs. + // Diffs are needed for always increasing counters only because we store them as deltas. + // This includes cpu usage and psi total value. + if c.nodeScrapePoint == nil { + c.nodeScrapePoint = currScrape + return nil } + if batch == nil { + return nil + } + + batch.Items = append(batch.Items, &castaipb.StatsItem{Data: &castaipb.StatsItem_Node{ + Node: &castaipb.NodeStats{ + NodeName: c.nodeName, + CpuStats: getCPUStatsDiff(c.nodeScrapePoint.cpuStat, currScrape.cpuStat), + MemoryStats: getMemoryStatsDiff(c.nodeScrapePoint.memStat, currScrape.memStat), + IoStats: getIOStatsDiff(c.nodeScrapePoint.ioStat, currScrape.ioStat), + }, + }}) + + c.nodeScrapePoint.cpuStat = currScrape.cpuStat + c.nodeScrapePoint.memStat = currScrape.memStat + c.nodeScrapePoint.ioStat = currScrape.ioStat + return nil }(); err != nil { if c.log.IsEnabled(slog.LevelDebug) { @@ -134,13 +184,39 @@ func (c *Controller) scrapeNodeStats(batch *castaipb.StatsBatch) { } } -func getCPUStatsDiff(prev, curr *containerStatsScrapePoint) *castaipb.CpuStats { +func getCPUStatsDiff(prev, curr *castaipb.CpuStats) *castaipb.CpuStats { return &castaipb.CpuStats{ - TotalUsage: curr.cpuStat.TotalUsage - prev.cpuStat.TotalUsage, - UsageInKernelmode: curr.cpuStat.UsageInKernelmode - prev.cpuStat.UsageInKernelmode, - UsageInUsermode: curr.cpuStat.UsageInUsermode - prev.cpuStat.UsageInUsermode, - ThrottledPeriods: curr.cpuStat.ThrottledPeriods, - ThrottledTime: curr.cpuStat.ThrottledTime, - Psi: curr.cpuStat.Psi, + TotalUsage: curr.TotalUsage - prev.TotalUsage, + UsageInKernelmode: curr.UsageInKernelmode - prev.UsageInKernelmode, + UsageInUsermode: curr.UsageInUsermode - prev.UsageInUsermode, + ThrottledPeriods: curr.ThrottledPeriods, + ThrottledTime: curr.ThrottledTime, + Psi: getPSIStatsDiff(prev.Psi, curr.Psi), + } +} + +func getMemoryStatsDiff(prev, curr *castaipb.MemoryStats) *castaipb.MemoryStats { + return &castaipb.MemoryStats{ + Cache: curr.Cache, + Usage: curr.Usage, + SwapOnlyUsage: curr.SwapOnlyUsage, + Psi: getPSIStatsDiff(prev.Psi, curr.Psi), + } +} + +func getIOStatsDiff(prev, curr *castaipb.IOStats) *castaipb.IOStats { + return &castaipb.IOStats{ + Psi: getPSIStatsDiff(prev.Psi, curr.Psi), + } +} + +func getPSIStatsDiff(prev, curr *castaipb.PSIStats) *castaipb.PSIStats { + return &castaipb.PSIStats{ + Some: &castaipb.PSIData{ + Total: curr.Some.Total - prev.Some.Total, + }, + Full: &castaipb.PSIData{ + Total: curr.Full.Total - prev.Full.Total, + }, } } diff --git a/cmd/controller/state/castai_controller_test.go b/cmd/controller/state/castai_controller_test.go index 46ec77aa..17549eb6 100644 --- a/cmd/controller/state/castai_controller_test.go +++ b/cmd/controller/state/castai_controller_test.go @@ -124,7 +124,7 @@ func (t testGrpcClient) LogsWriteStream(ctx context.Context, opts ...grpc.CallOp return nil, nil } -func (t testGrpcClient) ContainerStatsWriteStream(ctx context.Context, opts ...grpc.CallOption) (castaipb.RuntimeSecurityAgentAPI_ContainerStatsWriteStreamClient, error) { +func (t testGrpcClient) StatsWriteStream(ctx context.Context, opts ...grpc.CallOption) (castaipb.RuntimeSecurityAgentAPI_StatsWriteStreamClient, error) { return nil, nil } diff --git a/cmd/mock-server/main.go b/cmd/mock-server/main.go index 03558ba9..ecd6b594 100644 --- a/cmd/mock-server/main.go +++ b/cmd/mock-server/main.go @@ -170,7 +170,7 @@ func (m *MockServer) StatsWriteStream(server castaipb.RuntimeSecurityAgentAPI_St } node := v.GetNode() if node != nil { - m.log.Debugf("node_stats, node=%s,cpu=%v, mem=%v", node.NodeName, cont.CpuStats, cont.MemoryStats) + m.log.Debugf("node_stats, node=%s,cpu=%v, mem=%v", node.NodeName, node.CpuStats, node.MemoryStats) } } } diff --git a/pkg/castai/client_test.go b/pkg/castai/client_test.go index 5bda8faf..791366c8 100644 --- a/pkg/castai/client_test.go +++ b/pkg/castai/client_test.go @@ -154,7 +154,7 @@ func (*testServer) KubernetesDeltaIngest(castaipb.RuntimeSecurityAgentAPI_Kubern panic("unimplemented") } -func (t *testServer) ContainerStatsWriteStream(server castaipb.RuntimeSecurityAgentAPI_ContainerStatsWriteStreamServer) error { +func (t *testServer) StatsWriteStream(server castaipb.RuntimeSecurityAgentAPI_StatsWriteStreamServer) error { //TODO implement me panic("implement me") } diff --git a/pkg/cgroup/cgroup_memory.go b/pkg/cgroup/cgroup_memory.go index eb79698f..c8fb1613 100644 --- a/pkg/cgroup/cgroup_memory.go +++ b/pkg/cgroup/cgroup_memory.go @@ -3,10 +3,7 @@ package cgroup import ( "bufio" "errors" - "math" "os" - "strconv" - "strings" castaipb "github.com/castai/kvisor/api/v1/runtime" "golang.org/x/sys/unix" @@ -48,15 +45,6 @@ func statMemoryV2(dirPath string, stats *Stats) error { return err } stats.MemoryStats.SwapOnlyUsage = swapOnlyUsage - swapUsage := swapOnlyUsage - // As cgroup v1 reports SwapUsage values as mem+swap combined, - // while in cgroup v2 swap values do not include memory, - // report combined mem+swap for v1 compatibility. - swapUsage.Usage += memoryUsage.Usage - if swapUsage.Limit != math.MaxUint64 { - swapUsage.Limit += memoryUsage.Limit - } - stats.MemoryStats.SwapUsage = swapUsage return nil } @@ -113,7 +101,7 @@ func statMemoryV1(dirPath string, stats *Stats) error { } } - memoryUsage, err := getMemoryDataV1(dirPath, "") // TODO: Why empty? + memoryUsage, err := getMemoryDataV1(dirPath, "") if err != nil { return err } @@ -122,7 +110,6 @@ func statMemoryV1(dirPath string, stats *Stats) error { if err != nil { return err } - stats.MemoryStats.SwapUsage = swapUsage stats.MemoryStats.SwapOnlyUsage = &castaipb.MemoryData{ Usage: swapUsage.Usage - memoryUsage.Usage, } @@ -165,66 +152,3 @@ func getMemoryDataV1(path, name string) (*castaipb.MemoryData, error) { return &memoryData, nil } - -var _ = rootStatsFromMeminfo - -func rootStatsFromMeminfo(stats *Stats) error { - const file = "/proc/meminfo" - f, err := os.Open(file) - if err != nil { - return err - } - defer f.Close() - - // Fields we are interested in. - var ( - memTotal uint64 - memFree uint64 - swapFree uint64 - swapTotal uint64 - ) - mem := map[string]*uint64{ - "MemTotal": &memTotal, - "MemFree": &memFree, - "SwapFree": &swapFree, - "SwapTotal": &swapTotal, - } - - found := 0 - sc := bufio.NewScanner(f) - for sc.Scan() { - parts := strings.SplitN(sc.Text(), ":", 3) - if len(parts) != 2 { - // Should not happen. - continue - } - k := parts[0] - p, ok := mem[k] - if !ok { - // Unknown field -- not interested. - continue - } - vStr := strings.TrimSpace(strings.TrimSuffix(parts[1], " kB")) - *p, err = strconv.ParseUint(vStr, 10, 64) - if err != nil { - return &parseError{File: file, Err: errors.New("bad value for " + k)} - } - - found++ - if found == len(mem) { - // Got everything we need -- skip the rest. - break - } - } - if err := sc.Err(); err != nil { - return &parseError{Path: "", File: file, Err: err} - } - - stats.MemoryStats.Usage.Usage = memTotal - memFree - stats.MemoryStats.Usage.Limit = math.MaxUint64 - stats.MemoryStats.SwapUsage.Usage = (swapTotal - swapFree) * 1024 - stats.MemoryStats.SwapUsage.Limit = math.MaxUint64 - stats.MemoryStats.SwapUsage.Usage += stats.MemoryStats.Usage.Usage - - return nil -} diff --git a/pkg/cgroup/psi.go b/pkg/cgroup/psi.go index eabf318b..baad4945 100644 --- a/pkg/cgroup/psi.go +++ b/pkg/cgroup/psi.go @@ -60,27 +60,15 @@ func parsePSIData(psi []string) (*castaipb.PSIData, error) { if len(kv) != 2 { return nil, fmt.Errorf("invalid psi data: %q", f) } - var pv *float64 switch kv[0] { - case "avg10": - pv = &data.Avg10 - case "avg60": - pv = &data.Avg60 - case "avg300": - pv = &data.Avg300 case "total": v, err := strconv.ParseUint(kv[1], 10, 64) if err != nil { return nil, fmt.Errorf("invalid %s PSI value: %w", kv[0], err) } data.Total = v - } - if pv != nil { - v, err := strconv.ParseFloat(kv[1], 64) - if err != nil { - return nil, fmt.Errorf("invalid %s PSI value: %w", kv[0], err) - } - *pv = v + // For now we care only about total value. + return &data, nil } } return &data, nil diff --git a/pkg/proc/proc.go b/pkg/proc/proc.go index d18a37ea..bd696a00 100644 --- a/pkg/proc/proc.go +++ b/pkg/proc/proc.go @@ -268,13 +268,13 @@ func (p *Proc) GetMeminfoStats() (*castaipb.MemoryStats, error) { } memUsage := memTotal - memFree - swapUsage := ((swapTotal - swapFree) * 1024) + memUsage + swapUsage := (swapTotal - swapFree) * 1024 return &castaipb.MemoryStats{ Usage: &castaipb.MemoryData{ Usage: memUsage, }, - SwapUsage: &castaipb.MemoryData{ + SwapOnlyUsage: &castaipb.MemoryData{ Usage: swapUsage, }, }, nil