From d302a0aaff4dc8c89593c14c85075f558614703d Mon Sep 17 00:00:00 2001 From: Joseph Anttila Hall Date: Tue, 12 Mar 2024 22:12:41 -0700 Subject: [PATCH 1/2] Add Drain message, and generated files. - make gen-proto - go mod vendor --- konnectivity-client/proto/client/client.pb.go | 145 ++++++++++++----- konnectivity-client/proto/client/client.proto | 7 + .../proto/client/client.pb.go | 147 +++++++++++++----- .../proto/client/client.proto | 7 + .../proto/client/client_grpc.pb.go | 2 +- 5 files changed, 234 insertions(+), 74 deletions(-) diff --git a/konnectivity-client/proto/client/client.pb.go b/konnectivity-client/proto/client/client.pb.go index da1e37c18..0831f3e63 100644 --- a/konnectivity-client/proto/client/client.pb.go +++ b/konnectivity-client/proto/client/client.pb.go @@ -43,6 +43,7 @@ const ( PacketType_CLOSE_RSP PacketType = 3 PacketType_DATA PacketType = 4 PacketType_DIAL_CLS PacketType = 5 + PacketType_DRAIN PacketType = 6 ) // Enum value maps for PacketType. @@ -54,6 +55,7 @@ var ( 3: "CLOSE_RSP", 4: "DATA", 5: "DIAL_CLS", + 6: "DRAIN", } PacketType_value = map[string]int32{ "DIAL_REQ": 0, @@ -62,6 +64,7 @@ var ( "CLOSE_RSP": 3, "DATA": 4, "DIAL_CLS": 5, + "DRAIN": 6, } ) @@ -106,6 +109,7 @@ type Packet struct { // *Packet_CloseRequest // *Packet_CloseResponse // *Packet_CloseDial + // *Packet_Drain Payload isPacket_Payload `protobuf_oneof:"payload"` } @@ -197,6 +201,13 @@ func (x *Packet) GetCloseDial() *CloseDial { return nil } +func (x *Packet) GetDrain() *Drain { + if x, ok := x.GetPayload().(*Packet_Drain); ok { + return x.Drain + } + return nil +} + type isPacket_Payload interface { isPacket_Payload() } @@ -225,6 +236,10 @@ type Packet_CloseDial struct { CloseDial *CloseDial `protobuf:"bytes,7,opt,name=closeDial,proto3,oneof"` } +type Packet_Drain struct { + Drain *Drain `protobuf:"bytes,8,opt,name=drain,proto3,oneof"` +} + func (*Packet_DialRequest) isPacket_Payload() {} func (*Packet_DialResponse) isPacket_Payload() {} @@ -237,6 +252,8 @@ func (*Packet_CloseResponse) isPacket_Payload() {} func (*Packet_CloseDial) isPacket_Payload() {} +func (*Packet_Drain) isPacket_Payload() {} + type DialRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -522,6 +539,44 @@ func (x *CloseDial) GetRandom() int64 { return 0 } +type Drain struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *Drain) Reset() { + *x = Drain{} + if protoimpl.UnsafeEnabled { + mi := &file_konnectivity_client_proto_client_client_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Drain) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Drain) ProtoMessage() {} + +func (x *Drain) ProtoReflect() protoreflect.Message { + mi := &file_konnectivity_client_proto_client_client_proto_msgTypes[6] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Drain.ProtoReflect.Descriptor instead. +func (*Drain) Descriptor() ([]byte, []int) { + return file_konnectivity_client_proto_client_client_proto_rawDescGZIP(), []int{6} +} + type Data struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -538,7 +593,7 @@ type Data struct { func (x *Data) Reset() { *x = Data{} if protoimpl.UnsafeEnabled { - mi := &file_konnectivity_client_proto_client_client_proto_msgTypes[6] + mi := &file_konnectivity_client_proto_client_client_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -551,7 +606,7 @@ func (x *Data) String() string { func (*Data) ProtoMessage() {} func (x *Data) ProtoReflect() protoreflect.Message { - mi := &file_konnectivity_client_proto_client_client_proto_msgTypes[6] + mi := &file_konnectivity_client_proto_client_client_proto_msgTypes[7] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -564,7 +619,7 @@ func (x *Data) ProtoReflect() protoreflect.Message { // Deprecated: Use Data.ProtoReflect.Descriptor instead. func (*Data) Descriptor() ([]byte, []int) { - return file_konnectivity_client_proto_client_client_proto_rawDescGZIP(), []int{6} + return file_konnectivity_client_proto_client_client_proto_rawDescGZIP(), []int{7} } func (x *Data) GetConnectID() int64 { @@ -594,7 +649,7 @@ var file_konnectivity_client_proto_client_client_proto_rawDesc = []byte{ 0x0a, 0x2d, 0x6b, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x76, 0x69, 0x74, 0x79, 0x2d, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x2f, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, - 0xd1, 0x02, 0x0a, 0x06, 0x50, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x1f, 0x0a, 0x04, 0x74, 0x79, + 0xf1, 0x02, 0x0a, 0x06, 0x50, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x1f, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x0b, 0x2e, 0x50, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x30, 0x0a, 0x0b, 0x64, 0x69, 0x61, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, @@ -614,7 +669,9 @@ var file_konnectivity_client_proto_client_client_proto_rawDesc = []byte{ 0x6c, 0x6f, 0x73, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2a, 0x0a, 0x09, 0x63, 0x6c, 0x6f, 0x73, 0x65, 0x44, 0x69, 0x61, 0x6c, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x43, 0x6c, 0x6f, 0x73, 0x65, 0x44, 0x69, 0x61, 0x6c, 0x48, 0x00, 0x52, 0x09, 0x63, - 0x6c, 0x6f, 0x73, 0x65, 0x44, 0x69, 0x61, 0x6c, 0x42, 0x09, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, + 0x6c, 0x6f, 0x73, 0x65, 0x44, 0x69, 0x61, 0x6c, 0x12, 0x1e, 0x0a, 0x05, 0x64, 0x72, 0x61, 0x69, + 0x6e, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x06, 0x2e, 0x44, 0x72, 0x61, 0x69, 0x6e, 0x48, + 0x00, 0x52, 0x05, 0x64, 0x72, 0x61, 0x69, 0x6e, 0x42, 0x09, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x22, 0x5b, 0x0a, 0x0b, 0x44, 0x69, 0x61, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x18, @@ -636,26 +693,27 @@ var file_konnectivity_client_proto_client_client_proto_rawDesc = []byte{ 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x49, 0x44, 0x22, 0x23, 0x0a, 0x09, 0x43, 0x6c, 0x6f, 0x73, 0x65, 0x44, 0x69, 0x61, 0x6c, 0x12, 0x16, 0x0a, 0x06, 0x72, 0x61, 0x6e, 0x64, 0x6f, 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x72, 0x61, - 0x6e, 0x64, 0x6f, 0x6d, 0x22, 0x4e, 0x0a, 0x04, 0x44, 0x61, 0x74, 0x61, 0x12, 0x1c, 0x0a, 0x09, - 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, - 0x09, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x49, 0x44, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, - 0x72, 0x6f, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, - 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, - 0x64, 0x61, 0x74, 0x61, 0x2a, 0x5e, 0x0a, 0x0a, 0x50, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x54, 0x79, - 0x70, 0x65, 0x12, 0x0c, 0x0a, 0x08, 0x44, 0x49, 0x41, 0x4c, 0x5f, 0x52, 0x45, 0x51, 0x10, 0x00, - 0x12, 0x0c, 0x0a, 0x08, 0x44, 0x49, 0x41, 0x4c, 0x5f, 0x52, 0x53, 0x50, 0x10, 0x01, 0x12, 0x0d, - 0x0a, 0x09, 0x43, 0x4c, 0x4f, 0x53, 0x45, 0x5f, 0x52, 0x45, 0x51, 0x10, 0x02, 0x12, 0x0d, 0x0a, - 0x09, 0x43, 0x4c, 0x4f, 0x53, 0x45, 0x5f, 0x52, 0x53, 0x50, 0x10, 0x03, 0x12, 0x08, 0x0a, 0x04, - 0x44, 0x41, 0x54, 0x41, 0x10, 0x04, 0x12, 0x0c, 0x0a, 0x08, 0x44, 0x49, 0x41, 0x4c, 0x5f, 0x43, - 0x4c, 0x53, 0x10, 0x05, 0x32, 0x2f, 0x0a, 0x0c, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x53, 0x65, 0x72, - 0x76, 0x69, 0x63, 0x65, 0x12, 0x1f, 0x0a, 0x05, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x12, 0x07, 0x2e, - 0x50, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x1a, 0x07, 0x2e, 0x50, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x22, - 0x00, 0x28, 0x01, 0x30, 0x01, 0x42, 0x46, 0x5a, 0x44, 0x73, 0x69, 0x67, 0x73, 0x2e, 0x6b, 0x38, - 0x73, 0x2e, 0x69, 0x6f, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2d, 0x6e, - 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x2d, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x2f, 0x6b, 0x6f, 0x6e, - 0x6e, 0x65, 0x63, 0x74, 0x69, 0x76, 0x69, 0x74, 0x79, 0x2d, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, - 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x62, 0x06, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x6e, 0x64, 0x6f, 0x6d, 0x22, 0x07, 0x0a, 0x05, 0x44, 0x72, 0x61, 0x69, 0x6e, 0x22, 0x4e, 0x0a, + 0x04, 0x44, 0x61, 0x74, 0x61, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, + 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, + 0x74, 0x49, 0x44, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, + 0x61, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x2a, 0x69, 0x0a, + 0x0a, 0x50, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0c, 0x0a, 0x08, 0x44, + 0x49, 0x41, 0x4c, 0x5f, 0x52, 0x45, 0x51, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x44, 0x49, 0x41, + 0x4c, 0x5f, 0x52, 0x53, 0x50, 0x10, 0x01, 0x12, 0x0d, 0x0a, 0x09, 0x43, 0x4c, 0x4f, 0x53, 0x45, + 0x5f, 0x52, 0x45, 0x51, 0x10, 0x02, 0x12, 0x0d, 0x0a, 0x09, 0x43, 0x4c, 0x4f, 0x53, 0x45, 0x5f, + 0x52, 0x53, 0x50, 0x10, 0x03, 0x12, 0x08, 0x0a, 0x04, 0x44, 0x41, 0x54, 0x41, 0x10, 0x04, 0x12, + 0x0c, 0x0a, 0x08, 0x44, 0x49, 0x41, 0x4c, 0x5f, 0x43, 0x4c, 0x53, 0x10, 0x05, 0x12, 0x09, 0x0a, + 0x05, 0x44, 0x52, 0x41, 0x49, 0x4e, 0x10, 0x06, 0x32, 0x2f, 0x0a, 0x0c, 0x50, 0x72, 0x6f, 0x78, + 0x79, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x1f, 0x0a, 0x05, 0x50, 0x72, 0x6f, 0x78, + 0x79, 0x12, 0x07, 0x2e, 0x50, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x1a, 0x07, 0x2e, 0x50, 0x61, 0x63, + 0x6b, 0x65, 0x74, 0x22, 0x00, 0x28, 0x01, 0x30, 0x01, 0x42, 0x46, 0x5a, 0x44, 0x73, 0x69, 0x67, + 0x73, 0x2e, 0x6b, 0x38, 0x73, 0x2e, 0x69, 0x6f, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x65, 0x72, 0x76, + 0x65, 0x72, 0x2d, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x2d, 0x70, 0x72, 0x6f, 0x78, 0x79, + 0x2f, 0x6b, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x76, 0x69, 0x74, 0x79, 0x2d, 0x63, 0x6c, + 0x69, 0x65, 0x6e, 0x74, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x63, 0x6c, 0x69, 0x65, 0x6e, + 0x74, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -671,7 +729,7 @@ func file_konnectivity_client_proto_client_client_proto_rawDescGZIP() []byte { } var file_konnectivity_client_proto_client_client_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_konnectivity_client_proto_client_client_proto_msgTypes = make([]protoimpl.MessageInfo, 7) +var file_konnectivity_client_proto_client_client_proto_msgTypes = make([]protoimpl.MessageInfo, 8) var file_konnectivity_client_proto_client_client_proto_goTypes = []interface{}{ (PacketType)(0), // 0: PacketType (*Packet)(nil), // 1: Packet @@ -680,23 +738,25 @@ var file_konnectivity_client_proto_client_client_proto_goTypes = []interface{}{ (*CloseRequest)(nil), // 4: CloseRequest (*CloseResponse)(nil), // 5: CloseResponse (*CloseDial)(nil), // 6: CloseDial - (*Data)(nil), // 7: Data + (*Drain)(nil), // 7: Drain + (*Data)(nil), // 8: Data } var file_konnectivity_client_proto_client_client_proto_depIdxs = []int32{ 0, // 0: Packet.type:type_name -> PacketType 2, // 1: Packet.dialRequest:type_name -> DialRequest 3, // 2: Packet.dialResponse:type_name -> DialResponse - 7, // 3: Packet.data:type_name -> Data + 8, // 3: Packet.data:type_name -> Data 4, // 4: Packet.closeRequest:type_name -> CloseRequest 5, // 5: Packet.closeResponse:type_name -> CloseResponse 6, // 6: Packet.closeDial:type_name -> CloseDial - 1, // 7: ProxyService.Proxy:input_type -> Packet - 1, // 8: ProxyService.Proxy:output_type -> Packet - 8, // [8:9] is the sub-list for method output_type - 7, // [7:8] is the sub-list for method input_type - 7, // [7:7] is the sub-list for extension type_name - 7, // [7:7] is the sub-list for extension extendee - 0, // [0:7] is the sub-list for field type_name + 7, // 7: Packet.drain:type_name -> Drain + 1, // 8: ProxyService.Proxy:input_type -> Packet + 1, // 9: ProxyService.Proxy:output_type -> Packet + 9, // [9:10] is the sub-list for method output_type + 8, // [8:9] is the sub-list for method input_type + 8, // [8:8] is the sub-list for extension type_name + 8, // [8:8] is the sub-list for extension extendee + 0, // [0:8] is the sub-list for field type_name } func init() { file_konnectivity_client_proto_client_client_proto_init() } @@ -778,6 +838,18 @@ func file_konnectivity_client_proto_client_client_proto_init() { } } file_konnectivity_client_proto_client_client_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Drain); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_konnectivity_client_proto_client_client_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Data); i { case 0: return &v.state @@ -797,6 +869,7 @@ func file_konnectivity_client_proto_client_client_proto_init() { (*Packet_CloseRequest)(nil), (*Packet_CloseResponse)(nil), (*Packet_CloseDial)(nil), + (*Packet_Drain)(nil), } type x struct{} out := protoimpl.TypeBuilder{ @@ -804,7 +877,7 @@ func file_konnectivity_client_proto_client_client_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_konnectivity_client_proto_client_client_proto_rawDesc, NumEnums: 1, - NumMessages: 7, + NumMessages: 8, NumExtensions: 0, NumServices: 1, }, diff --git a/konnectivity-client/proto/client/client.proto b/konnectivity-client/proto/client/client.proto index 811278e62..006e0ac83 100644 --- a/konnectivity-client/proto/client/client.proto +++ b/konnectivity-client/proto/client/client.proto @@ -30,6 +30,7 @@ enum PacketType { CLOSE_RSP = 3; DATA = 4; DIAL_CLS = 5; + DRAIN = 6; } message Packet { @@ -42,6 +43,7 @@ message Packet { CloseRequest closeRequest = 5; CloseResponse closeResponse = 6; CloseDial closeDial = 7; + Drain drain = 8; } } @@ -85,6 +87,11 @@ message CloseDial { int64 random = 1; } +message Drain { + // A hint from an Agent to Server that it is pending termination. + // A Server should prefer non-draining agents for new dials. +} + message Data { // connectID to connect to int64 connectID = 1; diff --git a/vendor/sigs.k8s.io/apiserver-network-proxy/konnectivity-client/proto/client/client.pb.go b/vendor/sigs.k8s.io/apiserver-network-proxy/konnectivity-client/proto/client/client.pb.go index 6af92b448..0831f3e63 100644 --- a/vendor/sigs.k8s.io/apiserver-network-proxy/konnectivity-client/proto/client/client.pb.go +++ b/vendor/sigs.k8s.io/apiserver-network-proxy/konnectivity-client/proto/client/client.pb.go @@ -15,7 +15,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.27.1 -// protoc v3.12.4 +// protoc v3.21.12 // source: konnectivity-client/proto/client/client.proto package client @@ -43,6 +43,7 @@ const ( PacketType_CLOSE_RSP PacketType = 3 PacketType_DATA PacketType = 4 PacketType_DIAL_CLS PacketType = 5 + PacketType_DRAIN PacketType = 6 ) // Enum value maps for PacketType. @@ -54,6 +55,7 @@ var ( 3: "CLOSE_RSP", 4: "DATA", 5: "DIAL_CLS", + 6: "DRAIN", } PacketType_value = map[string]int32{ "DIAL_REQ": 0, @@ -62,6 +64,7 @@ var ( "CLOSE_RSP": 3, "DATA": 4, "DIAL_CLS": 5, + "DRAIN": 6, } ) @@ -106,6 +109,7 @@ type Packet struct { // *Packet_CloseRequest // *Packet_CloseResponse // *Packet_CloseDial + // *Packet_Drain Payload isPacket_Payload `protobuf_oneof:"payload"` } @@ -197,6 +201,13 @@ func (x *Packet) GetCloseDial() *CloseDial { return nil } +func (x *Packet) GetDrain() *Drain { + if x, ok := x.GetPayload().(*Packet_Drain); ok { + return x.Drain + } + return nil +} + type isPacket_Payload interface { isPacket_Payload() } @@ -225,6 +236,10 @@ type Packet_CloseDial struct { CloseDial *CloseDial `protobuf:"bytes,7,opt,name=closeDial,proto3,oneof"` } +type Packet_Drain struct { + Drain *Drain `protobuf:"bytes,8,opt,name=drain,proto3,oneof"` +} + func (*Packet_DialRequest) isPacket_Payload() {} func (*Packet_DialResponse) isPacket_Payload() {} @@ -237,6 +252,8 @@ func (*Packet_CloseResponse) isPacket_Payload() {} func (*Packet_CloseDial) isPacket_Payload() {} +func (*Packet_Drain) isPacket_Payload() {} + type DialRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -522,6 +539,44 @@ func (x *CloseDial) GetRandom() int64 { return 0 } +type Drain struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *Drain) Reset() { + *x = Drain{} + if protoimpl.UnsafeEnabled { + mi := &file_konnectivity_client_proto_client_client_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Drain) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Drain) ProtoMessage() {} + +func (x *Drain) ProtoReflect() protoreflect.Message { + mi := &file_konnectivity_client_proto_client_client_proto_msgTypes[6] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Drain.ProtoReflect.Descriptor instead. +func (*Drain) Descriptor() ([]byte, []int) { + return file_konnectivity_client_proto_client_client_proto_rawDescGZIP(), []int{6} +} + type Data struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -538,7 +593,7 @@ type Data struct { func (x *Data) Reset() { *x = Data{} if protoimpl.UnsafeEnabled { - mi := &file_konnectivity_client_proto_client_client_proto_msgTypes[6] + mi := &file_konnectivity_client_proto_client_client_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -551,7 +606,7 @@ func (x *Data) String() string { func (*Data) ProtoMessage() {} func (x *Data) ProtoReflect() protoreflect.Message { - mi := &file_konnectivity_client_proto_client_client_proto_msgTypes[6] + mi := &file_konnectivity_client_proto_client_client_proto_msgTypes[7] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -564,7 +619,7 @@ func (x *Data) ProtoReflect() protoreflect.Message { // Deprecated: Use Data.ProtoReflect.Descriptor instead. func (*Data) Descriptor() ([]byte, []int) { - return file_konnectivity_client_proto_client_client_proto_rawDescGZIP(), []int{6} + return file_konnectivity_client_proto_client_client_proto_rawDescGZIP(), []int{7} } func (x *Data) GetConnectID() int64 { @@ -594,7 +649,7 @@ var file_konnectivity_client_proto_client_client_proto_rawDesc = []byte{ 0x0a, 0x2d, 0x6b, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x76, 0x69, 0x74, 0x79, 0x2d, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x2f, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, - 0xd1, 0x02, 0x0a, 0x06, 0x50, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x1f, 0x0a, 0x04, 0x74, 0x79, + 0xf1, 0x02, 0x0a, 0x06, 0x50, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x1f, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x0b, 0x2e, 0x50, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x30, 0x0a, 0x0b, 0x64, 0x69, 0x61, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, @@ -614,7 +669,9 @@ var file_konnectivity_client_proto_client_client_proto_rawDesc = []byte{ 0x6c, 0x6f, 0x73, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2a, 0x0a, 0x09, 0x63, 0x6c, 0x6f, 0x73, 0x65, 0x44, 0x69, 0x61, 0x6c, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x43, 0x6c, 0x6f, 0x73, 0x65, 0x44, 0x69, 0x61, 0x6c, 0x48, 0x00, 0x52, 0x09, 0x63, - 0x6c, 0x6f, 0x73, 0x65, 0x44, 0x69, 0x61, 0x6c, 0x42, 0x09, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, + 0x6c, 0x6f, 0x73, 0x65, 0x44, 0x69, 0x61, 0x6c, 0x12, 0x1e, 0x0a, 0x05, 0x64, 0x72, 0x61, 0x69, + 0x6e, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x06, 0x2e, 0x44, 0x72, 0x61, 0x69, 0x6e, 0x48, + 0x00, 0x52, 0x05, 0x64, 0x72, 0x61, 0x69, 0x6e, 0x42, 0x09, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x22, 0x5b, 0x0a, 0x0b, 0x44, 0x69, 0x61, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x18, @@ -636,26 +693,27 @@ var file_konnectivity_client_proto_client_client_proto_rawDesc = []byte{ 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x49, 0x44, 0x22, 0x23, 0x0a, 0x09, 0x43, 0x6c, 0x6f, 0x73, 0x65, 0x44, 0x69, 0x61, 0x6c, 0x12, 0x16, 0x0a, 0x06, 0x72, 0x61, 0x6e, 0x64, 0x6f, 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x72, 0x61, - 0x6e, 0x64, 0x6f, 0x6d, 0x22, 0x4e, 0x0a, 0x04, 0x44, 0x61, 0x74, 0x61, 0x12, 0x1c, 0x0a, 0x09, - 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, - 0x09, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x49, 0x44, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, - 0x72, 0x6f, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, - 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, - 0x64, 0x61, 0x74, 0x61, 0x2a, 0x5e, 0x0a, 0x0a, 0x50, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x54, 0x79, - 0x70, 0x65, 0x12, 0x0c, 0x0a, 0x08, 0x44, 0x49, 0x41, 0x4c, 0x5f, 0x52, 0x45, 0x51, 0x10, 0x00, - 0x12, 0x0c, 0x0a, 0x08, 0x44, 0x49, 0x41, 0x4c, 0x5f, 0x52, 0x53, 0x50, 0x10, 0x01, 0x12, 0x0d, - 0x0a, 0x09, 0x43, 0x4c, 0x4f, 0x53, 0x45, 0x5f, 0x52, 0x45, 0x51, 0x10, 0x02, 0x12, 0x0d, 0x0a, - 0x09, 0x43, 0x4c, 0x4f, 0x53, 0x45, 0x5f, 0x52, 0x53, 0x50, 0x10, 0x03, 0x12, 0x08, 0x0a, 0x04, - 0x44, 0x41, 0x54, 0x41, 0x10, 0x04, 0x12, 0x0c, 0x0a, 0x08, 0x44, 0x49, 0x41, 0x4c, 0x5f, 0x43, - 0x4c, 0x53, 0x10, 0x05, 0x32, 0x2f, 0x0a, 0x0c, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x53, 0x65, 0x72, - 0x76, 0x69, 0x63, 0x65, 0x12, 0x1f, 0x0a, 0x05, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x12, 0x07, 0x2e, - 0x50, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x1a, 0x07, 0x2e, 0x50, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x22, - 0x00, 0x28, 0x01, 0x30, 0x01, 0x42, 0x46, 0x5a, 0x44, 0x73, 0x69, 0x67, 0x73, 0x2e, 0x6b, 0x38, - 0x73, 0x2e, 0x69, 0x6f, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2d, 0x6e, - 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x2d, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x2f, 0x6b, 0x6f, 0x6e, - 0x6e, 0x65, 0x63, 0x74, 0x69, 0x76, 0x69, 0x74, 0x79, 0x2d, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, - 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x62, 0x06, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x6e, 0x64, 0x6f, 0x6d, 0x22, 0x07, 0x0a, 0x05, 0x44, 0x72, 0x61, 0x69, 0x6e, 0x22, 0x4e, 0x0a, + 0x04, 0x44, 0x61, 0x74, 0x61, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, + 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, + 0x74, 0x49, 0x44, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, + 0x61, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x2a, 0x69, 0x0a, + 0x0a, 0x50, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0c, 0x0a, 0x08, 0x44, + 0x49, 0x41, 0x4c, 0x5f, 0x52, 0x45, 0x51, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x44, 0x49, 0x41, + 0x4c, 0x5f, 0x52, 0x53, 0x50, 0x10, 0x01, 0x12, 0x0d, 0x0a, 0x09, 0x43, 0x4c, 0x4f, 0x53, 0x45, + 0x5f, 0x52, 0x45, 0x51, 0x10, 0x02, 0x12, 0x0d, 0x0a, 0x09, 0x43, 0x4c, 0x4f, 0x53, 0x45, 0x5f, + 0x52, 0x53, 0x50, 0x10, 0x03, 0x12, 0x08, 0x0a, 0x04, 0x44, 0x41, 0x54, 0x41, 0x10, 0x04, 0x12, + 0x0c, 0x0a, 0x08, 0x44, 0x49, 0x41, 0x4c, 0x5f, 0x43, 0x4c, 0x53, 0x10, 0x05, 0x12, 0x09, 0x0a, + 0x05, 0x44, 0x52, 0x41, 0x49, 0x4e, 0x10, 0x06, 0x32, 0x2f, 0x0a, 0x0c, 0x50, 0x72, 0x6f, 0x78, + 0x79, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x1f, 0x0a, 0x05, 0x50, 0x72, 0x6f, 0x78, + 0x79, 0x12, 0x07, 0x2e, 0x50, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x1a, 0x07, 0x2e, 0x50, 0x61, 0x63, + 0x6b, 0x65, 0x74, 0x22, 0x00, 0x28, 0x01, 0x30, 0x01, 0x42, 0x46, 0x5a, 0x44, 0x73, 0x69, 0x67, + 0x73, 0x2e, 0x6b, 0x38, 0x73, 0x2e, 0x69, 0x6f, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x65, 0x72, 0x76, + 0x65, 0x72, 0x2d, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x2d, 0x70, 0x72, 0x6f, 0x78, 0x79, + 0x2f, 0x6b, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x76, 0x69, 0x74, 0x79, 0x2d, 0x63, 0x6c, + 0x69, 0x65, 0x6e, 0x74, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x63, 0x6c, 0x69, 0x65, 0x6e, + 0x74, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -671,7 +729,7 @@ func file_konnectivity_client_proto_client_client_proto_rawDescGZIP() []byte { } var file_konnectivity_client_proto_client_client_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_konnectivity_client_proto_client_client_proto_msgTypes = make([]protoimpl.MessageInfo, 7) +var file_konnectivity_client_proto_client_client_proto_msgTypes = make([]protoimpl.MessageInfo, 8) var file_konnectivity_client_proto_client_client_proto_goTypes = []interface{}{ (PacketType)(0), // 0: PacketType (*Packet)(nil), // 1: Packet @@ -680,23 +738,25 @@ var file_konnectivity_client_proto_client_client_proto_goTypes = []interface{}{ (*CloseRequest)(nil), // 4: CloseRequest (*CloseResponse)(nil), // 5: CloseResponse (*CloseDial)(nil), // 6: CloseDial - (*Data)(nil), // 7: Data + (*Drain)(nil), // 7: Drain + (*Data)(nil), // 8: Data } var file_konnectivity_client_proto_client_client_proto_depIdxs = []int32{ 0, // 0: Packet.type:type_name -> PacketType 2, // 1: Packet.dialRequest:type_name -> DialRequest 3, // 2: Packet.dialResponse:type_name -> DialResponse - 7, // 3: Packet.data:type_name -> Data + 8, // 3: Packet.data:type_name -> Data 4, // 4: Packet.closeRequest:type_name -> CloseRequest 5, // 5: Packet.closeResponse:type_name -> CloseResponse 6, // 6: Packet.closeDial:type_name -> CloseDial - 1, // 7: ProxyService.Proxy:input_type -> Packet - 1, // 8: ProxyService.Proxy:output_type -> Packet - 8, // [8:9] is the sub-list for method output_type - 7, // [7:8] is the sub-list for method input_type - 7, // [7:7] is the sub-list for extension type_name - 7, // [7:7] is the sub-list for extension extendee - 0, // [0:7] is the sub-list for field type_name + 7, // 7: Packet.drain:type_name -> Drain + 1, // 8: ProxyService.Proxy:input_type -> Packet + 1, // 9: ProxyService.Proxy:output_type -> Packet + 9, // [9:10] is the sub-list for method output_type + 8, // [8:9] is the sub-list for method input_type + 8, // [8:8] is the sub-list for extension type_name + 8, // [8:8] is the sub-list for extension extendee + 0, // [0:8] is the sub-list for field type_name } func init() { file_konnectivity_client_proto_client_client_proto_init() } @@ -778,6 +838,18 @@ func file_konnectivity_client_proto_client_client_proto_init() { } } file_konnectivity_client_proto_client_client_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Drain); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_konnectivity_client_proto_client_client_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Data); i { case 0: return &v.state @@ -797,6 +869,7 @@ func file_konnectivity_client_proto_client_client_proto_init() { (*Packet_CloseRequest)(nil), (*Packet_CloseResponse)(nil), (*Packet_CloseDial)(nil), + (*Packet_Drain)(nil), } type x struct{} out := protoimpl.TypeBuilder{ @@ -804,7 +877,7 @@ func file_konnectivity_client_proto_client_client_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_konnectivity_client_proto_client_client_proto_rawDesc, NumEnums: 1, - NumMessages: 7, + NumMessages: 8, NumExtensions: 0, NumServices: 1, }, diff --git a/vendor/sigs.k8s.io/apiserver-network-proxy/konnectivity-client/proto/client/client.proto b/vendor/sigs.k8s.io/apiserver-network-proxy/konnectivity-client/proto/client/client.proto index 811278e62..006e0ac83 100644 --- a/vendor/sigs.k8s.io/apiserver-network-proxy/konnectivity-client/proto/client/client.proto +++ b/vendor/sigs.k8s.io/apiserver-network-proxy/konnectivity-client/proto/client/client.proto @@ -30,6 +30,7 @@ enum PacketType { CLOSE_RSP = 3; DATA = 4; DIAL_CLS = 5; + DRAIN = 6; } message Packet { @@ -42,6 +43,7 @@ message Packet { CloseRequest closeRequest = 5; CloseResponse closeResponse = 6; CloseDial closeDial = 7; + Drain drain = 8; } } @@ -85,6 +87,11 @@ message CloseDial { int64 random = 1; } +message Drain { + // A hint from an Agent to Server that it is pending termination. + // A Server should prefer non-draining agents for new dials. +} + message Data { // connectID to connect to int64 connectID = 1; diff --git a/vendor/sigs.k8s.io/apiserver-network-proxy/konnectivity-client/proto/client/client_grpc.pb.go b/vendor/sigs.k8s.io/apiserver-network-proxy/konnectivity-client/proto/client/client_grpc.pb.go index b8d07fe55..5a0d6a2a8 100644 --- a/vendor/sigs.k8s.io/apiserver-network-proxy/konnectivity-client/proto/client/client_grpc.pb.go +++ b/vendor/sigs.k8s.io/apiserver-network-proxy/konnectivity-client/proto/client/client_grpc.pb.go @@ -16,7 +16,7 @@ limitations under the License. // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.2.0 -// - protoc v3.12.4 +// - protoc v3.21.12 // source: konnectivity-client/proto/client/client.proto package client From f7a7f0cde84a3a433ceec26da9a809f1a06cac0e Mon Sep 17 00:00:00 2001 From: Joseph Anttila Hall Date: Mon, 11 Mar 2024 21:33:03 -0700 Subject: [PATCH 2/2] Agent drain: implement first half. - Agent sends drain signal to Server. - Server only logs it. --- cmd/agent/app/server.go | 39 +++++++++++++++++++++++++++----- pkg/agent/client.go | 20 +++++++++++++++- pkg/agent/client_test.go | 34 ++++++++++++++++++++++++++++ pkg/agent/clientset.go | 5 +++- pkg/server/server.go | 2 ++ tests/framework/agent.go | 26 +++++++++++++++++++-- tests/proxy_test.go | 49 ++++++++++++++++++++++++++++++++++++++++ 7 files changed, 165 insertions(+), 10 deletions(-) diff --git a/cmd/agent/app/server.go b/cmd/agent/app/server.go index 8cc3cfd0e..8c087de53 100644 --- a/cmd/agent/app/server.go +++ b/cmd/agent/app/server.go @@ -24,10 +24,13 @@ import ( "net" "net/http" "net/http/pprof" + "os" + "os/signal" "runtime" runpprof "runtime/pprof" "strconv" "strings" + "syscall" "time" "github.com/prometheus/client_golang/prometheus/promhttp" @@ -49,8 +52,8 @@ func NewAgentCommand(a *Agent, o *options.GrpcProxyAgentOptions) *cobra.Command Use: "agent", Long: `A gRPC agent, Connects to the proxy and then allows traffic to be forwarded to it.`, RunE: func(cmd *cobra.Command, args []string) error { - stopCh := make(chan struct{}) - return a.Run(o, stopCh) + drainCh, stopCh := SetupSignalHandler() + return a.Run(o, drainCh, stopCh) }, } @@ -64,13 +67,13 @@ type Agent struct { cs *agent.ClientSet } -func (a *Agent) Run(o *options.GrpcProxyAgentOptions, stopCh <-chan struct{}) error { +func (a *Agent) Run(o *options.GrpcProxyAgentOptions, drainCh, stopCh <-chan struct{}) error { o.Print() if err := o.Validate(); err != nil { return fmt.Errorf("failed to validate agent options with %v", err) } - cs, err := a.runProxyConnection(o, stopCh) + cs, err := a.runProxyConnection(o, drainCh, stopCh) if err != nil { return fmt.Errorf("failed to run proxy connection with %v", err) } @@ -92,7 +95,31 @@ func (a *Agent) Run(o *options.GrpcProxyAgentOptions, stopCh <-chan struct{}) er return nil } -func (a *Agent) runProxyConnection(o *options.GrpcProxyAgentOptions, stopCh <-chan struct{}) (*agent.ClientSet, error) { +var shutdownSignals = []os.Signal{os.Interrupt, syscall.SIGTERM} + +func SetupSignalHandler() (drainCh, stopCh <-chan struct{}) { + drain := make(chan struct{}) + stop := make(chan struct{}) + c := make(chan os.Signal, 2) + signal.Notify(c, shutdownSignals...) + labels := runpprof.Labels( + "core", "signalHandler", + ) + go runpprof.Do(context.Background(), labels, func(context.Context) { handleSignals(c, drain, stop) }) + + return drain, stop +} + +func handleSignals(signalCh chan os.Signal, drainCh, stopCh chan struct{}) { + s := <-signalCh + klog.V(2).InfoS("Received first signal", "signal", s) + close(drainCh) + s = <-signalCh + klog.V(2).InfoS("Received second signal", "signal", s) + close(stopCh) +} + +func (a *Agent) runProxyConnection(o *options.GrpcProxyAgentOptions, drainCh, stopCh <-chan struct{}) (*agent.ClientSet, error) { var tlsConfig *tls.Config var err error if tlsConfig, err = util.GetClientTLSConfig(o.CaCert, o.AgentCert, o.AgentKey, o.ProxyServerHost, o.AlpnProtos); err != nil { @@ -106,7 +133,7 @@ func (a *Agent) runProxyConnection(o *options.GrpcProxyAgentOptions, stopCh <-ch }), } cc := o.ClientSetConfig(dialOptions...) - cs := cc.NewAgentClientSet(stopCh) + cs := cc.NewAgentClientSet(drainCh, stopCh) cs.Serve() return cs, nil diff --git a/pkg/agent/client.go b/pkg/agent/client.go index 16b1df244..11159632f 100644 --- a/pkg/agent/client.go +++ b/pkg/agent/client.go @@ -137,7 +137,11 @@ type Client struct { address string opts []grpc.DialOption conn *grpc.ClientConn - stopCh chan struct{} + + drainCh <-chan struct{} + drainOnce sync.Once + stopCh chan struct{} + // locks sendLock sync.Mutex recvLock sync.Mutex @@ -158,6 +162,7 @@ func newAgentClient(address, agentID, agentIdentifiers string, cs *ClientSet, op agentIdentifiers: agentIdentifiers, opts: opts, probeInterval: cs.probeInterval, + drainCh: cs.drainCh, stopCh: make(chan struct{}), serviceAccountTokenPath: cs.serviceAccountTokenPath, connManager: newConnectionManager(), @@ -325,6 +330,19 @@ func (a *Client) Serve() { case <-a.stopCh: klog.V(2).InfoS("stop agent client.") return + case <-a.drainCh: + a.drainOnce.Do(func() { + klog.V(2).InfoS("drain agent client", "serverID", a.serverID, "agentID", a.agentID) + drainPkt := &client.Packet{ + Type: client.PacketType_DRAIN, + Payload: &client.Packet_Drain{ + Drain: &client.Drain{}, + }, + } + if err := a.Send(drainPkt); err != nil { + klog.ErrorS(err, "drain failure", "") + } + }) default: } diff --git a/pkg/agent/client_test.go b/pkg/agent/client_test.go index 87b61b1b9..6e69ebbcd 100644 --- a/pkg/agent/client_test.go +++ b/pkg/agent/client_test.go @@ -343,6 +343,40 @@ func TestFailedSend_DialResp_GRPC(t *testing.T) { }() } +func TestDrain(t *testing.T) { + var stream agent.AgentService_ConnectClient + drainCh := make(chan struct{}) + stopCh := make(chan struct{}) + cs := &ClientSet{ + clients: make(map[string]*Client), + drainCh: drainCh, + stopCh: stopCh, + } + testClient := &Client{ + connManager: newConnectionManager(), + drainCh: drainCh, + stopCh: stopCh, + cs: cs, + } + testClient.stream, stream = pipe() + + // Start agent + go testClient.Serve() + defer close(stopCh) + + // Simulate pod first shutdown signal + close(drainCh) + + // Expect to receive DRAIN packet from (Agent) Client + pkt, err := stream.Recv() + if err != nil { + t.Fatal(err) + } + if pkt.Type != client.PacketType_DRAIN { + t.Errorf("expect PacketType_DRAIN; got %v", pkt.Type) + } +} + // fakeStream implements AgentService_ConnectClient type fakeStream struct { grpc.ClientStream diff --git a/pkg/agent/clientset.go b/pkg/agent/clientset.go index c5adcf4f7..6a2510a84 100644 --- a/pkg/agent/clientset.go +++ b/pkg/agent/clientset.go @@ -52,6 +52,8 @@ type ClientSet struct { dialOptions []grpc.DialOption // file path contains service account token serviceAccountTokenPath string + // channel to signal that the agent is pending termination. + drainCh <-chan struct{} // channel to signal shutting down the client set. Primarily for test. stopCh <-chan struct{} @@ -141,7 +143,7 @@ type ClientSetConfig struct { SyncForever bool } -func (cc *ClientSetConfig) NewAgentClientSet(stopCh <-chan struct{}) *ClientSet { +func (cc *ClientSetConfig) NewAgentClientSet(drainCh, stopCh <-chan struct{}) *ClientSet { return &ClientSet{ clients: make(map[string]*Client), agentID: cc.AgentID, @@ -154,6 +156,7 @@ func (cc *ClientSetConfig) NewAgentClientSet(stopCh <-chan struct{}) *ClientSet serviceAccountTokenPath: cc.ServiceAccountTokenPath, warnOnChannelLimit: cc.WarnOnChannelLimit, syncForever: cc.SyncForever, + drainCh: drainCh, stopCh: stopCh, } } diff --git a/pkg/server/server.go b/pkg/server/server.go index d65d5c4be..b3f883c93 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -990,6 +990,8 @@ func (s *ProxyServer) serveRecvBackend(backend Backend, agentID string, recvCh < klog.V(5).InfoS("CLOSE_RSP sent to frontend", "connectionID", resp.ConnectID) } + case client.PacketType_DRAIN: + klog.V(2).InfoS("agent is draining", "agentID", agentID) default: klog.V(5).InfoS("Ignoring unrecognized packet from backend", "packet", pkt, "agentID", agentID) } diff --git a/tests/framework/agent.go b/tests/framework/agent.go index 15b9ae2e2..6d446fca6 100644 --- a/tests/framework/agent.go +++ b/tests/framework/agent.go @@ -27,6 +27,7 @@ import ( "path/filepath" "strconv" "sync" + "syscall" "testing" "time" @@ -53,6 +54,7 @@ type AgentRunner interface { type Agent interface { GetConnectedServerCount() (int, error) Ready() bool + Drain() Stop() Metrics() metricstest.AgentTester } @@ -66,9 +68,10 @@ func (*InProcessAgentRunner) Start(t testing.TB, opts AgentOpts) (Agent, error) } ctx, cancel := context.WithCancel(context.Background()) + drainCh := make(chan struct{}) stopCh := make(chan struct{}) go func() { - if err := a.Run(o, stopCh); err != nil { + if err := a.Run(o, drainCh, stopCh); err != nil { log.Printf("ERROR running agent: %v", err) cancel() } @@ -84,6 +87,7 @@ func (*InProcessAgentRunner) Start(t testing.TB, opts AgentOpts) (Agent, error) pa := &inProcessAgent{ client: a.ClientSet(), + drainCh: drainCh, stopCh: stopCh, healthAddr: healthAddr, } @@ -94,12 +98,21 @@ func (*InProcessAgentRunner) Start(t testing.TB, opts AgentOpts) (Agent, error) type inProcessAgent struct { client *agent.ClientSet + drainOnce sync.Once + drainCh chan struct{} + stopOnce sync.Once stopCh chan struct{} healthAddr string } +func (a *inProcessAgent) Drain() { + a.drainOnce.Do(func() { + close(a.drainCh) + }) +} + func (a *inProcessAgent) Stop() { a.stopOnce.Do(func() { close(a.stopCh) @@ -160,7 +173,16 @@ type externalAgent struct { cmd *exec.Cmd metrics *metricstest.Tester - stopOnce sync.Once + drainOnce sync.Once + stopOnce sync.Once +} + +func (a *externalAgent) Drain() { + a.drainOnce.Do(func() { + if err := a.cmd.Process.Signal(syscall.SIGTERM); err != nil { + log.Fatalf("Error draining agent process: %v", err) + } + }) } func (a *externalAgent) Stop() { diff --git a/tests/proxy_test.go b/tests/proxy_test.go index bdf0de942..708b1c798 100644 --- a/tests/proxy_test.go +++ b/tests/proxy_test.go @@ -789,6 +789,55 @@ func TestFailedDial_HTTPCONN(t *testing.T) { resetAllMetrics() // For clean shutdown. } +func TestProxyHandle_AfterDrain(t *testing.T) { + expectCleanShutdown(t) + + server := httptest.NewServer(newEchoServer("hello")) + defer server.Close() + + ps := runGRPCProxyServer(t) + defer ps.Stop() + + a := runAgent(t, ps.AgentAddr()) + defer a.Stop() + waitForConnectedServerCount(t, 1, a) + + // Drain agent + a.Drain() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + tunnel, err := createSingleUseGrpcTunnel(ctx, ps.FrontAddr()) + if err != nil { + t.Fatal(err) + } + + c := &http.Client{ + Transport: &http.Transport{ + DialContext: tunnel.DialContext, + }, + } + + req, err := http.NewRequest("GET", server.URL, nil) + if err != nil { + t.Fatal(err) + } + + r, err := c.Do(req) + if err != nil { + t.Fatal(err) + } + defer r.Body.Close() + + data, err := io.ReadAll(r.Body) + if err != nil { + t.Fatal(err) + } + if string(data) != "hello" { + t.Errorf("expect %v; got %v", "hello", string(data)) + } +} + func runGRPCProxyServer(t testing.TB) framework.ProxyServer { return runGRPCProxyServerWithServerCount(t, 1) }