Skip to content

Commit

Permalink
[occm] feat : add custom tags to load balancer and other resources us…
Browse files Browse the repository at this point in the history 
…ing service annotation
  • Loading branch information
@yorubad-dev
yorubad-dev committed Nov 13, 2023
1 parent 95a3823 commit 954ee65
Show file tree
Hide file tree
Showing 3 changed files with 164 additions and 12 deletions.
5 changes: 5 additions & 0 deletions ...cloud-controller-manager/expose-applications-using-loadbalancer-type-service.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -236,6 +236,11 @@ Request Body:
This annotation is automatically added and it contains the floating ip address of the load balancer service.
When using `loadbalancer.openstack.org/hostname` annotation it is the only place to see the real address of the load balancer.

- `loadbalancer.openstack.org/custom-tags`

Allows to specify custom tags that all load balancer resources for that Service will be tagged with.
Tags are arbitrary strings, to specify multiple tags separate them using a comma `,` in the annotation.

### Switching between Floating Subnets by using preconfigured Classes

If you have multiple `FloatingIPPools` and/or `FloatingIPSubnets` it might be desirable to offer the user logical meanings for `LoadBalancers` like `internetFacing` or `DMZ` instead of requiring the user to select a dedicated network or subnet ID at the service object level as an annotation.
Expand Down
59 changes: 49 additions & 10 deletions pkg/openstack/loadbalancer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@ const (
ServiceAnnotationLoadBalancerXForwardedFor = "loadbalancer.openstack.org/x-forwarded-for"
ServiceAnnotationLoadBalancerFlavorID = "loadbalancer.openstack.org/flavor-id"
ServiceAnnotationLoadBalancerAvailabilityZone = "loadbalancer.openstack.org/availability-zone"
ServiceAnnotationLoadBalancerCustomTags = "loadbalancer.openstack.org/custom-tags"
// ServiceAnnotationLoadBalancerEnableHealthMonitor defines whether to create health monitor for the load balancer
// pool, if not specified, use 'create-monitor' config. The health monitor can be created or deleted dynamically.
ServiceAnnotationLoadBalancerEnableHealthMonitor = "loadbalancer.openstack.org/enable-health-monitor"
Expand Down Expand Up @@ -469,6 +470,7 @@ func (lbaas *LbaasV2) createOctaviaLoadBalancer(name, clusterName string, servic

if svcConf.supportLBTags {
createOpts.Tags = []string{svcConf.lbName}
createOpts.Tags = append(createOpts.Tags, lbaas.getCustomLoadBalancerTags(service, svcConf)...)
}

if svcConf.flavorID != "" {
Expand Down Expand Up @@ -508,8 +510,8 @@ func (lbaas *LbaasV2) createOctaviaLoadBalancer(name, clusterName string, servic

if !lbaas.opts.ProviderRequiresSerialAPICalls {
for portIndex, port := range service.Spec.Ports {
listenerCreateOpt := lbaas.buildListenerCreateOpt(port, svcConf, cpoutil.Sprintf255(listenerFormat, portIndex, name))
members, newMembers, err := lbaas.buildBatchUpdateMemberOpts(port, nodes, svcConf)
listenerCreateOpt := lbaas.buildListenerCreateOpt(port, svcConf, service, cpoutil.Sprintf255(listenerFormat, portIndex, name))
members, newMembers, err := lbaas.buildBatchUpdateMemberOpts(port, nodes, svcConf, service)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -606,6 +608,23 @@ func (lbaas *LbaasV2) getLoadBalancerLegacyName(_ context.Context, _ string, ser
return cloudprovider.DefaultLoadBalancerName(service)
}

// Returns a list of custom loadbalancer tags for the supported service resources.
func (lbaas *LbaasV2) getCustomLoadBalancerTags(service *corev1.Service, svcConf *serviceConfig) []string {
if !svcConf.supportLBTags {
return nil
}

annotationVal := getStringFromServiceAnnotation(service, ServiceAnnotationLoadBalancerCustomTags, "")

tags := strings.Split(annotationVal, ",")

for i, tag := range tags {
tags[i] = strings.TrimSpace(tag)
}

return tags
}

// The LB needs to be configured with instance addresses on the same
// subnet as the LB (aka opts.SubnetID). Currently, we're just
// guessing that the node's InternalIP is the right address.
Expand Down Expand Up @@ -896,10 +915,16 @@ func (lbaas *LbaasV2) deleteOctaviaListeners(lbID string, listenerList []listene
return nil
}

func (lbaas *LbaasV2) createFloatingIP(msg string, floatIPOpts floatingips.CreateOpts) (*floatingips.FloatingIP, error) {
func (lbaas *LbaasV2) createFloatingIP(msg string, floatIPOpts floatingips.CreateOpts, service *corev1.Service, svcConf *serviceConfig) (*floatingips.FloatingIP, error) {
klog.V(4).Infof("%s floating ip with opts %+v", msg, floatIPOpts)
mc := metrics.NewMetricContext("floating_ip", "create")
floatIP, err := floatingips.Create(lbaas.network, floatIPOpts).Extract()

tags := lbaas.getCustomLoadBalancerTags(service, svcConf)

if _, err := neutrontags.ReplaceAll(lbaas.network, "floatingips", floatIP.ID, neutrontags.ReplaceAllOpts{Tags: tags}).Extract(); err != nil {
return nil, fmt.Errorf("failed to add custom tags %s to floatingIPs %s with a projectID (%s)", tags, floatIP.ID, floatIP.ProjectID)
}
err = PreserveGopherError(err)
if mc.ObserveRequest(err) != nil {
return floatIP, fmt.Errorf("error creating LB floatingip: %s", err)
Expand Down Expand Up @@ -1037,7 +1062,7 @@ func (lbaas *LbaasV2) ensureFloatingIP(clusterName string, service *corev1.Servi
svcConf.lbPublicSubnetSpec, svcConf.lbPublicNetworkID)
for _, subnet := range foundSubnets {
floatIPOpts.SubnetID = subnet.ID
floatIP, err = lbaas.createFloatingIP(fmt.Sprintf("Trying subnet %s for creating", subnet.Name), floatIPOpts)
floatIP, err = lbaas.createFloatingIP(fmt.Sprintf("Trying subnet %s for creating", subnet.Name), floatIPOpts, service, svcConf)
if err == nil {
foundSubnet = subnet
break
Expand All @@ -1054,7 +1079,7 @@ func (lbaas *LbaasV2) ensureFloatingIP(clusterName string, service *corev1.Servi
floatIPOpts.SubnetID = svcConf.lbPublicSubnetSpec.subnetID
}
floatIPOpts.FloatingIP = loadBalancerIP
floatIP, err = lbaas.createFloatingIP("Creating", floatIPOpts)
floatIP, err = lbaas.createFloatingIP("Creating", floatIPOpts, service, svcConf)
if err != nil {
return "", err
}
Expand Down Expand Up @@ -1237,7 +1262,7 @@ func (lbaas *LbaasV2) ensureOctaviaPool(lbID string, name string, listener *list
curMembers.Insert(fmt.Sprintf("%s-%s-%d-%d", m.Name, m.Address, m.ProtocolPort, m.MonitorPort))
}

members, newMembers, err := lbaas.buildBatchUpdateMemberOpts(port, nodes, svcConf)
members, newMembers, err := lbaas.buildBatchUpdateMemberOpts(port, nodes, svcConf, service)
if err != nil {
return nil, err
}
Expand All @@ -1254,6 +1279,8 @@ func (lbaas *LbaasV2) ensureOctaviaPool(lbID string, name string, listener *list
}

func (lbaas *LbaasV2) buildPoolCreateOpt(listenerProtocol string, service *corev1.Service, svcConf *serviceConfig, name string) v2pools.CreateOpts {
customTags := lbaas.getCustomLoadBalancerTags(service, svcConf)

// By default, use the protocol of the listener
poolProto := v2pools.Protocol(listenerProtocol)
if svcConf.enableProxyProtocol {
Expand Down Expand Up @@ -1284,14 +1311,17 @@ func (lbaas *LbaasV2) buildPoolCreateOpt(listenerProtocol string, service *corev
Protocol: poolProto,
LBMethod: lbmethod,
Persistence: persistence,
Tags: customTags,
}
}

// buildBatchUpdateMemberOpts returns v2pools.BatchUpdateMemberOpts array for Services and Nodes alongside a list of member names
func (lbaas *LbaasV2) buildBatchUpdateMemberOpts(port corev1.ServicePort, nodes []*corev1.Node, svcConf *serviceConfig) ([]v2pools.BatchUpdateMemberOpts, sets.Set[string], error) {
func (lbaas *LbaasV2) buildBatchUpdateMemberOpts(port corev1.ServicePort, nodes []*corev1.Node, svcConf *serviceConfig, service *corev1.Service) ([]v2pools.BatchUpdateMemberOpts, sets.Set[string], error) {
var members []v2pools.BatchUpdateMemberOpts
newMembers := sets.New[string]()

customTags := lbaas.getCustomLoadBalancerTags(service, svcConf)

for _, node := range nodes {
addr, err := nodeAddressForLB(node, svcConf.preferredIPFamily)
if err != nil {
Expand All @@ -1315,6 +1345,7 @@ func (lbaas *LbaasV2) buildBatchUpdateMemberOpts(port corev1.ServicePort, nodes
ProtocolPort: int(port.NodePort),
Name: &node.Name,
SubnetID: memberSubnetID,
Tags: customTags,
}
if svcConf.healthCheckNodePort > 0 && lbaas.canUseHTTPMonitor(port) {
member.MonitorPort = &svcConf.healthCheckNodePort
Expand All @@ -1327,13 +1358,13 @@ func (lbaas *LbaasV2) buildBatchUpdateMemberOpts(port corev1.ServicePort, nodes
}

// Make sure the listener is created for Service
func (lbaas *LbaasV2) ensureOctaviaListener(lbID string, name string, curListenerMapping map[listenerKey]*listeners.Listener, port corev1.ServicePort, svcConf *serviceConfig, _ *corev1.Service) (*listeners.Listener, error) {
func (lbaas *LbaasV2) ensureOctaviaListener(lbID string, name string, curListenerMapping map[listenerKey]*listeners.Listener, port corev1.ServicePort, svcConf *serviceConfig, service *corev1.Service) (*listeners.Listener, error) {
listener, isPresent := curListenerMapping[listenerKey{
Protocol: getListenerProtocol(port.Protocol, svcConf),
Port: int(port.Port),
}]
if !isPresent {
listenerCreateOpt := lbaas.buildListenerCreateOpt(port, svcConf, name)
listenerCreateOpt := lbaas.buildListenerCreateOpt(port, svcConf, service, name)
listenerCreateOpt.LoadbalancerID = lbID

klog.V(2).Infof("Creating listener for port %d using protocol %s", int(port.Port), listenerCreateOpt.Protocol)
Expand Down Expand Up @@ -1419,7 +1450,7 @@ func (lbaas *LbaasV2) ensureOctaviaListener(lbID string, name string, curListene
}

// buildListenerCreateOpt returns listeners.CreateOpts for a specific Service port and configuration
func (lbaas *LbaasV2) buildListenerCreateOpt(port corev1.ServicePort, svcConf *serviceConfig, name string) listeners.CreateOpts {
func (lbaas *LbaasV2) buildListenerCreateOpt(port corev1.ServicePort, svcConf *serviceConfig, service *corev1.Service, name string) listeners.CreateOpts {
listenerCreateOpt := listeners.CreateOpts{
Name: name,
Protocol: listeners.Protocol(port.Protocol),
Expand All @@ -1429,6 +1460,8 @@ func (lbaas *LbaasV2) buildListenerCreateOpt(port corev1.ServicePort, svcConf *s

if svcConf.supportLBTags {
listenerCreateOpt.Tags = []string{svcConf.lbName}
// add custom tags to LB listener
listenerCreateOpt.Tags = append(listenerCreateOpt.Tags, lbaas.getCustomLoadBalancerTags(service, svcConf)...)
}

if openstackutil.IsOctaviaFeatureSupported(lbaas.lb, openstackutil.OctaviaFeatureTimeout, lbaas.opts.LBProvider) {
Expand Down Expand Up @@ -2317,6 +2350,12 @@ func (lbaas *LbaasV2) ensureAndUpdateOctaviaSecurityGroup(clusterName string, ap
return fmt.Errorf("failed to create Security Group for loadbalancer service %s/%s: %v", apiService.Namespace, apiService.Name, err)
}
lbSecGroupID = lbSecGroup.ID

tags := lbaas.getCustomLoadBalancerTags(apiService, svcConf)

if _, err := neutrontags.ReplaceAll(lbaas.network, "security-group", lbSecGroupID, neutrontags.ReplaceAllOpts{Tags: tags}).Extract(); err != nil {
return fmt.Errorf("failed to add custom tags %s to security group %s (%s)", tags, lbSecGroupID, lbSecGroupName)
}
}

mc := metrics.NewMetricContext("subnet", "get")
Expand Down
112 changes: 110 additions & 2 deletions pkg/openstack/loadbalancer_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -708,6 +708,7 @@ func TestLbaasV2_checkListenerPorts(t *testing.T) {
})
}
}

func TestLbaasV2_createLoadBalancerStatus(t *testing.T) {
type fields struct {
LoadBalancer LoadBalancer
Expand Down Expand Up @@ -1831,6 +1832,7 @@ func TestBuildBatchUpdateMemberOpts(t *testing.T) {
nodes []*corev1.Node
port corev1.ServicePort
svcConf *serviceConfig
service *corev1.Service
expectedLen int
expectedNewMembersCount int
}{
Expand Down Expand Up @@ -1909,7 +1911,7 @@ func TestBuildBatchUpdateMemberOpts(t *testing.T) {
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
lbaas := &LbaasV2{}
members, newMembers, err := lbaas.buildBatchUpdateMemberOpts(tc.port, tc.nodes, tc.svcConf)
members, newMembers, err := lbaas.buildBatchUpdateMemberOpts(tc.port, tc.nodes, tc.svcConf, tc.service)
assert.Len(t, members, tc.expectedLen)
assert.NoError(t, err)

Expand Down Expand Up @@ -2298,6 +2300,7 @@ func TestBuildListenerCreateOpt(t *testing.T) {
name string
port corev1.ServicePort
svcConf *serviceConfig
service *corev1.Service
expectedCreateOpt listeners.CreateOpts
}{
{
Expand Down Expand Up @@ -2400,9 +2403,114 @@ func TestBuildListenerCreateOpt(t *testing.T) {
},
},
}
createOpt := lbaas.buildListenerCreateOpt(tc.port, tc.svcConf, tc.name)
createOpt := lbaas.buildListenerCreateOpt(tc.port, tc.svcConf, tc.service, tc.name)
assert.Equal(t, tc.expectedCreateOpt, createOpt)
})
}
}

func TestLbaasV2_customLoadBalancerListenerTag(t *testing.T) {
type testArgs struct {
service *corev1.Service
svcConf *serviceConfig
}
tests := []struct {
name string
lbaas LbaasV2
testArgs testArgs
want []string
}{
{
name: "Single Custom Tag in Annotation With Disabled 'svcconfig.supportLBTags'",
testArgs: testArgs{
service: &corev1.Service{
ObjectMeta: v1.ObjectMeta{
Annotations: map[string]string{ServiceAnnotationLoadBalancerCustomTags: "single-custom-tag"},
},
},
svcConf: &serviceConfig{
supportLBTags: false,
},
},
want: nil,
},
{
name: "Empty Custom Tag in Annotation With Disabled 'svcconfig.supportLBTags'",
testArgs: testArgs{
service: &corev1.Service{
ObjectMeta: v1.ObjectMeta{
Annotations: map[string]string{ServiceAnnotationLoadBalancerCustomTags: ""},
},
},
svcConf: &serviceConfig{
supportLBTags: false,
},
},
want: nil,
},
{
name: "Multiple Custom Tag in Annotation With Disabled 'svcconfig.supportLBTags'",
testArgs: testArgs{
service: &corev1.Service{
ObjectMeta: v1.ObjectMeta{
Annotations: map[string]string{ServiceAnnotationLoadBalancerCustomTags: "tag1, tag2, tag3, tag4, multiple-custom-tag"},
},
},
svcConf: &serviceConfig{
supportLBTags: false,
},
},
want: nil,
},
{
name: "Empty Custom Tag in Annotation With Enabled 'svcconfig.supportLBTags'",
testArgs: testArgs{
service: &corev1.Service{
ObjectMeta: v1.ObjectMeta{
Annotations: map[string]string{ServiceAnnotationLoadBalancerCustomTags: ""},
},
},
svcConf: &serviceConfig{
supportLBTags: true,
},
},
want: []string{""},
},
{
name: "Valid Single Custom Tag in Annotation With Enabled 'svcconfig.supportLBTags'",
testArgs: testArgs{
service: &corev1.Service{
ObjectMeta: v1.ObjectMeta{
Annotations: map[string]string{ServiceAnnotationLoadBalancerCustomTags: "single-custom-tag"},
},
},
svcConf: &serviceConfig{
supportLBTags: true,
},
},
want: []string{"single-custom-tag"},
},
{
name: "Multiple Custom Tag in Annotation With Enabled 'svcconfig.supportLBTags'",
testArgs: testArgs{
service: &corev1.Service{
ObjectMeta: v1.ObjectMeta{
Annotations: map[string]string{ServiceAnnotationLoadBalancerCustomTags: "tag1, tag2, tag3, tag4, multiple-custom-tag"},
},
},
svcConf: &serviceConfig{
supportLBTags: true,
},
},
want: []string{"tag1", "tag2", "tag3", "tag4", "multiple-custom-tag"},
},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := tt.lbaas.getCustomLoadBalancerTags(tt.testArgs.service, tt.testArgs.svcConf)

assert.ElementsMatch(t, tt.want, got)
})
}
}

0 comments on commit 954ee65

Please sign in to comment.