From e7d0df511fc50679cdfdf16e583be4ecb9a0f553 Mon Sep 17 00:00:00 2001 From: Tao Zou Date: Wed, 17 Jan 2024 09:59:05 +0800 Subject: [PATCH] check license from nsxt side While init, it will check the license from nsxt side. If CONTAINER license is disable, it will reboot. If DFW license is disable, security policy will be only response for DELETE operation. It will check license periodically. Test Done: no CONTAINER license 1. if no CONTAINER license, nsx-operator should reset CONTAINER license enable, DFW disable 1. nsx-operator could bootup 2. security policy failed to create or update 3. security policy could be deleted CONTAINER license enable, DFW enable -> CONTAINER/DFW disable 1. nsx-operator restart due to DFW changed --- cmd/main.go | 17 ++++ pkg/metrics/metrics.go | 2 + pkg/nsx/client.go | 29 ++++++ pkg/nsx/cluster.go | 63 ++++++++++++ pkg/nsx/cluster_test.go | 71 +++++++++++++ pkg/nsx/services/securitypolicy/firewall.go | 5 + pkg/nsx/services/vpc/vpc.go | 4 + pkg/nsx/services/vpc/vpc_test.go | 2 + pkg/nsx/util/license.go | 65 ++++++++++++ pkg/nsx/util/license_test.go | 105 ++++++++++++++++++++ pkg/nsx/util/utils.go | 2 +- 11 files changed, 364 insertions(+), 1 deletion(-) create mode 100644 pkg/nsx/util/license.go create mode 100644 pkg/nsx/util/license_test.go diff --git a/cmd/main.go b/cmd/main.go index ae252f72a..cf0d7852b 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -169,6 +169,11 @@ func main() { NSXClient: nsxClient, NSXConfig: cf, } + err = commonService.NSXClient.ValidateLicense(true) + if err != nil { + os.Exit(1) + } + go updateLicensePeriodically(nsxClient, metrics.LicenseTimeout*time.Second) var vpcService *vpc.VPCService @@ -263,3 +268,15 @@ func updateHealthMetricsPeriodically(nsxClient *nsx.Client) { } } } + +func updateLicensePeriodically(nsxClient *nsx.Client, interval time.Duration) { + for { + select { + case <-time.After(interval): + } + err := nsxClient.ValidateLicense(false) + if err != nil { + os.Exit(1) + } + } +} diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 748a08fae..bb46d531f 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -22,6 +22,8 @@ const ( ControllerDeleteSuccessTotalKey = "controller_delete_success_total" ControllerDeleteFailTotalKey = "controller_delete_fail_total" ScrapeTimeout = 30 + // LicenseTimeout is the timeout for checking license status. + LicenseTimeout = 2 * 3600 ) var log = logger.Log diff --git a/pkg/nsx/client.go b/pkg/nsx/client.go index bba1f19ad..146f56a83 100644 --- a/pkg/nsx/client.go +++ b/pkg/nsx/client.go @@ -31,6 +31,7 @@ import ( "github.com/vmware-tanzu/nsx-operator/pkg/config" "github.com/vmware-tanzu/nsx-operator/pkg/nsx/ratelimiter" + "github.com/vmware-tanzu/nsx-operator/pkg/nsx/util" ) const ( @@ -264,3 +265,31 @@ func (client *Client) NSXCheckVersion(feature int) bool { func (client *Client) FeatureEnabled(feature int) bool { return client.NSXVerChecker.featureSupported[feature] == true } + +// ValidateLicense validates NSX license. init is used to indicate whether nsx-operator is init or not +// if not init, nsx-operator will check if license has been updated. +// once license updated, operator will restart +// if FeatureContainer license is false, operatore will restart +func (client *Client) ValidateLicense(init bool) error { + log.Info("Checking NSX license") + oldContainerLicense := util.IsLicensed(util.FeatureContainer) + oldDfwLicense := util.IsLicensed(util.FeatureDFW) + err := client.NSXChecker.cluster.FetchLicense() + if err != nil { + return err + } + if !util.IsLicensed(util.FeatureContainer) { + err = errors.New("NSX license check failed") + log.Error(err, "container license is not supported") + return err + } + if !init { + newContainerLicense := util.IsLicensed(util.FeatureContainer) + newDfwLicense := util.IsLicensed(util.FeatureDFW) + if newContainerLicense != oldContainerLicense || newDfwLicense != oldDfwLicense { + log.Info("license updated, reset", "container license new value", newContainerLicense, "DFW license new value", newDfwLicense, "container license old value", oldContainerLicense, "DFW license old value", oldDfwLicense) + return errors.New("license updated") + } + } + return nil +} diff --git a/pkg/nsx/cluster.go b/pkg/nsx/cluster.go index 8baeff552..23e6c4a06 100644 --- a/pkg/nsx/cluster.go +++ b/pkg/nsx/cluster.go @@ -5,9 +5,11 @@ package nsx import ( "context" + "crypto/rand" "crypto/tls" "errors" "fmt" + "math/big" "net" "net/http" "os" @@ -39,6 +41,12 @@ const ( const ( EnvoyUrlWithCert = "http://%s:%d/external-cert/http1/%s" EnvoyUrlWithThumbprint = "http://%s:%d/external-tp/http1/%s/%s" + LicenseAPI = "api/v1/licenses/licensed-features" +) + +const ( + maxNSXGetRetries = 10 + NSXGetDelay = 2 * time.Second ) // Cluster consists of endpoint and provides http.Client used to send http requests. @@ -418,3 +426,58 @@ func (nsxVersion *NsxVersion) featureSupported(feature int) bool { } return false } + +func (cluster *Cluster) createHttpRequest(api string, ep *Endpoint) (*http.Request, error) { + return http.NewRequest("GET", fmt.Sprintf("%s://%s/%s", ep.Scheme(), ep.Host(), api), nil) +} + +func (cluster *Cluster) getLicenseFromNsx() (*util.NsxLicense, error) { + ep := cluster.endpoints[0] + req, err := cluster.createHttpRequest(LicenseAPI, ep) + if err != nil { + log.Error(err, "failed to create http request") + return nil, err + } + err = ep.UpdateHttpRequestAuth(req) + if err != nil { + log.Error(err, "keep alive update auth error") + return nil, err + } + + resp, err := ep.noBalancerClient.Do(req) + if err != nil { + log.Error(err, "failed to get nsx license") + return nil, err + } + nsxLicense := &util.NsxLicense{} + err, _ = util.HandleHTTPResponse(resp, nsxLicense, true) + return nsxLicense, err +} + +func (cluster *Cluster) getLicenseWithRetries(delay time.Duration, maxRetry int) (*util.NsxLicense, error) { + var err error + for i := 0; i < maxRetry; i++ { + nsxLicense, err := cluster.getLicenseFromNsx() + if err != nil { + log.Error(err, "failed to get nsx license") + rand, err := rand.Int(rand.Reader, big.NewInt(1000)) + if err != nil { + log.Error(err, "failed to generate random number") + return nil, err + } + time.Sleep(delay + time.Duration(rand.Int64())*time.Millisecond) + } else { + return nsxLicense, nil + } + } + return nil, err +} + +func (cluster *Cluster) FetchLicense() error { + nsxLicense, err := cluster.getLicenseWithRetries(NSXGetDelay, maxNSXGetRetries) + if err != nil { + return err + } + util.UpdateFeatureLicense(nsxLicense) + return nil +} diff --git a/pkg/nsx/cluster_test.go b/pkg/nsx/cluster_test.go index de191a3eb..ac7785575 100644 --- a/pkg/nsx/cluster_test.go +++ b/pkg/nsx/cluster_test.go @@ -4,8 +4,11 @@ package nsx import ( + "bytes" + "encoding/json" "errors" "fmt" + "io" "net/http" "net/http/httptest" "reflect" @@ -17,6 +20,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/vmware-tanzu/nsx-operator/pkg/nsx/ratelimiter" + nsxutil "github.com/vmware-tanzu/nsx-operator/pkg/nsx/util" ) func TestNewCluster(t *testing.T) { @@ -344,3 +348,70 @@ func TestCluster_CreateServerUrl(t *testing.T) { }) } } + +func TestGetLicenseFromNsx(t *testing.T) { + address := address{ + host: "1.2.3.4", + scheme: "https", + } + // Success case + cluster := &Cluster{endpoints: []*Endpoint{{ + provider: &address, + }}} + + // Request creation failure + patch := gomonkey.ApplyFunc(http.NewRequest, + func(method, url string, body io.Reader) (*http.Request, error) { + return nil, errors.New("request error") + }) + license, err := cluster.getLicenseFromNsx() + assert.Error(t, err) + assert.Nil(t, license) + patch.Reset() + + // HTTP error + patch = gomonkey.ApplyFunc((*http.Client).Do, + func(client *http.Client, req *http.Request) (*http.Response, error) { + return nil, errors.New("http error") + }) + + license, err = cluster.getLicenseFromNsx() + assert.Error(t, err) + assert.Nil(t, license) + patch.Reset() + + // normal case + patch = gomonkey.ApplyFunc((*http.Client).Do, + func(client *http.Client, req *http.Request) (*http.Response, error) { + res := &nsxutil.NsxLicense{ + Results: []struct { + FeatureName string `json:"feature_name"` + IsLicensed bool `json:"is_licensed"` + }{{ + FeatureName: "CONTAINER", + IsLicensed: true, + }, + { + FeatureName: "DFW", + IsLicensed: true, + }, + }, + ResultCount: 2, + } + + jsonBytes, _ := json.Marshal(res) + + return &http.Response{ + StatusCode: 200, + Body: io.NopCloser(bytes.NewReader(jsonBytes)), + Header: http.Header{ + "Content-Type": []string{"application/json"}, + }, + Request: req, + }, nil + }) + defer patch.Reset() + _, err = cluster.getLicenseFromNsx() + assert.Nil(t, err) + +} diff --git a/pkg/nsx/services/securitypolicy/firewall.go b/pkg/nsx/services/securitypolicy/firewall.go index 3630dc9b7..c34241d06 100644 --- a/pkg/nsx/services/securitypolicy/firewall.go +++ b/pkg/nsx/services/securitypolicy/firewall.go @@ -16,6 +16,7 @@ import ( "github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha1" "github.com/vmware-tanzu/nsx-operator/pkg/logger" "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" + nsxutil "github.com/vmware-tanzu/nsx-operator/pkg/nsx/util" "github.com/vmware-tanzu/nsx-operator/pkg/util" ) @@ -127,6 +128,10 @@ func InitializeSecurityPolicy(service common.Service, vpcService common.VPCServi } func (service *SecurityPolicyService) CreateOrUpdateSecurityPolicy(obj interface{}) error { + if !nsxutil.IsLicensed(nsxutil.FeatureDFW) { + log.Info("no DFW license, skip creating SecurityPolicy.") + return nsxutil.RestrictionError{Desc: "no DFW license"} + } var err error switch obj.(type) { case *networkingv1.NetworkPolicy: diff --git a/pkg/nsx/services/vpc/vpc.go b/pkg/nsx/services/vpc/vpc.go index c640d31b4..6f4389ca5 100644 --- a/pkg/nsx/services/vpc/vpc.go +++ b/pkg/nsx/services/vpc/vpc.go @@ -22,6 +22,7 @@ import ( "github.com/vmware-tanzu/nsx-operator/pkg/nsx" "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/realizestate" + nsxutil "github.com/vmware-tanzu/nsx-operator/pkg/nsx/util" "github.com/vmware-tanzu/nsx-operator/pkg/util" ) @@ -614,6 +615,9 @@ func (service *VPCService) CreateOrUpdateAVIRule(vpc *model.Vpc, namespace strin if !enableAviAllowRule { return nil } + if !nsxutil.IsLicensed(nsxutil.FeatureDFW) { + return nil + } vpcInfo, err := common.ParseVPCResourcePath(*vpc.Path) if err != nil { log.Error(err, "failed to parse VPC Resource Path: ", *vpc.Path) diff --git a/pkg/nsx/services/vpc/vpc_test.go b/pkg/nsx/services/vpc/vpc_test.go index adac76d49..0d77d82f1 100644 --- a/pkg/nsx/services/vpc/vpc_test.go +++ b/pkg/nsx/services/vpc/vpc_test.go @@ -21,6 +21,7 @@ import ( "github.com/vmware-tanzu/nsx-operator/pkg/nsx" "github.com/vmware-tanzu/nsx-operator/pkg/nsx/ratelimiter" "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" + "github.com/vmware-tanzu/nsx-operator/pkg/nsx/util" ) var ( @@ -471,6 +472,7 @@ func TestCreateOrUpdateAVIRule(t *testing.T) { sp := model.SecurityPolicy{ Path: &sppath1, } + util.UpdateLicense(util.FeatureDFW, true) // security policy not found spClient.SP = sp diff --git a/pkg/nsx/util/license.go b/pkg/nsx/util/license.go new file mode 100644 index 000000000..573ad4909 --- /dev/null +++ b/pkg/nsx/util/license.go @@ -0,0 +1,65 @@ +package util + +import ( + "sync" +) + +const ( + FeatureContainer = "CONTAINER" + FeatureDFW = "DFW" + LicenseContainerNetwork = "CONTAINER_NETWORK" + LicenseContainerSecurity = "DFW" + LicenseContainer = "CONTAINER" +) + +var ( + container_network = false + container_security = false + licenseMutex sync.Mutex + licenseMap = map[string]bool{FeatureContainer: container_network, FeatureDFW: container_security} + Feature_license_map = map[string][]string{FeatureContainer: {LicenseContainerNetwork, + LicenseContainerSecurity}, + FeatureDFW: {LicenseContainerSecurity}} + Features_to_check = []string{FeatureContainer, FeatureDFW} +) + +type NsxLicense struct { + Results []struct { + FeatureName string `json:"feature_name"` + IsLicensed bool `json:"is_licensed"` + } `json:"results"` + ResultCount int `json:"result_count"` +} + +func IsLicensed(feature string) bool { + licenseMutex.Lock() + defer licenseMutex.Unlock() + return licenseMap[feature] +} + +func UpdateLicense(feature string, isLicensed bool) { + licenseMutex.Lock() + licenseMap[feature] = isLicensed + licenseMutex.Unlock() +} + +func searchLicense(licenses *NsxLicense, licenseNames []string) bool { + license := false + for _, feature := range licenses.Results { + for _, licenseName := range licenseNames { + if feature.FeatureName == licenseName { + license = license || feature.IsLicensed + } + } + } + return license +} + +func UpdateFeatureLicense(licenses *NsxLicense) { + for _, feature := range Features_to_check { + licenseNames := Feature_license_map[feature] + license := searchLicense(licenses, licenseNames) + UpdateLicense(feature, license) + log.V(1).Info("update license", "feature", feature, "license", license) + } +} diff --git a/pkg/nsx/util/license_test.go b/pkg/nsx/util/license_test.go new file mode 100644 index 000000000..c932de3f3 --- /dev/null +++ b/pkg/nsx/util/license_test.go @@ -0,0 +1,105 @@ +package util + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestIsLicensed(t *testing.T) { + licenseMap[FeatureContainer] = true + assert.True(t, IsLicensed(FeatureContainer)) + + licenseMap[FeatureDFW] = false + assert.False(t, IsLicensed(FeatureDFW)) +} + +func TestUpdateLicense(t *testing.T) { + UpdateLicense(FeatureDFW, true) + assert.True(t, licenseMap[FeatureDFW]) + + UpdateLicense(FeatureDFW, false) + assert.False(t, licenseMap[FeatureDFW]) +} + +func TestSearchLicense(t *testing.T) { + licenses := &NsxLicense{ + Results: []struct { + FeatureName string `json:"feature_name"` + IsLicensed bool `json:"is_licensed"` + }{ + { + FeatureName: LicenseContainer, + IsLicensed: true, + }, + { + FeatureName: LicenseContainerSecurity, + IsLicensed: false, + }, + }, + } + + // Search for license that exists + assert.True(t, searchLicense(licenses, []string{FeatureContainer})) + + // Search for license that does not exist + assert.False(t, searchLicense(licenses, []string{"IDFW"})) + + // Search with empty results + licenses.Results = []struct { + FeatureName string `json:"feature_name"` + IsLicensed bool `json:"is_licensed"` + }{} + assert.False(t, searchLicense(licenses, []string{FeatureContainer})) + + licenses = &NsxLicense{ + Results: []struct { + FeatureName string `json:"feature_name"` + IsLicensed bool `json:"is_licensed"` + }{ + { + FeatureName: LicenseContainerNetwork, + IsLicensed: true, + }, + { + FeatureName: LicenseContainerSecurity, + IsLicensed: false, + }, + { + FeatureName: LicenseContainer, + IsLicensed: true, + }, + }, + } + assert.True(t, searchLicense(licenses, []string{FeatureContainer})) +} + +func TestUpdateFeatureLicense(t *testing.T) { + + // Normal case + licenses := &NsxLicense{ + Results: []struct { + FeatureName string `json:"feature_name"` + IsLicensed bool `json:"is_licensed"` + }{ + {FeatureName: LicenseContainerSecurity, IsLicensed: true}, + {FeatureName: LicenseContainer, IsLicensed: true}, + }, + } + + UpdateFeatureLicense(licenses) + assert.True(t, IsLicensed(FeatureDFW)) + assert.True(t, IsLicensed(FeatureContainer)) + + // Old version format + UpdateFeatureLicense(licenses) + assert.True(t, IsLicensed(FeatureDFW)) + assert.True(t, IsLicensed(FeatureContainer)) + + // Empty license list + licenses.Results = nil + UpdateFeatureLicense(licenses) + assert.False(t, IsLicensed(FeatureDFW)) + assert.False(t, IsLicensed(FeatureContainer)) + +} diff --git a/pkg/nsx/util/utils.go b/pkg/nsx/util/utils.go index 42d0feae3..a67e1ac9b 100644 --- a/pkg/nsx/util/utils.go +++ b/pkg/nsx/util/utils.go @@ -247,7 +247,7 @@ func HandleHTTPResponse(response *http.Response, result interface{}, debug bool) defer response.Body.Close() if !(response.StatusCode == http.StatusOK || response.StatusCode == http.StatusAccepted) { err := errors.New("received HTTP Error") - log.Error(err, "handle http response", "status", response.StatusCode, "requestUrl", response.Request.URL, "response body", string(body)) + log.Error(err, "handle http response", "status", response.StatusCode, "request URL", response.Request.URL, "response body", string(body)) return err, nil } if err != nil || body == nil {