From c24d0601e996647b3811d735c04115f8e16ab755 Mon Sep 17 00:00:00 2001 From: Patrick Kremer Date: Mon, 9 Jan 2023 18:19:17 -0600 Subject: [PATCH 01/26] feat: Split authentication into separate class file --- VMCImportExport.py | 237 +++++++++++++++++++++--------------------- sddc_import_export.py | 36 +++---- vmc_auth.py | 45 ++++++++ 3 files changed, 181 insertions(+), 137 deletions(-) create mode 100644 vmc_auth.py diff --git a/VMCImportExport.py b/VMCImportExport.py index 8873f58..a85ec66 100755 --- a/VMCImportExport.py +++ b/VMCImportExport.py @@ -1,12 +1,10 @@ # SDDC Import/Export for VMware Cloud on AWS - ################################################################################ -### Copyright 2020-2021 VMware, Inc. +### Copyright 2020-2023 VMware, Inc. ### SPDX-License-Identifier: BSD-2-Clause ################################################################################ - import configparser # parsing config file import datetime import glob @@ -22,13 +20,13 @@ from prettytable import PrettyTable from zipfile import ZipFile +import vmc_auth + class VMCImportExport: """A class to handle importing and exporting portions of a VMC SDDC""" def __init__(self,configPath="./config_ini/config.ini", vmcConfigPath="./config_ini/vmc.ini", awsConfigPath="./config/aws.ini", vCenterConfigPath="./config_ini/vcenter.ini"): - self.access_token = None - self.access_token_expiration = None - self.activeRefreshToken = None + self.vmc_auth = None self.proxy_url = None self.proxy_url_short = None self.lastJSONResponse = None @@ -98,6 +96,7 @@ def ConfigLoader(self): else: self.strProdURL = vmcConfig.get("vmcConfig", "strProdURL") self.strCSPProdURL = vmcConfig.get("vmcConfig", "strCSPProdURL") + self.vmc_auth = vmc_auth.VMCAuth(strCSPProdURL=self.strCSPProdURL) self.source_refresh_token = vmcConfig.get("vmcConfig", "source_refresh_token") self.source_org_id = vmcConfig.get("vmcConfig", "source_org_id") self.source_sddc_id = vmcConfig.get("vmcConfig", "source_sddc_id") @@ -406,7 +405,7 @@ def exportOnPremDFWRule(self): return True def importOnPremServices(self): - self.check_access_token_expiration() + self.vmc_auth.check_access_token_expiration() """Import all services from a JSON file""" fname = self.import_path / self.services_filename try: @@ -437,7 +436,7 @@ def importOnPremServices(self): modified_entry[k] = entry[k] service_entries.append(modified_entry) json_data["service_entries"]=service_entries - myHeader = {"Content-Type": "application/json","Accept": "application/json", 'csp-auth-token': self.access_token } + myHeader = {"Content-Type": "application/json","Accept": "application/json", 'csp-auth-token': self.vmc_auth.access_token } myURL = self.proxy_url + "/policy/api/v1/infra/services/" + service["id"] #print(myURL) #print(json_data) @@ -457,7 +456,7 @@ def importOnPremServices(self): def importOnPremGroup(self): """Import all CGW groups from a JSON file""" - self.check_access_token_expiration() + self.vmc_auth.check_access_token_expiration() fname = self.import_path / self.cgw_groups_filename try: with open(fname) as filehandle: @@ -482,7 +481,7 @@ def importOnPremGroup(self): payload["resource_type"]=group["resource_type"] payload["display_name"]=group["display_name"] if self.import_mode == "live": - myHeader = {"Content-Type": "application/json","Accept": "application/json", 'csp-auth-token': self.access_token } + myHeader = {"Content-Type": "application/json","Accept": "application/json", 'csp-auth-token': self.vmc_auth.access_token } myURL = self.proxy_url + "/policy/api/v1/infra/domains/cgw/groups/" + group["id"] if "expression" in group: group_expression = group["expression"] @@ -511,7 +510,7 @@ def importOnPremGroup(self): def importOnPremDFWRule(self): """Import all DFW Rules from a JSON file""" - self.check_access_token_expiration() + self.vmc_auth.check_access_token_expiration() fname = self.import_path / self.dfw_import_filename fname_detailed = self.import_path / self.dfw_detailed_import_filename try: @@ -537,7 +536,7 @@ def importOnPremDFWRule(self): payload["sequence_number"] = cmap["sequence_number"] payload["stateful"] = cmap["stateful"] if self.import_mode == 'live': - myHeader = {"Content-Type": "application/json","Accept": "application/json", 'csp-auth-token': self.access_token } + myHeader = {"Content-Type": "application/json","Accept": "application/json", 'csp-auth-token': self.vmc_auth.access_token } myURL = self.proxy_url_short + "/policy/api/v1/infra/domains/cgw/security-policies/" + cmap["id"] json_data = json.dumps(payload) if self.sync_mode is True: @@ -568,7 +567,7 @@ def importOnPremDFWRule(self): if self.import_mode == 'live': myURL = self.proxy_url + "/policy/api/v1/infra/domains/cgw/security-policies/" + cmap["id"] + "/rules/" + commEnt["id"] - myHeader = {"Content-Type": "application/json","Accept": "application/json", 'csp-auth-token': self.access_token } + myHeader = {"Content-Type": "application/json","Accept": "application/json", 'csp-auth-token': self.vmc_auth.access_token } json_data = json.dumps(payload) if self.sync_mode is True: response = requests.patch(myURL,headers=myHeader,data=json_data) @@ -816,7 +815,7 @@ def exportSDDCDFWRule(self): def importSDDCDFWRule(self): """Import all DFW Rules from a JSON file""" - self.check_access_token_expiration() + self.vmc_auth.check_access_token_expiration() fname = self.import_path / self.dfw_import_filename fname_detailed = self.import_path / self.dfw_detailed_import_filename try: @@ -841,7 +840,7 @@ def importSDDCDFWRule(self): payload["sequence_number"] = cmap["sequence_number"] payload["stateful"] = cmap["stateful"] if self.import_mode == 'live': - myHeader = {"Content-Type": "application/json","Accept": "application/json", 'csp-auth-token': self.access_token } + myHeader = {"Content-Type": "application/json","Accept": "application/json", 'csp-auth-token': self.vmc_auth.access_token } myURL = self.proxy_url_short + "/policy/api/v1/infra/domains/cgw/security-policies/" + cmap["id"] json_data = json.dumps(payload) if self.sync_mode is True: @@ -868,7 +867,7 @@ def importSDDCDFWRule(self): payload["disabled"] = commEnt["disabled"] if self.import_mode == 'live': myURL = self.proxy_url + "/policy/api/v1/infra/domains/cgw/security-policies/" + cmap["id"] + "/rules/" + commEnt["id"] - myHeader = {"Content-Type": "application/json","Accept": "application/json", 'csp-auth-token': self.access_token } + myHeader = {"Content-Type": "application/json","Accept": "application/json", 'csp-auth-token': self.vmc_auth.access_token } json_data = json.dumps(payload) if self.sync_mode is True: response = requests.patch(myURL,headers=myHeader,data=json_data) @@ -1112,7 +1111,7 @@ def exportVPNBGPNeighbors(self): def getVPNl3sensitivedata(self,l3vpnid): """ Retrieve sensitive data such as IPSEC preshared keys from an L3VPN configuration""" - myHeader = {'csp-auth-token': self.access_token} + myHeader = {'csp-auth-token': self.vmc_auth.access_token} myURL = (self.proxy_url_short + f'/policy/api/v1/infra/tier-0s/vmc/locale-services/default/ipsec-vpn-services/default/sessions/{l3vpnid}?action=show_sensitive_data') try: response = requests.get(myURL, headers=myHeader) @@ -1161,7 +1160,7 @@ def exportVPNl3config(self): def importCGWNetworks(self): """Imports CGW network semgements from a JSON file""" - self.check_access_token_expiration() + self.vmc_auth.check_access_token_expiration() fname = self.import_path / self.network_import_filename try: with open(fname) as filehandle: @@ -1196,7 +1195,7 @@ def importCGWNetworks(self): if "advanced_config" in n: json_data["advanced_config"] = n["advanced_config"] if self.import_mode == "live": - myHeader = {"Content-Type": "application/json","Accept": "application/json", 'csp-auth-token': self.access_token } + myHeader = {"Content-Type": "application/json","Accept": "application/json", 'csp-auth-token': self.vmc_auth.access_token } myURL = (self.proxy_url + "/policy/api/v1/infra/tier-1s/cgw/segments/" + n['id']) if self.sync_mode is True: response = requests.patch(myURL, headers=myHeader, json=json_data) @@ -1226,7 +1225,7 @@ def importCGWNetworks(self): def import_flex_segments(self): """Imports flexible segments from a JSON file""" - self.check_access_token_expiration() + self.vmc_auth.check_access_token_expiration() fname = self.import_path / self.flex_segment_import_filename try: with open (fname) as filehandle: @@ -1258,7 +1257,7 @@ def import_flex_segments(self): json_data['advanced_config'] = f['advanced_config'] uri_path = f['path'] if self.import_mode == 'live': - my_header = {"Content-Type": "application/json","Accept": "application/json", 'csp-auth-token': self.access_token} + my_header = {"Content-Type": "application/json","Accept": "application/json", 'csp-auth-token': self.vmc_auth.access_token} my_url = f'{self.proxy_url}/policy/api/v1{uri_path}' if self.sync_mode is True: response = requests.patch(my_url, headers = my_header, json = json_data) @@ -1282,7 +1281,7 @@ def import_flex_segments(self): return table def importCGWDHCPStaticBindings(self): - self.check_access_token_expiration() + self.vmc_auth.check_access_token_expiration() fname = self.import_path / self.network_dhcp_static_binding_filename try: with open(fname) as filehandle: @@ -1299,7 +1298,7 @@ def importCGWDHCPStaticBindings(self): payload[x] = binding[x] if self.import_mode == 'live': - myHeader = {"Content-Type": "application/json","Accept": "application/json", 'csp-auth-token': self.access_token } + myHeader = {"Content-Type": "application/json","Accept": "application/json", 'csp-auth-token': self.vmc_auth.access_token } myURL = self.proxy_url + "/policy/api/v1" + binding['path'] if self.sync_mode is True: response = requests.patch(myURL, headers=myHeader, json=payload) @@ -1317,7 +1316,7 @@ def importCGWDHCPStaticBindings(self): def importSDDCServices(self): """Import all services from a JSON file""" - self.check_access_token_expiration() + self.vmc_auth.check_access_token_expiration() fname = self.import_path / self.services_filename try: with open(fname) as filehandle: @@ -1346,7 +1345,7 @@ def importSDDCServices(self): modified_entry[k] = entry[k] service_entries.append(modified_entry) json_data["service_entries"]=service_entries - myHeader = {"Content-Type": "application/json","Accept": "application/json", 'csp-auth-token': self.access_token } + myHeader = {"Content-Type": "application/json","Accept": "application/json", 'csp-auth-token': self.vmc_auth.access_token } myURL = self.proxy_url + "/policy/api/v1/infra/services/" + service["id"] #print(myURL) #print(json_data) @@ -1365,7 +1364,7 @@ def importSDDCServices(self): def import_mcgw(self): """Import Tier-1 gateways from a JSON file""" - self.check_access_token_expiration() + self.vmc_auth.check_access_token_expiration() fname = self.import_path / self.mcgw_import_filename try: with open(fname) as filehandle: @@ -1381,7 +1380,7 @@ def import_mcgw(self): json_data['type'] = mcgw['type'] if 'dhcp_config_paths' in mcgw: json_data['dhcp_config_paths'] = mcgw['dhcp_config_paths'] - my_header = {"Content-Type": "application/json", "Accept": "application/json", "csp-auth-token": self.access_token} + my_header = {"Content-Type": "application/json", "Accept": "application/json", "csp-auth-token": self.vmc_auth.access_token} my_url = self.proxy_url + '/policy/api/v1/infra/tier-1s/' + mcgw['id'] if self.sync_mode is True: response = requests.patch(my_url, headers=my_header, json=json_data) @@ -1396,7 +1395,7 @@ def import_mcgw(self): def import_mcgw_static_routes(self): """Import Tier-1 Gateway static routes from a JSON file""" - self.check_access_token_expiration() + self.vmc_auth.check_access_token_expiration() fname = self.import_path / self.mcgw_static_route_import_filename try: with open(fname) as filehandle: @@ -1415,7 +1414,7 @@ def import_mcgw_static_routes(self): json_data['resource_type'] = r['resource_type'] path = r['path'] my_header = {"Content-Type": "application/json", "Accept": "application/json", - "csp-auth-token": self.access_token} + "csp-auth-token": self.vmc_auth.access_token} my_url = f'{self.proxy_url}/policy/api/v1{path}' if self.sync_mode is True: response = requests.patch(my_url, headers=my_header, json=json_data) @@ -1432,7 +1431,7 @@ def import_mcgw_static_routes(self): def import_mcgw_fw(self): """Import Tier-1 Gateway firewall policies and rules from a JSON file""" - self.check_access_token_expiration() + self.vmc_auth.check_access_token_expiration() fname = self.import_path / self.mcgw_fw_import_filename try: with open(fname) as filehandle: @@ -1450,7 +1449,7 @@ def import_mcgw_fw(self): json_policy_data['display_name'] = policy['display_name'] json_policy_data['category'] = policy['category'] path = policy['path'] - my_header = {"Content-Type": "application/json", "Accept": "application/json", "csp-auth-token": self.access_token} + my_header = {"Content-Type": "application/json", "Accept": "application/json", "csp-auth-token": self.vmc_auth.access_token} my_url = f'{self.proxy_url}/policy/api/v1{path}' if self.sync_mode is True: response = requests.patch(my_url, headers=my_header, json=json_policy_data) @@ -1480,7 +1479,7 @@ def import_mcgw_fw(self): json_rule_data['tag'] = r['tag'] path = r['path'] my_header = {"Content-Type": "application/json", "Accept": "application/json", - "csp-auth-token": self.access_token} + "csp-auth-token": self.vmc_auth.access_token} my_url = f'{self.proxy_url}/policy/api/v1{path}' if self.sync_mode is True: response = requests.patch(my_url, headers=my_header, json=json_rule_data) @@ -1497,7 +1496,7 @@ def import_mcgw_fw(self): def import_ral(self): """Import SDDC Route Aggregation lists from JSON""" - self.check_access_token_expiration() + self.vmc_auth.check_access_token_expiration() fname = self.import_path / self.ral_import_filename try: with open(fname) as filehandle: @@ -1514,7 +1513,7 @@ def import_ral(self): json_data['id'] = r['id'] path = r['path'] # print(json.dumps(json_data, indent=2)) - my_header = {"Content-Type": "application/json", "Accept": "application/json", "csp-auth-token": self.access_token} + my_header = {"Content-Type": "application/json", "Accept": "application/json", "csp-auth-token": self.vmc_auth.access_token} my_url = f'{self.proxy_url}/cloud-service/api/v1{path}' response = requests.put(my_url, headers=my_header, json=json_data) if response.status_code == 200: @@ -1528,7 +1527,7 @@ def import_ral(self): def import_route_config(self): """Imports SDDC route configuration from JSON""" - self.check_access_token_expiration() + self.vmc_auth.check_access_token_expiration() fname = self.import_path / self.route_config_import_filename try: with open(fname) as filehandle: @@ -1545,7 +1544,7 @@ def import_route_config(self): json_data['aggregation_route_config'] = r['aggregation_route_config'] json_data['connectivity_endpoint_path'] = r['connectivity_endpoint_path'] my_header = {"Content-Type": "application/json", "Accept": "application/json", - "csp-auth-token": self.access_token} + "csp-auth-token": self.vmc_auth.access_token} my_url = f'{self.proxy_url}/cloud-service/api/v1/infra/external/route/configs/{r["id"]}' response = requests.put(my_url, headers=my_header, json=json_data) if response.status_code == 200: @@ -1605,9 +1604,9 @@ def syncRolesToDestinationUsers(self): print('Could not find user with email ' + email) def invokeCSPGET(self,url: str) -> requests.Response: - self.check_access_token_expiration() + self.vmc_auth.check_access_token_expiration() try: - response = requests.get(url,headers= {"Authorization":"Bearer " + self.access_token}) + response = requests.get(url,headers= {"Authorization":"Bearer " + self.vmc_auth.access_token}) if response.status_code != 200: self.lastJSONResponse = f'API Call Status {response.status_code}, text:{response.text}' return response @@ -1617,8 +1616,8 @@ def invokeCSPGET(self,url: str) -> requests.Response: def invokeVMCGET(self,url: str) -> requests.Response: """Invokes a VMC On AWS GET request""" - self.check_access_token_expiration() - myHeader = {'csp-auth-token': self.access_token} + self.vmc_auth.check_access_token_expiration() + myHeader = {'csp-auth-token': self.vmc_auth.access_token} attempts = 1 status_code = 0 try: @@ -1641,8 +1640,8 @@ def invokeVMCGET(self,url: str) -> requests.Response: def invokeVMCPUT(self, url: str,json_data: str) -> requests.Response: """Invokes a VMC on AWS PUT request""" - self.check_access_token_expiration() - myHeader = {"Content-Type": "application/json","Accept": "application/json", 'csp-auth-token': self.access_token } + self.vmc_auth.check_access_token_expiration() + myHeader = {"Content-Type": "application/json","Accept": "application/json", 'csp-auth-token': self.vmc_auth.access_token } try: response = requests.put(url,headers=myHeader,data=json_data) if response.status_code != 200: @@ -1654,8 +1653,8 @@ def invokeVMCPUT(self, url: str,json_data: str) -> requests.Response: def invokeVMCPATCH(self, url: str,json_data: str) -> requests.Response: """Invokes a VMC on AWS PATCH request""" - self.check_access_token_expiration() - myHeader = {"Content-Type": "application/json","Accept": "application/json", 'csp-auth-token': self.access_token } + self.vmc_auth.check_access_token_expiration() + myHeader = {"Content-Type": "application/json","Accept": "application/json", 'csp-auth-token': self.vmc_auth.access_token } try: response = requests.patch(url,headers=myHeader,data=json_data) if response.status_code != 200: @@ -1677,9 +1676,9 @@ def invokeNSXTGET(self,url: str) -> requests.Response: return None def findRandomTestbedVM(self) -> str: - self.check_access_token_expiration() + self.vmc_auth.check_access_token_expiration() """Looks for any of the first 100 VMs available in NSX-T - used to generate realistic group members for a testbed""" - myHeader = {"Content-Type": "application/json","Accept": "application/json", 'csp-auth-token': self.access_token } + myHeader = {"Content-Type": "application/json","Accept": "application/json", 'csp-auth-token': self.vmc_auth.access_token } myURL = self.proxy_url + '/policy/api/v1/search/aggregate?page_size=100' json_data = {"primary":{"resource_type":"VirtualMachine","filters":[{"field_names":"!tags.tag","value":"nsx_policy_internal"},{"field_names":"!display_name","value":"(\"NSX-Edge-0\" OR \"NSX-Edge-1\" OR \"NSX-Manager-0\" OR \"NSX-Manager-1\" OR \"NSX-Manager-2\" OR \"vcenter\")"}]},"related":[{"resource_type":"TransportNode OR HostNode","join_condition":"id:source.target_id","alias":"TransportNode"},{"resource_type":"VirtualNetworkInterface","join_condition":"owner_vm_id:external_id","alias":"VirtualNetworkInterface"},{"resource_type":"HostNode","join_condition":"id:host_id","alias":"HostNode","size":0},{"resource_type":"DiscoveredNode","join_condition":"external_id:$2.discovered_node_id","alias":"DiscoveredNode","size":0},{"resource_type":"ComputeManager","join_condition":"id:$3.origin_id","alias":"ComputeManager"}],"data_source":"ALL"} response = requests.post(myURL, headers=myHeader, data=json.dumps(json_data)) @@ -1696,8 +1695,8 @@ def findRandomTestbedVM(self) -> str: def createSDDCCGWGroup(self, group_name: str, vm_name_to_add: str = None): """Creates a new CGW Group""" - self.check_access_token_expiration() - myHeader = {"Content-Type": "application/json","Accept": "application/json", 'csp-auth-token': self.access_token } + self.vmc_auth.check_access_token_expiration() + myHeader = {"Content-Type": "application/json","Accept": "application/json", 'csp-auth-token': self.vmc_auth.access_token } myURL = self.proxy_url + "/policy/api/v1/infra/domains/cgw/groups/" + group_name if vm_name_to_add is None: @@ -1740,7 +1739,7 @@ def createSDDCCGWGroup(self, group_name: str, vm_name_to_add: str = None): def deleteAllSDDCCGWGroups(self): """ Just what it sounds like - delete every single CGW group. Use with caution""" - self.check_access_token_expiration() + self.vmc_auth.check_access_token_expiration() myURL = self.proxy_url + "/policy/api/v1/infra/domains/cgw/groups" response = self.invokeVMCGET(myURL) if response is None or response.status_code != 200: @@ -1762,8 +1761,8 @@ def deleteAllSDDCCGWGroups(self): def deleteSDDCCGWGroup(self, group_name: str): """Deletes a CGW Group""" - self.check_access_token_expiration() - myHeader = {"Content-Type": "application/json","Accept": "application/json", 'csp-auth-token': self.access_token } + self.vmc_auth.check_access_token_expiration() + myHeader = {"Content-Type": "application/json","Accept": "application/json", 'csp-auth-token': self.vmc_auth.access_token } myURL = self.proxy_url + "/policy/api/v1/infra/domains/cgw/groups/" + group_name #json_data = {"display_name":group_name, "id":group_name } @@ -1780,7 +1779,7 @@ def deleteSDDCCGWGroup(self, group_name: str): def exportSDDCCGWGroups(self): """Exports the CGW groups to a JSON file""" - self.check_access_token_expiration() + self.vmc_auth.check_access_token_expiration() debug_mode = False debug_page_size = 20 @@ -1820,9 +1819,9 @@ def exportSDDCCGWGroups(self): def enable_advanced_firewall_dest(self) -> bool: """Enable the NSX advanced firewall in the destination SDDC""" - self.check_access_token_expiration() + self.vmc_auth.check_access_token_expiration() myURL = (self.strProdURL + f'/vmc/skynet/api/orgs/{self.dest_org_id}/sddcs/{self.dest_sddc_id}/nsx-advanced-addon?enable=true') - myHeader = {"Authorization":"Bearer " + self.access_token} + myHeader = {"Authorization":"Bearer " + self.vmc_auth.access_token} if self.import_mode == "live": response = requests.post(myURL,headers=myHeader) if response is None or (response.status_code != 200 and response.status_code != 201 and response.status_code != 202): @@ -1837,7 +1836,7 @@ def enable_advanced_firewall_dest(self) -> bool: return True def import_advanced_firewall(self): - self.check_access_token_expiration() + self.vmc_auth.check_access_token_expiration() if self.dest_sddc_enable_nsx_advanced_addon is False: if self.nsx_adv_fw_allow_enable is True: print("nsx_adv_fw_allow_enable set to True, attempting to enable the NSX Advanced Firewall in the destination SDDC...") @@ -1856,7 +1855,7 @@ def import_advanced_firewall(self): def importSDDCCGWRule(self): """Import all CGW Rules from a JSON file""" - self.check_access_token_expiration() + self.vmc_auth.check_access_token_expiration() fname = self.import_path / self.cgw_import_filename try: with open(fname) as filehandle: @@ -1892,7 +1891,7 @@ def importSDDCCGWRule(self): payload["destination_groups"]=rule["destination_groups"] if self.import_mode == "live": json_data = json.dumps(payload) - myHeader = {"Content-Type": "application/json","Accept": "application/json", 'csp-auth-token': self.access_token } + myHeader = {"Content-Type": "application/json","Accept": "application/json", 'csp-auth-token': self.vmc_auth.access_token } myURL = self.proxy_url + "/policy/api/v1/infra/domains/cgw/gateway-policies/default/rules/" + rule["id"] json_data = json.dumps(payload) if self.sync_mode is True: @@ -1915,7 +1914,7 @@ def importSDDCCGWRule(self): def importSDDCCGWGroup(self): """Import all CGW groups from a JSON file""" - self.check_access_token_expiration() + self.vmc_auth.check_access_token_expiration() fname = self.import_path / self.cgw_groups_filename try: with open(fname) as filehandle: @@ -1941,7 +1940,7 @@ def importSDDCCGWGroup(self): payload["resource_type"]=group["resource_type"] payload["display_name"]=group["display_name"] if self.import_mode == "live": - myHeader = {"Content-Type": "application/json","Accept": "application/json", 'csp-auth-token': self.access_token } + myHeader = {"Content-Type": "application/json","Accept": "application/json", 'csp-auth-token': self.vmc_auth.access_token } myURL = self.proxy_url + "/policy/api/v1/infra/domains/cgw/groups/" + group["id"] if "expression" in group: group_expression = group["expression"] @@ -1973,9 +1972,9 @@ def importSDDCCGWGroup(self): def importServiceAccess(self): """Imports SDDC Service Access config from a JSON file""" - self.check_access_token_expiration() + self.vmc_auth.check_access_token_expiration() # First, retrieve the linked VPC ID - myHeader = {'csp-auth-token': self.access_token} + myHeader = {'csp-auth-token': self.vmc_auth.access_token} myURL = (self.proxy_url + '/cloud-service/api/v1/infra/linked-vpcs') try: response = requests.get(myURL,headers=myHeader) @@ -2006,7 +2005,7 @@ def importServiceAccess(self): payload['name'] = svcaccess['name'] payload['enabled'] = svcaccess['enabled'] if self.import_mode == 'live': - myHeader = {"Content-Type": "application/json","Accept": "application/json", 'csp-auth-token': self.access_token } + myHeader = {"Content-Type": "application/json","Accept": "application/json", 'csp-auth-token': self.vmc_auth.access_token } myURL = (self.proxy_url + '/cloud-service/api/v1/infra/linked-vpcs/' + linked_vpc_id + '/connected-services/' + payload['name']) json_data = json.dumps(payload) if self.sync_mode is True: @@ -2024,14 +2023,14 @@ def importServiceAccess(self): return True def importVPNLocalBGP(self): - self.check_access_token_expiration() + self.vmc_auth.check_access_token_expiration() fname = self.import_path / self.vpn_local_bgp_filename with open(fname) as filehandle: local_bgp = json.load(filehandle) payload = {} payload["local_as_num"] = local_bgp ["local_as_num"] if self.import_mode == 'live': - myHeader = {"Content-Type": "application/json","Accept": "application/json", 'csp-auth-token': self.access_token } + myHeader = {"Content-Type": "application/json","Accept": "application/json", 'csp-auth-token': self.vmc_auth.access_token } myURL = self.proxy_url + "/policy/api/v1/infra/tier-0s/vmc/locale-services/default/bgp" json_data = json.dumps(payload) # Always using PATCH here because this BGP object always exists in any SDDC @@ -2046,7 +2045,7 @@ def importVPNLocalBGP(self): def importVPNBGPNeighbors(self): - self.check_access_token_expiration() + self.vmc_auth.check_access_token_expiration() fname = self.import_path / self.vpn_bgp_filename with open(fname) as filehandle: bgpdata = json.load(filehandle) @@ -2075,7 +2074,7 @@ def importVPNBGPNeighbors(self): if "overridden" in bgpentry: payload["overridden"]=bgpentry["overridden"] if self.import_mode == 'live': - myHeader = {"Content-Type": "application/json","Accept": "application/json", 'csp-auth-token': self.access_token } + myHeader = {"Content-Type": "application/json","Accept": "application/json", 'csp-auth-token': self.vmc_auth.access_token } myURL = self.proxy_url + "/policy/api/v1/infra/tier-0s/vmc/locale-services/default/bgp/neighbors/" + bgpentry["id"] json_data = json.dumps(payload) if self.sync_mode is True: @@ -2092,7 +2091,7 @@ def importVPNBGPNeighbors(self): def importVPNTunnelProfiles(self): - self.check_access_token_expiration() + self.vmc_auth.check_access_token_expiration() fname = self.import_path / self.vpn_tunnel_filename with open(fname) as filehandle: tunps = json.load(filehandle) @@ -2109,7 +2108,7 @@ def importVPNTunnelProfiles(self): payload["resource_type"]=tunp["resource_type"] payload["display_name"]=tunp["display_name"] if self.import_mode == 'live': - myHeader = {"Content-Type": "application/json","Accept": "application/json", 'csp-auth-token': self.access_token } + myHeader = {"Content-Type": "application/json","Accept": "application/json", 'csp-auth-token': self.vmc_auth.access_token } myURL = self.proxy_url + "/policy/api/v1/infra/ipsec-vpn-tunnel-profiles/" + tunp["id"] json_data = json.dumps(payload) if self.sync_mode is True: @@ -2125,7 +2124,7 @@ def importVPNTunnelProfiles(self): print("TEST MODE - Tunnel Profile " + payload["display_name"] + " created by " + tunp["_create_user"] + " would have been imported.") def importVPNl2config(self): - self.check_access_token_expiration() + self.vmc_auth.check_access_token_expiration() fname = self.import_path / self.vpn_l2_filename with open(fname) as filehandle: l2vpns = json.load(filehandle) @@ -2147,7 +2146,7 @@ def importVPNl2config(self): payload["overridden"]=l2vpn["overridden"] json_data = json.dumps(payload) if self.import_mode == 'live': - myHeader = {"Content-Type": "application/json","Accept": "application/json", 'csp-auth-token': self.access_token } + myHeader = {"Content-Type": "application/json","Accept": "application/json", 'csp-auth-token': self.vmc_auth.access_token } myURL = self.proxy_url + f'/policy/api/v1/infra/tier-0s/vmc/locale-services/default/l2vpn-services/default/sessions/{payload["id"]}' if self.sync_mode is True: l2vpnresp = requests.patch(myURL,headers=myHeader,data=json_data) @@ -2164,7 +2163,7 @@ def importVPNl2config(self): def importVPNl3config(self): - self.check_access_token_expiration() + self.vmc_auth.check_access_token_expiration() fname = self.import_path / self.vpn_l3_filename with open(fname) as filehandle: l3vpns = json.load(filehandle) @@ -2251,7 +2250,7 @@ def importVPNl3config(self): payload["dpd_profile_path"]=l3vpn["dpd_profile_path"] json_data = json.dumps(payload) if self.import_mode == 'live': - myHeader = {"Content-Type": "application/json","Accept": "application/json", 'csp-auth-token': self.access_token } + myHeader = {"Content-Type": "application/json","Accept": "application/json", 'csp-auth-token': self.vmc_auth.access_token } myURL = self.proxy_url + f'/policy/api/v1/infra/tier-0s/vmc/locale-services/default/ipsec-vpn-services/default/sessions/{payload["id"]}' if self.sync_mode is True: l3vpnresp = requests.patch(myURL,headers=myHeader,data=json_data) @@ -2267,7 +2266,7 @@ def importVPNl3config(self): def importVPNIKEProfiles(self): """Import all""" - self.check_access_token_expiration() + self.vmc_auth.check_access_token_expiration() fname = self.import_path / self.vpn_ike_filename with open(fname) as filehandle: ikeps = json.load(filehandle) @@ -2286,7 +2285,7 @@ def importVPNIKEProfiles(self): payload["overridden"]=ikep["overridden"] json_data = json.dumps(payload) if self.import_mode == 'live': - myHeader = {"Content-Type": "application/json","Accept": "application/json", 'csp-auth-token': self.access_token } + myHeader = {"Content-Type": "application/json","Accept": "application/json", 'csp-auth-token': self.vmc_auth.access_token } myURL = self.proxy_url + "/policy/api/v1/infra/ipsec-vpn-ike-profiles/" + ikep["id"] if self.sync_mode is True: createikepresp = requests.patch(myURL,headers=myHeader,data=json_data) @@ -2302,7 +2301,7 @@ def importVPNIKEProfiles(self): def importSDDCMGWRule(self): """Import all MGW Rules from a JSON file""" - self.check_access_token_expiration() + self.vmc_auth.check_access_token_expiration() fname = self.import_path / self.mgw_import_filename try: with open(fname) as filehandle: @@ -2338,7 +2337,7 @@ def importSDDCMGWRule(self): payload["destination_groups"]=rule["destination_groups"] if self.import_mode == "live": json_data = json.dumps(payload) - myHeader = {"Content-Type": "application/json","Accept": "application/json", 'csp-auth-token': self.access_token } + myHeader = {"Content-Type": "application/json","Accept": "application/json", 'csp-auth-token': self.vmc_auth.access_token } myURL = self.proxy_url + "/policy/api/v1/infra/domains/mgw/gateway-policies/default/rules/" + rule["id"] json_data = json.dumps(payload) if self.sync_mode is True: @@ -2353,7 +2352,7 @@ def importSDDCMGWRule(self): def importSDDCMGWGroup(self): """Import all MGW groups from a JSON file""" - self.check_access_token_expiration() + self.vmc_auth.check_access_token_expiration() fname = self.import_path / self.mgw_groups_filename try: with open(fname) as filehandle: @@ -2381,7 +2380,7 @@ def importSDDCMGWGroup(self): if "expression" in group: payload["expression"]=group["expression"] if self.import_mode == "live": - myHeader = {"Content-Type": "application/json","Accept": "application/json", 'csp-auth-token': self.access_token } + myHeader = {"Content-Type": "application/json","Accept": "application/json", 'csp-auth-token': self.vmc_auth.access_token } myURL = self.proxy_url + "/policy/api/v1/infra/domains/mgw/groups/" + group["id"] json_data = json.dumps(payload) if self.sync_mode is True: @@ -2396,7 +2395,7 @@ def importSDDCMGWGroup(self): def exportSDDCNat(self): """Exports the NAT rules to a JSON file""" - self.check_access_token_expiration() + self.vmc_auth.check_access_token_expiration() myURL = (self.proxy_url + "/policy/api/v1/infra/tier-1s/cgw/nat/USER/nat-rules") response = self.invokeVMCGET(myURL) if response is None or response.status_code != 200: @@ -2411,7 +2410,7 @@ def exportSDDCNat(self): def importSDDCNats(self): """Imports SDDC NAT from a JSON file""" - self.check_access_token_expiration() + self.vmc_auth.check_access_token_expiration() fname = self.import_path / self.nat_import_filename with open(fname) as filehandle: nat = json.load(filehandle) @@ -2438,7 +2437,7 @@ def importSDDCNats(self): json_data["translated_network"] = public_ip_old_new[old_ip] new_ip_name_dash = json_data["translated_network"].replace(".","-") myURL = (self.proxy_url + "/policy/api/v1/infra/tier-1s/cgw/nat/USER/nat-rules/" + (n['display_name']).replace(" ", "-") + "-" + new_ip_name_dash) - myHeader = {'csp-auth-token': self.access_token} + myHeader = {'csp-auth-token': self.vmc_auth.access_token} response = requests.put(myURL, headers=myHeader, json=json_data) json_response_status_code = response.status_code print("NAT Rule " + n['display_name'] + " has been imported.") @@ -2450,7 +2449,7 @@ def importSDDCNats(self): json_data["service"] = n["service"] new_ip_name_dash = json_data["destination_network"].replace(".","-") myURL = (self.proxy_url + "/policy/api/v1/infra/tier-1s/cgw/nat/USER/nat-rules/" + (n['display_name']).replace(" ", "-") + "-" + new_ip_name_dash) - myHeader = {'csp-auth-token': self.access_token} + myHeader = {'csp-auth-token': self.vmc_auth.access_token} response = requests.put(myURL, headers=myHeader, json=json_data) json_response_status_code = response.status_code print("NAT Rule " + n['display_name'] + " has been imported.") @@ -2476,8 +2475,8 @@ def exportSDDCListPublicIP(self): def importSDDCPublicIPs(self): """Import all Public IP addresses from a JSON file""" - self.check_access_token_expiration() - myHeader = {"Content-Type": "application/json","Accept": "application/json", 'csp-auth-token': self.access_token} + self.vmc_auth.check_access_token_expiration() + myHeader = {"Content-Type": "application/json","Accept": "application/json", 'csp-auth-token': self.vmc_auth.access_token} proxy_url_short = (self.proxy_url).rstrip("sks-nsxt-manager") aDict = {} fname = self.import_path / self.public_import_filename @@ -2494,7 +2493,7 @@ def importSDDCPublicIPs(self): } if self.import_mode == "live": public_ip_response = requests.put(myURL, headers=myHeader, json=public_ip_request_data) - myHeader = {'csp-auth-token': self.access_token} + myHeader = {'csp-auth-token': self.vmc_auth.access_token} myURL = (self.proxy_url + "/cloud-service/api/v1/infra/public-ips/" + public_name) response = requests.get(myURL, headers=myHeader) json_response = response.json() @@ -2516,7 +2515,7 @@ def importSDDCPublicIPs(self): return aDict def importVPN(self): - self.check_access_token_expiration() + self.vmc_auth.check_access_token_expiration() successval = True print("Beginning IKE Profiles...") @@ -2568,36 +2567,36 @@ def importVPN(self): return successval - def getAccessToken(self,myRefreshToken): - """ Gets the Access Token using the Refresh Token """ - self.activeRefreshToken = myRefreshToken - params = {'api_token': myRefreshToken} - headers = {'Content-Type': 'application/x-www-form-urlencoded'} - response = requests.post(f'{self.strCSPProdURL}/csp/gateway/am/api/auth/api-tokens/authorize', params=params, headers=headers) - jsonResponse = response.json() - #print(jsonResponse) - try: - self.access_token = jsonResponse['access_token'] - expires_in = jsonResponse['expires_in'] - expirestime = datetime.datetime.now() + datetime.timedelta(seconds=expires_in) - self.access_token_expiration = expirestime - print(f'Token expires at {expirestime}') - except: - self.access_token = None - self.access_token_expiration = None - return self.access_token - - def check_access_token_expiration(self) -> None: - """Retrieve a new access token if it is near expiration""" - time_to_expire = self.access_token_expiration - datetime.datetime.now() - if time_to_expire.total_seconds() <= 100: - print('Access token expired, attempting to refresh...') - self.getAccessToken(self.activeRefreshToken) + # def getAccessToken(self,myRefreshToken): + # """ Gets the Access Token using the Refresh Token """ + # self.activeRefreshToken = myRefreshToken + # params = {'api_token': myRefreshToken} + # headers = {'Content-Type': 'application/x-www-form-urlencoded'} + # response = requests.post(f'{self.strCSPProdURL}/csp/gateway/am/api/auth/api-tokens/authorize', params=params, headers=headers) + # jsonResponse = response.json() + # #print(jsonResponse) + # try: + # self.access_token = jsonResponse['access_token'] + # expires_in = jsonResponse['expires_in'] + # expirestime = datetime.datetime.now() + datetime.timedelta(seconds=expires_in) + # self.access_token_expiration = expirestime + # print(f'Token expires at {expirestime}') + # except: + # self.access_token = None + # self.access_token_expiration = None + # return self.access_token + + # def check_access_token_expiration(self) -> None: + # """Retrieve a new access token if it is near expiration""" + # time_to_expire = self.access_token_expiration - datetime.datetime.now() + # if time_to_expire.total_seconds() <= 100: + # print('Access token expired, attempting to refresh...') + # self.getAccessToken(self.activeRefreshToken) def getNSXTproxy(self, org_id, sddc_id): """ Gets the Reverse Proxy URL """ - self.check_access_token_expiration() - myHeader = {'csp-auth-token': self.access_token} + self.vmc_auth.check_access_token_expiration() + myHeader = {'csp-auth-token': self.vmc_auth.access_token} myURL = f'{self.strProdURL}/vmc/api/orgs/{org_id}/sddcs/{sddc_id}' response = requests.get(myURL, headers=myHeader) json_response = response.json() @@ -2711,8 +2710,8 @@ def loadSourceOrgData(self): def loadOrgData(self,orgID): """Download the JSON for an organization object""" - self.check_access_token_expiration() - myHeader = {'csp-auth-token': self.access_token} + self.vmc_auth.check_access_token_expiration() + myHeader = {'csp-auth-token': self.vmc_auth.access_token} myURL = self.strProdURL + "/vmc/api/orgs/" + orgID try: response = requests.get(myURL,headers=myHeader) @@ -2780,8 +2779,8 @@ def exportSourceSDDCData(self): def loadSDDCData(self,orgID,sddcID): """Download the JSON for an SDDC object""" - self.check_access_token_expiration() - myHeader = {'csp-auth-token': self.access_token} + self.vmc_auth.check_access_token_expiration() + myHeader = {'csp-auth-token': self.vmc_auth.access_token} myURL = self.strProdURL + "/vmc/api/orgs/" + orgID + "/sddcs/" + sddcID try: response = requests.get(myURL,headers=myHeader) @@ -2796,7 +2795,7 @@ def loadSDDCData(self,orgID,sddcID): def loadSDDCNSX(self, orgID, sddcID): """Loads SDDC URLs and credentials""" - self.check_access_token_expiration() + self.vmc_auth.check_access_token_expiration() myURL = self.strProdURL + f'/api/network/{orgID}/core/deployments/{sddcID}/nsx' response = self.invokeCSPGET(myURL) if response is None or response.status_code != 200: @@ -2805,7 +2804,7 @@ def loadSDDCNSX(self, orgID, sddcID): return response.json() def searchOrgUser(self,orgid,userSearchTerm): - self.check_access_token_expiration() + self.vmc_auth.check_access_token_expiration() myURL = (self.strCSPProdURL + "/csp/gateway/am/api/orgs/" + orgid + "/users/search?userSearchTerm=" + userSearchTerm) response = self.invokeCSPGET(myURL) if response is None or response.status_code != 200: diff --git a/sddc_import_export.py b/sddc_import_export.py index 37c1d95..2da8524 100755 --- a/sddc_import_export.py +++ b/sddc_import_export.py @@ -5,9 +5,8 @@ # SDDC Import/Export for VMware Cloud on AWS - ################################################################################ -### Copyright 2020-2021 VMware, Inc. +### Copyright 2020-2023 VMware, Inc. ### SPDX-License-Identifier: BSD-2-Clause ################################################################################ @@ -188,8 +187,8 @@ def main(args): if intent_name == "rolesync": no_intent_found = False - ioObj.getAccessToken(ioObj.source_refresh_token) - if (ioObj.access_token == ""): + ioObj.vmc_auth.getAccessToken(ioObj.source_refresh_token) + if (ioObj.vmc_auth.access_token == ""): print("Unable to retrieve source access token. Server response:{}".format(ioObj.lastJSONResponse)) sys.exit() print (f'Looking up template user {ioObj.RoleSyncSourceUserEmail}') @@ -203,8 +202,8 @@ def main(args): retval = ioObj.convertServiceRolePayload(template_user_roles) payload = {} payload = ioObj.convertedServiceRolePayload - ioObj.getAccessToken(ioObj.dest_refresh_token) - if (ioObj.access_token == ""): + ioObj.vmc_auth.getAccessToken(ioObj.dest_refresh_token) + if (ioObj.vmc_auth.access_token == ""): print("Unable to retrieve source access token. Server response:{}".format(ioObj.lastJSONResponse)) sys.exit() if ioObj.import_mode == "live": @@ -238,8 +237,8 @@ def main(args): print('Testbed mode:',ioObj.import_mode) - ioObj.getAccessToken(ioObj.dest_refresh_token) - if (ioObj.access_token == ""): + ioObj.vmc_auth.getAccessToken(ioObj.dest_refresh_token) + if (ioObj.vmc_auth.access_token == ""): print("Unable to retrieve access token. Server response:{}".format(ioObj.lastJSONResponse)) sys.exit() @@ -347,8 +346,8 @@ def main(args): no_intent_found = False print('Import mode:', ioObj.import_mode) - ioObj.getAccessToken(ioObj.dest_refresh_token) - if (ioObj.access_token == ""): + ioObj.vmc_auth.getAccessToken(ioObj.dest_refresh_token) + if (ioObj.vmc_auth.access_token == ""): print("Unable to retrieve access token. Server response:{}".format(ioObj.lastJSONResponse)) sys.exit() @@ -399,8 +398,8 @@ def main(args): if intent_name == "check-vmc-ini": no_intent_found = False - ioObj.getAccessToken(ioObj.source_refresh_token) - if (ioObj.access_token == ""): + ioObj.vmc_auth.getAccessToken(ioObj.source_refresh_token) + if (ioObj.vmc_auth.access_token == ""): print("Unable to retrieve access token. Server response:{}".format(ioObj.lastJSONResponse)) sys.exit() @@ -421,8 +420,8 @@ def main(args): print(f'Export configuration: Org {ioObj.source_org_display_name} ({ioObj.source_org_id}), SDDC {ioObj.source_sddc_name} ({ioObj.source_sddc_id}), SDDC version {ioObj.source_sddc_version}') - ioObj.getAccessToken(ioObj.dest_refresh_token) - if (ioObj.access_token == ""): + ioObj.vmc_auth.getAccessToken(ioObj.dest_refresh_token) + if (ioObj.vmc_auth.access_token == ""): print("Unable to retrieve access token. Server response:{}".format(ioObj.lastJSONResponse)) sys.exit() @@ -446,8 +445,9 @@ def main(args): if intent_name == "export" or intent_name == "export-import": no_intent_found = False - ioObj.getAccessToken(ioObj.source_refresh_token) - if (ioObj.access_token == ""): + ioObj.vmc_auth.getAccessToken(ioObj.source_refresh_token) + + if (ioObj.vmc_auth.access_token == ""): print("Unable to retrieve access token. Server response:{}".format(ioObj.lastJSONResponse)) sys.exit() @@ -738,8 +738,8 @@ def main(args): print('Import mode:',ioObj.import_mode) - ioObj.getAccessToken(ioObj.dest_refresh_token) - if (ioObj.access_token == ""): + ioObj.vmc_auth.getAccessToken(ioObj.dest_refresh_token) + if (ioObj.vmc_auth.access_token == ""): print("Unable to retrieve access token. Server response:{}".format(ioObj.lastJSONResponse)) sys.exit() diff --git a/vmc_auth.py b/vmc_auth.py new file mode 100644 index 0000000..5237ce9 --- /dev/null +++ b/vmc_auth.py @@ -0,0 +1,45 @@ +## API authentication module for VMware Cloud on AWS + +################################################################################ +### Copyright 2020-2023 VMware, Inc. +### SPDX-License-Identifier: BSD-2-Clause +################################################################################ + +import json +import requests +import datetime + +class VMCAuth: + def __init__(self, strCSPProdURL: str): + self.access_token = None + self.access_token_expiration = None + self.activeRefreshToken = None + self.strCSPProdURL = strCSPProdURL + + def getAccessToken(self,myRefreshToken): + """ Gets the Access Token using the Refresh Token """ + self.activeRefreshToken = myRefreshToken + params = {'api_token': myRefreshToken} + headers = {'Content-Type': 'application/x-www-form-urlencoded'} + + try: + response = requests.post(f'{self.strCSPProdURL}/csp/gateway/am/api/auth/api-tokens/authorize', params=params, headers=headers) + jsonResponse = response.json() + self.access_token = jsonResponse['access_token'] + expires_in = jsonResponse['expires_in'] + expirestime = datetime.datetime.now() + datetime.timedelta(seconds=expires_in) + self.access_token_expiration = expirestime + print(f'Token expires at {expirestime}') + except: + self.access_token = None + self.access_token_expiration = None + print(jsonResponse) + return self.access_token + + def check_access_token_expiration(self) -> None: + """Retrieve a new access token if it is near expiration""" + if self.access_token_expiration is not None: + time_to_expire = self.access_token_expiration - datetime.datetime.now() + if time_to_expire.total_seconds() <= 100: + print('Access token expired, attempting to refresh...') + self.getAccessToken(self.activeRefreshToken) \ No newline at end of file From 5166643787fc5093404f5e213c9a0c566e5c4155 Mon Sep 17 00:00:00 2001 From: Chris White Date: Wed, 18 Jan 2023 08:50:14 -0500 Subject: [PATCH 02/26] Update requirements.txt --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index d3f98b0..dbc584e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -certifi==2020.6.20 +certifi==2022.12.07 chardet==3.0.4 configparser==5.0.1 idna==2.10 @@ -7,4 +7,4 @@ requests==2.26.0 urllib3==1.26.6 wcwidth==0.2.5 boto3==1.16.39 -#git+https://github.com/vmware/vsphere-automation-sdk-python.git@v7.0.2.0 \ No newline at end of file +#git+https://github.com/vmware/vsphere-automation-sdk-python.git@v7.0.2.0 From 822146894692c3273cfb7ffaf3911d39209846fe Mon Sep 17 00:00:00 2001 From: Chris White Date: Tue, 4 Apr 2023 08:50:11 -0400 Subject: [PATCH 03/26] Update requirements.txt Signed-off-by: Chris White --- requirements.txt | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/requirements.txt b/requirements.txt index dbc584e..81dad16 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,10 @@ certifi==2022.12.07 -chardet==3.0.4 -configparser==5.0.1 -idna==2.10 +chardet==5.1.0 +configparser==5.3.0 +idna==3.4 PTable==0.9.2 -requests==2.26.0 -urllib3==1.26.6 -wcwidth==0.2.5 -boto3==1.16.39 +requests==2.28.2 +urllib3==1.26.15 +wcwidth==0.2.6 +boto3==1.26.105 #git+https://github.com/vmware/vsphere-automation-sdk-python.git@v7.0.2.0 From 6bca1e6cca6c5d877ed78ee615f2a808c0f57b7f Mon Sep 17 00:00:00 2001 From: Chris White Date: Fri, 19 May 2023 12:29:13 -0400 Subject: [PATCH 04/26] BUG: correct SDDC L3VPN import function Signed-off-by: Chris White --- .gitignore | 7 ++- VMCImportExport.py | 108 ++++++++++++++++++++++++++++++++++++++++-- sddc_import_export.py | 2 +- 3 files changed, 112 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 585b27a..244c2b2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,7 @@ *.json -__pycache__ \ No newline at end of file +__pycache__ +.DS_Store +CODE-OF-CONDUCT.md +CONTRIBUTING.md +LICENSE.txt +NOTICE.txt diff --git a/VMCImportExport.py b/VMCImportExport.py index a85ec66..0908e9d 100755 --- a/VMCImportExport.py +++ b/VMCImportExport.py @@ -273,6 +273,64 @@ def ConfigLoader(self): self.RoleSyncSourceUserEmail = config.get("exportConfig","role_sync_source_user_email") self.RoleSyncDestUserEmails = config.get("importConfig","role_sync_dest_user_emails").split('|') + def error_handling(self, response): + """Helper function to properly report API errors""" + code = response.status_code + print(f'API call failed with status code {code}.') + if code == 301: + print(f'Error {code}: "Moved Permanently"') + print("Request must be reissued to a different controller node.") + print( + "The controller node has been replaced by a new node that should be used for this and all future requests.") + elif code == 307: + print(f'Error {code}: "Temporary Redirect"') + print("Request should be reissued to a different controller node.") + print( + "The controller node is requesting the client make further requests against the controller node specified in the Location header. Clients should continue to use the new server until directed otherwise by the new controller node.") + elif code == 400: + print(f'Error {code}: "Bad Request"') + print("Request was improperly formatted or contained an invalid parameter.") + elif code == 401: + print(f'Error {code}: "Unauthorized"') + print("The client has not authenticated.") + print("It's likely your refresh token is out of date or otherwise incorrect.") + elif code == 403: + print(f'Error {code}: "Forbidden"') + print("The client does not have sufficient privileges to execute the request.") + print("The API is likely in read-only mode, or a request was made to modify a read-only property.") + print("It's likely your refresh token does not provide sufficient access.") + elif code == 409: + print(f'Error {code}: "Temporary Redirect"') + print( + "The request can not be performed because it conflicts with configuration on a different entity, or because another client modified the same entity.") + print( + "If the conflict arose because of a conflict with a different entity, modify the conflicting configuration. If the problem is due to a concurrent update, re-fetch the resource, apply the desired update, and reissue the request.") + elif code == 412: + print(f'Error {code}: "Precondition Failed"') + print( + "The request can not be performed because a precondition check failed. Usually, this means that the client sent a PUT or PATCH request with an out-of-date _revision property, probably because some other client has modified the entity since it was retrieved. The client should re-fetch the entry, apply any desired changes, and re-submit the operation.") + elif code == 500: + print(f'Error {code}: "Internal Server Error"') + print( + "An internal error occurred while executing the request. If the problem persists, perform diagnostic system tests, or contact your support representative.") + elif code == 503: + print(f'Error {code}: "Service Unavailable"') + print( + "The request can not be performed because the associated resource could not be reached or is temporarily busy. Please confirm the ORG ID and SDDC ID entries in your config.ini are correct.") + else: + print(f'Error: {code}: Unknown error') + try: + json_response = response.json() + if 'error_message' in json_response: + print(json_response['error_message']) + if 'related_errors' in json_response: + print("Related Errors") + for r in json_response['related_errors']: + print(r['error_message']) + except: + print("No additional information in the error response.") + return None + def purgeJSONfiles(self): """Removes the JSON export files before a new export""" files = glob.glob(self.export_folder + '/*.json') @@ -2123,6 +2181,41 @@ def importVPNTunnelProfiles(self): else: print("TEST MODE - Tunnel Profile " + payload["display_name"] + " created by " + tunp["_create_user"] + " would have been imported.") + def importVPNDPDProfiles(self): + self.vmc_auth.check_access_token_expiration() + fname = self.import_path / self.vpn_dpd_filename + with open(fname) as filehandle: + dpdps = json.load(filehandle) + payload = {} + for dpdp in dpdps: + if dpdp["_create_user"]!= 'admin' and dpdp['_create_user'] != "admin;admin" and dpdp['_create_user'] != "system": + payload['id'] = dpdp['id'] + payload['display_name'] = dpdp['display_name'] + payload['dpd_probe_mode'] = dpdp['dpd_probe_mode'] + payload['dpd_probe_interval'] = dpdp['dpd_probe_interval'] + payload['retry_count'] = dpdp['retry_count'] + payload['enabled'] = dpdp['enabled'] + profile_url = dpdp['path'] + if self.import_mode == 'live': + my_header = {"Content-Type": "application/json","Accept": "application/json", 'csp-auth-token': self.vmc_auth.access_token } + my_url = f"{self.proxy_url}/policy/api/v1{profile_url}" + json_data = json.dumps(payload) + if self.sync_mode is True: + response = requests.patch(my_url, headers=my_header, data=json_data) + else: + response = requests.put(my_url, headers=my_header, data=json_data) + if response.status_code == 200: + print(f"DPD Profile {payload['display_name']} has been imported") + return True + else: + self.error_handling(response) + return False + else: + print(f"TEST MODE - DPD Profile {payload['display_name']} created by {payload['_create_user']} would have been imported") + else: + pass + + def importVPNl2config(self): self.vmc_auth.check_access_token_expiration() fname = self.import_path / self.vpn_l2_filename @@ -2258,9 +2351,10 @@ def importVPNl3config(self): l3vpnresp = requests.put(myURL,headers=myHeader,data=json_data) if l3vpnresp.status_code == 200: print("L3VPN " + payload["id"] + " has been imported.") + return True else: - print(f'API Call Status {l3vpnresp.status_code}, text:{l3vpnresp.text}') - print(json_data) + self.error_handling(l3vpnresp) + return False else: print("TEST MODE - L3VPN " + l3vpn["id"] + " created by " + l3vpn["_create_user"] + " would have been imported.") @@ -2292,7 +2386,7 @@ def importVPNIKEProfiles(self): else: createikepresp = requests.put(myURL,headers=myHeader,data=json_data) if createikepresp.status_code == 200: - print("VPN Profile " + payload["display_name"] + " has been imported.") + print("IKE Profile " + payload["display_name"] + " has been imported.") else: print(f'API Call Status {createikepresp.status_code}, text:{createikepresp.text}') print(json_data) @@ -2526,6 +2620,7 @@ def importVPN(self): else: print('IKE Profiles imported.') print("Beginning VPN Tunnel Profiles...") + retval = self.importVPNTunnelProfiles() if retval is False: successval = False @@ -2533,6 +2628,13 @@ def importVPN(self): else: print('Tunnel profiles imported.') + retval = self.importVPNDPDProfiles() + if retval is False: + successval = False + print('DPD Profile import failure: ', self.lastJSONResponse) + else: + print('DPD Profiles imported.') + print("Beginning BGP Neighbors...") retval = self.importVPNBGPNeighbors() if retval is False: diff --git a/sddc_import_export.py b/sddc_import_export.py index 2da8524..e0c8c37 100755 --- a/sddc_import_export.py +++ b/sddc_import_export.py @@ -899,7 +899,7 @@ def main(args): print("Import has been concluded. Thank you for using SDDC Import/Export for VMware Cloud on AWS.") - if no_intent_found == True: + if no_intent_found: print("\nWelcome to sddc_import_export!") print("\nHere are the currently supported commands: ") print("\nTo export your source SDDC to JSON") From 34fbce69b5451a681f7389039ca57bb4b717ec54 Mon Sep 17 00:00:00 2001 From: Chris White Date: Fri, 19 May 2023 14:51:24 -0400 Subject: [PATCH 05/26] BUG: Fixed SDDC L3 VPN DPD Profile import Signed-off-by: Chris White --- VMCImportExport.py | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/VMCImportExport.py b/VMCImportExport.py index 0908e9d..8bce94a 100755 --- a/VMCImportExport.py +++ b/VMCImportExport.py @@ -2096,8 +2096,7 @@ def importVPNLocalBGP(self): if bgppresp.status_code == 200: print("Local BGP config has been imported.") else: - print(f'API Call Status {bgppresp.status_code}, text:{bgppresp.text}') - print(json_data) + self.error_handling(bgppresp) else: print("TEST MODE - Local BGP config would have been imported.") @@ -2142,8 +2141,7 @@ def importVPNBGPNeighbors(self): if bgppresp.status_code == 200: print("BGP neighbor " + payload["display_name"] + " has been imported.") else: - print(f'API Call Status {bgppresp.status_code}, text:{bgppresp.text}') - print(json_data) + self.error_handling(bgppresp) else: print("TEST MODE - BGP Neighbor " + payload["display_name"] + " created by " + bgpentry["_create_user"] + " would have been imported.") @@ -2176,8 +2174,7 @@ def importVPNTunnelProfiles(self): if tunpresp.status_code == 200: print("Tunnel Profile " + payload["display_name"] + " has been imported.") else: - print(f'API Call Status {tunpresp.status_code}, text:{tunpresp.text}') - print(json_data) + self.error_handling(tunpresp) else: print("TEST MODE - Tunnel Profile " + payload["display_name"] + " created by " + tunp["_create_user"] + " would have been imported.") @@ -2188,13 +2185,14 @@ def importVPNDPDProfiles(self): dpdps = json.load(filehandle) payload = {} for dpdp in dpdps: - if dpdp["_create_user"]!= 'admin' and dpdp['_create_user'] != "admin;admin" and dpdp['_create_user'] != "system": + if dpdp['_system_owned'] is False: payload['id'] = dpdp['id'] payload['display_name'] = dpdp['display_name'] payload['dpd_probe_mode'] = dpdp['dpd_probe_mode'] payload['dpd_probe_interval'] = dpdp['dpd_probe_interval'] payload['retry_count'] = dpdp['retry_count'] payload['enabled'] = dpdp['enabled'] + payload['_create_user'] = dpdp['_create_user'] profile_url = dpdp['path'] if self.import_mode == 'live': my_header = {"Content-Type": "application/json","Accept": "application/json", 'csp-auth-token': self.vmc_auth.access_token } @@ -2206,14 +2204,10 @@ def importVPNDPDProfiles(self): response = requests.put(my_url, headers=my_header, data=json_data) if response.status_code == 200: print(f"DPD Profile {payload['display_name']} has been imported") - return True else: self.error_handling(response) - return False else: print(f"TEST MODE - DPD Profile {payload['display_name']} created by {payload['_create_user']} would have been imported") - else: - pass def importVPNl2config(self): @@ -2248,8 +2242,7 @@ def importVPNl2config(self): if l2vpnresp.status_code == 200: print("L2VPN " + payload["id"] + " has been imported.") else: - print(f'API Call Status {l2vpnresp.status_code}, text:{l2vpnresp.text}') - print(json_data) + self.error_handling(l2vpnresp) else: print("TEST MODE - L2VPN " + l2vpn["id"] + " created by " + l2vpn["_create_user"] + " would have been imported.") return True @@ -2351,10 +2344,8 @@ def importVPNl3config(self): l3vpnresp = requests.put(myURL,headers=myHeader,data=json_data) if l3vpnresp.status_code == 200: print("L3VPN " + payload["id"] + " has been imported.") - return True else: self.error_handling(l3vpnresp) - return False else: print("TEST MODE - L3VPN " + l3vpn["id"] + " created by " + l3vpn["_create_user"] + " would have been imported.") @@ -2388,8 +2379,7 @@ def importVPNIKEProfiles(self): if createikepresp.status_code == 200: print("IKE Profile " + payload["display_name"] + " has been imported.") else: - print(f'API Call Status {createikepresp.status_code}, text:{createikepresp.text}') - print(json_data) + self.error_handling(createikepresp) else: print("TEST MODE - IKE Profile " + ikep["display_name"] + " created by " + ikep["_create_user"] + " would have been imported.") From 6ce9448a818259df105632fd010f6f2341d1c52d Mon Sep 17 00:00:00 2001 From: Chris White Date: Mon, 22 May 2023 11:13:03 -0400 Subject: [PATCH 06/26] FEATURE: Add support for Managed Prefix List Signed-off-by: Chris White --- VMCImportExport.py | 53 ++++++++++++++++++++++++++++++++++++++++++- config_ini/config.ini | 11 ++++++++- sddc_import_export.py | 20 ++++++++++++++++ 3 files changed, 82 insertions(+), 2 deletions(-) diff --git a/VMCImportExport.py b/VMCImportExport.py index 8bce94a..59efa11 100755 --- a/VMCImportExport.py +++ b/VMCImportExport.py @@ -179,7 +179,13 @@ def ConfigLoader(self): self.mcgw_fw_import = self.loadConfigFlag(config, "importConfig", "mcgw_fw_import") self.mcgw_fw_import_filename = self.loadConfigFilename(config, "importConfig", "mcgw_fw_import_filename") - #DDC Route Aggregation Lists and Route Configurations + #Connected VPC Managed Prefix List MOde + self.mpl_export = self.loadConfigFlag(config, 'exportConfig', 'mpl_export') + self.mpl_export_filename = self.loadConfigFilename(config, 'exportConfig', 'mpl_export_filename') + self.mpl_import = self.loadConfigFlag(config, 'importConfig', 'mpl_import') + self.mpl_import_filename = self.loadConfigFilename(config, 'importConfig', 'mpl_import_filename') + + #SDDC Route Aggregation Lists and Route Configurations self.ral_export = self.loadConfigFlag(config, "exportConfig", "ral_export") self.ral_export_filename = self.loadConfigFilename(config, "exportConfig", "ral_export_filename") self.route_config_export = self.loadConfigFlag(config, "exportConfig", "route_config_export") @@ -820,6 +826,22 @@ def export_mcgw_fw(self): json.dump(mcgw_fw_policy_json, outfile, indent=4) return True + + def export_mpl(self): + """Exports Connected VPC Managed Prefix List""" + my_url = f'{self.proxy_url}/cloud-service/api/v1/infra/linked-vpcs' + response = self.invokeCSPGET(my_url) + if response is None or response.status_code != 200: + self.error_handling(response) + return False + json_response = response.json() + mpl_response = json_response['results'] + fname = self.export_path / self.mpl_export_filename + with open(fname, 'w') as outfile: + json.dump(mpl_response, outfile, indent=4) + return True + + def export_ral(self): """Exports the SDDCs Route Aggregation List(s)""" my_url = f'{self.proxy_url}/cloud-service/api/v1/infra/external/route/aggregations' @@ -1552,6 +1574,35 @@ def import_mcgw_fw(self): else: print(f"TEST MODE - Tier 1 Gateway firewall policy and rules would have been imported.") + + def import_mpl(self): + """Import/Configuration Connected VPC Managed Prefix List""" + self.vmc_auth.check_access_token_expiration() + fname = self.import_path / self.mpl_import_filename + try: + with open(fname) as filehandle: + mpl = json.load(filehandle) + except: + print(f'Import failed - unable to open {fname}') + return + if self.import_mode == 'live': + for m in mpl: + if m['linked_vpc_managed_prefix_list_info']['managed_prefix_list_mode'] == 'ENABLED': + vpc_id = m['linked_vpc_id'] + my_header = {"Content-Type": "application/json", "Accept": "application/json", "csp-auth-token": self.vmc_auth.access_token} + my_url = f'{self.proxy_url}/cloud-service/api/v1/linked-vpcs/{vpc_id}?action=enable_managed_prefix_list_mode' + response = requests.post(my_url, headers=my_header) + if response.status_code == 200: + result = "SUCCESS" + print('Enabled Managed Prefix List Mode. Proceed to the AWS Management Console and Resource Access Manager to accept the share') + else: + self.error_handling(response) + result = "FAIL" + else: + print("TEST MODE - Connected VPC Managed Prefix List mode would have been enabled") + + + def import_ral(self): """Import SDDC Route Aggregation lists from JSON""" self.vmc_auth.check_access_token_expiration() diff --git a/config_ini/config.ini b/config_ini/config.ini index d42985d..3dc812d 100755 --- a/config_ini/config.ini +++ b/config_ini/config.ini @@ -43,6 +43,7 @@ nat_export_filename = natrules.json #Export VPN configuration? vpn_export = True +t1_vpn_export = False # Export service access? service_access_export = True @@ -64,6 +65,10 @@ mcgw_fw_export_filename = mcgw_fw.json mcgw_static_routes_export = True mcgw_static_routes_export_filename = mcgw_static_routes.json +# Export Connected VPC Managed Prefix List configuration +mpl_export = True +mpl_export_filename = mpl.json + # Export route aggregation lists and route configuration ral_export = True ral_export_filename = ral.json @@ -127,7 +132,7 @@ import_mode_live_warning = True # Import services? Only disable this if you truly know what you are doing. # Firewall groups are dependent on Services. If you skip Services, Groups that are # dependent on those services will fail to import -services_import = True +services_import = False # Import groups? Only disable this if you truly know what you are doing. # Firewall rules are dependent on Groups. If you skip Groups, Firewall rules @@ -162,6 +167,10 @@ mcgw_static_route_import_filename = mcgw_static_routes.json mcgw_fw_import = True mcgw_fw_import_filename = mcgw_fw.json +# Import Conncted VPC Managed Prefix List +mpl_import = True +mpl_import_filename = mpl.json + #Import Route Aggregation Lists and Route Configuration? ral_import = True ral_import_filename = ral.json diff --git a/sddc_import_export.py b/sddc_import_export.py index e0c8c37..eb8b1a2 100755 --- a/sddc_import_export.py +++ b/sddc_import_export.py @@ -562,6 +562,16 @@ def main(args): else: print("Multi-T1 Firewall Policy and Rules export skipped") + if ioObj.mpl_export is True: + print("Beginning Connected VPC Managed Prefix List export") + retval = ioObj.export_mpl() + if retval is True: + print("Connected VPC Managed Prefix List settings exported") + else: + print(f'Connected VPC Managed Prefix List export error: {ioObj.lastJSONResponse}') + else: + print("Connected VPC Managed Prefix List export skipped") + if ioObj.ral_export is True: print('Beginning Route Aggegration list export') retval = ioObj.export_ral() @@ -849,12 +859,22 @@ def main(args): print('Tier-1 Gateway import is set to false, this can cause import error is Tier-1 Gateway objects are missing.') ioObj.import_mcgw_fw() + if ioObj.mpl_import is True: + print("Beginning import of Connected VPC Managed Prefix List configuration") + ioObj.import_mpl() + else: + print("Connected VPC Managed Prefix List import skipped...") + if ioObj.ral_import is True: print('Beginning import of SDDC Route Aggregation Lists...') + if ioObj.mpl_import is False: + print("Managed Prefix List import is set to false. If MPL is not enabled, Route Aggregation lists will not be imported successfully") ioObj.import_ral() if ioObj.route_config_import is True: print('Beginning import of SDDC Route Configurations...') + if ioObj.mpl_import is False: + print("Managed Prefix List import is set to false. If MPL is not enabled, Route Configuration will not be imported successfully") if ioObj.ral_import is False: print('Import of Route Configurations may be impacted by missing Route Aggregations lists') ioObj.import_route_config() From 3614f8fae7fc0ff91e02926c1ddc8c288924cf78 Mon Sep 17 00:00:00 2001 From: Chris White Date: Tue, 23 May 2023 07:53:42 -0400 Subject: [PATCH 07/26] Update requirements.txt Signed-off-by: Chris White --- requirements.txt | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 81dad16..b17ab0f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,11 @@ -certifi==2022.12.07 +certifi==2023.5.7 chardet==5.1.0 configparser==5.3.0 idna==3.4 PTable==0.9.2 -requests==2.28.2 +requests==2.31.0 urllib3==1.26.15 wcwidth==0.2.6 -boto3==1.26.105 +boto3==1.26.137 +prettytable == 3.7.0 #git+https://github.com/vmware/vsphere-automation-sdk-python.git@v7.0.2.0 From 6bc8c7c5f1ff04a4c3972098beec8fb59686d5d7 Mon Sep 17 00:00:00 2001 From: Chris White Date: Thu, 25 May 2023 15:16:24 -0400 Subject: [PATCH 08/26] Export Tier-1 VPN info Signed-off-by: Chris White --- VMCImportExport.py | 92 +++++++++++++++++++++++++++++++++++++++++++ config_ini/config.ini | 5 ++- sddc_import_export.py | 12 +++++- 3 files changed, 107 insertions(+), 2 deletions(-) diff --git a/VMCImportExport.py b/VMCImportExport.py index 59efa11..fe30fee 100755 --- a/VMCImportExport.py +++ b/VMCImportExport.py @@ -238,6 +238,10 @@ def ConfigLoader(self): self.vpn_l3_filename = self.loadConfigFilename(config,"importConfig","vpn_l3_filename") self.vpn_l2_filename = self.loadConfigFilename(config,"importConfig","vpn_l2_filename") self.vpn_disable_on_import = self.loadConfigFlag(config,"importConfig","vpn_disable_on_import") + self.tier1_vpn_export = self.loadConfigFlag(config, 'exportConfig', 't1_vpn_export') + self.tier1_vpn_export_filename = self.loadConfigFilename(config, 'exportConfig', 't1_vpn_export_filename') + self.tier1_vpn_service_filename = self.loadConfigFilename(config, 'exportConfig', 't1_vpn_service_filename') + self.tier1_vpn_le_filename = self.loadConfigFilename(config, 'exportConfig', 't1_vpn_localendpoint_filename') #Service Access self.service_access_export = self.loadConfigFlag(config,"exportConfig","service_access_export") @@ -1189,6 +1193,94 @@ def exportVPNBGPNeighbors(self): json.dump(bgp_neighbors, outfile,indent=4) return True + + def export_tier1_vpn(self): + """Exports the Tier-1 VPN Services""" + t1_url = f'{self.proxy_url}/policy/api/v1/infra/tier-1s' + t1_response = self.invokeVMCGET(t1_url) + if t1_response.status_code != 200: + self.error_handling(t1_response) + return False + t1_json = t1_response.json() + t1_lst = [] + for t in t1_json['results']: + if t['_create_user'] != 'admin': + t1_lst.append(t['id']) + if self.vpn_export is False: + self.exportVPNIKEProfiles() + self.exportVPNTunnelProfiles() + self.exportVPNDPDProfiles() + self.exportVPNBGPNeighbors() + self.exportVPNLocalBGP() + t1_vpn_service_dict = {} + t1_vpn_le_dict = {} + t1_vpn_dict = {} + for t in t1_lst: + t1_vpn_service_url = f'{self.proxy_url}/policy/api/v1/infra/tier-1s/{t}/ipsec-vpn-services' + t1_vpn_service_response = self.invokeVMCGET(t1_vpn_service_url) + if t1_vpn_service_response.status_code == 200: + t1_vpn_service_json = t1_vpn_service_response.json() + t1_vpn_service = t1_vpn_service_json['results'] + t1_vpn_service_dict[t] = t1_vpn_service + t1_vpn_service_id = t1_vpn_service[0]['id'] + else: + self.error_handling(t1_vpn_service_response) + return False + + t1_vpn_le_url = f'{self.proxy_url}/policy/api/v1/infra/tier-1s/{t}/ipsec-vpn-services/{t1_vpn_service_id}/local-endpoints' + t1_vpn_le_response = self.invokeVMCGET(t1_vpn_le_url) + if t1_vpn_le_response.status_code == 200: + t1_vpn_le_json = t1_vpn_le_response.json() + t1_vpn_le = t1_vpn_le_json['results'] + if t1_vpn_le: + t1_vpn_le_dict[t1_vpn_service_id] = t1_vpn_le + else: + pass + else: + self.error_handling(t1_vpn_le_response) + return False + + t1_vpn_url = f'{self.proxy_url}/policy/api/v1/infra/tier-1s/{t}/ipsec-vpn-services/{t1_vpn_service_id}/sessions' + t1_vpn_response = self.invokeCSPGET(t1_vpn_url) + if t1_vpn_response.status_code == 200: + t1_vpn_json = t1_vpn_response.json() + t1_vpn_json = t1_vpn_json['results'] + if t1_vpn_json: + for v in t1_vpn_json: + if self.sddc_info_hide_sensitive_data is True: + t1_vpn_dict[t1_vpn_service_id] = v + else: + t1_vpn_id = v['id'] + t1_vpn_sen_url = f'{self.proxy_url}/policy/api/v1/infra/tier-1s/{t}/ipsec-vpn-services/{t1_vpn_service_id}/sessions/{t1_vpn_id}?action=show_sensitive_data' + t1_vpn_sen_response = self.invokeCSPGET(t1_vpn_sen_url) + if t1_vpn_sen_response.status_code == 200: + t1_vpn_sen_json = t1_vpn_sen_response.json() + # print(json.dumps(t1_vpn_sen_json, indent=2)) + t1_vpn_dict[t1_vpn_service_id] = t1_vpn_sen_json + else: + self.error_handling(t1_vpn_sen_response) + return False + else: + pass + else: + self.error_handling(t1_vpn_response) + return False + + fname = self.export_path / self.tier1_vpn_service_filename + with open(fname, 'w') as outfile: + json.dump(t1_vpn_service_dict, outfile, indent=4) + + lname = self.export_path / self.tier1_vpn_le_filename + with open(lname, 'w') as lefile: + json.dump(t1_vpn_le_dict, lefile, indent=4) + + vname = f'{self.export_path}/{self.tier1_vpn_export_filename}' + with open(vname, 'w') as outfile: + json.dump(t1_vpn_dict, outfile, indent=4) + + return True + + def getVPNl3sensitivedata(self,l3vpnid): """ Retrieve sensitive data such as IPSEC preshared keys from an L3VPN configuration""" myHeader = {'csp-auth-token': self.vmc_auth.access_token} diff --git a/config_ini/config.ini b/config_ini/config.ini index 3dc812d..f1a23cb 100755 --- a/config_ini/config.ini +++ b/config_ini/config.ini @@ -43,7 +43,10 @@ nat_export_filename = natrules.json #Export VPN configuration? vpn_export = True -t1_vpn_export = False +t1_vpn_export = True +t1_vpn_export_filename = t1vpn.json +t1_vpn_service_filename = t1vpn_service.json +t1_vpn_localendpoint_filename = t1vpn_le.json # Export service access? service_access_export = True diff --git a/sddc_import_export.py b/sddc_import_export.py index eb8b1a2..5b3fe01 100755 --- a/sddc_import_export.py +++ b/sddc_import_export.py @@ -38,7 +38,7 @@ """ import boto3 import sys -MIN_PYTHON = (3,6) +MIN_PYTHON = (3,10) assert sys.version_info >= MIN_PYTHON, f"Python {'.'.join([str(n) for n in MIN_PYTHON])} or newer is required." import argparse @@ -662,6 +662,16 @@ def main(args): else: print("VPN export skipped.") + if ioObj.tier1_vpn_export is True: + print("Beginning export of Tier-1 VPNs") + retval = ioObj.export_tier1_vpn() + if retval is True: + print("Tier-1 VPNs exported") + else: + print("Tier-1 VPN export error") + else: + print("Tier-1 VPN export skipped.") + if ioObj.export_history is True: retval = ioObj.zipJSONfiles() if retval is False: From 9cd771fc9076c860897a18833294959b75143589 Mon Sep 17 00:00:00 2001 From: Chris White Date: Fri, 2 Jun 2023 08:40:59 -0400 Subject: [PATCH 09/26] Feature: IPv6 Enablement, Flex segment profile export, T1 VPN Export Signed-off-by: Chris White --- VMCImportExport.py | 63 ++++++++++++++++++++++++++++++++++++++++++- config_ini/config.ini | 11 +++++++- sddc_import_export.py | 16 ++++++++--- 3 files changed, 85 insertions(+), 5 deletions(-) diff --git a/VMCImportExport.py b/VMCImportExport.py index fe30fee..73b8b29 100755 --- a/VMCImportExport.py +++ b/VMCImportExport.py @@ -117,6 +117,7 @@ def ConfigLoader(self): self.max_export_history_files = int(config.get("exportConfig", "max_export_history_files")) self.export_type = self.loadConfigFilename(config,"exportConfig","export_type") self.import_mode_live_warning = self.loadConfigFlag(config,"importConfig","import_mode_live_warning") + self.enable_ipv6 = self.loadConfigFlag(config, 'importConfig', 'enable_ipv6') # vCenter self.srcvCenterURL = vCenterConfig.get("vCenterConfig","srcvCenterURL") @@ -210,9 +211,11 @@ def ConfigLoader(self): #Flexible Segments self.flex_segment_export = self.loadConfigFlag(config, "exportConfig", "flex_segment_export") self.flex_segment_export_filename = self.loadConfigFilename(config, "exportConfig", "flex_segment_export_filename") + self.flex_segment_disc_prof_export_filename = self.loadConfigFilename(config, 'exportConfig', 'flex_segment_disc_prof_export_filename') self.flex_segment_import = self.loadConfigFlag(config, "importConfig", "flex_segment_import") self.flex_segment_import_filename = self.loadConfigFilename(config, "importConfig", "flex_segment_import_filename") self.flex_segment_import_exclude_list = self.loadConfigRegex(config, "importConfig", "flex_segment_import_exclude_list", '|') + self.flex_segment_disc_prof_import_filename = self.loadConfigFilename(config, 'importConfig', 'flex_segment_disc_prof_import_filename') #Public IP self.public_export = self.loadConfigFlag(config,"exportConfig","public_export") @@ -699,6 +702,30 @@ def export_flexible_segments(self): with open (fname, 'w') as outfile: json.dump(flex_segments, outfile, indent=4) return True + + def export_flexible_segment_disc_bindings(self): + """Exports the MAC and IP Discovery binding maps for each flexible segment to JSON""" + flex_seg_bind = {} + flex_seg_url = f'{self.proxy_url}/policy/api/v1/infra/segments' + flex_seg_resp = self.invokeCSPGET(flex_seg_url) + flex_seg_json = flex_seg_resp.json() + flex_seg_json = flex_seg_json['results'] + flex_seg_id = [] + for f in flex_seg_json: + flex_seg_name = f['id'] + flex_seg_id.append(flex_seg_name) + + for x in flex_seg_id: + my_url = f'{self.proxy_url}/policy/api/v1/infra/segments/{x}/segment-discovery-profile-binding-maps' + response = self.invokeCSPGET(my_url) + json_response = response.json() + disc_bind_map = json_response['results'] + flex_seg_bind[x] = disc_bind_map + + fname = self.export_path / self.flex_segment_disc_prof_export_filename + with open (fname, 'w') as outfile: + json.dump(flex_seg_bind, outfile, indent=4) + return True def exportSDDCMGWRule(self): """Exports the MGW firewall rules to a JSON file""" @@ -1255,7 +1282,6 @@ def export_tier1_vpn(self): t1_vpn_sen_response = self.invokeCSPGET(t1_vpn_sen_url) if t1_vpn_sen_response.status_code == 200: t1_vpn_sen_json = t1_vpn_sen_response.json() - # print(json.dumps(t1_vpn_sen_json, indent=2)) t1_vpn_dict[t1_vpn_service_id] = t1_vpn_sen_json else: self.error_handling(t1_vpn_sen_response) @@ -2741,6 +2767,41 @@ def importSDDCPublicIPs(self): json.dump(aDict, outfile,indent=4) return aDict + def enable_sddc_ipv6(self): + """Enable IPv6 on destination SDDC if enalbed on source SDDC""" + self.vmc_auth.check_access_token_expiration() + fname = self.import_path / self.sddc_info_filename + with open(fname) as filehandle: + source_sddc_info = json.load(filehandle) + if self.import_mode == 'live': + if source_sddc_info['resource_config']['ipv6_enabled'] is True: + my_header = {"Content-Type": "application/json", "Accept": "application/json", + 'csp-auth-token': self.vmc_auth.access_token} + my_url = f'{self.strProdURL}/api/network/{self.dest_org_id}/aws/operations' + json_body = { + "type": "ENABLE_IPV6", + "resource_type": "deployment", + "resource_id": self.dest_sddc_id, + "config": { + "type": "AwsEnableIpv6Config" + } + } + response = requests.post(my_url, json=json_body, headers=my_header) + if response.status_code == 201: + print(f"Enabling IPv6 on SDDC, please wait...") + time.sleep(180) + return True + else: + self.error_handling(response) + return False + else: + print(f'IPv6 not enalbed on source SDDC and will not be enabled on destination SDDC') + return False + else: + print(f'IPv6 would have been enabled on the destination SDDC') + return + + def importVPN(self): self.vmc_auth.check_access_token_expiration() successval = True diff --git a/config_ini/config.ini b/config_ini/config.ini index f1a23cb..32a1293 100755 --- a/config_ini/config.ini +++ b/config_ini/config.ini @@ -32,6 +32,7 @@ network_dhcp_static_binding_export = False # Export segments attached to non-default Tier-1 gateways? flex_segment_export = True flex_segment_export_filename = flex_seg.json +flex_segment_disc_prof_export_filename = flex_seg_disc_prof.json # Export the list of public IP addresses? public_export = True @@ -132,10 +133,13 @@ import_mode = test # Set this to false if you are absolutely sure you have your script configured correctly and want to run it automatically import_mode_live_warning = True +# Script will attempt to enable IPv6 on the destination SDDC if it was enabled on the source SDDC +enable_ipv6 = True + # Import services? Only disable this if you truly know what you are doing. # Firewall groups are dependent on Services. If you skip Services, Groups that are # dependent on those services will fail to import -services_import = False +services_import = True # Import groups? Only disable this if you truly know what you are doing. # Firewall rules are dependent on Groups. If you skip Groups, Firewall rules @@ -197,6 +201,7 @@ flex_segment_import = True flex_segment_import_filename = flex_seg.json # Python regex match on network display name, pipe-delimited. See README for examples. flex_segment_import_exclude_list = L2E_ +flex_segment_disc_prof_import_filename = flex_seg_disc_prof.json # Import the list of public IP addresses? public_import = True @@ -209,6 +214,10 @@ nat_import_filename = natrules.json # Import VPN configuration? vpn_import = True +t1_vpn_import = False +t1_vpn_import_filename = t1vpn.json +t1_vpn_service_import_filename = t1vpn_service.json +t1_vpn_localendpoint_import_filename = t1vpn_le.json # Automatically disable VPN tunnels when importing them vpn_disable_on_import = True diff --git a/sddc_import_export.py b/sddc_import_export.py index 5b3fe01..ec0d0db 100755 --- a/sddc_import_export.py +++ b/sddc_import_export.py @@ -605,12 +605,13 @@ def main(args): if ioObj.flex_segment_export is True: print("Beginning flexible segment export...") retval = ioObj.export_flexible_segments() - if retval is True: - print("Flexible segment exported.") + retval2 = ioObj.export_flexible_segment_disc_bindings() + if retval is True and retval2 is True: + print("Flexible segment and segment discovery bindings exported.") else: print(f"Flexible segment export error: {ioObj.lastJSONResponse}") else: - print("Flexible segment export skipped.") + print("Flexible segment and segment discovery profile bindings export skipped.") if ioObj.dfw_export is True: print("Beginning DFW export...") @@ -794,6 +795,15 @@ def main(args): else: print("Live import will proceed") + if ioObj.enable_ipv6 is True: + ipv6_enable_status = ioObj.enable_sddc_ipv6() + if ipv6_enable_status is True: + print(f'IPv6 enalbed on {ioObj.dest_sddc_name}') + else: + print(f'IPv6 not enabled on {ioObj.dest_sddc_name}') + else: + print('IPv6 enablement skipped') + if ioObj.network_import is True: print("Beginning CGW network import...") import_table = ioObj.importCGWNetworks() From d148fa22a801c3eb87393182057cfdc15e0444f5 Mon Sep 17 00:00:00 2001 From: Chris White Date: Mon, 5 Jun 2023 11:34:15 -0400 Subject: [PATCH 10/26] BUG: Flex segment import did not work for disconnected segments Signed-off-by: Chris White --- VMCImportExport.py | 50 ++++++++++++++++++++++++++-------------------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/VMCImportExport.py b/VMCImportExport.py index 73b8b29..a7bb19a 100755 --- a/VMCImportExport.py +++ b/VMCImportExport.py @@ -1448,11 +1448,12 @@ def import_flex_segments(self): json_data = {} json_data['id'] = f['id'] json_data['display_name'] = f['display_name'] - json_data['connectivity_path'] = f['connectivity_path'] json_data['type'] = f['type'] json_data['resource_type'] = f['resource_type'] - json_data['subnets'] = f['subnets'] json_data['advanced_config'] = f['advanced_config'] + if f['type'] == 'ROUTED': + json_data['connectivity_path'] = f['connectivity_path'] + json_data['subnets'] = f['subnets'] uri_path = f['path'] if self.import_mode == 'live': my_header = {"Content-Type": "application/json","Accept": "application/json", 'csp-auth-token': self.vmc_auth.access_token} @@ -2774,29 +2775,34 @@ def enable_sddc_ipv6(self): with open(fname) as filehandle: source_sddc_info = json.load(filehandle) if self.import_mode == 'live': - if source_sddc_info['resource_config']['ipv6_enabled'] is True: - my_header = {"Content-Type": "application/json", "Accept": "application/json", - 'csp-auth-token': self.vmc_auth.access_token} - my_url = f'{self.strProdURL}/api/network/{self.dest_org_id}/aws/operations' - json_body = { - "type": "ENABLE_IPV6", - "resource_type": "deployment", - "resource_id": self.dest_sddc_id, - "config": { - "type": "AwsEnableIpv6Config" + dest_sddc_json = self.loadSDDCData(self.dest_org_id, self.dest_sddc_id) + if dest_sddc_json['resource_config']['ipv6_enabled'] is True: + print(f"IPv6 already enabled on {self.dest_sddc_name}...skipping") + return True + else: + if source_sddc_info['resource_config']['ipv6_enabled'] is True: + my_header = {"Content-Type": "application/json", "Accept": "application/json", + 'csp-auth-token': self.vmc_auth.access_token} + my_url = f'{self.strProdURL}/api/network/{self.dest_org_id}/aws/operations' + json_body = { + "type": "ENABLE_IPV6", + "resource_type": "deployment", + "resource_id": self.dest_sddc_id, + "config": { + "type": "AwsEnableIpv6Config" + } } - } - response = requests.post(my_url, json=json_body, headers=my_header) - if response.status_code == 201: - print(f"Enabling IPv6 on SDDC, please wait...") - time.sleep(180) - return True + response = requests.post(my_url, json=json_body, headers=my_header) + if response.status_code == 201: + print(f"Enabling IPv6 on SDDC, please wait...") + time.sleep(180) + return True + else: + self.error_handling(response) + return False else: - self.error_handling(response) + print(f'IPv6 not enalbed on source SDDC and will not be enabled on destination SDDC') return False - else: - print(f'IPv6 not enalbed on source SDDC and will not be enabled on destination SDDC') - return False else: print(f'IPv6 would have been enabled on the destination SDDC') return From 7c926d8d89e90226acfe5f64473fc0d0e4305c1f Mon Sep 17 00:00:00 2001 From: Chris White Date: Mon, 5 Jun 2023 14:17:31 -0400 Subject: [PATCH 11/26] FEATURE: Add import for Flex Segment Discovery Binding Map Signed-off-by: Chris White --- VMCImportExport.py | 36 ++++++++++++++++++++++++++++++++++++ config_ini/config.ini | 4 ++-- sddc_import_export.py | 1 + 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/VMCImportExport.py b/VMCImportExport.py index a7bb19a..1b2670f 100755 --- a/VMCImportExport.py +++ b/VMCImportExport.py @@ -1479,6 +1479,42 @@ def import_flex_segments(self): table.add_row([import_results[i]['display_name'], import_results[i]['result'], import_results[i]['result_note'], import_results[i]['id']]) return table + def import_flex_seg_disc_binding_map(self): + """Imports Segment profile binding maps for all imported flexible segments""" + self.vmc_auth.check_access_token_expiration() + fname = self.import_path / self.flex_segment_disc_prof_export_filename + try: + with open (fname) as filehandle: + binding_maps = json.load(filehandle) + except: + print(f"Import failed - unable to open {filehandle}") + return + for b in binding_maps.values(): + if b: + json_data = {} + json_data['mac_discovery_profile_path'] = b[0]['mac_discovery_profile_path'] + json_data['ip_discovery_profile_path'] = b[0]['ip_discovery_profile_path'] + json_data['resource_type'] = b[0]['resource_type'] + json_data['id'] = b[0]['id'] + json_data['display_name'] = b[0]['display_name'] + uri_path = b[0]['path'] + + if self.import_mode == 'live': + my_header = {"Content-Type": "application/json", "Accept": "application/json", + 'csp-auth-token': self.vmc_auth.access_token} + my_url = f'{self.proxy_url}/policy/api/v1{uri_path}' + response = requests.put(my_url, headers = my_header, json = json_data) + if response.status_code == 200: + print(f'Discovery binding map has been updated for segment {b[0]["parent_path"]}') + else: + self.error_handling(response) + else: + print(f'TEST MODE - Discovery binding map for segment {b[0]["parent_path"]} would have been imported') + else: + pass + + + def importCGWDHCPStaticBindings(self): self.vmc_auth.check_access_token_expiration() fname = self.import_path / self.network_dhcp_static_binding_filename diff --git a/config_ini/config.ini b/config_ini/config.ini index 32a1293..503d5ef 100755 --- a/config_ini/config.ini +++ b/config_ini/config.ini @@ -127,11 +127,11 @@ import_folder = json # # import_mode = live # - Changes will be made to the destination SDDC -import_mode = test +import_mode = live # Script will warn, ask for a confirmation before continuing in live mode # Set this to false if you are absolutely sure you have your script configured correctly and want to run it automatically -import_mode_live_warning = True +import_mode_live_warning = False # Script will attempt to enable IPv6 on the destination SDDC if it was enabled on the source SDDC enable_ipv6 = True diff --git a/sddc_import_export.py b/sddc_import_export.py index ec0d0db..b958ccc 100755 --- a/sddc_import_export.py +++ b/sddc_import_export.py @@ -912,6 +912,7 @@ def main(args): import_table = ioObj.import_flex_segments() print("Import results:\n") print(import_table) + ioObj.import_flex_seg_disc_binding_map() if ioObj.public_import is True: print("Beginning Public IP import...") From 362d9186b6c446ed14e6879553f53600f0bc9408 Mon Sep 17 00:00:00 2001 From: Chris White Date: Fri, 9 Jun 2023 08:25:03 -0400 Subject: [PATCH 12/26] BUG: Tier-1 VPN export errors if no Tier-1 VPN configured Signed-off-by: Chris White --- VMCImportExport.py | 100 +++++++++++++++++++++++---------------------- 1 file changed, 52 insertions(+), 48 deletions(-) diff --git a/VMCImportExport.py b/VMCImportExport.py index 1b2670f..0b1e70d 100755 --- a/VMCImportExport.py +++ b/VMCImportExport.py @@ -1248,61 +1248,67 @@ def export_tier1_vpn(self): if t1_vpn_service_response.status_code == 200: t1_vpn_service_json = t1_vpn_service_response.json() t1_vpn_service = t1_vpn_service_json['results'] - t1_vpn_service_dict[t] = t1_vpn_service - t1_vpn_service_id = t1_vpn_service[0]['id'] - else: - self.error_handling(t1_vpn_service_response) - return False - - t1_vpn_le_url = f'{self.proxy_url}/policy/api/v1/infra/tier-1s/{t}/ipsec-vpn-services/{t1_vpn_service_id}/local-endpoints' - t1_vpn_le_response = self.invokeVMCGET(t1_vpn_le_url) - if t1_vpn_le_response.status_code == 200: - t1_vpn_le_json = t1_vpn_le_response.json() - t1_vpn_le = t1_vpn_le_json['results'] - if t1_vpn_le: - t1_vpn_le_dict[t1_vpn_service_id] = t1_vpn_le - else: - pass - else: - self.error_handling(t1_vpn_le_response) - return False + if t1_vpn_service: + t1_vpn_service_dict[t] = t1_vpn_service + t1_vpn_service_id = t1_vpn_service[0]['id'] + + t1_vpn_le_url = f'{self.proxy_url}/policy/api/v1/infra/tier-1s/{t}/ipsec-vpn-services/{t1_vpn_service_id}/local-endpoints' + t1_vpn_le_response = self.invokeVMCGET(t1_vpn_le_url) + if t1_vpn_le_response.status_code == 200: + t1_vpn_le_json = t1_vpn_le_response.json() + t1_vpn_le = t1_vpn_le_json['results'] + if t1_vpn_le: + t1_vpn_le_dict[t1_vpn_service_id] = t1_vpn_le + else: + pass + else: + self.error_handling(t1_vpn_le_response) + return False - t1_vpn_url = f'{self.proxy_url}/policy/api/v1/infra/tier-1s/{t}/ipsec-vpn-services/{t1_vpn_service_id}/sessions' - t1_vpn_response = self.invokeCSPGET(t1_vpn_url) - if t1_vpn_response.status_code == 200: - t1_vpn_json = t1_vpn_response.json() - t1_vpn_json = t1_vpn_json['results'] - if t1_vpn_json: - for v in t1_vpn_json: - if self.sddc_info_hide_sensitive_data is True: - t1_vpn_dict[t1_vpn_service_id] = v + t1_vpn_url = f'{self.proxy_url}/policy/api/v1/infra/tier-1s/{t}/ipsec-vpn-services/{t1_vpn_service_id}/sessions' + t1_vpn_response = self.invokeCSPGET(t1_vpn_url) + if t1_vpn_response.status_code == 200: + t1_vpn_json = t1_vpn_response.json() + t1_vpn_json = t1_vpn_json['results'] + if t1_vpn_json: + for v in t1_vpn_json: + if self.sddc_info_hide_sensitive_data is True: + t1_vpn_dict[t1_vpn_service_id] = v + else: + t1_vpn_id = v['id'] + t1_vpn_sen_url = f'{self.proxy_url}/policy/api/v1/infra/tier-1s/{t}/ipsec-vpn-services/{t1_vpn_service_id}/sessions/{t1_vpn_id}?action=show_sensitive_data' + t1_vpn_sen_response = self.invokeCSPGET(t1_vpn_sen_url) + if t1_vpn_sen_response.status_code == 200: + t1_vpn_sen_json = t1_vpn_sen_response.json() + t1_vpn_dict[t1_vpn_service_id] = t1_vpn_sen_json + else: + self.error_handling(t1_vpn_sen_response) + return False else: - t1_vpn_id = v['id'] - t1_vpn_sen_url = f'{self.proxy_url}/policy/api/v1/infra/tier-1s/{t}/ipsec-vpn-services/{t1_vpn_service_id}/sessions/{t1_vpn_id}?action=show_sensitive_data' - t1_vpn_sen_response = self.invokeCSPGET(t1_vpn_sen_url) - if t1_vpn_sen_response.status_code == 200: - t1_vpn_sen_json = t1_vpn_sen_response.json() - t1_vpn_dict[t1_vpn_service_id] = t1_vpn_sen_json - else: - self.error_handling(t1_vpn_sen_response) - return False + pass + else: + self.error_handling(t1_vpn_response) + return False else: pass else: - self.error_handling(t1_vpn_response) + self.error_handling(t1_vpn_service_response) return False - fname = self.export_path / self.tier1_vpn_service_filename - with open(fname, 'w') as outfile: - json.dump(t1_vpn_service_dict, outfile, indent=4) + if t1_vpn_service_dict: + fname = self.export_path / self.tier1_vpn_service_filename + with open(fname, 'w') as outfile: + json.dump(t1_vpn_service_dict, outfile, indent=4) - lname = self.export_path / self.tier1_vpn_le_filename - with open(lname, 'w') as lefile: - json.dump(t1_vpn_le_dict, lefile, indent=4) + if t1_vpn_le_dict: + lname = self.export_path / self.tier1_vpn_le_filename + with open(lname, 'w') as lefile: + json.dump(t1_vpn_le_dict, lefile, indent=4) - vname = f'{self.export_path}/{self.tier1_vpn_export_filename}' - with open(vname, 'w') as outfile: - json.dump(t1_vpn_dict, outfile, indent=4) + if t1_vpn_dict: + vname = f'{self.export_path}/{self.tier1_vpn_export_filename}' + with open(vname, 'w') as outfile: + json.dump(t1_vpn_dict, outfile, indent=4) return True @@ -1513,8 +1519,6 @@ def import_flex_seg_disc_binding_map(self): else: pass - - def importCGWDHCPStaticBindings(self): self.vmc_auth.check_access_token_expiration() fname = self.import_path / self.network_dhcp_static_binding_filename From 38b27011f64716fdf9aeab506ec8f57113ab0173 Mon Sep 17 00:00:00 2001 From: Chris White Date: Fri, 9 Jun 2023 11:15:00 -0400 Subject: [PATCH 13/26] BUG: Tier-1 gateway test import does not behave correctly Signed-off-by: Chris White --- VMCImportExport.py | 177 ++++++++++++++++++++++++--------------------- 1 file changed, 96 insertions(+), 81 deletions(-) diff --git a/VMCImportExport.py b/VMCImportExport.py index 0b1e70d..7d0310e 100755 --- a/VMCImportExport.py +++ b/VMCImportExport.py @@ -1601,6 +1601,7 @@ def importSDDCServices(self): else: print("TEST MODE - Service",service["display_name"],"would have been imported.") + def import_mcgw(self): """Import Tier-1 gateways from a JSON file""" self.vmc_auth.check_access_token_expiration() @@ -1613,24 +1614,27 @@ def import_mcgw(self): return for mcgw in mcgws.values(): json_data = {} + json_data['id'] = mcgw['id'] + json_data['display_name'] = mcgw['display_name'] + json_data['type'] = mcgw['type'] + if 'dhcp_config_paths' in mcgw: + json_data['dhcp_config_paths'] = mcgw['dhcp_config_paths'] if self.import_mode == "live": - json_data['id'] = mcgw['id'] - json_data['display_name'] = mcgw['display_name'] - json_data['type'] = mcgw['type'] - if 'dhcp_config_paths' in mcgw: - json_data['dhcp_config_paths'] = mcgw['dhcp_config_paths'] - my_header = {"Content-Type": "application/json", "Accept": "application/json", "csp-auth-token": self.vmc_auth.access_token} - my_url = self.proxy_url + '/policy/api/v1/infra/tier-1s/' + mcgw['id'] - if self.sync_mode is True: - response = requests.patch(my_url, headers=my_header, json=json_data) - else: - response = requests.put(my_url, headers=my_header, json=json_data) - if response.status_code == 200: - result = "SUCCESS" - print('Added {}'.format(json_data['display_name'])) + my_header = {"Content-Type": "application/json", "Accept": "application/json", "csp-auth-token": self.vmc_auth.access_token} + my_url = self.proxy_url + '/policy/api/v1/infra/tier-1s/' + mcgw['id'] + if self.sync_mode is True: + response = requests.patch(my_url, headers=my_header, json=json_data) + else: + response = requests.put(my_url, headers=my_header, json=json_data) + if response.status_code == 200: + result = "SUCCESS" + print('Added {}'.format(json_data['display_name'])) + else: + result = "FAIL" + self.error_handling(response) else: - result = "FAIL" - print(f'API Call Status {response.status_code}, text:{response.text}') + print(f'TEST MODE - {mcgw["id"]} would have been imported') + def import_mcgw_static_routes(self): """Import Tier-1 Gateway static routes from a JSON file""" @@ -1642,18 +1646,20 @@ def import_mcgw_static_routes(self): except: print(f'Import failed - unable to open {fname}') return - if self.import_mode == 'live': - for route in routes.values(): - for r in route['results']: - json_data = {} - json_data['display_name'] = r['display_name'] - json_data['id'] = r['id'] - json_data['network'] = r['network'] - json_data['next_hops'] = r['next_hops'] - json_data['resource_type'] = r['resource_type'] - path = r['path'] + + for route in routes.values(): + for r in route['results']: + + json_data = {} + json_data['display_name'] = r['display_name'] + json_data['id'] = r['id'] + json_data['network'] = r['network'] + json_data['next_hops'] = r['next_hops'] + json_data['resource_type'] = r['resource_type'] + path = r['path'] + if self.import_mode == 'live': my_header = {"Content-Type": "application/json", "Accept": "application/json", - "csp-auth-token": self.vmc_auth.access_token} + "csp-auth-token": self.vmc_auth.access_token} my_url = f'{self.proxy_url}/policy/api/v1{path}' if self.sync_mode is True: response = requests.patch(my_url, headers=my_header, json=json_data) @@ -1664,9 +1670,10 @@ def import_mcgw_static_routes(self): print('Added {}'.format(json_data['display_name'])) else: result = "FAIL" - print(f'API Call Status {response.status_code}, text:{response.text}') - else: - print(f"TEST MODE - Tier 1 Gateway static routes would have been imported.") + self.error_handling(response) + else: + print(f"TEST MODE - Tier 1 Gateway static routes for {r['id']} would have been imported.") + def import_mcgw_fw(self): """Import Tier-1 Gateway firewall policies and rules from a JSON file""" @@ -1678,16 +1685,17 @@ def import_mcgw_fw(self): except: print(f'Import failed - unable to open {fname}') return - if self.import_mode == 'live': - for policy in rules.values(): - # print(json.dumps(policy, indent=2)) - # import and create the top level firewall policy - json_policy_data = {} - json_policy_data['resource_type'] = policy['resource_type'] - json_policy_data['id'] = policy['id'] - json_policy_data['display_name'] = policy['display_name'] - json_policy_data['category'] = policy['category'] - path = policy['path'] + + for policy in rules.values(): + # print(json.dumps(policy, indent=2)) + # import and create the top level firewall policy + json_policy_data = {} + json_policy_data['resource_type'] = policy['resource_type'] + json_policy_data['id'] = policy['id'] + json_policy_data['display_name'] = policy['display_name'] + json_policy_data['category'] = policy['category'] + path = policy['path'] + if self.import_mode == 'live': my_header = {"Content-Type": "application/json", "Accept": "application/json", "csp-auth-token": self.vmc_auth.access_token} my_url = f'{self.proxy_url}/policy/api/v1{path}' if self.sync_mode is True: @@ -1699,7 +1707,7 @@ def import_mcgw_fw(self): print(f'Added {json_policy_data["id"]} firewall policy') else: result = "FAIL" - print(f'API Call Status {response.status_code}, text:{response.text}') + self.error_handling(response) json_rule_data = {} rules = policy['rules'] for r in rules: @@ -1718,7 +1726,7 @@ def import_mcgw_fw(self): json_rule_data['tag'] = r['tag'] path = r['path'] my_header = {"Content-Type": "application/json", "Accept": "application/json", - "csp-auth-token": self.vmc_auth.access_token} + "csp-auth-token": self.vmc_auth.access_token} my_url = f'{self.proxy_url}/policy/api/v1{path}' if self.sync_mode is True: response = requests.patch(my_url, headers=my_header, json=json_rule_data) @@ -1729,9 +1737,9 @@ def import_mcgw_fw(self): print(f'Added {json_rule_data["display_name"]} firewall rule') else: result = "FAIL" - print(f'API Call Status {response.status_code}, text:{response.text}') - else: - print(f"TEST MODE - Tier 1 Gateway firewall policy and rules would have been imported.") + self.error_handling(response) + else: + print(f"TEST MODE - Tier 1 Gateway {policy['id']} firewall policy and rules would have been imported.") def import_mpl(self): @@ -1744,22 +1752,26 @@ def import_mpl(self): except: print(f'Import failed - unable to open {fname}') return - if self.import_mode == 'live': - for m in mpl: + + for m in mpl: if m['linked_vpc_managed_prefix_list_info']['managed_prefix_list_mode'] == 'ENABLED': vpc_id = m['linked_vpc_id'] - my_header = {"Content-Type": "application/json", "Accept": "application/json", "csp-auth-token": self.vmc_auth.access_token} - my_url = f'{self.proxy_url}/cloud-service/api/v1/linked-vpcs/{vpc_id}?action=enable_managed_prefix_list_mode' - response = requests.post(my_url, headers=my_header) - if response.status_code == 200: - result = "SUCCESS" - print('Enabled Managed Prefix List Mode. Proceed to the AWS Management Console and Resource Access Manager to accept the share') + if self.import_mode == 'live': + my_header = {"Content-Type": "application/json", "Accept": "application/json", "csp-auth-token": self.vmc_auth.access_token} + my_url = f'{self.proxy_url}/cloud-service/api/v1/linked-vpcs/{vpc_id}?action=enable_managed_prefix_list_mode' + response = requests.post(my_url, headers=my_header) + if response.status_code == 200: + result = "SUCCESS" + print('Enabled Managed Prefix List Mode. Proceed to the AWS Management Console and Resource Access Manager to accept the share') + else: + self.error_handling(response) + result = "FAIL" else: - self.error_handling(response) + print("TEST MODE - Connected VPC Managed Prefix List mode would have been enabled") result = "FAIL" - else: - print("TEST MODE - Connected VPC Managed Prefix List mode would have been enabled") - + else: + print(f'Source SDDC did not have Managed Prefix List enabled...skipping') + pass def import_ral(self): @@ -1772,15 +1784,15 @@ def import_ral(self): except: print(f'Import failed - unable to open {fname}') return - if self.import_mode == 'live': - for r in ral: - json_data = {} - json_data['display_name'] = r['display_name'] - json_data['prefixes'] = r['prefixes'] - json_data['resource_type'] = r['resource_type'] - json_data['id'] = r['id'] - path = r['path'] - # print(json.dumps(json_data, indent=2)) + for r in ral: + json_data = {} + json_data['display_name'] = r['display_name'] + json_data['prefixes'] = r['prefixes'] + json_data['resource_type'] = r['resource_type'] + json_data['id'] = r['id'] + path = r['path'] + # print(json.dumps(json_data, indent=2)) + if self.import_mode == 'live': my_header = {"Content-Type": "application/json", "Accept": "application/json", "csp-auth-token": self.vmc_auth.access_token} my_url = f'{self.proxy_url}/cloud-service/api/v1{path}' response = requests.put(my_url, headers=my_header, json=json_data) @@ -1789,9 +1801,10 @@ def import_ral(self): print(f'Added {json_data["display_name"]} route aggregation list') else: result = "FAIL" - print(f'API Call Status {response.status_code}, text:{response.text}') - else: - print(f'TEST Mode - Route Aggregation lists would have been imported') + self.error_handling(response) + else: + print(f'TEST Mode - Route Aggregation lists {json_data["id"]} would have been imported') + def import_route_config(self): """Imports SDDC route configuration from JSON""" @@ -1803,16 +1816,17 @@ def import_route_config(self): except: print(f'Import failed - unable to open {fname}') return - if self.import_mode == 'live': - for r in config: - json_data = {} - json_data['display_name'] = r['display_name'] - json_data['resource_type'] = r['resource_type'] - json_data['id'] = r['id'] - json_data['aggregation_route_config'] = r['aggregation_route_config'] - json_data['connectivity_endpoint_path'] = r['connectivity_endpoint_path'] + dest_sddc = self.loadSDDCData(self.dest_org_id, self.dest_sddc_id) + for r in config: + json_data = {} + json_data['display_name'] = r['display_name'] + json_data['resource_type'] = r['resource_type'] + json_data['id'] = r['id'] + json_data['aggregation_route_config'] = r['aggregation_route_config'] + json_data['connectivity_endpoint_path'] = r['connectivity_endpoint_path'] + if self.import_mode == 'live': my_header = {"Content-Type": "application/json", "Accept": "application/json", - "csp-auth-token": self.vmc_auth.access_token} + "csp-auth-token": self.vmc_auth.access_token} my_url = f'{self.proxy_url}/cloud-service/api/v1/infra/external/route/configs/{r["id"]}' response = requests.put(my_url, headers=my_header, json=json_data) if response.status_code == 200: @@ -1820,9 +1834,10 @@ def import_route_config(self): print(f'Added {json_data["display_name"]} route configuration') else: result = "FAIL" - print(f'API Call Status {response.status_code}, text:{response.text}') - else: - print(f'TEST Mode - Route configuration would have been imported') + self.error_handling(response) + else: + print(f'TEST Mode - Route configuration {json_data["id"]} would have been imported') + def convertServiceRolePayload(self, sourcePayload: str) -> bool: """Converts a ServiceRole payload from its default format to the format required to add it to a User. Saves results to convertedServiceRolePayload """ From 89bfbac2beee83e1d4bd8bce70cf6d24a9257fc7 Mon Sep 17 00:00:00 2001 From: Chris White Date: Tue, 13 Jun 2023 10:09:30 -0400 Subject: [PATCH 14/26] BUG: Route agg list now handles IPv6 appropriately Signed-off-by: Chris White --- Dockerfile | 2 +- VMCImportExport.py | 6 ++++-- config_ini/config.ini | 8 ++++---- sddc_import_export.py | 2 +- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index deee0d9..3dccb72 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM bitnami/python:3.9.5-prod +FROM bitnami/python:latest WORKDIR /tmp/scripts diff --git a/VMCImportExport.py b/VMCImportExport.py index 7d0310e..c777649 100755 --- a/VMCImportExport.py +++ b/VMCImportExport.py @@ -1791,7 +1791,10 @@ def import_ral(self): json_data['resource_type'] = r['resource_type'] json_data['id'] = r['id'] path = r['path'] - # print(json.dumps(json_data, indent=2)) + if 'address_family' in r: + json_data['address_family'] = r['address_family'] + else: + json_data['address_family'] = 'IPv4' if self.import_mode == 'live': my_header = {"Content-Type": "application/json", "Accept": "application/json", "csp-auth-token": self.vmc_auth.access_token} my_url = f'{self.proxy_url}/cloud-service/api/v1{path}' @@ -1816,7 +1819,6 @@ def import_route_config(self): except: print(f'Import failed - unable to open {fname}') return - dest_sddc = self.loadSDDCData(self.dest_org_id, self.dest_sddc_id) for r in config: json_data = {} json_data['display_name'] = r['display_name'] diff --git a/config_ini/config.ini b/config_ini/config.ini index 503d5ef..e6e6c27 100755 --- a/config_ini/config.ini +++ b/config_ini/config.ini @@ -50,7 +50,7 @@ t1_vpn_service_filename = t1vpn_service.json t1_vpn_localendpoint_filename = t1vpn_le.json # Export service access? -service_access_export = True +service_access_export = False # Export the distributed firewall? dfw_export = True @@ -127,11 +127,11 @@ import_folder = json # # import_mode = live # - Changes will be made to the destination SDDC -import_mode = live +import_mode = test # Script will warn, ask for a confirmation before continuing in live mode # Set this to false if you are absolutely sure you have your script configured correctly and want to run it automatically -import_mode_live_warning = False +import_mode_live_warning = True # Script will attempt to enable IPv6 on the destination SDDC if it was enabled on the source SDDC enable_ipv6 = True @@ -214,7 +214,7 @@ nat_import_filename = natrules.json # Import VPN configuration? vpn_import = True -t1_vpn_import = False +t1_vpn_import = True t1_vpn_import_filename = t1vpn.json t1_vpn_service_import_filename = t1vpn_service.json t1_vpn_localendpoint_import_filename = t1vpn_le.json diff --git a/sddc_import_export.py b/sddc_import_export.py index b958ccc..ce85e97 100755 --- a/sddc_import_export.py +++ b/sddc_import_export.py @@ -798,7 +798,7 @@ def main(args): if ioObj.enable_ipv6 is True: ipv6_enable_status = ioObj.enable_sddc_ipv6() if ipv6_enable_status is True: - print(f'IPv6 enalbed on {ioObj.dest_sddc_name}') + print(f'IPv6 enabled on {ioObj.dest_sddc_name}') else: print(f'IPv6 not enabled on {ioObj.dest_sddc_name}') else: From 19f9f694e1b09cefb26bed6d71edcd4979bf575b Mon Sep 17 00:00:00 2001 From: Chris White Date: Mon, 26 Jun 2023 14:19:18 -0400 Subject: [PATCH 15/26] FEATURE: Add tag support for various NSX objects Signed-off-by: Chris White --- VMCImportExport.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/VMCImportExport.py b/VMCImportExport.py index c777649..f9db367 100755 --- a/VMCImportExport.py +++ b/VMCImportExport.py @@ -1398,6 +1398,9 @@ def importCGWNetworks(self): resultNote += "No subnets found." if "advanced_config" in n: json_data["advanced_config"] = n["advanced_config"] + if 'tags' in n: + json_data['tags'] = n['tags'] + if self.import_mode == "live": myHeader = {"Content-Type": "application/json","Accept": "application/json", 'csp-auth-token': self.vmc_auth.access_token } myURL = (self.proxy_url + "/policy/api/v1/infra/tier-1s/cgw/segments/" + n['id']) @@ -1457,6 +1460,8 @@ def import_flex_segments(self): json_data['type'] = f['type'] json_data['resource_type'] = f['resource_type'] json_data['advanced_config'] = f['advanced_config'] + if 'tags' in f: + json_data['tags'] = f['tags'] if f['type'] == 'ROUTED': json_data['connectivity_path'] = f['connectivity_path'] json_data['subnets'] = f['subnets'] @@ -1571,6 +1576,9 @@ def importSDDCServices(self): json_data["resource_type"]=service["resource_type"] json_data["display_name"]=service["display_name"] json_data["service_type"]=service["service_type"] + if 'tags' in service: + json_data['tags'] = service['tags'] + service_entries = [] for entry in service["service_entries"]: #print("-------------------------------------------------") @@ -1619,6 +1627,9 @@ def import_mcgw(self): json_data['type'] = mcgw['type'] if 'dhcp_config_paths' in mcgw: json_data['dhcp_config_paths'] = mcgw['dhcp_config_paths'] + if 'tags' in mcgw: + json_data['tags'] = mcgw['tags'] + if self.import_mode == "live": my_header = {"Content-Type": "application/json", "Accept": "application/json", "csp-auth-token": self.vmc_auth.access_token} my_url = self.proxy_url + '/policy/api/v1/infra/tier-1s/' + mcgw['id'] @@ -2224,6 +2235,9 @@ def importSDDCCGWGroup(self): payload["id"]=group["id"] payload["resource_type"]=group["resource_type"] payload["display_name"]=group["display_name"] + if 'tags' in group: + payload['tags'] = group['tags'] + if self.import_mode == "live": myHeader = {"Content-Type": "application/json","Accept": "application/json", 'csp-auth-token': self.vmc_auth.access_token } myURL = self.proxy_url + "/policy/api/v1/infra/domains/cgw/groups/" + group["id"] @@ -2688,6 +2702,8 @@ def importSDDCMGWGroup(self): payload["id"]=group["id"] payload["resource_type"]=group["resource_type"] payload["display_name"]=group["display_name"] + if 'tags' in group: + payload['tags'] = group['tags'] if "expression" in group: payload["expression"]=group["expression"] if self.import_mode == "live": From afe54db663c941ba1aecbb441af2658b4c2d0923 Mon Sep 17 00:00:00 2001 From: Chris White Date: Wed, 28 Jun 2023 12:41:48 -0400 Subject: [PATCH 16/26] FEATURE: Implement boto3 to automate linked VPC resource share acceptance and linked VPC route table programming Signed-off-by: Chris White --- VMCImportExport.py | 90 ++++++++++++++++++++++++++++++++++++++++++- config_ini/aws.ini | 7 +++- config_ini/config.ini | 3 ++ 3 files changed, 98 insertions(+), 2 deletions(-) diff --git a/VMCImportExport.py b/VMCImportExport.py index f9db367..e8ac933 100755 --- a/VMCImportExport.py +++ b/VMCImportExport.py @@ -15,6 +15,7 @@ import re import time import sys +import boto3 from pathlib import Path from prettytable import PrettyTable @@ -76,6 +77,9 @@ def __init__(self,configPath="./config_ini/config.ini", vmcConfigPath="./config_ self.convertedServiceRolePayload = "" self.RoleSyncSourceUserEmail = "" self.RoleSyncDestUserEmails = {} + self.aws_import_access_key_id = "" + self.aws_import_secret_access_key = "" + self.aws_import_session_token = "" self.ConfigLoader() def ConfigLoader(self): @@ -185,6 +189,8 @@ def ConfigLoader(self): self.mpl_export_filename = self.loadConfigFilename(config, 'exportConfig', 'mpl_export_filename') self.mpl_import = self.loadConfigFlag(config, 'importConfig', 'mpl_import') self.mpl_import_filename = self.loadConfigFilename(config, 'importConfig', 'mpl_import_filename') + self.automate_ram_acceptance = self.loadConfigFlag(config, 'importConfig', 'automate_ram_acceptance') + self.automate_vpc_route_table_programming = self.loadConfigFlag(config, 'importConfig', 'automate_vpc_route_table_programming') #SDDC Route Aggregation Lists and Route Configurations self.ral_export = self.loadConfigFlag(config, "exportConfig", "ral_export") @@ -264,6 +270,9 @@ def ConfigLoader(self): self.aws_s3_export_access_id = self.loadConfigFilename(awsConfig,"awsConfig","aws_s3_export_access_id") self.aws_s3_export_access_secret = self.loadConfigFilename(awsConfig,"awsConfig","aws_s3_export_access_secret") self.aws_s3_export_bucket = self.loadConfigFilename(awsConfig,"awsConfig","aws_s3_export_bucket") + self.aws_import_access_key_id = self.loadConfigFilename(awsConfig, 'awsConfig', 'aws_import_access_key_id') + self.aws_import_secret_access_key = self.loadConfigFilename(awsConfig, 'awsConfig', 'aws_import_secret_access_key') + self.aws_import_session_token = self.loadConfigFilename(awsConfig, 'awsConfig', 'aws_import_session_token') #DFW self.dfw_export = self.loadConfigFlag(config,"exportConfig","dfw_export") @@ -1773,7 +1782,30 @@ def import_mpl(self): response = requests.post(my_url, headers=my_header) if response.status_code == 200: result = "SUCCESS" - print('Enabled Managed Prefix List Mode. Proceed to the AWS Management Console and Resource Access Manager to accept the share') + print('Enabling Managed Prefix List Mode.') + if self.automate_ram_acceptance is True: + time.sleep(20) + ram_response = self.aws_ram_accept(vpc_id) + if ram_response is True: + linked_vpn_url = f'{self.proxy_url}/cloud-service/api/v1/linked-vpcs/{vpc_id}' + vpc_resp_json = self.invokeCSPGET(linked_vpn_url) + vpc_resp = vpc_resp_json.json() + mpl_status = vpc_resp['linked_vpc_managed_prefix_list_info']['managed_prefix_list_mode'] + while mpl_status != "ENABLED": + time.sleep(20) + print(f'Waiting for MPL to be enabled') + vpc_resp_json = self.invokeCSPGET(linked_vpn_url) + vpc_resp = vpc_resp_json.json() + mpl_status = vpc_resp['linked_vpc_managed_prefix_list_info']['managed_prefix_list_mode'] + if mpl_status == "ENABLED" and self.automate_vpc_route_table_programming is True: + rt_lst = m['linked_vpc_managed_prefix_list_info']['managed_prefix_lists'][0]['programming_info']['route_table_ids'] + active_eni = m['linked_vpc_managed_prefix_list_info']['managed_prefix_lists'][0]['programming_info']['active_eni'] + prefix_list_id = m['linked_vpc_managed_prefix_list_info']['managed_prefix_lists'][0]['id'] + vpc_resp = self.vpc_rt_prog(rt_lst, active_eni, prefix_list_id) + else: + print('Resourse share acceptance failed. Continuing with import, please check the AWS Management Console') + else: + print('Proceed to the AWS Management Console and the Resource Access Manager service to accept the share') else: self.error_handling(response) result = "FAIL" @@ -1785,6 +1817,62 @@ def import_mpl(self): pass + def aws_ram_accept(self, vpc_id): + """Function to accept Resource Access Share""" + self.vmc_auth.check_access_token_expiration() + my_header = {"Content-Type": "application/json", "Accept": "application/json", "csp-auth-token": self.vmc_auth.access_token} + my_url = f'{self.proxy_url}/cloud-service/api/v1/linked-vpcs/{vpc_id}' + response = requests.get(my_url, headers=my_header) + json_response = response.json() + ram_arn = [] + ram_arn.append(json_response['linked_vpc_managed_prefix_list_info']['aws_resource_share_info']['aws_resource_share_arn']) + + if self.aws_import_session_token: + session = boto3.Session(aws_access_key_id=self.aws_import_access_key_id, aws_secret_access_key=self.aws_import_secret_access_key, aws_session_token=self.aws_import_session_token) + else: + session = boto3.Session(aws_access_key_id=self.aws_import_access_key_id, aws_secret_access_key=self.aws_import_secret_access_key) + + ram = session.client('ram') + + ram_share_arn = ram.get_resource_share_invitations(resourceShareArns=ram_arn) + ram_arn_id = ram_share_arn['resourceShareInvitations'][0]['resourceShareInvitationArn'] + ram_response = ram.accept_resource_share_invitation(resourceShareInvitationArn=ram_arn_id) + + ram_status = ram_response['resourceShareInvitation']['status'] + match ram_status: + case 'ACCEPTED': + print (f'Resource Share {ram_response["resourceShareInvitation"]["resourceShareName"]} accepted. Managed Prefix List has been accepted.') + return True + case 'REJECTED': + print (f'Resource Share {ram_response["resourceShareInvitation"]["resourceShareName"]} has been rejected. Please check your AWS credentials.') + return False + case 'EXPIRED': + print (f'The resource share {ram_response["resourceShareInvitation"]["resourceShareName"]} has expired. Please try again.') + return False + case other: + print (f'An unknown error has occured.') + return False + + + def vpc_rt_prog(self, rt_lst, active_eni, prefix_list_id): + """Function to program linked VPC route tables""" + self.vmc_auth.check_access_token_expiration() + + if self.aws_import_session_token: + session = boto3.Session(aws_access_key_id=self.aws_import_access_key_id, aws_secret_access_key=self.aws_import_secret_access_key, aws_session_token=self.aws_import_session_token) + else: + session = boto3.Session(aws_access_key_id=self.aws_import_access_key_id, aws_secret_access_key=self.aws_import_secret_access_key) + + vpc = session.client('ec2') + + for r in rt_lst: + response = vpc.create_route(RouteTableId=r, DestinationPrefixListId=prefix_list_id, NetworkInterfaceId=active_eni) + if response['Return'] == True: + print(f'VPC Route Table {r} has been programmed with MPL') + else: + print('Route application failure') + + def import_ral(self): """Import SDDC Route Aggregation lists from JSON""" self.vmc_auth.check_access_token_expiration() diff --git a/config_ini/aws.ini b/config_ini/aws.ini index e4c2798..c0128a5 100644 --- a/config_ini/aws.ini +++ b/config_ini/aws.ini @@ -7,4 +7,9 @@ aws_s3_export_access_id = aws_s3_export_access_secret = # Populate this with the S3 bucket name. The sample Lambda code uses a command line parameter to override this configuration -aws_s3_export_bucket = \ No newline at end of file +aws_s3_export_bucket = + +#AWS credntials for Connected VPC Configuraton +aws_import_access_key_id= +aws_import_secret_access_key= +aws_import_session_token= \ No newline at end of file diff --git a/config_ini/config.ini b/config_ini/config.ini index e6e6c27..a666719 100755 --- a/config_ini/config.ini +++ b/config_ini/config.ini @@ -177,6 +177,9 @@ mcgw_fw_import_filename = mcgw_fw.json # Import Conncted VPC Managed Prefix List mpl_import = True mpl_import_filename = mpl.json +#Automatically accept the Resource Share. Users must provide an AWS Access Key and Secret Access Key in the aws.ini file +automate_ram_acceptance = True +automate_vpc_route_table_programming = True #Import Route Aggregation Lists and Route Configuration? ral_import = True From 448b5e58aee47c80c527768da4febdfa81e61ade Mon Sep 17 00:00:00 2001 From: Tom Twyman Date: Thu, 6 Jul 2023 12:29:57 -0400 Subject: [PATCH 17/26] Added import and export for NSX Advanced Firewall Signed-off-by: Tom Twyman --- .DS_Store | Bin 8196 -> 8196 bytes .gitignore | 1 + CODE-OF-CONDUCT.md | 0 CONTRIBUTING.md | 0 Dockerfile | 0 LICENSE.txt | 0 NOTICE.txt | 0 VMCImportExport.py | 404 +++++++++++++++++- api-test.py | 0 config_ini/.gitignore | 4 + config_ini/aws.ini | 0 config_ini/config.ini | 96 +++-- config_ini/config.ini.vcdr.sample | 0 config_ini/vcenter.ini | 0 config_ini/vmc.ini | 12 +- invoke_lambda.py | 0 json/.gitignore | 0 .../Export_NSX-T_FW_config_from_an_SDDC.py | 0 reference/Export_SDDC_config.py | 0 reference/Import_NSX-T_FW_config.py | 0 reference/Import_SDDC_config.py | 0 reference/README.md | 0 reference/pyVMC.py | 0 requirements.txt | 0 sddc_import_export.py | 18 +- testvmc.py | 0 vcenter.py | 0 vmc.py | 0 vmc_auth.py | 0 29 files changed, 466 insertions(+), 69 deletions(-) mode change 100644 => 100755 .DS_Store mode change 100644 => 100755 .gitignore mode change 100644 => 100755 CODE-OF-CONDUCT.md mode change 100644 => 100755 CONTRIBUTING.md mode change 100644 => 100755 Dockerfile mode change 100644 => 100755 LICENSE.txt mode change 100644 => 100755 NOTICE.txt mode change 100644 => 100755 api-test.py create mode 100755 config_ini/.gitignore mode change 100644 => 100755 config_ini/aws.ini mode change 100644 => 100755 config_ini/config.ini.vcdr.sample mode change 100644 => 100755 config_ini/vcenter.ini mode change 100644 => 100755 config_ini/vmc.ini mode change 100644 => 100755 invoke_lambda.py mode change 100644 => 100755 json/.gitignore mode change 100644 => 100755 reference/Export_NSX-T_FW_config_from_an_SDDC.py mode change 100644 => 100755 reference/Export_SDDC_config.py mode change 100644 => 100755 reference/Import_NSX-T_FW_config.py mode change 100644 => 100755 reference/Import_SDDC_config.py mode change 100644 => 100755 reference/README.md mode change 100644 => 100755 reference/pyVMC.py mode change 100644 => 100755 requirements.txt mode change 100644 => 100755 testvmc.py mode change 100644 => 100755 vcenter.py mode change 100644 => 100755 vmc.py mode change 100644 => 100755 vmc_auth.py diff --git a/.DS_Store b/.DS_Store old mode 100644 new mode 100755 index b03f85c0862ca7432f5d524968a93b4e51a169db..bcf3577bf1886c4939c3e8eb50ed02061d3b642d GIT binary patch literal 8196 zcmeHM%Wl&^6upz0CLyYn1&<0L)xZXcP!gmHDuiT0_y8JI78Hdz&con%LY|Z;k~b_` zHY|Y^-@t}1V2fC>V}ZmD36S^!?mQBWiQRUEikPWp&Sd7?$(b{*-RlAXDGr|^^Vjxh21ChxGk>5;YhC=wakd{OQ0p)S!yB1$8zOQehb1i4UzU+?k&*Qq0A>#plN zbX24R-|6w#vY1GxQ|U_fRQgsPjcpX{c=8M7V_(BlSw*t{M@B$8q&v88UoW4BmPtPfGP4$%a{OgR{ZVrby+s8lZ0E@%7#5MrN_9_^A1iVDbi+hBS2XvTtBz z=;I|;4=eb!70pt$tUP=EQC|EJ>#3^tW`^`7@=RX_hFsSWF5(5>#$VSuJb{j?adGFz z`FN(P^XmqEDQ_Gx>#AD!XRCUXrvenApZ{orF4}@Jz4CXMdPEbESQuUi(vh zW^CTKf$}5d3KFEi1`CeT+@p}7+5#h)cQXDwv&oEUIBrwZH7qNySh94PAV`8#DLob+ zv-5FRb#qRKnjK6FNs5>`7ByI~DSW)*cFuSw2d?bJ0&v$0z=_gw zrRx!C|A3LQt+;VgPsmjc;nqdBy;*&nFzQd)W-e_x3As9@nyH+rx_Y~5GGk}J)sMTZ zb=a}9Zl`CMkma1CAJECp7}sNDO|GlhNIeS;RBM9Z zb#L0NNPP~iRaSk�W6cZ{eaI}`#7Jkl&F2Te7po3bWyx3(RU(m&3ZEI;aZ?%X9Q z1N22KO-ppDvC;32wkygAn|Z(LxIG!&qNv`d>Als}D*FUskZIdBjiRcLZsz!TFo-fc zDHW)LR@eo-FbEpt;Vhhk7vNQxhRg6ayayk_C-50ugX{1Wd<);h&+rTU3cuk}tituU z5o)D1nfPm0}E>VCe0 z^YxC^Yc|$z-qO5dSNq()F51cYl=67QiQ*BcKoD;$7qF7;y%Jg{Z79X;gTz1mxWGsx zFXPtXP@O=$kv-xlp@z^VfygbFd0Oe75Lv2%T+WTRNfCj_Dlg&MR;f)SI?EMYd%v_( zAlAxDxpt2f71;wP1HW$w#Y92I#at}O#5dp~%)p26F){E4d=1~hPY5XBO04nNSBLf3 zfLn1JMz9%MaX0QE9=2mA?#FKI!CvCx5Dw!ACQv0V9`m@E!4r5APvInS@>x7XjC>BC z_c?hUFW~Dqy@-Kb1@3W)Z#tsk&5V*|!2V?y?R)#9j5CDU92#> zDK0%1s=RcZq?eA9ocqI&+F>$fK2cvzaY;Q?{_zh1^c(l@|8xz`|KR*DWT^j*^Z$37 C$CV)f diff --git a/.gitignore b/.gitignore old mode 100644 new mode 100755 index 244c2b2..0400792 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ CODE-OF-CONDUCT.md CONTRIBUTING.md LICENSE.txt NOTICE.txt +venv-impexp diff --git a/CODE-OF-CONDUCT.md b/CODE-OF-CONDUCT.md old mode 100644 new mode 100755 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md old mode 100644 new mode 100755 diff --git a/Dockerfile b/Dockerfile old mode 100644 new mode 100755 diff --git a/LICENSE.txt b/LICENSE.txt old mode 100644 new mode 100755 diff --git a/NOTICE.txt b/NOTICE.txt old mode 100644 new mode 100755 diff --git a/VMCImportExport.py b/VMCImportExport.py index c6be493..332b39d 100755 --- a/VMCImportExport.py +++ b/VMCImportExport.py @@ -284,8 +284,21 @@ def ConfigLoader(self): #Advanced Firewall self.nsx_adv_fw_export = self.loadConfigFlag(config,"exportConfig","nsx_adv_fw_export") - self.nsx_adv_fw_import = self.loadConfigFlag(config,"importConfig","nsx_adv_fw_import") - self.nsx_adv_fw_allow_enable = self.loadConfigFlag(config,"importConfig","nsx_adv_fw_allow_enable") + self.nsx_adv_fw_settings_export_filename = self.loadConfigFilename(config,"exportConfig","nsx_adv_fw_settings_export_filename") + self.nsx_adv_fw_sigs_export_filename = self.loadConfigFilename(config, "exportConfig","nsx_adv_fw_sigs_export_filename") + self.nsx_adv_fw_profiles_export_filename = self.loadConfigFilename(config,"exportConfig","nsx_adv_fw_profiles_export_filename") + self.nsx_adv_fw_policies_export_filename = self.loadConfigFilename(config,"exportConfig","nsx_adv_fw_policies_export_filename") + self.nsx_adv_fw_rules_export_filename = self.loadConfigFilename(config,"exportConfig","nsx_adv_fw_rules_export_filename") + + self.nsx_adv_fw_import = self.loadConfigFlag(config,"importConfig","nsx_adv_fw_import") + self.nsx_adv_fw_allow_enable = self.loadConfigFlag(config,"importConfig","nsx_adv_fw_allow_enable") + self.nsx_adv_fw_settings_import_filename = self.loadConfigFilename(config,"importConfig","nsx_adv_fw_settings_import_filename") + # self.nsx_adv_fw_sigs_import_filename = self.loadConfigFilename(config, "importConfig","nsx_adv_fw_sigs_import_filename") + self.nsx_adv_fw_profiles_import_filename = self.loadConfigFilename(config,"importConfig","nsx_adv_fw_profiles_import_filename") + self.nsx_adv_fw_policies_import_filename = self.loadConfigFilename(config,"importConfig","nsx_adv_fw_policies_import_filename") + self.nsx_adv_fw_rules_import_filename = self.loadConfigFilename(config,"importConfig","nsx_adv_fw_rules_import_filename") + + #SDDC Info self.sddc_info_filename = self.loadConfigFilename(config,"exportConfig","sddc_info_filename") @@ -866,7 +879,6 @@ def export_mcgw_fw(self): json.dump(mcgw_fw_policy_json, outfile, indent=4) return True - def export_mpl(self): """Exports Connected VPC Managed Prefix List""" my_url = f'{self.proxy_url}/cloud-service/api/v1/infra/linked-vpcs' @@ -881,6 +893,117 @@ def export_mpl(self): json.dump(mpl_response, outfile, indent=4) return True + def export_advanced_firewall(self): + """Exports NSX Advanced Firewall settings, profiles, policies and rules""" + successval = True + + retval = self.export_nsx_adv_fw_settings() + if retval is False: + successval = False + print('NSX Advanced Firewall settings export failure: ', self.lastJSONResponse) + else: + print('NSX Advanced Firewall settings exported.') + + retval = self.export_nsx_adv_fw_exclusions() + if retval is False: + successval = False + print('NSX Advanced Firewall exclusion export failure: ', self.lastJSONResponse) + elif retval is None: + print("No exclusions to export.") + else: + print('NSX Advanced Firewall exclusions exported.') + + retval = self.export_ids_profiles() + if retval is False: + successval = False + print('NSX Advanced Firewall profiles export failure: ', self.lastJSONResponse) + else: + print('NSX Advanced Firewall profiles exported.') + + retval = self.export_ids_policies() + if retval[0] is False: + successval = False + print('NSX Advanced Firewall policies export failure: ', self.lastJSONResponse) + else: + print('NSX Advanced Firewall policies exported.') + pol_count = retval[1] + pol_list = retval[2] + rule_set = [] + if pol_count == 0: + print("There are no policies and therefore no rules to export - skipping NSX Adv FW rules.") + return successval + else: + for policy in pol_list: + retval = self.export_ids_rules(policy) + if retval[0] is False: + successval = False + print('NSX Advanced Firewall rules export failure: ', self.lastJSONResponse) + else: + rule_set.append(retval[1]) + print(f'NSX Advanced Firewall rules exported for policy {policy}.') + fname = self.export_path / f'{self.nsx_adv_fw_rules_export_filename}' + with open(fname, 'w') as outfile: + json.dump(rule_set, outfile,indent=4) + return successval + + def export_nsx_adv_fw_settings(self): + my_url = f'{self.proxy_url_short}/policy/api/v1/infra/settings/firewall/security/intrusion-services' + response = self.invokeCSPGET(my_url) + if response is None or response.status_code != 200: + return False + json_response = response.json() + fname = self.export_path / self.nsx_adv_fw_settings_export_filename + with open(fname, 'w') as outfile: + json.dump(json_response, outfile,indent=4) + return True + + def export_nsx_adv_fw_exclusions(self): + my_url = f'{self.proxy_url_short}/policy/api/v1/infra/settings/firewall/security/intrusion-services/global-signatures' + response = self.invokeCSPGET(my_url) + if response is None or response.status_code != 200: + return False + json_response = response.json() + if json_response['results']: + nsxaf_sigs = json_response['results'] + fname = self.export_path / self.nsx_adv_fw_sigs_export_filename + with open(fname, 'w') as outfile: + json.dump(nsxaf_sigs, outfile,indent=4) + return True + else: + return None + + def export_ids_profiles(self): + my_url = f'{self.proxy_url_short}/policy/api/v1/infra/settings/firewall/security/intrusion-services/profiles' + response = self.invokeCSPGET(my_url) + if response is None or response.status_code != 200: + return False + json_response = response.json() + nsxaf_profiles = json_response['results'] + fname = self.export_path / self.nsx_adv_fw_profiles_export_filename + with open(fname, 'w') as outfile: + json.dump(nsxaf_profiles, outfile,indent=4) + return True + + def export_ids_policies(self): + my_url = f'{self.proxy_url_short}/policy/api/v1/infra/domains/cgw/intrusion-service-policies' + response = self.invokeCSPGET(my_url) + json_response = response.json() + nsxaf_policies = json_response['results'] + policy_count = json_response['result_count'] + fname = self.export_path / self.nsx_adv_fw_policies_export_filename + with open(fname, 'w') as outfile: + json.dump(nsxaf_policies, outfile,indent=4) + policy_list=[] + for policy in nsxaf_policies: + policy_list.append(policy['id']) + return True, policy_count, policy_list + + def export_ids_rules(self, ids_policy_name): + my_url = f'{self.proxy_url_short}/policy/api/v1/infra/domains/cgw/intrusion-service-policies/{ids_policy_name}/rules' + response = self.invokeCSPGET(my_url) + json_response = response.json() + nsxaf_rules = json_response['results'][0] + return True, nsxaf_rules def export_ral(self): """Exports the SDDCs Route Aggregation List(s)""" @@ -2201,6 +2324,73 @@ def exportSDDCCGWGroups(self): return True + def import_advanced_firewall(self): + """Imports NSX Advanced Firewall settings, profiles, policies and rules""" + self.vmc_auth.check_access_token_expiration() + if self.dest_sddc_enable_nsx_advanced_addon is False: + if self.nsx_adv_fw_allow_enable is True: + print("nsx_adv_fw_allow_enable set to True, attempting to enable the NSX Advanced Firewall in the destination SDDC...") + retval = self.enable_advanced_firewall_dest() + if retval is False: + print("ERROR - Failed to enable NSX Advanced Firewall - unable to import") + return + print("NSX advanced firewall has been enabled.") + else: + print("ERROR - Unable to import advanced firewall config - the advanced firewall add-on is disabled in the destination SDDC. You can try to automatically enable the feature with the `nsx_adv_fw_allow_enable` flag in config.ini") + return + + # wait 5 seconds before proceeding or you will get an error; service needs time to activate on NSX manager. + time.sleep(5) + + # import setting JSON + # capture the following from settings: + # "oversubscription": "DROPPED" - used to avoid triggering bug! must be set before autoupdate + # "auto_update": use for evaluating configuration of auto update + # "resource_type": "IdsSettings" - used to set NSX AF values + # "path": "/infra/settings/firewall/security/intrusion-services" + + # Enable auto update + retval = self.enable_nsx_ids_auto_update() + if retval is False: + print('NSX Advanced Firewall autoupdate failed: ', self.lastJSONResponse) + else: + print('NSX Advanced Firewall autoupdate configured successfully.') + # Update signatures + retval = self.nsx_ids_update_signatures() + if retval is False: + print('Automatic update of NSX Advanced Firewall signatures failed: ', self.lastJSONResponse) + else: + print('NSX Advanced Firewall signature update initiated.') + + # Enable all clusters + retval = self.enable_nsx_ids_all_clusters() + if retval is False: + print('NSX Advanced Firewall cluster enable failed: ', self.lastJSONResponse) + else: + print('NSX Advanced Firewall clusters enabled successfully.') + + # Import NSX AF IDS profiles + # skip the import of system-created profile(s) + retval = self.patch_ips_profile() + if retval is False: + print('NSX Advanced Firewall profile import failed: ', self.lastJSONResponse) + else: + print('NSX Advanced Firewall profiles imported.') + + # Import NSX AF IDS policies + retval = self.put_ids_policy() + if retval is False: + print('NSX Advanced Firewall policy import failed: ', self.lastJSONResponse) + else: + print('NSX Advanced Firewall policies imported.') + + # Import NSX AF IDS rules + retval = self.put_ids_rule() + if retval is False: + print('NSX Advanced Firewall rule import failed: ', self.lastJSONResponse) + else: + print('NSX Advanced Firewall rules imported.') + def enable_advanced_firewall_dest(self) -> bool: """Enable the NSX advanced firewall in the destination SDDC""" self.vmc_auth.check_access_token_expiration() @@ -2213,28 +2403,206 @@ def enable_advanced_firewall_dest(self) -> bool: print(f'API Call Status {response.status_code}, text:{response.text}') return False else: - print(f'Enabled NSX Advanced Firewall in dest SDDC {self.dest_sddc_id}') + print(f'Enabled NSX Advanced Firewall in dest SDDC {self.dest_sddc_id}') else: print(f'TEST MODE - Would have enabled NSX Advanced Firewall in SDDC {self.dest_sddc_id}') return True - def import_advanced_firewall(self): - self.vmc_auth.check_access_token_expiration() - if self.dest_sddc_enable_nsx_advanced_addon is False: - if self.nsx_adv_fw_allow_enable is True: - print("nsx_adv_fw_allow_enable set to True, attempting to enable the NSX Advanced Firewall in the destination SDDC...") - retval = self.enable_advanced_firewall_dest() - if retval is False: - print("ERROR - Failed to enable NSX Advanced Firewall - unable to import") - return + def enable_nsx_ids_auto_update(self): + myHeader = {"Authorization":"Bearer " + self.vmc_auth.access_token} + # myURL = f"{self.proxy_url_short}/policy/api/v1/infra/settings/firewall/security/intrusion-services" + myURL = f'{self.proxy_url_short}/policy/api/v1/infra/settings/firewall/security/intrusion-services' + json_data = { + "auto_update": True, + "oversubscription": "DROPPED" + } + response = requests.patch(myURL, headers=myHeader, json=json_data) + status = response.status_code + if status == 202: + return response + else: + self.error_handling(response) + return False - print("NSX advanced firewall has been enabled.") - else: - print("ERROR - Unable to import advanced firewall config - the advanced firewall add-on is disabled in the destination SDDC. You can try to automatically enable the feature with the `nsx_adv_fw_allow_enable` flag in config.ini") - return + def nsx_ids_update_signatures(self): + myHeader = {"Authorization":"Bearer " + self.vmc_auth.access_token} + myURL = f"{self.proxy_url_short}/policy/api/v1/infra/settings/firewall/security/intrusion-services/signatures?action=update_signatures" + response = requests.post(myURL, headers=myHeader) + status = response.status_code + if status == 202: + return response + else: + self.error_handling(response) + return False + + def enable_nsx_ids_all_clusters(self): + clusters_json = self.get_nsx_ids_cluster_enabled() + if clusters_json is not None: + cluster_array = clusters_json['results'] + for i in cluster_array: + targetID = i['id'] + ids_status = i['ids_enabled'] + if ids_status == False: + json_body = { + "ids_enabled": True, + "cluster": { + "target_id": targetID + } + } + response = self.enable_nsx_ids_cluster(targetID, json_body) + if response.status_code != 200: + print("Something went wrong. Please check your syntax and try again.") + sys.exit(1) + else: + pass + else: + pass + else: + print("Something went wrong. Please check your syntax and try again.") + sys.exit(1) + + def get_nsx_ids_cluster_enabled(self): + myURL = f"{self.proxy_url_short}/policy/api/v1/infra/settings/firewall/security/intrusion-services/cluster-configs" + myHeader = {"Authorization":"Bearer " + self.vmc_auth.access_token} + response = requests.get(myURL, headers=myHeader) + if response.status_code == 200: + json_response = response.json() + return json_response + else: + print("There was an error. Check the syntax.") + print (f'API call failed with status code {response.status_code}. URL: {myURL}.') + print(json_response['error_message']) + return None + + def enable_nsx_ids_cluster(self, targetID, json_data): + myURL = f"{self.proxy_url_short}/policy/api/v1/infra/settings/firewall/security/intrusion-services/cluster-configs/{targetID}" + myHeader = {"Authorization":"Bearer " + self.vmc_auth.access_token} + response = requests.patch(myURL, headers=myHeader, json=json_data) + if response.status_code == 200: + return response + else: + print("There was an error. Check the syntax.") + print (f'API call failed with status code {response.status_code}. URL: {myURL}.') + print(response['error_message']) + return None - print("Feature not implemented") + def patch_ips_profile(self): + # self.vmc_auth.check_access_token_expiration() + fname = self.import_path / self.nsx_adv_fw_profiles_import_filename + try: + with open(fname) as filehandle: + profiles = json.load(filehandle) + except: + print(f'Import failed - unable to open {fname}') + return + for profile in profiles: + if profile: + json_data = {} + if profile['_create_user'] == "system": + pass + else: + # stage the necessary JSON payload + keep_keys = ["resource_type", "id", "display_name", "profile_severity", "criteria"] + def without_keys(d, keys): + return {x: d[x] for x in d if x in keys} + json_data = without_keys(profile, keep_keys) + + # json_data['resource_type']= "IdsProfile" + # json_data['id'] = profile['id'] + # json_data['display_name'] = profile['display_name'] + # if profile["profile_severity"]: + # json_data['profile_severity'] = profile["profile_severity"] + # if profile["criteria"]: + # json_data['criteria'] = profile["criteria"] + + if self.import_mode == "live": + myHeader = {"Authorization":"Bearer " + self.vmc_auth.access_token} + my_url = f'{self.proxy_url}/policy/api/v1/infra/settings/firewall/security/intrusion-services/profiles/{profile["display_name"]}' + if self.sync_mode is True: + response = requests.patch(my_url, headers=myHeader, json=json_data) + else: + response = requests.put(my_url, headers=myHeader, json=json_data) + if response.status_code == 200: + result = "SUCCESS" + print('Added {}'.format(json_data['display_name'])) + else: + result = "FAIL" + self.error_handling(response) + else: + print(f'TEST MODE - IDS profile {profile["display_name"]} would have been imported.') + + def put_ids_policy(self): + fname = self.import_path / self.nsx_adv_fw_policies_import_filename + try: + with open(fname) as filehandle: + policies = json.load(filehandle) + except: + print(f'Import failed - unable to open {fname}') + return + for pol in policies: + if pol: + json_data = {} + if pol['_create_user'] == "system": + pass + else: + # stage the necessary JSON payload + json_data['resource_type']= "IdsSecurityPolicy" + json_data['id'] = pol['id'] + json_data['display_name'] = pol['display_name'] + # print(json.dumps(json_data, indent=2)) + + if self.import_mode == "live": + myHeader = {"Authorization":"Bearer " + self.vmc_auth.access_token} + my_url = f'{self.proxy_url}/policy/api/v1/infra/domains/cgw/intrusion-service-policies/{pol["display_name"]}' + if self.sync_mode is True: + response = requests.patch(my_url, headers=myHeader, json=json_data) + else: + response = requests.put(my_url, headers=myHeader, json=json_data) + if response.status_code == 200: + result = "SUCCESS" + print('Added {}'.format(json_data['display_name'])) + else: + result = "FAIL" + self.error_handling(response) + else: + print(f'TEST MODE - IDS policy {pol["display_name"]} would have been imported.') + + def put_ids_rule(self): + fname = self.import_path / self.nsx_adv_fw_rules_import_filename + try: + with open(fname) as filehandle: + rules = json.load(filehandle) + except: + print(f'Import failed - unable to open {fname}') + return + for rule in rules: + if rule: + if rule['_create_user'] == "system": + pass + else: + policy_name = rule['parent_path'][46:] + print(policy_name) + keep_keys = ["action", "ids_profiles", "resource_type", "id", "display_name", "sources_excluded", "destinations_excluded", "source_groups", "destination_groups", "services", "scope", "direction", "tag", "ip_protocol"] + def without_keys(d, keys): + return {x: d[x] for x in d if x in keys} + json_data = without_keys(rule, keep_keys) + + if self.import_mode == "live": + myHeader = {"Authorization":"Bearer " + self.vmc_auth.access_token} + my_url = f'{self.proxy_url_short}/policy/api/v1/infra/domains/cgw/intrusion-service-policies/{policy_name}/rules/{rule["display_name"]}' + if self.sync_mode is True: + response = requests.patch(my_url, headers=myHeader, json=json_data) + else: + response = requests.put(my_url, headers=myHeader, json=json_data) + if response.status_code == 200: + result = "SUCCESS" + print('Added {}'.format(json_data['display_name'])) + else: + result = "FAIL" + self.error_handling(response) + else: + print(f'TEST MODE - IDS rule {rule["display_name"]} would have been imported.') def importSDDCCGWRule(self): """Import all CGW Rules from a JSON file""" diff --git a/api-test.py b/api-test.py old mode 100644 new mode 100755 diff --git a/config_ini/.gitignore b/config_ini/.gitignore new file mode 100755 index 0000000..854c187 --- /dev/null +++ b/config_ini/.gitignore @@ -0,0 +1,4 @@ +*.zip +*.json +*.ini + diff --git a/config_ini/aws.ini b/config_ini/aws.ini old mode 100644 new mode 100755 diff --git a/config_ini/config.ini b/config_ini/config.ini index a666719..6c79473 100755 --- a/config_ini/config.ini +++ b/config_ini/config.ini @@ -18,11 +18,11 @@ cgw_export = True cgw_export_filename = cgw.json # Export the management gateway settings? -mgw_export = True +mgw_export = False mgw_export_filename = mgw.json # Export the segments configured on the default compute gateway? -network_export = True +network_export = False network_export_filename = cgw-networks.json # Export DHCP static bindings for networks configured on the compute gateway? # This will significantly slow down the export process, only enable this if you use @@ -30,21 +30,21 @@ network_export_filename = cgw-networks.json network_dhcp_static_binding_export = False # Export segments attached to non-default Tier-1 gateways? -flex_segment_export = True +flex_segment_export = False flex_segment_export_filename = flex_seg.json flex_segment_disc_prof_export_filename = flex_seg_disc_prof.json # Export the list of public IP addresses? -public_export = True +public_export = False public_export_filename = public.json # Export the NAT rules, including Public IP addresses? -nat_export = True +nat_export = False nat_export_filename = natrules.json #Export VPN configuration? -vpn_export = True -t1_vpn_export = True +vpn_export = False +t1_vpn_export = False t1_vpn_export_filename = t1vpn.json t1_vpn_service_filename = t1vpn_service.json t1_vpn_localendpoint_filename = t1vpn_le.json @@ -53,36 +53,41 @@ t1_vpn_localendpoint_filename = t1vpn_le.json service_access_export = False # Export the distributed firewall? -dfw_export = True +dfw_export = False dfw_export_filename = dfw.json dfw_detailed_export_filename = dfw_details.json # Export the NSX advanced firewall? This setting will have no effect if the # NSX advanced firewall add-on is not enabled in the source SDDC -nsx_adv_fw_export = False +nsx_adv_fw_export = True +nsx_adv_fw_settings_export_filename = nsx_adv_fw_settings.json +nsx_adv_fw_sigs_export_filename = nsx_adv_fw_sigs.json +nsx_adv_fw_profiles_export_filename = nsx_adv_fw_profiles.json +nsx_adv_fw_policies_export_filename = nsx_adv_fw_policies.json +nsx_adv_fw_rules_export_filename = nsx_adv_fw_rules.json # Export the multiple tier-1 gateway configuration and firewall rules. Feature added with M18. -mcgw_export = True +mcgw_export = False mcgw_export_filename = mcgw.json -mcgw_fw_export = True +mcgw_fw_export = False mcgw_fw_export_filename = mcgw_fw.json -mcgw_static_routes_export = True +mcgw_static_routes_export = False mcgw_static_routes_export_filename = mcgw_static_routes.json # Export Connected VPC Managed Prefix List configuration -mpl_export = True +mpl_export = False mpl_export_filename = mpl.json # Export route aggregation lists and route configuration -ral_export = True +ral_export = False ral_export_filename = ral.json -route_config_export = True +route_config_export = False route_config_export_filename = route_config.json # Filename for SDDC info - the base SDDC congfiguration from /vmc/api/orgs/*/sddc sddc_info_filename = sddc_info.json # If True, this will hide sensitive SDDC data like cloud_password -sddc_info_hide_sensitive_data = True +sddc_info_hide_sensitive_data = False # Keep previous versions of the exported JSON files? export_history = False @@ -125,31 +130,31 @@ import_folder = json # import_mode = test # - No changes will be made to the destination SDDC # -# import_mode = live +import_mode = live # - Changes will be made to the destination SDDC -import_mode = test +# import_mode = test # Script will warn, ask for a confirmation before continuing in live mode # Set this to false if you are absolutely sure you have your script configured correctly and want to run it automatically import_mode_live_warning = True # Script will attempt to enable IPv6 on the destination SDDC if it was enabled on the source SDDC -enable_ipv6 = True +enable_ipv6 = False # Import services? Only disable this if you truly know what you are doing. # Firewall groups are dependent on Services. If you skip Services, Groups that are # dependent on those services will fail to import -services_import = True +services_import = False # Import groups? Only disable this if you truly know what you are doing. # Firewall rules are dependent on Groups. If you skip Groups, Firewall rules # dependent on those groups will fail to import. Both CGW and DFW rules are dependent # on Compute Groups -compute_groups_import = True -management_groups_import = True +compute_groups_import = False +management_groups_import = False # Import compute gateway settings? -cgw_import = True +cgw_import = False cgw_groups_filename = cgw_groups.json cgw_import_filename = cgw.json # Python regex match on CGW group display name, pipe-delimited. See README for examples. @@ -158,7 +163,7 @@ cgw_groups_import_exclude_list = cgw_import_exclude_list = # Import the management gateway settings? -mgw_import = True +mgw_import = False mgw_groups_filename = mgw_groups.json mgw_import_filename = mgw.json # Python regex match on MGW group display name, pipe-delimited. See README for examples. @@ -167,28 +172,28 @@ mgw_groups_import_exclude_list = mgw_import_exclude_list = # Import additional Tier-1 Gateways and Firewall Policy/Rules? -mcgw_import = True +mcgw_import = False mcgw_import_filename = mcgw.json -mcgw_static_route_import = True +mcgw_static_route_import = False mcgw_static_route_import_filename = mcgw_static_routes.json -mcgw_fw_import = True +mcgw_fw_import = False mcgw_fw_import_filename = mcgw_fw.json # Import Conncted VPC Managed Prefix List -mpl_import = True +mpl_import = False mpl_import_filename = mpl.json #Automatically accept the Resource Share. Users must provide an AWS Access Key and Secret Access Key in the aws.ini file -automate_ram_acceptance = True -automate_vpc_route_table_programming = True +automate_ram_acceptance = False +automate_vpc_route_table_programming = False #Import Route Aggregation Lists and Route Configuration? -ral_import = True +ral_import = False ral_import_filename = ral.json -route_config_import = True +route_config_import = False route_config_import_filename = route_config.json # Import the compute gateway networks? -network_import = True +network_import = False network_import_filename = cgw-networks.json # Python regex match on network display name, pipe-delimited. See README for examples. network_import_exclude_list = L2E_ @@ -200,36 +205,36 @@ network_import_max_networks = -1 network_dhcp_static_binding_import = False # Import flexible segments? -flex_segment_import = True +flex_segment_import = False flex_segment_import_filename = flex_seg.json # Python regex match on network display name, pipe-delimited. See README for examples. flex_segment_import_exclude_list = L2E_ flex_segment_disc_prof_import_filename = flex_seg_disc_prof.json # Import the list of public IP addresses? -public_import = True +public_import = False public_import_filename = public.json public_ip_old_new_filename = public_ip_old_new.json # Import the NAT rules across, alongside the public IP addresses? -nat_import = True +nat_import = False nat_import_filename = natrules.json # Import VPN configuration? -vpn_import = True -t1_vpn_import = True +vpn_import = False +t1_vpn_import = False t1_vpn_import_filename = t1vpn.json t1_vpn_service_import_filename = t1vpn_service.json t1_vpn_localendpoint_import_filename = t1vpn_le.json # Automatically disable VPN tunnels when importing them -vpn_disable_on_import = True +vpn_disable_on_import = False # Import service access? -service_access_import = True +service_access_import = False # Import the distributed firewall? -dfw_import = True +dfw_import = False dfw_import_filename = dfw.json dfw_detailed_import_filename = dfw_details.json @@ -239,8 +244,13 @@ dfw_detailed_import_filename = dfw_details.json # firewall add-on is not enable in the destination SDDC. The script will # automatically enable the advanced firewall addon if you set # nsx_adv_fw_allow_enable to True -nsx_adv_fw_allow_enable = False -nsx_adv_fw_import = False +nsx_adv_fw_import = True +nsx_adv_fw_allow_enable = True +nsx_adv_fw_settings_import_filename = nsx_adv_fw_settings.json +nsx_adv_fw_sigs_import_filename = nsx_adv_fw_sigs.json +nsx_adv_fw_profiles_import_filename = nsx_adv_fw_profiles.json +nsx_adv_fw_policies_import_filename = nsx_adv_fw_policies.json +nsx_adv_fw_rules_import_filename = nsx_adv_fw_rules.json # vCenter Import Options diff --git a/config_ini/config.ini.vcdr.sample b/config_ini/config.ini.vcdr.sample old mode 100644 new mode 100755 diff --git a/config_ini/vcenter.ini b/config_ini/vcenter.ini old mode 100644 new mode 100755 diff --git a/config_ini/vmc.ini b/config_ini/vmc.ini old mode 100644 new mode 100755 index 254b50e..6669d55 --- a/config_ini/vmc.ini +++ b/config_ini/vmc.ini @@ -11,11 +11,11 @@ strGovProdURL = https://www.vmc-us-gov.vmware.com strGovCSPProdURL = https://console.cloud-us-gov.vmware.com # Refresh tokens generated in the VMC console. Users have a separate token in each org -source_refresh_token = x -dest_refresh_token = x +source_refresh_token = xFQma5YcMOmdfs0o6G5GhKDIKkiqxXLlM2CnjJgolNJrdcMfx1pfyQdBnVKpa8Hk +dest_refresh_token = xFQma5YcMOmdfs0o6G5GhKDIKkiqxXLlM2CnjJgolNJrdcMfx1pfyQdBnVKpa8Hk # Organization and SDDC IDs are easily found in the support tab of any SDDC -source_org_id = x -source_sddc_id = x -dest_org_id = x -dest_sddc_id = x \ No newline at end of file +source_org_id = 2e7fa64c-ff5b-4031-9c0c-f8f629587e10 +source_sddc_id = ae0fce8d-25ef-4c37-8e65-bd0d52e6f9ed +dest_org_id = 2e7fa64c-ff5b-4031-9c0c-f8f629587e10 +dest_sddc_id = ae0fce8d-25ef-4c37-8e65-bd0d52e6f9ed diff --git a/invoke_lambda.py b/invoke_lambda.py old mode 100644 new mode 100755 diff --git a/json/.gitignore b/json/.gitignore old mode 100644 new mode 100755 diff --git a/reference/Export_NSX-T_FW_config_from_an_SDDC.py b/reference/Export_NSX-T_FW_config_from_an_SDDC.py old mode 100644 new mode 100755 diff --git a/reference/Export_SDDC_config.py b/reference/Export_SDDC_config.py old mode 100644 new mode 100755 diff --git a/reference/Import_NSX-T_FW_config.py b/reference/Import_NSX-T_FW_config.py old mode 100644 new mode 100755 diff --git a/reference/Import_SDDC_config.py b/reference/Import_SDDC_config.py old mode 100644 new mode 100755 diff --git a/reference/README.md b/reference/README.md old mode 100644 new mode 100755 diff --git a/reference/pyVMC.py b/reference/pyVMC.py old mode 100644 new mode 100755 diff --git a/requirements.txt b/requirements.txt old mode 100644 new mode 100755 diff --git a/sddc_import_export.py b/sddc_import_export.py index ce85e97..9fc3e28 100755 --- a/sddc_import_export.py +++ b/sddc_import_export.py @@ -302,8 +302,6 @@ def main(args): grp_name = f'cgw-test-group-{i:04}' retval = ioObj.deleteSDDCCGWGroup(grp_name) - - if intent_name == "import-vcenter": no_intent_found = False if ioObj.import_vcenter_folders: @@ -643,6 +641,22 @@ def main(args): else: print("NAT rules export skipped.") + if ioObj.nsx_adv_fw_export is True: + if (ioObj.cgw_export is False): + print("NSX Advanced Firewall export is enabled, but CGW export is not.") + print("Please enable export of Compute Gateway settings to capture all CGW Groups, else import of NSX AF settings and rules may fail.") + print("Exiting.") + sys.exit() + print("Beginning NSX Advanced Firewall export...") + retval = ioObj.export_advanced_firewall() + if retval is True: + print("NSX Advanced Firewall exported.") + else: + print("NSX Advanced Firewall export error: {}.".format(ioObj.lastJSONResponse)) + else: + print("NSX Advanced Firewall export skipped.") + + if ioObj.service_access_export is True: print("Beginning Service Access export...") retval = ioObj.exportServiceAccess() diff --git a/testvmc.py b/testvmc.py old mode 100644 new mode 100755 diff --git a/vcenter.py b/vcenter.py old mode 100644 new mode 100755 diff --git a/vmc.py b/vmc.py old mode 100644 new mode 100755 diff --git a/vmc_auth.py b/vmc_auth.py old mode 100644 new mode 100755 From 05c07f1acb302d8df208e5f9da871cc4a3e9f2dc Mon Sep 17 00:00:00 2001 From: Tom Twyman Date: Thu, 6 Jul 2023 12:36:22 -0400 Subject: [PATCH 18/26] removed unnecessary comments Signed-off-by: Tom Twyman --- VMCImportExport.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/VMCImportExport.py b/VMCImportExport.py index 332b39d..330d66e 100755 --- a/VMCImportExport.py +++ b/VMCImportExport.py @@ -2508,14 +2508,6 @@ def without_keys(d, keys): return {x: d[x] for x in d if x in keys} json_data = without_keys(profile, keep_keys) - # json_data['resource_type']= "IdsProfile" - # json_data['id'] = profile['id'] - # json_data['display_name'] = profile['display_name'] - # if profile["profile_severity"]: - # json_data['profile_severity'] = profile["profile_severity"] - # if profile["criteria"]: - # json_data['criteria'] = profile["criteria"] - if self.import_mode == "live": myHeader = {"Authorization":"Bearer " + self.vmc_auth.access_token} my_url = f'{self.proxy_url}/policy/api/v1/infra/settings/firewall/security/intrusion-services/profiles/{profile["display_name"]}' From 3db7c2afdc2e6841b48d110adfb713af556760ef Mon Sep 17 00:00:00 2001 From: Tom Twyman Date: Tue, 18 Jul 2023 09:12:56 -0400 Subject: [PATCH 19/26] updated NSX AF policy import to capture category Signed-off-by: Tom Twyman --- VMCImportExport.py | 8 ++-- config_ini/config.ini | 86 +++++++++++++++++++++---------------------- config_ini/vmc.ini | 12 +++--- sddc_import_export.py | 8 +++- 4 files changed, 60 insertions(+), 54 deletions(-) diff --git a/VMCImportExport.py b/VMCImportExport.py index 330d66e..5cf1ce8 100755 --- a/VMCImportExport.py +++ b/VMCImportExport.py @@ -2539,10 +2539,10 @@ def put_ids_policy(self): pass else: # stage the necessary JSON payload - json_data['resource_type']= "IdsSecurityPolicy" - json_data['id'] = pol['id'] - json_data['display_name'] = pol['display_name'] - # print(json.dumps(json_data, indent=2)) + keep_keys = ["resource_type", "id", "display_name", "category"] + def without_keys(d, keys): + return {x: d[x] for x in d if x in keys} + json_data = without_keys(pol, keep_keys) if self.import_mode == "live": myHeader = {"Authorization":"Bearer " + self.vmc_auth.access_token} diff --git a/config_ini/config.ini b/config_ini/config.ini index 6c79473..c616b5e 100755 --- a/config_ini/config.ini +++ b/config_ini/config.ini @@ -18,11 +18,11 @@ cgw_export = True cgw_export_filename = cgw.json # Export the management gateway settings? -mgw_export = False +mgw_export = True mgw_export_filename = mgw.json # Export the segments configured on the default compute gateway? -network_export = False +network_export = True network_export_filename = cgw-networks.json # Export DHCP static bindings for networks configured on the compute gateway? # This will significantly slow down the export process, only enable this if you use @@ -30,30 +30,30 @@ network_export_filename = cgw-networks.json network_dhcp_static_binding_export = False # Export segments attached to non-default Tier-1 gateways? -flex_segment_export = False +flex_segment_export = True flex_segment_export_filename = flex_seg.json flex_segment_disc_prof_export_filename = flex_seg_disc_prof.json # Export the list of public IP addresses? -public_export = False +public_export = True public_export_filename = public.json # Export the NAT rules, including Public IP addresses? -nat_export = False +nat_export = True nat_export_filename = natrules.json #Export VPN configuration? -vpn_export = False -t1_vpn_export = False +vpn_export = True +t1_vpn_export = True t1_vpn_export_filename = t1vpn.json t1_vpn_service_filename = t1vpn_service.json t1_vpn_localendpoint_filename = t1vpn_le.json # Export service access? -service_access_export = False +service_access_export = True # Export the distributed firewall? -dfw_export = False +dfw_export = True dfw_export_filename = dfw.json dfw_detailed_export_filename = dfw_details.json @@ -67,27 +67,27 @@ nsx_adv_fw_policies_export_filename = nsx_adv_fw_policies.json nsx_adv_fw_rules_export_filename = nsx_adv_fw_rules.json # Export the multiple tier-1 gateway configuration and firewall rules. Feature added with M18. -mcgw_export = False +mcgw_export = True mcgw_export_filename = mcgw.json -mcgw_fw_export = False +mcgw_fw_export = True mcgw_fw_export_filename = mcgw_fw.json -mcgw_static_routes_export = False +mcgw_static_routes_export = True mcgw_static_routes_export_filename = mcgw_static_routes.json # Export Connected VPC Managed Prefix List configuration -mpl_export = False +mpl_export = True mpl_export_filename = mpl.json # Export route aggregation lists and route configuration -ral_export = False +ral_export = True ral_export_filename = ral.json -route_config_export = False +route_config_export = True route_config_export_filename = route_config.json # Filename for SDDC info - the base SDDC congfiguration from /vmc/api/orgs/*/sddc sddc_info_filename = sddc_info.json # If True, this will hide sensitive SDDC data like cloud_password -sddc_info_hide_sensitive_data = False +sddc_info_hide_sensitive_data = True # Keep previous versions of the exported JSON files? export_history = False @@ -108,7 +108,7 @@ export_purge_before_run = True # vCenter Export Options # Must configure vcenter.ini -export_vcenter_folders = False +export_vcenter_folders = True # CSP Role sync options # The source user - the template user account with roles that need to be synced to other user accounts @@ -130,31 +130,31 @@ import_folder = json # import_mode = test # - No changes will be made to the destination SDDC # -import_mode = live +# import_mode = live # - Changes will be made to the destination SDDC -# import_mode = test +import_mode = test # Script will warn, ask for a confirmation before continuing in live mode # Set this to false if you are absolutely sure you have your script configured correctly and want to run it automatically import_mode_live_warning = True # Script will attempt to enable IPv6 on the destination SDDC if it was enabled on the source SDDC -enable_ipv6 = False +enable_ipv6 = True # Import services? Only disable this if you truly know what you are doing. # Firewall groups are dependent on Services. If you skip Services, Groups that are # dependent on those services will fail to import -services_import = False +services_import = True # Import groups? Only disable this if you truly know what you are doing. # Firewall rules are dependent on Groups. If you skip Groups, Firewall rules # dependent on those groups will fail to import. Both CGW and DFW rules are dependent # on Compute Groups -compute_groups_import = False -management_groups_import = False +compute_groups_import = True +management_groups_import = True # Import compute gateway settings? -cgw_import = False +cgw_import = True cgw_groups_filename = cgw_groups.json cgw_import_filename = cgw.json # Python regex match on CGW group display name, pipe-delimited. See README for examples. @@ -163,7 +163,7 @@ cgw_groups_import_exclude_list = cgw_import_exclude_list = # Import the management gateway settings? -mgw_import = False +mgw_import = True mgw_groups_filename = mgw_groups.json mgw_import_filename = mgw.json # Python regex match on MGW group display name, pipe-delimited. See README for examples. @@ -172,28 +172,28 @@ mgw_groups_import_exclude_list = mgw_import_exclude_list = # Import additional Tier-1 Gateways and Firewall Policy/Rules? -mcgw_import = False +mcgw_import = True mcgw_import_filename = mcgw.json -mcgw_static_route_import = False +mcgw_static_route_import = True mcgw_static_route_import_filename = mcgw_static_routes.json -mcgw_fw_import = False +mcgw_fw_import = True mcgw_fw_import_filename = mcgw_fw.json # Import Conncted VPC Managed Prefix List -mpl_import = False +mpl_import = True mpl_import_filename = mpl.json #Automatically accept the Resource Share. Users must provide an AWS Access Key and Secret Access Key in the aws.ini file -automate_ram_acceptance = False -automate_vpc_route_table_programming = False +automate_ram_acceptance = True +automate_vpc_route_table_programming = True #Import Route Aggregation Lists and Route Configuration? -ral_import = False +ral_import = True ral_import_filename = ral.json -route_config_import = False +route_config_import = True route_config_import_filename = route_config.json # Import the compute gateway networks? -network_import = False +network_import = True network_import_filename = cgw-networks.json # Python regex match on network display name, pipe-delimited. See README for examples. network_import_exclude_list = L2E_ @@ -205,36 +205,36 @@ network_import_max_networks = -1 network_dhcp_static_binding_import = False # Import flexible segments? -flex_segment_import = False +flex_segment_import = True flex_segment_import_filename = flex_seg.json # Python regex match on network display name, pipe-delimited. See README for examples. flex_segment_import_exclude_list = L2E_ flex_segment_disc_prof_import_filename = flex_seg_disc_prof.json # Import the list of public IP addresses? -public_import = False +public_import = True public_import_filename = public.json public_ip_old_new_filename = public_ip_old_new.json # Import the NAT rules across, alongside the public IP addresses? -nat_import = False +nat_import = True nat_import_filename = natrules.json # Import VPN configuration? -vpn_import = False -t1_vpn_import = False +vpn_import = True +t1_vpn_import = True t1_vpn_import_filename = t1vpn.json t1_vpn_service_import_filename = t1vpn_service.json t1_vpn_localendpoint_import_filename = t1vpn_le.json # Automatically disable VPN tunnels when importing them -vpn_disable_on_import = False +vpn_disable_on_import = True # Import service access? -service_access_import = False +service_access_import = True # Import the distributed firewall? -dfw_import = False +dfw_import = True dfw_import_filename = dfw.json dfw_detailed_import_filename = dfw_details.json @@ -255,7 +255,7 @@ nsx_adv_fw_rules_import_filename = nsx_adv_fw_rules.json # vCenter Import Options # Must configure vcenter.ini -import_vcenter_folders = False +import_vcenter_folders = True # CSP Role sync options # A pipe delimited list of email addresses - these accounts will have the roles synchronized with roles attached to the source user diff --git a/config_ini/vmc.ini b/config_ini/vmc.ini index 6669d55..a195a6a 100755 --- a/config_ini/vmc.ini +++ b/config_ini/vmc.ini @@ -11,11 +11,11 @@ strGovProdURL = https://www.vmc-us-gov.vmware.com strGovCSPProdURL = https://console.cloud-us-gov.vmware.com # Refresh tokens generated in the VMC console. Users have a separate token in each org -source_refresh_token = xFQma5YcMOmdfs0o6G5GhKDIKkiqxXLlM2CnjJgolNJrdcMfx1pfyQdBnVKpa8Hk -dest_refresh_token = xFQma5YcMOmdfs0o6G5GhKDIKkiqxXLlM2CnjJgolNJrdcMfx1pfyQdBnVKpa8Hk +source_refresh_token = +dest_refresh_token = # Organization and SDDC IDs are easily found in the support tab of any SDDC -source_org_id = 2e7fa64c-ff5b-4031-9c0c-f8f629587e10 -source_sddc_id = ae0fce8d-25ef-4c37-8e65-bd0d52e6f9ed -dest_org_id = 2e7fa64c-ff5b-4031-9c0c-f8f629587e10 -dest_sddc_id = ae0fce8d-25ef-4c37-8e65-bd0d52e6f9ed +source_org_id = +source_sddc_id = +dest_org_id = +dest_sddc_id = diff --git a/sddc_import_export.py b/sddc_import_export.py index 9fc3e28..6a08e4f 100755 --- a/sddc_import_export.py +++ b/sddc_import_export.py @@ -642,7 +642,7 @@ def main(args): print("NAT rules export skipped.") if ioObj.nsx_adv_fw_export is True: - if (ioObj.cgw_export is False): + if (ioObj.cgw_export is False or ioObj.network_export is False): print("NSX Advanced Firewall export is enabled, but CGW export is not.") print("Please enable export of Compute Gateway settings to capture all CGW Groups, else import of NSX AF settings and rules may fail.") print("Exiting.") @@ -949,6 +949,12 @@ def main(args): ioObj.importVPN() if ioObj.nsx_adv_fw_import is True: + if (ioObj.cgw_import is False): + print("NSX Advanced Firewall export is enabled, but CGW export is not.") + print("Please enable export of Compute Gateway settings to capture all CGW Groups and segments, else import of NSX AF settings and rules may fail.") + print("Exiting.") + sys.exit() + print("Beginning NSX advanced firewall import...") ioObj.import_advanced_firewall() From 948955498ba93a6a176f9b4b59b716e9b4bd604c Mon Sep 17 00:00:00 2001 From: Tom Twyman Date: Tue, 18 Jul 2023 09:50:19 -0400 Subject: [PATCH 20/26] updated messaging for NSX AF import and export warnings Signed-off-by: Tom Twyman --- VMCImportExport.py | 43 ++++++++++++++++++++++++++----------------- sddc_import_export.py | 6 +----- 2 files changed, 27 insertions(+), 22 deletions(-) diff --git a/VMCImportExport.py b/VMCImportExport.py index 5cf1ce8..9e76c98 100755 --- a/VMCImportExport.py +++ b/VMCImportExport.py @@ -2417,24 +2417,30 @@ def enable_nsx_ids_auto_update(self): "auto_update": True, "oversubscription": "DROPPED" } - response = requests.patch(myURL, headers=myHeader, json=json_data) - status = response.status_code - if status == 202: - return response + if self.import_mode == "live": + response = requests.patch(myURL, headers=myHeader, json=json_data) + status = response.status_code + if status == 202: + return response + else: + self.error_handling(response) + return False else: - self.error_handling(response) - return False + print(f'TEST MODE - Would have enabled signature auto-update for NSX Advanced Firewall in SDDC {self.dest_sddc_id}') def nsx_ids_update_signatures(self): myHeader = {"Authorization":"Bearer " + self.vmc_auth.access_token} myURL = f"{self.proxy_url_short}/policy/api/v1/infra/settings/firewall/security/intrusion-services/signatures?action=update_signatures" - response = requests.post(myURL, headers=myHeader) - status = response.status_code - if status == 202: - return response + if self.import_mode == "live": + response = requests.post(myURL, headers=myHeader) + status = response.status_code + if status == 202: + return response + else: + self.error_handling(response) + return False else: - self.error_handling(response) - return False + print(f'TEST MODE - Would have updated signatures for NSX Advanced Firewall in SDDC {self.dest_sddc_id}') def enable_nsx_ids_all_clusters(self): clusters_json = self.get_nsx_ids_cluster_enabled() @@ -2450,12 +2456,15 @@ def enable_nsx_ids_all_clusters(self): "target_id": targetID } } - response = self.enable_nsx_ids_cluster(targetID, json_body) - if response.status_code != 200: - print("Something went wrong. Please check your syntax and try again.") - sys.exit(1) + if self.import_mode == "live": + response = self.enable_nsx_ids_cluster(targetID, json_body) + if response.status_code != 200: + print("Something went wrong. Please check your syntax and try again.") + sys.exit(1) + else: + pass else: - pass + print(f'TEST MODE - Would have enabled NSX Advanced Firewall on cluster {targetID}') else: pass else: diff --git a/sddc_import_export.py b/sddc_import_export.py index 6a08e4f..376045d 100755 --- a/sddc_import_export.py +++ b/sddc_import_export.py @@ -644,9 +644,7 @@ def main(args): if ioObj.nsx_adv_fw_export is True: if (ioObj.cgw_export is False or ioObj.network_export is False): print("NSX Advanced Firewall export is enabled, but CGW export is not.") - print("Please enable export of Compute Gateway settings to capture all CGW Groups, else import of NSX AF settings and rules may fail.") - print("Exiting.") - sys.exit() + print("Please enable export of Compute Gateway settings to capture all CGW Groups AND Segments, else import of NSX AF settings and rules may fail.") print("Beginning NSX Advanced Firewall export...") retval = ioObj.export_advanced_firewall() if retval is True: @@ -952,8 +950,6 @@ def main(args): if (ioObj.cgw_import is False): print("NSX Advanced Firewall export is enabled, but CGW export is not.") print("Please enable export of Compute Gateway settings to capture all CGW Groups and segments, else import of NSX AF settings and rules may fail.") - print("Exiting.") - sys.exit() print("Beginning NSX advanced firewall import...") ioObj.import_advanced_firewall() From 237994b6474c45c466a42520b37b333944129438 Mon Sep 17 00:00:00 2001 From: Chris White Date: Wed, 19 Jul 2023 09:23:35 -0400 Subject: [PATCH 21/26] BUG: Missing indent Signed-off-by: Chris White --- VMCImportExport.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/VMCImportExport.py b/VMCImportExport.py index 9e76c98..11bde14 100755 --- a/VMCImportExport.py +++ b/VMCImportExport.py @@ -2423,7 +2423,7 @@ def enable_nsx_ids_auto_update(self): if status == 202: return response else: - self.error_handling(response) + self.error_handling(response) return False else: print(f'TEST MODE - Would have enabled signature auto-update for NSX Advanced Firewall in SDDC {self.dest_sddc_id}') @@ -2437,7 +2437,7 @@ def nsx_ids_update_signatures(self): if status == 202: return response else: - self.error_handling(response) + self.error_handling(response) return False else: print(f'TEST MODE - Would have updated signatures for NSX Advanced Firewall in SDDC {self.dest_sddc_id}') From 4fc877c7ac4641c7e44d3e4f9f04e3ebf9f6ed68 Mon Sep 17 00:00:00 2001 From: Chris White Date: Thu, 20 Jul 2023 12:14:58 -0400 Subject: [PATCH 22/26] Enhancement: MPL enhancement to wait for resource share creation Signed-off-by: Chris White --- VMCImportExport.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/VMCImportExport.py b/VMCImportExport.py index 11bde14..91f7e31 100755 --- a/VMCImportExport.py +++ b/VMCImportExport.py @@ -1907,7 +1907,6 @@ def import_mpl(self): result = "SUCCESS" print('Enabling Managed Prefix List Mode.') if self.automate_ram_acceptance is True: - time.sleep(20) ram_response = self.aws_ram_accept(vpc_id) if ram_response is True: linked_vpn_url = f'{self.proxy_url}/cloud-service/api/v1/linked-vpcs/{vpc_id}' @@ -1947,6 +1946,16 @@ def aws_ram_accept(self, vpc_id): my_url = f'{self.proxy_url}/cloud-service/api/v1/linked-vpcs/{vpc_id}' response = requests.get(my_url, headers=my_header) json_response = response.json() + mpl_info = json_response['linked_vpc_managed_prefix_list_info'] + mpl_status = mpl_info.get('aws_resource_share_info') + + #wait for resource share to be created + while mpl_status == None: + response = requests.get(my_url, headers=my_header) + json_response = response.json() + mpl_info = json_response['linked_vpc_managed_prefix_list_info'] + mpl_status = mpl_info.get('aws_resource_share_info') + ram_arn = [] ram_arn.append(json_response['linked_vpc_managed_prefix_list_info']['aws_resource_share_info']['aws_resource_share_arn']) From bfe480d4e9bba2f499f99e9b2085de6d4c48e90d Mon Sep 17 00:00:00 2001 From: Chris White Date: Thu, 20 Jul 2023 13:30:17 -0400 Subject: [PATCH 23/26] Enhancement: Modified import URLs to utilize path variable Signed-off-by: Chris White --- VMCImportExport.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/VMCImportExport.py b/VMCImportExport.py index 91f7e31..d2c206b 100755 --- a/VMCImportExport.py +++ b/VMCImportExport.py @@ -2528,7 +2528,8 @@ def without_keys(d, keys): if self.import_mode == "live": myHeader = {"Authorization":"Bearer " + self.vmc_auth.access_token} - my_url = f'{self.proxy_url}/policy/api/v1/infra/settings/firewall/security/intrusion-services/profiles/{profile["display_name"]}' + #'Path' key is grabbed the exported JSON + my_url = f'{self.proxy_url}/policy/api/v1{profile["path"]}' if self.sync_mode is True: response = requests.patch(my_url, headers=myHeader, json=json_data) else: @@ -2564,7 +2565,8 @@ def without_keys(d, keys): if self.import_mode == "live": myHeader = {"Authorization":"Bearer " + self.vmc_auth.access_token} - my_url = f'{self.proxy_url}/policy/api/v1/infra/domains/cgw/intrusion-service-policies/{pol["display_name"]}' + #'Path' key is grabbed the exported JSON + my_url = f'{self.proxy_url}/policy/api/v1{pol["path"]}' if self.sync_mode is True: response = requests.patch(my_url, headers=myHeader, json=json_data) else: @@ -2591,8 +2593,6 @@ def put_ids_rule(self): if rule['_create_user'] == "system": pass else: - policy_name = rule['parent_path'][46:] - print(policy_name) keep_keys = ["action", "ids_profiles", "resource_type", "id", "display_name", "sources_excluded", "destinations_excluded", "source_groups", "destination_groups", "services", "scope", "direction", "tag", "ip_protocol"] def without_keys(d, keys): return {x: d[x] for x in d if x in keys} @@ -2600,7 +2600,8 @@ def without_keys(d, keys): if self.import_mode == "live": myHeader = {"Authorization":"Bearer " + self.vmc_auth.access_token} - my_url = f'{self.proxy_url_short}/policy/api/v1/infra/domains/cgw/intrusion-service-policies/{policy_name}/rules/{rule["display_name"]}' + #'Path' key is grabbed the exported JSON + my_url = f'{self.proxy_url_short}/policy/api/v1{rule["path"]}' if self.sync_mode is True: response = requests.patch(my_url, headers=myHeader, json=json_data) else: @@ -2614,6 +2615,7 @@ def without_keys(d, keys): else: print(f'TEST MODE - IDS rule {rule["display_name"]} would have been imported.') + def importSDDCCGWRule(self): """Import all CGW Rules from a JSON file""" From 9dc65ed56bc0a91125f728a13152f547ad37be03 Mon Sep 17 00:00:00 2001 From: Chris White Date: Fri, 21 Jul 2023 13:58:25 -0400 Subject: [PATCH 24/26] BUG: Boto3 missing 'region_name' variable Signed-off-by: Chris White --- VMCImportExport.py | 21 +++++++++++++++++---- config_ini/aws.ini | 6 +++--- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/VMCImportExport.py b/VMCImportExport.py index d2c206b..c16f649 100755 --- a/VMCImportExport.py +++ b/VMCImportExport.py @@ -80,6 +80,7 @@ def __init__(self,configPath="./config_ini/config.ini", vmcConfigPath="./config_ self.aws_import_access_key_id = "" self.aws_import_secret_access_key = "" self.aws_import_session_token = "" + self.aws_dest_sddc_region = "" self.ConfigLoader() def ConfigLoader(self): @@ -1907,6 +1908,7 @@ def import_mpl(self): result = "SUCCESS" print('Enabling Managed Prefix List Mode.') if self.automate_ram_acceptance is True: + self.get_sddc_region(self.dest_org_id, self.dest_sddc_id) ram_response = self.aws_ram_accept(vpc_id) if ram_response is True: linked_vpn_url = f'{self.proxy_url}/cloud-service/api/v1/linked-vpcs/{vpc_id}' @@ -1960,9 +1962,9 @@ def aws_ram_accept(self, vpc_id): ram_arn.append(json_response['linked_vpc_managed_prefix_list_info']['aws_resource_share_info']['aws_resource_share_arn']) if self.aws_import_session_token: - session = boto3.Session(aws_access_key_id=self.aws_import_access_key_id, aws_secret_access_key=self.aws_import_secret_access_key, aws_session_token=self.aws_import_session_token) + session = boto3.Session(aws_access_key_id=self.aws_import_access_key_id, aws_secret_access_key=self.aws_import_secret_access_key, aws_session_token=self.aws_import_session_token, region_name=self.aws_dest_sddc_region) else: - session = boto3.Session(aws_access_key_id=self.aws_import_access_key_id, aws_secret_access_key=self.aws_import_secret_access_key) + session = boto3.Session(aws_access_key_id=self.aws_import_access_key_id, aws_secret_access_key=self.aws_import_secret_access_key, region=self.aws_dest_sddc_region) ram = session.client('ram') @@ -1991,9 +1993,9 @@ def vpc_rt_prog(self, rt_lst, active_eni, prefix_list_id): self.vmc_auth.check_access_token_expiration() if self.aws_import_session_token: - session = boto3.Session(aws_access_key_id=self.aws_import_access_key_id, aws_secret_access_key=self.aws_import_secret_access_key, aws_session_token=self.aws_import_session_token) + session = boto3.Session(aws_access_key_id=self.aws_import_access_key_id, aws_secret_access_key=self.aws_import_secret_access_key, aws_session_token=self.aws_import_session_token, region_name=self.aws_dest_sddc_region) else: - session = boto3.Session(aws_access_key_id=self.aws_import_access_key_id, aws_secret_access_key=self.aws_import_secret_access_key) + session = boto3.Session(aws_access_key_id=self.aws_import_access_key_id, aws_secret_access_key=self.aws_import_secret_access_key, region_name=self.aws_dest_sddc_region) vpc = session.client('ec2') @@ -2003,6 +2005,17 @@ def vpc_rt_prog(self, rt_lst, active_eni, prefix_list_id): print(f'VPC Route Table {r} has been programmed with MPL') else: print('Route application failure') + + + def get_sddc_region(self, org_id, sddc_id): + """Get given SDDC region""" + self.vmc_auth.check_access_token_expiration() + + sddc_info = self.loadSDDCData(org_id, sddc_id) + malformed_region_id = sddc_info['resource_config']['region'] + malformed_region_id = malformed_region_id.replace("_", "-") + malformed_region_id = malformed_region_id.lower() + self.aws_dest_sddc_region = malformed_region_id def import_ral(self): diff --git a/config_ini/aws.ini b/config_ini/aws.ini index c0128a5..04fb13b 100755 --- a/config_ini/aws.ini +++ b/config_ini/aws.ini @@ -10,6 +10,6 @@ aws_s3_export_access_secret = aws_s3_export_bucket = #AWS credntials for Connected VPC Configuraton -aws_import_access_key_id= -aws_import_secret_access_key= -aws_import_session_token= \ No newline at end of file +aws_import_access_key_id = +aws_import_secret_access_key = +aws_import_session_token = \ No newline at end of file From bfc1bf4a2179b52ab698d16f29d2dd43f5ec882b Mon Sep 17 00:00:00 2001 From: Chris White Date: Mon, 24 Jul 2023 09:03:26 -0400 Subject: [PATCH 25/26] Update README for main push Signed-off-by: Chris White --- README.md | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 63423ae..68652c2 100755 --- a/README.md +++ b/README.md @@ -39,8 +39,7 @@ The SDDC Import/Export for VMware Cloud on AWS tool enable customers to save and There are many situations when customers want to migrate from an existing SDDC to a different one. While HCX addresses the data migration challenge, this tool offers customers the ability to copy the configuration from a source to a destination SDDC. A few example migration scenarios are: -- SDDC to SDDC migration from bare-metal (i3) to a different bare-metal type (i3en) -- SDDC to SDDC migration from VMware-based org to an AWS-based org +- SDDC to SDDC migration from bare-metal (i3) to a different bare-metal type (i3en, i4i) - SDDC to SDDC migration from region (i.e. London) to a different region (i.e. Dublin). Other use cases are: @@ -140,6 +139,7 @@ Version 1.3 introduced the ability to filter out objects during an import. The f - CGW firewall rule - CGW firewall group - CGW network segment +- Flexible segment - MGW firewall rule - MGW firewall group @@ -177,6 +177,14 @@ aws_s3_export_access_id = "" aws_s3_export_access_secret = "" aws_s3_export_bucket = "" ``` + +The aws.ini also includes an option to include credentials for the customer-owned AWS account connected to the SDDC. This allows for automatic acceptance of a resource share for the managed prefix list feature as well as configuration of multiple VPC route tables. This is separate from the S3 configuration above and must be filled out for the configuration to work correctly. +``` +aws_import_access_key_id = +aws_import_secret_access_key = +aws_import_session_token = +``` + The aws.ini configuration can also be passed via command line. Use sddc_import_export --help for syntax. ### 1.3.7. Update vcenter.ini (optional) @@ -216,13 +224,22 @@ If all of the export options are enabled, this will export a set of files: - cgw.json - dfw_details.json - dfw.json +- dhcp-static-bindings.json +- flex_seg_disc_prof.json - flex_seg.json -- mcgw.json -- mcgw-fw.json +- mcgw_fw.json - mcgw_static_routes.json +- mcgw.json - mgw_groups.json - mgw.json +- mpl.json - natrules.json +- nsx_adv_fw_policies.json +- nsx_adv_fw_profiles.json +- nsx_adv_fw_rules.json +- nsx_adv_fw_settings.json +- nsx_adv_fw_sigs.json +- public_ip_old_new.json - public.json - ral.json - route_config.json @@ -230,6 +247,9 @@ If all of the export options are enabled, this will export a set of files: - sddc_info.json - service_access.json - services.json +- t1vpn.json +- t1vpn_service.json +- t1vpn_le.json - vpn-bgp.json - vpn-dpd.json - vpn-ike.json From a5405ebd44db4818adcde922679ee19dba5b22a6 Mon Sep 17 00:00:00 2001 From: Chris White Date: Mon, 24 Jul 2023 09:12:32 -0400 Subject: [PATCH 26/26] Update config.ini --- config_ini/config.ini | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/config_ini/config.ini b/config_ini/config.ini index c616b5e..1df60bd 100755 --- a/config_ini/config.ini +++ b/config_ini/config.ini @@ -108,7 +108,7 @@ export_purge_before_run = True # vCenter Export Options # Must configure vcenter.ini -export_vcenter_folders = True +export_vcenter_folders = False # CSP Role sync options # The source user - the template user account with roles that need to be synced to other user accounts @@ -255,8 +255,8 @@ nsx_adv_fw_rules_import_filename = nsx_adv_fw_rules.json # vCenter Import Options # Must configure vcenter.ini -import_vcenter_folders = True +import_vcenter_folders = False # CSP Role sync options # A pipe delimited list of email addresses - these accounts will have the roles synchronized with roles attached to the source user -role_sync_dest_user_emails = \ No newline at end of file +role_sync_dest_user_emails =