From b2e85cefeaafd338b8bcee44ef85142672d1cd46 Mon Sep 17 00:00:00 2001 From: OmriGez Date: Thu, 15 Aug 2024 17:58:17 +0300 Subject: [PATCH 1/3] updated actions endpoint --- main.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/main.py b/main.py index 38f42e4..c68d9ac 100644 --- a/main.py +++ b/main.py @@ -93,14 +93,6 @@ def getScorecards(): resp = res.json()["scorecards"] return resp -def getSpecificActions(blueprints): - returnActions = [] - for blueprint in blueprints: - print(f"Getting actions for blueprint {blueprint}") - res = requests.get(f'{API_URL}/blueprints/{blueprint}/actions', headers=old_headers) - resp = res.json()["actions"] - returnActions += resp - return returnActions def getActions(): print("Getting actions") @@ -233,7 +225,8 @@ def postActions(actions): action.pop("updatedAt", None) action.pop("createdBy", None) action.pop("updatedBy", None) - res = requests.post(f'{API_URL}/blueprints/{blueprint}/actions', headers=new_headers, json=action) + action.pop("id", None) + res = requests.post(f'{API_URL}/actions', headers=new_headers, json=action) if res.ok != True: print(f"error posting action: " + json.dumps(res.json())) teamError = True @@ -268,7 +261,6 @@ def main(): if SPECIFIC: #check if we are backing up specific blueprints blueprints = getSpecificBlueprints(specificBlueprints) scorecards = getSpecificScorecards(specificBlueprints) - actions = getSpecificActions(specificBlueprints) else: blueprints = getBlueprints() scorecards = getScorecards() From de1e7f12befa38953fe5e28a79ee0d94543b1d12 Mon Sep 17 00:00:00 2001 From: OmriGez Date: Sun, 22 Sep 2024 17:34:22 +0300 Subject: [PATCH 2/3] added additional data handling --- clean.py | 72 +++++++++++++++++++++++++++++++++++++++++++------ main.py | 81 +++++++++++++++++++++++++++++++++++++++++++++----------- 2 files changed, 130 insertions(+), 23 deletions(-) diff --git a/clean.py b/clean.py index 27d13d6..b7803d6 100644 --- a/clean.py +++ b/clean.py @@ -16,20 +16,76 @@ systemBlueprints = ["_user", "_team"] +print("Deleting actions") +res = requests.get(f'{API_URL}/actions?version=v2', headers=headers) +actions = res.json()["actions"] +for action in actions: + print(f"deleting action {action['identifier']}") + res = requests.delete(f'{API_URL}/actions/{action["identifier"]}', headers=headers) + if res.ok != True: + print("error while deleting action: "+ json.dumps(res.json())) + +print("Deleting pages") +res = requests.get(f'{API_URL}/pages', headers=headers) +pages = res.json()["pages"] +for page in pages: + print(f"deleting page {page['identifier']}") + res = requests.delete(f'{API_URL}/pages/{page["identifier"]}', headers=headers) + if res.ok != True: + print("error while deleting page: "+ json.dumps(res.json())) + print("Getting blueprints") res = requests.get(f'{API_URL}/blueprints', headers=headers) -resp = res.json()["blueprints"] +blueprints = res.json()["blueprints"] +failed_blueprints = [] -for blueprint in resp: +for blueprint in blueprints: if blueprint["identifier"] not in systemBlueprints: - print(f"deleting entities of blueprint {blueprint['identifier']}") + print(f"Deleting entities of blueprint {blueprint['identifier']}") res = requests.delete(f'{API_URL}/blueprints/{blueprint["identifier"]}/all-entities', headers=headers) - if res.ok != True: - print("error while deleting entities: "+ json.dumps(res.json())) - print(f"deleting blueprint {blueprint['identifier']}") + if not res.ok: + print(f"Error while deleting entities: {json.dumps(res.json())}") + failed_blueprints.append(blueprint["identifier"]) + continue + + print(f"Deleting blueprint {blueprint['identifier']}") res = requests.delete(f'{API_URL}/blueprints/{blueprint["identifier"]}', headers=headers) - if res.ok != True: - print("error while deleting blueprint: "+ json.dumps(res.json())) + if not res.ok: + print(f"Error while deleting blueprint: {json.dumps(res.json())}") + failed_blueprints.append(blueprint["identifier"]) + + return failed_blueprints + +for attempt in range(5): + if not failed_blueprints: + break + + print(f"Retrying deletion... Attempt {attempt + 1}") + remaining_failed = [] + + for identifier in failed_blueprints: + print(f"Retrying deletion of blueprint {identifier}") + res = requests.delete(f'{API_URL}/blueprints/{identifier}/all-entities', headers=headers) + if not res.ok: + print(f"Error while deleting entities: {json.dumps(res.json())}") + remaining_failed.append(identifier) + continue + + res = requests.delete(f'{API_URL}/blueprints/{identifier}', headers=headers) + if not res.ok: + print(f"Error while deleting blueprint: {json.dumps(res.json())}") + remaining_failed.append(identifier) + + failed_blueprints = remaining_failed + if failed_blueprints: + print(f"Some deletions still failed. Retrying in 5 seconds...") + time.sleep(5) + +if failed_blueprints: + print(f"Failed to delete the following blueprints after {retries} attempts: {failed_blueprints}") +else: + print("All blueprints deleted successfully.") + print("Getting teams") res = requests.get(f'{API_URL}/teams', headers=headers) diff --git a/main.py b/main.py index c68d9ac..3e8da66 100644 --- a/main.py +++ b/main.py @@ -7,6 +7,7 @@ import openpyxl API_URL = 'https://api.getport.io/v1' +US_API_URL = 'https://api.us.getport.io/v1' global error error = False @@ -48,7 +49,7 @@ if PORT_NEW_CLIENT_ID != "" or PORT_NEW_CLIENT_SECRET != "": global new_headers new_credentials = { 'clientId': PORT_NEW_CLIENT_ID, 'clientSecret': PORT_NEW_CLIENT_SECRET } - new_credentials = requests.post(f'{API_URL}/auth/access_token', json=new_credentials) + new_credentials = requests.post(f'{US_API_URL}/auth/access_token', json=new_credentials) new_access_token = new_credentials.json()["accessToken"] new_headers = { 'Authorization': f'Bearer {new_access_token}' @@ -56,7 +57,7 @@ def getNewToken(): global new_headers - new_credentials = requests.post(f'{API_URL}/auth/access_token', json=new_credentials) + new_credentials = requests.post(f'{US_API_URL}/auth/access_token', json=new_credentials) new_access_token = new_credentials.json()["accessToken"] new_headers = { 'Authorization': f'Bearer {new_access_token}' @@ -93,13 +94,22 @@ def getScorecards(): resp = res.json()["scorecards"] return resp +def getSidebar(): + res = requests.get(f'{API_URL}/sidebars/catalog', headers=old_headers) + sidebar = res.json()['sidebar']['items'] def getActions(): print("Getting actions") - res = requests.get(f'{API_URL}/actions', headers=old_headers) + res = requests.get(f'{API_URL}/actions?version=v2', headers=old_headers) resp = res.json()["actions"] return resp +def getPages(): + print("Getting pages") + res = requests.get(f'{API_URL}/pages', headers=old_headers) + resp = res.json()["pages"] + return resp + def getTeams(): print("Getting teams") res = requests.get(f'{API_URL}/teams', headers=old_headers) @@ -124,7 +134,7 @@ def postBlueprints(blueprints): bp["aggregationProperties"] = {} bp["relations"] = {} bp["mirrorProperties"] = {} - res = requests.post(f'{API_URL}/blueprints', headers=new_headers, json=bp) + res = requests.post(f'{US_API_URL}/blueprints?create_catalog_page=false', headers=new_headers, json=bp) if res.ok != True: print("error posting blueprint:" + json.dumps(res.json())) error = True @@ -134,17 +144,42 @@ def postBlueprints(blueprints): blueprint.pop("teamInheritance", None) blueprint["aggregationProperties"] = {} blueprint["mirrorProperties"] = {} - res = requests.patch(f'{API_URL}/blueprints/{blueprint["identifier"]}', headers=new_headers, json=blueprint) + res = requests.patch(f'{US_API_URL}/blueprints/{blueprint["identifier"]}', headers=new_headers, json=blueprint) if res.ok != True: print("error patching blueprint:" + json.dumps(res.json())) error = True for blueprint in blueprints: # send blueprint with everything print(f"patching blueprint {blueprint['identifier']} with mirror properties") - res = requests.patch(f'{API_URL}/blueprints/{blueprint["identifier"]}', headers=new_headers, json=blueprint) + res = requests.patch(f'{US_API_URL}/blueprints/{blueprint["identifier"]}', headers=new_headers, json=blueprint) if res.ok != True: print("error patching blueprint:" + json.dumps(res.json())) error = True +def postPages(pages, sidebar): + for catalog_item in sidebar: + if catalog_item['sidebarType'] == 'page': #catalog_item is a page + pageToPost = next((page for page in pages if page['identifier'] == catalog_item['identifier']), None) + print(f"posting page {catalog_item['identifier']}") + res = requests.post(f'{US_API_URL}/pages', headers=new_headers, json=pageToPost) + if res.ok != True: + print("error posting page:" + json.dumps(res.json())) + error = True + else: #catalog_item is a folder + catalog_item.pop("createdAt", None) + catalog_item.pop("updatedAt", None) + catalog_item.pop("createdBy", None) + catalog_item.pop("updatedBy", None) + catalog_item.pop("_orgId", None) + catalog_item.pop("_id", None) + catalog_item.pop("sidebar", None) + catalog_item.pop("sidebarType", None) + res = requests.post(f'{US_API_URL}/sidebars/catalog/folders', headers=new_headers, json=catalog_item) + if res.ok != True: + print("error posting catalog item:" + json.dumps(res.json())) + error = True + + + def postExcelEntities(entities, blueprints): for item in entities.values(): for entity in item: @@ -204,7 +239,7 @@ def postScorecards(scorecards): scorecard.pop("createdBy", None) scorecard.pop("updatedBy", None) blueprint = scorecard.pop("blueprint", None) - res = requests.post(f'{API_URL}/blueprints/{blueprint}/scorecards', headers=new_headers, json=scorecard) + res = requests.post(f'{US_API_URL}/blueprints/{blueprint}/scorecards', headers=new_headers, json=scorecard) if res.ok != True: print(f"error posting scorecard:" + json.dumps(res.json())) error = True @@ -215,7 +250,6 @@ def postActions(actions): print("Posting actions") for action in actions: print(f"posting action {action['identifier']}") - action.pop("id", None) blueprint = action.pop("blueprint", None) if pd.isna(action.get("description", "")): # check if description is NaN action["description"] = "" # set description to empty string @@ -226,7 +260,7 @@ def postActions(actions): action.pop("createdBy", None) action.pop("updatedBy", None) action.pop("id", None) - res = requests.post(f'{API_URL}/actions', headers=new_headers, json=action) + res = requests.post(f'{US_API_URL}/actions', headers=new_headers, json=action) if res.ok != True: print(f"error posting action: " + json.dumps(res.json())) teamError = True @@ -251,7 +285,7 @@ def postTeams(teams): for team in teams: if pd.isna(team.get("description", "")): # check if description is NaN team["description"] = "" # set description to empty string - res = requests.post(f'{API_URL}/teams', headers=new_headers, json=team) + res = requests.post(f'{US_API_URL}/teams', headers=new_headers, json=team) if res.ok != True: print(f"error posting team {team['name']} :" + json.dumps(res.json())) error = True @@ -265,6 +299,8 @@ def main(): blueprints = getBlueprints() scorecards = getScorecards() actions = getActions() + sidebar = getSidebar() + pages = getPages() teams = getTeams() entities = {} for blueprint in blueprints: @@ -280,6 +316,10 @@ def main(): json.dump(actions, outfile) with open('bk-teams.json', 'w') as outfile: json.dump(teams, outfile) + with open('bk-pages.json', 'w') as outfile: + json.dump(pages, outfile) + with open('bk-sidebar', 'w') as outfile: + json.dump(sidebar, outfile) with open('bk-entities.json', 'w') as outfile: json.dump(entities, outfile) else: @@ -287,11 +327,15 @@ def main(): df_scorecards = pd.DataFrame(scorecards).map(lambda x: json.dumps(x) if isinstance(x, dict) or isinstance(x,list) else x) df_actions = pd.DataFrame(actions).map(lambda x: json.dumps(x) if isinstance(x, dict) or isinstance(x,list) else x) df_teams = pd.DataFrame(teams).map(lambda x: json.dumps(x) if isinstance(x, dict) or isinstance(x,list) else x) + df_pages = pd.DataFrame(pages).map(lambda x: json.dumps(x) if isinstance(x, dict) or isinstance(x,list) else x) + df_sidebar = pd.DataFrame(sidebar).map(lambda x: json.dumps(x) if isinstance(x, dict) or isinstance(x,list) else x) with pd.ExcelWriter('bk-data.xlsx') as writer: df_blueprints.to_excel(writer, sheet_name='Blueprints', index=False) df_scorecards.to_excel(writer, sheet_name='Scorecards', index=False) df_actions.to_excel(writer, sheet_name='Actions', index=False) df_teams.to_excel(writer, sheet_name='Teams', index=False) + df_pages.to_excel(writer, sheet_name='Pages', index=False) + df_sidebar.to_excel(writer, sheet_name='Sidebar', index=False) for blueprint, entity_list in entities.items(): for entity in entity_list: for key in entity["properties"]: @@ -303,7 +347,7 @@ def main(): entity.pop("relations", None) df = pd.DataFrame(entity_list).map(lambda x: json.dumps(x) if isinstance(x, dict) or isinstance(x,list) else x) df.to_excel(writer, sheet_name=blueprint, index=False) - + if RUN_MODE == "migrate" or RUN_MODE == "restore": if RUN_MODE == "restore": if FORMAT == "tar": @@ -315,6 +359,10 @@ def main(): actions = json.load(json_file) with open('bk-teams.json') as json_file: teams = json.load(json_file) + with open('bk-pages') as json_file: + pages = json.load(json_file) + with open('bk-sidebar') as json_file: + sidebar = json.load(json_file) with open('bk-entities.json') as json_file: entities = json.load(json_file) else: @@ -322,10 +370,12 @@ def main(): scorecards = parser((pd.read_excel(os.getenv("FILE_NAME"), sheet_name='Scorecards').to_dict(orient='records'))) actions = parser((pd.read_excel(os.getenv("FILE_NAME"), sheet_name='Actions').to_dict(orient='records'))) teams = parser((pd.read_excel(os.getenv("FILE_NAME"), sheet_name='Teams').to_dict(orient='records'))) + pages = parser((pd.read_excel(os.getenv("FILE_NAME"), sheet_name='Pages').to_dict(orient='records'))) + sidebar = parser((pd.read_excel(os.getenv("FILE_NAME"), sheet_name='Sidebar').to_dict(orient='records'))) entities = {} sheet_names = pd.ExcelFile(os.getenv("FILE_NAME")).sheet_names for sheet in sheet_names: - if sheet not in ['Blueprints', 'Scorecards', 'Actions', 'Teams']: # Skip non-entity sheets + if sheet not in ['Blueprints', 'Scorecards', 'Actions', 'Teams', 'Pages', 'Sidebar']: # Skip non-entity sheets df = parser(pd.read_excel(os.getenv("FILE_NAME"), sheet_name=sheet).to_dict(orient='records')) for entity in df: entity["properties"] = {} @@ -344,11 +394,12 @@ def main(): postBlueprints(blueprints) postScorecards(scorecards) postActions(actions) + postPages(pages, sidebar) postTeams(teams) if FORMAT == "tar": - postEntities(entities, blueprints) + postEntities(entities, blueprints) else: - postExcelEntities(entities, blueprints) + postExcelEntities(entities, blueprints) if error: print("Errors occured during migration, please check logs") elif teamError: @@ -357,4 +408,4 @@ def main(): print("No errors were caught during migration") if __name__ == "__main__": - main() + main() \ No newline at end of file From c4057dae28bf0883e4327877328d51c2d1abe5cf Mon Sep 17 00:00:00 2001 From: OmriGez Date: Sun, 22 Sep 2024 17:35:36 +0300 Subject: [PATCH 3/3] reverted US API --- main.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/main.py b/main.py index 3e8da66..df0ca22 100644 --- a/main.py +++ b/main.py @@ -49,7 +49,7 @@ if PORT_NEW_CLIENT_ID != "" or PORT_NEW_CLIENT_SECRET != "": global new_headers new_credentials = { 'clientId': PORT_NEW_CLIENT_ID, 'clientSecret': PORT_NEW_CLIENT_SECRET } - new_credentials = requests.post(f'{US_API_URL}/auth/access_token', json=new_credentials) + new_credentials = requests.post(f'{API_URL}/auth/access_token', json=new_credentials) new_access_token = new_credentials.json()["accessToken"] new_headers = { 'Authorization': f'Bearer {new_access_token}' @@ -57,7 +57,7 @@ def getNewToken(): global new_headers - new_credentials = requests.post(f'{US_API_URL}/auth/access_token', json=new_credentials) + new_credentials = requests.post(f'{API_URL}/auth/access_token', json=new_credentials) new_access_token = new_credentials.json()["accessToken"] new_headers = { 'Authorization': f'Bearer {new_access_token}' @@ -134,7 +134,7 @@ def postBlueprints(blueprints): bp["aggregationProperties"] = {} bp["relations"] = {} bp["mirrorProperties"] = {} - res = requests.post(f'{US_API_URL}/blueprints?create_catalog_page=false', headers=new_headers, json=bp) + res = requests.post(f'{API_URL}/blueprints?create_catalog_page=false', headers=new_headers, json=bp) if res.ok != True: print("error posting blueprint:" + json.dumps(res.json())) error = True @@ -144,13 +144,13 @@ def postBlueprints(blueprints): blueprint.pop("teamInheritance", None) blueprint["aggregationProperties"] = {} blueprint["mirrorProperties"] = {} - res = requests.patch(f'{US_API_URL}/blueprints/{blueprint["identifier"]}', headers=new_headers, json=blueprint) + res = requests.patch(f'{API_URL}/blueprints/{blueprint["identifier"]}', headers=new_headers, json=blueprint) if res.ok != True: print("error patching blueprint:" + json.dumps(res.json())) error = True for blueprint in blueprints: # send blueprint with everything print(f"patching blueprint {blueprint['identifier']} with mirror properties") - res = requests.patch(f'{US_API_URL}/blueprints/{blueprint["identifier"]}', headers=new_headers, json=blueprint) + res = requests.patch(f'{API_URL}/blueprints/{blueprint["identifier"]}', headers=new_headers, json=blueprint) if res.ok != True: print("error patching blueprint:" + json.dumps(res.json())) error = True @@ -160,7 +160,7 @@ def postPages(pages, sidebar): if catalog_item['sidebarType'] == 'page': #catalog_item is a page pageToPost = next((page for page in pages if page['identifier'] == catalog_item['identifier']), None) print(f"posting page {catalog_item['identifier']}") - res = requests.post(f'{US_API_URL}/pages', headers=new_headers, json=pageToPost) + res = requests.post(f'{API_URL}/pages', headers=new_headers, json=pageToPost) if res.ok != True: print("error posting page:" + json.dumps(res.json())) error = True @@ -173,7 +173,7 @@ def postPages(pages, sidebar): catalog_item.pop("_id", None) catalog_item.pop("sidebar", None) catalog_item.pop("sidebarType", None) - res = requests.post(f'{US_API_URL}/sidebars/catalog/folders', headers=new_headers, json=catalog_item) + res = requests.post(f'{API_URL}/sidebars/catalog/folders', headers=new_headers, json=catalog_item) if res.ok != True: print("error posting catalog item:" + json.dumps(res.json())) error = True @@ -239,7 +239,7 @@ def postScorecards(scorecards): scorecard.pop("createdBy", None) scorecard.pop("updatedBy", None) blueprint = scorecard.pop("blueprint", None) - res = requests.post(f'{US_API_URL}/blueprints/{blueprint}/scorecards', headers=new_headers, json=scorecard) + res = requests.post(f'{API_URL}/blueprints/{blueprint}/scorecards', headers=new_headers, json=scorecard) if res.ok != True: print(f"error posting scorecard:" + json.dumps(res.json())) error = True @@ -260,7 +260,7 @@ def postActions(actions): action.pop("createdBy", None) action.pop("updatedBy", None) action.pop("id", None) - res = requests.post(f'{US_API_URL}/actions', headers=new_headers, json=action) + res = requests.post(f'{API_URL}/actions', headers=new_headers, json=action) if res.ok != True: print(f"error posting action: " + json.dumps(res.json())) teamError = True @@ -285,7 +285,7 @@ def postTeams(teams): for team in teams: if pd.isna(team.get("description", "")): # check if description is NaN team["description"] = "" # set description to empty string - res = requests.post(f'{US_API_URL}/teams', headers=new_headers, json=team) + res = requests.post(f'{API_URL}/teams', headers=new_headers, json=team) if res.ok != True: print(f"error posting team {team['name']} :" + json.dumps(res.json())) error = True