From 1940bf107fd284ed48354d3d5849c94e681e2975 Mon Sep 17 00:00:00 2001 From: Zach Wright Date: Mon, 16 Dec 2024 15:59:15 -0500 Subject: [PATCH 01/17] KPMP-5665: Move to derived --- scripts/packageChecker/packageChecker.py | 101 ++++++++++++++--------- scripts/packageChecker/requirements.txt | 3 +- 2 files changed, 66 insertions(+), 38 deletions(-) diff --git a/scripts/packageChecker/packageChecker.py b/scripts/packageChecker/packageChecker.py index 09eb54f7..7d6718aa 100644 --- a/scripts/packageChecker/packageChecker.py +++ b/scripts/packageChecker/packageChecker.py @@ -2,9 +2,11 @@ import logging import requests import os +import shutil import csv from dotenv import load_dotenv import numpy as np +import argparse load_dotenv() @@ -14,12 +16,13 @@ slack_url = "https://hooks.slack.com/services/" + slack_passcode data_directory = os.environ.get('data_directory') + class PackageChecker: def __init__(self): try: mongo_client = pymongo.MongoClient("mongodb://localhost:27017/", serverSelectionTimeoutMS=5000) - self.dataLake= mongo_client['dataLake'] + self.dataLake = mongo_client['dataLake'] except: print("Unable to connect to database") @@ -30,16 +33,16 @@ def get_expected_files(self, package): expected_file_names.append(file_info['fileName']) return expected_file_names - def find_empty_packages(self): + def find_empty_packages(self, move_derived=False): empty_package_list = [] missing_package_list = [] extra_package_list = [] - + missing_files_header = ["Package ID", "Missing Files"] missing_files_csv = open("missing_files.csv", "w") missing_writer = csv.writer(missing_files_csv) missing_writer.writerow(missing_files_header) - + extra_files_header = ['Package ID', 'Extra Files'] extra_files_csv = open("extra_files.csv", "w") extra_writer = csv.writer(extra_files_csv) @@ -47,12 +50,11 @@ def find_empty_packages(self): packages = self.dataLake.packages.find({}) mongo_files = self.dataLake.files.find({}) mongo_files_col_list = [] - + for file_names in mongo_files: - file_name = file_names['fileName'] - mongo_files_col_list.append(file_name) - - + file_name = file_names['fileName'] + mongo_files_col_list.append(file_name) + for package in packages: package_id = package["_id"] package_states = self.dataLake.state.find({"packageId": package_id}).sort("stateChangeDate", -1).limit(1) @@ -69,45 +71,46 @@ def find_empty_packages(self): for file in files: ext = os.path.splitext(file) if not ext[1] == ".bfmemo" and not "expression_matrix" in ext[0]: - actual_file_names.append(file) - + actual_file_names.append(file) + if file == "metadata.json" and len(files) == 1: empty_package_list.append(package_id) - - if (not set(expected_file_names).issubset(set(actual_file_names))) and not all(p == "metadata.json" for p in actual_file_names): + if (not set(expected_file_names).issubset(set(actual_file_names))) and not all( + p == "metadata.json" for p in actual_file_names): empty_package_list.append(package_id) - - missing_files_list = set(expected_file_names).difference(set(actual_file_names)) + + missing_files_list = set(expected_file_names).difference(set(actual_file_names)) missing_files_list = ', '.join(missing_files_list) disk_files = set(actual_file_names).difference(set(expected_file_names)) - + if "metadata.json" in disk_files: - disk_files.remove("metadata.json") - + disk_files.remove("metadata.json") + disk_files = ", ".join(disk_files) files_list = np.setdiff1d(disk_files, mongo_files_col_list) extra_files_list = list(files_list) - + if '' in extra_files_list: - extra_files_list.remove("") - + extra_files_list.remove("") + if len(missing_files_list) != 0 and file_name not in missing_files_list: - data = [ - [package_id, missing_files_list] - ] - missing_writer.writerows(data) - + data = [ + [package_id, missing_files_list] + ] + missing_writer.writerows(data) + if len(extra_files_list) != 0 and file_name not in extra_files_list: - extra_package_list.append(package_id) - data = [ - [package_id, extra_files_list] - ] - extra_writer.writerows(data) - + extra_package_list.append(package_id) + data = [ + [package_id, extra_files_list] + ] + extra_writer.writerows(data) + self.move_file_to_derived(directory, file_name) + except: missing_package_list.append(package_id) - + missing_files_csv.close() extra_files_csv.close() if len(empty_package_list) > 0: @@ -115,7 +118,7 @@ def find_empty_packages(self): requests.post( slack_url, headers={'Content-type': 'application/json', }, - data='{"text":"' + message+'"}') + data='{"text":"' + message + '"}') if len(missing_package_list) > 0: message = "Missing package directories for packages: " + ', '.join(missing_package_list) requests.post( @@ -125,11 +128,35 @@ def find_empty_packages(self): if len(extra_package_list) > 0: message = "Extra files for packages: " + ', '.join(extra_package_list) requests.post( - slack_url, - headers={'Content-type': 'application/json', }, + slack_url, + headers={'Content-type': 'application/json', }, data='{"text":"' + message + '"}') + def move_file_to_derived(self, package_directory, file_name): + derived_dir = package_directory + "/derived" + file_path = package_directory + file_name + print(f"Moving file '{file_path}' to '{derived_dir}'.") + + # if not os.path.exists(derived_dir): + # os.makedirs(derived_dir) + # + # try: + # shutil.move(file_path, derived_dir) + # print(f"File '{file_path}' moved to '{derived_dir}' successfully.") + # except FileNotFoundError: + # print(f"Error: File '{source_file}' not found.") + # except Exception as e: + # print(f"An error occurred: {e}") + if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "-m", + "--move_derived", + required=False, + action='store_true' + ) + args = parser.parse_args() checker = PackageChecker() - checker.find_empty_packages() \ No newline at end of file + checker.find_empty_packages(args.move_derived) diff --git a/scripts/packageChecker/requirements.txt b/scripts/packageChecker/requirements.txt index 385c24b9..9dd33031 100644 --- a/scripts/packageChecker/requirements.txt +++ b/scripts/packageChecker/requirements.txt @@ -1,3 +1,4 @@ pymongo slack_sdk -numpy \ No newline at end of file +numpy +shutil \ No newline at end of file From d9697b2af3529ac1e9279db42a953c796550c0a8 Mon Sep 17 00:00:00 2001 From: Zach Wright Date: Mon, 16 Dec 2024 16:01:10 -0500 Subject: [PATCH 02/17] KPMP-5665: Use that var --- scripts/packageChecker/packageChecker.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/packageChecker/packageChecker.py b/scripts/packageChecker/packageChecker.py index 7d6718aa..cf679ac2 100644 --- a/scripts/packageChecker/packageChecker.py +++ b/scripts/packageChecker/packageChecker.py @@ -106,7 +106,8 @@ def find_empty_packages(self, move_derived=False): [package_id, extra_files_list] ] extra_writer.writerows(data) - self.move_file_to_derived(directory, file_name) + if (move_derived): + self.move_file_to_derived(directory, file_name) except: missing_package_list.append(package_id) From a0deceaaebfa63168f6261e95798bf588694e524 Mon Sep 17 00:00:00 2001 From: Zach Wright Date: Tue, 17 Dec 2024 10:39:27 -0500 Subject: [PATCH 03/17] KPMP-5665: turn off slack for now --- scripts/packageChecker/packageChecker.py | 26 ++++++++++++------------ 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/scripts/packageChecker/packageChecker.py b/scripts/packageChecker/packageChecker.py index cf679ac2..b02a5e43 100644 --- a/scripts/packageChecker/packageChecker.py +++ b/scripts/packageChecker/packageChecker.py @@ -12,7 +12,7 @@ slack_passcode = os.environ.get('slack_passcode') logger = logging.getLogger("packageChecker") -logging.basicConfig(level=logging.ERROR) +logging.basicConfig(level=logging.DEBUG) slack_url = "https://hooks.slack.com/services/" + slack_passcode data_directory = os.environ.get('data_directory') @@ -116,22 +116,22 @@ def find_empty_packages(self, move_derived=False): extra_files_csv.close() if len(empty_package_list) > 0: message = "Missing files in packages: " + ', '.join(empty_package_list) - requests.post( - slack_url, - headers={'Content-type': 'application/json', }, - data='{"text":"' + message + '"}') + # requests.post( + # slack_url, + # headers={'Content-type': 'application/json', }, + # data='{"text":"' + message + '"}') if len(missing_package_list) > 0: message = "Missing package directories for packages: " + ', '.join(missing_package_list) - requests.post( - slack_url, - headers={'Content-type': 'application/json', }, - data='{"text":"' + message + '"}') + # requests.post( + # slack_url, + # headers={'Content-type': 'application/json', }, + # data='{"text":"' + message + '"}') if len(extra_package_list) > 0: message = "Extra files for packages: " + ', '.join(extra_package_list) - requests.post( - slack_url, - headers={'Content-type': 'application/json', }, - data='{"text":"' + message + '"}') + # requests.post( + # slack_url, + # headers={'Content-type': 'application/json', }, + # data='{"text":"' + message + '"}') def move_file_to_derived(self, package_directory, file_name): derived_dir = package_directory + "/derived" From 24d2a251add98e6fc107d9db50a0247388765f42 Mon Sep 17 00:00:00 2001 From: Zach Wright Date: Tue, 17 Dec 2024 11:01:22 -0500 Subject: [PATCH 04/17] KPMP-5665: add that package_id --- scripts/packageChecker/packageChecker.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/packageChecker/packageChecker.py b/scripts/packageChecker/packageChecker.py index b02a5e43..5ca5a7ae 100644 --- a/scripts/packageChecker/packageChecker.py +++ b/scripts/packageChecker/packageChecker.py @@ -61,8 +61,8 @@ def find_empty_packages(self, move_derived=False): for state in package_states: if state['state'] == "UPLOAD_SUCCEEDED": try: - directory = data_directory + "/package_"; - files = os.listdir(directory + package_id) + directory = data_directory + "/package_" + package_id; + files = os.listdir(directory) expected_file_names = self.get_expected_files(package) actual_file_names = [] if len(files) == 0: @@ -135,7 +135,7 @@ def find_empty_packages(self, move_derived=False): def move_file_to_derived(self, package_directory, file_name): derived_dir = package_directory + "/derived" - file_path = package_directory + file_name + file_path = package_directory + "/" + file_name print(f"Moving file '{file_path}' to '{derived_dir}'.") # if not os.path.exists(derived_dir): From bdff89ed36f744f014c166de9500028388e2fc5d Mon Sep 17 00:00:00 2001 From: Zach Wright Date: Tue, 17 Dec 2024 11:04:41 -0500 Subject: [PATCH 05/17] KPMP-5665: parse out those rows --- scripts/packageChecker/packageChecker.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/packageChecker/packageChecker.py b/scripts/packageChecker/packageChecker.py index 5ca5a7ae..b137f9c9 100644 --- a/scripts/packageChecker/packageChecker.py +++ b/scripts/packageChecker/packageChecker.py @@ -107,7 +107,9 @@ def find_empty_packages(self, move_derived=False): ] extra_writer.writerows(data) if (move_derived): - self.move_file_to_derived(directory, file_name) + for row in data: + for file in row[1]: + self.move_file_to_derived(directory, file_name) except: missing_package_list.append(package_id) From 43a6c41fa76a13210e440f8b5e8acb121e753b8c Mon Sep 17 00:00:00 2001 From: Zach Wright Date: Tue, 17 Dec 2024 11:09:13 -0500 Subject: [PATCH 06/17] KPMP-5665: use that name --- scripts/packageChecker/packageChecker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/packageChecker/packageChecker.py b/scripts/packageChecker/packageChecker.py index b137f9c9..c3613502 100644 --- a/scripts/packageChecker/packageChecker.py +++ b/scripts/packageChecker/packageChecker.py @@ -109,7 +109,7 @@ def find_empty_packages(self, move_derived=False): if (move_derived): for row in data: for file in row[1]: - self.move_file_to_derived(directory, file_name) + self.move_file_to_derived(directory, file) except: missing_package_list.append(package_id) From 8a2c78b99c5eee942d04745c096f722d576b9444 Mon Sep 17 00:00:00 2001 From: Zach Wright Date: Tue, 17 Dec 2024 11:13:54 -0500 Subject: [PATCH 07/17] KPMP-5665: use file list --- scripts/packageChecker/packageChecker.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/packageChecker/packageChecker.py b/scripts/packageChecker/packageChecker.py index c3613502..0365af1f 100644 --- a/scripts/packageChecker/packageChecker.py +++ b/scripts/packageChecker/packageChecker.py @@ -107,9 +107,8 @@ def find_empty_packages(self, move_derived=False): ] extra_writer.writerows(data) if (move_derived): - for row in data: - for file in row[1]: - self.move_file_to_derived(directory, file) + for file in extra_files_list: + self.move_file_to_derived(directory, file) except: missing_package_list.append(package_id) From 06f83ee0d2ca7748bf9b07135341b608cbeee5af Mon Sep 17 00:00:00 2001 From: Zach Wright Date: Tue, 17 Dec 2024 11:15:35 -0500 Subject: [PATCH 08/17] KPMP-5665: split --- scripts/packageChecker/packageChecker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/packageChecker/packageChecker.py b/scripts/packageChecker/packageChecker.py index 0365af1f..35432ab3 100644 --- a/scripts/packageChecker/packageChecker.py +++ b/scripts/packageChecker/packageChecker.py @@ -107,7 +107,7 @@ def find_empty_packages(self, move_derived=False): ] extra_writer.writerows(data) if (move_derived): - for file in extra_files_list: + for file in extra_files_list.split(", "): self.move_file_to_derived(directory, file) except: From c8bbed8b2305796da3f9df97d186cb81756971e1 Mon Sep 17 00:00:00 2001 From: Zach Wright Date: Tue, 17 Dec 2024 12:19:54 -0500 Subject: [PATCH 09/17] KPMP-5665: handle pesky np string --- scripts/packageChecker/packageChecker.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/packageChecker/packageChecker.py b/scripts/packageChecker/packageChecker.py index 35432ab3..c63387c3 100644 --- a/scripts/packageChecker/packageChecker.py +++ b/scripts/packageChecker/packageChecker.py @@ -107,8 +107,10 @@ def find_empty_packages(self, move_derived=False): ] extra_writer.writerows(data) if (move_derived): - for file in extra_files_list.split(", "): - self.move_file_to_derived(directory, file) + for file_name in str(extra_files_list).split(", "): + clean_file_name = file_name.replace("[", "").replace("]", "").replace(",", "").replace("'", "") + if clean_file_name != 'derived': + self.move_file_to_derived(directory, clean_file_name) except: missing_package_list.append(package_id) From ebc110320ca6949d97d4217b2b6c7695b7a2bd53 Mon Sep 17 00:00:00 2001 From: Zach Wright Date: Tue, 17 Dec 2024 12:33:59 -0500 Subject: [PATCH 10/17] KPMP-5665: skip svs --- scripts/packageChecker/packageChecker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/packageChecker/packageChecker.py b/scripts/packageChecker/packageChecker.py index c63387c3..b960eaab 100644 --- a/scripts/packageChecker/packageChecker.py +++ b/scripts/packageChecker/packageChecker.py @@ -109,7 +109,7 @@ def find_empty_packages(self, move_derived=False): if (move_derived): for file_name in str(extra_files_list).split(", "): clean_file_name = file_name.replace("[", "").replace("]", "").replace(",", "").replace("'", "") - if clean_file_name != 'derived': + if clean_file_name != 'derived' and 'svs' not in clean_file_name: self.move_file_to_derived(directory, clean_file_name) except: From e40cef94f7d3a77f9fa15bc9bda136cec5edefe8 Mon Sep 17 00:00:00 2001 From: Zach Wright Date: Tue, 17 Dec 2024 12:43:36 -0500 Subject: [PATCH 11/17] KPMP-5665: uncomment move, exit after 1st --- scripts/packageChecker/packageChecker.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/scripts/packageChecker/packageChecker.py b/scripts/packageChecker/packageChecker.py index b960eaab..18369e76 100644 --- a/scripts/packageChecker/packageChecker.py +++ b/scripts/packageChecker/packageChecker.py @@ -111,6 +111,7 @@ def find_empty_packages(self, move_derived=False): clean_file_name = file_name.replace("[", "").replace("]", "").replace(",", "").replace("'", "") if clean_file_name != 'derived' and 'svs' not in clean_file_name: self.move_file_to_derived(directory, clean_file_name) + os._exit(0) except: missing_package_list.append(package_id) @@ -141,16 +142,16 @@ def move_file_to_derived(self, package_directory, file_name): file_path = package_directory + "/" + file_name print(f"Moving file '{file_path}' to '{derived_dir}'.") - # if not os.path.exists(derived_dir): - # os.makedirs(derived_dir) - # - # try: - # shutil.move(file_path, derived_dir) - # print(f"File '{file_path}' moved to '{derived_dir}' successfully.") - # except FileNotFoundError: - # print(f"Error: File '{source_file}' not found.") - # except Exception as e: - # print(f"An error occurred: {e}") + if not os.path.exists(derived_dir): + os.makedirs(derived_dir) + + try: + shutil.move(file_path, derived_dir) + print(f"File '{file_path}' moved to '{derived_dir}' successfully.") + except FileNotFoundError: + print(f"Error: File '{source_file}' not found.") + except Exception as e: + print(f"An error occurred: {e}") if __name__ == "__main__": From 1571425ab05776e47abc7d05351ce61a20a65d2c Mon Sep 17 00:00:00 2001 From: Zach Wright Date: Tue, 17 Dec 2024 12:58:49 -0500 Subject: [PATCH 12/17] KPMP-5665: keep expression, skip derived --- scripts/packageChecker/packageChecker.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/scripts/packageChecker/packageChecker.py b/scripts/packageChecker/packageChecker.py index 18369e76..430180cb 100644 --- a/scripts/packageChecker/packageChecker.py +++ b/scripts/packageChecker/packageChecker.py @@ -70,7 +70,7 @@ def find_empty_packages(self, move_derived=False): else: for file in files: ext = os.path.splitext(file) - if not ext[1] == ".bfmemo" and not "expression_matrix" in ext[0]: + if not ext[1] == ".bfmemo" and not "derived" in ext[0]: actual_file_names.append(file) if file == "metadata.json" and len(files) == 1: @@ -142,16 +142,16 @@ def move_file_to_derived(self, package_directory, file_name): file_path = package_directory + "/" + file_name print(f"Moving file '{file_path}' to '{derived_dir}'.") - if not os.path.exists(derived_dir): - os.makedirs(derived_dir) - - try: - shutil.move(file_path, derived_dir) - print(f"File '{file_path}' moved to '{derived_dir}' successfully.") - except FileNotFoundError: - print(f"Error: File '{source_file}' not found.") - except Exception as e: - print(f"An error occurred: {e}") + # if not os.path.exists(derived_dir): + # os.makedirs(derived_dir) + # + # try: + # shutil.move(file_path, derived_dir) + # print(f"File '{file_path}' moved to '{derived_dir}' successfully.") + # except FileNotFoundError: + # print(f"Error: File '{source_file}' not found.") + # except Exception as e: + # print(f"An error occurred: {e}") if __name__ == "__main__": From 11310fa86aed39fa7638c8bd483aa4b76b276cc9 Mon Sep 17 00:00:00 2001 From: Zach Wright Date: Tue, 17 Dec 2024 14:13:57 -0500 Subject: [PATCH 13/17] KPMP-5665: make sure it's a file --- scripts/packageChecker/packageChecker.py | 27 +++++++++++++----------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/scripts/packageChecker/packageChecker.py b/scripts/packageChecker/packageChecker.py index 430180cb..620848a3 100644 --- a/scripts/packageChecker/packageChecker.py +++ b/scripts/packageChecker/packageChecker.py @@ -140,18 +140,21 @@ def find_empty_packages(self, move_derived=False): def move_file_to_derived(self, package_directory, file_name): derived_dir = package_directory + "/derived" file_path = package_directory + "/" + file_name - print(f"Moving file '{file_path}' to '{derived_dir}'.") - - # if not os.path.exists(derived_dir): - # os.makedirs(derived_dir) - # - # try: - # shutil.move(file_path, derived_dir) - # print(f"File '{file_path}' moved to '{derived_dir}' successfully.") - # except FileNotFoundError: - # print(f"Error: File '{source_file}' not found.") - # except Exception as e: - # print(f"An error occurred: {e}") + if os.path.isfile(file_path): + print(f"Moving file '{file_path}' to '{derived_dir}'.") + # if not os.path.exists(derived_dir): + # os.makedirs(derived_dir) + # + # try: + # shutil.move(file_path, derived_dir) + # print(f"File '{file_path}' moved to '{derived_dir}' successfully.") + # except FileNotFoundError: + # print(f"Error: File '{source_file}' not found.") + # except Exception as e: + # print(f"An error occurred: {e}") + else: + print(f"'{file_path}' is not a file. Skipping.") + if __name__ == "__main__": From 62dd74ba0b408dc56427859365b4ffc77e73b58e Mon Sep 17 00:00:00 2001 From: Zach Wright Date: Tue, 17 Dec 2024 14:14:24 -0500 Subject: [PATCH 14/17] KPMP-5665: don't exit --- scripts/packageChecker/packageChecker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/packageChecker/packageChecker.py b/scripts/packageChecker/packageChecker.py index 620848a3..1a6111c7 100644 --- a/scripts/packageChecker/packageChecker.py +++ b/scripts/packageChecker/packageChecker.py @@ -111,7 +111,7 @@ def find_empty_packages(self, move_derived=False): clean_file_name = file_name.replace("[", "").replace("]", "").replace(",", "").replace("'", "") if clean_file_name != 'derived' and 'svs' not in clean_file_name: self.move_file_to_derived(directory, clean_file_name) - os._exit(0) + # os._exit(0) except: missing_package_list.append(package_id) From 0cf3e442da2f18e7d3912a54ef6afb202d51b591 Mon Sep 17 00:00:00 2001 From: Zach Wright Date: Tue, 17 Dec 2024 14:27:52 -0500 Subject: [PATCH 15/17] KPMP-5665: don't exit, uncomment move code --- scripts/packageChecker/packageChecker.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/scripts/packageChecker/packageChecker.py b/scripts/packageChecker/packageChecker.py index 1a6111c7..30f4228a 100644 --- a/scripts/packageChecker/packageChecker.py +++ b/scripts/packageChecker/packageChecker.py @@ -111,7 +111,6 @@ def find_empty_packages(self, move_derived=False): clean_file_name = file_name.replace("[", "").replace("]", "").replace(",", "").replace("'", "") if clean_file_name != 'derived' and 'svs' not in clean_file_name: self.move_file_to_derived(directory, clean_file_name) - # os._exit(0) except: missing_package_list.append(package_id) @@ -142,16 +141,16 @@ def move_file_to_derived(self, package_directory, file_name): file_path = package_directory + "/" + file_name if os.path.isfile(file_path): print(f"Moving file '{file_path}' to '{derived_dir}'.") - # if not os.path.exists(derived_dir): - # os.makedirs(derived_dir) - # - # try: - # shutil.move(file_path, derived_dir) - # print(f"File '{file_path}' moved to '{derived_dir}' successfully.") - # except FileNotFoundError: - # print(f"Error: File '{source_file}' not found.") - # except Exception as e: - # print(f"An error occurred: {e}") + if not os.path.exists(derived_dir): + os.makedirs(derived_dir) + + try: + shutil.move(file_path, derived_dir) + print(f"File '{file_path}' moved to '{derived_dir}' successfully.") + except FileNotFoundError: + print(f"Error: File '{source_file}' not found.") + except Exception as e: + print(f"An error occurred: {e}") else: print(f"'{file_path}' is not a file. Skipping.") From 607bc8e393990b27d44046dc3526de80326cec6b Mon Sep 17 00:00:00 2001 From: Zach Wright Date: Tue, 17 Dec 2024 16:03:57 -0500 Subject: [PATCH 16/17] KPMP-5665: uncomment slack stuff --- scripts/packageChecker/packageChecker.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/scripts/packageChecker/packageChecker.py b/scripts/packageChecker/packageChecker.py index 30f4228a..7bef7f6c 100644 --- a/scripts/packageChecker/packageChecker.py +++ b/scripts/packageChecker/packageChecker.py @@ -119,22 +119,22 @@ def find_empty_packages(self, move_derived=False): extra_files_csv.close() if len(empty_package_list) > 0: message = "Missing files in packages: " + ', '.join(empty_package_list) - # requests.post( - # slack_url, - # headers={'Content-type': 'application/json', }, - # data='{"text":"' + message + '"}') + requests.post( + slack_url, + headers={'Content-type': 'application/json', }, + data='{"text":"' + message + '"}') if len(missing_package_list) > 0: message = "Missing package directories for packages: " + ', '.join(missing_package_list) - # requests.post( - # slack_url, - # headers={'Content-type': 'application/json', }, - # data='{"text":"' + message + '"}') + requests.post( + slack_url, + headers={'Content-type': 'application/json', }, + data='{"text":"' + message + '"}') if len(extra_package_list) > 0: message = "Extra files for packages: " + ', '.join(extra_package_list) - # requests.post( - # slack_url, - # headers={'Content-type': 'application/json', }, - # data='{"text":"' + message + '"}') + requests.post( + slack_url, + headers={'Content-type': 'application/json', }, + data='{"text":"' + message + '"}') def move_file_to_derived(self, package_directory, file_name): derived_dir = package_directory + "/derived" From 6d0079041416231267723e4ecda0ea3a0bd27562 Mon Sep 17 00:00:00 2001 From: Zach Wright Date: Tue, 17 Dec 2024 16:06:27 -0500 Subject: [PATCH 17/17] KPMP-5665: add help --- scripts/packageChecker/packageChecker.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/packageChecker/packageChecker.py b/scripts/packageChecker/packageChecker.py index 7bef7f6c..a9635ec5 100644 --- a/scripts/packageChecker/packageChecker.py +++ b/scripts/packageChecker/packageChecker.py @@ -162,7 +162,8 @@ def move_file_to_derived(self, package_directory, file_name): "-m", "--move_derived", required=False, - action='store_true' + action='store_true', + help='move extra files to derived folder' ) args = parser.parse_args() checker = PackageChecker()