From 7f86f3486640a05d25fdbe2527b557422062c4bd Mon Sep 17 00:00:00 2001 From: Ruge Li Date: Thu, 11 Jul 2024 13:51:49 -0700 Subject: [PATCH 1/6] upload combined gradients --- cellpack/autopack/DBRecipeHandler.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/cellpack/autopack/DBRecipeHandler.py b/cellpack/autopack/DBRecipeHandler.py index c5b9bf22..38d19621 100644 --- a/cellpack/autopack/DBRecipeHandler.py +++ b/cellpack/autopack/DBRecipeHandler.py @@ -483,8 +483,16 @@ def upload_gradients(self, gradients): def upload_single_object(self, obj_name, obj_data): # replace gradient name with path to check if gradient exists in db if "gradient" in obj_data[obj_name]: - grad_name = obj_data[obj_name]["gradient"] - obj_data[obj_name]["gradient"] = self.grad_to_path_map[grad_name] + # single gradient + if isinstance(obj_data[obj_name]["gradient"], str): + grad_name = obj_data[obj_name]["gradient"] + obj_data[obj_name]["gradient"] = self.grad_to_path_map[grad_name] + # combined gradients + elif isinstance(obj_data[obj_name]["gradient"], list): + new_grad_list = [] + for grad in obj_data[obj_name]["gradient"]: + new_grad_list.append({grad: self.grad_to_path_map[grad]}) + obj_data[obj_name]["gradient"] = new_grad_list object_doc = ObjectDoc(name=obj_name, settings=obj_data[obj_name]) _, doc_id = object_doc.should_write(self.db) if doc_id: From 190b58fadae67759fe77f2d890143b9b9e823412 Mon Sep 17 00:00:00 2001 From: Ruge Li Date: Sun, 14 Jul 2024 20:21:10 -0700 Subject: [PATCH 2/6] resolve list of gradients --- cellpack/autopack/DBRecipeHandler.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/cellpack/autopack/DBRecipeHandler.py b/cellpack/autopack/DBRecipeHandler.py index 38d19621..4a496605 100644 --- a/cellpack/autopack/DBRecipeHandler.py +++ b/cellpack/autopack/DBRecipeHandler.py @@ -86,8 +86,15 @@ def as_dict(self): @staticmethod def get_reference_in_obj(downloaded_data, db): for key in CompositionDoc.KEY_TO_DICT_MAPPING: - if key in downloaded_data and db.is_reference(downloaded_data[key]): - downloaded_data[key], _ = db.get_doc_by_ref(downloaded_data[key]) + if key in downloaded_data: + # single gradient and inherited object + if db.is_reference(downloaded_data[key]): + downloaded_data[key], _ = db.get_doc_by_ref(downloaded_data[key]) + # combined gradients + elif isinstance(downloaded_data[key], list): + for gradient in downloaded_data[key]: + for gradient_name, path in gradient.items(): + gradient[gradient_name], _ = db.get_doc_by_ref(path) @staticmethod def get_reference_data(key_or_dict, db): From 167b7027d6118b179d9bbe97a697d2b12b66c1ea Mon Sep 17 00:00:00 2001 From: Ruge Li Date: Sun, 14 Jul 2024 20:42:24 -0700 Subject: [PATCH 3/6] resolve local obj data for comparing --- cellpack/autopack/DBRecipeHandler.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/cellpack/autopack/DBRecipeHandler.py b/cellpack/autopack/DBRecipeHandler.py index 4a496605..4b4cddcb 100644 --- a/cellpack/autopack/DBRecipeHandler.py +++ b/cellpack/autopack/DBRecipeHandler.py @@ -156,9 +156,19 @@ def resolve_object_data(self, object_data, prep_recipe_data): Resolve the object data from the local data. """ for key in CompositionDoc.KEY_TO_DICT_MAPPING: - if key in object_data and isinstance(object_data[key], str): - target_dict = CompositionDoc.KEY_TO_DICT_MAPPING[key] - object_data[key] = prep_recipe_data[target_dict][object_data[key]] + if key in object_data: + # single gradient and inherited object + if isinstance(object_data[key], str): + target_dict = CompositionDoc.KEY_TO_DICT_MAPPING[key] + object_data[key] = prep_recipe_data[target_dict][object_data[key]] + # combined gradients + elif isinstance(object_data[key], list): + new_grad_list = [] + for grad in object_data[key]: + new_grad_list.append( + {grad: prep_recipe_data["gradients"][grad]} + ) + object_data[key] = new_grad_list def resolve_local_regions(self, local_data, recipe_data, db): """ From f3e710894cd8871f876de8eacee7a486aa21694f Mon Sep 17 00:00:00 2001 From: Ruge Li Date: Tue, 16 Jul 2024 12:58:16 -0700 Subject: [PATCH 4/6] download combined gradients --- cellpack/autopack/DBRecipeHandler.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/cellpack/autopack/DBRecipeHandler.py b/cellpack/autopack/DBRecipeHandler.py index 4b4cddcb..292ad290 100644 --- a/cellpack/autopack/DBRecipeHandler.py +++ b/cellpack/autopack/DBRecipeHandler.py @@ -713,10 +713,20 @@ def _get_grad_and_obj(obj_data, obj_dict, grad_dict): obj_name = obj_data["name"] for key, target_dict in CompositionDoc.KEY_TO_DICT_MAPPING.items(): if key in obj_data: - item_name = obj_data[key]["name"] - target_dict = grad_dict if key == "gradient" else obj_dict - target_dict[item_name] = obj_data[key] - obj_dict[obj_name][key] = item_name + # single gradient and inherited object + if isinstance(obj_data[key], dict): + item_name = obj_data[key]["name"] + target_dict = grad_dict if key == "gradient" else obj_dict + target_dict[item_name] = obj_data[key] + obj_dict[obj_name][key] = item_name + # combined gradients + elif key == "gradient" and isinstance(obj_data[key], list): + new_grad_list = [] + for grad in obj_data[key]: + for name in grad: + grad_dict[name] = grad[name] + new_grad_list.append(name) + obj_dict[obj_name][key] = new_grad_list return obj_dict, grad_dict @staticmethod From c0ddf2adfb062725cab8d467b4ff2643375b188d Mon Sep 17 00:00:00 2001 From: Ruge Li Date: Wed, 17 Jul 2024 14:51:20 -0700 Subject: [PATCH 5/6] resolve reference in combined grads --- cellpack/autopack/DBRecipeHandler.py | 29 ++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/cellpack/autopack/DBRecipeHandler.py b/cellpack/autopack/DBRecipeHandler.py index 292ad290..073e4724 100644 --- a/cellpack/autopack/DBRecipeHandler.py +++ b/cellpack/autopack/DBRecipeHandler.py @@ -151,6 +151,19 @@ def gradient_list_to_dict(prep_recipe_data): gradient_dict[gradient["name"]] = gradient prep_recipe_data["gradients"] = gradient_dict + @staticmethod + def create_combined_gradient_list(key, obj_data, prep_data): + """ + When the gradients are combined, fetch and replace gradient data in a list. + key --> the key in the object data that we want to modify its value + obj_data --> the object data that contains the gradient list + prep_data --> the data that contains the gradients (raw data or path) we want to fetch + """ + new_grad_list = [] + for grad in obj_data[key]: + new_grad_list.append({grad: prep_data[grad]}) + obj_data[key] = new_grad_list + def resolve_object_data(self, object_data, prep_recipe_data): """ Resolve the object data from the local data. @@ -163,12 +176,9 @@ def resolve_object_data(self, object_data, prep_recipe_data): object_data[key] = prep_recipe_data[target_dict][object_data[key]] # combined gradients elif isinstance(object_data[key], list): - new_grad_list = [] - for grad in object_data[key]: - new_grad_list.append( - {grad: prep_recipe_data["gradients"][grad]} - ) - object_data[key] = new_grad_list + self.create_combined_gradient_list( + key, object_data, prep_recipe_data["gradients"] + ) def resolve_local_regions(self, local_data, recipe_data, db): """ @@ -506,10 +516,9 @@ def upload_single_object(self, obj_name, obj_data): obj_data[obj_name]["gradient"] = self.grad_to_path_map[grad_name] # combined gradients elif isinstance(obj_data[obj_name]["gradient"], list): - new_grad_list = [] - for grad in obj_data[obj_name]["gradient"]: - new_grad_list.append({grad: self.grad_to_path_map[grad]}) - obj_data[obj_name]["gradient"] = new_grad_list + CompositionDoc.create_combined_gradient_list( + "gradient", obj_data[obj_name], self.grad_to_path_map + ) object_doc = ObjectDoc(name=obj_name, settings=obj_data[obj_name]) _, doc_id = object_doc.should_write(self.db) if doc_id: From aaf8c27437c69c731ffc2fd44ca22a479c97ad3b Mon Sep 17 00:00:00 2001 From: Ruge Li Date: Wed, 17 Jul 2024 14:53:13 -0700 Subject: [PATCH 6/6] rename the helper function --- cellpack/autopack/DBRecipeHandler.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cellpack/autopack/DBRecipeHandler.py b/cellpack/autopack/DBRecipeHandler.py index 073e4724..3ad9b5b2 100644 --- a/cellpack/autopack/DBRecipeHandler.py +++ b/cellpack/autopack/DBRecipeHandler.py @@ -152,7 +152,7 @@ def gradient_list_to_dict(prep_recipe_data): prep_recipe_data["gradients"] = gradient_dict @staticmethod - def create_combined_gradient_list(key, obj_data, prep_data): + def resolve_combined_gradient(key, obj_data, prep_data): """ When the gradients are combined, fetch and replace gradient data in a list. key --> the key in the object data that we want to modify its value @@ -176,7 +176,7 @@ def resolve_object_data(self, object_data, prep_recipe_data): object_data[key] = prep_recipe_data[target_dict][object_data[key]] # combined gradients elif isinstance(object_data[key], list): - self.create_combined_gradient_list( + self.resolve_combined_gradient( key, object_data, prep_recipe_data["gradients"] ) @@ -516,7 +516,7 @@ def upload_single_object(self, obj_name, obj_data): obj_data[obj_name]["gradient"] = self.grad_to_path_map[grad_name] # combined gradients elif isinstance(obj_data[obj_name]["gradient"], list): - CompositionDoc.create_combined_gradient_list( + CompositionDoc.resolve_combined_gradient( "gradient", obj_data[obj_name], self.grad_to_path_map ) object_doc = ObjectDoc(name=obj_name, settings=obj_data[obj_name])