From 40fcdf05b318fea674894118157a381cbad57fe1 Mon Sep 17 00:00:00 2001 From: sylvanie85 Date: Wed, 24 Jul 2024 13:34:28 +0000 Subject: [PATCH 1/6] new pipeline_id column --- datastore/db/bytebase/schema_nachet_0.0.11.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/datastore/db/bytebase/schema_nachet_0.0.11.sql b/datastore/db/bytebase/schema_nachet_0.0.11.sql index 63b54df9..f143b9bc 100644 --- a/datastore/db/bytebase/schema_nachet_0.0.11.sql +++ b/datastore/db/bytebase/schema_nachet_0.0.11.sql @@ -59,6 +59,7 @@ IF (EXISTS (SELECT 1 FROM information_schema.schemata WHERE schema_name = 'nache "user_id" uuid NOT NULL REFERENCES "nachet_0.0.11".users(id), "feedback_user_id" uuid, "verified" boolean DEFAULT false NOT NULL, + "pipeline_id" uuid NOT NULL REFERENCES "nachet_0.0.11"."pipeline"(id), FOREIGN KEY ("picture_id") REFERENCES "nachet_0.0.11"."picture"(id) ON DELETE CASCADE ); From c64f8104c7d70f505185e7315ff86c8efde10ca6 Mon Sep 17 00:00:00 2001 From: sylvanie85 Date: Wed, 24 Jul 2024 19:06:34 +0000 Subject: [PATCH 2/6] save pipeline_id --- datastore/__init__.py | 10 +++++- datastore/db/queries/inference/__init__.py | 8 +++-- .../db/queries/machine_learning/__init__.py | 35 ++++++++++++++++++- tests/test_datastore.py | 2 +- 4 files changed, 49 insertions(+), 6 deletions(-) diff --git a/datastore/__init__.py b/datastore/__init__.py index 7f92c5e5..8d9f5814 100644 --- a/datastore/__init__.py +++ b/datastore/__init__.py @@ -450,11 +450,19 @@ async def register_inference_result( """ try: trimmed_inference = inference_metadata.build_inference_import(inference_dict) + + model_name = inference_dict["models"][0]["name"] + model_id = machine_learning.get_model_id_from_name(cursor, model_name) + pipeline_id = machine_learning.get_pipeline_by_model_id(cursor, model_id) + inference_dict["pipeline_id"] = str(pipeline_id) + inference_id = inference.new_inference( - cursor, trimmed_inference, user_id, picture_id, type + cursor, trimmed_inference, user_id, picture_id, type, pipeline_id ) nb_object = int(inference_dict["totalBoxes"]) inference_dict["inference_id"] = str(inference_id) + + # loop through the boxes for box_index in range(nb_object): # TODO: adapt for multiple types of objects diff --git a/datastore/db/queries/inference/__init__.py b/datastore/db/queries/inference/__init__.py index 8d3ae70f..9f8a3d7c 100644 --- a/datastore/db/queries/inference/__init__.py +++ b/datastore/db/queries/inference/__init__.py @@ -23,7 +23,7 @@ class InferenceAlreadyVerifiedError(Exception): """ -def new_inference(cursor, inference, user_id: str, picture_id:str,type): +def new_inference(cursor, inference, user_id: str, picture_id:str,type, pipeline_id:str): """ This function uploads a new inference to the database. @@ -42,10 +42,11 @@ def new_inference(cursor, inference, user_id: str, picture_id:str,type): inference( inference, picture_id, - user_id + user_id, + pipeline_id ) VALUES - (%s,%s,%s) + (%s,%s,%s,%s) RETURNING id """ cursor.execute( @@ -54,6 +55,7 @@ def new_inference(cursor, inference, user_id: str, picture_id:str,type): inference, picture_id, user_id, + pipeline_id, ), ) inference_id=cursor.fetchone()[0] diff --git a/datastore/db/queries/machine_learning/__init__.py b/datastore/db/queries/machine_learning/__init__.py index 51cbab4d..5c855120 100644 --- a/datastore/db/queries/machine_learning/__init__.py +++ b/datastore/db/queries/machine_learning/__init__.py @@ -240,7 +240,40 @@ def new_pipeline_model(cursor,pipeline_id,model_id): return pipeline_model_id except(Exception): raise PipelineCreationError("Error: pipeline model not uploaded") - + +def get_pipeline_by_model_id(cursor,model_id:str): + """ + This function gets the pipeline from the model id. + + Parameters: + - cursor (cursor): The cursor of the database. + - model_id (str): The UUID of the model. + + Returns: + - The UUID of the pipeline. + """ + try: + query = """ + SELECT + pm.pipeline_id + FROM + pipeline_model as pm + WHERE + pm.model_id = %s + """ + cursor.execute( + query, + ( + model_id, + ) + ) + pipeline_id=cursor.fetchone()[0] + return pipeline_id + except(ValueError): + raise NonExistingTaskEWarning(f"Warning: the given model '{model_id}' was not found") + except(Exception): + raise PipelineCreationError("Error: pipeline not found") + def new_model(cursor, model,name,endpoint_name,task_id:int): """ This function creates a new model in the database. diff --git a/tests/test_datastore.py b/tests/test_datastore.py index 9e3876f5..538f33bc 100644 --- a/tests/test_datastore.py +++ b/tests/test_datastore.py @@ -204,7 +204,7 @@ def test_register_inference_result(self): result = asyncio.run(datastore.register_inference_result(self.cursor,self.user_id,self.inference, picture_id, model_id)) #self.cursor.execute("SELECT result FROM inference WHERE picture_id=%s AND model_id=%s",(picture_id,model_id,)) - self.assertTrue(validator.is_valid_uuid(result["inferenceId"])) + self.assertTrue(validator.is_valid_uuid(result["inference_id"])) def test_create_picture_set(self): """ From 81d6b41377e95f4787ef8832f92f24647f349b66 Mon Sep 17 00:00:00 2001 From: sylvanie85 Date: Fri, 26 Jul 2024 16:09:49 +0000 Subject: [PATCH 3/6] fixes --- datastore/Nachet/__init__.py | 19 ++++-- .../db/bytebase/schema_nachet_0.0.11.sql | 40 +------------ .../db/queries/machine_learning/__init__.py | 58 +++++-------------- 3 files changed, 31 insertions(+), 86 deletions(-) diff --git a/datastore/Nachet/__init__.py b/datastore/Nachet/__init__.py index 38df7ac0..843c4886 100644 --- a/datastore/Nachet/__init__.py +++ b/datastore/Nachet/__init__.py @@ -280,11 +280,18 @@ async def register_inference_result( """ try: trimmed_inference = inference_metadata.build_inference_import(inference_dict) + + model_name = inference_dict["models"][0]["name"] + pipeline_id = machine_learning.get_pipeline_id_from_model_name(cursor, model_name) + inference_dict["pipeline_id"] = str(pipeline_id) + inference_id = inference.new_inference( - cursor, trimmed_inference, user_id, picture_id, type + cursor, trimmed_inference, user_id, picture_id, type, pipeline_id ) nb_object = int(inference_dict["totalBoxes"]) inference_dict["inference_id"] = str(inference_id) + + # loop through the boxes for box_index in range(nb_object): # TODO: adapt for multiple types of objects @@ -343,14 +350,14 @@ async def new_correction_inference_feedback(cursor, inference_dict, type: int = TODO: doc """ try: - if "inference_id" in inference_dict.keys(): - inference_id = inference_dict["inference_id"] + if "inferenceId" in inference_dict.keys(): + inference_id = inference_dict["inferenceId"] else: raise InferenceFeedbackError( "Error: inference_id not found in the given infence_dict" ) - if "user_id" in inference_dict.keys(): - user_id = inference_dict["user_id"] + if "userId" in inference_dict.keys(): + user_id = inference_dict["userId"] if not (user.is_a_user_id(cursor, user_id)): raise InferenceFeedbackError( f"Error: user_id {user_id} not found in the database" @@ -371,7 +378,7 @@ async def new_correction_inference_feedback(cursor, inference_dict, type: int = f"Error: Inference {inference_id} is already verified" ) for object in inference_dict["boxes"]: - box_id = object["box_id"] + box_id = object["boxId"] seed_name = object["label"] seed_id = object["classId"] # flag_seed = False diff --git a/datastore/db/bytebase/schema_nachet_0.0.11.sql b/datastore/db/bytebase/schema_nachet_0.0.11.sql index 0e399157..d32cc093 100644 --- a/datastore/db/bytebase/schema_nachet_0.0.11.sql +++ b/datastore/db/bytebase/schema_nachet_0.0.11.sql @@ -60,7 +60,6 @@ IF (EXISTS (SELECT 1 FROM information_schema.schemata WHERE schema_name = 'nache "user_id" uuid NOT NULL REFERENCES "nachet_0.0.11".users(id), "feedback_user_id" uuid, "verified" boolean DEFAULT false NOT NULL, - "pipeline_id" uuid NOT NULL REFERENCES "nachet_0.0.11"."pipeline"(id), FOREIGN KEY ("picture_id") REFERENCES "nachet_0.0.11"."picture"(id) ON DELETE CASCADE ); @@ -110,7 +109,9 @@ IF (EXISTS (SELECT 1 FROM information_schema.schemata WHERE schema_name = 'nache "is_default" boolean NOT NULL default false, "data" json NOT NULL ); - + + Alter table "nachet_0.0.11"."inference" ADD "pipeline_id" uuid REFERENCES "nachet_0.0.11"."pipeline"(id); + CREATE TRIGGER "pipeline_default_trigger" BEFORE insert OR UPDATE ON "nachet_0.0.11"."pipeline" FOR EACH ROW EXECUTE FUNCTION "nachet_0.0.11".pipeline_default_trigger(); @@ -167,41 +168,6 @@ FOR EACH ROW WHEN (NEW.verified = true) EXECUTE FUNCTION verified_inference(); - - -- Trigger function for the `inference` table - CREATE OR REPLACE FUNCTION "nachet_0.0.11".update_inference_timestamp() - RETURNS TRIGGER AS $$ - BEGIN - NEW.update_at = CURRENT_TIMESTAMP; - RETURN NEW; - END; - $$ LANGUAGE plpgsql; - - -- Trigger for the `inference` table - CREATE TRIGGER inference_update_before - BEFORE UPDATE ON "nachet_0.0.11".inference - FOR EACH ROW - EXECUTE FUNCTION "nachet_0.0.11".update_inference_timestamp(); - - -- Trigger function for the `object` table - CREATE OR REPLACE FUNCTION "nachet_0.0.11".update_object_timestamp() - RETURNS TRIGGER AS $$ - BEGIN - NEW.update_at = CURRENT_TIMESTAMP; - RETURN NEW; - END; - $$ LANGUAGE plpgsql; - - updated_at - - -- Trigger for the `inference` table - CREATE TRIGGER object_update_before - BEFORE UPDATE ON "nachet_0.0.11".object - FOR EACH ROW - EXECUTE FUNCTION "nachet_0.0.11".update_object_timestamp(); - - - INSERT INTO "nachet_0.0.11".seed(name) VALUES ('Brassica napus'), ('Brassica juncea'), diff --git a/datastore/db/queries/machine_learning/__init__.py b/datastore/db/queries/machine_learning/__init__.py index 5c855120..fb03fb0a 100644 --- a/datastore/db/queries/machine_learning/__init__.py +++ b/datastore/db/queries/machine_learning/__init__.py @@ -241,36 +241,38 @@ def new_pipeline_model(cursor,pipeline_id,model_id): except(Exception): raise PipelineCreationError("Error: pipeline model not uploaded") -def get_pipeline_by_model_id(cursor,model_id:str): +def get_pipeline_id_from_model_name(cursor,model_name:str): """ - This function gets the pipeline from the model id. + This function gets the pipeline id from the model name. Parameters: - cursor (cursor): The cursor of the database. - - model_id (str): The UUID of the model. + - model_name (str): The name of the model. Returns: - - The UUID of the pipeline. + - The UUID of the model. """ try: query = """ SELECT pm.pipeline_id - FROM - pipeline_model as pm - WHERE - pm.model_id = %s + FROM + pipeline_model as pm + LEFT JOIN + model as m on m.id=pm.model_id + WHERE + m.name= %s """ cursor.execute( query, ( - model_id, - ) + model_name, + ), ) - pipeline_id=cursor.fetchone()[0] - return pipeline_id + model_id=cursor.fetchone()[0] + return model_id except(ValueError): - raise NonExistingTaskEWarning(f"Warning: the given model '{model_id}' was not found") + raise NonExistingTaskEWarning(f"Warning: the given model '{model_name}' was not found") except(Exception): raise PipelineCreationError("Error: pipeline not found") @@ -377,36 +379,6 @@ def is_a_model(cursor,model_id:str): except(Exception): raise PipelineCreationError("Error: model not found") -def get_model_id_from_name(cursor,model_name:str): - """ - This function gets the model id from the model name. - - Parameters: - - cursor (cursor): The cursor of the database. - - model_name (str): The name of the model. - - Returns: - - The UUID of the model. - """ - try: - query = """ - SELECT id - FROM model - WHERE name = %s - """ - cursor.execute( - query, - ( - model_name, - ), - ) - model_id=cursor.fetchone()[0] - return model_id - except(ValueError): - raise NonExistingTaskEWarning(f"Warning: the given model '{model_name}' was not found") - except(Exception): - raise PipelineCreationError("Error: model not found") - def get_model_id_from_endpoint(cursor,endpoint_name:str): """ This function gets the model id from the endpoint name. From 28e0fec9f1fb2c8d6843af8604e6ee8eba7d9dd2 Mon Sep 17 00:00:00 2001 From: sylvanie85 Date: Fri, 26 Jul 2024 18:14:53 +0000 Subject: [PATCH 4/6] fix tests --- .../db/queries/machine_learning/__init__.py | 30 +++++++++++++++++++ tests/Nachet/test_datastore.py | 16 +++++++--- 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/datastore/db/queries/machine_learning/__init__.py b/datastore/db/queries/machine_learning/__init__.py index fb03fb0a..a5855ef9 100644 --- a/datastore/db/queries/machine_learning/__init__.py +++ b/datastore/db/queries/machine_learning/__init__.py @@ -379,6 +379,36 @@ def is_a_model(cursor,model_id:str): except(Exception): raise PipelineCreationError("Error: model not found") +def get_model_id_from_name(cursor,model_name:str): + """ + This function gets the model id from the model name. + + Parameters: + - cursor (cursor): The cursor of the database. + - model_name (str): The name of the model. + + Returns: + - The UUID of the model. + """ + try: + query = """ + SELECT id + FROM model + WHERE name = %s + """ + cursor.execute( + query, + ( + model_name, + ), + ) + model_id=cursor.fetchone()[0] + return model_id + except(ValueError): + raise NonExistingTaskEWarning(f"Warning: the given model '{model_name}' was not found") + except(Exception): + raise PipelineCreationError("Error: model not found") + def get_model_id_from_endpoint(cursor,endpoint_name:str): """ This function gets the model id from the endpoint name. diff --git a/tests/Nachet/test_datastore.py b/tests/Nachet/test_datastore.py index 3be2c2c7..5a1ee72c 100644 --- a/tests/Nachet/test_datastore.py +++ b/tests/Nachet/test_datastore.py @@ -712,14 +712,19 @@ def setUp(self): self.cursor, self.user_id, self.inference, picture_id, model_id ) ) - self.registered_inference["user_id"] = self.user_id + # to match frontend field names : + self.registered_inference["inferenceId"] = self.registered_inference["inference_id"] + self.registered_inference.pop("inference_id") + self.registered_inference["userId"] = self.user_id self.mock_box = {"topX": 123, "topY": 456, "bottomX": 789, "bottomY": 123} - self.inference_id = self.registered_inference.get("inference_id") + self.inference_id = self.registered_inference.get("inferenceId") self.boxes_id = [] self.top_id = [] self.unreal_seed_id = Nachet.seed.new_seed(self.cursor, "unreal_seed") for box in self.registered_inference["boxes"]: - self.boxes_id.append(box["box_id"]) + box["boxId"] = box["box_id"] + box.pop("box_id") + self.boxes_id.append(box["boxId"]) self.top_id.append(box["top_id"]) box["classId"] = Nachet.seed.get_seed_id(self.cursor, box["label"]) @@ -872,7 +877,7 @@ def test_new_correction_inference_feedback_box_edited(self): ) for box in self.registered_inference["boxes"]: object_db = Nachet.inference.get_inference_object( - self.cursor, box["box_id"] + self.cursor, box["boxId"] ) # The new box metadata must be updated self.assertDictEqual(object_db[1], self.mock_box) @@ -950,3 +955,6 @@ def test_new_correction_inference_feedback_unknown_seed(self): self.assertTrue(validator.is_valid_uuid(str(object_db[4]))) # valid column must be true self.assertTrue(object_db[6]) + +if __name__ == "__main__": + unittest.main() From cf1600db15746db3c9c7aca397f13b0585821283 Mon Sep 17 00:00:00 2001 From: sylvanie85 Date: Mon, 29 Jul 2024 17:38:57 +0000 Subject: [PATCH 5/6] quik fix --- datastore/db/queries/machine_learning/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/datastore/db/queries/machine_learning/__init__.py b/datastore/db/queries/machine_learning/__init__.py index a5855ef9..c706bae9 100644 --- a/datastore/db/queries/machine_learning/__init__.py +++ b/datastore/db/queries/machine_learning/__init__.py @@ -6,6 +6,8 @@ class NonExistingTaskEWarning(UserWarning): pass class PipelineCreationError(Exception): pass +class NonExistingModelError(Exception): + pass def new_pipeline(cursor, pipeline,pipeline_name, model_ids, active:bool=False): """ @@ -272,7 +274,7 @@ def get_pipeline_id_from_model_name(cursor,model_name:str): model_id=cursor.fetchone()[0] return model_id except(ValueError): - raise NonExistingTaskEWarning(f"Warning: the given model '{model_name}' was not found") + raise NonExistingModelError(f"Warning: the given model '{model_name}' was not found") except(Exception): raise PipelineCreationError("Error: pipeline not found") @@ -405,7 +407,7 @@ def get_model_id_from_name(cursor,model_name:str): model_id=cursor.fetchone()[0] return model_id except(ValueError): - raise NonExistingTaskEWarning(f"Warning: the given model '{model_name}' was not found") + raise NonExistingModelError(f"Error: the given model '{model_name}' was not found") except(Exception): raise PipelineCreationError("Error: model not found") From 183105169a4e76f97db059ba2938b5f68055e33e Mon Sep 17 00:00:00 2001 From: sylvanie85 Date: Mon, 29 Jul 2024 17:45:27 +0000 Subject: [PATCH 6/6] quik fix --- datastore/db/queries/machine_learning/__init__.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/datastore/db/queries/machine_learning/__init__.py b/datastore/db/queries/machine_learning/__init__.py index c706bae9..1cb3a938 100644 --- a/datastore/db/queries/machine_learning/__init__.py +++ b/datastore/db/queries/machine_learning/__init__.py @@ -6,7 +6,7 @@ class NonExistingTaskEWarning(UserWarning): pass class PipelineCreationError(Exception): pass -class NonExistingModelError(Exception): +class PipelineNotFoundError(Exception): pass def new_pipeline(cursor, pipeline,pipeline_name, model_ids, active:bool=False): @@ -273,10 +273,8 @@ def get_pipeline_id_from_model_name(cursor,model_name:str): ) model_id=cursor.fetchone()[0] return model_id - except(ValueError): - raise NonExistingModelError(f"Warning: the given model '{model_name}' was not found") except(Exception): - raise PipelineCreationError("Error: pipeline not found") + raise PipelineNotFoundError(f"Error: error finding pipelin for model {model_name}") def new_model(cursor, model,name,endpoint_name,task_id:int): """ @@ -406,10 +404,8 @@ def get_model_id_from_name(cursor,model_name:str): ) model_id=cursor.fetchone()[0] return model_id - except(ValueError): - raise NonExistingModelError(f"Error: the given model '{model_name}' was not found") except(Exception): - raise PipelineCreationError("Error: model not found") + raise PipelineNotFoundError(f"Error: model not found for model name : {model_name}") def get_model_id_from_endpoint(cursor,endpoint_name:str): """