From 344ab5515be32407aca0e4eaa324ddff484ccb9b Mon Sep 17 00:00:00 2001 From: sylvanie85 Date: Mon, 15 Jul 2024 19:49:10 +0000 Subject: [PATCH 01/27] fix and documentation #69 --- datastore/db/queries/picture/__init__.py | 6 +++-- doc/nachet-manage-folders.md | 33 ++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/datastore/db/queries/picture/__init__.py b/datastore/db/queries/picture/__init__.py index 9c51d2a6..ffc5a331 100644 --- a/datastore/db/queries/picture/__init__.py +++ b/datastore/db/queries/picture/__init__.py @@ -342,8 +342,10 @@ def change_picture_set_id(cursor, user_id, old_picture_set_id, new_picture_set_i - picture_set_id (str): The UUID of the PictureSet to retrieve the pictures from. """ try : - if get_picture_set_owner_id(cursor, old_picture_set_id) != user_id or get_picture_set_owner_id(cursor, new_picture_set_id) != user_id: - raise PictureUpdateError(f"Error: picture set not own by user :{user_id}") + if get_picture_set_owner_id(cursor, old_picture_set_id) != user_id : + raise PictureUpdateError(f"Error: old picture set not own by user :{user_id}") + if get_picture_set_owner_id(cursor, new_picture_set_id) != user_id : + raise PictureUpdateError(f"Error: new picture set not own by user :{user_id}") query = """ UPDATE picture diff --git a/doc/nachet-manage-folders.md b/doc/nachet-manage-folders.md index 2f0a05a2..3c6b3bb5 100644 --- a/doc/nachet-manage-folders.md +++ b/doc/nachet-manage-folders.md @@ -66,6 +66,10 @@ training purposes. Our solution is to request confirmation from the user, who can decide to delete pictures from his container but let us save them, or he can delete everything anyway, for example if there has been a missed click. +Users have asked to be able to access the pictures of folders in the directory +section on frontend. We want them to be able to see each pictures name. Then a +user can select a folder this will get all pictures and their inferences. + ## Prerequisites - The user must be signed in and have an Azure Storage Container @@ -137,3 +141,32 @@ note left of FE : "Are you sure ? Everything in this folder will be deleted and end ``` + +### Get folder content user case + +```mermaid +sequenceDiagram + participant User + participant FE as Frontend + participant BE as Backend + participant DS as Datastore + + User->>FE: ApplicationStart() + FE-->>BE: /directories + BE->>DS: get_picture_sets_info(user_id) + loop for each picture set + DS-->DS: get_pictures(user_id, picture_set_id) + end + DS-->>BE: List of picture_set with pictures name + BE-->>FE: Response : List of picture_set with pictures name + User->>FE: Select Folder + FE->>BE: /get-folder-content + BE->>DS: get_picture_set_content(user_id, picture_set_id) + loop for each picture + DS-->DS: get_picture_hash(user_id, picture_id) + DS-->DS: get_picture_inference(user_id, picture_id) + end + DS-->>BE: Return pictures with inferences + BE-->>FE: Send pictures + FE-->>User: Display folder content (list of pictures) +``` From 154647f2fcd4cb2188813df14032e7d83e47dc0f Mon Sep 17 00:00:00 2001 From: sylvanie85 Date: Wed, 17 Jul 2024 20:00:12 +0000 Subject: [PATCH 02/27] update manage folders doc --- doc/nachet-manage-folders.md | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/doc/nachet-manage-folders.md b/doc/nachet-manage-folders.md index 3c6b3bb5..a467dc69 100644 --- a/doc/nachet-manage-folders.md +++ b/doc/nachet-manage-folders.md @@ -157,16 +157,24 @@ sequenceDiagram loop for each picture set DS-->DS: get_pictures(user_id, picture_set_id) end - DS-->>BE: List of picture_set with pictures name + DS-->>BE: List of picture_set with pictures data BE-->>FE: Response : List of picture_set with pictures name User->>FE: Select Folder FE->>BE: /get-folder-content - BE->>DS: get_picture_set_content(user_id, picture_set_id) - loop for each picture - DS-->DS: get_picture_hash(user_id, picture_id) - DS-->DS: get_picture_inference(user_id, picture_id) - end + BE->>DS: get_pictures_inferences(user_id, picture_set_id) + DS-->DS: get_pictures_with_inferencse(user_id, picture_set_id) DS-->>BE: Return pictures with inferences - BE-->>FE: Send pictures + BE->>DS: get_pictures_blobs(user_id, picture_set_id) + DS-->DS: get_blobs(user_id, picture_set_id) + DS-->>BE: Return pictures blobs + BE-->>BE: cache pictures and inferences + BE-->>FE: Response : True FE-->>User: Display folder content (list of pictures) + User->>FE: Select Picture + FE->>BE: /get-picture + BE->>BE: get_picture_from_cache(user_id, picture_id) + BE-->>FE: Response : Picture with hash and inference + FE-->>User: Display picture with inference if exist + + ``` From 3bf22bf296ec86c16df6ecebe33b0a8edf2d0173 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Werbrouck?= <122839953+Francois-Werbrouck@users.noreply.github.com> Date: Wed, 17 Jul 2024 14:36:34 +0000 Subject: [PATCH 03/27] Issue #68 : split datastore --- datastore/FertiScan/__init__.py | 47 +++ datastore/Nachet/__init__.py | 660 ++++++++++++++++++++++++++++++++ datastore/db/__init__.py | 18 - 3 files changed, 707 insertions(+), 18 deletions(-) create mode 100644 datastore/FertiScan/__init__.py create mode 100644 datastore/Nachet/__init__.py diff --git a/datastore/FertiScan/__init__.py b/datastore/FertiScan/__init__.py new file mode 100644 index 00000000..a722c459 --- /dev/null +++ b/datastore/FertiScan/__init__.py @@ -0,0 +1,47 @@ +import os +from dotenv import load_dotenv + + +load_dotenv() + +FERTISCAN_DB_URL = os.environ.get("FERTISCAN_DB_URL") +if FERTISCAN_DB_URL is None or FERTISCAN_DB_URL == "": + # raise ValueError("FERTISCAN_DB_URL is not set") + print("Warning: FERTISCAN_DB_URL not set") + +FERTISCAN_SCHEMA = os.environ.get("FERTISCAN_SCHEMA") +if FERTISCAN_SCHEMA is None or FERTISCAN_SCHEMA == "": + # raise ValueError("FERTISCAN_SCHEMA is not set") + print("Warning: FERTISCAN_SCHEMA not set") + +FERTISCAN_STORAGE_URL = os.environ.get("FERTISCAN_STORAGE_URL") +if FERTISCAN_STORAGE_URL is None or FERTISCAN_STORAGE_URL == "": + # raise ValueError("FERTISCAN_STORAGE_URL is not set") + print("Warning: FERTISCAN_STORAGE_URL not set") + +async def register_analysis(cursor,container_client, analysis_dict,picture_id :str,picture,folder = "General"): + """ + Register an analysis in the database + + Parameters: + - cursor: The cursor object to interact with the database. + - container_client: The container client of the user. + - analysis_dict (dict): The analysis to register in a dict string (soon to be json loaded). + - picture: The picture encoded to upload. + + Returns: + - The analysis_dict with the analysis_id added. + """ + try: + if picture_id is None or picture_id == "": + #picture_id = str(uuid.uuid4()) + print('test') + # if not azure_storage.is_a_folder(container_client, folder): + # azure_storage.create_folder(container_client, folder) + # azure_storage.upload_image(container_client, folder, picture, picture_id) + #analysis_id = analysis.new_analysis(cursor, json.dumps(analysis_dict)) + #analysis_dict["analysis_id"] = str(analysis_id) + return None + except Exception as e: + print(e.__str__()) + raise Exception("Datastore Unhandled Error") diff --git a/datastore/Nachet/__init__.py b/datastore/Nachet/__init__.py new file mode 100644 index 00000000..3ba515aa --- /dev/null +++ b/datastore/Nachet/__init__.py @@ -0,0 +1,660 @@ +import os +from dotenv import load_dotenv +import datastore.db.queries.inference as inference +import datastore.db.queries.machine_learning as machine_learning +import datastore.db.metadata.machine_learning as ml_metadata +import datastore.db.metadata.inference as inference_metadata +import datastore.db.metadata.validator as validator +import datastore.db.queries.seed as seed +import datastore.db.queries.user as user +import datastore.db.queries.picture as picture +import datastore.db.metadata.picture_set as data_picture_set +import datastore.blob.azure_storage_api as azure_storage +import json +from azure.storage.blob import BlobServiceClient,ContainerClient +from datastore import ( + get_user_container_client, + BlobUploadError, + FolderCreationError, + UserNotOwnerError, + +) +load_dotenv() + +NACHET_BLOB_ACCOUNT = os.environ.get("NACHET_BLOB_ACCOUNT") +if NACHET_BLOB_ACCOUNT is None or NACHET_BLOB_ACCOUNT == "": + raise ValueError("NACHET_BLOB_ACCOUNT is not set") + +NACHET_BLOB_KEY = os.environ.get("NACHET_BLOB_KEY") +if NACHET_BLOB_KEY is None or NACHET_BLOB_KEY == "": + raise ValueError("NACHET_BLOB_KEY is not set") + +NACHET_STORAGE_URL = os.environ.get("NACHET_STORAGE_URL") +if NACHET_STORAGE_URL is None or NACHET_STORAGE_URL == "": + raise ValueError("NACHET_STORAGE_URL is not set") + +DEV_USER_EMAIL = os.environ.get("DEV_USER_EMAIL") +if DEV_USER_EMAIL is None or DEV_USER_EMAIL == "": + # raise ValueError("DEV_USER_EMAIL is not set") + print("Warning: DEV_USER_EMAIL not set") + +NACHET_DB_URL = os.environ.get("NACHET_DB_URL") +if NACHET_DB_URL is None or NACHET_DB_URL == "": + raise ValueError("NACHET_DB_URL is not set") + +NACHET_SCHEMA = os.environ.get("NACHET_SCHEMA") +if NACHET_SCHEMA is None or NACHET_SCHEMA == "": + raise ValueError("NACHET_SCHEMA is not set") + +class InferenceCreationError(Exception): + pass + +class InferenceFeedbackError(Exception): + pass + + +class MLRetrievalError(Exception): + pass + + +async def upload_picture_unknown(cursor, user_id, picture_hash, container_client, picture_set_id=None): + """ + Upload a picture that we don't know the seed to the user container + + Parameters: + - cursor: The cursor object to interact with the database. + - user_id (str): The UUID of the user. + - picture (str): The image to upload. + - container_client: The container client of the user. + """ + try: + + if not user.is_a_user_id(cursor=cursor, user_id=user_id): + raise user.UserNotFoundError( + f"User not found based on the given id: {user_id}" + ) + + empty_picture = json.dumps([]) + + default_picture_set = str(user.get_default_picture_set(cursor, user_id)) + if picture_set_id is None or str(picture_set_id) == default_picture_set: + picture_set_id = default_picture_set + folder_name = "General" + else : + folder_name = picture.get_picture_set_name(cursor, picture_set_id) + if folder_name is None : + folder_name = picture_set_id + + # Create picture instance in DB + picture_id = picture.new_picture_unknown( + cursor=cursor, + picture=empty_picture, + picture_set_id=picture_set_id, + ) + # Upload the picture to the Blob Storage + response = await azure_storage.upload_image( + container_client, folder_name, picture_set_id, picture_hash, picture_id + ) + # Update the picture metadata in the DB + data = { + "link": f"{folder_name}/" + str(picture_id), + "description": "Uploaded through the API", + } + + if not response: + raise BlobUploadError("Error uploading the picture") + + picture.update_picture_metadata(cursor, picture_id, json.dumps(data),0) + + return picture_id + except BlobUploadError or azure_storage.UploadImageError: + raise BlobUploadError("Error uploading the picture") + except Exception as e: + print(e) + raise Exception("Datastore Unhandled Error") + +async def upload_picture_known(cursor, user_id, picture_hash, container_client, seed_id, picture_set_id=None, nb_seeds=None, zoom_level=None): + """ + Upload a picture that the seed is known to the user container + + Parameters: + - cursor: The cursor object to interact with the database. + - user_id (str): The UUID of the user. + - picture (str): The image to upload. + - container_client: The container client of the user. + - seed_id: The UUID of the seed on the image. + - picture_set_id: The UUID of the picture set where to add the picture. + - nb_seeds: The number of seeds on the picture. + - zoom_level: The zoom level of the picture. + """ + try: + + if not user.is_a_user_id(cursor=cursor, user_id=user_id): + raise user.UserNotFoundError( + f"User not found based on the given id: {user_id}" + ) + + empty_picture = json.dumps([]) + # Create picture instance in DB + if picture_set_id is None : + picture_set_id = user.get_default_picture_set(cursor, user_id) + picture_id = picture.new_picture( + cursor=cursor, + picture=empty_picture, + picture_set_id=picture_set_id, + seed_id=seed_id, + ) + # Upload the picture to the Blob Storage + folder_name = picture.get_picture_set_name(cursor, picture_set_id) + if folder_name is None: + folder_name = picture_set_id + + response = await azure_storage.upload_image( + container_client, folder_name, picture_set_id, picture_hash, picture_id + ) + picture_link = container_client.url + "/" + str(folder_name) + "/" + str(picture_id) + # Create picture metadata and update DB instance (with link to Azure blob) + """ + data = picture_metadata.build_picture( + pic_encoded=picture_hash, + link=picture_link, + nb_seeds=nb_seeds, + zoom=zoom_level, + description="upload_picture_set script", + ) + """ + data = { + "link": picture_link, + "nb_seeds":nb_seeds, + "zoom":zoom_level, + "description": "Uploaded through the API", + } + if not response: + raise BlobUploadError("Error uploading the picture") + + picture.update_picture_metadata(cursor, picture_id, json.dumps(data),0) + + return picture_id + except BlobUploadError or azure_storage.UploadImageError: + raise BlobUploadError("Error uploading the picture") + except (user.UserNotFoundError) as e: + raise e + except Exception as e: + print(e) + raise Exception("Datastore Unhandled Error") + +async def upload_pictures(cursor, user_id, picture_set_id, container_client, pictures, seed_name: str, zoom_level: float = None, nb_seeds: int = None) : + """ + Upload an array of pictures that the seed is known to the user container + + Parameters: + - cursor: The cursor object to interact with the database. + - user_id (str): The UUID of the user. + - picture (str): The image to upload. + - container_client: The container client of the user. + - pictures array: array of images to upload + - seed_name: The name of the seed on the images. + - picture_set_id: The UUID of the picture set where to add the pictures. + - nb_seeds: The number of seeds on the picture. + - zoom_level: The zoom level of the picture. + + Returns: + array of the new pictures UUID + """ + try: + + if not seed.is_seed_registered(cursor=cursor, seed_name=seed_name): + raise seed.SeedNotFoundError( + f"Seed not found based on the given name: {seed_name}" + ) + seed_id = seed.get_seed_id(cursor=cursor, seed_name=seed_name) + + pictures_id = [] + for picture_encoded in pictures: + id = await upload_picture_known(cursor, user_id, picture_encoded, container_client, seed_id, picture_set_id, nb_seeds, zoom_level) + pictures_id.append(id) + + return pictures_id + except (seed.SeedNotFoundError) as e: + raise e + except user.UserNotFoundError as e: + raise e + except Exception: + raise BlobUploadError("An error occured during the upload of the pictures") + +async def register_inference_result( + cursor, + user_id: str, + inference_dict, + picture_id: str, + pipeline_id: str, + type: int = 1, +): + """ + Register an inference result in the database + + Parameters: + - cursor: The cursor object to interact with the database. + - user_id (str): The UUID of the user. + - inference (str): The inference to register in a dict string (soon to be json loaded). + - picture_id (str): The UUID of the picture. + - pipeline_id (str): The UUID of the pipeline. + + Returns: + - The inference_dict with the inference_id, box_id and top_id added. + """ + try: + trimmed_inference = inference_metadata.build_inference_import(inference_dict) + inference_id = inference.new_inference( + cursor, trimmed_inference, user_id, picture_id, type + ) + nb_object = int(inference_dict["totalBoxes"]) + inference_dict["inferenceId"] = str(inference_id) + # loop through the boxes + for box_index in range(nb_object): + # TODO: adapt for multiple types of objects + if type == 1: + # TODO : adapt for the seed_id in the inference_dict + top_id = seed.get_seed_id( + cursor, inference_dict["boxes"][box_index]["label"] + ) + inference_dict["boxes"][box_index]["object_type_id"] = 1 + else: + raise inference.InferenceCreationError("Error: type not recognized") + box = inference_metadata.build_object_import( + inference_dict["boxes"][box_index] + ) + object_inference_id = inference.new_inference_object( + cursor, inference_id, box, type, False + ) + inference_dict["boxes"][box_index]["boxId"] = str(object_inference_id) + # loop through the topN Prediction + top_score = -1 + if "topN" in inference_dict["boxes"][box_index]: + for topN in inference_dict["boxes"][box_index]["topN"]: + + # Retrieve the right seed_id + seed_id = seed.get_seed_id(cursor, topN["label"]) + id = inference.new_seed_object( + cursor, seed_id, object_inference_id, topN["score"] + ) + topN["object_id"] = str(id) + if topN["score"] > top_score: + top_score = topN["score"] + top_id = id + else: + seed_id = seed.get_seed_id(cursor, inference_dict["boxes"][box_index]["label"]) + top_id = inference.new_seed_object(cursor, seed_id, object_inference_id, inference_dict["boxes"][box_index]["score"]) + inference.set_inference_object_top_id(cursor, object_inference_id, top_id) + inference_dict["boxes"][box_index]["top_id"] = str(top_id) + + return inference_dict + except ValueError: + raise ValueError("The value of 'totalBoxes' is not an integer.") + except Exception as e: + print(e.__str__()) + raise Exception("Unhandled Error") + +async def new_correction_inference_feedback(cursor,inference_dict, type: int = 1): + """ + TODO: doc + """ + try: + if "inferenceId" in inference_dict.keys(): + inference_id = inference_dict["inferenceId"] + else: + raise InferenceFeedbackError("Error: inference_id not found in the given infence_dict") + if "userId" in inference_dict.keys(): + user_id = inference_dict["userId"] + if not (user.is_a_user_id(cursor, user_id)): + raise InferenceFeedbackError(f"Error: user_id {user_id} not found in the database") + else: + raise InferenceFeedbackError("Error: user_id not found in the given infence_dict") + # if infence_dict["totalBoxes"] != len(inference_dict["boxes"] & infence_dict["totalBoxes"] > 0 ): + # if len(inference_dict["boxes"]) == 0: + # raise InferenceFeedbackError("Error: No boxes found in the given inference_dict") + # else if len(inference_dict["boxes"]) > infence_dict["totalBoxes"]: + # raise InferenceFeedbackError("Error: There are more boxes than the totalBoxes") + # else if len(inference_dict["boxes"]) < infence_dict["totalBoxes"]: + # raise InferenceFeedbackError("Error: There are less boxes than the totalBoxes") + if inference.is_inference_verified(cursor, inference_id): + raise InferenceFeedbackError(f"Error: Inference {inference_id} is already verified") + for object in inference_dict["boxes"]: + box_id = object["boxId"] + seed_name = object["label"] + seed_id = object["classId"] + # flag_seed = False + # flag_box_metadata = False + valid = False + box_metadata = object["box"] + + if box_id =="": + # This is a new box created by the user + + # Check if the seed is known + if seed_id == "" and seed_name == "": + raise InferenceFeedbackError("Error: seed_name and seed_id not found in the new box. We don't know what to do with it and this should not happen.") + if seed_id == "": + if seed.is_seed_registered(cursor, seed_name): + # Mistake from the FE, the seed is known in the database + seed_id = seed.get_seed_id(cursor, seed_name) + else: + #unknown seed + seed_id = seed.new_seed(cursor, seed_name) + # Create the new object + object_id = inference.new_inference_object(cursor, inference_id, box_metadata, 1,True) + seed_object_id = inference.new_seed_object(cursor, seed_id, object_id, 0) + # Set the verified_id to the seed_object_id + inference.set_inference_object_verified_id(cursor, object_id, seed_object_id) + valid = True + else: + if (inference.is_object_verified(cursor, box_id)): + raise InferenceFeedbackError(f"Error: Object {box_id} is already verified") + # This is a box that was created by the pipeline so it should be within the database + object_db = inference.get_inference_object(cursor, box_id) + object_metadata = object_db[1] + object_id = object_db[0] + + # Check if there are difference between the metadata + if not (inference_metadata.compare_object_metadata(box_metadata, object_metadata["box"])): + # Update the object metadata + # flag_box_metadata = True + inference.set_object_box_metadata(cursor, box_id, json.dumps(box_metadata)) + + # Check if the seed is known + if seed_id == "": + if seed_name == "": + # box has been deleted by the user + valid = False + else: + valid = True + if(seed.is_seed_registered(cursor, seed_name)): + # The seed is known in the database and it was a mistake from the FE + seed_id = seed.get_seed_id(cursor, seed_name) + else: # The seed is not known in the database + seed_id = seed.new_seed(cursor, seed_name) + seed_object_id = inference.new_seed_object(cursor, seed_id, object_id, 0) + inference.set_inference_object_verified_id(cursor, object_id, seed_object_id) + else: + #Box is still valid + valid = True + # Check if a new seed has been selected + top_inference_id = inference.get_inference_object_top_id(cursor, object_db[0]) + new_top_id = inference.get_seed_object_id(cursor, seed_id, box_id ) + + if new_top_id is None: + # Seed selected was not an inference guess, we need to create a new seed_object + new_top_id=inference.new_seed_object(cursor, seed_id, box_id, 0) + inference.set_inference_object_verified_id(cursor, box_id, new_top_id) + # flag_seed = True + if top_inference_id != new_top_id: + # Seed was not correctly identified, set the verified_id to the correct seed_object.id + # flag_seed = True + inference.set_inference_object_verified_id(cursor, box_id, new_top_id) + else: + # Seed was correctly identified, set the verified_id to the top_id + # flag_seed = False + inference.set_inference_object_verified_id(cursor, box_id, top_inference_id) + + # Update the object validity + inference.set_inference_object_valid(cursor, box_id, valid) + inference.verify_inference_status(cursor, inference_id, user_id) + except InferenceFeedbackError: + raise + except Exception as e: + print(e.__str__()) + raise Exception("Datastore Unhandled Error") + +async def new_perfect_inference_feeback(cursor, inference_id, user_id, boxes_id) : + """ + Update objects when a perfect feedback is sent by a user and update the inference if all the objects in it are verified. + + Args: + cursor: The cursor object to interact with the database. + inference_id (str): id of the inference on which feedback is given + user_id (str): id of the user giving a feedback + boxes_id (str array): array of id of the objects that are correctly identified + """ + try: + # Check if user exists + if not user.is_a_user_id(cursor=cursor, user_id=user_id): + raise user.UserNotFoundError( + f"User not found based on the given id: {user_id}" + ) + # Check if boxes_id exists + for box_id in boxes_id : + if not inference.check_inference_object_exist(cursor, box_id): + raise inference.InferenceObjectNotFoundError( + f"Error: could not get inference object for id {box_id}" + ) + # Check if inference exists + if not inference.check_inference_exist(cursor, inference_id): + raise inference.InferenceNotFoundError( + f"Inference not found based on the given id: {inference_id}" + ) + + if inference.is_inference_verified(cursor, inference_id): + raise inference.InferenceAlreadyVerifiedError( + f"Can't add feedback to a verified inference, id: {inference_id}" + ) + + for object_id in boxes_id: + top_inference_id = inference.get_inference_object_top_id(cursor, object_id) + inference.set_inference_object_verified_id(cursor, object_id, top_inference_id ) + inference.set_inference_object_valid(cursor, object_id, True) + + inference.verify_inference_status(cursor, inference_id, user_id) + + except (user.UserNotFoundError, inference.InferenceObjectNotFoundError, inference.InferenceNotFoundError, inference.InferenceAlreadyVerifiedError) as e: + raise e + except Exception as e: + print(e) + raise Exception(f"Datastore Unhandled Error : {e}") + +async def import_ml_structure_from_json_version(cursor, ml_version: dict): + """ + TODO: build tests + """ + pipelines = ml_version["pipelines"] + models = ml_version["models"] + # Create the models + for model in models: + model_db = ml_metadata.build_model_import(model) + task_id = machine_learning.get_task_id(cursor, model["task"]) + model_name = model["model_name"] + endpoint_name = model["endpoint_name"] + machine_learning.new_model(cursor, model_db, model_name, endpoint_name, task_id) + # Create the pipelines + for pipeline in pipelines: + pipeline_db = ml_metadata.build_pipeline_import(pipeline) + pipeline_name = pipeline["pipeline_name"] + model_ids = [] + # Establish the relationship between the pipelines and its models + for name_model in pipeline["models"]: + model_id = 0 + model_id = machine_learning.get_model_id_from_name(cursor, name_model) + if validator.is_valid_uuid(model_id): + model_ids.append(model_id) + else: + raise ValueError(f"Model {name_model} not found") + machine_learning.new_pipeline(cursor, pipeline_db, pipeline_name, model_ids) + +async def get_ml_structure(cursor): + """ + This function retrieves the machine learning structure from the database. + + Returns a usable json object with the machine learning structure for the FE and BE + """ + try: + ml_structure = {"pipelines": [], "models": []} + pipelines = machine_learning.get_active_pipeline(cursor) + if len(pipelines)==0: + raise MLRetrievalError("No Active pipelines found in the database.") + model_list = [] + for pipeline in pipelines: + # (id, name, active:bool, is_default: bool, data, model_ids: array) + pipeline_name = pipeline[1] + pipeline_id = pipeline[0] + default = pipeline[3] + model_ids = pipeline[5] + pipeline_dict = ml_metadata.build_pipeline_export( + pipeline[4], pipeline_name, pipeline_id, default, model_ids + ) + ml_structure["pipelines"].append(pipeline_dict) + for model_id in model_ids: + if model_id not in model_list: + model_list.append(model_id) + model_db = machine_learning.get_model(cursor, model_id) + # (id, name, endpoint_name, task_name, data,version: str) + model_name = model_db[1] + model_endpoint = model_db[2] + model_task = model_db[3] + model_version = model_db[5] + model_dict = ml_metadata.build_model_export( + model_db[4], + model_id, + model_name, + model_endpoint, + model_task, + model_version, + ) + ml_structure["models"].append(model_dict) + return ml_structure + except MLRetrievalError: + raise + except Exception as e: + print(e) + raise Exception("Datastore Unhandled Error") + +async def get_seed_info(cursor): + """ + This function retrieves the seed information from the database. + + Returns a usable json object with the seed information for the FE and BE + """ + seeds = seed.get_all_seeds(cursor) + seed_dict = {"seeds": []} + for seed_db in seeds: + seed_id = seed_db[0] + seed_name = seed_db[1] + seed_dict["seeds"].append({"seed_id": seed_id, "seed_name": seed_name}) + return seed_dict + +async def delete_picture_set_with_archive(cursor, user_id, picture_set_id, container_client): + """ + Delete a picture set from the database and the blob storage but archives inferences and pictures in dev container + + Args: + cursor: The cursor object to interact with the database. + user_id (str): id of the user + picture_set_id (str): id of the picture set to delete + container_client: The container client of the user. + """ + try: + # Check if user exists + if not user.is_a_user_id(cursor=cursor, user_id=user_id): + raise user.UserNotFoundError( + f"User not found based on the given id: {user_id}" + ) + # Check if picture set exists + if not picture.is_a_picture_set_id(cursor, picture_set_id): + raise picture.PictureSetNotFoundError( + f"Picture set not found based on the given id: {picture_set_id}" + ) + # Check user is owner of the picture set + if picture.get_picture_set_owner_id(cursor, picture_set_id) != user_id: + raise UserNotOwnerError( + f"User can't delete this folder, user uuid :{user_id}, folder name : {picture_set_id}" + ) + # Check if the picture set is the default picture set + general_folder_id = str(user.get_default_picture_set(cursor, user_id)) + if general_folder_id == picture_set_id: + raise picture.PictureSetDeleteError( + f"User can't delete the default picture set, user uuid :{user_id}" + ) + + folder_name = picture.get_picture_set_name(cursor, picture_set_id) + if folder_name is None : + folder_name = picture_set_id + validated_pictures = picture.get_validated_pictures(cursor, picture_set_id) + + dev_user_id = user.get_user_id(cursor, DEV_USER_EMAIL) + dev_container_client = await get_user_container_client(dev_user_id) + + if not await azure_storage.is_a_folder(dev_container_client, str(user_id)): + await azure_storage.create_folder(dev_container_client, str(user_id)) + + picture_set = data_picture_set.build_picture_set(dev_user_id, len(validated_pictures)) + dev_picture_set_id = picture.new_picture_set( + cursor=cursor, picture_set=picture_set, user_id=dev_user_id, folder_name=folder_name + ) + + folder_created = await azure_storage.create_dev_container_folder(dev_container_client, str(picture_set_id), folder_name, user_id) + if not folder_created: + raise FolderCreationError(f"Error while creating this folder : {picture_set_id}") + + for picture_id in picture.get_validated_pictures(cursor, picture_set_id): + picture_metadata = picture.get_picture(cursor, picture_id) + blob_name = f"{folder_name}/{str(picture_id)}.png" + # change the link in the metadata + picture_metadata["link"] = f"{user_id}/{folder_name}/{picture_id}" + picture.update_picture_metadata(cursor, picture_id, json.dumps(picture_metadata), 0) + # set picture set to dev one + picture.update_picture_picture_set_id(cursor, picture_id, dev_picture_set_id) + # move the picture to the dev container + new_blob_name = "{}/{}/{}.png".format(user_id, folder_name, picture_id) + await azure_storage.move_blob(blob_name, new_blob_name, dev_picture_set_id, container_client, dev_container_client) + + if len(picture.get_validated_pictures(cursor, picture_set_id)) > 0: + raise picture.PictureSetDeleteError( + f"Can't delete the folder, there are still validated pictures in it, folder name : {picture_set_id}" + ) + + # Delete the folder in the blob storage + await azure_storage.delete_folder(container_client, picture_set_id) + # Delete the picture set + picture.delete_picture_set(cursor, picture_set_id) + + return dev_picture_set_id + except (user.UserNotFoundError, picture.PictureSetNotFoundError, picture.PictureSetDeleteError, UserNotOwnerError) as e: + raise e + except Exception as e: + print(f"Datastore Unhandled Error : {e}") + raise Exception("Datastore Unhandled Error") + +async def find_validated_pictures(cursor, user_id, picture_set_id): + """ + Find pictures that have been validated by the user in the given picture set + + Args: + cursor: The cursor object to interact with the database. + user_id (str): id of the user that should be the owner of the picture set + picture_set_id (str): id of the picture set + + Returns: + list of picture_id + """ + try: + # Check if user exists + if not user.is_a_user_id(cursor=cursor, user_id=user_id): + raise user.UserNotFoundError( + f"User not found based on the given id: {user_id}" + ) + # Check if picture set exists + if not picture.is_a_picture_set_id(cursor, picture_set_id): + raise picture.PictureSetNotFoundError( + f"Picture set not found based on the given id: {picture_set_id}" + ) + # Check user is owner of the picture set + if picture.get_picture_set_owner_id(cursor, picture_set_id) != user_id: + raise UserNotOwnerError( + f"User isn't owner of this folder, user uuid :{user_id}, folder uuid : {picture_set_id}" + ) + + validated_pictures_id = picture.get_validated_pictures(cursor, picture_set_id) + return validated_pictures_id + except (user.UserNotFoundError, picture.PictureSetNotFoundError, UserNotOwnerError) as e: + raise e + except Exception as e: + print(e) + raise Exception("Datastore Unhandled Error") diff --git a/datastore/db/__init__.py b/datastore/db/__init__.py index 3479f885..d58e006e 100644 --- a/datastore/db/__init__.py +++ b/datastore/db/__init__.py @@ -7,24 +7,6 @@ load_dotenv() -NACHET_DB_URL = os.environ.get("NACHET_DB_URL") -if NACHET_DB_URL is None or NACHET_DB_URL == "": - raise ValueError("NACHET_DB_URL is not set") - -NACHET_SCHEMA = os.environ.get("NACHET_SCHEMA") -if NACHET_SCHEMA is None or NACHET_SCHEMA == "": - raise ValueError("NACHET_SCHEMA is not set") - -FERTISCAN_DB_URL = os.environ.get("FERTISCAN_DB_URL") -if FERTISCAN_DB_URL is None or FERTISCAN_DB_URL == "": - # raise ValueError("FERTISCAN_DB_URL is not set") - print("Warning: FERTISCAN_DB_URL not set") - -FERTISCAN_SCHEMA = os.environ.get("FERTISCAN_SCHEMA") -if FERTISCAN_SCHEMA is None or FERTISCAN_SCHEMA == "": - # raise ValueError("FERTISCAN_SCHEMA is not set") - print("Warning: FERTISCAN_SCHEMA not set") - # def connect_db(): # """Connect to the postgresql database and return the connection.""" # connection = psycopg.connect( From 815672467b6c9800b795e8c9f8bc6eacb1f84abb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Werbrouck?= <122839953+Francois-Werbrouck@users.noreply.github.com> Date: Wed, 17 Jul 2024 14:43:56 +0000 Subject: [PATCH 04/27] Issue #68: Split Datastore --- datastore/FertiScan/__init__.py | 9 +- datastore/Nachet/__init__.py | 312 +++++++++---- datastore/__init__.py | 798 -------------------------------- 3 files changed, 223 insertions(+), 896 deletions(-) diff --git a/datastore/FertiScan/__init__.py b/datastore/FertiScan/__init__.py index a722c459..b0b309d6 100644 --- a/datastore/FertiScan/__init__.py +++ b/datastore/FertiScan/__init__.py @@ -19,7 +19,9 @@ # raise ValueError("FERTISCAN_STORAGE_URL is not set") print("Warning: FERTISCAN_STORAGE_URL not set") -async def register_analysis(cursor,container_client, analysis_dict,picture_id :str,picture,folder = "General"): +async def register_analysis( + cursor, container_client, analysis_dict, picture_id: str, picture, folder="General" +): """ Register an analysis in the database @@ -34,12 +36,11 @@ async def register_analysis(cursor,container_client, analysis_dict,picture_id :s """ try: if picture_id is None or picture_id == "": - #picture_id = str(uuid.uuid4()) - print('test') + picture_id = str(uuid.uuid4()) # if not azure_storage.is_a_folder(container_client, folder): # azure_storage.create_folder(container_client, folder) # azure_storage.upload_image(container_client, folder, picture, picture_id) - #analysis_id = analysis.new_analysis(cursor, json.dumps(analysis_dict)) + # analysis_id = analysis.new_analysis(cursor, json.dumps(analysis_dict)) #analysis_dict["analysis_id"] = str(analysis_id) return None except Exception as e: diff --git a/datastore/Nachet/__init__.py b/datastore/Nachet/__init__.py index 3ba515aa..7626f4b4 100644 --- a/datastore/Nachet/__init__.py +++ b/datastore/Nachet/__init__.py @@ -56,8 +56,11 @@ class InferenceFeedbackError(Exception): class MLRetrievalError(Exception): pass - -async def upload_picture_unknown(cursor, user_id, picture_hash, container_client, picture_set_id=None): + + +async def upload_picture_unknown( + cursor, user_id, picture_hash, container_client, picture_set_id=None +): """ Upload a picture that we don't know the seed to the user container @@ -68,23 +71,23 @@ async def upload_picture_unknown(cursor, user_id, picture_hash, container_client - container_client: The container client of the user. """ try: - + if not user.is_a_user_id(cursor=cursor, user_id=user_id): raise user.UserNotFoundError( f"User not found based on the given id: {user_id}" ) - + empty_picture = json.dumps([]) - + default_picture_set = str(user.get_default_picture_set(cursor, user_id)) if picture_set_id is None or str(picture_set_id) == default_picture_set: picture_set_id = default_picture_set folder_name = "General" - else : + else: folder_name = picture.get_picture_set_name(cursor, picture_set_id) - if folder_name is None : + if folder_name is None: folder_name = picture_set_id - + # Create picture instance in DB picture_id = picture.new_picture_unknown( cursor=cursor, @@ -100,11 +103,11 @@ async def upload_picture_unknown(cursor, user_id, picture_hash, container_client "link": f"{folder_name}/" + str(picture_id), "description": "Uploaded through the API", } - + if not response: raise BlobUploadError("Error uploading the picture") - - picture.update_picture_metadata(cursor, picture_id, json.dumps(data),0) + + picture.update_picture_metadata(cursor, picture_id, json.dumps(data), 0) return picture_id except BlobUploadError or azure_storage.UploadImageError: @@ -113,7 +116,17 @@ async def upload_picture_unknown(cursor, user_id, picture_hash, container_client print(e) raise Exception("Datastore Unhandled Error") -async def upload_picture_known(cursor, user_id, picture_hash, container_client, seed_id, picture_set_id=None, nb_seeds=None, zoom_level=None): + +async def upload_picture_known( + cursor, + user_id, + picture_hash, + container_client, + seed_id, + picture_set_id=None, + nb_seeds=None, + zoom_level=None, +): """ Upload a picture that the seed is known to the user container @@ -128,15 +141,15 @@ async def upload_picture_known(cursor, user_id, picture_hash, container_client, - zoom_level: The zoom level of the picture. """ try: - + if not user.is_a_user_id(cursor=cursor, user_id=user_id): raise user.UserNotFoundError( f"User not found based on the given id: {user_id}" ) - + empty_picture = json.dumps([]) # Create picture instance in DB - if picture_set_id is None : + if picture_set_id is None: picture_set_id = user.get_default_picture_set(cursor, user_id) picture_id = picture.new_picture( cursor=cursor, @@ -148,11 +161,13 @@ async def upload_picture_known(cursor, user_id, picture_hash, container_client, folder_name = picture.get_picture_set_name(cursor, picture_set_id) if folder_name is None: folder_name = picture_set_id - + response = await azure_storage.upload_image( container_client, folder_name, picture_set_id, picture_hash, picture_id ) - picture_link = container_client.url + "/" + str(folder_name) + "/" + str(picture_id) + picture_link = ( + container_client.url + "/" + str(folder_name) + "/" + str(picture_id) + ) # Create picture metadata and update DB instance (with link to Azure blob) """ data = picture_metadata.build_picture( @@ -165,25 +180,35 @@ async def upload_picture_known(cursor, user_id, picture_hash, container_client, """ data = { "link": picture_link, - "nb_seeds":nb_seeds, - "zoom":zoom_level, + "nb_seeds": nb_seeds, + "zoom": zoom_level, "description": "Uploaded through the API", } if not response: raise BlobUploadError("Error uploading the picture") - - picture.update_picture_metadata(cursor, picture_id, json.dumps(data),0) + + picture.update_picture_metadata(cursor, picture_id, json.dumps(data), 0) return picture_id except BlobUploadError or azure_storage.UploadImageError: raise BlobUploadError("Error uploading the picture") - except (user.UserNotFoundError) as e: + except user.UserNotFoundError as e: raise e except Exception as e: print(e) raise Exception("Datastore Unhandled Error") -async def upload_pictures(cursor, user_id, picture_set_id, container_client, pictures, seed_name: str, zoom_level: float = None, nb_seeds: int = None) : + +async def upload_pictures( + cursor, + user_id, + picture_set_id, + container_client, + pictures, + seed_name: str, + zoom_level: float = None, + nb_seeds: int = None, +): """ Upload an array of pictures that the seed is known to the user container @@ -202,26 +227,36 @@ async def upload_pictures(cursor, user_id, picture_set_id, container_client, pic array of the new pictures UUID """ try: - + if not seed.is_seed_registered(cursor=cursor, seed_name=seed_name): raise seed.SeedNotFoundError( f"Seed not found based on the given name: {seed_name}" ) seed_id = seed.get_seed_id(cursor=cursor, seed_name=seed_name) - + pictures_id = [] for picture_encoded in pictures: - id = await upload_picture_known(cursor, user_id, picture_encoded, container_client, seed_id, picture_set_id, nb_seeds, zoom_level) + id = await upload_picture_known( + cursor, + user_id, + picture_encoded, + container_client, + seed_id, + picture_set_id, + nb_seeds, + zoom_level, + ) pictures_id.append(id) return pictures_id - except (seed.SeedNotFoundError) as e: + except seed.SeedNotFoundError as e: raise e except user.UserNotFoundError as e: raise e except Exception: raise BlobUploadError("An error occured during the upload of the pictures") + async def register_inference_result( cursor, user_id: str, @@ -239,7 +274,7 @@ async def register_inference_result( - inference (str): The inference to register in a dict string (soon to be json loaded). - picture_id (str): The UUID of the picture. - pipeline_id (str): The UUID of the pipeline. - + Returns: - The inference_dict with the inference_id, box_id and top_id added. """ @@ -283,8 +318,15 @@ async def register_inference_result( top_score = topN["score"] top_id = id else: - seed_id = seed.get_seed_id(cursor, inference_dict["boxes"][box_index]["label"]) - top_id = inference.new_seed_object(cursor, seed_id, object_inference_id, inference_dict["boxes"][box_index]["score"]) + seed_id = seed.get_seed_id( + cursor, inference_dict["boxes"][box_index]["label"] + ) + top_id = inference.new_seed_object( + cursor, + seed_id, + object_inference_id, + inference_dict["boxes"][box_index]["score"], + ) inference.set_inference_object_top_id(cursor, object_inference_id, top_id) inference_dict["boxes"][box_index]["top_id"] = str(top_id) @@ -295,7 +337,8 @@ async def register_inference_result( print(e.__str__()) raise Exception("Unhandled Error") -async def new_correction_inference_feedback(cursor,inference_dict, type: int = 1): + +async def new_correction_inference_feedback(cursor, inference_dict, type: int = 1): """ TODO: doc """ @@ -303,13 +346,19 @@ async def new_correction_inference_feedback(cursor,inference_dict, type: int = 1 if "inferenceId" in inference_dict.keys(): inference_id = inference_dict["inferenceId"] else: - raise InferenceFeedbackError("Error: inference_id not found in the given infence_dict") + raise InferenceFeedbackError( + "Error: inference_id not found in the given infence_dict" + ) if "userId" in inference_dict.keys(): user_id = inference_dict["userId"] if not (user.is_a_user_id(cursor, user_id)): - raise InferenceFeedbackError(f"Error: user_id {user_id} not found in the database") + raise InferenceFeedbackError( + f"Error: user_id {user_id} not found in the database" + ) else: - raise InferenceFeedbackError("Error: user_id not found in the given infence_dict") + raise InferenceFeedbackError( + "Error: user_id not found in the given infence_dict" + ) # if infence_dict["totalBoxes"] != len(inference_dict["boxes"] & infence_dict["totalBoxes"] > 0 ): # if len(inference_dict["boxes"]) == 0: # raise InferenceFeedbackError("Error: No boxes found in the given inference_dict") @@ -318,7 +367,9 @@ async def new_correction_inference_feedback(cursor,inference_dict, type: int = 1 # else if len(inference_dict["boxes"]) < infence_dict["totalBoxes"]: # raise InferenceFeedbackError("Error: There are less boxes than the totalBoxes") if inference.is_inference_verified(cursor, inference_id): - raise InferenceFeedbackError(f"Error: Inference {inference_id} is already verified") + raise InferenceFeedbackError( + f"Error: Inference {inference_id} is already verified" + ) for object in inference_dict["boxes"]: box_id = object["boxId"] seed_name = object["label"] @@ -327,75 +378,105 @@ async def new_correction_inference_feedback(cursor,inference_dict, type: int = 1 # flag_box_metadata = False valid = False box_metadata = object["box"] - - if box_id =="": + + if box_id == "": # This is a new box created by the user - + # Check if the seed is known if seed_id == "" and seed_name == "": - raise InferenceFeedbackError("Error: seed_name and seed_id not found in the new box. We don't know what to do with it and this should not happen.") + raise InferenceFeedbackError( + "Error: seed_name and seed_id not found in the new box. We don't know what to do with it and this should not happen." + ) if seed_id == "": if seed.is_seed_registered(cursor, seed_name): # Mistake from the FE, the seed is known in the database seed_id = seed.get_seed_id(cursor, seed_name) else: - #unknown seed + # unknown seed seed_id = seed.new_seed(cursor, seed_name) # Create the new object - object_id = inference.new_inference_object(cursor, inference_id, box_metadata, 1,True) - seed_object_id = inference.new_seed_object(cursor, seed_id, object_id, 0) + object_id = inference.new_inference_object( + cursor, inference_id, box_metadata, 1, True + ) + seed_object_id = inference.new_seed_object( + cursor, seed_id, object_id, 0 + ) # Set the verified_id to the seed_object_id - inference.set_inference_object_verified_id(cursor, object_id, seed_object_id) + inference.set_inference_object_verified_id( + cursor, object_id, seed_object_id + ) valid = True - else: - if (inference.is_object_verified(cursor, box_id)): - raise InferenceFeedbackError(f"Error: Object {box_id} is already verified") + else: + if inference.is_object_verified(cursor, box_id): + raise InferenceFeedbackError( + f"Error: Object {box_id} is already verified" + ) # This is a box that was created by the pipeline so it should be within the database object_db = inference.get_inference_object(cursor, box_id) object_metadata = object_db[1] object_id = object_db[0] # Check if there are difference between the metadata - if not (inference_metadata.compare_object_metadata(box_metadata, object_metadata["box"])): + if not ( + inference_metadata.compare_object_metadata( + box_metadata, object_metadata["box"] + ) + ): # Update the object metadata # flag_box_metadata = True - inference.set_object_box_metadata(cursor, box_id, json.dumps(box_metadata)) - + inference.set_object_box_metadata( + cursor, box_id, json.dumps(box_metadata) + ) + # Check if the seed is known if seed_id == "": - if seed_name == "": + if seed_name == "": # box has been deleted by the user valid = False else: valid = True - if(seed.is_seed_registered(cursor, seed_name)): + if seed.is_seed_registered(cursor, seed_name): # The seed is known in the database and it was a mistake from the FE seed_id = seed.get_seed_id(cursor, seed_name) - else: # The seed is not known in the database + else: # The seed is not known in the database seed_id = seed.new_seed(cursor, seed_name) - seed_object_id = inference.new_seed_object(cursor, seed_id, object_id, 0) - inference.set_inference_object_verified_id(cursor, object_id, seed_object_id) - else: - #Box is still valid + seed_object_id = inference.new_seed_object( + cursor, seed_id, object_id, 0 + ) + inference.set_inference_object_verified_id( + cursor, object_id, seed_object_id + ) + else: + # Box is still valid valid = True # Check if a new seed has been selected - top_inference_id = inference.get_inference_object_top_id(cursor, object_db[0]) - new_top_id = inference.get_seed_object_id(cursor, seed_id, box_id ) - + top_inference_id = inference.get_inference_object_top_id( + cursor, object_db[0] + ) + new_top_id = inference.get_seed_object_id(cursor, seed_id, box_id) + if new_top_id is None: # Seed selected was not an inference guess, we need to create a new seed_object - new_top_id=inference.new_seed_object(cursor, seed_id, box_id, 0) - inference.set_inference_object_verified_id(cursor, box_id, new_top_id) + new_top_id = inference.new_seed_object( + cursor, seed_id, box_id, 0 + ) + inference.set_inference_object_verified_id( + cursor, box_id, new_top_id + ) # flag_seed = True if top_inference_id != new_top_id: # Seed was not correctly identified, set the verified_id to the correct seed_object.id # flag_seed = True - inference.set_inference_object_verified_id(cursor, box_id, new_top_id) + inference.set_inference_object_verified_id( + cursor, box_id, new_top_id + ) else: # Seed was correctly identified, set the verified_id to the top_id # flag_seed = False - inference.set_inference_object_verified_id(cursor, box_id, top_inference_id) - + inference.set_inference_object_verified_id( + cursor, box_id, top_inference_id + ) + # Update the object validity inference.set_inference_object_valid(cursor, box_id, valid) inference.verify_inference_status(cursor, inference_id, user_id) @@ -405,10 +486,11 @@ async def new_correction_inference_feedback(cursor,inference_dict, type: int = 1 print(e.__str__()) raise Exception("Datastore Unhandled Error") -async def new_perfect_inference_feeback(cursor, inference_id, user_id, boxes_id) : + +async def new_perfect_inference_feeback(cursor, inference_id, user_id, boxes_id): """ Update objects when a perfect feedback is sent by a user and update the inference if all the objects in it are verified. - + Args: cursor: The cursor object to interact with the database. inference_id (str): id of the inference on which feedback is given @@ -422,7 +504,7 @@ async def new_perfect_inference_feeback(cursor, inference_id, user_id, boxes_id) f"User not found based on the given id: {user_id}" ) # Check if boxes_id exists - for box_id in boxes_id : + for box_id in boxes_id: if not inference.check_inference_object_exist(cursor, box_id): raise inference.InferenceObjectNotFoundError( f"Error: could not get inference object for id {box_id}" @@ -432,25 +514,33 @@ async def new_perfect_inference_feeback(cursor, inference_id, user_id, boxes_id) raise inference.InferenceNotFoundError( f"Inference not found based on the given id: {inference_id}" ) - + if inference.is_inference_verified(cursor, inference_id): raise inference.InferenceAlreadyVerifiedError( f"Can't add feedback to a verified inference, id: {inference_id}" ) - + for object_id in boxes_id: top_inference_id = inference.get_inference_object_top_id(cursor, object_id) - inference.set_inference_object_verified_id(cursor, object_id, top_inference_id ) + inference.set_inference_object_verified_id( + cursor, object_id, top_inference_id + ) inference.set_inference_object_valid(cursor, object_id, True) - + inference.verify_inference_status(cursor, inference_id, user_id) - - except (user.UserNotFoundError, inference.InferenceObjectNotFoundError, inference.InferenceNotFoundError, inference.InferenceAlreadyVerifiedError) as e: + + except ( + user.UserNotFoundError, + inference.InferenceObjectNotFoundError, + inference.InferenceNotFoundError, + inference.InferenceAlreadyVerifiedError, + ) as e: raise e except Exception as e: print(e) raise Exception(f"Datastore Unhandled Error : {e}") - + + async def import_ml_structure_from_json_version(cursor, ml_version: dict): """ TODO: build tests @@ -479,6 +569,7 @@ async def import_ml_structure_from_json_version(cursor, ml_version: dict): raise ValueError(f"Model {name_model} not found") machine_learning.new_pipeline(cursor, pipeline_db, pipeline_name, model_ids) + async def get_ml_structure(cursor): """ This function retrieves the machine learning structure from the database. @@ -488,7 +579,7 @@ async def get_ml_structure(cursor): try: ml_structure = {"pipelines": [], "models": []} pipelines = machine_learning.get_active_pipeline(cursor) - if len(pipelines)==0: + if len(pipelines) == 0: raise MLRetrievalError("No Active pipelines found in the database.") model_list = [] for pipeline in pipelines: @@ -526,6 +617,7 @@ async def get_ml_structure(cursor): print(e) raise Exception("Datastore Unhandled Error") + async def get_seed_info(cursor): """ This function retrieves the seed information from the database. @@ -540,7 +632,10 @@ async def get_seed_info(cursor): seed_dict["seeds"].append({"seed_id": seed_id, "seed_name": seed_name}) return seed_dict -async def delete_picture_set_with_archive(cursor, user_id, picture_set_id, container_client): + +async def delete_picture_set_with_archive( + cursor, user_id, picture_set_id, container_client +): """ Delete a picture set from the database and the blob storage but archives inferences and pictures in dev container @@ -572,39 +667,58 @@ async def delete_picture_set_with_archive(cursor, user_id, picture_set_id, conta raise picture.PictureSetDeleteError( f"User can't delete the default picture set, user uuid :{user_id}" ) - + folder_name = picture.get_picture_set_name(cursor, picture_set_id) - if folder_name is None : + if folder_name is None: folder_name = picture_set_id validated_pictures = picture.get_validated_pictures(cursor, picture_set_id) - + dev_user_id = user.get_user_id(cursor, DEV_USER_EMAIL) dev_container_client = await get_user_container_client(dev_user_id) - + if not await azure_storage.is_a_folder(dev_container_client, str(user_id)): await azure_storage.create_folder(dev_container_client, str(user_id)) - - picture_set = data_picture_set.build_picture_set(dev_user_id, len(validated_pictures)) + + picture_set = data_picture_set.build_picture_set( + dev_user_id, len(validated_pictures) + ) dev_picture_set_id = picture.new_picture_set( - cursor=cursor, picture_set=picture_set, user_id=dev_user_id, folder_name=folder_name + cursor=cursor, + picture_set=picture_set, + user_id=dev_user_id, + folder_name=folder_name, ) - folder_created = await azure_storage.create_dev_container_folder(dev_container_client, str(picture_set_id), folder_name, user_id) + folder_created = await azure_storage.create_dev_container_folder( + dev_container_client, str(picture_set_id), folder_name, user_id + ) if not folder_created: - raise FolderCreationError(f"Error while creating this folder : {picture_set_id}") - + raise FolderCreationError( + f"Error while creating this folder : {picture_set_id}" + ) + for picture_id in picture.get_validated_pictures(cursor, picture_set_id): picture_metadata = picture.get_picture(cursor, picture_id) blob_name = f"{folder_name}/{str(picture_id)}.png" # change the link in the metadata picture_metadata["link"] = f"{user_id}/{folder_name}/{picture_id}" - picture.update_picture_metadata(cursor, picture_id, json.dumps(picture_metadata), 0) + picture.update_picture_metadata( + cursor, picture_id, json.dumps(picture_metadata), 0 + ) # set picture set to dev one - picture.update_picture_picture_set_id(cursor, picture_id, dev_picture_set_id) + picture.update_picture_picture_set_id( + cursor, picture_id, dev_picture_set_id + ) # move the picture to the dev container new_blob_name = "{}/{}/{}.png".format(user_id, folder_name, picture_id) - await azure_storage.move_blob(blob_name, new_blob_name, dev_picture_set_id, container_client, dev_container_client) - + await azure_storage.move_blob( + blob_name, + new_blob_name, + dev_picture_set_id, + container_client, + dev_container_client, + ) + if len(picture.get_validated_pictures(cursor, picture_set_id)) > 0: raise picture.PictureSetDeleteError( f"Can't delete the folder, there are still validated pictures in it, folder name : {picture_set_id}" @@ -614,14 +728,20 @@ async def delete_picture_set_with_archive(cursor, user_id, picture_set_id, conta await azure_storage.delete_folder(container_client, picture_set_id) # Delete the picture set picture.delete_picture_set(cursor, picture_set_id) - + return dev_picture_set_id - except (user.UserNotFoundError, picture.PictureSetNotFoundError, picture.PictureSetDeleteError, UserNotOwnerError) as e: + except ( + user.UserNotFoundError, + picture.PictureSetNotFoundError, + picture.PictureSetDeleteError, + UserNotOwnerError, + ) as e: raise e except Exception as e: print(f"Datastore Unhandled Error : {e}") raise Exception("Datastore Unhandled Error") + async def find_validated_pictures(cursor, user_id, picture_set_id): """ Find pictures that have been validated by the user in the given picture set @@ -650,10 +770,14 @@ async def find_validated_pictures(cursor, user_id, picture_set_id): raise UserNotOwnerError( f"User isn't owner of this folder, user uuid :{user_id}, folder uuid : {picture_set_id}" ) - + validated_pictures_id = picture.get_validated_pictures(cursor, picture_set_id) return validated_pictures_id - except (user.UserNotFoundError, picture.PictureSetNotFoundError, UserNotOwnerError) as e: + except ( + user.UserNotFoundError, + picture.PictureSetNotFoundError, + UserNotOwnerError, + ) as e: raise e except Exception as e: print(e) diff --git a/datastore/__init__.py b/datastore/__init__.py index 7f92c5e5..c253582a 100644 --- a/datastore/__init__.py +++ b/datastore/__init__.py @@ -4,47 +4,15 @@ """ import datastore.db.queries.user as user -import datastore.db.queries.inference as inference -import datastore.db.queries.machine_learning as machine_learning import datastore.db.queries.picture as picture -import datastore.db.metadata.machine_learning as ml_metadata -import datastore.db.metadata.inference as inference_metadata -import datastore.db.metadata.validator as validator -import datastore.db.queries.seed as seed import datastore.db.metadata.picture_set as data_picture_set import datastore.blob as blob import datastore.blob.azure_storage_api as azure_storage -import uuid -import json from azure.storage.blob import BlobServiceClient, ContainerClient -import os from dotenv import load_dotenv load_dotenv() -NACHET_BLOB_ACCOUNT = os.environ.get("NACHET_BLOB_ACCOUNT") -if NACHET_BLOB_ACCOUNT is None or NACHET_BLOB_ACCOUNT == "": - raise ValueError("NACHET_BLOB_ACCOUNT is not set") - -NACHET_BLOB_KEY = os.environ.get("NACHET_BLOB_KEY") -if NACHET_BLOB_KEY is None or NACHET_BLOB_KEY == "": - raise ValueError("NACHET_BLOB_KEY is not set") - -NACHET_STORAGE_URL = os.environ.get("NACHET_STORAGE_URL") -if NACHET_STORAGE_URL is None or NACHET_STORAGE_URL == "": - raise ValueError("NACHET_STORAGE_URL is not set") - -FERTISCAN_STORAGE_URL = os.environ.get("FERTISCAN_STORAGE_URL") -if FERTISCAN_STORAGE_URL is None or FERTISCAN_STORAGE_URL == "": - # raise ValueError("FERTISCAN_STORAGE_URL is not set") - print("Warning: FERTISCAN_STORAGE_URL not set") - -DEV_USER_EMAIL = os.environ.get("DEV_USER_EMAIL") -if DEV_USER_EMAIL is None or DEV_USER_EMAIL == "": - # raise ValueError("DEV_USER_EMAIL is not set") - print("Warning: DEV_USER_EMAIL not set") - - class UserAlreadyExistsError(Exception): pass @@ -53,10 +21,6 @@ class UserNotOwnerError(Exception): pass -class MLRetrievalError(Exception): - pass - - class BlobUploadError(Exception): pass @@ -69,13 +33,6 @@ class FolderCreationError(Exception): pass -class InferenceCreationError(Exception): - pass - - -class InferenceFeedbackError(Exception): - pass - class User: def __init__(self, email: str, id: str = None, tier: str = "user"): @@ -228,581 +185,6 @@ async def create_picture_set( raise BlobUploadError("An error occured during the upload of the picture set") -async def upload_picture_unknown( - cursor, user_id, picture_hash, container_client, picture_set_id=None -): - """ - Upload a picture that we don't know the seed to the user container - - Parameters: - - cursor: The cursor object to interact with the database. - - user_id (str): The UUID of the user. - - picture (str): The image to upload. - - container_client: The container client of the user. - """ - try: - - if not user.is_a_user_id(cursor=cursor, user_id=user_id): - raise user.UserNotFoundError( - f"User not found based on the given id: {user_id}" - ) - - empty_picture = json.dumps([]) - - default_picture_set = str(user.get_default_picture_set(cursor, user_id)) - if picture_set_id is None or str(picture_set_id) == default_picture_set: - picture_set_id = default_picture_set - folder_name = "General" - else: - folder_name = picture.get_picture_set_name(cursor, picture_set_id) - if folder_name is None: - folder_name = picture_set_id - - # Create picture instance in DB - picture_id = picture.new_picture_unknown( - cursor=cursor, - picture=empty_picture, - picture_set_id=picture_set_id, - ) - # Upload the picture to the Blob Storage - response = await azure_storage.upload_image( - container_client, folder_name, picture_set_id, picture_hash, picture_id - ) - # Update the picture metadata in the DB - data = { - "link": f"{folder_name}/" + str(picture_id), - "description": "Uploaded through the API", - } - - if not response: - raise BlobUploadError("Error uploading the picture") - - picture.update_picture_metadata(cursor, picture_id, json.dumps(data), 0) - - return picture_id - except BlobUploadError or azure_storage.UploadImageError: - raise BlobUploadError("Error uploading the picture") - except Exception as e: - print(e) - raise Exception("Datastore Unhandled Error") - - -async def upload_picture_known( - cursor, - user_id, - picture_hash, - container_client, - seed_id, - picture_set_id=None, - nb_seeds=None, - zoom_level=None, -): - """ - Upload a picture that the seed is known to the user container - - Parameters: - - cursor: The cursor object to interact with the database. - - user_id (str): The UUID of the user. - - picture (str): The image to upload. - - container_client: The container client of the user. - - seed_id: The UUID of the seed on the image. - - picture_set_id: The UUID of the picture set where to add the picture. - - nb_seeds: The number of seeds on the picture. - - zoom_level: The zoom level of the picture. - """ - try: - - if not user.is_a_user_id(cursor=cursor, user_id=user_id): - raise user.UserNotFoundError( - f"User not found based on the given id: {user_id}" - ) - - empty_picture = json.dumps([]) - # Create picture instance in DB - if picture_set_id is None: - picture_set_id = user.get_default_picture_set(cursor, user_id) - picture_id = picture.new_picture( - cursor=cursor, - picture=empty_picture, - picture_set_id=picture_set_id, - seed_id=seed_id, - ) - # Upload the picture to the Blob Storage - folder_name = picture.get_picture_set_name(cursor, picture_set_id) - if folder_name is None: - folder_name = picture_set_id - - response = await azure_storage.upload_image( - container_client, folder_name, picture_set_id, picture_hash, picture_id - ) - picture_link = ( - container_client.url + "/" + str(folder_name) + "/" + str(picture_id) - ) - # Create picture metadata and update DB instance (with link to Azure blob) - """ - data = picture_metadata.build_picture( - pic_encoded=picture_hash, - link=picture_link, - nb_seeds=nb_seeds, - zoom=zoom_level, - description="upload_picture_set script", - ) - """ - data = { - "link": picture_link, - "nb_seeds": nb_seeds, - "zoom": zoom_level, - "description": "Uploaded through the API", - } - if not response: - raise BlobUploadError("Error uploading the picture") - - picture.update_picture_metadata(cursor, picture_id, json.dumps(data), 0) - - return picture_id - except BlobUploadError or azure_storage.UploadImageError: - raise BlobUploadError("Error uploading the picture") - except user.UserNotFoundError as e: - raise e - except Exception as e: - print(e) - raise Exception("Datastore Unhandled Error") - - -async def upload_pictures( - cursor, - user_id, - picture_set_id, - container_client, - pictures, - seed_name: str, - zoom_level: float = None, - nb_seeds: int = None, -): - """ - Upload an array of pictures that the seed is known to the user container - - Parameters: - - cursor: The cursor object to interact with the database. - - user_id (str): The UUID of the user. - - picture (str): The image to upload. - - container_client: The container client of the user. - - pictures array: array of images to upload - - seed_name: The name of the seed on the images. - - picture_set_id: The UUID of the picture set where to add the pictures. - - nb_seeds: The number of seeds on the picture. - - zoom_level: The zoom level of the picture. - - Returns: - array of the new pictures UUID - """ - try: - - if not seed.is_seed_registered(cursor=cursor, seed_name=seed_name): - raise seed.SeedNotFoundError( - f"Seed not found based on the given name: {seed_name}" - ) - seed_id = seed.get_seed_id(cursor=cursor, seed_name=seed_name) - - pictures_id = [] - for picture_encoded in pictures: - id = await upload_picture_known( - cursor, - user_id, - picture_encoded, - container_client, - seed_id, - picture_set_id, - nb_seeds, - zoom_level, - ) - pictures_id.append(id) - - return pictures_id - except seed.SeedNotFoundError as e: - raise e - except user.UserNotFoundError as e: - raise e - except Exception: - raise BlobUploadError("An error occured during the upload of the pictures") - - -async def register_inference_result( - cursor, - user_id: str, - inference_dict, - picture_id: str, - pipeline_id: str, - type: int = 1, -): - """ - Register an inference result in the database - - Parameters: - - cursor: The cursor object to interact with the database. - - user_id (str): The UUID of the user. - - inference (str): The inference to register in a dict string (soon to be json loaded). - - picture_id (str): The UUID of the picture. - - pipeline_id (str): The UUID of the pipeline. - - Returns: - - The inference_dict with the inference_id, box_id and top_id added. - """ - try: - trimmed_inference = inference_metadata.build_inference_import(inference_dict) - inference_id = inference.new_inference( - cursor, trimmed_inference, user_id, picture_id, type - ) - nb_object = int(inference_dict["totalBoxes"]) - inference_dict["inference_id"] = str(inference_id) - # loop through the boxes - for box_index in range(nb_object): - # TODO: adapt for multiple types of objects - if type == 1: - # TODO : adapt for the seed_id in the inference_dict - top_id = seed.get_seed_id( - cursor, inference_dict["boxes"][box_index]["label"] - ) - inference_dict["boxes"][box_index]["object_type_id"] = 1 - else: - raise inference.InferenceCreationError("Error: type not recognized") - box = inference_metadata.build_object_import( - inference_dict["boxes"][box_index] - ) - object_inference_id = inference.new_inference_object( - cursor, inference_id, box, type, False - ) - inference_dict["boxes"][box_index]["box_id"] = str(object_inference_id) - # loop through the topN Prediction - top_score = -1 - if "topN" in inference_dict["boxes"][box_index]: - for topN in inference_dict["boxes"][box_index]["topN"]: - - # Retrieve the right seed_id - seed_id = seed.get_seed_id(cursor, topN["label"]) - id = inference.new_seed_object( - cursor, seed_id, object_inference_id, topN["score"] - ) - topN["object_id"] = str(id) - if topN["score"] > top_score: - top_score = topN["score"] - top_id = id - else: - seed_id = seed.get_seed_id( - cursor, inference_dict["boxes"][box_index]["label"] - ) - top_id = inference.new_seed_object( - cursor, - seed_id, - object_inference_id, - inference_dict["boxes"][box_index]["score"], - ) - inference.set_inference_object_top_id(cursor, object_inference_id, top_id) - inference_dict["boxes"][box_index]["top_id"] = str(top_id) - - return inference_dict - except ValueError: - raise ValueError("The value of 'totalBoxes' is not an integer.") - except Exception as e: - print(e.__str__()) - raise Exception("Unhandled Error") - - -async def new_correction_inference_feedback(cursor, inference_dict, type: int = 1): - """ - TODO: doc - """ - try: - if "inferenceId" in inference_dict.keys(): - inference_id = inference_dict["inferenceId"] - else: - raise InferenceFeedbackError( - "Error: inference_id not found in the given infence_dict" - ) - if "userId" in inference_dict.keys(): - user_id = inference_dict["userId"] - if not (user.is_a_user_id(cursor, user_id)): - raise InferenceFeedbackError( - f"Error: user_id {user_id} not found in the database" - ) - else: - raise InferenceFeedbackError( - "Error: user_id not found in the given infence_dict" - ) - # if infence_dict["totalBoxes"] != len(inference_dict["boxes"] & infence_dict["totalBoxes"] > 0 ): - # if len(inference_dict["boxes"]) == 0: - # raise InferenceFeedbackError("Error: No boxes found in the given inference_dict") - # else if len(inference_dict["boxes"]) > infence_dict["totalBoxes"]: - # raise InferenceFeedbackError("Error: There are more boxes than the totalBoxes") - # else if len(inference_dict["boxes"]) < infence_dict["totalBoxes"]: - # raise InferenceFeedbackError("Error: There are less boxes than the totalBoxes") - if inference.is_inference_verified(cursor, inference_id): - raise InferenceFeedbackError( - f"Error: Inference {inference_id} is already verified" - ) - for object in inference_dict["boxes"]: - box_id = object["boxId"] - seed_name = object["label"] - seed_id = object["classId"] - # flag_seed = False - # flag_box_metadata = False - valid = False - box_metadata = object["box"] - - if box_id == "": - # This is a new box created by the user - - # Check if the seed is known - if seed_id == "" and seed_name == "": - raise InferenceFeedbackError( - "Error: seed_name and seed_id not found in the new box. We don't know what to do with it and this should not happen." - ) - if seed_id == "": - if seed.is_seed_registered(cursor, seed_name): - # Mistake from the FE, the seed is known in the database - seed_id = seed.get_seed_id(cursor, seed_name) - else: - # unknown seed - seed_id = seed.new_seed(cursor, seed_name) - # Create the new object - object_id = inference.new_inference_object( - cursor, inference_id, box_metadata, 1, True - ) - seed_object_id = inference.new_seed_object( - cursor, seed_id, object_id, 0 - ) - # Set the verified_id to the seed_object_id - inference.set_inference_object_verified_id( - cursor, object_id, seed_object_id - ) - valid = True - else: - if inference.is_object_verified(cursor, box_id): - raise InferenceFeedbackError( - f"Error: Object {box_id} is already verified" - ) - # This is a box that was created by the pipeline so it should be within the database - object_db = inference.get_inference_object(cursor, box_id) - object_metadata = object_db[1] - object_id = object_db[0] - - # Check if there are difference between the metadata - if not ( - inference_metadata.compare_object_metadata( - box_metadata, object_metadata["box"] - ) - ): - # Update the object metadata - # flag_box_metadata = True - inference.set_object_box_metadata( - cursor, box_id, json.dumps(box_metadata) - ) - - # Check if the seed is known - if seed_id == "": - if seed_name == "": - # box has been deleted by the user - valid = False - else: - valid = True - if seed.is_seed_registered(cursor, seed_name): - # The seed is known in the database and it was a mistake from the FE - seed_id = seed.get_seed_id(cursor, seed_name) - else: # The seed is not known in the database - seed_id = seed.new_seed(cursor, seed_name) - seed_object_id = inference.new_seed_object( - cursor, seed_id, object_id, 0 - ) - inference.set_inference_object_verified_id( - cursor, object_id, seed_object_id - ) - else: - # Box is still valid - valid = True - # Check if a new seed has been selected - top_inference_id = inference.get_inference_object_top_id( - cursor, object_db[0] - ) - new_top_id = inference.get_seed_object_id(cursor, seed_id, box_id) - - if new_top_id is None: - # Seed selected was not an inference guess, we need to create a new seed_object - new_top_id = inference.new_seed_object( - cursor, seed_id, box_id, 0 - ) - inference.set_inference_object_verified_id( - cursor, box_id, new_top_id - ) - # flag_seed = True - if top_inference_id != new_top_id: - # Seed was not correctly identified, set the verified_id to the correct seed_object.id - # flag_seed = True - inference.set_inference_object_verified_id( - cursor, box_id, new_top_id - ) - else: - # Seed was correctly identified, set the verified_id to the top_id - # flag_seed = False - inference.set_inference_object_verified_id( - cursor, box_id, top_inference_id - ) - - # Update the object validity - inference.set_inference_object_valid(cursor, box_id, valid) - inference.verify_inference_status(cursor, inference_id, user_id) - except InferenceFeedbackError: - raise - except Exception as e: - print(e.__str__()) - raise Exception("Datastore Unhandled Error") - - -async def new_perfect_inference_feeback(cursor, inference_id, user_id, boxes_id): - """ - Update objects when a perfect feedback is sent by a user and update the inference if all the objects in it are verified. - - Args: - cursor: The cursor object to interact with the database. - inference_id (str): id of the inference on which feedback is given - user_id (str): id of the user giving a feedback - boxes_id (str array): array of id of the objects that are correctly identified - """ - try: - # Check if user exists - if not user.is_a_user_id(cursor=cursor, user_id=user_id): - raise user.UserNotFoundError( - f"User not found based on the given id: {user_id}" - ) - # Check if boxes_id exists - for box_id in boxes_id: - if not inference.check_inference_object_exist(cursor, box_id): - raise inference.InferenceObjectNotFoundError( - f"Error: could not get inference object for id {box_id}" - ) - # Check if inference exists - if not inference.check_inference_exist(cursor, inference_id): - raise inference.InferenceNotFoundError( - f"Inference not found based on the given id: {inference_id}" - ) - - if inference.is_inference_verified(cursor, inference_id): - raise inference.InferenceAlreadyVerifiedError( - f"Can't add feedback to a verified inference, id: {inference_id}" - ) - - for object_id in boxes_id: - top_inference_id = inference.get_inference_object_top_id(cursor, object_id) - inference.set_inference_object_verified_id( - cursor, object_id, top_inference_id - ) - inference.set_inference_object_valid(cursor, object_id, True) - - inference.verify_inference_status(cursor, inference_id, user_id) - - except ( - user.UserNotFoundError, - inference.InferenceObjectNotFoundError, - inference.InferenceNotFoundError, - inference.InferenceAlreadyVerifiedError, - ) as e: - raise e - except Exception as e: - print(e) - raise Exception(f"Datastore Unhandled Error : {e}") - - -async def import_ml_structure_from_json_version(cursor, ml_version: dict): - """ - TODO: build tests - """ - pipelines = ml_version["pipelines"] - models = ml_version["models"] - # Create the models - for model in models: - model_db = ml_metadata.build_model_import(model) - task_id = machine_learning.get_task_id(cursor, model["task"]) - model_name = model["model_name"] - endpoint_name = model["endpoint_name"] - machine_learning.new_model(cursor, model_db, model_name, endpoint_name, task_id) - # Create the pipelines - for pipeline in pipelines: - pipeline_db = ml_metadata.build_pipeline_import(pipeline) - pipeline_name = pipeline["pipeline_name"] - model_ids = [] - # Establish the relationship between the pipelines and its models - for name_model in pipeline["models"]: - model_id = 0 - model_id = machine_learning.get_model_id_from_name(cursor, name_model) - if validator.is_valid_uuid(model_id): - model_ids.append(model_id) - else: - raise ValueError(f"Model {name_model} not found") - machine_learning.new_pipeline(cursor, pipeline_db, pipeline_name, model_ids) - - -async def get_ml_structure(cursor): - """ - This function retrieves the machine learning structure from the database. - - Returns a usable json object with the machine learning structure for the FE and BE - """ - try: - ml_structure = {"pipelines": [], "models": []} - pipelines = machine_learning.get_active_pipeline(cursor) - if len(pipelines) == 0: - raise MLRetrievalError("No Active pipelines found in the database.") - model_list = [] - for pipeline in pipelines: - # (id, name, active:bool, is_default: bool, data, model_ids: array) - pipeline_name = pipeline[1] - pipeline_id = pipeline[0] - default = pipeline[3] - model_ids = pipeline[5] - pipeline_dict = ml_metadata.build_pipeline_export( - pipeline[4], pipeline_name, pipeline_id, default, model_ids - ) - ml_structure["pipelines"].append(pipeline_dict) - for model_id in model_ids: - if model_id not in model_list: - model_list.append(model_id) - model_db = machine_learning.get_model(cursor, model_id) - # (id, name, endpoint_name, task_name, data,version: str) - model_name = model_db[1] - model_endpoint = model_db[2] - model_task = model_db[3] - model_version = model_db[5] - model_dict = ml_metadata.build_model_export( - model_db[4], - model_id, - model_name, - model_endpoint, - model_task, - model_version, - ) - ml_structure["models"].append(model_dict) - return ml_structure - except MLRetrievalError: - raise - except Exception as e: - print(e) - raise Exception("Datastore Unhandled Error") - - -async def get_seed_info(cursor): - """ - This function retrieves the seed information from the database. - - Returns a usable json object with the seed information for the FE and BE - """ - seeds = seed.get_all_seeds(cursor) - seed_dict = {"seeds": []} - for seed_db in seeds: - seed_id = seed_db[0] - seed_name = seed_db[1] - seed_dict["seeds"].append({"seed_id": seed_id, "seed_name": seed_name}) - return seed_dict - - async def get_picture_sets_info(cursor, user_id: str): """This function retrieves the picture sets names and number of pictures from the database. @@ -874,183 +256,3 @@ async def delete_picture_set_permanently( except Exception as e: print(e) raise Exception("Datastore Unhandled Error") - - -async def delete_picture_set_with_archive( - cursor, user_id, picture_set_id, container_client -): - """ - Delete a picture set from the database and the blob storage but archives inferences and pictures in dev container - - Args: - cursor: The cursor object to interact with the database. - user_id (str): id of the user - picture_set_id (str): id of the picture set to delete - container_client: The container client of the user. - """ - try: - # Check if user exists - if not user.is_a_user_id(cursor=cursor, user_id=user_id): - raise user.UserNotFoundError( - f"User not found based on the given id: {user_id}" - ) - # Check if picture set exists - if not picture.is_a_picture_set_id(cursor, picture_set_id): - raise picture.PictureSetNotFoundError( - f"Picture set not found based on the given id: {picture_set_id}" - ) - # Check user is owner of the picture set - if picture.get_picture_set_owner_id(cursor, picture_set_id) != user_id: - raise UserNotOwnerError( - f"User can't delete this folder, user uuid :{user_id}, folder name : {picture_set_id}" - ) - # Check if the picture set is the default picture set - general_folder_id = str(user.get_default_picture_set(cursor, user_id)) - if general_folder_id == picture_set_id: - raise picture.PictureSetDeleteError( - f"User can't delete the default picture set, user uuid :{user_id}" - ) - - folder_name = picture.get_picture_set_name(cursor, picture_set_id) - if folder_name is None: - folder_name = picture_set_id - validated_pictures = picture.get_validated_pictures(cursor, picture_set_id) - - dev_user_id = user.get_user_id(cursor, DEV_USER_EMAIL) - dev_container_client = await get_user_container_client(dev_user_id) - - if not await azure_storage.is_a_folder(dev_container_client, str(user_id)): - await azure_storage.create_folder(dev_container_client, str(user_id)) - - picture_set = data_picture_set.build_picture_set( - dev_user_id, len(validated_pictures) - ) - dev_picture_set_id = picture.new_picture_set( - cursor=cursor, - picture_set=picture_set, - user_id=dev_user_id, - folder_name=folder_name, - ) - - folder_created = await azure_storage.create_dev_container_folder( - dev_container_client, str(picture_set_id), folder_name, user_id - ) - if not folder_created: - raise FolderCreationError( - f"Error while creating this folder : {picture_set_id}" - ) - - for picture_id in picture.get_validated_pictures(cursor, picture_set_id): - picture_metadata = picture.get_picture(cursor, picture_id) - blob_name = f"{folder_name}/{str(picture_id)}.png" - # change the link in the metadata - picture_metadata["link"] = f"{user_id}/{folder_name}/{picture_id}" - picture.update_picture_metadata( - cursor, picture_id, json.dumps(picture_metadata), 0 - ) - # set picture set to dev one - picture.update_picture_picture_set_id( - cursor, picture_id, dev_picture_set_id - ) - # move the picture to the dev container - new_blob_name = "{}/{}/{}.png".format(user_id, folder_name, picture_id) - await azure_storage.move_blob( - blob_name, - new_blob_name, - dev_picture_set_id, - container_client, - dev_container_client, - ) - - if len(picture.get_validated_pictures(cursor, picture_set_id)) > 0: - raise picture.PictureSetDeleteError( - f"Can't delete the folder, there are still validated pictures in it, folder name : {picture_set_id}" - ) - - # Delete the folder in the blob storage - await azure_storage.delete_folder(container_client, picture_set_id) - # Delete the picture set - picture.delete_picture_set(cursor, picture_set_id) - - return dev_picture_set_id - except ( - user.UserNotFoundError, - picture.PictureSetNotFoundError, - picture.PictureSetDeleteError, - UserNotOwnerError, - ) as e: - raise e - except Exception as e: - print(f"Datastore Unhandled Error : {e}") - raise Exception("Datastore Unhandled Error") - - -async def find_validated_pictures(cursor, user_id, picture_set_id): - """ - Find pictures that have been validated by the user in the given picture set - - Args: - cursor: The cursor object to interact with the database. - user_id (str): id of the user that should be the owner of the picture set - picture_set_id (str): id of the picture set - - Returns: - list of picture_id - """ - try: - # Check if user exists - if not user.is_a_user_id(cursor=cursor, user_id=user_id): - raise user.UserNotFoundError( - f"User not found based on the given id: {user_id}" - ) - # Check if picture set exists - if not picture.is_a_picture_set_id(cursor, picture_set_id): - raise picture.PictureSetNotFoundError( - f"Picture set not found based on the given id: {picture_set_id}" - ) - # Check user is owner of the picture set - if picture.get_picture_set_owner_id(cursor, picture_set_id) != user_id: - raise UserNotOwnerError( - f"User isn't owner of this folder, user uuid :{user_id}, folder uuid : {picture_set_id}" - ) - - validated_pictures_id = picture.get_validated_pictures(cursor, picture_set_id) - return validated_pictures_id - except ( - user.UserNotFoundError, - picture.PictureSetNotFoundError, - UserNotOwnerError, - ) as e: - raise e - except Exception as e: - print(e) - raise Exception("Datastore Unhandled Error") - - -async def register_analysis( - cursor, container_client, analysis_dict, picture_id: str, picture, folder="General" -): - """ - Register an analysis in the database - - Parameters: - - cursor: The cursor object to interact with the database. - - container_client: The container client of the user. - - analysis_dict (dict): The analysis to register in a dict string (soon to be json loaded). - - picture: The picture encoded to upload. - - Returns: - - The analysis_dict with the analysis_id added. - """ - try: - if picture_id is None or picture_id == "": - picture_id = str(uuid.uuid4()) - # if not azure_storage.is_a_folder(container_client, folder): - # azure_storage.create_folder(container_client, folder) - # azure_storage.upload_image(container_client, folder, picture, picture_id) - # analysis_id = analysis.new_analysis(cursor, json.dumps(analysis_dict)) - #analysis_dict["analysis_id"] = str(analysis_id) - return None - except Exception as e: - print(e.__str__()) - raise Exception("Datastore Unhandled Error") From 0a0e95c5c81f57c0d3cbfa6097a54650c38a3133 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Werbrouck?= <122839953+Francois-Werbrouck@users.noreply.github.com> Date: Wed, 17 Jul 2024 15:11:18 +0000 Subject: [PATCH 05/27] Issue #68: Doc update --- doc/{ => FertiScan}/fertiScan-architecture.md | 0 doc/{ => Nachet}/deployment-mass-import.md | 0 doc/{ => Nachet}/inference-feedback.md | 0 doc/{ => Nachet}/inference-results.md | 0 doc/{ => Nachet}/nachet-architecture.md | 0 doc/{ => Nachet}/nachet-manage-folders.md | 0 doc/{ => Nachet}/trusted-user-upload.md | 0 doc/bytebase-doc.md | 17 +- doc/datastore.md | 8 +- doc/metadata-doc.md | 278 +----------------- 10 files changed, 25 insertions(+), 278 deletions(-) rename doc/{ => FertiScan}/fertiScan-architecture.md (100%) rename doc/{ => Nachet}/deployment-mass-import.md (100%) rename doc/{ => Nachet}/inference-feedback.md (100%) rename doc/{ => Nachet}/inference-results.md (100%) rename doc/{ => Nachet}/nachet-architecture.md (100%) rename doc/{ => Nachet}/nachet-manage-folders.md (100%) rename doc/{ => Nachet}/trusted-user-upload.md (100%) diff --git a/doc/fertiScan-architecture.md b/doc/FertiScan/fertiScan-architecture.md similarity index 100% rename from doc/fertiScan-architecture.md rename to doc/FertiScan/fertiScan-architecture.md diff --git a/doc/deployment-mass-import.md b/doc/Nachet/deployment-mass-import.md similarity index 100% rename from doc/deployment-mass-import.md rename to doc/Nachet/deployment-mass-import.md diff --git a/doc/inference-feedback.md b/doc/Nachet/inference-feedback.md similarity index 100% rename from doc/inference-feedback.md rename to doc/Nachet/inference-feedback.md diff --git a/doc/inference-results.md b/doc/Nachet/inference-results.md similarity index 100% rename from doc/inference-results.md rename to doc/Nachet/inference-results.md diff --git a/doc/nachet-architecture.md b/doc/Nachet/nachet-architecture.md similarity index 100% rename from doc/nachet-architecture.md rename to doc/Nachet/nachet-architecture.md diff --git a/doc/nachet-manage-folders.md b/doc/Nachet/nachet-manage-folders.md similarity index 100% rename from doc/nachet-manage-folders.md rename to doc/Nachet/nachet-manage-folders.md diff --git a/doc/trusted-user-upload.md b/doc/Nachet/trusted-user-upload.md similarity index 100% rename from doc/trusted-user-upload.md rename to doc/Nachet/trusted-user-upload.md diff --git a/doc/bytebase-doc.md b/doc/bytebase-doc.md index e1b436ee..4242878c 100644 --- a/doc/bytebase-doc.md +++ b/doc/bytebase-doc.md @@ -1,11 +1,10 @@ -# Workflow +# Bytebase -``` mermaid ---- -title: Bytebase workflow ---- -flowchart LR; +## Context - Project --> Database - Project --> GitOps -``` +We are storing our database architecture in a folder named bytebase. +[Bytebase](https://www.bytebase.com/docs/introduction/what-is-bytebase/) is a +version management tool for database, just like github. It also enables us to +connect to a database server on a cloud service. This is important because the +network policy we are working under doesn't allow us to make connection on port +1532. diff --git a/doc/datastore.md b/doc/datastore.md index 534e3881..0ebda026 100644 --- a/doc/datastore.md +++ b/doc/datastore.md @@ -46,7 +46,7 @@ your valuable data. ``` mermaid --- -title: Nachet Layers +title: Project Layers --- flowchart LR; FE(Frontend) @@ -68,8 +68,8 @@ flowchart LR; ``` ## Database Architecture - -### Needs + For more detail on each app database architecture go check [Nachet Architecture](nachetArchitecture.md) and [Fertiscan Architecture](FertiscanArchitecture.md). +### Global Needs - A User must be able to take a picture on the app and it must be saved in the blob Storage. @@ -79,5 +79,3 @@ flowchart LR; - A User can verify the result of a picture that went through the pipeline and the changes are saved for training. -- A User can verify an inference by confirming the positive result, or entering - the right value. diff --git a/doc/metadata-doc.md b/doc/metadata-doc.md index 73d921a9..a5c68527 100644 --- a/doc/metadata-doc.md +++ b/doc/metadata-doc.md @@ -2,118 +2,39 @@ ## Context +The Datastore receives from its Backend JSON files related to pictures which we +also have to register. Therefore, we would need a module in place to manipulate +all the JSON and format them for our DB needs. The modules needs to also be able +to rebuild a JSON to be used by the other layers (BE & FE) + The following documentation provide an overview of the metadata importation -process for the Nachet pipeline. We outline each steps of the workflow, -illustrating how data progresses until it becomes usable by our models. +process . We outline each steps of the workflow, illustrating how data +progresses until it becomes usable by our applications. + Additionally, the upcoming process are showcased with the expected files structure. ``` mermaid --- -title: Nachet data upload +title: Data upload --- flowchart LR; FE(Frontend) BE(Backend) - file[/Folder/] classDef foo stroke:#f00 - file --> FE FE-->BE subgraph dbProcess [New Process] MD(Datastore) DB[(Database)] blob[(Blob)] end - BE -- TODO --- MD + BE -- Work in progress --- MD MD --> DB MD --> blob - -``` - -As shown above, we are currently working on a process to validate the files -uploaded to the cloud. However, since Nachet is still a work in progress, here's -the current workflow for our user to upload their images for the models. - -## Workflow: Metadata upload to Azure cloud (Currently) - -``` mermaid - -sequenceDiagram; - actor User - actor DataScientist - participant Azure Portal - participant Azure Storage Explorer - - alt New User - User->>DataScientist: Storage subscription key request - Note right of User: Email - DataScientist->>Azure Portal: Create Storage Blob - create participant Azure Storage - Azure Portal-)Azure Storage: Create() - DataScientist->>Azure Storage: Retrieve Key - DataScientist->>User: Send back subscription key - Note left of DataScientist: Email - User-)Azure Storage Explorer: SubscribeToStorage(key) - end - loop for each project Folder - User-)Azure Storage Explorer: Upload(Folder) - Azure Storage Explorer-) Azure Storage: Save(folder) - end - -``` - -This workflow showcase the two options that a user will face to upload data. The -first one being he's a first time user. Therefore, the current process for a -first time user is to contact the AI-Lab team and subscribe to the Blob storage -with a given subscription key. - -## Sequence of processing metadata for model (Currently) - -``` mermaid - -sequenceDiagram; - actor DataScientist - participant Azure Portal - participant Notebook - participant Azure Storage - DataScientist->>Azure Storage: Check files structure - alt Wrong structure - DataScientist->>Azure Portal: Create Alt. Azure Storage - Azure Portal-)Azure Storage: Create(Alt) - create participant Azure Storage Alt - DataScientist-)Azure Storage Alt: Copy files - DataScientist-)Azure Storage Alt: Rework structure + Edit files/folders - DataScientist-)Azure Storage Alt: Replace Storage content with Alt. Storage content - destroy Azure Storage Alt - Azure Storage Alt -) Azure Storage: Export files - end - DataScientist->>Azure Storage: Retrieve extraction code - DataScientist->>Notebook: Edit parameters of extraction code commands - - DataScientist-) Notebook: Run extraction code - Note left of Notebook: output source needs to be specified - Notebook -) Azure Storage: Processing files into metadata - DataScientist->> Azure Storage: Use files to train the model - ``` -This sequence illustrate the manual task done by our team to maintain the -storage of user's data. - -### Legend - -| Element | Description | -| ---------------------- | -------------------------------------------------------------------------------------------------------------------------- | -| User | Anyone wanting to upload data. | -| DataScientist | Member of the AI-Lab Team. | -| Azure Portal | Interface managing Azure's services | -| Azure Storage | Interface storing data in the cloud. | -| Azure Storage Explorer | Application with GUI offering a user friendly access to a Azure Storage without granting full acess To the Azure Services. | -| NoteBook | Azure Service enabling to run code with Azure Storage structure | -| Folder | All project folder should follow the files structure presented bellow. | - ## Development As explained in the context we aim to implement an architecture for user @@ -123,8 +44,8 @@ will be able to remove all manual metadata maintenance. Once the validation process is complete, the upload process will come into play, enabling the distribution of the files between the BLOB storage and a PostgreSQL database. - We are currently working on such process, which will be handled by the backend - part of nachet once it is finished. + We are currently improving the process to make it more efficient across all our + project. ``` mermaid @@ -152,177 +73,6 @@ flowchart LR; ``` -## Finished processes - -### [Deployment mass import](deployment-mass-import.md) - -### [Trusted User Upload](trusted-user-upload.md) - -## Database - -We plan on storing the metadata of the user's files in a postgreSQL Database. -The database should have the following structure: - -``` mermaid - ---- -title: Nachet DB Structure ---- -erDiagram - user{ - uuid id PK - string email - timestamp registration_date - timestamp updated_at - integer permission_id - } - picture_set{ - uuid id PK - json picture_set - uuid owner_id FK - timestamp upload_date - } - picture{ - uuid id PK - json picture - uuid picture_set_id FK - uuid parent FK - int nb_object - boolean verified - timestamp upload_date - } - picture_seed{ - uuid id PK - uuid picture_id FK - uuid seed_id FK - timestamp upload_date - } - seed{ - uuid id PK - string name - json information - uuid object_type_id - timestamp upload_date - } - object_type{ - integer id PK - text name - } - inference{ - uuid id PK - json inference - uuid picture_id FK - uuid user_id FK - timestamp upload_date - } - model{ - uuid id PK - text name - text endpoint_name - integer task_id FK - uuid active_version FK - } - model_version{ - uuid id PK - uuid model_id FK - json data - text version - timestamp upload_date - } - object{ - uuid id PK - json box_metadata - uuid inference_id FK - integer type_id - uuid verified_id - boolean valid - uuid top_inference FK - timestamp upload_date - timestamp updated_at - } - seed_object{ - uuid id PK - uuid seed_id FK - uuid object_id FK - } - pipeline{ - uuid id PK - text name - boolean active - boolean is_default - json data - } - pipeline_model{ - uuid id PK - uuid pipeline_id FK - uuid model_id FK - } - pipeline_default{ - uuid id PK - uuid pipeline_id FK - uuid user_id FK - timestamp update_date - } - group{ - uuid id PK - text name - int permission_id FK - uuid owner_id FK - timestamp upload_date - } - permission{ - int id - text name - } - user_group{ - uuid id - uuid user_id - uuid group_id - timestamp upload_date - - } - group_container{ - uuid id - uuid group_id - uuid container_id - timestamp upload_date - } - container{ - uuid id PK - uuid owner_id FK - boolean public - timestamp creation_date - timestamp updated_at - } - - user ||--|{ picture_set: uploads - picture_set ||--o{picture: contains - picture ||--o{picture: cropped - picture |o--o{picture_seed: has - picture_seed }o--o| seed: has - object_type ||--|| seed: is - user ||--o{ inference: requests - inference ||--|| picture: infers - inference }o--|| pipeline: uses - inference ||--o{ object: detects - object ||--o{ seed_object: is - seed_object }o--|| seed: is - object }o--|| object_type: is - user ||--||pipeline_default: select - pipeline_default }o--||pipeline: is - pipeline }o--o{ pipeline_model: uses - model }o--o{ pipeline_model: uses - model_version ||--o{ model: has - user }o--o{ user_group: apart - user_group }o--o{group: represent - permission ||--|| user: has - permission ||--|| group: has - group }o--o{group_container: has - container }o--o{group_container: represent - container ||--o{picture_set: contains - -``` - ## Blob Storage Finally the picture uploaded by the users will need to be stored in a blob @@ -356,8 +106,8 @@ Storage account: ## Consequences - Implementing this structure and introducing the new backend features in Nachet - will result in the following impact: + Implementing this structure and introducing the new backend features will + result in the following impact: - **Automation of file structure maintenance:** This process will autonomously From 411db7046cfcf23c02ca04918cb75d754af02bc8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Werbrouck?= <122839953+Francois-Werbrouck@users.noreply.github.com> Date: Wed, 17 Jul 2024 15:17:59 +0000 Subject: [PATCH 06/27] Issue #68: Fix lint --- datastore/FertiScan/__init__.py | 2 +- datastore/__init__.py | 4 ++-- datastore/db/__init__.py | 24 +++++++----------------- doc/datastore.md | 3 ++- doc/metadata-doc.md | 2 +- 5 files changed, 13 insertions(+), 22 deletions(-) diff --git a/datastore/FertiScan/__init__.py b/datastore/FertiScan/__init__.py index b0b309d6..55a97b16 100644 --- a/datastore/FertiScan/__init__.py +++ b/datastore/FertiScan/__init__.py @@ -1,6 +1,6 @@ import os from dotenv import load_dotenv - +import uuid load_dotenv() diff --git a/datastore/__init__.py b/datastore/__init__.py index c253582a..86b59d36 100644 --- a/datastore/__init__.py +++ b/datastore/__init__.py @@ -113,7 +113,7 @@ async def new_user(cursor, email, connection_string, tier="user") -> User: async def get_user_container_client( - user_id, tier="user", storage_url=NACHET_STORAGE_URL + user_id, storage_url, account, key, tier="user" ): """ Get the container client of a user @@ -123,7 +123,7 @@ async def get_user_container_client( Returns: ContainerClient object """ - sas = blob.get_account_sas(NACHET_BLOB_ACCOUNT, NACHET_BLOB_KEY) + sas = blob.get_account_sas(account, key) # Get the container client container_client = await azure_storage.mount_container( storage_url, user_id, True, tier, sas diff --git a/datastore/db/__init__.py b/datastore/db/__init__.py index d58e006e..a43a64ed 100644 --- a/datastore/db/__init__.py +++ b/datastore/db/__init__.py @@ -1,25 +1,14 @@ """ This module contains the function interacting with the database directly. """ -import os + import psycopg from dotenv import load_dotenv load_dotenv() -# def connect_db(): -# """Connect to the postgresql database and return the connection.""" -# connection = psycopg.connect( -# conninfo=NACHET_DB_URL, -# autocommit=False, -# options=f"-c search_path={NACHET_SCHEMA},public") -# assert connection.info.encoding == 'utf-8', ( -# 'Encoding is not UTF8: ' + connection.info.encoding) -# # psycopg.extras.register_uuid() -# return connection - -def connect_db(conn_str : str = NACHET_DB_URL, schema :str = NACHET_SCHEMA): +def connect_db(conn_str: str, schema: str): """Connect to the postgresql database and return the connection.""" connection = psycopg.connect( conninfo=conn_str, @@ -45,14 +34,15 @@ def end_query(connection, cursor): connection.close() -def create_search_path(connection, cur,schema = NACHET_SCHEMA): +def create_search_path(connection, cur, schema): cur.execute(f"""SET search_path TO "{schema}";""") connection.commit() + if __name__ == "__main__": - connection = connect_db(FERTISCAN_DB_URL,FERTISCAN_SCHEMA) - print(FERTISCAN_DB_URL) + connection = connect_db("test", "test") + print("test") cur = cursor(connection) - create_search_path(connection, cur,FERTISCAN_SCHEMA) + create_search_path(connection, cur, "test") end_query(connection, cur) print("Connection to the database successful") diff --git a/doc/datastore.md b/doc/datastore.md index 0ebda026..a713d3e1 100644 --- a/doc/datastore.md +++ b/doc/datastore.md @@ -68,7 +68,9 @@ flowchart LR; ``` ## Database Architecture + For more detail on each app database architecture go check [Nachet Architecture](nachetArchitecture.md) and [Fertiscan Architecture](FertiscanArchitecture.md). + ### Global Needs - A User must be able to take a picture on the app and it must be saved in the @@ -78,4 +80,3 @@ flowchart LR; - A User can verify the result of a picture that went through the pipeline and the changes are saved for training. - diff --git a/doc/metadata-doc.md b/doc/metadata-doc.md index a5c68527..aa7ec342 100644 --- a/doc/metadata-doc.md +++ b/doc/metadata-doc.md @@ -45,7 +45,7 @@ process is complete, the upload process will come into play, enabling the distribution of the files between the BLOB storage and a PostgreSQL database. We are currently improving the process to make it more efficient across all our - project. + project. ``` mermaid From 237228fdbb2a82386dc88f478302b4519cc46fce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Werbrouck?= <122839953+Francois-Werbrouck@users.noreply.github.com> Date: Wed, 17 Jul 2024 15:33:14 +0000 Subject: [PATCH 07/27] Issue #68: PyProject update --- pyproject.toml | 3 ++- pyproject_Nachet.toml | 47 ++++++++++++++++++++++++++++++++++++++++ pyproject_fertiscan.toml | 47 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 96 insertions(+), 1 deletion(-) create mode 100644 pyproject_Nachet.toml create mode 100644 pyproject_fertiscan.toml diff --git a/pyproject.toml b/pyproject.toml index d67925b3..57a8834e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,9 +4,10 @@ build-backend = "setuptools.build_meta" [project] name = "nachet_datastore" -version = "0.0.1" +version = "0.1.0" authors = [ { name="Francois Werbrouck", email="francois.werbrouck@inspection.gc.ca" }, + { name="Sylvanie You", email=Sylvanie.You@inspection.gc.ca" ] description = "Data management python layer" readme = "README.md" diff --git a/pyproject_Nachet.toml b/pyproject_Nachet.toml new file mode 100644 index 00000000..57a8834e --- /dev/null +++ b/pyproject_Nachet.toml @@ -0,0 +1,47 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "nachet_datastore" +version = "0.1.0" +authors = [ + { name="Francois Werbrouck", email="francois.werbrouck@inspection.gc.ca" }, + { name="Sylvanie You", email=Sylvanie.You@inspection.gc.ca" +] +description = "Data management python layer" +readme = "README.md" +requires-python = ">=3.11" +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", +] +dynamic = ["dependencies"] + + +license = { file = "LICENSE" } + +keywords = ["Nachet","ailab"] + +[tool.setuptools] +packages = ["datastore", "datastore.db", "datastore.bin", +"datastore.db.queries", "datastore.db.queries.picture", +"datastore.db.queries.seed", "datastore.db.queries.user", +"datastore.db.queries.machine_learning","datastore.db.queries.inference", "datastore.db.queries.analysis", +"datastore.db.metadata","datastore.db.metadata.picture", "datastore.db.metadata.picture_set", +"datastore.db.metadata.validator", +"datastore.db.metadata.machine_learning","datastore.db.metadata.inference", +"datastore.blob","datastore.blob.azure_storage_api"] + +[tool.setuptools.dynamic] +dependencies = {file = ["requirements.txt"]} + +# https://setuptools.pypa.io/en/latest/userguide/pyproject_config.html +# [tool.setuptools.packages.find] +# where = ["ailab"] + +[project.urls] +"Homepage" = "https://github.com/ai-cfia/nachet-datastore" +"Bug Tracker" = "https://github.com/ai-cfia/nachet-datastore/issues" +Repository = "https://github.com/ai-cfia/nachet-datastore" diff --git a/pyproject_fertiscan.toml b/pyproject_fertiscan.toml new file mode 100644 index 00000000..41b4a5f2 --- /dev/null +++ b/pyproject_fertiscan.toml @@ -0,0 +1,47 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "fertiscan_datastore" +version = "0.0.1" +authors = [ + { name="Francois Werbrouck", email="francois.werbrouck@inspection.gc.ca" } +] +description = "Data management python layer" +readme = "README.md" +requires-python = ">=3.11" +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", +] +dynamic = ["dependencies"] + + +license = { file = "LICENSE" } + +keywords = ["FertiScan","ailab"] + +[tool.setuptools] +packages = ["datastore", "datastore.FertiScan", "datastore.bin","datastore.db", +"datastore.db.queries", "datastore.db.queries.picture", +"datastore.db.queries.user", +"datastore.db.queries.inspection","datastore.db.queries.label","datastore.db.queries.metric", +"datastore.db.queries.nutrients","datastore.db.queries.organization","datastore.db.queries.specification", +"datastore.db.queries.sub_label", +"datastore.db.metadata","datastore.db.metadata.picture", "datastore.db.metadata.picture_set", +"datastore.db.metadata.validator", +"datastore.blob","datastore.blob.azure_storage_api"] + +[tool.setuptools.dynamic] +dependencies = {file = ["requirements.txt"]} + +# https://setuptools.pypa.io/en/latest/userguide/pyproject_config.html +# [tool.setuptools.packages.find] +# where = ["ailab"] + +[project.urls] +"Homepage" = "https://github.com/ai-cfia/nachet-datastore" +"Bug Tracker" = "https://github.com/ai-cfia/nachet-datastore/issues" +Repository = "https://github.com/ai-cfia/nachet-datastore" From dd0ac7c4eb74608a9915f1c3fd157d170dab26ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Werbrouck?= <122839953+Francois-Werbrouck@users.noreply.github.com> Date: Thu, 18 Jul 2024 14:34:53 +0000 Subject: [PATCH 08/27] Issue #68: rebase --- datastore/Nachet/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datastore/Nachet/__init__.py b/datastore/Nachet/__init__.py index 7626f4b4..f0215995 100644 --- a/datastore/Nachet/__init__.py +++ b/datastore/Nachet/__init__.py @@ -284,7 +284,7 @@ async def register_inference_result( cursor, trimmed_inference, user_id, picture_id, type ) nb_object = int(inference_dict["totalBoxes"]) - inference_dict["inferenceId"] = str(inference_id) + inference_dict["inference_id"] = str(inference_id) # loop through the boxes for box_index in range(nb_object): # TODO: adapt for multiple types of objects @@ -302,7 +302,7 @@ async def register_inference_result( object_inference_id = inference.new_inference_object( cursor, inference_id, box, type, False ) - inference_dict["boxes"][box_index]["boxId"] = str(object_inference_id) + inference_dict["boxes"][box_index]["box_id"] = str(object_inference_id) # loop through the topN Prediction top_score = -1 if "topN" in inference_dict["boxes"][box_index]: From bfd51d64adba5d322b635d470fda914af6e3252c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Werbrouck?= <122839953+Francois-Werbrouck@users.noreply.github.com> Date: Thu, 18 Jul 2024 14:39:07 +0000 Subject: [PATCH 09/27] Issue #68: fix lint --- doc/datastore.md | 4 +++- pyproject.toml | 2 +- pyproject_Nachet.toml | 2 +- tests/test_azure_storage.py | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/doc/datastore.md b/doc/datastore.md index a713d3e1..b779ac3f 100644 --- a/doc/datastore.md +++ b/doc/datastore.md @@ -69,7 +69,9 @@ flowchart LR; ## Database Architecture - For more detail on each app database architecture go check [Nachet Architecture](nachetArchitecture.md) and [Fertiscan Architecture](FertiscanArchitecture.md). + For more detail on each app database architecture go check [Nachet + Architecture](nachetArchitecture.md) and [Fertiscan + Architecture](FertiscanArchitecture.md). ### Global Needs diff --git a/pyproject.toml b/pyproject.toml index 57a8834e..b3b1dd5a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "nachet_datastore" version = "0.1.0" authors = [ { name="Francois Werbrouck", email="francois.werbrouck@inspection.gc.ca" }, - { name="Sylvanie You", email=Sylvanie.You@inspection.gc.ca" + { name="Sylvanie You", email="Sylvanie.You@inspection.gc.ca" ] description = "Data management python layer" readme = "README.md" diff --git a/pyproject_Nachet.toml b/pyproject_Nachet.toml index 57a8834e..b3b1dd5a 100644 --- a/pyproject_Nachet.toml +++ b/pyproject_Nachet.toml @@ -7,7 +7,7 @@ name = "nachet_datastore" version = "0.1.0" authors = [ { name="Francois Werbrouck", email="francois.werbrouck@inspection.gc.ca" }, - { name="Sylvanie You", email=Sylvanie.You@inspection.gc.ca" + { name="Sylvanie You", email="Sylvanie.You@inspection.gc.ca" ] description = "Data management python layer" readme = "README.md" diff --git a/tests/test_azure_storage.py b/tests/test_azure_storage.py index acab6d69..80e28915 100644 --- a/tests/test_azure_storage.py +++ b/tests/test_azure_storage.py @@ -243,7 +243,7 @@ def setUp(self): def tearDown(self): self.container_client.delete_container() - 0 + def test_get_folder_uuid(self): result = asyncio.run(get_folder_uuid(self.container_client,self.folder_name)) self.assertEqual(result, self.folder_uuid) From 25dc3e912fb90b88be3a4faf786db59c08f4db68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Werbrouck?= <122839953+Francois-Werbrouck@users.noreply.github.com> Date: Thu, 18 Jul 2024 14:42:41 +0000 Subject: [PATCH 10/27] Issue #68: update links --- doc/datastore.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/datastore.md b/doc/datastore.md index b779ac3f..eede97f9 100644 --- a/doc/datastore.md +++ b/doc/datastore.md @@ -70,8 +70,8 @@ flowchart LR; ## Database Architecture For more detail on each app database architecture go check [Nachet - Architecture](nachetArchitecture.md) and [Fertiscan - Architecture](FertiscanArchitecture.md). + Architecture](Nachet/nachet-architecture.md) and [Fertiscan + Architecture](FertiScan/fertiScan-architecture.md). ### Global Needs From ce9fcbf8d9d8d3662ac10b6172fbb23404034025 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Werbrouck?= <122839953+Francois-Werbrouck@users.noreply.github.com> Date: Thu, 18 Jul 2024 14:43:43 +0000 Subject: [PATCH 11/27] Issue #68: Pyproject fix --- pyproject.toml | 2 +- pyproject_Nachet.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index b3b1dd5a..2868e1a3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "nachet_datastore" version = "0.1.0" authors = [ { name="Francois Werbrouck", email="francois.werbrouck@inspection.gc.ca" }, - { name="Sylvanie You", email="Sylvanie.You@inspection.gc.ca" + { name="Sylvanie You", email="Sylvanie.You@inspection.gc.ca"} ] description = "Data management python layer" readme = "README.md" diff --git a/pyproject_Nachet.toml b/pyproject_Nachet.toml index b3b1dd5a..2868e1a3 100644 --- a/pyproject_Nachet.toml +++ b/pyproject_Nachet.toml @@ -7,7 +7,7 @@ name = "nachet_datastore" version = "0.1.0" authors = [ { name="Francois Werbrouck", email="francois.werbrouck@inspection.gc.ca" }, - { name="Sylvanie You", email="Sylvanie.You@inspection.gc.ca" + { name="Sylvanie You", email="Sylvanie.You@inspection.gc.ca"} ] description = "Data management python layer" readme = "README.md" From d3ade03c9a33cdff66484add766f764e031df6b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Werbrouck?= <122839953+Francois-Werbrouck@users.noreply.github.com> Date: Thu, 18 Jul 2024 14:45:23 +0000 Subject: [PATCH 12/27] Issue #68: fix lint --- datastore/Nachet/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/datastore/Nachet/__init__.py b/datastore/Nachet/__init__.py index f0215995..0c342cf8 100644 --- a/datastore/Nachet/__init__.py +++ b/datastore/Nachet/__init__.py @@ -11,7 +11,6 @@ import datastore.db.metadata.picture_set as data_picture_set import datastore.blob.azure_storage_api as azure_storage import json -from azure.storage.blob import BlobServiceClient,ContainerClient from datastore import ( get_user_container_client, BlobUploadError, From 29d43ba772c54e5c24f73d5bb715f9fb81188c53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Werbrouck?= <122839953+Francois-Werbrouck@users.noreply.github.com> Date: Thu, 18 Jul 2024 15:02:19 +0000 Subject: [PATCH 13/27] Issue #68: update Nachet packages --- pyproject.toml | 4 ++-- pyproject_Nachet.toml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 2868e1a3..56bfcd22 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,10 +25,10 @@ license = { file = "LICENSE" } keywords = ["Nachet","ailab"] [tool.setuptools] -packages = ["datastore", "datastore.db", "datastore.bin", +packages = ["datastore", "datastore.Nachet" "datastore.db", "datastore.bin", "datastore.db.queries", "datastore.db.queries.picture", "datastore.db.queries.seed", "datastore.db.queries.user", -"datastore.db.queries.machine_learning","datastore.db.queries.inference", "datastore.db.queries.analysis", +"datastore.db.queries.machine_learning","datastore.db.queries.inference", "datastore.db.metadata","datastore.db.metadata.picture", "datastore.db.metadata.picture_set", "datastore.db.metadata.validator", "datastore.db.metadata.machine_learning","datastore.db.metadata.inference", diff --git a/pyproject_Nachet.toml b/pyproject_Nachet.toml index 2868e1a3..56bfcd22 100644 --- a/pyproject_Nachet.toml +++ b/pyproject_Nachet.toml @@ -25,10 +25,10 @@ license = { file = "LICENSE" } keywords = ["Nachet","ailab"] [tool.setuptools] -packages = ["datastore", "datastore.db", "datastore.bin", +packages = ["datastore", "datastore.Nachet" "datastore.db", "datastore.bin", "datastore.db.queries", "datastore.db.queries.picture", "datastore.db.queries.seed", "datastore.db.queries.user", -"datastore.db.queries.machine_learning","datastore.db.queries.inference", "datastore.db.queries.analysis", +"datastore.db.queries.machine_learning","datastore.db.queries.inference", "datastore.db.metadata","datastore.db.metadata.picture", "datastore.db.metadata.picture_set", "datastore.db.metadata.validator", "datastore.db.metadata.machine_learning","datastore.db.metadata.inference", From b0d85ce68ed08928b7240c35d9d462cda48ba294 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Werbrouck?= <122839953+Francois-Werbrouck@users.noreply.github.com> Date: Thu, 18 Jul 2024 15:13:50 +0000 Subject: [PATCH 14/27] Issue #68: pyproject update --- pyproject.toml | 2 +- pyproject_Nachet.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 56bfcd22..1869ab5a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ license = { file = "LICENSE" } keywords = ["Nachet","ailab"] [tool.setuptools] -packages = ["datastore", "datastore.Nachet" "datastore.db", "datastore.bin", +packages = ["datastore", "datastore.Nachet", "datastore.db", "datastore.bin", "datastore.db.queries", "datastore.db.queries.picture", "datastore.db.queries.seed", "datastore.db.queries.user", "datastore.db.queries.machine_learning","datastore.db.queries.inference", diff --git a/pyproject_Nachet.toml b/pyproject_Nachet.toml index 56bfcd22..1869ab5a 100644 --- a/pyproject_Nachet.toml +++ b/pyproject_Nachet.toml @@ -25,7 +25,7 @@ license = { file = "LICENSE" } keywords = ["Nachet","ailab"] [tool.setuptools] -packages = ["datastore", "datastore.Nachet" "datastore.db", "datastore.bin", +packages = ["datastore", "datastore.Nachet", "datastore.db", "datastore.bin", "datastore.db.queries", "datastore.db.queries.picture", "datastore.db.queries.seed", "datastore.db.queries.user", "datastore.db.queries.machine_learning","datastore.db.queries.inference", From ebee5c63832deb6e26293e6b7f181f3ec5e3b184 Mon Sep 17 00:00:00 2001 From: sylvanie85 Date: Thu, 18 Jul 2024 18:11:52 +0000 Subject: [PATCH 15/27] new functions to retrieve pictures #69 --- datastore/__init__.py | 27 +++++++++++-- datastore/db/queries/picture/__init__.py | 48 ++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 3 deletions(-) diff --git a/datastore/__init__.py b/datastore/__init__.py index 7f92c5e5..24cd6429 100644 --- a/datastore/__init__.py +++ b/datastore/__init__.py @@ -805,6 +805,7 @@ async def get_seed_info(cursor): async def get_picture_sets_info(cursor, user_id: str): """This function retrieves the picture sets names and number of pictures from the database. + This also retrieve for each picture in the picture set their name, if an inference exist and if the picture is validated. Args: user_id (str): id of the user @@ -813,13 +814,33 @@ async def get_picture_sets_info(cursor, user_id: str): if not user.is_a_user_id(cursor=cursor, user_id=user_id): raise user.UserNotFoundError(f"User not found based on the given id: {user_id}") - result = {} + result = [] picture_sets = picture.get_user_picture_sets(cursor, user_id) for picture_set in picture_sets: + picture_set_info = {} picture_set_id = picture_set[0] picture_set_name = picture_set[1] - nb_picture = picture.count_pictures(cursor, picture_set_id) - result[str(picture_set_id)] = [picture_set_name, nb_picture] + + picture_set_info["picture_set_id"] = picture_set_id + picture_set_info["folder_name"] = picture_set_name + + pictures = picture.get_picture_set_pictures(cursor, picture_set_id) + picture_set_info["nb_pictures"] = len(pictures) + + picture_set_info["pictures"] = [] + for pic in pictures : + picture_info = {} + picture_id = pic[0] + picture_info["picture_id"] = picture_id + + is_validated = picture.is_picture_validated(cursor, picture_id) + inference_exist = picture.check_picture_inference_exist(cursor, picture_id) + picture_info["is_validated"] = is_validated + picture_info["inference_exist"] = inference_exist + + picture_set_info["pictures"].append(picture_info) + result.append(picture_set_info) + print(result) return result diff --git a/datastore/db/queries/picture/__init__.py b/datastore/db/queries/picture/__init__.py index ffc5a331..1ee30a52 100644 --- a/datastore/db/queries/picture/__init__.py +++ b/datastore/db/queries/picture/__init__.py @@ -331,6 +331,54 @@ def get_validated_pictures(cursor, picture_set_id: str): return result except Exception: raise GetPictureError(f"Error: Error while getting validated pictures for picture_set:{picture_set_id}") + +def is_picture_validated(cursor, picture_id: str): + """ + This functions check if a picture is validated. Therefore, there should exists picture_seed entity for this picture. + + Parameters: + - cursor (cursor): The cursor of the database. + - picture_id (str): The UUID of the picture to check. + """ + try : + query = """ + SELECT + 1 + FROM + picture_seed + WHERE + picture_id = %s + """ + cursor.execute(query, (picture_id,)) + result = cursor.fetchone()[0] + return result + except Exception: + raise GetPictureError( + f"Error: could not check if the picture {picture_id} is validated") + +def check_picture_inference_exist(cursor, picture_id: str): + """ + This functions check whether a picture is associated with an inference. + + Parameters: + - cursor (cursor): The cursor of the database. + - picture_id (str): The UUID of the picture to check. + """ + try : + query = """ + SELECT + 1 + FROM + inference + WHERE + picture_id = %s + """ + cursor.execute(query, (picture_id,)) + result = cursor.fetchone()[0] + return result + except Exception: + raise GetPictureError( + f"Error: could not check if the picture {picture_id} has an existing inference") def change_picture_set_id(cursor, user_id, old_picture_set_id, new_picture_set_id): """ From 573a9ade9afbf9bbf45c0dca02d4fd4f38b263a2 Mon Sep 17 00:00:00 2001 From: sylvanie85 Date: Fri, 19 Jul 2024 14:31:35 +0000 Subject: [PATCH 16/27] Retrieve pictures and inferences for a given folder #69 --- datastore/Nachet/__init__.py | 40 ++++++++++++++++++++++++ datastore/db/queries/picture/__init__.py | 24 ++++++++++++++ tests/test_datastore.py | 31 ++++++++++++------ 3 files changed, 85 insertions(+), 10 deletions(-) diff --git a/datastore/Nachet/__init__.py b/datastore/Nachet/__init__.py index 0c342cf8..b459bf28 100644 --- a/datastore/Nachet/__init__.py +++ b/datastore/Nachet/__init__.py @@ -631,6 +631,46 @@ async def get_seed_info(cursor): seed_dict["seeds"].append({"seed_id": seed_id, "seed_name": seed_name}) return seed_dict +async def get_pictures_inferences(cursor, user_id: str, picture_set_id: str): + """ + Retrieves inferences (if exist) of each picture in the given picture_set + + Args: + cursor: The cursor object to interact with the database. + user_id (str): id of the user + picture_set_id (str): id of the picture set + """ + try: + # Check if user exists + if not user.is_a_user_id(cursor=cursor, user_id=user_id): + raise user.UserNotFoundError( + f"User not found based on the given id: {user_id}" + ) + # Check if picture set exists + if not picture.is_a_picture_set_id(cursor, picture_set_id): + raise picture.PictureSetNotFoundError( + f"Picture set not found based on the given id: {picture_set_id}" + ) + # Check user is owner of the picture set + if picture.get_picture_set_owner_id(cursor, picture_set_id) != user_id: + raise UserNotOwnerError( + f"User can't delete this folder, user uuid :{user_id}, folder name : {picture_set_id}" + ) + result=[] + pictures = picture.get_picture_set_pictures(cursor, picture_set_id) + for pic in pictures: + picture_id = pic[0] + if picture.check_picture_inference_exist(cursor, picture_id): + inf = picture.get_picture_inference(cursor, picture_id) + result.append(inf) + print(result) + return result + except ( + user.UserNotFoundError, + picture.PictureSetNotFoundError, + UserNotOwnerError, + ) as e: + raise e async def delete_picture_set_with_archive( cursor, user_id, picture_set_id, container_client diff --git a/datastore/db/queries/picture/__init__.py b/datastore/db/queries/picture/__init__.py index 1ee30a52..99d4b731 100644 --- a/datastore/db/queries/picture/__init__.py +++ b/datastore/db/queries/picture/__init__.py @@ -380,6 +380,30 @@ def check_picture_inference_exist(cursor, picture_id: str): raise GetPictureError( f"Error: could not check if the picture {picture_id} has an existing inference") +def get_picture_inference(cursor, picture_id: str): + """ + This functions retrieve inference of a given picture + + Parameters: + - cursor (cursor): The cursor of the database. + - picture_id (str): The UUID of the picture. + """ + try : + query = """ + SELECT + * + FROM + inference + WHERE + picture_id = %s + """ + cursor.execute(query, (picture_id,)) + result = cursor.fetchone()[0] + return result + except Exception: + raise GetPictureError( + f"Error: could not get inference for the picture {picture_id}") + def change_picture_set_id(cursor, user_id, old_picture_set_id, new_picture_set_id): """ This function change picture_set_id of all pictures in a picture_set to a new one. diff --git a/tests/test_datastore.py b/tests/test_datastore.py index 9e3876f5..329e5bc8 100644 --- a/tests/test_datastore.py +++ b/tests/test_datastore.py @@ -12,6 +12,7 @@ import asyncio import datastore.db.__init__ as db import datastore.__init__ as datastore +import datastore.Nachet as nachet_datastore import datastore.db.metadata.validator as validator import datastore.db.queries.seed as seed_query from copy import deepcopy @@ -34,7 +35,7 @@ class test_ml_structure(unittest.TestCase): def setUp(self): with open("tests/ml_structure_exemple.json") as file: self.ml_dict= json.load(file) - self.con = db.connect_db(DB_CONNECTION_STRING) + self.con = db.connect_db(DB_CONNECTION_STRING,DB_SCHEMA) self.cursor = self.con.cursor() db.create_search_path(self.con, self.cursor,DB_SCHEMA) @@ -86,7 +87,7 @@ def test_get_ml_structure_eeror(self): class test_user(unittest.TestCase): def setUp(self): - self.con = db.connect_db(DB_CONNECTION_STRING) + self.con = db.connect_db(DB_CONNECTION_STRING,DB_SCHEMA) self.cursor = self.con.cursor() db.create_search_path(self.con, self.cursor,DB_SCHEMA) self.user_email="test@email" @@ -163,7 +164,7 @@ def test_get_user_container_client(self): class test_picture(unittest.TestCase): def setUp(self): - self.con = db.connect_db(DB_CONNECTION_STRING) + self.con = db.connect_db(DB_CONNECTION_STRING,DB_SCHEMA) self.cursor = self.con.cursor() db.create_search_path(self.con, self.cursor,DB_SCHEMA) self.connection_str=BLOB_CONNECTION_STRING @@ -293,7 +294,7 @@ def test_upload_pictures_error_seed_not_found(self): class test_feedback(unittest.TestCase): def setUp(self): - self.con = db.connect_db(DB_CONNECTION_STRING) + self.con = db.connect_db(DB_CONNECTION_STRING,DB_SCHEMA) self.cursor = self.con.cursor() db.create_search_path(self.con, self.cursor,DB_SCHEMA) self.connection_str=BLOB_CONNECTION_STRING @@ -487,7 +488,7 @@ def test_new_correction_inference_feedback_unknown_seed(self): class test_picture_set(unittest.TestCase): def setUp(self): - self.con = db.connect_db(DB_CONNECTION_STRING) + self.con = db.connect_db(DB_CONNECTION_STRING,DB_SCHEMA) self.cursor = self.con.cursor() db.create_search_path(self.con, self.cursor,DB_SCHEMA) self.connection_str=BLOB_CONNECTION_STRING @@ -507,13 +508,13 @@ def setUp(self): self.picture_set_id = asyncio.run(datastore.create_picture_set(self.cursor, self.container_client, 0, self.user_id, self.folder_name)) self.pictures_id =[] for i in range(3) : - picture_id = asyncio.run(datastore.upload_picture_unknown(self.cursor, self.user_id, self.pic_encoded,self.container_client, self.picture_set_id)) + picture_id = asyncio.run(nachet_datastore.upload_picture_unknown(self.cursor, self.user_id, self.pic_encoded,self.container_client, self.picture_set_id)) self.pictures_id.append(picture_id) with open("tests/inference_result.json") as file: self.inference= json.load(file) self.dev_user_id=datastore.user.get_user_id(self.cursor, os.environ["DEV_USER_EMAIL"]) - self.dev_container_client = asyncio.run(datastore.get_user_container_client(self.dev_user_id)) + self.dev_container_client = asyncio.run(datastore.get_user_container_client(self.dev_user_id, BLOB_CONNECTION_STRING)) def tearDown(self): self.con.rollback() @@ -527,8 +528,18 @@ def test_get_picture_sets_info(self) : picture_sets_info = asyncio.run(datastore.get_picture_sets_info(self.cursor, self.user_id)) self.assertEqual(len(picture_sets_info), 2) - self.assertEqual(picture_sets_info.get(str(self.picture_set_id))[1], 3) - self.assertEqual(picture_sets_info.get(str(self.picture_set_id))[0], self.folder_name) + for picture_set in picture_sets_info : + if picture_set["picture_set_id"] == self.picture_set_id : + self.assertEqual(picture_set["picture_set_id"], self.picture_set_id) + self.assertEqual(picture_set["folder_name"], self.folder_name) + self.assertEqual(picture_set["nb_pictures"], 3) + self.assertTrue("pictures" in picture_set) + ids = [pic["picture_id"] for pic in picture_set["pictures"]] + self.assertEqual(set(ids), set(self.pictures_id)) + for pic in picture_set["pictures"] : + pic["is_verified"] = False + pic["inference_exist"] = False + self.picture_set_id = asyncio.run(datastore.create_picture_set(self.cursor, self.container_client, 0, self.user_id, self.folder_name + "2")) @@ -761,7 +772,7 @@ def test_delete_picture_set_with_archive_error_default_folder(self): class test_analysis(unittest.TestCase): def setUp(self): - self.con = db.connect_db(DB_CONNECTION_STRING) + self.con = db.connect_db(DB_CONNECTION_STRING,DB_SCHEMA) self.cursor = self.con.cursor() db.create_search_path(self.con, self.cursor,DB_SCHEMA) self.connection_str=BLOB_CONNECTION_STRING From 5a93c684d0fec6f4e5e3ff4eeb60aa9ff1b83781 Mon Sep 17 00:00:00 2001 From: sylvanie85 Date: Fri, 19 Jul 2024 20:43:59 +0000 Subject: [PATCH 17/27] get_pictures_inferences and tests --- datastore/Nachet/__init__.py | 110 +++++++++++++++++++-- datastore/__init__.py | 5 +- datastore/db/queries/inference/__init__.py | 20 ++++ datastore/db/queries/picture/__init__.py | 31 +++--- datastore/db/queries/seed/__init__.py | 26 +++++ tests/test_datastore.py | 64 +++++++++--- 6 files changed, 216 insertions(+), 40 deletions(-) diff --git a/datastore/Nachet/__init__.py b/datastore/Nachet/__init__.py index b459bf28..aa6c3eb5 100644 --- a/datastore/Nachet/__init__.py +++ b/datastore/Nachet/__init__.py @@ -445,7 +445,9 @@ async def new_correction_inference_feedback(cursor, inference_dict, type: int = inference.set_inference_object_verified_id( cursor, object_id, seed_object_id ) - else: + + # If a seed is selected by the user or if it is a known seed that the FE has not recognized + if seed_id != "": # Box is still valid valid = True # Check if a new seed has been selected @@ -656,20 +658,110 @@ async def get_pictures_inferences(cursor, user_id: str, picture_set_id: str): raise UserNotOwnerError( f"User can't delete this folder, user uuid :{user_id}, folder name : {picture_set_id}" ) - result=[] + result={} pictures = picture.get_picture_set_pictures(cursor, picture_set_id) + for pic in pictures: picture_id = pic[0] + if picture.check_picture_inference_exist(cursor, picture_id): - inf = picture.get_picture_inference(cursor, picture_id) - result.append(inf) + inference = picture.get_picture_inference(cursor, picture_id) + inference_id = inference[0] + inference_data = json.loads(inference[1]) + boxes = inference.get_objects_by_inference(cursor, inference_id) + + print(boxes) + for object in boxes: + box_id = object[0] + + box_metadata = object[1] + json.loads(box_metadata) + box = box_metadata.get("box") + color = box_metadata.get("color") + overlapping = box_metadata.get("overlapping") + overlappingIndices = box_metadata.get("overlappingIndices") + + topN = inference.get_seed_object_by_object_id(cursor, box_id) + print(json.dumps(topN, indent=4)) + for seed_obj in topN : + object_id = seed_obj[0] + seed_id = seed_obj[1] + seed_name = seed.get_seed_name(cursor, seed_obj[1]) + score = seed_obj[2] + seed_obj = {"label": seed_name, "object_id": object_id, "score": score} + if seed_id == top_id: + top_score = score + + print(json.dumps(topN, indent=4)) + top_id = inference.get_inference_object_top_id(cursor, box_id) + top_label = seed.get_seed_name(cursor, top_id) + if inference.is_object_verified(cursor, box_id): + top_score = 1 + + object = {"box" : box, + "box_id": box_id, + "color" : color, + "label" : top_label, + "object_type_id" : 1, + "overlapping" : overlapping, + "overlappingIndices" : overlappingIndices, + "score" : top_score, + "topN": topN, + "top_id" : top_id} + + print(boxes) + + inference = { + "boxes": boxes, + "filename": inference_data.get("filename"), + "inference_id": inference_id, + "labelOccurence" : inference_data.get("labelOccurence"), + "totalBoxes" : inference_data.get("totalBoxes"), + } + result[picture_id] = inference + else : + result[picture_id] = None print(result) return result - except ( - user.UserNotFoundError, - picture.PictureSetNotFoundError, - UserNotOwnerError, - ) as e: + except(user.UserNotFoundError,picture.PictureSetNotFoundError,UserNotOwnerError) as e: + raise e + +async def get_pictures_blobs(cursor, user_id: str, container_client, picture_set_id: str): + """ + Retrieves blobs of each picture in the given picture_set + + Args: + cursor: The cursor object to interact with the database. + user_id (str): id of the user + picture_set_id (str): id of the picture set + """ + try: + # Check if user exists + if not user.is_a_user_id(cursor=cursor, user_id=user_id): + raise user.UserNotFoundError( + f"User not found based on the given id: {user_id}" + ) + # Check if picture set exists + if not picture.is_a_picture_set_id(cursor, picture_set_id): + raise picture.PictureSetNotFoundError( + f"Picture set not found based on the given id: {picture_set_id}" + ) + # Check user is owner of the picture set + if picture.get_picture_set_owner_id(cursor, picture_set_id) != user_id: + raise UserNotOwnerError( + f"User can't delete this folder, user uuid :{user_id}, folder name : {picture_set_id}" + ) + result=[] + folder_name = picture.get_picture_set_name(cursor, picture_set_id) + pictures = picture.get_picture_set_pictures(cursor, picture_set_id) + for pic in pictures: + picture_id = pic[0] + blob_name = "{}/{}.png".format(folder_name, picture_id) + picture_hash = azure_storage.get_blob(container_client, blob_name) + result.append(picture_hash) + print(result) + return result + except(user.UserNotFoundError,picture.PictureSetNotFoundError,UserNotOwnerError) as e: raise e async def delete_picture_set_with_archive( diff --git a/datastore/__init__.py b/datastore/__init__.py index e9de0acf..291ee7ed 100644 --- a/datastore/__init__.py +++ b/datastore/__init__.py @@ -203,7 +203,7 @@ async def get_picture_sets_info(cursor, user_id: str): picture_set_id = picture_set[0] picture_set_name = picture_set[1] - picture_set_info["picture_set_id"] = picture_set_id + picture_set_info["picture_set_id"] = str(picture_set_id) picture_set_info["folder_name"] = picture_set_name pictures = picture.get_picture_set_pictures(cursor, picture_set_id) @@ -213,7 +213,7 @@ async def get_picture_sets_info(cursor, user_id: str): for pic in pictures : picture_info = {} picture_id = pic[0] - picture_info["picture_id"] = picture_id + picture_info["picture_id"] = str(picture_id) is_validated = picture.is_picture_validated(cursor, picture_id) inference_exist = picture.check_picture_inference_exist(cursor, picture_id) @@ -222,7 +222,6 @@ async def get_picture_sets_info(cursor, user_id: str): picture_set_info["pictures"].append(picture_info) result.append(picture_set_info) - print(result) return result diff --git a/datastore/db/queries/inference/__init__.py b/datastore/db/queries/inference/__init__.py index 8d3ae70f..893fa51c 100644 --- a/datastore/db/queries/inference/__init__.py +++ b/datastore/db/queries/inference/__init__.py @@ -519,3 +519,23 @@ def get_seed_object_id(cursor, seed_id: str, object_id:str): return res except Exception: raise Exception(f"Error: could not get seed_object_id for seed_id {seed_id} for object {object_id}") + +def get_seed_object_by_object_id(cursor, object_id: str): + try: + query = """ + SELECT + so.id, + so.seed_id, + so.score + FROM + seed_obj so + WHERE + so.object_id = %s + """ + cursor.execute(query, (object_id,)) + res = cursor.fetchall() + return res + except Exception: + raise Exception(f"Error: could not get seed_object for object {object_id}") + + \ No newline at end of file diff --git a/datastore/db/queries/picture/__init__.py b/datastore/db/queries/picture/__init__.py index 99d4b731..3535abe7 100644 --- a/datastore/db/queries/picture/__init__.py +++ b/datastore/db/queries/picture/__init__.py @@ -26,6 +26,7 @@ class GetPictureError(Exception): class PictureSetDeleteError(Exception): pass +import json """ This module contains all the queries related to the Picture and PictureSet tables. """ @@ -342,12 +343,14 @@ def is_picture_validated(cursor, picture_id: str): """ try : query = """ - SELECT - 1 - FROM - picture_seed - WHERE - picture_id = %s + SELECT EXISTS( + SELECT + 1 + FROM + picture_seed + WHERE + picture_id = %s + ) """ cursor.execute(query, (picture_id,)) result = cursor.fetchone()[0] @@ -366,12 +369,14 @@ def check_picture_inference_exist(cursor, picture_id: str): """ try : query = """ - SELECT - 1 - FROM - inference - WHERE - picture_id = %s + SELECT EXISTS( + SELECT + 1 + FROM + inference + WHERE + picture_id = %s + ) """ cursor.execute(query, (picture_id,)) result = cursor.fetchone()[0] @@ -391,7 +396,7 @@ def get_picture_inference(cursor, picture_id: str): try : query = """ SELECT - * + id, inference FROM inference WHERE diff --git a/datastore/db/queries/seed/__init__.py b/datastore/db/queries/seed/__init__.py index ac1add1e..44302032 100644 --- a/datastore/db/queries/seed/__init__.py +++ b/datastore/db/queries/seed/__init__.py @@ -85,6 +85,32 @@ def get_seed_id(cursor, seed_name: str) -> str: except Exception: raise Exception("unhandled error") +def get_seed_name(cursor, seed_id:str) -> str : + """ + This function retrieves the name of a seed from the database. + + Parameters: + - cursor (cursor): The cursor of the database. + - seed_id (str): The id of the seed. + + Returns: + - The name of the seed. + """ + try: + query = """ + SELECT + name + FROM + seed + WHERE + id = %s + """ + cursor.execute(query, (seed_id,)) + return cursor.fetchone()[0] + except TypeError: + raise SeedNotFoundError("Error: seed not found") + except Exception: + raise Exception("unhandled error") def new_seed(cursor, seed_name: str): """ diff --git a/tests/test_datastore.py b/tests/test_datastore.py index 329e5bc8..4b0bd5d0 100644 --- a/tests/test_datastore.py +++ b/tests/test_datastore.py @@ -31,6 +31,14 @@ if BLOB_CONNECTION_STRING is None or BLOB_CONNECTION_STRING == "": raise ValueError("NACHET_STORAGE_URL_TESTING is not set") +BLOB_ACCOUNT = os.environ["NACHET_BLOB_ACCOUNT"] +if BLOB_ACCOUNT is None or BLOB_ACCOUNT == "": + raise ValueError("NACHET_BLOB_ACCOUNT is not set") + +BLOB_KEY = os.environ["NACHET_BLOB_KEY"] +if BLOB_KEY is None or BLOB_KEY == "": + raise ValueError("NACHET_BLOB_KEY is not set") + class test_ml_structure(unittest.TestCase): def setUp(self): with open("tests/ml_structure_exemple.json") as file: @@ -501,7 +509,7 @@ def setUp(self): #self.picture_hash= asyncio.run(azure_storage.generate_hash(self.pic_encoded)) self.container_name='test-container' self.user_id=datastore.User.get_id(self.user_obj) - self.container_client = asyncio.run(datastore.get_user_container_client(self.user_id,'test-user')) + self.container_client = asyncio.run(datastore.get_user_container_client(self.user_id,BLOB_CONNECTION_STRING,BLOB_ACCOUNT,BLOB_KEY,'test-user')) self.seed_name = "test-name" self.seed_id = seed_query.new_seed(self.cursor, self.seed_name) self.folder_name = "test_folder" @@ -514,40 +522,59 @@ def setUp(self): self.inference= json.load(file) self.dev_user_id=datastore.user.get_user_id(self.cursor, os.environ["DEV_USER_EMAIL"]) - self.dev_container_client = asyncio.run(datastore.get_user_container_client(self.dev_user_id, BLOB_CONNECTION_STRING)) + self.dev_container_client = asyncio.run(datastore.get_user_container_client(self.dev_user_id, BLOB_CONNECTION_STRING, BLOB_ACCOUNT, BLOB_KEY)) def tearDown(self): self.con.rollback() self.container_client.delete_container() db.end_query(self.con, self.cursor) + + def assert_picture_set_info(self, picture_set, expected_picture_set_id, expected_folder_name, expected_nb_pictures, expected_pictures_info): + self.assertEqual(picture_set["picture_set_id"], expected_picture_set_id) + self.assertEqual(picture_set["folder_name"], expected_folder_name) + self.assertEqual(picture_set["nb_pictures"], expected_nb_pictures) + for pic, expected_pic in zip(picture_set["pictures"], expected_pictures_info): + self.assert_picture_info(pic, expected_pic) + # Helper pour valider les informations des images individuelles + def assert_picture_info(self, picture_info, expected_pic_info): + self.assertEqual(picture_info["picture_id"], expected_pic_info["picture_id"]) + self.assertEqual(picture_info["is_verified"], expected_pic_info["is_verified"]) + self.assertEqual(picture_info["inference_exist"], expected_pic_info["inference_exist"]) + def test_get_picture_sets_info(self) : """ Test the get_picture_sets_info function """ picture_sets_info = asyncio.run(datastore.get_picture_sets_info(self.cursor, self.user_id)) + self.assertEqual(len(picture_sets_info), 2) + for picture_set in picture_sets_info : if picture_set["picture_set_id"] == self.picture_set_id : - self.assertEqual(picture_set["picture_set_id"], self.picture_set_id) - self.assertEqual(picture_set["folder_name"], self.folder_name) - self.assertEqual(picture_set["nb_pictures"], 3) - self.assertTrue("pictures" in picture_set) - ids = [pic["picture_id"] for pic in picture_set["pictures"]] - self.assertEqual(set(ids), set(self.pictures_id)) - for pic in picture_set["pictures"] : - pic["is_verified"] = False - pic["inference_exist"] = False - + expected_pictures_info = [ + {"picture_id": pid, "is_verified": False, "inference_exist": False} + for pid in self.pictures_id ] + self.assert_picture_set_info(picture_set, self.picture_set_id, self.folder_name, 3, expected_pictures_info) self.picture_set_id = asyncio.run(datastore.create_picture_set(self.cursor, self.container_client, 0, self.user_id, self.folder_name + "2")) + picture_id = asyncio.run(nachet_datastore.upload_picture_unknown(self.cursor, self.user_id, self.pic_encoded,self.container_client, self.picture_set_id)) + inference = asyncio.run(nachet_datastore.register_inference_result(self.cursor,self.user_id,self.inference, picture_id, "test_model_id")) + asyncio.run(nachet_datastore.new_perfect_inference_feeback(self.cursor, inference["inference_id"], self.user_id, [box["box_id"] for box in inference["boxes"]])) picture_sets_info = asyncio.run(datastore.get_picture_sets_info(self.cursor, self.user_id)) + self.assertEqual(len(picture_sets_info), 3) - self.assertEqual(picture_sets_info.get(str(self.picture_set_id))[1], 0) - self.assertEqual(picture_sets_info.get(str(self.picture_set_id))[0], self.folder_name + "2") + for picture_set in picture_sets_info : + if picture_set["picture_set_id"] == self.picture_set_id : + expected_pictures_info = [ + {"picture_id": picture_id, "is_verified": True, "inference_exist": True} + ] + self.assert_picture_set_info(picture_set, self.picture_set_id, self.folder_name+"2", 1, expected_pictures_info) + + def test_get_picture_sets_info_error_user_not_found(self): """ This test checks if the get_picture_sets_info function correctly raise an exception if the user given doesn't exist in db @@ -564,7 +591,14 @@ def test_get_picture_sets_info_error_connection_error(self): with self.assertRaises(Exception): asyncio.run(datastore.get_picture_sets_info(mock_cursor, self.user_id)) - + def test_get_pictures_inferences(self): + """ + This test checks if the get_pictures_inferences function correctly returns the inferences of each pictures a picture set + """ + + picture_sets_info = asyncio.run(nachet_datastore.get_pictures_inferences(self.cursor, self.user_id, self.picture_set_id)) + + def test_find_validated_pictures(self): """ This test checks if the find_validated_pictures function correctly returns the validated pictures of a picture_set From f0ea270f7dfed48fb696a19a7c0c448613b206d8 Mon Sep 17 00:00:00 2001 From: sylvanie85 Date: Tue, 23 Jul 2024 17:54:43 +0000 Subject: [PATCH 18/27] #69 functions and tests --- datastore/Nachet/__init__.py | 123 +++++--------------- datastore/db/metadata/inference/__init__.py | 102 +++++++++++++++- datastore/db/queries/picture/__init__.py | 49 +++++++- datastore/db/queries/seed/__init__.py | 27 +++++ tests/test_datastore.py | 118 +++++++++++++++++-- 5 files changed, 316 insertions(+), 103 deletions(-) diff --git a/datastore/Nachet/__init__.py b/datastore/Nachet/__init__.py index aa6c3eb5..a7ec3941 100644 --- a/datastore/Nachet/__init__.py +++ b/datastore/Nachet/__init__.py @@ -633,9 +633,9 @@ async def get_seed_info(cursor): seed_dict["seeds"].append({"seed_id": seed_id, "seed_name": seed_name}) return seed_dict -async def get_pictures_inferences(cursor, user_id: str, picture_set_id: str): +async def get_picture_inference(cursor, user_id: str, picture_id: str): """ - Retrieves inferences (if exist) of each picture in the given picture_set + Retrieves inference (if exist) of the given picture Args: cursor: The cursor object to interact with the database. @@ -649,91 +649,37 @@ async def get_pictures_inferences(cursor, user_id: str, picture_set_id: str): f"User not found based on the given id: {user_id}" ) # Check if picture set exists - if not picture.is_a_picture_set_id(cursor, picture_set_id): - raise picture.PictureSetNotFoundError( - f"Picture set not found based on the given id: {picture_set_id}" + if not picture.is_a_picture_id(cursor, picture_id): + raise picture.PictureNotFoundError( + f"Picture not found based on the given id: {picture_id}" ) - # Check user is owner of the picture set - if picture.get_picture_set_owner_id(cursor, picture_set_id) != user_id: + # Check user is owner of the picture set where the picutre is + picture_set_id = picture.get_picture_picture_set_id(cursor, picture_id) + if str(picture.get_picture_set_owner_id(cursor, picture_set_id)) != user_id: raise UserNotOwnerError( - f"User can't delete this folder, user uuid :{user_id}, folder name : {picture_set_id}" + f"User can't access this picture, user uuid :{user_id}, picture : {picture_id}" ) - result={} - pictures = picture.get_picture_set_pictures(cursor, picture_set_id) - for pic in pictures: - picture_id = pic[0] + if picture.check_picture_inference_exist(cursor, picture_id): + inf = picture.get_picture_inference(cursor, picture_id) - if picture.check_picture_inference_exist(cursor, picture_id): - inference = picture.get_picture_inference(cursor, picture_id) - inference_id = inference[0] - inference_data = json.loads(inference[1]) - boxes = inference.get_objects_by_inference(cursor, inference_id) - - print(boxes) - for object in boxes: - box_id = object[0] - - box_metadata = object[1] - json.loads(box_metadata) - box = box_metadata.get("box") - color = box_metadata.get("color") - overlapping = box_metadata.get("overlapping") - overlappingIndices = box_metadata.get("overlappingIndices") - - topN = inference.get_seed_object_by_object_id(cursor, box_id) - print(json.dumps(topN, indent=4)) - for seed_obj in topN : - object_id = seed_obj[0] - seed_id = seed_obj[1] - seed_name = seed.get_seed_name(cursor, seed_obj[1]) - score = seed_obj[2] - seed_obj = {"label": seed_name, "object_id": object_id, "score": score} - if seed_id == top_id: - top_score = score - - print(json.dumps(topN, indent=4)) - top_id = inference.get_inference_object_top_id(cursor, box_id) - top_label = seed.get_seed_name(cursor, top_id) - if inference.is_object_verified(cursor, box_id): - top_score = 1 - - object = {"box" : box, - "box_id": box_id, - "color" : color, - "label" : top_label, - "object_type_id" : 1, - "overlapping" : overlapping, - "overlappingIndices" : overlappingIndices, - "score" : top_score, - "topN": topN, - "top_id" : top_id} - - print(boxes) - - inference = { - "boxes": boxes, - "filename": inference_data.get("filename"), - "inference_id": inference_id, - "labelOccurence" : inference_data.get("labelOccurence"), - "totalBoxes" : inference_data.get("totalBoxes"), - } - result[picture_id] = inference - else : - result[picture_id] = None - print(result) - return result - except(user.UserNotFoundError,picture.PictureSetNotFoundError,UserNotOwnerError) as e: + inf = inference_metadata.rebuild_inference(cursor, inf) + + return inf + else : + return None + + except(user.UserNotFoundError,picture.PictureNotFoundError,UserNotOwnerError) as e: raise e -async def get_pictures_blobs(cursor, user_id: str, container_client, picture_set_id: str): +async def get_picture_blob(cursor, user_id: str, container_client, picture_id: str): """ - Retrieves blobs of each picture in the given picture_set + Retrieves blob of the given picture Args: cursor: The cursor object to interact with the database. user_id (str): id of the user - picture_set_id (str): id of the picture set + picture_id (str): id of the picture set """ try: # Check if user exists @@ -742,26 +688,21 @@ async def get_pictures_blobs(cursor, user_id: str, container_client, picture_set f"User not found based on the given id: {user_id}" ) # Check if picture set exists - if not picture.is_a_picture_set_id(cursor, picture_set_id): - raise picture.PictureSetNotFoundError( - f"Picture set not found based on the given id: {picture_set_id}" + if not picture.is_a_picture_id(cursor, picture_id): + raise picture.PictureNotFoundError( + f"Picture set not found based on the given id: {picture_id}" ) - # Check user is owner of the picture set - if picture.get_picture_set_owner_id(cursor, picture_set_id) != user_id: + # Check user is owner of the picture set where the picutre is + picture_set_id = picture.get_picture_picture_set_id(cursor, picture_id) + if str(picture.get_picture_set_owner_id(cursor, picture_set_id)) != user_id: raise UserNotOwnerError( - f"User can't delete this folder, user uuid :{user_id}, folder name : {picture_set_id}" + f"User can't access this picture, user uuid :{user_id}, picture : {picture_id}" ) - result=[] folder_name = picture.get_picture_set_name(cursor, picture_set_id) - pictures = picture.get_picture_set_pictures(cursor, picture_set_id) - for pic in pictures: - picture_id = pic[0] - blob_name = "{}/{}.png".format(folder_name, picture_id) - picture_hash = azure_storage.get_blob(container_client, blob_name) - result.append(picture_hash) - print(result) - return result - except(user.UserNotFoundError,picture.PictureSetNotFoundError,UserNotOwnerError) as e: + blob_name = "{}/{}.png".format(folder_name, picture_id) + picture_hash = await azure_storage.get_blob(container_client, blob_name) + return picture_hash + except(user.UserNotFoundError,picture.PictureNotFoundError,UserNotOwnerError) as e: raise e async def delete_picture_set_with_archive( diff --git a/datastore/db/metadata/inference/__init__.py b/datastore/db/metadata/inference/__init__.py index ac516f91..70f3df57 100644 --- a/datastore/db/metadata/inference/__init__.py +++ b/datastore/db/metadata/inference/__init__.py @@ -5,7 +5,8 @@ """ import json - +import datastore.db.queries.seed as seed +import datastore.db.queries.inference as inference class MissingKeyError(Exception): pass @@ -74,3 +75,102 @@ def compare_object_metadata(object1:dict , object2:dict) -> bool: if object1[key] != object2[key]: return False return True + +def rebuild_inference(cursor, inf) : + """ + This function rebuilds the inference object from the database. + + Parameters: + - inference: The inference from the database to convert into inference result. + + Returns: + - The inference object as a dict. + """ + inference_id = str(inf[0]) + inference_data = json.loads(json.dumps(inf[1])) + + objects = inference.get_objects_by_inference(cursor, inference_id) + boxes = rebuild_boxes_export(cursor, objects) + + inf = { + "boxes": boxes, + "filename": inference_data.get("filename"), + "inference_id": inference_id, + "labelOccurrence" : inference_data.get("labelOccurrence"), + "totalBoxes" : inference_data.get("totalBoxes"), + } + return inf + + +def rebuild_boxes_export(cursor, objects) : + """ + This function rebuilds the boxes object from the database. + + Parameters: + - objects: The objects of an inference from the database to convert into boxes. + + Returns: + - The boxes object as an array of dict. + """ + boxes = [] + for object in objects: + box_id = str(object[0]) + + box_metadata = object[1] + box_metadata = json.loads(json.dumps(box_metadata)) + + box = box_metadata.get("box") + color = box_metadata.get("color") + overlapping = box_metadata.get("overlapping") + overlappingIndices = box_metadata.get("overlappingIndices") + + top_id = str(inference.get_inference_object_top_id(cursor, box_id)) + top_seed_id = str(seed.get_seed_object_seed_id(cursor, top_id)) + + seed_objects = inference.get_seed_object_by_object_id(cursor, box_id) + topN = rebuild_topN_export(cursor, seed_objects) + + top_score = 0 + if inference.is_object_verified(cursor, box_id): + top_score = 1 + else : + for seed_object in seed_objects: + score = seed_object[2] + seed_id = str(seed_object[1]) + if seed_id == top_seed_id: + top_score = score + top_label = seed.get_seed_name(cursor, top_seed_id) + + object = {"box" : box, + "box_id": box_id, + "color" : color, + "label" : top_label, + "object_type_id" : 1, + "overlapping" : overlapping, + "overlappingIndices" : overlappingIndices, + "score" : top_score, + "topN": topN, + "top_id" : top_id} + + boxes.append(object) + return boxes + + +def rebuild_topN_export(cursor, seed_objects) : + """ + This function rebuilds the topN object from the database. + + Parameters: + - seed_objects: The seed_objects from the database to convert into topN. + + Returns: + - The topN object as an array of dict. + """ + topN = [] + for seed_obj in seed_objects : + seed_obj = { + "label": seed.get_seed_name(cursor, str(seed_obj[1])), + "object_id": str(seed_obj[0]), + "score": seed_obj[2]} + topN.append(seed_obj) + return topN diff --git a/datastore/db/queries/picture/__init__.py b/datastore/db/queries/picture/__init__.py index 3535abe7..8c2eff3a 100644 --- a/datastore/db/queries/picture/__init__.py +++ b/datastore/db/queries/picture/__init__.py @@ -403,7 +403,7 @@ def get_picture_inference(cursor, picture_id: str): picture_id = %s """ cursor.execute(query, (picture_id,)) - result = cursor.fetchone()[0] + result = cursor.fetchone() return result except Exception: raise GetPictureError( @@ -519,6 +519,53 @@ def is_a_picture_set_id(cursor, picture_set_id): except Exception: raise Exception("unhandled error") +def is_a_picture_id(cursor, picture_id): + """ + This function checks if a picture_id exists in the database. + + Parameters: + - cursor (cursor): The cursor of the database. + - picture_id (str): The UUID of the picture to check. + """ + try: + query = """ + SELECT EXISTS( + SELECT + 1 + FROM + picture + WHERE + id = %s + ) + """ + cursor.execute(query, (picture_id,)) + res = cursor.fetchone()[0] + return res + except Exception: + raise Exception("unhandled error") + +def get_picture_picture_set_id(cursor, picture_id): + """ + This function retrieves the picture_set_id of a picture in the database. + + Parameters: + - cursor (cursor): The cursor of the database. + - picture_id (str): The UUID of the picture to retrieve the picture_set_id from. + """ + try: + query = """ + SELECT + picture_set_id + FROM + picture + WHERE + id = %s + """ + cursor.execute(query, (picture_id,)) + return str(cursor.fetchone()[0]) + except Exception: + raise PictureNotFoundError(f"Error: Picture not found:{picture_id}") + def get_picture_set_owner_id(cursor, picture_set_id): """ This function retrieves the owner_id of a picture_set. diff --git a/datastore/db/queries/seed/__init__.py b/datastore/db/queries/seed/__init__.py index 44302032..79054651 100644 --- a/datastore/db/queries/seed/__init__.py +++ b/datastore/db/queries/seed/__init__.py @@ -165,3 +165,30 @@ def is_seed_registered(cursor, seed_name: str) -> bool: return res except Exception: raise Exception("Error: could not check if seed name is a seed") + +def get_seed_object_seed_id(cursor, seed_object_id: str) -> str: + """ + This function retrieves the seed_id of a seed object. + + Parameters: + - cursor (cursor): The cursor of the database. + - seed_object_id (str): The id of the seed object. + + Returns: + - The seed_id of the seed object. + """ + try: + query = """ + SELECT + seed_id + FROM + seed_obj + WHERE + id = %s + """ + cursor.execute(query, (seed_object_id,)) + return cursor.fetchone()[0] + except TypeError: + raise SeedNotFoundError("Error: seed not found") + except Exception: + raise Exception("unhandled error") \ No newline at end of file diff --git a/tests/test_datastore.py b/tests/test_datastore.py index 4b0bd5d0..a9016efa 100644 --- a/tests/test_datastore.py +++ b/tests/test_datastore.py @@ -6,7 +6,7 @@ import os import unittest from unittest.mock import MagicMock, patch -from PIL import Image +from PIL import Image, ImageChops import json import uuid import asyncio @@ -185,7 +185,7 @@ def setUp(self): #self.picture_hash= asyncio.run(azure_storage.generate_hash(self.pic_encoded)) self.container_name='test-container' self.user_id=datastore.User.get_id(self.user_obj) - self.container_client = asyncio.run(datastore.get_user_container_client(self.user_id,'test-user')) + self.container_client = asyncio.run(datastore.get_user_container_client(self.user_id,BLOB_CONNECTION_STRING,BLOB_ACCOUNT,BLOB_KEY,'test-user')) self.seed_name = "test-name" self.seed_id = seed_query.new_seed(self.cursor, self.seed_name) with open("tests/inference_result.json") as file: @@ -300,6 +300,112 @@ def test_upload_pictures_error_seed_not_found(self): with self.assertRaises(datastore.seed.SeedNotFoundError): asyncio.run(datastore.upload_pictures(self.cursor, self.user_id, picture_set_id, self.container_client,[self.pic_encoded,self.pic_encoded,self.pic_encoded], "unknown_seed")) + def test_get_picture_inference(self): + """ + This test checks if the get_picture_inference function correctly returns the inference of a picture + """ + picture_id = asyncio.run(nachet_datastore.upload_picture_unknown(self.cursor, self.user_id, self.pic_encoded,self.container_client)) + inference = asyncio.run(nachet_datastore.register_inference_result(self.cursor,self.user_id,self.inference, picture_id, "test_model_id")) + + # delete models in the registered inference result because it's not retrieved by the get_picture_inference function + if 'models' in inference: + del inference['models'] + + picture_inference = asyncio.run(nachet_datastore.get_picture_inference(self.cursor, str(self.user_id), str(picture_id))) + + self.assertDictEqual(picture_inference,inference) + + def test_get_picture_inference_error_user_not_found(self): + """ + This test checks if the get_pictures_inferences function correctly raise an exception if the user given doesn't exist in db + """ + picture_id = asyncio.run(nachet_datastore.upload_picture_unknown(self.cursor, self.user_id, self.pic_encoded,self.container_client)) + with self.assertRaises(datastore.user.UserNotFoundError): + asyncio.run(nachet_datastore.get_picture_inference(self.cursor, str(uuid.uuid4()), str(picture_id))) + + def test_get_picture_inference_error_connection_error(self): + """ + This test checks if the get_pictures_inferences function correctly raise an exception if the connection to the db fails + """ + picture_id = asyncio.run(nachet_datastore.upload_picture_unknown(self.cursor, self.user_id, self.pic_encoded,self.container_client)) + mock_cursor = MagicMock() + mock_cursor.fetchall.side_effect = Exception("Connection error") + with self.assertRaises(Exception): + asyncio.run(nachet_datastore.get_picture_inference(mock_cursor, str(self.user_id), str(picture_id))) + + def test_get_picture_inference_error_picture_not_found(self): + """ + This test checks if the get_pictures_inferences function correctly raise an exception if the picture given doesn't exist in db + """ + with self.assertRaises(datastore.picture.PictureNotFoundError): + asyncio.run(nachet_datastore.get_picture_inference(self.cursor, str(self.user_id), str(uuid.uuid4()))) + + def test_get_picture_inference_error_not_owner(self): + """ + This test checks if the get_pictures_inferences function correctly raise an exception if the user is not the owner of the picture set + """ + picture_id = asyncio.run(nachet_datastore.upload_picture_unknown(self.cursor, self.user_id, self.pic_encoded,self.container_client)) + + not_owner_user_obj= asyncio.run(datastore.new_user(self.cursor,"notowner@email",self.connection_str,'test-user')) + not_owner_user_id=datastore.User.get_id(not_owner_user_obj) + + with self.assertRaises(nachet_datastore.UserNotOwnerError): + asyncio.run(nachet_datastore.get_picture_inference(self.cursor, str(not_owner_user_id), str(picture_id))) + + container_client = asyncio.run(datastore.get_user_container_client(not_owner_user_id,BLOB_CONNECTION_STRING,BLOB_ACCOUNT,BLOB_KEY,'test-user')) + container_client.delete_container() + + def test_get_picture_blob(self): + """ + This test checks if the get_picture_blob function correctly returns the blob of a picture + """ + picture_id = asyncio.run(nachet_datastore.upload_picture_unknown(self.cursor, self.user_id, self.pic_encoded,self.container_client)) + blob = asyncio.run(nachet_datastore.get_picture_blob(self.cursor, str(self.user_id), self.container_client, str(picture_id))) + blob_image = Image.frombytes("RGB", (1980, 1080), blob) + + difference = ImageChops.difference(blob_image, self.image) + self.assertTrue(difference.getbbox() is None) + + def test_get_picture_blob_error_user_not_found(self): + """ + This test checks if the get_pictures_inferences function correctly raise an exception if the user given doesn't exist in db + """ + picture_id = asyncio.run(nachet_datastore.upload_picture_unknown(self.cursor, self.user_id, self.pic_encoded,self.container_client)) + with self.assertRaises(datastore.user.UserNotFoundError): + asyncio.run(nachet_datastore.get_picture_blob(self.cursor, str(uuid.uuid4()), self.container_client, str(picture_id))) + + def test_get_picture_blob_error_connection_error(self): + """ + This test checks if the get_pictures_inferences function correctly raise an exception if the connection to the db fails + """ + picture_id = asyncio.run(nachet_datastore.upload_picture_unknown(self.cursor, self.user_id, self.pic_encoded,self.container_client)) + mock_cursor = MagicMock() + mock_cursor.fetchall.side_effect = Exception("Connection error") + with self.assertRaises(Exception): + asyncio.run(nachet_datastore.get_picture_blob(mock_cursor, str(self.user_id), self.container_client, str(picture_id))) + + def test_get_picture_blob_error_picture_set_not_found(self): + """ + This test checks if the get_pictures_inferences function correctly raise an exception if the picture set given doesn't exist in db + """ + with self.assertRaises(datastore.picture.PictureNotFoundError): + asyncio.run(nachet_datastore.get_picture_blob(self.cursor, str(self.user_id), self.container_client, str(uuid.uuid4()))) + + def test_get_picture_blob_error_not_owner(self): + """ + This test checks if the get_pictures_inferences function correctly raise an exception if the user is not the owner of the picture set + """ + picture_id = asyncio.run(nachet_datastore.upload_picture_unknown(self.cursor, self.user_id, self.pic_encoded,self.container_client)) + + not_owner_user_obj= asyncio.run(datastore.new_user(self.cursor,"notowner@email",self.connection_str,'test-user')) + not_owner_user_id=datastore.User.get_id(not_owner_user_obj) + + with self.assertRaises(nachet_datastore.UserNotOwnerError): + asyncio.run(nachet_datastore.get_picture_blob(self.cursor, str(not_owner_user_id), self.container_client, str(picture_id))) + + container_client = asyncio.run(datastore.get_user_container_client(not_owner_user_id,BLOB_CONNECTION_STRING,BLOB_ACCOUNT,BLOB_KEY,'test-user')) + container_client.delete_container() + class test_feedback(unittest.TestCase): def setUp(self): self.con = db.connect_db(DB_CONNECTION_STRING,DB_SCHEMA) @@ -591,14 +697,6 @@ def test_get_picture_sets_info_error_connection_error(self): with self.assertRaises(Exception): asyncio.run(datastore.get_picture_sets_info(mock_cursor, self.user_id)) - def test_get_pictures_inferences(self): - """ - This test checks if the get_pictures_inferences function correctly returns the inferences of each pictures a picture set - """ - - picture_sets_info = asyncio.run(nachet_datastore.get_pictures_inferences(self.cursor, self.user_id, self.picture_set_id)) - - def test_find_validated_pictures(self): """ This test checks if the find_validated_pictures function correctly returns the validated pictures of a picture_set From a06703a6d32da71c851469ada1d5a4e5cfb3eda9 Mon Sep 17 00:00:00 2001 From: sylvanie85 Date: Wed, 24 Jul 2024 19:41:14 +0000 Subject: [PATCH 19/27] retrieve models --- datastore/db/metadata/inference/__init__.py | 14 ++++++++++++++ datastore/db/queries/picture/__init__.py | 2 +- tests/test_datastore.py | 4 ---- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/datastore/db/metadata/inference/__init__.py b/datastore/db/metadata/inference/__init__.py index 70f3df57..58fe1a63 100644 --- a/datastore/db/metadata/inference/__init__.py +++ b/datastore/db/metadata/inference/__init__.py @@ -7,6 +7,7 @@ import json import datastore.db.queries.seed as seed import datastore.db.queries.inference as inference +import datastore.db.queries.machine_learning as machine_learning class MissingKeyError(Exception): pass @@ -88,6 +89,18 @@ def rebuild_inference(cursor, inf) : """ inference_id = str(inf[0]) inference_data = json.loads(json.dumps(inf[1])) + pipeline_id = str(inf[2]) + + models = [] + if pipeline_id is not None : + pipeline = machine_learning.get_pipeline(cursor, pipeline_id) + models_data = pipeline["models"] + version = pipeline["version"] + for model_name in models_data : + model = {} + model["name"] = model_name + model["version"] = version + models.append(model) objects = inference.get_objects_by_inference(cursor, inference_id) boxes = rebuild_boxes_export(cursor, objects) @@ -98,6 +111,7 @@ def rebuild_inference(cursor, inf) : "inference_id": inference_id, "labelOccurrence" : inference_data.get("labelOccurrence"), "totalBoxes" : inference_data.get("totalBoxes"), + "models" : models, } return inf diff --git a/datastore/db/queries/picture/__init__.py b/datastore/db/queries/picture/__init__.py index 8c2eff3a..c20f0b62 100644 --- a/datastore/db/queries/picture/__init__.py +++ b/datastore/db/queries/picture/__init__.py @@ -396,7 +396,7 @@ def get_picture_inference(cursor, picture_id: str): try : query = """ SELECT - id, inference + id, inference, pipeline_id FROM inference WHERE diff --git a/tests/test_datastore.py b/tests/test_datastore.py index a9016efa..ec038d98 100644 --- a/tests/test_datastore.py +++ b/tests/test_datastore.py @@ -307,10 +307,6 @@ def test_get_picture_inference(self): picture_id = asyncio.run(nachet_datastore.upload_picture_unknown(self.cursor, self.user_id, self.pic_encoded,self.container_client)) inference = asyncio.run(nachet_datastore.register_inference_result(self.cursor,self.user_id,self.inference, picture_id, "test_model_id")) - # delete models in the registered inference result because it's not retrieved by the get_picture_inference function - if 'models' in inference: - del inference['models'] - picture_inference = asyncio.run(nachet_datastore.get_picture_inference(self.cursor, str(self.user_id), str(picture_id))) self.assertDictEqual(picture_inference,inference) From 1371a9a5ba85d5a32fd67e7642d47cdb4d1241ab Mon Sep 17 00:00:00 2001 From: sylvanie85 Date: Fri, 26 Jul 2024 19:35:20 +0000 Subject: [PATCH 20/27] fix tests --- datastore/Nachet/__init__.py | 40 +++++++++ datastore/__init__.py | 41 --------- datastore/db/metadata/inference/__init__.py | 1 + tests/Nachet/test_datastore.py | 99 +++++++++++++++------ tests/test_datastore.py | 49 ---------- 5 files changed, 115 insertions(+), 115 deletions(-) diff --git a/datastore/Nachet/__init__.py b/datastore/Nachet/__init__.py index cd0ea906..f9a305e0 100644 --- a/datastore/Nachet/__init__.py +++ b/datastore/Nachet/__init__.py @@ -640,6 +640,46 @@ async def get_seed_info(cursor): seed_dict["seeds"].append({"seed_id": seed_id, "seed_name": seed_name}) return seed_dict +async def get_picture_sets_info(cursor, user_id: str): + """This function retrieves the picture sets names and number of pictures from the database. + This also retrieve for each picture in the picture set their name, if an inference exist and if the picture is validated. + + Args: + user_id (str): id of the user + """ + # Check if user exists + if not user.is_a_user_id(cursor=cursor, user_id=user_id): + raise user.UserNotFoundError(f"User not found based on the given id: {user_id}") + + result = [] + picture_sets = picture.get_user_picture_sets(cursor, user_id) + for picture_set in picture_sets: + picture_set_info = {} + picture_set_id = picture_set[0] + picture_set_name = picture_set[1] + + picture_set_info["picture_set_id"] = str(picture_set_id) + picture_set_info["folder_name"] = picture_set_name + + pictures = picture.get_picture_set_pictures(cursor, picture_set_id) + picture_set_info["nb_pictures"] = len(pictures) + + picture_set_info["pictures"] = [] + for pic in pictures : + picture_info = {} + picture_id = pic[0] + picture_info["picture_id"] = str(picture_id) + + is_validated = picture.is_picture_validated(cursor, picture_id) + inference_exist = picture.check_picture_inference_exist(cursor, picture_id) + picture_info["is_validated"] = is_validated + picture_info["inference_exist"] = inference_exist + + picture_set_info["pictures"].append(picture_info) + result.append(picture_set_info) + return result + + async def get_picture_inference(cursor, user_id: str, picture_id: str): """ Retrieves inference (if exist) of the given picture diff --git a/datastore/__init__.py b/datastore/__init__.py index 9572b449..1b36079b 100644 --- a/datastore/__init__.py +++ b/datastore/__init__.py @@ -182,47 +182,6 @@ async def create_picture_set( except Exception: raise BlobUploadError("An error occured during the upload of the picture set") - -async def get_picture_sets_info(cursor, user_id: str): - """This function retrieves the picture sets names and number of pictures from the database. - This also retrieve for each picture in the picture set their name, if an inference exist and if the picture is validated. - - Args: - user_id (str): id of the user - """ - # Check if user exists - if not user.is_a_user_id(cursor=cursor, user_id=user_id): - raise user.UserNotFoundError(f"User not found based on the given id: {user_id}") - - result = [] - picture_sets = picture.get_user_picture_sets(cursor, user_id) - for picture_set in picture_sets: - picture_set_info = {} - picture_set_id = picture_set[0] - picture_set_name = picture_set[1] - - picture_set_info["picture_set_id"] = str(picture_set_id) - picture_set_info["folder_name"] = picture_set_name - - pictures = picture.get_picture_set_pictures(cursor, picture_set_id) - picture_set_info["nb_pictures"] = len(pictures) - - picture_set_info["pictures"] = [] - for pic in pictures : - picture_info = {} - picture_id = pic[0] - picture_info["picture_id"] = str(picture_id) - - is_validated = picture.is_picture_validated(cursor, picture_id) - inference_exist = picture.check_picture_inference_exist(cursor, picture_id) - picture_info["is_validated"] = is_validated - picture_info["inference_exist"] = inference_exist - - picture_set_info["pictures"].append(picture_info) - result.append(picture_set_info) - return result - - async def delete_picture_set_permanently( cursor, user_id, picture_set_id, container_client ): diff --git a/datastore/db/metadata/inference/__init__.py b/datastore/db/metadata/inference/__init__.py index 58fe1a63..859c640b 100644 --- a/datastore/db/metadata/inference/__init__.py +++ b/datastore/db/metadata/inference/__init__.py @@ -112,6 +112,7 @@ def rebuild_inference(cursor, inf) : "labelOccurrence" : inference_data.get("labelOccurrence"), "totalBoxes" : inference_data.get("totalBoxes"), "models" : models, + "pipeline_id" : pipeline_id, } return inf diff --git a/tests/Nachet/test_datastore.py b/tests/Nachet/test_datastore.py index 9fa2b79f..bdd6eb36 100644 --- a/tests/Nachet/test_datastore.py +++ b/tests/Nachet/test_datastore.py @@ -252,10 +252,10 @@ def test_get_picture_inference(self): """ This test checks if the get_picture_inference function correctly returns the inference of a picture """ - picture_id = asyncio.run(datastore.upload_picture_unknown(self.cursor, self.user_id, self.pic_encoded,self.container_client)) - inference = asyncio.run(datastore.register_inference_result(self.cursor,self.user_id,self.inference, picture_id, "test_model_id")) + picture_id = asyncio.run(Nachet.upload_picture_unknown(self.cursor, self.user_id, self.pic_encoded,self.container_client)) + inference = asyncio.run(Nachet.register_inference_result(self.cursor,self.user_id,self.inference, picture_id, "test_model_id")) - picture_inference = asyncio.run(datastore.get_picture_inference(self.cursor, str(self.user_id), str(picture_id))) + picture_inference = asyncio.run(Nachet.get_picture_inference(self.cursor, str(self.user_id), str(picture_id))) self.assertDictEqual(picture_inference,inference) @@ -263,38 +263,38 @@ def test_get_picture_inference_error_user_not_found(self): """ This test checks if the get_pictures_inferences function correctly raise an exception if the user given doesn't exist in db """ - picture_id = asyncio.run(datastore.upload_picture_unknown(self.cursor, self.user_id, self.pic_encoded,self.container_client)) + picture_id = asyncio.run(Nachet.upload_picture_unknown(self.cursor, self.user_id, self.pic_encoded,self.container_client)) with self.assertRaises(datastore.user.UserNotFoundError): - asyncio.run(datastore.get_picture_inference(self.cursor, str(uuid.uuid4()), str(picture_id))) + asyncio.run(Nachet.get_picture_inference(self.cursor, str(uuid.uuid4()), str(picture_id))) def test_get_picture_inference_error_connection_error(self): """ This test checks if the get_pictures_inferences function correctly raise an exception if the connection to the db fails """ - picture_id = asyncio.run(datastore.upload_picture_unknown(self.cursor, self.user_id, self.pic_encoded,self.container_client)) + picture_id = asyncio.run(Nachet.upload_picture_unknown(self.cursor, self.user_id, self.pic_encoded,self.container_client)) mock_cursor = MagicMock() mock_cursor.fetchall.side_effect = Exception("Connection error") with self.assertRaises(Exception): - asyncio.run(datastore.get_picture_inference(mock_cursor, str(self.user_id), str(picture_id))) + asyncio.run(Nachet.get_picture_inference(mock_cursor, str(self.user_id), str(picture_id))) def test_get_picture_inference_error_picture_not_found(self): """ This test checks if the get_pictures_inferences function correctly raise an exception if the picture given doesn't exist in db """ with self.assertRaises(datastore.picture.PictureNotFoundError): - asyncio.run(datastore.get_picture_inference(self.cursor, str(self.user_id), str(uuid.uuid4()))) + asyncio.run(Nachet.get_picture_inference(self.cursor, str(self.user_id), str(uuid.uuid4()))) def test_get_picture_inference_error_not_owner(self): """ This test checks if the get_pictures_inferences function correctly raise an exception if the user is not the owner of the picture set """ - picture_id = asyncio.run(datastore.upload_picture_unknown(self.cursor, self.user_id, self.pic_encoded,self.container_client)) + picture_id = asyncio.run(Nachet.upload_picture_unknown(self.cursor, self.user_id, self.pic_encoded,self.container_client)) not_owner_user_obj= asyncio.run(datastore.new_user(self.cursor,"notowner@email",self.connection_str,'test-user')) not_owner_user_id=datastore.User.get_id(not_owner_user_obj) - with self.assertRaises(datastore.UserNotOwnerError): - asyncio.run(datastore.get_picture_inference(self.cursor, str(not_owner_user_id), str(picture_id))) + with self.assertRaises(Nachet.UserNotOwnerError): + asyncio.run(Nachet.get_picture_inference(self.cursor, str(not_owner_user_id), str(picture_id))) container_client = asyncio.run(datastore.get_user_container_client(not_owner_user_id,BLOB_CONNECTION_STRING,BLOB_ACCOUNT,BLOB_KEY,'test-user')) container_client.delete_container() @@ -303,8 +303,8 @@ def test_get_picture_blob(self): """ This test checks if the get_picture_blob function correctly returns the blob of a picture """ - picture_id = asyncio.run(datastore.upload_picture_unknown(self.cursor, self.user_id, self.pic_encoded,self.container_client)) - blob = asyncio.run(datastore.get_picture_blob(self.cursor, str(self.user_id), self.container_client, str(picture_id))) + picture_id = asyncio.run(Nachet.upload_picture_unknown(self.cursor, self.user_id, self.pic_encoded,self.container_client)) + blob = asyncio.run(Nachet.get_picture_blob(self.cursor, str(self.user_id), self.container_client, str(picture_id))) blob_image = Image.frombytes("RGB", (1980, 1080), blob) difference = ImageChops.difference(blob_image, self.image) @@ -314,38 +314,38 @@ def test_get_picture_blob_error_user_not_found(self): """ This test checks if the get_pictures_inferences function correctly raise an exception if the user given doesn't exist in db """ - picture_id = asyncio.run(datastore.upload_picture_unknown(self.cursor, self.user_id, self.pic_encoded,self.container_client)) + picture_id = asyncio.run(Nachet.upload_picture_unknown(self.cursor, self.user_id, self.pic_encoded,self.container_client)) with self.assertRaises(datastore.user.UserNotFoundError): - asyncio.run(datastore.get_picture_blob(self.cursor, str(uuid.uuid4()), self.container_client, str(picture_id))) + asyncio.run(Nachet.get_picture_blob(self.cursor, str(uuid.uuid4()), self.container_client, str(picture_id))) def test_get_picture_blob_error_connection_error(self): """ This test checks if the get_pictures_inferences function correctly raise an exception if the connection to the db fails """ - picture_id = asyncio.run(datastore.upload_picture_unknown(self.cursor, self.user_id, self.pic_encoded,self.container_client)) + picture_id = asyncio.run(Nachet.upload_picture_unknown(self.cursor, self.user_id, self.pic_encoded,self.container_client)) mock_cursor = MagicMock() mock_cursor.fetchall.side_effect = Exception("Connection error") with self.assertRaises(Exception): - asyncio.run(datastore.get_picture_blob(mock_cursor, str(self.user_id), self.container_client, str(picture_id))) + asyncio.run(Nachet.get_picture_blob(mock_cursor, str(self.user_id), self.container_client, str(picture_id))) def test_get_picture_blob_error_picture_set_not_found(self): """ This test checks if the get_pictures_inferences function correctly raise an exception if the picture set given doesn't exist in db """ with self.assertRaises(datastore.picture.PictureNotFoundError): - asyncio.run(datastore.get_picture_blob(self.cursor, str(self.user_id), self.container_client, str(uuid.uuid4()))) + asyncio.run(Nachet.get_picture_blob(self.cursor, str(self.user_id), self.container_client, str(uuid.uuid4()))) def test_get_picture_blob_error_not_owner(self): """ This test checks if the get_pictures_inferences function correctly raise an exception if the user is not the owner of the picture set """ - picture_id = asyncio.run(datastore.upload_picture_unknown(self.cursor, self.user_id, self.pic_encoded,self.container_client)) + picture_id = asyncio.run(Nachet.upload_picture_unknown(self.cursor, self.user_id, self.pic_encoded,self.container_client)) not_owner_user_obj= asyncio.run(datastore.new_user(self.cursor,"notowner@email",self.connection_str,'test-user')) not_owner_user_id=datastore.User.get_id(not_owner_user_obj) - with self.assertRaises(datastore.UserNotOwnerError): - asyncio.run(datastore.get_picture_blob(self.cursor, str(not_owner_user_id), self.container_client, str(picture_id))) + with self.assertRaises(Nachet.UserNotOwnerError): + asyncio.run(Nachet.get_picture_blob(self.cursor, str(not_owner_user_id), self.container_client, str(picture_id))) container_client = asyncio.run(datastore.get_user_container_client(not_owner_user_id,BLOB_CONNECTION_STRING,BLOB_ACCOUNT,BLOB_KEY,'test-user')) container_client.delete_container() @@ -413,6 +413,55 @@ def tearDown(self): self.container_client.delete_container() db.end_query(self.con, self.cursor) + def test_get_picture_sets_info(self) : + """ + Test the get_picture_sets_info function + """ + + picture_sets_info = asyncio.run(Nachet.get_picture_sets_info(self.cursor, self.user_id)) + + self.assertEqual(len(picture_sets_info), 2) + + for picture_set in picture_sets_info : + if picture_set["picture_set_id"] == self.picture_set_id : + expected_pictures_info = [ + {"picture_id": pid, "is_verified": False, "inference_exist": False} + for pid in self.pictures_id ] + self.assert_picture_set_info(picture_set, self.picture_set_id, self.folder_name, 3, expected_pictures_info) + + self.picture_set_id = asyncio.run(datastore.create_picture_set(self.cursor, self.container_client, 0, self.user_id, self.folder_name + "2")) + picture_id = asyncio.run(Nachet.upload_picture_unknown(self.cursor, self.user_id, self.pic_encoded,self.container_client, self.picture_set_id)) + inference = asyncio.run(Nachet.register_inference_result(self.cursor,self.user_id,self.inference, picture_id, "test_model_id")) + asyncio.run(Nachet.new_perfect_inference_feeback(self.cursor, inference["inference_id"], self.user_id, [box["box_id"] for box in inference["boxes"]])) + + picture_sets_info = asyncio.run(Nachet.get_picture_sets_info(self.cursor, self.user_id)) + + self.assertEqual(len(picture_sets_info), 3) + + for picture_set in picture_sets_info : + if picture_set["picture_set_id"] == self.picture_set_id : + expected_pictures_info = [ + {"picture_id": picture_id, "is_verified": True, "inference_exist": True} + ] + self.assert_picture_set_info(picture_set, self.picture_set_id, self.folder_name+"2", 1, expected_pictures_info) + + + def test_get_picture_sets_info_error_user_not_found(self): + """ + This test checks if the get_picture_sets_info function correctly raise an exception if the user given doesn't exist in db + """ + with self.assertRaises(datastore.user.UserNotFoundError): + asyncio.run(Nachet.get_picture_sets_info(self.cursor, uuid.uuid4())) + + def test_get_picture_sets_info_error_connection_error(self): + """ + This test checks if the get_picture_sets_info function correctly raise an exception if the connection to the db fails + """ + mock_cursor = MagicMock() + mock_cursor.fetchone.side_effect = Exception("Connection error") + with self.assertRaises(Exception): + asyncio.run(Nachet.get_picture_sets_info(mock_cursor, self.user_id)) + def test_find_validated_pictures(self): """ This test checks if the find_validated_pictures function correctly returns the validated pictures of a picture_set @@ -615,12 +664,12 @@ def test_delete_picture_set_with_archive(self): ) dev_nb_folders = len( - asyncio.run(datastore.get_picture_sets_info(self.cursor, self.dev_user_id)) + asyncio.run(Nachet.get_picture_sets_info(self.cursor, self.dev_user_id)) ) # Check there is the right number of picture sets in db for each user self.assertEqual( len( - asyncio.run(datastore.get_picture_sets_info(self.cursor, self.user_id)) + asyncio.run(Nachet.get_picture_sets_info(self.cursor, self.user_id)) ), 2, ) @@ -636,14 +685,14 @@ def test_delete_picture_set_with_archive(self): # Check there is the right number of picture sets in db for each user after moving self.assertEqual( len( - asyncio.run(datastore.get_picture_sets_info(self.cursor, self.user_id)) + asyncio.run(Nachet.get_picture_sets_info(self.cursor, self.user_id)) ), 1, ) self.assertEqual( len( asyncio.run( - datastore.get_picture_sets_info(self.cursor, self.dev_user_id) + Nachet.get_picture_sets_info(self.cursor, self.dev_user_id) ) ), dev_nb_folders + 1, diff --git a/tests/test_datastore.py b/tests/test_datastore.py index 264ef9f2..8bdb9853 100644 --- a/tests/test_datastore.py +++ b/tests/test_datastore.py @@ -216,56 +216,7 @@ def assert_picture_info(self, picture_info, expected_pic_info): self.assertEqual(picture_info["picture_id"], expected_pic_info["picture_id"]) self.assertEqual(picture_info["is_verified"], expected_pic_info["is_verified"]) self.assertEqual(picture_info["inference_exist"], expected_pic_info["inference_exist"]) - - def test_get_picture_sets_info(self) : - """ - Test the get_picture_sets_info function - """ - - picture_sets_info = asyncio.run(datastore.get_picture_sets_info(self.cursor, self.user_id)) - - self.assertEqual(len(picture_sets_info), 2) - - for picture_set in picture_sets_info : - if picture_set["picture_set_id"] == self.picture_set_id : - expected_pictures_info = [ - {"picture_id": pid, "is_verified": False, "inference_exist": False} - for pid in self.pictures_id ] - self.assert_picture_set_info(picture_set, self.picture_set_id, self.folder_name, 3, expected_pictures_info) - - self.picture_set_id = asyncio.run(datastore.create_picture_set(self.cursor, self.container_client, 0, self.user_id, self.folder_name + "2")) - picture_id = asyncio.run(nachet_datastore.upload_picture_unknown(self.cursor, self.user_id, self.pic_encoded,self.container_client, self.picture_set_id)) - inference = asyncio.run(nachet_datastore.register_inference_result(self.cursor,self.user_id,self.inference, picture_id, "test_model_id")) - asyncio.run(nachet_datastore.new_perfect_inference_feeback(self.cursor, inference["inference_id"], self.user_id, [box["box_id"] for box in inference["boxes"]])) - - picture_sets_info = asyncio.run(datastore.get_picture_sets_info(self.cursor, self.user_id)) - - self.assertEqual(len(picture_sets_info), 3) - - for picture_set in picture_sets_info : - if picture_set["picture_set_id"] == self.picture_set_id : - expected_pictures_info = [ - {"picture_id": picture_id, "is_verified": True, "inference_exist": True} - ] - self.assert_picture_set_info(picture_set, self.picture_set_id, self.folder_name+"2", 1, expected_pictures_info) - - - def test_get_picture_sets_info_error_user_not_found(self): - """ - This test checks if the get_picture_sets_info function correctly raise an exception if the user given doesn't exist in db - """ - with self.assertRaises(datastore.user.UserNotFoundError): - asyncio.run(datastore.get_picture_sets_info(self.cursor, uuid.uuid4())) - def test_get_picture_sets_info_error_connection_error(self): - """ - This test checks if the get_picture_sets_info function correctly raise an exception if the connection to the db fails - """ - mock_cursor = MagicMock() - mock_cursor.fetchone.side_effect = Exception("Connection error") - with self.assertRaises(Exception): - asyncio.run(datastore.get_picture_sets_info(mock_cursor, self.user_id)) - def test_delete_picture_set_permanently(self): """ This test checks the delete_picture_set_permanently function From 1a41ebf65f87fb3be3bb68603d628cc7f4cc71fc Mon Sep 17 00:00:00 2001 From: sylvanie85 Date: Fri, 26 Jul 2024 19:38:42 +0000 Subject: [PATCH 21/27] lint tests --- datastore/db/queries/inference/__init__.py | 2 -- datastore/db/queries/picture/__init__.py | 1 - datastore/db/queries/seed/__init__.py | 2 +- tests/test_datastore.py | 1 - 4 files changed, 1 insertion(+), 5 deletions(-) diff --git a/datastore/db/queries/inference/__init__.py b/datastore/db/queries/inference/__init__.py index 32617a0e..73f74820 100644 --- a/datastore/db/queries/inference/__init__.py +++ b/datastore/db/queries/inference/__init__.py @@ -540,5 +540,3 @@ def get_seed_object_by_object_id(cursor, object_id: str): return res except Exception: raise Exception(f"Error: could not get seed_object for object {object_id}") - - \ No newline at end of file diff --git a/datastore/db/queries/picture/__init__.py b/datastore/db/queries/picture/__init__.py index c20f0b62..a97d32d2 100644 --- a/datastore/db/queries/picture/__init__.py +++ b/datastore/db/queries/picture/__init__.py @@ -26,7 +26,6 @@ class GetPictureError(Exception): class PictureSetDeleteError(Exception): pass -import json """ This module contains all the queries related to the Picture and PictureSet tables. """ diff --git a/datastore/db/queries/seed/__init__.py b/datastore/db/queries/seed/__init__.py index 79054651..426eb8d4 100644 --- a/datastore/db/queries/seed/__init__.py +++ b/datastore/db/queries/seed/__init__.py @@ -191,4 +191,4 @@ def get_seed_object_seed_id(cursor, seed_object_id: str) -> str: except TypeError: raise SeedNotFoundError("Error: seed not found") except Exception: - raise Exception("unhandled error") \ No newline at end of file + raise Exception("unhandled error") diff --git a/tests/test_datastore.py b/tests/test_datastore.py index 8bdb9853..d88c1c99 100644 --- a/tests/test_datastore.py +++ b/tests/test_datastore.py @@ -11,7 +11,6 @@ import asyncio import datastore.db.__init__ as db import datastore.__init__ as datastore -import datastore.Nachet as nachet_datastore import datastore.db.metadata.validator as validator From 0b76149def3a83220703689d3a10439809773399 Mon Sep 17 00:00:00 2001 From: sylvanie85 Date: Fri, 26 Jul 2024 19:46:59 +0000 Subject: [PATCH 22/27] fix get picture set info --- datastore/__init__.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/datastore/__init__.py b/datastore/__init__.py index 1b36079b..321542a3 100644 --- a/datastore/__init__.py +++ b/datastore/__init__.py @@ -181,6 +181,25 @@ async def create_picture_set( raise e except Exception: raise BlobUploadError("An error occured during the upload of the picture set") + +async def get_picture_sets_info(cursor, user_id: str): + """This function retrieves the picture sets names and number of pictures from the database. + + Args: + user_id (str): id of the user + """ + # Check if user exists + if not user.is_a_user_id(cursor=cursor, user_id=user_id): + raise user.UserNotFoundError(f"User not found based on the given id: {user_id}") + + result = {} + picture_sets = picture.get_user_picture_sets(cursor, user_id) + for picture_set in picture_sets: + picture_set_id = picture_set[0] + picture_set_name = picture_set[1] + nb_picture = picture.count_pictures(cursor, picture_set_id) + result[str(picture_set_id)] = [picture_set_name, nb_picture] + return result async def delete_picture_set_permanently( cursor, user_id, picture_set_id, container_client From c46eb946c0b2e78a819e58aa53ce1c62818f2ac7 Mon Sep 17 00:00:00 2001 From: sylvanie85 Date: Tue, 30 Jul 2024 15:53:08 +0000 Subject: [PATCH 23/27] get-picture fixes --- datastore/Nachet/__init__.py | 9 ++++++--- datastore/db/queries/picture/__init__.py | 3 ++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/datastore/Nachet/__init__.py b/datastore/Nachet/__init__.py index f9a305e0..f1683799 100644 --- a/datastore/Nachet/__init__.py +++ b/datastore/Nachet/__init__.py @@ -745,10 +745,13 @@ async def get_picture_blob(cursor, user_id: str, container_client, picture_id: s raise UserNotOwnerError( f"User can't access this picture, user uuid :{user_id}, picture : {picture_id}" ) - folder_name = picture.get_picture_set_name(cursor, picture_set_id) + if str(user.get_default_picture_set(cursor, user_id)) == str(picture_set_id): + folder_name = "General" + else: + folder_name = picture.get_picture_set_name(cursor, picture_set_id) blob_name = "{}/{}.png".format(folder_name, picture_id) - picture_hash = await azure_storage.get_blob(container_client, blob_name) - return picture_hash + picture_blob = await azure_storage.get_blob(container_client, blob_name) + return picture_blob except(user.UserNotFoundError,picture.PictureNotFoundError,UserNotOwnerError) as e: raise e diff --git a/datastore/db/queries/picture/__init__.py b/datastore/db/queries/picture/__init__.py index a97d32d2..b72f5041 100644 --- a/datastore/db/queries/picture/__init__.py +++ b/datastore/db/queries/picture/__init__.py @@ -208,7 +208,8 @@ def get_picture_set_name(cursor, picture_set_id: str): id = %s """ cursor.execute(query, (picture_set_id, )) - return cursor.fetchone()[0] + name = cursor.fetchone()[0] + return name if name is not None else picture_set_id except Exception: raise PictureSetNotFoundError(f"Error: PictureSet not found:{picture_set_id}") From afb715aaedc167a4b7525469694a204ef5fa1a41 Mon Sep 17 00:00:00 2001 From: sylvanie85 Date: Tue, 30 Jul 2024 20:43:52 +0000 Subject: [PATCH 24/27] exception raise fo get_picture_sets --- datastore/Nachet/__init__.py | 66 +++++++++++++++++++----------------- 1 file changed, 35 insertions(+), 31 deletions(-) diff --git a/datastore/Nachet/__init__.py b/datastore/Nachet/__init__.py index f1683799..7bade957 100644 --- a/datastore/Nachet/__init__.py +++ b/datastore/Nachet/__init__.py @@ -56,7 +56,6 @@ class InferenceFeedbackError(Exception): class MLRetrievalError(Exception): pass - async def upload_picture_unknown( cursor, user_id, picture_hash, container_client, picture_set_id=None ): @@ -647,37 +646,42 @@ async def get_picture_sets_info(cursor, user_id: str): Args: user_id (str): id of the user """ - # Check if user exists - if not user.is_a_user_id(cursor=cursor, user_id=user_id): - raise user.UserNotFoundError(f"User not found based on the given id: {user_id}") - - result = [] - picture_sets = picture.get_user_picture_sets(cursor, user_id) - for picture_set in picture_sets: - picture_set_info = {} - picture_set_id = picture_set[0] - picture_set_name = picture_set[1] - - picture_set_info["picture_set_id"] = str(picture_set_id) - picture_set_info["folder_name"] = picture_set_name - - pictures = picture.get_picture_set_pictures(cursor, picture_set_id) - picture_set_info["nb_pictures"] = len(pictures) - - picture_set_info["pictures"] = [] - for pic in pictures : - picture_info = {} - picture_id = pic[0] - picture_info["picture_id"] = str(picture_id) - - is_validated = picture.is_picture_validated(cursor, picture_id) - inference_exist = picture.check_picture_inference_exist(cursor, picture_id) - picture_info["is_validated"] = is_validated - picture_info["inference_exist"] = inference_exist + try : + # Check if user exists + if not user.is_a_user_id(cursor=cursor, user_id=user_id): + raise user.UserNotFoundError(f"User not found based on the given id: {user_id}") + + result = [] + picture_sets = picture.get_user_picture_sets(cursor, user_id) + for picture_set in picture_sets: + picture_set_info = {} + picture_set_id = picture_set[0] + picture_set_name = picture_set[1] - picture_set_info["pictures"].append(picture_info) - result.append(picture_set_info) - return result + picture_set_info["picture_set_id"] = str(picture_set_id) + picture_set_info["folder_name"] = picture_set_name + + pictures = picture.get_picture_set_pictures(cursor, picture_set_id) + picture_set_info["nb_pictures"] = len(pictures) + + picture_set_info["pictures"] = [] + for pic in pictures : + picture_info = {} + picture_id = pic[0] + picture_info["picture_id"] = str(picture_id) + + is_validated = picture.is_picture_validated(cursor, picture_id) + inference_exist = picture.check_picture_inference_exist(cursor, picture_id) + picture_info["is_validated"] = is_validated + picture_info["inference_exist"] = inference_exist + + picture_set_info["pictures"].append(picture_info) + result.append(picture_set_info) + return result + except (user.UserNotFoundError, picture.GetPictureSetError, picture.GetPictureError ) as e : + raise e + except Exception as e: + raise picture.GetPictureSetError("An error occured while retrieving the picture sets") async def get_picture_inference(cursor, user_id: str, picture_id: str): From f5595084c013e30c332b882c6135825386241147 Mon Sep 17 00:00:00 2001 From: sylvanie85 Date: Wed, 31 Jul 2024 14:30:24 +0000 Subject: [PATCH 25/27] quik fix --- tests/nachet/test_datastore.py | 64 +++++++++++++++++----------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/tests/nachet/test_datastore.py b/tests/nachet/test_datastore.py index a99ecccb..cf13ba2c 100644 --- a/tests/nachet/test_datastore.py +++ b/tests/nachet/test_datastore.py @@ -256,10 +256,10 @@ def test_get_picture_inference(self): """ This test checks if the get_picture_inference function correctly returns the inference of a picture """ - picture_id = asyncio.run(Nachet.upload_picture_unknown(self.cursor, self.user_id, self.pic_encoded,self.container_client)) - inference = asyncio.run(Nachet.register_inference_result(self.cursor,self.user_id,self.inference, picture_id, "test_model_id")) + picture_id = asyncio.run(nachet.upload_picture_unknown(self.cursor, self.user_id, self.pic_encoded,self.container_client)) + inference = asyncio.run(nachet.register_inference_result(self.cursor,self.user_id,self.inference, picture_id, "test_model_id")) - picture_inference = asyncio.run(Nachet.get_picture_inference(self.cursor, str(self.user_id), str(picture_id))) + picture_inference = asyncio.run(nachet.get_picture_inference(self.cursor, str(self.user_id), str(picture_id))) self.assertDictEqual(picture_inference,inference) @@ -267,38 +267,38 @@ def test_get_picture_inference_error_user_not_found(self): """ This test checks if the get_pictures_inferences function correctly raise an exception if the user given doesn't exist in db """ - picture_id = asyncio.run(Nachet.upload_picture_unknown(self.cursor, self.user_id, self.pic_encoded,self.container_client)) + picture_id = asyncio.run(nachet.upload_picture_unknown(self.cursor, self.user_id, self.pic_encoded,self.container_client)) with self.assertRaises(datastore.user.UserNotFoundError): - asyncio.run(Nachet.get_picture_inference(self.cursor, str(uuid.uuid4()), str(picture_id))) + asyncio.run(nachet.get_picture_inference(self.cursor, str(uuid.uuid4()), str(picture_id))) def test_get_picture_inference_error_connection_error(self): """ This test checks if the get_pictures_inferences function correctly raise an exception if the connection to the db fails """ - picture_id = asyncio.run(Nachet.upload_picture_unknown(self.cursor, self.user_id, self.pic_encoded,self.container_client)) + picture_id = asyncio.run(nachet.upload_picture_unknown(self.cursor, self.user_id, self.pic_encoded,self.container_client)) mock_cursor = MagicMock() mock_cursor.fetchall.side_effect = Exception("Connection error") with self.assertRaises(Exception): - asyncio.run(Nachet.get_picture_inference(mock_cursor, str(self.user_id), str(picture_id))) + asyncio.run(nachet.get_picture_inference(mock_cursor, str(self.user_id), str(picture_id))) def test_get_picture_inference_error_picture_not_found(self): """ This test checks if the get_pictures_inferences function correctly raise an exception if the picture given doesn't exist in db """ with self.assertRaises(datastore.picture.PictureNotFoundError): - asyncio.run(Nachet.get_picture_inference(self.cursor, str(self.user_id), str(uuid.uuid4()))) + asyncio.run(nachet.get_picture_inference(self.cursor, str(self.user_id), str(uuid.uuid4()))) def test_get_picture_inference_error_not_owner(self): """ This test checks if the get_pictures_inferences function correctly raise an exception if the user is not the owner of the picture set """ - picture_id = asyncio.run(Nachet.upload_picture_unknown(self.cursor, self.user_id, self.pic_encoded,self.container_client)) + picture_id = asyncio.run(nachet.upload_picture_unknown(self.cursor, self.user_id, self.pic_encoded,self.container_client)) not_owner_user_obj= asyncio.run(datastore.new_user(self.cursor,"notowner@email",self.connection_str,'test-user')) not_owner_user_id=datastore.User.get_id(not_owner_user_obj) - with self.assertRaises(Nachet.UserNotOwnerError): - asyncio.run(Nachet.get_picture_inference(self.cursor, str(not_owner_user_id), str(picture_id))) + with self.assertRaises(nachet.UserNotOwnerError): + asyncio.run(nachet.get_picture_inference(self.cursor, str(not_owner_user_id), str(picture_id))) container_client = asyncio.run(datastore.get_user_container_client(not_owner_user_id,BLOB_CONNECTION_STRING,BLOB_ACCOUNT,BLOB_KEY,'test-user')) container_client.delete_container() @@ -307,8 +307,8 @@ def test_get_picture_blob(self): """ This test checks if the get_picture_blob function correctly returns the blob of a picture """ - picture_id = asyncio.run(Nachet.upload_picture_unknown(self.cursor, self.user_id, self.pic_encoded,self.container_client)) - blob = asyncio.run(Nachet.get_picture_blob(self.cursor, str(self.user_id), self.container_client, str(picture_id))) + picture_id = asyncio.run(nachet.upload_picture_unknown(self.cursor, self.user_id, self.pic_encoded,self.container_client)) + blob = asyncio.run(nachet.get_picture_blob(self.cursor, str(self.user_id), self.container_client, str(picture_id))) blob_image = Image.frombytes("RGB", (1980, 1080), blob) difference = ImageChops.difference(blob_image, self.image) @@ -318,38 +318,38 @@ def test_get_picture_blob_error_user_not_found(self): """ This test checks if the get_pictures_inferences function correctly raise an exception if the user given doesn't exist in db """ - picture_id = asyncio.run(Nachet.upload_picture_unknown(self.cursor, self.user_id, self.pic_encoded,self.container_client)) + picture_id = asyncio.run(nachet.upload_picture_unknown(self.cursor, self.user_id, self.pic_encoded,self.container_client)) with self.assertRaises(datastore.user.UserNotFoundError): - asyncio.run(Nachet.get_picture_blob(self.cursor, str(uuid.uuid4()), self.container_client, str(picture_id))) + asyncio.run(nachet.get_picture_blob(self.cursor, str(uuid.uuid4()), self.container_client, str(picture_id))) def test_get_picture_blob_error_connection_error(self): """ This test checks if the get_pictures_inferences function correctly raise an exception if the connection to the db fails """ - picture_id = asyncio.run(Nachet.upload_picture_unknown(self.cursor, self.user_id, self.pic_encoded,self.container_client)) + picture_id = asyncio.run(nachet.upload_picture_unknown(self.cursor, self.user_id, self.pic_encoded,self.container_client)) mock_cursor = MagicMock() mock_cursor.fetchall.side_effect = Exception("Connection error") with self.assertRaises(Exception): - asyncio.run(Nachet.get_picture_blob(mock_cursor, str(self.user_id), self.container_client, str(picture_id))) + asyncio.run(nachet.get_picture_blob(mock_cursor, str(self.user_id), self.container_client, str(picture_id))) def test_get_picture_blob_error_picture_set_not_found(self): """ This test checks if the get_pictures_inferences function correctly raise an exception if the picture set given doesn't exist in db """ with self.assertRaises(datastore.picture.PictureNotFoundError): - asyncio.run(Nachet.get_picture_blob(self.cursor, str(self.user_id), self.container_client, str(uuid.uuid4()))) + asyncio.run(nachet.get_picture_blob(self.cursor, str(self.user_id), self.container_client, str(uuid.uuid4()))) def test_get_picture_blob_error_not_owner(self): """ This test checks if the get_pictures_inferences function correctly raise an exception if the user is not the owner of the picture set """ - picture_id = asyncio.run(Nachet.upload_picture_unknown(self.cursor, self.user_id, self.pic_encoded,self.container_client)) + picture_id = asyncio.run(nachet.upload_picture_unknown(self.cursor, self.user_id, self.pic_encoded,self.container_client)) not_owner_user_obj= asyncio.run(datastore.new_user(self.cursor,"notowner@email",self.connection_str,'test-user')) not_owner_user_id=datastore.User.get_id(not_owner_user_obj) - with self.assertRaises(Nachet.UserNotOwnerError): - asyncio.run(Nachet.get_picture_blob(self.cursor, str(not_owner_user_id), self.container_client, str(picture_id))) + with self.assertRaises(nachet.UserNotOwnerError): + asyncio.run(nachet.get_picture_blob(self.cursor, str(not_owner_user_id), self.container_client, str(picture_id))) container_client = asyncio.run(datastore.get_user_container_client(not_owner_user_id,BLOB_CONNECTION_STRING,BLOB_ACCOUNT,BLOB_KEY,'test-user')) container_client.delete_container() @@ -424,7 +424,7 @@ def test_get_picture_sets_info(self) : Test the get_picture_sets_info function """ - picture_sets_info = asyncio.run(Nachet.get_picture_sets_info(self.cursor, self.user_id)) + picture_sets_info = asyncio.run(nachet.get_picture_sets_info(self.cursor, self.user_id)) self.assertEqual(len(picture_sets_info), 2) @@ -436,11 +436,11 @@ def test_get_picture_sets_info(self) : self.assert_picture_set_info(picture_set, self.picture_set_id, self.folder_name, 3, expected_pictures_info) self.picture_set_id = asyncio.run(datastore.create_picture_set(self.cursor, self.container_client, 0, self.user_id, self.folder_name + "2")) - picture_id = asyncio.run(Nachet.upload_picture_unknown(self.cursor, self.user_id, self.pic_encoded,self.container_client, self.picture_set_id)) - inference = asyncio.run(Nachet.register_inference_result(self.cursor,self.user_id,self.inference, picture_id, "test_model_id")) - asyncio.run(Nachet.new_perfect_inference_feeback(self.cursor, inference["inference_id"], self.user_id, [box["box_id"] for box in inference["boxes"]])) + picture_id = asyncio.run(nachet.upload_picture_unknown(self.cursor, self.user_id, self.pic_encoded,self.container_client, self.picture_set_id)) + inference = asyncio.run(nachet.register_inference_result(self.cursor,self.user_id,self.inference, picture_id, "test_model_id")) + asyncio.run(nachet.new_perfect_inference_feeback(self.cursor, inference["inference_id"], self.user_id, [box["box_id"] for box in inference["boxes"]])) - picture_sets_info = asyncio.run(Nachet.get_picture_sets_info(self.cursor, self.user_id)) + picture_sets_info = asyncio.run(nachet.get_picture_sets_info(self.cursor, self.user_id)) self.assertEqual(len(picture_sets_info), 3) @@ -457,7 +457,7 @@ def test_get_picture_sets_info_error_user_not_found(self): This test checks if the get_picture_sets_info function correctly raise an exception if the user given doesn't exist in db """ with self.assertRaises(datastore.user.UserNotFoundError): - asyncio.run(Nachet.get_picture_sets_info(self.cursor, uuid.uuid4())) + asyncio.run(nachet.get_picture_sets_info(self.cursor, uuid.uuid4())) def test_get_picture_sets_info_error_connection_error(self): """ @@ -466,7 +466,7 @@ def test_get_picture_sets_info_error_connection_error(self): mock_cursor = MagicMock() mock_cursor.fetchone.side_effect = Exception("Connection error") with self.assertRaises(Exception): - asyncio.run(Nachet.get_picture_sets_info(mock_cursor, self.user_id)) + asyncio.run(nachet.get_picture_sets_info(mock_cursor, self.user_id)) def test_find_validated_pictures(self): """ @@ -670,12 +670,12 @@ def test_delete_picture_set_with_archive(self): ) dev_nb_folders = len( - asyncio.run(Nachet.get_picture_sets_info(self.cursor, self.dev_user_id)) + asyncio.run(nachet.get_picture_sets_info(self.cursor, self.dev_user_id)) ) # Check there is the right number of picture sets in db for each user self.assertEqual( len( - asyncio.run(Nachet.get_picture_sets_info(self.cursor, self.user_id)) + asyncio.run(nachet.get_picture_sets_info(self.cursor, self.user_id)) ), 2, ) @@ -691,14 +691,14 @@ def test_delete_picture_set_with_archive(self): # Check there is the right number of picture sets in db for each user after moving self.assertEqual( len( - asyncio.run(Nachet.get_picture_sets_info(self.cursor, self.user_id)) + asyncio.run(nachet.get_picture_sets_info(self.cursor, self.user_id)) ), 1, ) self.assertEqual( len( asyncio.run( - Nachet.get_picture_sets_info(self.cursor, self.dev_user_id) + nachet.get_picture_sets_info(self.cursor, self.dev_user_id) ) ), dev_nb_folders + 1, From 2780839f3bb86113fd171676b24cb4128a2740dc Mon Sep 17 00:00:00 2001 From: sylvanie85 Date: Wed, 31 Jul 2024 15:12:12 +0000 Subject: [PATCH 26/27] fix lint test --- datastore/nachet/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datastore/nachet/__init__.py b/datastore/nachet/__init__.py index 211d7e3b..3879d8ec 100644 --- a/datastore/nachet/__init__.py +++ b/datastore/nachet/__init__.py @@ -678,7 +678,7 @@ async def get_picture_sets_info(cursor, user_id: str): except (user.UserNotFoundError, picture.GetPictureSetError, picture.GetPictureError ) as e : raise e except Exception as e: - raise picture.GetPictureSetError("An error occured while retrieving the picture sets") + raise picture.GetPictureSetError(f"An error occured while retrieving the picture sets : {e}") async def get_picture_inference(cursor, user_id: str, picture_id: str): From 2912796b0ae082f41d3469f21c5bba7dc1204b6a Mon Sep 17 00:00:00 2001 From: sylvanie85 Date: Thu, 1 Aug 2024 15:50:58 +0000 Subject: [PATCH 27/27] review --- datastore/db/queries/inference/__init__.py | 25 ++++++++++++++++++++++ datastore/db/queries/picture/__init__.py | 24 --------------------- datastore/nachet/__init__.py | 2 +- 3 files changed, 26 insertions(+), 25 deletions(-) diff --git a/datastore/db/queries/inference/__init__.py b/datastore/db/queries/inference/__init__.py index 73f74820..3689ca49 100644 --- a/datastore/db/queries/inference/__init__.py +++ b/datastore/db/queries/inference/__init__.py @@ -17,6 +17,7 @@ class InferenceObjectNotFoundError(Exception): class InferenceAlreadyVerifiedError(Exception): pass + """ INFERENCE TABLE QUERIES @@ -90,6 +91,30 @@ def get_inference(cursor, inference_id: str): except Exception: raise InferenceNotFoundError(f"Error: could not get inference {inference_id}") +def get_inference_by_picture_id(cursor, picture_id: str): + """ + This functions retrieve inference of a given picture + + Parameters: + - cursor (cursor): The cursor of the database. + - picture_id (str): The UUID of the picture. + """ + try : + query = """ + SELECT + id, inference, pipeline_id + FROM + inference + WHERE + picture_id = %s + """ + cursor.execute(query, (picture_id,)) + result = cursor.fetchone() + return result + except Exception: + raise InferenceNotFoundError( + f"Error: could not get inference for the picture {picture_id}") + def set_inference_feedback_user_id(cursor, inference_id, user_id): """ This function sets the feedback_user_id of an inference. diff --git a/datastore/db/queries/picture/__init__.py b/datastore/db/queries/picture/__init__.py index b72f5041..b57ee5af 100644 --- a/datastore/db/queries/picture/__init__.py +++ b/datastore/db/queries/picture/__init__.py @@ -385,30 +385,6 @@ def check_picture_inference_exist(cursor, picture_id: str): raise GetPictureError( f"Error: could not check if the picture {picture_id} has an existing inference") -def get_picture_inference(cursor, picture_id: str): - """ - This functions retrieve inference of a given picture - - Parameters: - - cursor (cursor): The cursor of the database. - - picture_id (str): The UUID of the picture. - """ - try : - query = """ - SELECT - id, inference, pipeline_id - FROM - inference - WHERE - picture_id = %s - """ - cursor.execute(query, (picture_id,)) - result = cursor.fetchone() - return result - except Exception: - raise GetPictureError( - f"Error: could not get inference for the picture {picture_id}") - def change_picture_set_id(cursor, user_id, old_picture_set_id, new_picture_set_id): """ This function change picture_set_id of all pictures in a picture_set to a new one. diff --git a/datastore/nachet/__init__.py b/datastore/nachet/__init__.py index 3879d8ec..2fb0bc12 100644 --- a/datastore/nachet/__init__.py +++ b/datastore/nachet/__init__.py @@ -709,7 +709,7 @@ async def get_picture_inference(cursor, user_id: str, picture_id: str): ) if picture.check_picture_inference_exist(cursor, picture_id): - inf = picture.get_picture_inference(cursor, picture_id) + inf = inference.get_inference_by_picture_id(cursor, picture_id) inf = inference_metadata.rebuild_inference(cursor, inf)